From d8206a6c9f37e1d45ed56887f7e626018f79e102 Mon Sep 17 00:00:00 2001 From: Bruno Garcia Date: Wed, 13 Oct 2021 09:05:47 +0200 Subject: [PATCH 01/14] Move handling of parameter validation to Task --- Task/task.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/Task/task.py b/Task/task.py index 79f0f55..8940129 100644 --- a/Task/task.py +++ b/Task/task.py @@ -1,4 +1,4 @@ -from typing import Callable, Dict, Optional, Union +from typing import Callable, Dict, Optional, Union, Tuple, Any from Helper import Log, Level @@ -10,6 +10,7 @@ def __init__(self, name: str, parent, params: Optional[Dict] = None, self.name = name self.params = {} if params is None else params + self.paramRules: Dict[str, Tuple[Any, bool]] = {} # Dict[: (, )] self.parent: ExecutorBase = parent self.logMethod = Log.Log if logMethod is None else logMethod self.condition = conditionMethod @@ -20,8 +21,11 @@ def Start(self) -> Dict: if self.condition is None or self.condition(): self.Log(Level.INFO, f"[Starting Task '{self.name}']") self.Log(Level.DEBUG, f'Params: {self.params}') - self.Run() - self.Log(Level.INFO, f"[Task '{self.name}' finished]") + if self.SanitizeParams(): + self.Run() + self.Log(Level.INFO, f"[Task '{self.name}' finished]") + else: + self.Log(Level.ERROR, f"[Task '{self.name}' cancelled due to incorrect parameters]") self.Log(Level.DEBUG, f'Params: {self.params}') else: self.Log(Level.INFO, f"[Task '{self.name}' not started (condition false)]") @@ -37,3 +41,15 @@ def Run(self) -> None: def Log(self, level: Union[Level, str], msg: str): self.logMethod(level, msg) self.LogMessages.append(msg) + + def SanitizeParams(self): + for key, value in self.paramRules.items(): + default, mandatory = value + if key not in self.params.keys(): + if mandatory: + self.Log(Level.ERROR, f"Parameter '{key}' is mandatory but was not configured for the task.") + return False + else: + self.params[key] = default + self.Log(Level.DEBUG, f"Parameter '{key}' set to default ('{str(default)}').") + return True From 3e666936f5ccdb9b57a0e1093ead8d0947dc22c8 Mon Sep 17 00:00:00 2001 From: Bruno Garcia Date: Wed, 13 Oct 2021 09:45:12 +0200 Subject: [PATCH 02/14] Update task parameter validation --- Executor/Tasks/Run/add_milestone.py | 10 +++--- Executor/Tasks/Run/cli_execute.py | 9 +++--- Executor/Tasks/Run/compress_files.py | 12 +++++-- Executor/Tasks/Run/csvToInflux.py | 32 ++++++++----------- Executor/Tasks/Run/delay.py | 3 +- Executor/Tasks/Run/message.py | 5 ++- Executor/Tasks/Run/publish_from_source.py | 28 ++++++++-------- .../Tasks/Run/single_slice_creation_time.py | 8 ++++- Executor/Tasks/Run/slice_creation_time.py | 25 +++++++-------- Executor/Tasks/Run/tap_execute.py | 9 ++++-- Task/task.py | 2 +- 11 files changed, 79 insertions(+), 64 deletions(-) diff --git a/Executor/Tasks/Run/add_milestone.py b/Executor/Tasks/Run/add_milestone.py index c48e998..181908e 100644 --- a/Executor/Tasks/Run/add_milestone.py +++ b/Executor/Tasks/Run/add_milestone.py @@ -5,11 +5,9 @@ class AddMilestone(Task): def __init__(self, logMethod, parent, params): super().__init__("Add Milestone", parent, params, logMethod, None) + self.paramRules = {'Milestone': (None, True)} def Run(self): - try: - milestone = self.params['Milestone'] - self.Log(Level.INFO, f"Adding milestone '{milestone}' to experiment.") - self.parent.AddMilestone(milestone) - except KeyError: - self.Log(Level.ERROR, "'Milestone' value not set") + milestone = self.params['Milestone'] + self.Log(Level.INFO, f"Adding milestone '{milestone}' to experiment.") + self.parent.AddMilestone(milestone) diff --git a/Executor/Tasks/Run/cli_execute.py b/Executor/Tasks/Run/cli_execute.py index 9963d52..e1a8548 100644 --- a/Executor/Tasks/Run/cli_execute.py +++ b/Executor/Tasks/Run/cli_execute.py @@ -5,10 +5,11 @@ class CliExecute(Task): def __init__(self, logMethod, parent, params): super().__init__("CLI Execute", parent, params, logMethod, None) + self.paramRules = { + 'Parameters': (None, True), + 'CWD': (None, True) + } def Run(self): - parameters = self.params['Parameters'] - cwd = self.params['CWD'] - - cli = Cli(parameters, cwd, self.logMethod) + cli = Cli(self.params['Parameters'], self.params['CWD'], self.logMethod) cli.Execute() diff --git a/Executor/Tasks/Run/compress_files.py b/Executor/Tasks/Run/compress_files.py index 7e8aa65..fbdf04a 100644 --- a/Executor/Tasks/Run/compress_files.py +++ b/Executor/Tasks/Run/compress_files.py @@ -6,13 +6,19 @@ class CompressFiles(Task): def __init__(self, logMethod, parent, params): super().__init__("Compress Files", parent, params, logMethod, None) + self.paramRules = { + 'Files': ([], False), + 'Folders': ([], False), + 'Output': (None, True) + } def Run(self): from Helper import Compress, IO - files = [abspath(f) for f in self.params.get("Files", [])] - folders = [abspath(f) for f in self.params.get("Folders", [])] - output = self.params.get("Output", "") + files = [abspath(f) for f in self.params["Files"]] + folders = [abspath(f) for f in self.params["Folders"]] + output = self.params["Output"] .get("Output", "") + self.Log(Level.INFO, f"Compressing files to output: {output}") for folder in folders: diff --git a/Executor/Tasks/Run/csvToInflux.py b/Executor/Tasks/Run/csvToInflux.py index deffe74..61ff8f2 100644 --- a/Executor/Tasks/Run/csvToInflux.py +++ b/Executor/Tasks/Run/csvToInflux.py @@ -6,26 +6,22 @@ class CsvToInflux(Task): def __init__(self, logMethod, parent, params): super().__init__("Csv To Influx", parent, params, logMethod, None) + self.paramRules = { + 'ExecutionId': (None, True), + 'CSV': (None, True), + 'Measurement': (None, True), + 'Delimiter': (',', False), + 'Timestamp': ('Timestamp', False), + 'Convert': (True, False) + } def Run(self): - try: executionId = self.params['ExecutionId'] - except KeyError: - self.Log(Level.ERROR, "ExecutionId value not defined, please review the Task configuration.") - return - - try: csvFile = self.params["CSV"] - except KeyError: - self.Log(Level.ERROR, "CSV file not defined, please review the Task configuration.") - return - - try: measurement = self.params["Measurement"] - except KeyError: - self.Log(Level.ERROR, "Measurement not defined, please review the Task configuration.") - return - - delimiter = self.params.get("Delimiter", ',') - timestamp = self.params.get("Timestamp", "Timestamp") - tryConvert = self.params.get("Convert", True) + executionId = self.params['ExecutionId'] + csvFile = self.params["CSV"] + measurement = self.params["Measurement"] + delimiter = self.params["Delimiter"] + timestamp = self.params["Timestamp"] + tryConvert = self.params["Convert"] try: from Helper import InfluxDb, InfluxPayload # Delayed to avoid cyclic imports diff --git a/Executor/Tasks/Run/delay.py b/Executor/Tasks/Run/delay.py index 5fc6e7f..ab32d42 100644 --- a/Executor/Tasks/Run/delay.py +++ b/Executor/Tasks/Run/delay.py @@ -6,9 +6,10 @@ class Delay(Task): def __init__(self, logMethod, parent, params): super().__init__("Delay", parent, params, logMethod, None) + self.paramRules = {'Time': (60, False)} def Run(self): - value = self.params.get('Time', 60) + value = self.params['Time'] try: time = int(value) if time < 0: diff --git a/Executor/Tasks/Run/message.py b/Executor/Tasks/Run/message.py index 3e84105..3048cdd 100644 --- a/Executor/Tasks/Run/message.py +++ b/Executor/Tasks/Run/message.py @@ -1,11 +1,14 @@ from Task import Task from Helper import Level -from time import sleep class Message(Task): def __init__(self, logMethod, parent, params): super().__init__("Message", parent, params, logMethod, None) + self.paramRules = { + 'Severity': ('INFO', False), + 'Message': (None, True) + } def Run(self): level = Level[self.params['Severity']] diff --git a/Executor/Tasks/Run/publish_from_source.py b/Executor/Tasks/Run/publish_from_source.py index 47a4dec..c57158c 100644 --- a/Executor/Tasks/Run/publish_from_source.py +++ b/Executor/Tasks/Run/publish_from_source.py @@ -7,26 +7,26 @@ class PublishFromSource(Task): def __init__(self, name, parent, params, logMethod): super().__init__(name, parent, params, logMethod, None) + self.paramRules = { + 'Pattern': (None, True), + 'Keys': (None, True), + 'Path': (None, False) # Mandatory only for PublishFromFile, handled below + } def Run(self): self.Log(Level.INFO, f'Running task {self.name} with params: {self.params}') - filePath = self.params.get("Path", None) - pattern = self.params.get("Pattern", None) - keys = self.params.get("Keys", None) + filePath = self.params["Path"] + pattern = self.params["Pattern"] + keys = self.params["Keys"] - if pattern is None: - self.raiseConfigError("Pattern") + self.Log(Level.DEBUG, f"Looking for pattern: '{pattern}'; Assigning groups as:") - if keys is None: - self.raiseConfigError("Keys") - else: - self.Log(Level.DEBUG, f"Looking for pattern: '{pattern}'; Assigning groups as:") - try: - for index, key in keys: - self.Log(Level.DEBUG, f" {index}: {key}") - except Exception as e: - raise RuntimeError(f"Invalid 'Keys' definition: {e}") + try: + for index, key in keys: + self.Log(Level.DEBUG, f" {index}: {key}") + except Exception as e: + raise RuntimeError(f"Invalid 'Keys' definition: {e}") regex = re.compile(pattern) diff --git a/Executor/Tasks/Run/single_slice_creation_time.py b/Executor/Tasks/Run/single_slice_creation_time.py index 44d195c..545cc29 100644 --- a/Executor/Tasks/Run/single_slice_creation_time.py +++ b/Executor/Tasks/Run/single_slice_creation_time.py @@ -8,11 +8,17 @@ class SingleSliceCreationTime(Task): def __init__(self, logMethod, parent, params): super().__init__("Single Slice Creation Time Measurement", parent, params, logMethod, None) + self.paramRules = { + 'ExecutionId': (None, True), + 'WaitForRunning': (None, True), + 'SliceId': (None, True), + 'Timeout': (None, False) + } def Run(self): executionId = self.params['ExecutionId'] waitForRunning = self.params['WaitForRunning'] - timeout = self.params.get('Timeout', None) + timeout = self.params['Timeout'] sliceId = self.params['SliceId'] count = 0 diff --git a/Executor/Tasks/Run/slice_creation_time.py b/Executor/Tasks/Run/slice_creation_time.py index 366ee20..7896180 100644 --- a/Executor/Tasks/Run/slice_creation_time.py +++ b/Executor/Tasks/Run/slice_creation_time.py @@ -13,23 +13,22 @@ class SliceCreationTime(Task): def __init__(self, logMethod, parent, params): super().__init__("Slice Creation Time Measurement", parent, params, logMethod, None) + self.paramRules = { + 'ExecutionId': (None, True), + 'NEST': (None, True), + 'Iterations': (25, False), + 'CSV': (None, False), + 'Timeout': (None, False) + } def Run(self): - executionId = self.params.get('ExecutionId', None) - nestFile = self.params.get("NEST", None) - iterations = self.params.get("Iterations", 25) - csvFile = self.params.get("CSV", None) - timeout = self.params.get("Timeout", None) + executionId = self.params['ExecutionId'] + nestFile = self.params["NEST"] + iterations = self.params["Iterations"] + csvFile = self.params["CSV"] + timeout = self.params["Timeout"] pollTime = 5 - if executionId is None: - self.Log(Level.ERROR, "ExecutionId value not defined, please review the Task configuration.") - return - - if nestFile is None: - self.Log(Level.ERROR, "NEST value not defined, please review the Task configuration.") - return - try: with open(nestFile, 'r', encoding='utf-8') as input: nestData = json.load(input) diff --git a/Executor/Tasks/Run/tap_execute.py b/Executor/Tasks/Run/tap_execute.py index cae12b7..471b58d 100644 --- a/Executor/Tasks/Run/tap_execute.py +++ b/Executor/Tasks/Run/tap_execute.py @@ -7,6 +7,11 @@ class TapExecute(Task): def __init__(self, logMethod, parent, params): super().__init__("Tap Execute", parent, params, logMethod, None) + self.paramRules = { + 'TestPlan': (None, True), + 'Externals': ({}, False), + 'GatherResults': (False, False) + } def Run(self): from Helper import IO, Compress @@ -16,8 +21,8 @@ def Run(self): self.Log(Level.CRITICAL, "Trying to run TapExecute Task while TAP is not enabled") else: tapPlan = self.params['TestPlan'] - externals = self.params.get('Externals', {}) - gatherResults = self.params.get('GatherResults', False) + externals = self.params['Externals'] + gatherResults = self.params['GatherResults'] tap = Tap(tapPlan, externals, self.logMethod) tap.Execute() diff --git a/Task/task.py b/Task/task.py index 8940129..017ac52 100644 --- a/Task/task.py +++ b/Task/task.py @@ -51,5 +51,5 @@ def SanitizeParams(self): return False else: self.params[key] = default - self.Log(Level.DEBUG, f"Parameter '{key}' set to default ('{str(default)}').") + self.Log(Level.DEBUG, f"Parameter '{key}' set to default ({str(default)}).") return True From c87018ee4a3e2dcb775dc2a6120c1c35dadcb605 Mon Sep 17 00:00:00 2001 From: Bruno Garcia Date: Wed, 13 Oct 2021 12:40:33 +0200 Subject: [PATCH 03/14] Implement RestApi task --- Executor/Tasks/Run/__init__.py | 1 + Executor/Tasks/Run/rest_api.py | 78 ++++++++++++++++++++++++++++++++++ REST/__init__.py | 2 +- 3 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 Executor/Tasks/Run/rest_api.py diff --git a/Executor/Tasks/Run/__init__.py b/Executor/Tasks/Run/__init__.py index 0574fd2..bed150a 100644 --- a/Executor/Tasks/Run/__init__.py +++ b/Executor/Tasks/Run/__init__.py @@ -10,3 +10,4 @@ from .csvToInflux import CsvToInflux from .add_milestone import AddMilestone from .publish_from_source import PublishFromPreviousTaskLog, PublishFromFile +from .rest_api import RestApi diff --git a/Executor/Tasks/Run/rest_api.py b/Executor/Tasks/Run/rest_api.py new file mode 100644 index 0000000..4160d8c --- /dev/null +++ b/Executor/Tasks/Run/rest_api.py @@ -0,0 +1,78 @@ +from Task import Task +from Helper import Level +from REST import RestClient, Payload + + +class RestApi(Task): + def __init__(self, logMethod, parent, params): + super().__init__("REST API", parent, params, logMethod, None) + self.paramRules = { + 'Host': (None, True), + 'Port': (None, True), + 'Endpoint': (None, True), + 'Https': (False, False), + 'Insecure': (False, False), + 'Method': ('GET', False), + 'Payload': ('{}', False), + 'PayloadMode': (None, False), + 'Responses': (None, False), + 'Timeout': (10, False), + 'Headers': (None, False) + } + + def Run(self): + client = RestClient(self.params['Host'], self.params['Port'], "", + self.params['Https'], self.params['Insecure']) + + endpoint = self.params['Endpoint'] + method = str(self.params['Method']).upper() + payload = self.params['Payload'] + payloadMode = self.params['PayloadMode'] + timeout = self.params['Timeout'] + headers = self.params['Headers'] + statusCodes = self.params['Responses'] + + if statusCodes is not None: + if not isinstance(statusCodes, (tuple, list)): + statusCodes = [statusCodes] + + if "Success" in statusCodes: + statusCodes.remove("Success") + statusCodes.extend([*range(200, 300)]) + + self.Log(Level.INFO, f"Sending '{method}' request to '{client.api_url}', endpoint '{endpoint}'.") + self.Log(Level.DEBUG, f"Timeout: {timeout}; Extra Headers: {headers}") + self.Log(Level.DEBUG, f"Payload: {payload}") + if statusCodes is not None: + self.Log(Level.DEBUG, f"Accepted status codes: {statusCodes}") + + match method: + case "GET": + response = client.HttpGet(endpoint, extra_headers=headers, timeout=timeout) + case "POST": + payloadMode = None if payloadMode is None else Payload[payloadMode] + response = client.HttpPost(endpoint, extra_headers=headers, + body=payload, payload=payloadMode, timeout=timeout) + case "PATCH": + response = client.HttpPatch(endpoint, extra_headers=headers, body=payload, timeout=timeout) + case "DELETE": + response = client.HttpDelete(endpoint, extra_headers=headers, timeout=timeout) + case _: + self.Log(Level.ERROR, f"Unsupported method '{method}'") + return + + status = client.ResponseStatusCode(response) + try: + data = client.ResponseToJson(response) + except RuntimeError: + try: + data = client.ResponseToRaw(response) + except RuntimeError: + data = None + + self.Log(Level.INFO, f"Status '{status}'; Response: '{data}'") + if statusCodes is not None: + if status not in statusCodes: + message = f"Unexpected status code received: {status}" + self.Log(Level.ERROR, message) + raise RuntimeError(message) diff --git a/REST/__init__.py b/REST/__init__.py index c36803d..730174f 100644 --- a/REST/__init__.py +++ b/REST/__init__.py @@ -1 +1 @@ -from .rest_client import RestClient +from .rest_client import RestClient, Payload From 4e24547491353e839a299f93b6071e33617486ca Mon Sep 17 00:00:00 2001 From: Bruno Garcia Date: Thu, 14 Oct 2021 09:22:32 +0200 Subject: [PATCH 04/14] Abort experiment execution on incorrect parameters, update documentation --- README.md | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++ Task/task.py | 4 +++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e2d282d..25df35a 100644 --- a/README.md +++ b/README.md @@ -462,6 +462,25 @@ expression pattern, publishing the groups found. Configuration values: > - While writing the `Keys` in the task configuration note that YAML does not have a syntax for tuples, use lists of two elements instead. - `Path` (only for Run.PublishFromFile): Path of the file to read +### Run.RestApi + +Provides direct access to the internal RestClient functionality, avoiding the need of using external utilities such as +`curl` for simple queries. Configuration values: +- `Host`: Location where the REST API is listening +- `Port`: Port where the REST API is listening +- `Endpoint`: Specific API endpoint where the request will be sent +- `Https`: Whether to use HTTPS or not, defaults to False +- `Insecure`: Whether to ignore certificate errors when using HTTPS, defaults to False +- `Method`: REST method to use, currently suported methods are `GET`, `POST`, `PATCH`, `DELETE` +- `Payload`: Data to send in JSON format (as a single string), defaults to `'{}'` +- `PayloadMode`: Field where the payload will be sent, possible values are: + - `Data`: The payload is saved in the `Body` field of the request. Also adds the `Content-Type=application/json` header + - `Form`: The payload is saved on the `Form` field of the request +- `Responses`: Set of expected responses as a single value or a list. The special value `Success` indicates any possible +success response (2xx). Set to `None` to disable the check. +- `Timeout`: Maximum time in seconds to wait for a response +- `Headers`: Additional headers to add to the request + ### Run.SingleSliceCreationTime Sends the Slice Creation Time reported by the Slice Manager to InfluxDb. This task will not perform any deployment by itself, and will only read the values for an slice deployed during the experiment pre-run stage. @@ -523,6 +542,43 @@ Separate values from the `Parameters` dictionary can also be expanded using the > a '.' the ELCM will fall back to looking at the Publish values (the default for Release A). If the collection > is not 'Publish' or 'Params', the expression will be replaced by `<>` +## Implementing additional tasks: + +The ELCM is designed to be extensible, and thus, it is possible to easily integrate additional tasks. The basic steps +for defining a new task are: + +1. Create a new Python file with a descriptive name, for example `compress_files.py`. This file must be saved in the +`/Execitor/Task/Run` subfolder of the ELCM +2. In this file, define a new class, for example `CompressFiles`, which inherits from `Task`. The constructor of this +class must have the signature displayed below: + ```python + class CompressFiles(Task): + def __init__(self, logMethod, parent, params): + super().__init__("Compress Files", parent, params, logMethod, None) + ``` + In general, the parameters received in the constructor will be sent directly to the superclass, along with a Task +name (in the first parameter). The last parameter is an optional "Condition" method. If the callable passed in this +parameter evaluates to False, the execution of the Task will be skipped. +3. Override the `Run` method of the Task class. If this method is not overridden, a `NotImplementedError` will be raised +at runtime. The following methods and fields are available: + - `self.name`: Contains the name of the Task. + - `self.parent`: Contains a reference to the Executor that called the task. Using this reference a Task can gain +access to additional information about the Experiment. + - `self.params`: Contains a dictionary with the parameters of the Task, expanded following the procedure described +in the previous section. + - `self.paramRules`: Set of validation rules to automatically apply on the parameters before the execution of the +task, with the following format: ```Dict[: (, )]```. If `` has not +been defined, but it's not ``, then it is assigned the `` value. If it's ` Dict: self.Run() self.Log(Level.INFO, f"[Task '{self.name}' finished]") else: - self.Log(Level.ERROR, f"[Task '{self.name}' cancelled due to incorrect parameters]") + message = f"[Task '{self.name}' cancelled due to incorrect parameters ({self.params})]" + self.Log(Level.ERROR, message) + raise RuntimeError(message) self.Log(Level.DEBUG, f'Params: {self.params}') else: self.Log(Level.INFO, f"[Task '{self.name}' not started (condition false)]") From 8221f3f5755f97cb4e9766e0f18e089401c627ae Mon Sep 17 00:00:00 2001 From: Bruno Garcia Date: Fri, 22 Oct 2021 10:25:25 +0200 Subject: [PATCH 05/14] Reorganize configuration --- Executor/Tasks/PostRun/farewell.py | 3 +- Executor/Tasks/PreRun/coordinate.py | 3 +- Executor/Tasks/PreRun/instantiate.py | 3 +- Executor/Tasks/Remote/base_remote_task.py | 2 +- Executor/Tasks/Run/tap_execute.py | 3 +- Executor/executor_base.py | 3 +- Experiment/experiment_run.py | 3 +- Experiment/variable_expander.py | 2 +- Helper/__init__.py | 1 - Helper/influx.py | 2 +- Helper/log.py | 2 +- Helper/tap_executor.py | 3 +- Interfaces/management.py | 3 +- Interfaces/portal.py | 2 +- README.md | 2 +- Scheduler/__init__.py | 2 +- Scheduler/east_west/routes.py | 3 +- Scheduler/execution/routes.py | 2 +- Scheduler/routes.py | 3 +- Settings/__init__.py | 1 + {Helper => Settings}/config.py | 62 ++--------------------- Settings/config_base.py | 57 +++++++++++++++++++++ {Helper => Settings}/default_config | 6 ++- 23 files changed, 95 insertions(+), 78 deletions(-) create mode 100644 Settings/__init__.py rename {Helper => Settings}/config.py (79%) create mode 100644 Settings/config_base.py rename {Helper => Settings}/default_config (92%) diff --git a/Executor/Tasks/PostRun/farewell.py b/Executor/Tasks/PostRun/farewell.py index 3c9971f..39a9667 100644 --- a/Executor/Tasks/PostRun/farewell.py +++ b/Executor/Tasks/PostRun/farewell.py @@ -1,5 +1,6 @@ from Task import Task -from Helper import Level, Config +from Helper import Level +from Settings import Config from time import sleep from Interfaces import RemoteApi diff --git a/Executor/Tasks/PreRun/coordinate.py b/Executor/Tasks/PreRun/coordinate.py index ed976d0..0530372 100644 --- a/Executor/Tasks/PreRun/coordinate.py +++ b/Executor/Tasks/PreRun/coordinate.py @@ -1,5 +1,6 @@ from Task import Task -from Helper import Level, Config +from Helper import Level +from Settings import Config from time import sleep from Interfaces import RemoteApi diff --git a/Executor/Tasks/PreRun/instantiate.py b/Executor/Tasks/PreRun/instantiate.py index 31c3320..cbcf1d1 100644 --- a/Executor/Tasks/PreRun/instantiate.py +++ b/Executor/Tasks/PreRun/instantiate.py @@ -1,5 +1,6 @@ from Task import Task -from Helper import Level, Config +from Helper import Level +from Settings import Config from Data import NsInfo from typing import List, Dict from Interfaces import Management diff --git a/Executor/Tasks/Remote/base_remote_task.py b/Executor/Tasks/Remote/base_remote_task.py index 6c24d5e..392f687 100644 --- a/Executor/Tasks/Remote/base_remote_task.py +++ b/Executor/Tasks/Remote/base_remote_task.py @@ -1,5 +1,5 @@ from Task import Task -from Helper import Config +from Settings import Config TIMEOUT = Config().EastWest.Timeout diff --git a/Executor/Tasks/Run/tap_execute.py b/Executor/Tasks/Run/tap_execute.py index 471b58d..ba34a3e 100644 --- a/Executor/Tasks/Run/tap_execute.py +++ b/Executor/Tasks/Run/tap_execute.py @@ -1,5 +1,6 @@ from Task import Task -from Helper import Tap, Config, Level +from Helper import Tap, Level +from Settings import Config from os.path import exists, join from datetime import datetime, timezone diff --git a/Executor/executor_base.py b/Executor/executor_base.py index f6e05d5..69f593c 100644 --- a/Executor/executor_base.py +++ b/Executor/executor_base.py @@ -1,4 +1,5 @@ -from Helper import Child, Level, Config +from Helper import Child, Level +from Settings import Config from typing import Dict, Optional, List from Data import ExperimentDescriptor from Composer import PlatformConfiguration diff --git a/Experiment/experiment_run.py b/Experiment/experiment_run.py index 3b4abd3..42cf490 100644 --- a/Experiment/experiment_run.py +++ b/Experiment/experiment_run.py @@ -4,7 +4,8 @@ from enum import Enum, unique from datetime import datetime, timezone from tempfile import TemporaryDirectory -from Helper import Config, Serialize, Log +from Helper import Serialize, Log +from Settings import Config from Interfaces import PortalApi from Composer import Composer, PlatformConfiguration from os.path import join, abspath diff --git a/Experiment/variable_expander.py b/Experiment/variable_expander.py index 25ac3fd..17d024f 100644 --- a/Experiment/variable_expander.py +++ b/Experiment/variable_expander.py @@ -2,7 +2,7 @@ from .experiment_run import ExperimentRun from Executor import ExecutorBase from re import finditer -from Helper import Config +from Settings import Config from json import dumps diff --git a/Helper/__init__.py b/Helper/__init__.py index 4ac3ac7..48e2f0b 100644 --- a/Helper/__init__.py +++ b/Helper/__init__.py @@ -1,6 +1,5 @@ from .log import Log, LogInfo from .log_level import Level -from .config import Config, TapConfig from .child import Child from .serialize import Serialize from .tap_executor import Tap diff --git a/Helper/influx.py b/Helper/influx.py index 063e9b6..79f8dfe 100644 --- a/Helper/influx.py +++ b/Helper/influx.py @@ -1,5 +1,5 @@ from influxdb import InfluxDBClient -from .config import Config +from Settings import Config from typing import Dict, List, Union from datetime import datetime, timezone from csv import DictWriter, DictReader, Dialect, QUOTE_NONE diff --git a/Helper/log.py b/Helper/log.py index 24e69d7..2dc6265 100644 --- a/Helper/log.py +++ b/Helper/log.py @@ -4,7 +4,7 @@ from flask import Flask from os.path import exists, join from os import makedirs -from .config import Config +from Settings import Config import traceback from typing import Union, Optional, List, Dict, Tuple from dataclasses import dataclass diff --git a/Helper/tap_executor.py b/Helper/tap_executor.py index 5438073..6376557 100644 --- a/Helper/tap_executor.py +++ b/Helper/tap_executor.py @@ -1,7 +1,8 @@ import subprocess import psutil import re -from Helper import Config, TapConfig, Level +from Helper import Level +from Settings import Config, TapConfig from typing import Dict, Optional, Callable from time import sleep diff --git a/Interfaces/management.py b/Interfaces/management.py index cdeb2c7..add37ca 100644 --- a/Interfaces/management.py +++ b/Interfaces/management.py @@ -1,5 +1,6 @@ from REST import RestClient -from Helper import Config, Log +from Helper import Log +from Settings import Config from typing import Dict, Optional, Tuple, List from Data import Metal, MetalUsage, NsInfo from Facility import Facility diff --git a/Interfaces/portal.py b/Interfaces/portal.py index e7a1965..9d0eb9e 100644 --- a/Interfaces/portal.py +++ b/Interfaces/portal.py @@ -2,7 +2,7 @@ import json from threading import Thread from typing import Optional, Union -from Helper.config import Portal as PortalConfig +from Settings.config import Portal as PortalConfig class PortalApi(RestClient): diff --git a/README.md b/README.md index 25df35a..b946277 100644 --- a/README.md +++ b/README.md @@ -465,7 +465,7 @@ expression pattern, publishing the groups found. Configuration values: ### Run.RestApi Provides direct access to the internal RestClient functionality, avoiding the need of using external utilities such as -`curl` for simple queries. Configuration values: +`curl` for simple requests. Configuration values: - `Host`: Location where the REST API is listening - `Port`: Port where the REST API is listening - `Endpoint`: Specific API endpoint where the request will be sent diff --git a/Scheduler/__init__.py b/Scheduler/__init__.py index b970371..4f7ef08 100644 --- a/Scheduler/__init__.py +++ b/Scheduler/__init__.py @@ -3,7 +3,7 @@ from Status import Status from flask_bootstrap import Bootstrap from flask_moment import Moment -from Helper import Config +from Settings import Config from Facility import Facility from .heartbeat import HeartBeat from dotenv import load_dotenv diff --git a/Scheduler/east_west/routes.py b/Scheduler/east_west/routes.py index 6f2e4c0..3063df3 100644 --- a/Scheduler/east_west/routes.py +++ b/Scheduler/east_west/routes.py @@ -2,7 +2,8 @@ from Scheduler.execution import handleExecutionResults, executionOrTombstone from flask import jsonify, request, json from Status import ExecutionQueue -from Helper import Config, InfluxDb +from Helper import InfluxDb +from Settings import Config notFound = {'success': False, 'message': 'Execution ID is not valid or experiment is not running'} diff --git a/Scheduler/execution/routes.py b/Scheduler/execution/routes.py index ca94556..1705ece 100644 --- a/Scheduler/execution/routes.py +++ b/Scheduler/execution/routes.py @@ -3,7 +3,7 @@ from Experiment import ExperimentRun, Tombstone from Scheduler.execution import bp from typing import Union, Optional -from Helper import Config +from Settings import Config from os.path import join, isfile, abspath diff --git a/Scheduler/routes.py b/Scheduler/routes.py index 5621a58..0a77ff0 100644 --- a/Scheduler/routes.py +++ b/Scheduler/routes.py @@ -4,7 +4,8 @@ from flask import render_template, make_response, request, flash, redirect, url_for from functools import wraps, update_wrapper from datetime import datetime -from Helper import Log, Serialize, LogInfo, Config +from Helper import Log, Serialize, LogInfo +from Settings import Config from Facility import Facility from typing import List, Dict from flask_paginate import Pagination, get_page_parameter diff --git a/Settings/__init__.py b/Settings/__init__.py new file mode 100644 index 0000000..4de995b --- /dev/null +++ b/Settings/__init__.py @@ -0,0 +1 @@ +from .config import Config, TapConfig diff --git a/Helper/config.py b/Settings/config.py similarity index 79% rename from Helper/config.py rename to Settings/config.py index c138ae6..dc8abfc 100644 --- a/Helper/config.py +++ b/Settings/config.py @@ -5,62 +5,8 @@ from typing import Dict, List, Tuple, Optional import logging import platform -from REST import RestClient -from .log_level import Level - - -class validable: - def __init__(self, data: Dict, section: str, - defaults: Dict[str, Tuple[Optional[object], "Level"]]): - self.data = data - self.section = section - self.defaults = defaults - - def _keyOrDefault(self, key: str): - if key in self.data.keys(): - return self.data[key] - else: - default = self.defaults.get(key, None) - return default[0] if default is not None else None - - @property - def Validation(self) -> List[Tuple['Level', str]]: - res = [] - for key in self.defaults.keys(): - if key not in self.data: - default, level = self.defaults[key] - defaultText = f", using default '{default}'" if default is not None else "" - res.append((level, f"'{key}' not defined under '{self.section}'{defaultText}")) - if len(res) == 0: - values = '; '.join([f'{key}: {self.data[key]}' for key in self.defaults.keys()]) - res.append((Level.INFO, f'{self.section} [{values}]')) - return res - - -class restApi(validable): - def __init__(self, data: Dict, section: str, defaults: Dict[str, Tuple[Optional[object], "Level"]]): - if 'Host' not in defaults.keys(): defaults['Host'] = (None, Level.ERROR) - if 'Port' not in defaults.keys(): defaults['Port'] = (None, Level.ERROR) - super().__init__(data, section, defaults) - - @property - def Host(self): - return self._keyOrDefault('Host') - - @property - def Port(self): - return self._keyOrDefault('Port') - - @property - def Validation(self) -> List[Tuple['Level', str]]: - res = super().Validation - if all([e[0] == Level.INFO for e in res]): - # No errors, but check if a rest server can be created with the configuration - try: - _ = RestClient(self.Host, self.Port, "") - except Exception as e: - res.append((Level.ERROR, f'Exception creating {self.section} client: {e}')) - return res +from Helper.log_level import Level +from .config_base import validable, restApi class Grafana(restApi): @@ -283,13 +229,13 @@ def __init__(self): def Reload(self): if not exists(Config.FILENAME): - copy('Helper/default_config', Config.FILENAME) + copy('Settings/default_config', Config.FILENAME) try: with open(Config.FILENAME, 'r', encoding='utf-8') as file: Config.data = yaml.safe_load(file) except Exception as e: - from .log import Log + from Helper import Log Log.C(f"Exception while loading config file: {e}") return diff --git a/Settings/config_base.py b/Settings/config_base.py new file mode 100644 index 0000000..e4265ad --- /dev/null +++ b/Settings/config_base.py @@ -0,0 +1,57 @@ +from typing import Dict, List, Tuple, Optional +from Helper.log_level import Level +from REST import RestClient + + +class validable: + def __init__(self, data: Dict, section: str, + defaults: Dict[str, Tuple[Optional[object], "Level"]]): + self.data = data + self.section = section + self.defaults = defaults + + def _keyOrDefault(self, key: str): + if key in self.data.keys(): + return self.data[key] + else: + default = self.defaults.get(key, None) + return default[0] if default is not None else None + + @property + def Validation(self) -> List[Tuple['Level', str]]: + res = [] + for key in self.defaults.keys(): + if key not in self.data: + default, level = self.defaults[key] + defaultText = f", using default '{default}'" if default is not None else "" + res.append((level, f"'{key}' not defined under '{self.section}'{defaultText}")) + if len(res) == 0: + values = '; '.join([f'{key}: {self.data[key]}' for key in self.defaults.keys()]) + res.append((Level.INFO, f'{self.section} [{values}]')) + return res + + +class restApi(validable): + def __init__(self, data: Dict, section: str, defaults: Dict[str, Tuple[Optional[object], "Level"]]): + if 'Host' not in defaults.keys(): defaults['Host'] = (None, Level.ERROR) + if 'Port' not in defaults.keys(): defaults['Port'] = (None, Level.ERROR) + super().__init__(data, section, defaults) + + @property + def Host(self): + return self._keyOrDefault('Host') + + @property + def Port(self): + return self._keyOrDefault('Port') + + @property + def Validation(self) -> List[Tuple['Level', str]]: + res = super().Validation + if all([e[0] == Level.INFO for e in res]): + # No errors, but check if a rest server can be created with the configuration + try: + _ = RestClient(self.Host, self.Port, "") + except Exception as e: + res.append((Level.ERROR, f'Exception creating {self.section} client: {e}')) + return res \ No newline at end of file diff --git a/Helper/default_config b/Settings/default_config similarity index 92% rename from Helper/default_config rename to Settings/default_config index 7003a73..143da1b 100644 --- a/Helper/default_config +++ b/Settings/default_config @@ -44,4 +44,8 @@ EastWest: Port: port1 ExampleRemote2: Host: host1 - Port: port1 \ No newline at end of file + Port: port1 +Evolved5g: + Jenkins: + User: + Password: \ No newline at end of file From 96a15783a4c017a92fe34386857cbedda46eca1d Mon Sep 17 00:00:00 2001 From: Bruno Garcia Date: Fri, 22 Oct 2021 11:20:39 +0200 Subject: [PATCH 06/14] Add Evolved5g configuration file --- .gitignore | 1 + Scheduler/__init__.py | 19 ++++++++++++------- Settings/__init__.py | 1 + Settings/config.py | 24 +++++------------------- Settings/config_base.py | 26 +++++++++++++++++++++++++- Settings/default_config | 4 ---- Settings/default_evolved_config | 4 ++++ Settings/evolved_config.py | 33 +++++++++++++++++++++++++++++++++ 8 files changed, 81 insertions(+), 31 deletions(-) create mode 100644 Settings/default_evolved_config create mode 100644 Settings/evolved_config.py diff --git a/.gitignore b/.gitignore index 55860ba..b7a3c51 100644 --- a/.gitignore +++ b/.gitignore @@ -101,6 +101,7 @@ ENV/ Persistence/ config.yml facility.yml +evolved5g.yml *.log.* TestCases/ UEs/ diff --git a/Scheduler/__init__.py b/Scheduler/__init__.py index 4f7ef08..6211c4f 100644 --- a/Scheduler/__init__.py +++ b/Scheduler/__init__.py @@ -3,7 +3,7 @@ from Status import Status from flask_bootstrap import Bootstrap from flask_moment import Moment -from Settings import Config +from Settings import Config, EvolvedConfig from Facility import Facility from .heartbeat import HeartBeat from dotenv import load_dotenv @@ -11,15 +11,20 @@ load_dotenv(dotenv_path='./.flaskenv', verbose=True) +def _showValidation(name, validation): + print(f"{name} validation:") + for level, message in validation: + print(f" {level.name:8}: {message}", flush=True) + + config = Config() -print("Config validation:") -for level, message in config.Validation: - print(f" {level.name:8}: {message}", flush=True) +_showValidation("Config", config.Validation) + +evolvedConfig = EvolvedConfig() +_showValidation("Evolved5g Config", evolvedConfig.Validation) Facility.Reload() -print("Facility validation:") -for level, message in Facility.Validation: - print(f" {level.name:8}: {message}", flush=True) +_showValidation("Facility", Facility.Validation) app = Flask(__name__) app.secret_key = os.getenv('SECRET_KEY') diff --git a/Settings/__init__.py b/Settings/__init__.py index 4de995b..fe46d30 100644 --- a/Settings/__init__.py +++ b/Settings/__init__.py @@ -1 +1,2 @@ from .config import Config, TapConfig +from .evolved_config import EvolvedConfig diff --git a/Settings/config.py b/Settings/config.py index dc8abfc..ec574f8 100644 --- a/Settings/config.py +++ b/Settings/config.py @@ -1,12 +1,10 @@ -import yaml from os.path import exists, abspath, realpath, join from os import getenv -from shutil import copy from typing import Dict, List, Tuple, Optional import logging import platform from Helper.log_level import Level -from .config_base import validable, restApi +from .config_base import validable, restApi, ConfigBase class Grafana(restApi): @@ -217,28 +215,16 @@ def HostIp(self): return self._keyOrDefault("HostIp") def Facility(self): return self._keyOrDefault("Facility") -class Config: +class Config(ConfigBase): FILENAME = 'config.yml' data = None Validation: List[Tuple['Level', str]] = [] def __init__(self): - if Config.data is None: - self.Reload() - - def Reload(self): - if not exists(Config.FILENAME): - copy('Settings/default_config', Config.FILENAME) - - try: - with open(Config.FILENAME, 'r', encoding='utf-8') as file: - Config.data = yaml.safe_load(file) - except Exception as e: - from Helper import Log - Log.C(f"Exception while loading config file: {e}") - return - + super().__init__('config.yml', 'Settings/default_config') + if self.data is None: + Config.data = self.Reload() self.Validate() @property diff --git a/Settings/config_base.py b/Settings/config_base.py index e4265ad..7a98a35 100644 --- a/Settings/config_base.py +++ b/Settings/config_base.py @@ -1,3 +1,6 @@ +import yaml +from os.path import exists +from shutil import copy from typing import Dict, List, Tuple, Optional from Helper.log_level import Level from REST import RestClient @@ -54,4 +57,25 @@ def Validation(self) -> List[Tuple['Level', str]]: _ = RestClient(self.Host, self.Port, "") except Exception as e: res.append((Level.ERROR, f'Exception creating {self.section} client: {e}')) - return res \ No newline at end of file + return res + + +class ConfigBase: + def __init__(self, filename: str, defaultsFile: str): + self.filename = filename + self.defaultsFile = defaultsFile + + def Reload(self) -> Dict: + if not exists(self.filename): + copy(self.defaultsFile, self.filename) + + try: + with open(self.filename, 'r', encoding='utf-8') as file: + return yaml.safe_load(file) + except Exception as e: + from Helper import Log + Log.C(f"Exception while loading {self.filename} file: {e}") + return {} + + def Validate(self): + raise NotImplementedError diff --git a/Settings/default_config b/Settings/default_config index 143da1b..cbeb582 100644 --- a/Settings/default_config +++ b/Settings/default_config @@ -45,7 +45,3 @@ EastWest: ExampleRemote2: Host: host1 Port: port1 -Evolved5g: - Jenkins: - User: - Password: \ No newline at end of file diff --git a/Settings/default_evolved_config b/Settings/default_evolved_config new file mode 100644 index 0000000..ba31edc --- /dev/null +++ b/Settings/default_evolved_config @@ -0,0 +1,4 @@ +Jenkins: + Enabled: False + User: + Password: \ No newline at end of file diff --git a/Settings/evolved_config.py b/Settings/evolved_config.py new file mode 100644 index 0000000..8f2f108 --- /dev/null +++ b/Settings/evolved_config.py @@ -0,0 +1,33 @@ +from typing import Dict, List, Tuple, Optional +from Helper.log_level import Level +from .config_base import validable, restApi, ConfigBase + + +class JenkisApi(restApi): + def __init__(self, data: Dict): + defaults = { + 'Enabled': (False, Level.WARNING) + } + super().__init__(data, 'Portal', defaults) + + @property + def Enabled(self): + return self._keyOrDefault('Enabled') + + +class EvolvedConfig(ConfigBase): + data = None + Validation: List[Tuple['Level', str]] = [] + + def __init__(self): + super().__init__('evolved5g.yml', 'Settings/default_evolved_config') + if self.data is None: + EvolvedConfig.data = self.Reload() + self.Validate() + + @property + def Portal(self): + return JenkisApi(EvolvedConfig.data.get('Portal', {})) + + def Validate(self): + pass From ee247cc2cc9551887952eaa303b1d472e43c3966 Mon Sep 17 00:00:00 2001 From: Bruno Garcia Date: Fri, 22 Oct 2021 12:18:25 +0200 Subject: [PATCH 07/14] Add Evolved5g config to web interface --- .gitignore | 1 + Scheduler/routes.py | 43 ++++++++++++++++++++++------------ Scheduler/templates/index.html | 7 ++++++ Settings/config.py | 13 +++++++--- Settings/evolved_config.py | 24 ++++++++++++------- 5 files changed, 62 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index b7a3c51..7095ade 100644 --- a/.gitignore +++ b/.gitignore @@ -103,6 +103,7 @@ config.yml facility.yml evolved5g.yml *.log.* +.flaskenv TestCases/ UEs/ Resources/ diff --git a/Scheduler/routes.py b/Scheduler/routes.py index 0a77ff0..8cc87a4 100644 --- a/Scheduler/routes.py +++ b/Scheduler/routes.py @@ -5,7 +5,7 @@ from functools import wraps, update_wrapper from datetime import datetime from Helper import Log, Serialize, LogInfo -from Settings import Config +from Settings import Config, EvolvedConfig from Facility import Facility from typing import List, Dict from flask_paginate import Pagination, get_page_parameter @@ -28,12 +28,14 @@ def no_cache(*args, **kwargs): @nocache def index(): config = Config() + evolved = EvolvedConfig() configLog = LogInfo.FromTuple(config.Validation) + evolvedLog = LogInfo.FromTuple(evolved.Validation) facilityLog = LogInfo.FromTuple(Facility.Validation) resources = Facility.Resources() return render_template('index.html', executionId=Status.PeekNextId(), executions=ExecutionQueue.Retrieve(), resources=resources, - configLog=configLog, facilityLog=facilityLog) + configLog=configLog, evolvedLog=evolvedLog, facilityLog=facilityLog) @app.route("/log") @@ -62,20 +64,31 @@ def history(): @app.route("/reload_config") def reloadConfig(): - config = Config() - config.Reload() - Log.I("Configuration reloaded:") - for level, message in config.Validation: - Log.Log(level, message) - flash("Reloaded configuration") - return redirect(url_for('index')) + try: + configurations = [('Configuration', Config(forceReload=True)), + ('Evolved5g Configuration', EvolvedConfig(forceReload=True))] + + for name, config in configurations: + Log.I(f"{name} reloaded:") + for level, message in config.Validation: + Log.Log(level, message) + + flash("Reloaded all configurations", "info") + except Exception as e: + flash(f"Exception while reloading configurations: {e}", "error") + finally: + return redirect(url_for('index')) @app.route("/reload_facility") def reloadFacility(): - Facility.Reload() - Log.I("Facility reloaded:") - for level, message in Facility.Validation: - Log.Log(level, message) - flash("Reloaded Facility") - return redirect(url_for('index')) + try: + Facility.Reload() + Log.I("Facility reloaded:") + for level, message in Facility.Validation: + Log.Log(level, message) + flash("Reloaded Facility", "info") + except Exception as e: + flash(f"Exception while reloading facility: {e}", "error") + finally: + return redirect(url_for('index')) diff --git a/Scheduler/templates/index.html b/Scheduler/templates/index.html index 31d26e1..ee18911 100644 --- a/Scheduler/templates/index.html +++ b/Scheduler/templates/index.html @@ -53,6 +53,13 @@

Diagnostics


+
+
+ {# Some identifiers are generated from the accordion name, so whitespaces are not usable #} + {{ logView.logAccordionCard('evolvedAccordion', 'Evolved5g', evolvedLog) }} +
+
+
{{ logView.logAccordionCard('facilityAccordion', 'Facility', facilityLog) }} diff --git a/Settings/config.py b/Settings/config.py index ec574f8..7e4a9ca 100644 --- a/Settings/config.py +++ b/Settings/config.py @@ -124,6 +124,13 @@ def __init__(self, data: Dict): def Enabled(self): return self._keyOrDefault('Enabled') + @property + def Validation(self) -> List[Tuple['Level', str]]: + if self.Enabled: + return super().Validation + else: + return [(Level.INFO, "Portal is disabled")] + class SliceManager(restApi): def __init__(self, data: Dict): @@ -221,11 +228,11 @@ class Config(ConfigBase): data = None Validation: List[Tuple['Level', str]] = [] - def __init__(self): + def __init__(self, forceReload = False): super().__init__('config.yml', 'Settings/default_config') - if self.data is None: + if self.data is None or forceReload: Config.data = self.Reload() - self.Validate() + self.Validate() @property def Logging(self): diff --git a/Settings/evolved_config.py b/Settings/evolved_config.py index 8f2f108..a1afb27 100644 --- a/Settings/evolved_config.py +++ b/Settings/evolved_config.py @@ -3,31 +3,39 @@ from .config_base import validable, restApi, ConfigBase -class JenkisApi(restApi): +class JenkinsApi(restApi): def __init__(self, data: Dict): defaults = { 'Enabled': (False, Level.WARNING) } - super().__init__(data, 'Portal', defaults) + super().__init__(data, 'JenkinsApi', defaults) @property def Enabled(self): return self._keyOrDefault('Enabled') + @property + def Validation(self) -> List[Tuple['Level', str]]: + if self.Enabled: + return super().Validation + else: + return [(Level.INFO, "Jenkins API is disabled")] + class EvolvedConfig(ConfigBase): data = None Validation: List[Tuple['Level', str]] = [] - def __init__(self): + def __init__(self, forceReload=False): super().__init__('evolved5g.yml', 'Settings/default_evolved_config') - if self.data is None: + if self.data is None or forceReload: EvolvedConfig.data = self.Reload() - self.Validate() + self.Validate() @property - def Portal(self): - return JenkisApi(EvolvedConfig.data.get('Portal', {})) + def JenkinsApi(self): + return JenkinsApi(EvolvedConfig.data.get('JenkinsApi', {})) def Validate(self): - pass + for entry in [self.JenkinsApi, ]: + EvolvedConfig.Validation.extend(entry.Validation) From ed17b9bc036e450e9ec1cc5731fcf5b3e4382cda Mon Sep 17 00:00:00 2001 From: Bruno Garcia Date: Mon, 25 Oct 2021 09:36:15 +0200 Subject: [PATCH 08/14] Add empty Jenkins tasks --- Executor/Tasks/Evolved5g/__init__.py | 1 + Executor/Tasks/Evolved5g/jenkins_api.py | 30 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 Executor/Tasks/Evolved5g/__init__.py create mode 100644 Executor/Tasks/Evolved5g/jenkins_api.py diff --git a/Executor/Tasks/Evolved5g/__init__.py b/Executor/Tasks/Evolved5g/__init__.py new file mode 100644 index 0000000..48ea979 --- /dev/null +++ b/Executor/Tasks/Evolved5g/__init__.py @@ -0,0 +1 @@ +from .jenkins_api import JenkinsBuild, JenkinsStatus diff --git a/Executor/Tasks/Evolved5g/jenkins_api.py b/Executor/Tasks/Evolved5g/jenkins_api.py new file mode 100644 index 0000000..6f108c5 --- /dev/null +++ b/Executor/Tasks/Evolved5g/jenkins_api.py @@ -0,0 +1,30 @@ +from Task import Task +from Settings import EvolvedConfig +from Helper import Level + + +class JenkinsBase(Task): + def __init__(self, name, parent, params, logMethod): + super().__init__(name, parent, params, logMethod, None) + self.config = EvolvedConfig().JenkinsApi + + def Run(self): + raise NotImplementedError + + +class JenkinsBuild(JenkinsBase): + def __init__(self, logMethod, parent, params): + super().__init__("Jenkins Build", parent, params, logMethod) + self.paramRules = {} + + def Run(self): + pass + + +class JenkinsStatus(JenkinsBase): + def __init__(self, logMethod, parent, params): + super().__init__("Jenkins Status", parent, params, logMethod) + self.paramRules = {} + + def Run(self): + pass From 6fbea96a979b3df2191a0626124ab08776da71b6 Mon Sep 17 00:00:00 2001 From: Bruno Garcia Date: Mon, 25 Oct 2021 14:52:56 +0200 Subject: [PATCH 09/14] Add Jenkins API, implement token retrieval --- Interfaces/__init__.py | 1 + Interfaces/evolved5g_jenkins.py | 41 +++++++++++++++++++++++++++++++++ REST/rest_client.py | 4 +++- 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 Interfaces/evolved5g_jenkins.py diff --git a/Interfaces/__init__.py b/Interfaces/__init__.py index fd13c52..c554046 100644 --- a/Interfaces/__init__.py +++ b/Interfaces/__init__.py @@ -1,3 +1,4 @@ from .management import Management from .portal import PortalApi from .remote import RemoteApi +from .evolved5g_jenkins import Evolved5gJenkinsApi diff --git a/Interfaces/evolved5g_jenkins.py b/Interfaces/evolved5g_jenkins.py new file mode 100644 index 0000000..22ed83c --- /dev/null +++ b/Interfaces/evolved5g_jenkins.py @@ -0,0 +1,41 @@ +from REST import RestClient, Payload +from typing import List, Tuple, Dict, Optional +from Helper import Log +from datetime import datetime, timezone + + +class Evolved5gJenkinsApi(RestClient): + EXPIRY_MARGIN = 60 + + def __init__(self, host, port, user, password): + super().__init__(host, port, '', https=True, insecure=False) + self.basicAuth = {"username": user, "password": password} + self.token = None + self.expiry = 0 + + def RenewToken(self): + response = self.HttpPost("/api/auth", payload=Payload.Data, body=self.basicAuth) + status = self.ResponseStatusCode(response) + + if status != 200: + raise RuntimeError(f"Unexpected status {status} retrieving token: {self.ResponseToRaw(response)}") + try: + data = self.ResponseToJson(response) + self.token = data['access_token'] + delay = float(data['expires_in']) + if delay >= self.EXPIRY_MARGIN: + delay = delay - self.EXPIRY_MARGIN + self.expiry = datetime.now(timezone.utc).timestamp() + delay + except Exception as e: + raise RuntimeError(f'Could not extract token information: {e}') from e + + def MaybeRenewToken(self): + current = datetime.now(timezone.utc).timestamp() + if self.token is None or current >= self.expiry: + self.RenewToken() + + def TriggerBuild(self): + pass + + def CheckStatus(self): + pass diff --git a/REST/rest_client.py b/REST/rest_client.py index 87ba9c3..7b34ce2 100644 --- a/REST/rest_client.py +++ b/REST/rest_client.py @@ -20,7 +20,9 @@ class RestClient: FILENAME_PATTERN = re.compile(r'.*filename="?(.*)"?') def __init__(self, api_host, api_port, suffix, https=False, insecure=False): - self.api_url = f'http{"s" if https else ""}://{api_host}:{api_port}{suffix}' + protocol = f'http{"s" if https else ""}://' + port = api_port if api_port is not None else (443 if https else 80) + self.api_url = f'{protocol}{api_host}:{port}{suffix}' kw = {'maxsize': 1, 'headers': self.HEADERS} if https and insecure: From 011ebfcda078be3a88a0055d7cbcc39a19acc7c1 Mon Sep 17 00:00:00 2001 From: Bruno Garcia Date: Tue, 26 Oct 2021 09:51:14 +0200 Subject: [PATCH 10/14] Fix config, add client creation --- Executor/Tasks/Evolved5g/jenkins_api.py | 21 ++++++++++++++++++--- Settings/config_base.py | 2 +- Settings/default_evolved_config | 4 +++- Settings/evolved_config.py | 14 +++++++++++++- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/Executor/Tasks/Evolved5g/jenkins_api.py b/Executor/Tasks/Evolved5g/jenkins_api.py index 6f108c5..65dcfde 100644 --- a/Executor/Tasks/Evolved5g/jenkins_api.py +++ b/Executor/Tasks/Evolved5g/jenkins_api.py @@ -1,5 +1,6 @@ from Task import Task from Settings import EvolvedConfig +from Interfaces import Evolved5gJenkinsApi from Helper import Level @@ -7,9 +8,21 @@ class JenkinsBase(Task): def __init__(self, name, parent, params, logMethod): super().__init__(name, parent, params, logMethod, None) self.config = EvolvedConfig().JenkinsApi + self.client = None def Run(self): - raise NotImplementedError + try: + self.client = self.getApiClient() + except Exception as e: + self.Log(Level.Error, f"Unable to create Jenkins API client: {e}") + self.client = None + + def getApiClient(self) -> Evolved5gJenkinsApi: + if not self.config.Enabled: + raise RuntimeError(f"Trying to run {self.name} Task while Jenkins API is not enabled") + + return Evolved5gJenkinsApi(self.config.Host, self.config.Port, + self.config.User, self.config.Password) class JenkinsBuild(JenkinsBase): @@ -18,7 +31,8 @@ def __init__(self, logMethod, parent, params): self.paramRules = {} def Run(self): - pass + super().Run() + if self.client is None: return class JenkinsStatus(JenkinsBase): @@ -27,4 +41,5 @@ def __init__(self, logMethod, parent, params): self.paramRules = {} def Run(self): - pass + super().Run() + if self.client is None: return diff --git a/Settings/config_base.py b/Settings/config_base.py index 7a98a35..4e3f28e 100644 --- a/Settings/config_base.py +++ b/Settings/config_base.py @@ -54,7 +54,7 @@ def Validation(self) -> List[Tuple['Level', str]]: if all([e[0] == Level.INFO for e in res]): # No errors, but check if a rest server can be created with the configuration try: - _ = RestClient(self.Host, self.Port, "") + _ = RestClient(self.Host, self.Port if self.Port is not None else 80, "") except Exception as e: res.append((Level.ERROR, f'Exception creating {self.section} client: {e}')) return res diff --git a/Settings/default_evolved_config b/Settings/default_evolved_config index ba31edc..78bfb45 100644 --- a/Settings/default_evolved_config +++ b/Settings/default_evolved_config @@ -1,4 +1,6 @@ -Jenkins: +JenkinsApi: Enabled: False + Host: + Port: User: Password: \ No newline at end of file diff --git a/Settings/evolved_config.py b/Settings/evolved_config.py index a1afb27..69c7839 100644 --- a/Settings/evolved_config.py +++ b/Settings/evolved_config.py @@ -6,7 +6,9 @@ class JenkinsApi(restApi): def __init__(self, data: Dict): defaults = { - 'Enabled': (False, Level.WARNING) + 'Enabled': (False, Level.WARNING), + 'User': (None, Level.ERROR), + 'Password': (None, Level.ERROR), } super().__init__(data, 'JenkinsApi', defaults) @@ -14,6 +16,14 @@ def __init__(self, data: Dict): def Enabled(self): return self._keyOrDefault('Enabled') + @property + def User(self): + return self._keyOrDefault('User') + + @property + def Password(self): + return self._keyOrDefault('Password') + @property def Validation(self) -> List[Tuple['Level', str]]: if self.Enabled: @@ -37,5 +47,7 @@ def JenkinsApi(self): return JenkinsApi(EvolvedConfig.data.get('JenkinsApi', {})) def Validate(self): + EvolvedConfig.Validation = [] + for entry in [self.JenkinsApi, ]: EvolvedConfig.Validation.extend(entry.Validation) From 50bf3ff9a739f5bb0804c0fbb056734fd68b5647 Mon Sep 17 00:00:00 2001 From: Bruno Garcia Date: Tue, 26 Oct 2021 13:06:41 +0200 Subject: [PATCH 11/14] Implement tasks, first pass implementation of API methods --- Executor/Tasks/Evolved5g/__init__.py | 2 +- Executor/Tasks/Evolved5g/jenkins_api.py | 43 ++++++++++++++++++++++--- Interfaces/evolved5g_jenkins.py | 40 ++++++++++++++++++++--- 3 files changed, 76 insertions(+), 9 deletions(-) diff --git a/Executor/Tasks/Evolved5g/__init__.py b/Executor/Tasks/Evolved5g/__init__.py index 48ea979..aca04fb 100644 --- a/Executor/Tasks/Evolved5g/__init__.py +++ b/Executor/Tasks/Evolved5g/__init__.py @@ -1 +1 @@ -from .jenkins_api import JenkinsBuild, JenkinsStatus +from .jenkins_api import JenkinsJob, JenkinsStatus diff --git a/Executor/Tasks/Evolved5g/jenkins_api.py b/Executor/Tasks/Evolved5g/jenkins_api.py index 65dcfde..6fb276b 100644 --- a/Executor/Tasks/Evolved5g/jenkins_api.py +++ b/Executor/Tasks/Evolved5g/jenkins_api.py @@ -25,21 +25,56 @@ def getApiClient(self) -> Evolved5gJenkinsApi: self.config.User, self.config.Password) -class JenkinsBuild(JenkinsBase): +class JenkinsJob(JenkinsBase): def __init__(self, logMethod, parent, params): - super().__init__("Jenkins Build", parent, params, logMethod) - self.paramRules = {} + super().__init__("Jenkins Job", parent, params, logMethod) + self.paramRules = { + 'Instance': (None, True), + 'Job': (None, True), + 'GitUrl': (None, True), + 'GitBranch': (None, True), + 'Version': ('1.0', False), + 'PublishKey': ('JenkinsJobId', False), + } def Run(self): super().Run() if self.client is None: return + instance = self.params["Instance"] + job = self.params["Job"] + url = self.params["GitUrl"] + branch = self.params["GitBranch"] + version = self.params["Version"] + + self.Log(Level.DEBUG, + f"Trying to trigger job '{job}' on instance '{instance}' ({url}|{branch}|{version})") + + try: + jobId = self.client.TriggerJob(instance, job, url, branch, version) + self.Log(Level.INFO, f"Triggered '{job}'. Received Job Id: {jobId}") + self.Publish(self.params["PublishKey"], jobId) + except Exception as e: + self.Log(Level.ERROR, f"Unable to trigger job: {e}") + class JenkinsStatus(JenkinsBase): def __init__(self, logMethod, parent, params): super().__init__("Jenkins Status", parent, params, logMethod) - self.paramRules = {} + self.paramRules = { + 'JobId': (None, True), + 'PublishKey': ('JenkinsJobStatus', False), + } def Run(self): super().Run() if self.client is None: return + + jobId = self.params['JobId'] + + try: + status = self.client.CheckJob(jobId) + self.Log(Level.INFO, f"Status of job '{jobId}': {status}") + self.Publish(self.params["PublishKey"], status) + except Exception as e: + self.Log(Level.ERROR, f"Unable to check job '{jobId}' status: {e}") diff --git a/Interfaces/evolved5g_jenkins.py b/Interfaces/evolved5g_jenkins.py index 22ed83c..6552e90 100644 --- a/Interfaces/evolved5g_jenkins.py +++ b/Interfaces/evolved5g_jenkins.py @@ -34,8 +34,40 @@ def MaybeRenewToken(self): if self.token is None or current >= self.expiry: self.RenewToken() - def TriggerBuild(self): - pass + def getExtraHeaders(self): + self.MaybeRenewToken() + return {"Content-Type": "application/json", "Authorization": self.token} - def CheckStatus(self): - pass + def TriggerJob(self, instance: str, job: str, repository: str, branch: str, version: str) -> str: + headers = self.getExtraHeaders() + payload = { + "instance": instance, + "job": job, + "parameters": { + "VERSION": version, "GIT_URL": repository, "GIT_BRANCH": branch + } + } + + try: + response = self.HttpPost("/api/executions", payload=Payload.Data, body=payload, extra_headers=headers) + return "" # TODO + except Exception as e: + raise RuntimeError(f"Unable to trigger job: {e}") from e + + def CheckJob(self, jobId: str) -> str: + headers = self.getExtraHeaders() + + try: + response = self.HttpGet(f"/api/executions/{jobId}", extra_headers=headers) + status = self.ResponseStatusCode(response) + + if 200 <= status <= 299: + return "Correct" # TODO + elif status == 401: + return "Unauthorized" + elif status == 404: + return "Not Found" + else: + raise RuntimeError(f"Unrecognized status code: {status}") + except Exception as e: + raise RuntimeError(f"Unable to retrieve job status: {e}") from e From 282d1b4ca4ddfadde1fbea14389c7d7c751fce50 Mon Sep 17 00:00:00 2001 From: nanitebased Date: Thu, 4 Nov 2021 11:23:23 +0100 Subject: [PATCH 12/14] Handle Jenkins responses --- Executor/Tasks/Evolved5g/jenkins_api.py | 5 +++-- Interfaces/evolved5g_jenkins.py | 19 ++++++++++++------- Scheduler/__init__.py | 1 + 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Executor/Tasks/Evolved5g/jenkins_api.py b/Executor/Tasks/Evolved5g/jenkins_api.py index 6fb276b..2ef3b15 100644 --- a/Executor/Tasks/Evolved5g/jenkins_api.py +++ b/Executor/Tasks/Evolved5g/jenkins_api.py @@ -73,8 +73,9 @@ def Run(self): jobId = self.params['JobId'] try: - status = self.client.CheckJob(jobId) - self.Log(Level.INFO, f"Status of job '{jobId}': {status}") + status, message = self.client.CheckJob(jobId) + message = message if message is not None else "" + self.Log(Level.INFO, f"Status of job '{jobId}': {status} ('{message}')") self.Publish(self.params["PublishKey"], status) except Exception as e: self.Log(Level.ERROR, f"Unable to check job '{jobId}' status: {e}") diff --git a/Interfaces/evolved5g_jenkins.py b/Interfaces/evolved5g_jenkins.py index 6552e90..78d289a 100644 --- a/Interfaces/evolved5g_jenkins.py +++ b/Interfaces/evolved5g_jenkins.py @@ -1,6 +1,4 @@ from REST import RestClient, Payload -from typing import List, Tuple, Dict, Optional -from Helper import Log from datetime import datetime, timezone @@ -50,11 +48,16 @@ def TriggerJob(self, instance: str, job: str, repository: str, branch: str, vers try: response = self.HttpPost("/api/executions", payload=Payload.Data, body=payload, extra_headers=headers) - return "" # TODO + status = self.ResponseStatusCode(response) + if 200 <= status <= 299: + data = self.ResponseToJson(response) + return data['id'] + else: + raise RuntimeError(f"Unexpected status code {status} received.") except Exception as e: raise RuntimeError(f"Unable to trigger job: {e}") from e - def CheckJob(self, jobId: str) -> str: + def CheckJob(self, jobId: str) -> (str, None | str): headers = self.getExtraHeaders() try: @@ -62,11 +65,13 @@ def CheckJob(self, jobId: str) -> str: status = self.ResponseStatusCode(response) if 200 <= status <= 299: - return "Correct" # TODO + data = self.ResponseToJson(response) + return data['status'], data.get('console_log', None) + elif status == 401: - return "Unauthorized" + return "401 - Unauthorized", None elif status == 404: - return "Not Found" + return "404 - Not Found", None else: raise RuntimeError(f"Unrecognized status code: {status}") except Exception as e: diff --git a/Scheduler/__init__.py b/Scheduler/__init__.py index 6211c4f..1818485 100644 --- a/Scheduler/__init__.py +++ b/Scheduler/__init__.py @@ -11,6 +11,7 @@ load_dotenv(dotenv_path='./.flaskenv', verbose=True) + def _showValidation(name, validation): print(f"{name} validation:") for level, message in validation: From 7c386b0665e5546fee0902493b2a645baacdf07d Mon Sep 17 00:00:00 2001 From: nanitebased Date: Thu, 4 Nov 2021 12:24:06 +0100 Subject: [PATCH 13/14] Allow checking the status of non-running experiments --- Scheduler/execution/routes.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Scheduler/execution/routes.py b/Scheduler/execution/routes.py index 1705ece..4994785 100644 --- a/Scheduler/execution/routes.py +++ b/Scheduler/execution/routes.py @@ -38,15 +38,18 @@ def view(executionId: int): @bp.route('/json') def json(executionId: int): - execution = ExecutionQueue.Find(executionId) + execution = executionOrTombstone(executionId) coarse = status = 'ERR' percent = 0 messages = [] if execution is not None: coarse = execution.CoarseStatus.name - status = execution.Status - percent = execution.PerCent - messages = execution.Messages + if isinstance(execution, Tombstone): + status = "Not Running" + else: + status = execution.Status + percent = execution.PerCent + messages = execution.Messages return jsonify({ 'Coarse': coarse, 'Status': status, 'PerCent': percent, 'Messages': messages From 42547d55e681deb941de5dfb396555ccb24c242e Mon Sep 17 00:00:00 2001 From: nanitebased Date: Fri, 5 Nov 2021 09:16:44 +0100 Subject: [PATCH 14/14] Update documentation --- CHANGELOG.md | 6 ++++++ README.md | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e59dd4..d8a363b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +**05/11/2021** [Version 3.0.1] + + - Implement RestApi, JenkinsJob, JenkinsStatus tasks + - Add separate EVOLVED-5G configuration file + - Allow checking the status of finished experiments + **11/10/2021** [Version 3.0.0] - Initial EVOLVED-5G release diff --git a/README.md b/README.md index b946277..8050439 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,16 @@ The ELCM instance can be configured by editing the `config.yml` file. The values containing 'Host' and 'Port' values in the same format as in the `Portal` or `SliceManager` sections. Defaults to two (invalid) example entries. +An additional configuration file (`evolved5g.yml`) contains the configuration values for the EVOLVED-5G tasks. The +configuration values in this file are: + +* JenkinsApi: Configuration values for the EVOLVED-5G CI/CD infrastructure. + * Enabled: Boolean value indicating if the Jenkins API will be used. Defaults to `False`. + * Host: Address where the Jenkins API is listening + * Port: Port where the API is listening. If empty, the default ports (80 for http, 443 for https) will be used + * User: Provided user name + * Password: Provided password + ## Facility Configuration (Platform registry) The exposed capabilities and functionality of the facility are defined by a set of files distributed in 4 folders @@ -542,6 +552,31 @@ Separate values from the `Parameters` dictionary can also be expanded using the > a '.' the ELCM will fall back to looking at the Publish values (the default for Release A). If the collection > is not 'Publish' or 'Params', the expression will be replaced by `<>` +## EVOLVED-5G specific tasks: + +The following is a list of tasks specifically tailored for use on the EVOLVED-5G H2020 project. Configuration +values for these tasks can be set in the `evolved5g.yml` file at the root of the ELCM folder. + +### Evolved5g.JenkinsJob + +Initiates the execution of a Jenkins pipeline in the CI/CD infrastructure. The returned job ID will be published +as a variable for use later in the same experiment. Configuration values: + +- Instance: Address of the instance where the pipeline will be launched. +- Job: Kind of job to launch. +- GitUrl: URL of the GitHub repository that contains the NetApp code. +- GitBranch: Repository branch that will be used by the pipeline. +- Version: Pipeline version, defaults to `'1.0'` +- PublishKey: Name of the key that will be used for storing the returned job id, defaults to `JenkinsJobId` + +### Evolved5g.JenkinsStatus + +Checks the status of the specified pipeline, and publishes the obtained value for later use in the same experiment. +Configuration values: + +- JobId: Pipeline to check. Can be expanded from a previous JenkinsJob using `'@[Params.JenkinsJobId]'`. +- PublishKey: Name of the key that will be used for storing the returned status, defaults to `JenkinsJobStatus` + ## Implementing additional tasks: The ELCM is designed to be extensible, and thus, it is possible to easily integrate additional tasks. The basic steps @@ -557,7 +592,7 @@ class must have the signature displayed below: super().__init__("Compress Files", parent, params, logMethod, None) ``` In general, the parameters received in the constructor will be sent directly to the superclass, along with a Task -name (in the first parameter). The last parameter is an optional "Condition" method. If the callable passed in this +name (in the first parameter). The last parameter is an optional "Condition" method. If the `callable` passed in this parameter evaluates to False, the execution of the Task will be skipped. 3. Override the `Run` method of the Task class. If this method is not overridden, a `NotImplementedError` will be raised at runtime. The following methods and fields are available: