[resotocore][feat] Separate custom core command configuration (#780)

* [resotocore][feat] Separate custom core command configuration

resotocore/resotocore/cli/cli.py

@@ -229,7 +229,7 @@ def __init__(
         alias_names: Dict[str, str],
     ):
         dependencies.extend(cli=self)
-        alias_templates = [AliasTemplate.from_config(alias) for alias in dependencies.config.cli.alias_templates]
+        alias_templates = [AliasTemplate.from_config(cmd) for cmd in dependencies.config.custom_commands.commands]
         help_cmd = HelpCommand(dependencies, parts, alias_names, alias_templates)
         cmds = {p.name: p for p in parts + [help_cmd]}
         alias_cmds = {alias: cmds[name] for alias, name in alias_names.items() if name in cmds and alias not in cmds}

resotocore/resotocore/config/config_handler_service.py

@@ -120,7 +120,7 @@ async def config_yaml(self, cfg_id: str, revision: bool = False) -> Optional[str
                         yaml_str += "\n"
                     yaml_str += key + ":" + part
                 else:
-                    yaml_str += yaml.dump({key: value}, sort_keys=False)
+                    yaml_str += yaml.dump({key: value}, sort_keys=False, allow_unicode=True)
 
             # mix the revision into the yaml document
             if revision and config.revision:

resotocore/resotocore/config/core_config_handler.py

@@ -8,11 +8,21 @@
 import yaml
 
 from resotocore.config import ConfigHandler, ConfigEntity, ConfigValidation
-from resotocore.core_config import CoreConfig, ResotoCoreConfigId, config_model, EditableConfig, ResotoCoreRoot
+from resotocore.core_config import (
+    CoreConfig,
+    ResotoCoreConfigId,
+    config_model,
+    EditableConfig,
+    ResotoCoreRoot,
+    ResotoCoreCommandsConfigId,
+    ResotoCoreCommandsRoot,
+    CustomCommandsConfig,
+)
 from resotocore.dependencies import empty_config
 from resotocore.message_bus import MessageBus, CoreMessage
 from resotocore.model.model import Kind
 from resotocore.model.typed_model import from_js
+from resotocore.types import Json
 from resotocore.util import deep_merge, restart_service, value_in_path
 from resotocore.worker_task_queue import WorkerTaskQueue, WorkerTaskDescription, WorkerTaskName, WorkerTask
 
@@ -36,59 +46,100 @@ def __init__(
         self.config_handler = config_handler
         self.exit_fn = exit_fn
 
+    @staticmethod
+    def validate_config_entry(task_data: Json) -> Optional[Json]:
+        def validate_core_config() -> Optional[Json]:
+            config = value_in_path(task_data, ["config", ResotoCoreRoot])
+            if isinstance(config, dict):
+                # try to read editable config, throws if there are errors
+                read = from_js(config, EditableConfig)
+                return read.validate()
+            else:
+                return {"error": "Expected a json object"}
+
+        def validate_commands_config() -> Optional[Json]:
+            config = value_in_path(task_data, ["config", ResotoCoreCommandsRoot])
+            if isinstance(config, dict):
+                # try to read editable config, throws if there are errors
+                read = from_js(config, CustomCommandsConfig)
+                return read.validate()
+            else:
+                return {"error": "Expected a json object"}
+
+        holder = value_in_path(task_data, ["config"])
+        if not isinstance(holder, dict):
+            return {"error": "Expected a json object in config"}
+        elif ResotoCoreRoot in holder:
+            return validate_core_config()
+        elif ResotoCoreCommandsRoot in holder:
+            return validate_commands_config()
+        else:
+            return {"error": "No known configuration found"}
+
     async def __validate_config(self) -> None:
         worker_id = "resotocore.config.validate"
-        description = WorkerTaskDescription(WorkerTaskName.validate_config, {"config_id": [ResotoCoreConfigId]})
+        description = WorkerTaskDescription(
+            WorkerTaskName.validate_config, {"config_id": [ResotoCoreConfigId, ResotoCoreCommandsConfigId]}
+        )
         async with self.worker_task_queue.attach(worker_id, [description]) as tasks:
             while True:
                 task: WorkerTask = await tasks.get()
                 try:
-                    config = value_in_path(task.data, ["config", ResotoCoreRoot])
-                    if isinstance(config, dict):
-                        # try to read editable config, throws if there are errors
-                        from_js(config, EditableConfig)
-                        errors = EditableConfig.validate_config(config)
-                        if errors:
-                            message = "Validation Errors:\n" + yaml.safe_dump(errors)
-                            await self.worker_task_queue.error_task(worker_id, task.id, message)
-                        else:
-                            await self.worker_task_queue.acknowledge_task(worker_id, task.id)
-                        continue
+                    errors = self.validate_config_entry(task.data)
+                    if errors:
+                        message = "Validation Errors:\n" + yaml.safe_dump(errors)
+                        await self.worker_task_queue.error_task(worker_id, task.id, message)
+                    else:
+                        await self.worker_task_queue.acknowledge_task(worker_id, task.id)
                 except Exception as ex:
                     log.warning("Error processing validate configuration task", exc_info=ex)
                     await self.worker_task_queue.error_task(worker_id, task.id, str(ex))
-                    continue
-                # safeguard, if we should ever come here
-                await self.worker_task_queue.error_task(worker_id, task.id, "Failing to process the task!")
 
     async def __handle_events(self) -> None:
         async with self.message_bus.subscribe("resotocore.config.update", [CoreMessage.ConfigUpdated]) as events:
             while True:
                 event = await events.get()
-                if event.data.get("id") == ResotoCoreConfigId:
-                    log.info("Core config was updated. Restart to take effect.")
+                event_id = event.data.get("id")
+                if event_id in (ResotoCoreConfigId, ResotoCoreCommandsConfigId):
+                    log.info(f"Core config was updated: {event_id} Restart to take effect.")
                     # stop the process and rely on os to restart the service
                     self.exit_fn()
 
     async def __update_config(self) -> None:
+        # in case the internal configuration holds new properties, we update the existing config always.
         try:
-            # in case the internal configuration holds new properties, we update the existing config always.
             existing = await self.config_handler.get_config(ResotoCoreConfigId)
             empty = empty_config().json()
             updated = deep_merge(empty, existing.config) if existing else empty
             if existing is None or updated != existing.config:
                 await self.config_handler.put_config(ConfigEntity(ResotoCoreConfigId, updated), False)
                 log.info("Default resoto config updated.")
+
         except Exception as ex:
             log.error(f"Could not update resoto default configuration: {ex}", exc_info=ex)
 
+        # make sure there is a default command configuration
+        # note: this configuration is only created one time and never updated
+        try:
+            existing_commands = await self.config_handler.get_config(ResotoCoreCommandsConfigId)
+            if existing_commands is None:
+                await self.config_handler.put_config(
+                    ConfigEntity(ResotoCoreCommandsConfigId, CustomCommandsConfig().json()), False
+                )
+                log.info("Default resoto commands config updated.")
+        except Exception as ex:
+            log.error(f"Could not update resoto command configuration: {ex}", exc_info=ex)
+
     async def __update_model(self) -> None:
         try:
             kinds = from_js(config_model(), List[Kind])
             await self.config_handler.update_configs_model(kinds)
             await self.config_handler.put_config_validation(
                 ConfigValidation(ResotoCoreConfigId, external_validation=True)
             )
+            await self.config_handler.put_config_validation(
+                ConfigValidation(ResotoCoreCommandsConfigId, external_validation=True)
+            )
             log.debug("Resoto core config model updated.")
         except Exception as ex:
             log.error(f"Could not update resoto core config model: {ex}", exc_info=ex)

resotocore/resotocore/core_config.py

@@ -20,9 +20,16 @@
 
 log = logging.getLogger(__name__)
 
+# ids used in the config store
 ResotoCoreConfigId = "resoto.core"
+ResotoCoreCommandsConfigId = "resoto.core.commands"
+
+# root note of the configuration value
 ResotoCoreRoot = "resotocore"
+ResotoCoreCommandsRoot = "custom_commands"
+
 ResotoCoreRootRE = re.compile(r"^resotocore[.]")
+
 # created by the docker build process
 GitHashFile = "/usr/local/etc/git-commit.HEAD"
 
@@ -56,8 +63,20 @@ def default_hosts() -> List[str]:
     return ["0.0.0.0"] if inside_docker() else ["localhost"]
 
 
+def validate_config(config: Json, clazz: type) -> Optional[Json]:
+    schema = schema_name(clazz)
+    v = Validator(schema=schema, allow_unknown=True)
+    result = v.validate(config, normalize=False)
+    return None if result else v.errors
+
+
+class ConfigObject:
+    def validate(self) -> Optional[Json]:
+        return validate_config(to_js(self), type(self))
+
+
 @dataclass()
-class CertificateConfig:
+class CertificateConfig(ConfigObject):
     kind: ClassVar[str] = f"{ResotoCoreRoot}_certificate_config"
     common_name: str = field(default="some.engineering", metadata={"description": "The common name of the certificate"})
     include_loopback: bool = field(default=True, metadata={"description": "Include loopback in certificate"})
@@ -70,7 +89,7 @@ class CertificateConfig:
 
 
 @dataclass()
-class ApiConfig:
+class ApiConfig(ConfigObject):
     kind: ClassVar[str] = f"{ResotoCoreRoot}_api_config"
 
     web_hosts: List[str] = field(
@@ -108,7 +127,7 @@ class ApiConfig:
 
 
 @dataclass()
-class DatabaseConfig:
+class DatabaseConfig(ConfigObject):
     kind: ClassVar[str] = f"{ResotoCoreRoot}_database_config"
     server: str = field(
         default="http://localhost:8529",
@@ -131,7 +150,7 @@ class DatabaseConfig:
 
 
 @dataclass(order=True, unsafe_hash=True, frozen=True)
-class AliasTemplateParameterConfig:
+class AliasTemplateParameterConfig(ConfigObject):
     kind: ClassVar[str] = f"{ResotoCoreRoot}_cli_alias_template_parameter"
     name: str = field(metadata=dict(description="The name of the parameter."))
     description: str = field(metadata=dict(description="The intent of this parameter."))
@@ -145,7 +164,7 @@ class AliasTemplateParameterConfig:
 
 
 @dataclass(order=True, unsafe_hash=True, frozen=True)
-class AliasTemplateConfig:
+class AliasTemplateConfig(ConfigObject):
     kind: ClassVar[str] = f"{ResotoCoreRoot}_cli_alias_template"
     name: str = field(metadata=dict(description="The name of the alias to execute."))
     info: str = field(metadata=dict(description="A one line sentence that describes the effect of this command."))
@@ -182,7 +201,7 @@ def alias_templates() -> List[AliasTemplateConfig]:
 
 
 @dataclass()
-class CLIConfig:
+class CLIConfig(ConfigObject):
     kind: ClassVar[str] = f"{ResotoCoreRoot}_cli_config"
     default_graph: str = field(
         default="resoto",
@@ -195,18 +214,30 @@ class CLIConfig:
             "Relative paths will be interpreted with respect to this section."
         },
     )
-    alias_templates: List[AliasTemplateConfig] = field(
+
+
+# Define rules to validate this config
+schema_registry.add(schema_name(CLIConfig), {})
+
+
+@dataclass()
+class CustomCommandsConfig(ConfigObject):
+    kind: ClassVar[str] = ResotoCoreCommandsRoot
+    commands: List[AliasTemplateConfig] = field(
         default_factory=alias_templates,
-        metadata={"description": "Here you can define all alias templates for the CLI."},
+        metadata={"description": "Here you can define all custom commands for the CLI."},
     )
 
+    def json(self) -> Json:
+        return {ResotoCoreCommandsRoot: to_js(self, strip_attr="kind")}
+
 
 # Define rules to validate this config
-schema_registry.add(schema_name(CLIConfig), {})
+schema_registry.add(schema_name(CustomCommandsConfig), {})
 
 
 @dataclass()
-class GraphUpdateConfig:
+class GraphUpdateConfig(ConfigObject):
     kind: ClassVar[str] = f"{ResotoCoreRoot}_graph_update_config"
     merge_max_wait_time_seconds: int = field(
         default=3600, metadata={"description": "Max waiting time to complete a merge graph action."}
@@ -234,7 +265,7 @@ def abort_after(self) -> timedelta:
 
 
 @dataclass()
-class RuntimeConfig:
+class RuntimeConfig(ConfigObject):
     kind: ClassVar[str] = f"{ResotoCoreRoot}_runtime_config"
     analytics_opt_out: bool = field(default=False, metadata={"description": "Stop collecting analytics data."})
     debug: bool = field(default=False, metadata={"description": "Enable debug logging and exception tracing."})
@@ -257,12 +288,13 @@ class RuntimeConfig:
 
 
 @dataclass()
-class CoreConfig:
+class CoreConfig(ConfigObject):
     api: ApiConfig
     cli: CLIConfig
     graph_update: GraphUpdateConfig
     runtime: RuntimeConfig
     db: DatabaseConfig
+    custom_commands: CustomCommandsConfig
     args: Namespace
 
     @property
@@ -274,7 +306,7 @@ def json(self) -> Json:
 
 
 @dataclass()
-class EditableConfig:
+class EditableConfig(ConfigObject):
     kind: ClassVar[str] = ResotoCoreRoot
     api: ApiConfig = field(
         default_factory=ApiConfig,
@@ -293,18 +325,10 @@ class EditableConfig:
         metadata={"description": "Runtime related properties."},
     )
 
-    def validate(self) -> Optional[Json]:
-        return self.validate_config(to_js(self))
-
-    @staticmethod
-    def validate_config(config: Json) -> Optional[Json]:
-        v = Validator(schema="EditableConfig", allow_unknown=True)
-        result = v.validate(config, normalize=False)
-        return None if result else v.errors
-
 
 def config_model() -> List[Json]:
-    return dataclasses_to_resotocore_model({EditableConfig}, allow_unknown_props=False)  # type: ignore
+    config_classes = {EditableConfig, CustomCommandsConfig}
+    return dataclasses_to_resotocore_model(config_classes, allow_unknown_props=False)  # type: ignore
 
 
 # Define rules to validate this config
@@ -320,7 +344,7 @@ def config_model() -> List[Json]:
 )
 
 
-def parse_config(args: Namespace, json_config: Json) -> CoreConfig:
+def parse_config(args: Namespace, core_config: Json, command_templates: Optional[Json] = None) -> CoreConfig:
     db = DatabaseConfig(
         server=args.graphdb_server,
         database=args.graphdb_database,
@@ -343,7 +367,7 @@ def parse_config(args: Namespace, json_config: Json) -> CoreConfig:
         set_from_cmd_line[ResotoCoreRootRE.sub("", key, 1)] = value
 
     # set the relevant value in the json config model
-    adjusted = json_config.get(ResotoCoreRoot) or {}
+    adjusted = core_config.get(ResotoCoreRoot) or {}
     for path, value in set_from_cmd_line.items():
         if value is not None:
             adjusted = set_value_in_path(value, path, adjusted)
@@ -355,7 +379,7 @@ def parse_config(args: Namespace, json_config: Json) -> CoreConfig:
         if isinstance(root, ComplexKind):
             adjusted = root.coerce(adjusted)
     except Exception as e:
-        log.warning("Can not adjust configuration: e", exc_info=e)
+        log.warning(f"Can not adjust configuration: {e}", exc_info=e)
 
     try:
         ed = from_js(adjusted, EditableConfig)
@@ -364,13 +388,31 @@ def parse_config(args: Namespace, json_config: Json) -> CoreConfig:
         log.error("Final configuration can not be parsed! Fall back to default configuration.", exc_info=e)
         ed = EditableConfig()
 
-    return CoreConfig(api=ed.api, cli=ed.cli, db=db, graph_update=ed.graph_update, runtime=ed.runtime, args=args)
+    commands_config = CustomCommandsConfig()
+    if command_templates:
+        try:
+            commands_config = from_js(command_templates.get(ResotoCoreCommandsRoot), CustomCommandsConfig)
+        except Exception as e:
+            log.error(f"Can not parse command templates. Fall back to defaults. Reason: {e}", exc_info=e)
+
+    return CoreConfig(
+        api=ed.api,
+        cli=ed.cli,
+        db=db,
+        graph_update=ed.graph_update,
+        runtime=ed.runtime,
+        custom_commands=commands_config,
+        args=args,
+    )
 
 
 def config_from_db(args: Namespace, db: StandardDatabase, collection_name: str = "configs") -> CoreConfig:
-    config_entity = db.collection("configs").get(ResotoCoreConfigId) if db.has_collection(collection_name) else None
+    configs = db.collection(collection_name) if db.has_collection(collection_name) else None
+    config_entity = configs.get(ResotoCoreConfigId) if configs else None
     config = config_entity.get("config") if config_entity else None  # ConfigEntity.config
     if config:
-        return parse_config(args, config)
+        command_config = configs.get(ResotoCoreCommandsConfigId)
+        command_config = command_config.get("config") if command_config else None
+        return parse_config(args, config, command_config)
     else:
         return parse_config(args, {})