From 9bbed8249633a3cde19d0d113ffc774f4b7b5ebe Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Tue, 2 Jul 2024 00:58:38 -0700 Subject: [PATCH] Implement config for actors * Add actor config to the leapp.conf config file * Add an attribute for configuration schema to LEAPP Actors * Create API to load config for actors and validate it against the schemas stored in actors. * Create a function to retrieve the configuration that an actor has specified. --- etc/leapp/leapp.conf | 2 + leapp/actors/__init__.py | 14 ++ leapp/actors/config.py | 337 +++++++++++++++++++++++++++ leapp/models/fields/__init__.py | 13 +- leapp/repository/actor_definition.py | 8 + leapp/repository/manager.py | 6 + 6 files changed, 377 insertions(+), 3 deletions(-) create mode 100644 leapp/actors/config.py diff --git a/etc/leapp/leapp.conf b/etc/leapp/leapp.conf index d641b3000..a96dcce6a 100644 --- a/etc/leapp/leapp.conf +++ b/etc/leapp/leapp.conf @@ -4,3 +4,5 @@ repo_path=/etc/leapp/repos.d/ [database] path=/var/lib/leapp/leapp.db +[actor_config] +path=/etc/leapp/actor_conf.d/ diff --git a/leapp/actors/__init__.py b/leapp/actors/__init__.py index 9d83bf105..0828aba69 100644 --- a/leapp/actors/__init__.py +++ b/leapp/actors/__init__.py @@ -2,6 +2,7 @@ import os import sys +from leapp.actors.config import retrieve_config from leapp.compat import string_types from leapp.dialogs import Dialog from leapp.exceptions import (MissingActorAttributeError, RequestStopAfterPhase, StopActorExecution, @@ -41,6 +42,11 @@ class Actor(object): Write the actor's description as a docstring. """ + config_schemas = () + """ + Defines the structure of the configuration that the actor uses. + """ + consumes = () """ Tuple of :py:class:`leapp.models.Model` derived classes defined in the :ref:`repositories ` @@ -86,6 +92,7 @@ def serialize(self): 'path': os.path.dirname(sys.modules[type(self).__module__].__file__), 'class_name': type(self).__name__, 'description': self.description or type(self).__doc__, + 'config_schemas': self.config_schemas, 'consumes': [c.__name__ for c in self.consumes], 'produces': [p.__name__ for p in self.produces], 'tags': [t.__name__ for t in self.tags], @@ -100,6 +107,7 @@ def __init__(self, messaging=None, logger=None, config_model=None, skip_dialogs= This depends on the definition of such a configuration model being defined by the workflow and an actor that provides such a message. """ + Actor.current_instance = self install_translation_for_actor(type(self)) self._messaging = messaging @@ -107,8 +115,12 @@ def __init__(self, messaging=None, logger=None, config_model=None, skip_dialogs= self.skip_dialogs = skip_dialogs """ A configured logger instance for the current actor. """ + # self._configuration is the workflow configuration. + # self.config_schemas is the actor defined configuration. + # self.config is the actual actor configuration if config_model: self._configuration = next(self.consume(config_model), None) + self.config = retrieve_config(self.config_schemas) self._path = path @@ -470,6 +482,8 @@ def get_actor_metadata(actor): _get_attribute(actor, 'dialogs', _is_dialog_tuple, required=False, default_value=()), _get_attribute(actor, 'description', _is_type(string_types), required=False, default_value=actor.__doc__ or 'There has been no description provided for this actor.'), + _get_attribute(actor, 'config_schemas', _is_type(string_types), required=False, + default_value=actor.__doc__ or 'Description of the configuration used by this actor.') _get_attribute(actor, 'apis', _is_api_tuple, required=False, default_value=()) ]) diff --git a/leapp/actors/config.py b/leapp/actors/config.py new file mode 100644 index 000000000..4ee119ce4 --- /dev/null +++ b/leapp/actors/config.py @@ -0,0 +1,337 @@ +""" +Config file format: + yaml file like this: + +--- +# Note: have to add a fields.Map type before we can use yaml mappings. +section_name: + field1_name: value + field2_name: + - listitem1 + - listitem2 +section2_name: + field3_name: value + +Config files are any yaml files in /etc/leapp/actor_config.d/ +(This is settable in /etc/leapp/leapp.conf) + +""" +__metaclass__ = type + +import abc +import glob +import importlib +import logging +import os.path +import pkgutil +from collections import defaultdict + +import six +import yaml + +try: + # Compiled versions if available, for speed + from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper +except ImportError: + from yaml import SafeLoader, SafeDumper + + +_ACTOR_CONFIG = None +_ACTOR_CONFIG_VALIDATED = False + +log = logging.getLogger('leapp.actors.config') + + +class SchemaError(Exception): + """Raised when a schema fails validation.""" + + +class ValidationError(Exception): + """ + Raised when a config file fails to validate against any of the available schemas. + """ + + +@six.add_metaclass(abc.ABCMeta) +class Config: + """ + An Actor config schema looks like this. + + :: + class RHUIConfig(Config): + section = "rhui" + name = "file_map" + type_ = fields.Map(fields.String()) + description = 'Description here' + default = {"repo": "url"} + """ + @abc.abstractproperty + def section(self): + pass + + @abc.abstractproperty + def name(self): + pass + + @abc.abstractproperty + def type_(self): + pass + + @abc.abstractproperty + def description(self): + pass + + @abc.abstractproperty + def default(self): + pass + + @classmethod + def to_dict(cls): + """ + Return a dictionary representation of the config item that would be suitable for putting + into a config file. + """ + representation = { + cls.section: { + '{0}_description__'.format(cls.name): cls.description + } + } + ### TODO: Retrieve the default values from the type field. + # representation[cls.section][cls.name] = cls.type_.get_default() + + return representation + + +def _merge_config(configuration, new_config): + """ + Merge two dictionaries representing configuration. fields in new_config overwrite + any existing fields of the same name in the same section in configuration. + """ + for section_name, section in new_config.items(): + if section_name not in configuration: + configuration[section_name] = section + else: + for field_name, field in section: + configuration[section_name][field_name] = field + + +def _get_config(config_dir='/etc/leapp/actor_conf.d'): + """ + Read all configuration files from the config_dir and return a dict with their values. + """ + config_files = glob.glob(os.path.join(config_dir, '*'), recursive=True) + config_files = [f for f in config_files if f.endswith('.yml') or f.endswith('.yaml')] + config_files.sort() + + configuration = {} + for config_file in config_files: + with open(config_file) as f: + raw_cfg = f.read() + + try: + parsed_config = yaml.load(raw_cfg, SafeLoader) + except Exception as e: + log.warning("Warning: unparsable yaml file %s in the config directory." + " Error: %s", filename, str(e)) + raise + + _merge_config(configuration, parsed_config) + + return configuration + + +def _normalize_schemas(schemas): + """ + Merge all schemas into a single dictionary and validate them for errors we can detect. + """ + added_fields = set() + normalized_schema = {} + for schema in schemas: + for field in schema: + unique_name = (field.section, field.name) + + # Error if the field has been added by another schema + if unique_name in added_fields and added_fields[unique_name] != field: + # TODO: Also have information on what Actor contains the conflicting fields + message = "Two actors added incompatible values for {name}".format(name=unique_name)) + log.error(message) + raise SchemaError(message) + + # TODO: More validation here. + + # Store the fields from the schema in a way that we can easily look + # up while validating + added_fields.add(unique_name) + normalized_schema[field.section][field.name] = field + + return normalized_schema + + +def _validate_field_type(field_type, field_value): + """ + Return False if the field is not of the proper type. + """ + # TODO: I took a quick look at the Model code and this is what I came up + # with. This might not work right or there might be a much better way. + try: + field_type.create(field_value) + except Exception: + return False + return True + + +def _normalize_config(actor_config, schema): + for section_name, section in actor_config.items(): + for field_name in actor_config: + if (section_name, field_name) not in added_fields: + # TODO: Also have information about which config file contains the unknown field. + message = "A config file contained an unknown field: {name}".format(name=(section_name, field_name)) + log.warning(message) + + normalized_actor_config = {} + + for section_name, section in schema.items(): + for field_name, field in section.items(): + # TODO: We might be able to do this using the default piece of + # model.fields.Field(). Something using + # schema[section_name, field_name].type_ with the value from + # actor_config[section_name][field_name]. But looking at the Model + # code, I wasn't quite sure how this should be done so I think this + # will work for now. + + # For every item in the schema, either retrieve the value from the + # config files or set it to the default. + try: + value = actor_config[section_name][field_name] + except KeyError: + # Either section_name or field_name doesn't exist + section = actor_config[section_name] = actor_config.get(section_name, {}) + # May need to deepcopy default if these values are modified. + # However, it's probably an error if they are modified and we + # should possibly look into disallowing that. + value = section[field_name] = schema[section_name][field_name].default + + if not _validate_field(schema[section_name][field_name].type_, value): + raise ValidationError("Config value for {name} is not of the correct type".format(name=(section_name, field_name))) + + normalized_section = normalized_actor_config.get(section_name, {}) + normalized_section[field_name] = value + # If the section already exists, this is a no-op. Otherwise, it + # sets it to the newly created dict. + normalized_actor_config[section_name] = normalized_section + + return normalized_actor_config + + +def load(config_dir, schemas): + """ + Return Actor Configuration. + + :returns: a dict representing the configuration. + :raises ValueError: if the actor configuration does not match the schema. + + This function reads the config, validates it, and adds any default values. + """ + global _ACTOR_CONFIG + if _ACTOR_CONFIG: + return _ACTOR_CONFIG + + # TODO: Move this to the caller + schema = _normalize_schemas(schemas) + # End TODO + config = _get_config(config_dir) + config = _normalize_config(config, schema) + + _ACTOR_CONFIG = config + return _ACTOR_CONFIG + + +def retrieve_config(schema): + """Called by the actor to retrieve the actor configuration specific to this actor.""" + # TODO: This isn't good API. Since this function is called by the Actors, + # we *know* that this is okay to do (as the configuration will have already + # been loaded.) However, there's nothing in the API that ensures that this + # is the case. Need to redesign this. Can't think of how it should look + # right now because loading requires information that the Actor doesn't + # know. + global _ACTOR_CONFIG + all_actor_config = _ACTOR_CONFIG + + configuration = defaultdict(dict) + for field in schema: + configuration[field.section][field.name] = all_actor_config[field.section][field.name] + + return configuration + +# +# From this point down hasn't been re-evaluated to see if it's needed or how it +# fits into the bigger picture. Some of it definitely has been implemented in +# a different way above but not all of it. +# + +def parse_repo_config_files(): + repo_config = {} + for config in all_repository_config_schemas(): + section_name = config.section + + if section_name not in repo_config: + repo_config.update(config.to_dict()) + else: + if '{0}_description__'.format(config_item.name) in repo_config[config.section]: + raise Exception("Error: Two configuration items are declared with the same name Section: {0}, Key: {1}".format(config.section, config.name)) + + repo_config[config.section].update(config.to_dict()[config.section]) + + return repo_config + + +def parse_config_files(config_dir): + """ + Parse all configuration and return a dict with those values. + """ + config = parse_repo_config_files() + system_config = parse_system_config_files(config_dir) + + for section, config_items in system_config.items(): + if section not in config: + print('WARNING: config file contains an unused section: Section: {0}'.format(section)) + config.update[section] = config_items + else: + for key, value in config_items: + if '{0}_description__'.format(key) not in config[section]: + print('WARNING: config file contains an unused config entry: Section: {0}, Key{1}'.format(section, key)) + + config[section][key] = value + + return config + + +def format_config(): + """ + Read the configuration definitions from all of the known repositories and return a string that + can be used as an example config file. + + Example config file: + transaction: + to_install_description__: | + List of packages to be added to the upgrade transaction. + Signed packages which are already installed will be skipped. + to_remove_description__: | + List of packages to be removed from the upgrade transaction + initial-setup should be removed to avoid it asking for EULA acceptance during upgrade + to_remove: + - initial-setup + to_keep_description__: | + List of packages to be kept in the upgrade transaction + to_keep: + - leapp + - python2-leapp + - python3-leapp + - leapp-repository + - snactor + """ + return SafeDumper(yaml.dump(parse_config_files(), dumper=SafeDumper)) + if schemas != actor_config: + raise Exception("Invalid entries in the actor config files") + + global_ _ACTOR_CONFIG_VALIDATED = True diff --git a/leapp/models/fields/__init__.py b/leapp/models/fields/__init__.py index 5e24be44d..c2a8071a8 100644 --- a/leapp/models/fields/__init__.py +++ b/leapp/models/fields/__init__.py @@ -2,6 +2,12 @@ import copy import datetime import json +try: + # Python 3 + from collections.abc import Sequence +except ImportError: + # Python 2.7 + from collections import Sequence import six @@ -185,16 +191,17 @@ def _validate_builtin_value(self, value, name): self._validate(value=value, name=name, expected_type=self._builtin_type) def _validate(self, value, name, expected_type): - if not isinstance(expected_type, tuple): + if not isinstance(expected_type, Sequence): expected_type = (expected_type,) + if value is None and self._nullable: return - if not any(isinstance(value, t) for t in expected_type): + + if not isinstance(value, expected_type): names = ', '.join(['{}'.format(t.__name__) for t in expected_type]) raise ModelViolationError("Fields {} is of type: {} expected: {}".format(name, type(value).__name__, names)) - class Boolean(BuiltinField): """ Boolean field diff --git a/leapp/repository/actor_definition.py b/leapp/repository/actor_definition.py index 0e9cc0a3f..2f06f6e7a 100644 --- a/leapp/repository/actor_definition.py +++ b/leapp/repository/actor_definition.py @@ -175,6 +175,7 @@ def serialize(self): 'class_name': self.class_name, 'description': self.description, 'tags': self.tags, + 'config_schemas': self.config_schemas, 'consumes': self.consumes, 'produces': self.produces, 'apis': self.apis, @@ -279,6 +280,13 @@ def description(self): """ return self.discover()['description'] + @property + def config_schemas(self): + """ + :return: Actor config_schemas + """ + return self.discover()['config_schemas'] + @contextlib.contextmanager def injected_context(self): """ diff --git a/leapp/repository/manager.py b/leapp/repository/manager.py index 7d6cb2271..b1725263a 100644 --- a/leapp/repository/manager.py +++ b/leapp/repository/manager.py @@ -139,6 +139,12 @@ def actors(self): """ return tuple(itertools.chain(*[repo.actors for repo in self._repos.values()])) + @property + def config_schemas(self): + """ + """ + return tuple(itertools.chain(*[repo.config_schemas for repo in self._repos.values()])) + @property def topics(self): """