diff --git a/code/zato-cli/src/zato/cli/zato_command.py b/code/zato-cli/src/zato/cli/zato_command.py index 1b97c237a7..2f9a783f65 100755 --- a/code/zato-cli/src/zato/cli/zato_command.py +++ b/code/zato-cli/src/zato/cli/zato_command.py @@ -955,7 +955,7 @@ def main() -> 'any_': if missing: missing_noun = 'Option ' if len(missing) == 1 else 'Options ' - missing_verb = ' is ' if len(missing) == 1 else ' are ' + missing_verb = ' is ' if len(missing) == 1 else ' are ' missing.sort() sys.stdout.write( diff --git a/code/zato-cli/test/zato/test_enmasse.py b/code/zato-cli/test/zato/test_enmasse.py index 89b82c6f20..9547672262 100644 --- a/code/zato-cli/test/zato/test_enmasse.py +++ b/code/zato-cli/test/zato/test_enmasse.py @@ -37,7 +37,7 @@ # ################################################################################################################################ # ################################################################################################################################ -template = """ +template1 = """ channel_plain_http: - connection: channel @@ -47,7 +47,6 @@ name: /test/enmasse1/{test_suffix} params_pri: channel -params-over-msg sec_def: zato-no-security - service: pub.zato.ping service_name: pub.zato.ping transport: plain_http url_path: /test/enmasse1/{test_suffix} @@ -110,6 +109,49 @@ # ################################################################################################################################ # ################################################################################################################################ +template2 = """ + +security: + - name: Test Basic Auth Simple + username: "MyUser {test_suffix}" + password: "MyPassword" + type: basic_auth + realm: "My Realm" + + - name: Test Basic Auth Simple.2 + username: "MyUser {test_suffix}.2" + type: basic_auth + realm: "My Realm" + +channel_rest: + + - name: name: /test/enmasse1/simple/{test_suffix} + service: pub.zato.ping + url_path: /test/enmasse1/simple/{test_suffix} + +outgoing_rest: + + - name: Outgoing Rest Enmasse {test_suffix} + host: https://example.com + url_path: /enmasse/simple/{test_suffix} + + - name: Outgoing Rest Enmasse {test_suffix}.2 + host: https://example.com + url_path: /enmasse/simple/{test_suffix}.2 + data_format: form + +outgoing_ldap: + + - name: Enmasse LDAP {test_suffix} + username: 'CN=example.ldap,OU=example01,OU=Example,OU=Groups,DC=example,DC=corp' + auth_type: NTLM + server_list: 127.0.0.1:389 + password: {test_suffix} +""" + +# ################################################################################################################################ +# ################################################################################################################################ + class EnmasseTestCase(TestCase): # ################################################################################################################################ @@ -160,7 +202,7 @@ def _invoke_command(self, config_path:'str', require_ok:'bool'=True) -> 'Running command = get_zato_sh_command() # Invoke enmasse .. - out = command('enmasse', TestConfig.server_location, + out:'RunningCommand' = command('enmasse', TestConfig.server_location, '--import', '--input', config_path, '--replace-odb-objects', '--verbose') # .. if told to, make sure there was no error in stdout/stderr .. @@ -183,7 +225,7 @@ def _cleanup(self, test_suffix:'str') -> 'None': conn_name = f'test.enmasse.{test_suffix}' # Invoke the delete command .. - out = command( + out:'RunningCommand' = command( 'delete-wsx-outconn', '--path', TestConfig.server_location, '--name', conn_name @@ -194,7 +236,7 @@ def _cleanup(self, test_suffix:'str') -> 'None': # ################################################################################################################################ - def test_enmasse_ok(self) -> 'None': + def _test_enmasse_ok(self, template:'str') -> 'None': # sh from sh import ErrorReturnCode @@ -207,7 +249,7 @@ def test_enmasse_ok(self) -> 'None': smtp_config = self.get_smtp_config() - data = template.format(test_suffix=test_suffix, smtp_config=smtp_config) + data = template1.format(test_suffix=test_suffix, smtp_config=smtp_config) f = open_w(config_path) _ = f.write(data) @@ -221,9 +263,9 @@ def test_enmasse_ok(self) -> 'None': _ = self._invoke_command(config_path) except ErrorReturnCode as e: - stdout = e.stdout # type: bytes + stdout:'bytes' = e.stdout # type: bytes stdout = stdout.decode('utf8') # type: ignore - stderr = e.stderr + stderr:'str' = e.stderr self._warn_on_error(stdout, stderr) self.fail(f'Caught an exception while invoking enmasse; stdout -> {stdout}') @@ -231,6 +273,16 @@ def test_enmasse_ok(self) -> 'None': finally: self._cleanup(test_suffix) +# ################################################################################################################################ + + def test_enmasse_ok(self) -> 'None': + self._test_enmasse_ok(template1) + +# ################################################################################################################################ + + def test_enmasse_simple_ok(self) -> 'None': + self._test_enmasse_ok(template2) + # ################################################################################################################################ def test_enmasse_service_does_not_exit(self) -> 'None': @@ -244,7 +296,7 @@ def test_enmasse_service_does_not_exit(self) -> 'None': smtp_config = self.get_smtp_config() # Note that we replace pub.zato.ping with a service that certainly does not exist - data = template.replace('pub.zato.ping', 'zato-enmasse-service-does-not-exit') + data = template1.replace('pub.zato.ping', 'zato-enmasse-service-does-not-exit') data = data.format(test_suffix=test_suffix, smtp_config=smtp_config) f = open_w(config_path) diff --git a/code/zato-common/src/zato/common/api.py b/code/zato-common/src/zato/common/api.py index 05c53aaaca..bfc7a5cbe4 100644 --- a/code/zato-common/src/zato/common/api.py +++ b/code/zato-common/src/zato/common/api.py @@ -1674,6 +1674,7 @@ class HotDeploy: UserConfPrefix = 'user_conf' Source_Directory = 'src' User_Conf_Directory = 'user-conf' + Enmasse_File_Pattern = 'enmasse' # ################################################################################################################################ # ################################################################################################################################ diff --git a/code/zato-common/src/zato/common/hot_deploy_.py b/code/zato-common/src/zato/common/hot_deploy_.py index 7607d85c20..45f0391661 100644 --- a/code/zato-common/src/zato/common/hot_deploy_.py +++ b/code/zato-common/src/zato/common/hot_deploy_.py @@ -34,6 +34,7 @@ class HotDeployProject: ' core*/**', ' channel*/**', ' adapter*/**', + ' **/enmasse*.y*ml', ] # ################################################################################################################################ diff --git a/code/zato-server/src/zato/server/base/parallel/__init__.py b/code/zato-server/src/zato/server/base/parallel/__init__.py index 7d74081bb2..70b5e12002 100644 --- a/code/zato-server/src/zato/server/base/parallel/__init__.py +++ b/code/zato-server/src/zato/server/base/parallel/__init__.py @@ -88,7 +88,6 @@ from zato.common.odb.api import ODBManager from zato.common.odb.model import Cluster as ClusterModel from zato.common.typing_ import any_, anydict, anylist, anyset, callable_, dictlist, stranydict, strbytes, strlist, strnone - from zato.server.commands import CommandResult from zato.server.connection.cache import Cache, CacheAPI from zato.server.connection.connector.subprocess_.ipc import SubprocessIPC from zato.server.ext.zunicorn.arbiter import Arbiter @@ -510,6 +509,9 @@ def add_pickup_conf_from_local_path(self, paths:'str', source:'str') -> 'None': # Bunch from bunch import bunchify + # Local variables + path_patterns = [HotDeploy.User_Conf_Directory, HotDeploy.Enmasse_File_Pattern] + # We have hot-deployment configuration to process .. if paths: @@ -546,16 +548,26 @@ def add_pickup_conf_from_local_path(self, paths:'str', source:'str') -> 'None': } self.pickup_config[key_name] = bunchify(pickup_from) - # .. go through any of the paths potentially containing user configuration directories .. - for user_conf_path in Path(path).rglob(HotDeploy.User_Conf_Directory): + # .. go through all the path patterns that point to user configuration (including enmasse) .. + for path_pattern in path_patterns: + + # .. get a list of matching paths .. + user_conf_paths = Path(path).rglob(path_pattern) + user_conf_paths = list(user_conf_paths) - # .. and add each of them to hot-deployment. - self._add_user_conf_from_path(str(user_conf_path)) + # .. go through all the paths that matched .. + for user_conf_path in user_conf_paths: + + # .. and add each of them to hot-deployment. + self._add_user_conf_from_path(str(user_conf_path), source) # ################################################################################################################################ def add_user_conf_from_env(self) -> 'None': + # Local variables + env_keys = ['Zato_User_Conf_Dir', 'ZATO_USER_CONF_DIR'] + # Look up user-defined configuration directories .. paths = os.environ.get('ZATO_USER_CONF_DIR', '') @@ -563,31 +575,36 @@ def add_user_conf_from_env(self) -> 'None': if not paths: paths = os.environ.get('Zato_User_Conf_Dir', '') - # We have user-config details to process .. - if paths: + # Go through all the possible environment keys .. + for key in env_keys: - # .. support multiple entries .. - paths = paths.split(':') - paths = [elem.strip() for elem in paths] + # .. if we have user-config details to process .. + if paths := os.environ.get(key, ''): - # .. and the actual configuration. - for path in paths: - self._add_user_conf_from_path(path) + # .. support multiple entries .. + paths = paths.split(':') + paths = [elem.strip() for elem in paths] + + # .. and the actual configuration. + for path in paths: + source = f'env. variable found -> {key}' + self._add_user_conf_from_path(path, source) # ################################################################################################################################ - def _add_user_conf_from_path(self, path:'str') -> 'None': + def _add_user_conf_from_path(self, path:'str', source:'str') -> 'None': # Bunch from bunch import bunchify # Ignore files other than the below ones - suffixes = ['ini', 'conf'] + suffixes = ['ini', 'conf', 'yaml', 'yml'] patterns = ['*.' + elem for elem in suffixes] patterns_str = ', '.join(patterns) # Log what we are about to do .. - logger.info('Adding user-config from `%s` (env. variable found -> ZATO_USER_CONF_DIR)', path) + msg = f'Adding user-config from `{path}` ({source})' + logger.info(msg) # .. look up files inside the directory and add the path to each # .. to a list of what should be loaded on startup .. @@ -604,13 +621,21 @@ def _add_user_conf_from_path(self, path:'str') -> 'None': # .. use this prefix to indicate that it is a directory to deploy user configuration from .. key_name = '{}.{}'.format(HotDeploy.UserConfPrefix, _fs_safe_name) + # .. use a specific service if it is an enmasse file .. + if 'enmasse' in path: + service = 'zato.pickup.update-enmasse' + + # .. or default to the one for user config if it is not .. + else: + service= 'zato.pickup.update-user-conf' + # .. and store the configuration for later use now. pickup_from = { 'pickup_from': path, 'patterns': patterns_str, 'parse_on_pickup': False, 'delete_after_pickup': False, - 'services': 'zato.pickup.update-user-conf', + 'services': service, } self.pickup_config[key_name] = bunchify(pickup_from) @@ -728,7 +753,9 @@ def _normalize_service_source_path(name:'str') -> 'str': def _read_user_config_from_directory(self, dir_name:'str') -> 'None': - # We assume that it will be always one of these file name suffixes + # We assume that it will be always one of these file name suffixes, + # note that we are not reading enmasse (.yaml and .yml) files here, + # even though directories with enmasse files may be among what we have in self.user_conf_location_extra. suffixes_supported = ('.ini', '.conf') # User-config from ./config/repo/user-config @@ -797,13 +824,6 @@ def _run_stats_client(self, events_tcp_port:'int') -> 'None': self.stats_client.init('127.0.0.1', events_tcp_port) self.stats_client.run() -# ################################################################################################################################ - - def _on_enmasse_completed(self, result:'CommandResult') -> 'None': - - self.logger.info('Enmasse stdout -> `%s`', result.stdout.strip()) - self.logger.info('Enmasse stderr -> `%s`', result.stderr.strip()) - # ################################################################################################################################ def handle_enmasse_auto_from(self) -> 'None': @@ -824,8 +844,8 @@ def handle_enmasse_auto_from(self) -> 'None': # .. find all the enmasse files in this directory .. for file_path in sorted(path.iterdir()): - command = f'enmasse --import --replace-odb-objects --input {file_path} {self.base_dir} --verbose' - _ = commands.run_zato_cli_async(command, callback=self._on_enmasse_completed) + # .. and run enmasse with each of them. + _ = commands.run_enmasse_async(file_path) # ################################################################################################################################ diff --git a/code/zato-server/src/zato/server/commands.py b/code/zato-server/src/zato/server/commands.py index 92bf602b54..db7a89e8c6 100644 --- a/code/zato-server/src/zato/server/commands.py +++ b/code/zato-server/src/zato/server/commands.py @@ -133,17 +133,17 @@ def _append_result_details( # First, stdout .. try: - stdout = result.stdout.decode(encoding) + stdout:'str' = result.stdout.decode(encoding) except UnicodeDecodeError: - stdout = result.stdout.decode(encoding, 'replace') # type: str + stdout:'str' = result.stdout.decode(encoding, 'replace') # type: str if replace_char != Config.ReplaceChar: stdout = stdout.replace(Config.ReplaceChar, replace_char) # .. now, stderr .. try: - stderr = result.stderr.decode(encoding) + stderr:'str' = result.stderr.decode(encoding) except UnicodeDecodeError: - stderr = result.stderr.decode(encoding, 'replace') # type: str + stderr:'str' = result.stderr.decode(encoding, 'replace') # type: str if replace_char != Config.ReplaceChar: stderr = stderr.replace(Config.ReplaceChar, replace_char) @@ -200,8 +200,8 @@ def _run( timeout = cast_('float', timeout or None) # .. invoke the command .. - result = subprocess_run( - command, input=stdin, timeout=timeout, shell=True, capture_output=True) # type: CompletedProcess + result:'CompletedProcess' = subprocess_run( + command, input=stdin, timeout=timeout, shell=True, capture_output=True) # .. if we are here, it means that there was no timeout .. @@ -377,5 +377,19 @@ def run_zato_cli_async( return self.invoke_async(command, callback=callback) +# ################################################################################################################################ + + def run_enmasse_async(self, file_path:'str') -> 'CommandResult': + command = f'enmasse --import --replace --input {file_path} {self.server.base_dir} --verbose' + result = self.run_zato_cli_async(command, callback=self._on_enmasse_completed) + return result + +# ################################################################################################################################ + + def _on_enmasse_completed(self, result:'CommandResult') -> 'None': + + logger.info('Enmasse stdout -> `%s`', result.stdout.strip()) + logger.info('Enmasse stderr -> `%s`', result.stderr.strip()) + # ################################################################################################################################ # ################################################################################################################################ diff --git a/code/zato-server/src/zato/server/connection/http_soap/outgoing.py b/code/zato-server/src/zato/server/connection/http_soap/outgoing.py index ca4da6df2d..e8f3580e92 100644 --- a/code/zato-server/src/zato/server/connection/http_soap/outgoing.py +++ b/code/zato-server/src/zato/server/connection/http_soap/outgoing.py @@ -35,7 +35,7 @@ from zato.common.api import ContentType, CONTENT_TYPE, DATA_FORMAT, SEC_DEF_TYPE, URL_TYPE from zato.common.exception import Inactive, TimeoutException from zato.common.json_ import dumps, loads -from zato.common.marshal_.api import Model +from zato.common.marshal_.api import extract_model_class, is_list, Model from zato.common.typing_ import cast_ from zato.common.util.api import get_component_name from zato.common.util.open_ import open_rb @@ -46,7 +46,7 @@ if 0: from sqlalchemy.orm.session import Session as SASession - from zato.common.typing_ import any_, callnone, dictnone, stranydict, strdictnone, strstrdict, type_ + from zato.common.typing_ import any_, callnone, dictnone, list_, stranydict, strdictnone, strstrdict, type_ from zato.server.base.parallel import ParallelServer from zato.server.config import ConfigDict ConfigDict = ConfigDict @@ -631,11 +631,34 @@ def rest_call( raise Exception(response.text) # .. extract the underlying data .. - data = response.data # type: ignore + response_data = response.data # type: ignore # .. if we have a model, do make use of it here .. if model: - data:'Model' = model.from_dict(data) + + # .. if this model is actually a list .. + if is_list(model, True): + + # .. extract the underlying model .. + model_class:'type_[Model]' = extract_model_class(model) + + # .. build a list that we will map the response to .. + data:'list_[Model]' = [] + + # .. go through everything we had in the response .. + for item in response_data: + + # .. build an actual model instance .. + _item = model_class.from_dict(item) + + # .. and append it to the data that we are producing .. + data.append(_item) + else: + data:'Model' = model.from_dict(response_data) + + # .. if there is no model, use the response as-is .. + else: + data = response_data # .. run our callback, if there is any .. if callback: diff --git a/code/zato-server/src/zato/server/service/internal/pickup.py b/code/zato-server/src/zato/server/service/internal/pickup.py index 0e415c15bf..6801efe081 100644 --- a/code/zato-server/src/zato/server/service/internal/pickup.py +++ b/code/zato-server/src/zato/server/service/internal/pickup.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -Copyright (C) 2022, Zato Source s.r.o. https://zato.io +Copyright (C) 2023, Zato Source s.r.o. https://zato.io Licensed under LGPLv3, see LICENSE.txt for terms and conditions. """ @@ -119,6 +119,23 @@ class UpdateUserConf(_Updater): # ################################################################################################################################ # ################################################################################################################################ +class UpdateEnmasse(Service): + """ Runs an enmasse file if its contents is changed. + """ + def handle(self) -> 'None': + + # Add type hints .. + raw_request = cast_('stranydict', self.request.raw_request) + + # .. extract the path to the enmasse file .. + enmasse_file_path = raw_request['full_path'] + + # .. and execute it now. + _ = self.commands.run_enmasse_async(enmasse_file_path) + +# ################################################################################################################################ +# ################################################################################################################################ + class _OnUpdate(Service): """ Updates user configuration in memory and file system. """ @@ -199,7 +216,7 @@ def handle(self) -> 'None': # with self.lock('{}-{}-{}'.format(self.name, self.server.name, ctx.full_path)): # type: ignore with open(ctx.full_path, 'wb') as f: - f.write(ctx.data.encode('utf8')) + _ = f.write(ctx.data.encode('utf8')) try: # The file is saved so we can update our in-RAM mirror of it .. @@ -266,7 +283,7 @@ class OnUpdateStatic(_OnUpdate): update_type = 'static file' def sync_pickup_file_in_ram(self, ctx:'UpdateCtx') -> 'None': - self.server.static_config.read_file(ctx.full_path, ctx.file_name) + _ = self.server.static_config.read_file(ctx.full_path, ctx.file_name) # ################################################################################################################################ # ################################################################################################################################