Merge branch 'main' into support/3.2

code/zato-cli/src/zato/cli/zato_command.py

@@ -955,7 +955,7 @@ def main() -> 'any_':
 
             if missing:
                 missing_noun = 'Option ' if len(missing) == 1 else 'Options '
-                missing_verb = ' is ' if len(missing) == 1 else ' are '
+                missing_verb = ' is '    if len(missing) == 1 else ' are '
                 missing.sort()
 
                 sys.stdout.write(

code/zato-cli/test/zato/test_enmasse.py

@@ -37,7 +37,7 @@
 # ################################################################################################################################
 # ################################################################################################################################
 
-template = """
+template1 = """
 
 channel_plain_http:
   - connection: channel
@@ -47,7 +47,6 @@
     name: /test/enmasse1/{test_suffix}
     params_pri: channel -params-over-msg
     sec_def: zato-no-security
-    service: pub.zato.ping
     service_name: pub.zato.ping
     transport: plain_http
     url_path: /test/enmasse1/{test_suffix}
@@ -110,6 +109,49 @@
 # ################################################################################################################################
 # ################################################################################################################################
 
+template2 = """
+
+security:
+  - name: Test Basic Auth Simple
+    username: "MyUser {test_suffix}"
+    password: "MyPassword"
+    type: basic_auth
+    realm: "My Realm"
+
+  - name: Test Basic Auth Simple.2
+    username: "MyUser {test_suffix}.2"
+    type: basic_auth
+    realm: "My Realm"
+
+channel_rest:
+
+  - name: name: /test/enmasse1/simple/{test_suffix}
+    service: pub.zato.ping
+    url_path: /test/enmasse1/simple/{test_suffix}
+
+outgoing_rest:
+
+  - name: Outgoing Rest Enmasse {test_suffix}
+    host: https://example.com
+    url_path: /enmasse/simple/{test_suffix}
+
+  - name: Outgoing Rest Enmasse {test_suffix}.2
+    host: https://example.com
+    url_path: /enmasse/simple/{test_suffix}.2
+    data_format: form
+
+outgoing_ldap:
+
+  - name: Enmasse LDAP {test_suffix}
+    username: 'CN=example.ldap,OU=example01,OU=Example,OU=Groups,DC=example,DC=corp'
+    auth_type: NTLM
+    server_list: 127.0.0.1:389
+    password: {test_suffix}
+"""
+
+# ################################################################################################################################
+# ################################################################################################################################
+
 class EnmasseTestCase(TestCase):
 
 # ################################################################################################################################
@@ -160,7 +202,7 @@ def _invoke_command(self, config_path:'str', require_ok:'bool'=True) -> 'Running
         command = get_zato_sh_command()
 
         # Invoke enmasse ..
-        out = command('enmasse', TestConfig.server_location,
+        out:'RunningCommand' = command('enmasse', TestConfig.server_location,
             '--import', '--input', config_path, '--replace-odb-objects', '--verbose')
 
         # .. if told to, make sure there was no error in stdout/stderr ..
@@ -183,7 +225,7 @@ def _cleanup(self, test_suffix:'str') -> 'None':
         conn_name = f'test.enmasse.{test_suffix}'
 
         # Invoke the delete command ..
-        out = command(
+        out:'RunningCommand' = command(
             'delete-wsx-outconn',
             '--path', TestConfig.server_location,
             '--name', conn_name
@@ -194,7 +236,7 @@ def _cleanup(self, test_suffix:'str') -> 'None':
 
 # ################################################################################################################################
 
-    def test_enmasse_ok(self) -> 'None':
+    def _test_enmasse_ok(self, template:'str') -> 'None':
 
         # sh
         from sh import ErrorReturnCode
@@ -207,7 +249,7 @@ def test_enmasse_ok(self) -> 'None':
 
         smtp_config = self.get_smtp_config()
 
-        data = template.format(test_suffix=test_suffix, smtp_config=smtp_config)
+        data = template1.format(test_suffix=test_suffix, smtp_config=smtp_config)
 
         f = open_w(config_path)
         _ = f.write(data)
@@ -221,16 +263,26 @@ def test_enmasse_ok(self) -> 'None':
             _ = self._invoke_command(config_path)
 
         except ErrorReturnCode as e:
-            stdout = e.stdout # type: bytes
+            stdout:'bytes' = e.stdout # type: bytes
             stdout = stdout.decode('utf8') # type: ignore
-            stderr = e.stderr
+            stderr:'str' = e.stderr
 
             self._warn_on_error(stdout, stderr)
             self.fail(f'Caught an exception while invoking enmasse; stdout -> {stdout}')
 
         finally:
             self._cleanup(test_suffix)
 
+# ################################################################################################################################
+
+    def test_enmasse_ok(self) -> 'None':
+        self._test_enmasse_ok(template1)
+
+# ################################################################################################################################
+
+    def test_enmasse_simple_ok(self) -> 'None':
+        self._test_enmasse_ok(template2)
+
 # ################################################################################################################################
 
     def test_enmasse_service_does_not_exit(self) -> 'None':
@@ -244,7 +296,7 @@ def test_enmasse_service_does_not_exit(self) -> 'None':
         smtp_config = self.get_smtp_config()
 
         # Note that we replace pub.zato.ping with a service that certainly does not exist
-        data = template.replace('pub.zato.ping', 'zato-enmasse-service-does-not-exit')
+        data = template1.replace('pub.zato.ping', 'zato-enmasse-service-does-not-exit')
         data = data.format(test_suffix=test_suffix, smtp_config=smtp_config)
 
         f = open_w(config_path)

code/zato-common/src/zato/common/api.py

@@ -1674,6 +1674,7 @@ class HotDeploy:
     UserConfPrefix = 'user_conf'
     Source_Directory = 'src'
     User_Conf_Directory = 'user-conf'
+    Enmasse_File_Pattern = 'enmasse'
 
 # ################################################################################################################################
 # ################################################################################################################################

code/zato-common/src/zato/common/hot_deploy_.py

@@ -34,6 +34,7 @@ class HotDeployProject:
     '  core*/**',
     '  channel*/**',
     '  adapter*/**',
+    '  **/enmasse*.y*ml',
 ]
 
 # ################################################################################################################################

code/zato-server/src/zato/server/base/parallel/__init__.py

@@ -88,7 +88,6 @@
     from zato.common.odb.api import ODBManager
     from zato.common.odb.model import Cluster as ClusterModel
     from zato.common.typing_ import any_, anydict, anylist, anyset, callable_, dictlist, stranydict, strbytes, strlist, strnone
-    from zato.server.commands import CommandResult
     from zato.server.connection.cache import Cache, CacheAPI
     from zato.server.connection.connector.subprocess_.ipc import SubprocessIPC
     from zato.server.ext.zunicorn.arbiter import Arbiter
@@ -510,6 +509,9 @@ def add_pickup_conf_from_local_path(self, paths:'str', source:'str') -> 'None':
         # Bunch
         from bunch import bunchify
 
+        # Local variables
+        path_patterns = [HotDeploy.User_Conf_Directory, HotDeploy.Enmasse_File_Pattern]
+
         # We have hot-deployment configuration to process ..
         if paths:
 
@@ -546,48 +548,63 @@ def add_pickup_conf_from_local_path(self, paths:'str', source:'str') -> 'None':
                 }
                 self.pickup_config[key_name] = bunchify(pickup_from)
 
-                # .. go through any of the paths potentially containing user configuration directories ..
-                for user_conf_path in Path(path).rglob(HotDeploy.User_Conf_Directory):
+                # .. go through all the path patterns that point to user configuration (including enmasse) ..
+                for path_pattern in path_patterns:
+
+                    # .. get a list of matching paths ..
+                    user_conf_paths = Path(path).rglob(path_pattern)
+                    user_conf_paths = list(user_conf_paths)
 
-                    # .. and add each of them to hot-deployment.
-                    self._add_user_conf_from_path(str(user_conf_path))
+                    # .. go through all the paths that matched ..
+                    for user_conf_path in user_conf_paths:
+
+                        # .. and add each of them to hot-deployment.
+                        self._add_user_conf_from_path(str(user_conf_path), source)
 
 # ################################################################################################################################
 
     def add_user_conf_from_env(self) -> 'None':
 
+        # Local variables
+        env_keys = ['Zato_User_Conf_Dir', 'ZATO_USER_CONF_DIR']
+
         # Look up user-defined configuration directories ..
         paths = os.environ.get('ZATO_USER_CONF_DIR', '')
 
         # .. try the other name too ..
         if not paths:
             paths = os.environ.get('Zato_User_Conf_Dir', '')
 
-        # We have user-config details to process ..
-        if paths:
+        # Go through all the possible environment keys ..
+        for key in env_keys:
 
-            # .. support multiple entries ..
-            paths = paths.split(':')
-            paths = [elem.strip() for elem in paths]
+            # .. if we have user-config details to process ..
+            if paths := os.environ.get(key, ''):
 
-            # .. and the actual configuration.
-            for path in paths:
-                self._add_user_conf_from_path(path)
+                # .. support multiple entries ..
+                paths = paths.split(':')
+                paths = [elem.strip() for elem in paths]
+
+                # .. and the actual configuration.
+                for path in paths:
+                    source = f'env. variable found -> {key}'
+                    self._add_user_conf_from_path(path, source)
 
 # ################################################################################################################################
 
-    def _add_user_conf_from_path(self, path:'str') -> 'None':
+    def _add_user_conf_from_path(self, path:'str', source:'str') -> 'None':
 
         # Bunch
         from bunch import bunchify
 
         # Ignore files other than the below ones
-        suffixes = ['ini', 'conf']
+        suffixes = ['ini', 'conf', 'yaml', 'yml']
         patterns = ['*.' + elem for elem in suffixes]
         patterns_str = ', '.join(patterns)
 
         # Log what we are about to do ..
-        logger.info('Adding user-config from `%s` (env. variable found -> ZATO_USER_CONF_DIR)', path)
+        msg = f'Adding user-config from `{path}` ({source})'
+        logger.info(msg)
 
         # .. look up files inside the directory and add the path to each
         # .. to a list of what should be loaded on startup ..
@@ -604,13 +621,21 @@ def _add_user_conf_from_path(self, path:'str') -> 'None':
         # .. use this prefix to indicate that it is a directory to deploy user configuration from  ..
         key_name = '{}.{}'.format(HotDeploy.UserConfPrefix, _fs_safe_name)
 
+        # .. use a specific service if it is an enmasse file ..
+        if 'enmasse' in path:
+            service = 'zato.pickup.update-enmasse'
+
+        # .. or default to the one for user config if it is not ..
+        else:
+            service= 'zato.pickup.update-user-conf'
+
         # .. and store the configuration for later use now.
         pickup_from = {
             'pickup_from': path,
             'patterns': patterns_str,
             'parse_on_pickup': False,
             'delete_after_pickup': False,
-            'services': 'zato.pickup.update-user-conf',
+            'services': service,
         }
         self.pickup_config[key_name] = bunchify(pickup_from)
 
@@ -728,7 +753,9 @@ def _normalize_service_source_path(name:'str') -> 'str':
 
     def _read_user_config_from_directory(self, dir_name:'str') -> 'None':
 
-        # We assume that it will be always one of these file name suffixes
+        # We assume that it will be always one of these file name suffixes,
+        # note that we are not reading enmasse (.yaml and .yml) files here,
+        # even though directories with enmasse files may be among what we have in self.user_conf_location_extra.
         suffixes_supported = ('.ini', '.conf')
 
         # User-config from ./config/repo/user-config
@@ -797,13 +824,6 @@ def _run_stats_client(self, events_tcp_port:'int') -> 'None':
         self.stats_client.init('127.0.0.1', events_tcp_port)
         self.stats_client.run()
 
-# ################################################################################################################################
-
-    def _on_enmasse_completed(self, result:'CommandResult') -> 'None':
-
-        self.logger.info('Enmasse stdout -> `%s`', result.stdout.strip())
-        self.logger.info('Enmasse stderr -> `%s`', result.stderr.strip())
-
 # ################################################################################################################################
 
     def handle_enmasse_auto_from(self) -> 'None':
@@ -824,8 +844,8 @@ def handle_enmasse_auto_from(self) -> 'None':
         # .. find all the enmasse files in this directory ..
         for file_path in sorted(path.iterdir()):
 
-            command = f'enmasse --import --replace-odb-objects --input {file_path} {self.base_dir} --verbose'
-            _ = commands.run_zato_cli_async(command, callback=self._on_enmasse_completed)
+            # .. and run enmasse with each of them.
+            _ = commands.run_enmasse_async(file_path)
 
 # ################################################################################################################################
 

code/zato-server/src/zato/server/commands.py

@@ -133,17 +133,17 @@ def _append_result_details(
 
         # First, stdout ..
         try:
-            stdout = result.stdout.decode(encoding)
+            stdout:'str' = result.stdout.decode(encoding)
         except UnicodeDecodeError:
-            stdout = result.stdout.decode(encoding, 'replace') # type: str
+            stdout:'str' = result.stdout.decode(encoding, 'replace') # type: str
             if replace_char != Config.ReplaceChar:
                 stdout = stdout.replace(Config.ReplaceChar, replace_char)
 
         # .. now, stderr ..
         try:
-            stderr = result.stderr.decode(encoding)
+            stderr:'str' = result.stderr.decode(encoding)
         except UnicodeDecodeError:
-            stderr = result.stderr.decode(encoding, 'replace') # type: str
+            stderr:'str' = result.stderr.decode(encoding, 'replace') # type: str
             if replace_char != Config.ReplaceChar:
                 stderr = stderr.replace(Config.ReplaceChar, replace_char)
 
@@ -200,8 +200,8 @@ def _run(
             timeout = cast_('float', timeout or None)
 
             # .. invoke the command ..
-            result = subprocess_run(
-                command, input=stdin, timeout=timeout, shell=True, capture_output=True) # type: CompletedProcess
+            result:'CompletedProcess' = subprocess_run(
+                command, input=stdin, timeout=timeout, shell=True, capture_output=True)
 
             # .. if we are here, it means that there was no timeout ..
 
@@ -377,5 +377,19 @@ def run_zato_cli_async(
 
         return self.invoke_async(command, callback=callback)
 
+# ################################################################################################################################
+
+    def run_enmasse_async(self, file_path:'str') -> 'CommandResult':
+        command = f'enmasse --import --replace --input {file_path} {self.server.base_dir} --verbose'
+        result = self.run_zato_cli_async(command, callback=self._on_enmasse_completed)
+        return result
+
+# ################################################################################################################################
+
+    def _on_enmasse_completed(self, result:'CommandResult') -> 'None':
+
+        logger.info('Enmasse stdout -> `%s`', result.stdout.strip())
+        logger.info('Enmasse stderr -> `%s`', result.stderr.strip())
+
 # ################################################################################################################################
 # ################################################################################################################################