Merge branch 'main' into support/3.2

code/zato-common/src/zato/common/api.py

@@ -40,6 +40,7 @@ class OS_Env:
 zato_no_op_marker = 'zato_no_op_marker'
 
 SECRET_SHADOW = '******'
+Secret_Shadow = SECRET_SHADOW
 
 # TRACE1 logging level, even more details than DEBUG
 TRACE1 = 6

code/zato-common/src/zato/common/util/api.py

@@ -1081,9 +1081,12 @@ def parse_literal_dict(value:'str') -> 'str | anydict':
 
 # ################################################################################################################################
 
-def parse_extra_into_dict(lines, convert_bool=True):
+def parse_extra_into_dict(lines:'str | bytes', convert_bool:'bool'=True):
     """ Creates a dictionary out of key=value lines.
     """
+    if isinstance(lines, bytes):
+        lines = lines.decode('utf8')
+
     _extra = {}
 
     if lines:

code/zato-common/src/zato/common/util/config.py

@@ -8,24 +8,30 @@
 
 # stdlib
 import os
+from urllib.parse import parse_qsl, urlparse, urlunparse
 
 # Bunch
 from bunch import Bunch
 
-# Python 2/3 compatibility
-from builtins import bytes
-from zato.common.py23_.past.builtins import basestring
-
 # Zato
+from zato.common.api import Secret_Shadow
 from zato.common.const import SECRETS
 
+# ################################################################################################################################
+# ################################################################################################################################
+
+if 0:
+    from zato.common.typing_ import any_
+    from zato.server.base.parallel import ParallelServer
+
+# ################################################################################################################################
 # ################################################################################################################################
 
 def resolve_value(key, value, decrypt_func=None, _default=object(), _secrets=SECRETS):
     """ Resolves final value of a given variable by looking it up in environment if applicable.
     """
     # Skip non-resolvable items
-    if not isinstance(value, basestring):
+    if not isinstance(value, str):
         return value
 
     if not value:
@@ -69,3 +75,65 @@ def resolve_env_variables(data):
     return out
 
 # ################################################################################################################################
+
+def replace_query_string_items(server:'ParallelServer', data:'any_') -> 'str':
+
+    # Local variables
+    query_string_new = []
+
+    # Parse the data ..
+    data = urlparse(data)
+
+    # .. extract the query string ..
+    query_string = data.query
+    query_string = parse_qsl(query_string)
+
+    # .. convert the data to a list to make it possible to unparse it later on ..
+    data = list(data)
+
+    # .. replace all the required elements ..
+    for key, value in query_string:
+
+        # .. so we know if we matched something in the inner loops ..
+        should_continue = True
+
+        # .. check exact keys ..
+        for name in server.sio_config.secret_config.exact:
+            if key == name:
+                value = Secret_Shadow
+                should_continue = False
+                break
+
+        # .. check prefixes ..
+        if should_continue:
+            for name in server.sio_config.secret_config.prefixes:
+                if key.startswith(name):
+                    value = Secret_Shadow
+                    should_continue = should_continue
+                    break
+
+        # .. check suffixes ..
+        if should_continue:
+            for name in server.sio_config.secret_config.suffixes:
+                if key.endswith(name):
+                    value = Secret_Shadow
+                    break
+
+        # .. if we are here, either it means that the value was replaced ..
+        # .. or we are going to use as it was because it needed no replacing ..
+        query_string_new.append(f'{key}={value}')
+
+    # .. replace the query string ..
+    query_string_new = '&'.join(query_string_new)
+
+    # .. now, set the query string back ..
+    data[-2] = query_string_new
+
+    # .. build a full address once more ..
+    data = urlunparse(data)
+
+    # .. and return it to our caller.
+    return data
+
+# ################################################################################################################################
+# ################################################################################################################################

code/zato-common/test/zato/common/util/test_config.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+
+"""
+Copyright (C) 2023, Zato Source s.r.o. https://zato.io
+
+Licensed under LGPLv3, see LICENSE.txt for terms and conditions.
+"""
+
+# stdlib
+from unittest import main, TestCase
+
+# Zato
+from zato.common.api import Secret_Shadow
+from zato.common.typing_ import cast_
+from zato.common.util.config import replace_query_string_items
+
+# Zato - Cython
+from zato.simpleio import SecretConfig
+from zato.simpleio import SIOServerConfig
+
+# ################################################################################################################################
+# ################################################################################################################################
+
+if 0:
+    from zato.common.typing_ import any_
+    from zato.server.base.parallel import ParallelServer
+    ParallelServer = ParallelServer
+
+# ################################################################################################################################
+# ################################################################################################################################
+
+class ConfigTestCase(TestCase):
+
+    def test_replace_query_string_items(self):
+
+        class _Server:
+            sio_config = None
+
+        # Prepare individual pieces of the secrets configuration
+        exact = {'xApiKey', 'token'}
+        prefixes = {'secret_prefix_', 'password_prefix_'}
+        suffixes = {'_secret_suffix', '_password_suffix'}
+
+        # Build the configuration of secrets first
+        secret_config:'any_' = SecretConfig()
+        secret_config.exact = exact
+        secret_config.prefixes = prefixes
+        secret_config.suffixes = suffixes
+
+        sio_config:'any_' = SIOServerConfig()
+        sio_config.secret_config = secret_config
+
+        _server = cast_('ParallelServer', _Server())
+        _server.sio_config = sio_config
+
+        data01 = 'wss://hello?xApiKey=123'
+        data02 = 'wss://hello?xApiKey=123&token=456'
+        data03 = 'wss://hello?xApiKey=123&token=456&abc=111'
+        data04 = 'wss://hello?abc=111&xApiKey=123&token=456&zxc=456'
+        data05 = 'https://hello?secret_prefix_1=123&1_secret_suffix=456'
+        data06 = 'https://hello?password_prefix_2=123&2_password_suffix=456'
+
+        data01_expected = f'wss://hello?xApiKey={Secret_Shadow}'
+        data02_expected = f'wss://hello?xApiKey={Secret_Shadow}&token={Secret_Shadow}'
+        data03_expected = f'wss://hello?xApiKey={Secret_Shadow}&token={Secret_Shadow}&abc=111'
+        data04_expected = f'wss://hello?abc=111&xApiKey={Secret_Shadow}&token={Secret_Shadow}&zxc=456'
+        data05_expected = f'https://hello?secret_prefix_1={Secret_Shadow}&1_secret_suffix={Secret_Shadow}'
+        data06_expected = f'https://hello?password_prefix_2={Secret_Shadow}&2_password_suffix={Secret_Shadow}'
+
+        data01_replaced = replace_query_string_items(_server, data01)
+        data02_replaced = replace_query_string_items(_server, data02)
+        data03_replaced = replace_query_string_items(_server, data03)
+        data04_replaced = replace_query_string_items(_server, data04)
+        data05_replaced = replace_query_string_items(_server, data05)
+        data06_replaced = replace_query_string_items(_server, data06)
+
+        self.assertEqual(data01_replaced, data01_expected)
+        self.assertEqual(data02_replaced, data02_expected)
+        self.assertEqual(data03_replaced, data03_expected)
+        self.assertEqual(data04_replaced, data04_expected)
+        self.assertEqual(data05_replaced, data05_expected)
+        self.assertEqual(data06_replaced, data06_expected)
+
+# ################################################################################################################################
+# ################################################################################################################################
+
+if __name__ == '__main__':
+    _ = main()
+
+# ################################################################################################################################
+# ################################################################################################################################

code/zato-server/src/zato/server/connection/queue.py

@@ -20,6 +20,7 @@
 # Zato
 from zato.common.api import GENERIC as COMMON_GENERIC
 from zato.common.typing_ import cast_
+from zato.common.util.config import replace_query_string_items
 from zato.common.util.python_ import get_python_id
 
 # ################################################################################################################################
@@ -139,6 +140,11 @@ def __init__(
         self.max_attempts = max_attempts
         self.lock = RLock()
 
+        if isinstance(self.address, str): # type: ignore
+            self.log_address = replace_query_string_items(self.server, self.address)
+        else:
+            self.log_address = self.address
+
         # We are ready now
         self.logger = getLogger(self.__class__.__name__)
 
@@ -162,7 +168,7 @@ def put_client(self, client:'any_') -> 'bool':
                 log_func = self.logger.info
 
             if self.connection_exists():
-                log_func(msg, self.conn_name, self.address, self.conn_type)
+                log_func(msg, self.conn_name, self.log_address, self.conn_type)
 
             return is_accepted
 
@@ -226,7 +232,7 @@ def _build_queue(self) -> 'None':
                         num_attempts,
                         self.max_attempts,
                         self.conn_type,
-                        self.address,
+                        self.log_address,
                         self.conn_name
                     )
 
@@ -238,13 +244,13 @@ def _build_queue(self) -> 'None':
 
                 self.logger.info('%d/%d %s clients obtained to `%s` (%s) after %s (cap: %ss)',
                     self.queue.qsize(), self.queue_max_size,
-                    self.conn_type, self.address, self.conn_name, now - start, self.queue_build_cap)
+                    self.conn_type, self.log_address, self.conn_name, now - start, self.queue_build_cap)
 
                 if now >= build_until:
 
                     # Log the fact that the queue is not full yet
                     self.logger.info('Built %s/%s %s clients to `%s` within %s seconds, sleeping until %s (UTC)',
-                        self.queue.qsize(), self.queue.maxsize, self.conn_type, self.address, self.queue_build_cap,
+                        self.queue.qsize(), self.queue.maxsize, self.conn_type, self.log_address, self.queue_build_cap,
                         datetime.utcnow() + timedelta(seconds=self.queue_build_cap))
 
                     # Sleep for a predetermined time
@@ -261,9 +267,9 @@ def _build_queue(self) -> 'None':
 
             if self.keep_connecting and self.connection_exists():
                 self.logger.info('Obtained %d %s client%sto `%s` for `%s`', self.queue.maxsize, self.conn_type, suffix,
-                    self.address, self.conn_name)
+                    self.log_address, self.conn_name)
             else:
-                self.logger.info('Skipped building a queue to `%s` for `%s`', self.address, self.conn_name)
+                self.logger.info('Skipped building a queue to `%s` for `%s`', self.log_address, self.conn_name)
                 self.queue_building_stopped = True
 
             # Ok, got all the connections
@@ -290,7 +296,7 @@ def _spawn_add_client_func(self, count:'int'=1) -> 'None':
         """
         with self.lock:
             if self.queue.full():
-                logger.info('Queue fully prepared -> c:%d (%s %s)', count, self.address, self.conn_name)
+                logger.info('Queue fully prepared -> c:%d (%s %s)', count, self.log_address, self.conn_name)
                 return
             self._spawn_add_client_func_no_lock(count)
 

code/zato-server/test/zato/test_sio_eval.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 """
-Copyright (C) 2021, Zato Source s.r.o. https://zato.io
+Copyright (C) 2023, Zato Source s.r.o. https://zato.io
 
 Licensed under LGPLv3, see LICENSE.txt for terms and conditions.
 """
@@ -20,13 +20,6 @@
 # ################################################################################################################################
 # ################################################################################################################################
 
-if 0:
-    from zato.simpleio import SIOServerConfig
-    SIOServerConfig = SIOServerConfig
-
-# ################################################################################################################################
-# ################################################################################################################################
-
 class SIOEvalTestCase(BaseSIOTestCase):
 
     def test_eval_bool(self):