From ada32c49cba97cbec4211baff4583ad4263d211a Mon Sep 17 00:00:00 2001 From: Robert Betts Date: Wed, 13 Sep 2023 08:42:43 +0100 Subject: [PATCH 01/10] Corrected Licence reference in toml file --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6ddc62b..584cbcc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ keywords = ["python", "asynchrous", "api", "event", "rpc", "distributed", "edd", classifiers = [ 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.10', - 'License :: OSI Approved :: MIT License', + 'License :: OSI Approved :: Apache-2.0 license', 'Operating System :: OS Independent' ] packages = [ From 23f17f290a309fb827eaf2e481bd878ed089c265 Mon Sep 17 00:00:00 2001 From: Robert Betts Date: Thu, 14 Sep 2023 08:11:19 +0100 Subject: [PATCH 02/10] improved error handling --- README.md | 2 +- pyproject.toml | 10 +++++----- src/nuropb/rmq_api.py | 27 +++++++++++++++++++-------- src/nuropb/rmq_transport.py | 2 +- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index a1760cf..3e74ea2 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![codecov](https://codecov.io/gh/robertbetts/nuropb/branch/main/graph/badge.svg?token=DVSBZY794D)](https://codecov.io/gh/robertbetts/nuropb) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![CodeFactor](https://www.codefactor.io/repository/github/robertbetts/nuropb/badge)](https://www.codefactor.io/repository/github/robertbetts/nuropb) -[![License: MIT](https://img.shields.io/pypi/l/giteo)](https://www.apache.org/licenses/LICENSE-2.0.txt) +[![License: Apache 2.0](https://img.shields.io/pypi/l/giteo)](https://www.apache.org/licenses/LICENSE-2.0.txt) You have a Python class that you want to make available as a service to consumers. * You potentially want to scale this service horizontally many times over, likely at an unknown scale. diff --git a/pyproject.toml b/pyproject.toml index 584cbcc..d3073d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,12 +6,12 @@ authors = ["Robert Betts "] readme = "README.md" homepage = "https://github.com/robertbetts/nuropb" repository = "https://github.com/robertbetts/nuropb" -keywords = ["python", "asynchrous", "api", "event", "rpc", "distributed", "edd", "sevice-mesh"] +keywords = ["python", "asynchrous", "api", "event", "rpc", "distributed", "edd", "ddd", "sevice-mesh"] classifiers = [ - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.10', - 'License :: OSI Approved :: Apache-2.0 license', - 'Operating System :: OS Independent' + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.10", + "License :: OSI Approved :: Apache-2.0 license", + "Operating System :: OS Independent" ] packages = [ {include = "nuropb", from = "src"} diff --git a/src/nuropb/rmq_api.py b/src/nuropb/rmq_api.py index cedc87b..d9e708e 100644 --- a/src/nuropb/rmq_api.py +++ b/src/nuropb/rmq_api.py @@ -1,7 +1,6 @@ import logging from typing import Dict, Optional, Any, Union, cast from uuid import uuid4 -import asyncio from asyncio import Future from nuropb.interface import ( @@ -315,13 +314,25 @@ async def request( f"service: {service}\n" f"method: {method}\n" ) - self._transport.send_message( - exchange=self._transport.rpc_exchange, - routing_key=routing_key, - body=body, - properties=properties, - mandatory=True, - ) + try: + self._transport.send_message( + exchange=self._transport.rpc_exchange, + routing_key=routing_key, + body=body, + properties=properties, + mandatory=True, + ) + except Exception as e: + if rpc_response is False: + return { + "tag": "response", + "context": context, + "result": None, + "error": { + "error": f"{type(e).__name__}", + "description": f"Error sending request message: {e}", + } + } response: PayloadDict | None = None try: response = await response_future diff --git a/src/nuropb/rmq_transport.py b/src/nuropb/rmq_transport.py index c80c95d..01db66d 100644 --- a/src/nuropb/rmq_transport.py +++ b/src/nuropb/rmq_transport.py @@ -803,7 +803,7 @@ def on_message_returned( nuropb_type = properties.headers.get("nuropb_type", "unknown") nuropb_version = properties.headers.get("nuropb_version", "unknown") logger.warning( - f"Could not route {nuropb_type} message ro service {method.routing_key} " + f"Could not route {nuropb_type} message to service {method.routing_key} " f"correlation_id: {correlation_id} " f"trace_id: {trace_id} " f": {method.reply_code}, {method.reply_text}" From dd92e9258a56f82c7f2588e0674cc67409701ba6 Mon Sep 17 00:00:00 2001 From: Robert Betts Date: Sun, 17 Sep 2023 02:41:36 +0100 Subject: [PATCH 03/10] Added Context Manager and context authorisation --- README.md | 5 +- src/nuropb/contexts/__init__.py | 0 src/nuropb/contexts/context_manager.py | 145 ++++++++++++++++++++++ src/nuropb/contexts/decorators.py | 72 +++++++++++ src/nuropb/contexts/permissions.py | 18 +++ tests/test_context_authorise_decorator.py | 58 +++++++++ tests/test_context_manager.py | 85 +++++++++++++ tests/test_context_manager_decorator.py | 120 ++++++++++++++++++ 8 files changed, 501 insertions(+), 2 deletions(-) create mode 100644 src/nuropb/contexts/__init__.py create mode 100644 src/nuropb/contexts/context_manager.py create mode 100644 src/nuropb/contexts/decorators.py create mode 100644 src/nuropb/contexts/permissions.py create mode 100644 tests/test_context_authorise_decorator.py create mode 100644 tests/test_context_manager.py create mode 100644 tests/test_context_manager_decorator.py diff --git a/README.md b/README.md index 3e74ea2..e6ba25a 100644 --- a/README.md +++ b/README.md @@ -36,13 +36,14 @@ and ordered event streaming over Kafka. Kafka has also proved a great tool for a messages. Where does the name come from? NuroPb is a contraction of the word neural and the scientific symbol for Lead. Lead -associated with plumbing. So NuroPb is a a system's neural plumbing framework. +associated with plumbing. So NuroPb is a system's neural plumbing framework. ## Getting started - +Install the Python package ``` pip install nuropb ``` + diff --git a/src/nuropb/contexts/__init__.py b/src/nuropb/contexts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/nuropb/contexts/context_manager.py b/src/nuropb/contexts/context_manager.py new file mode 100644 index 0000000..8726a2d --- /dev/null +++ b/src/nuropb/contexts/context_manager.py @@ -0,0 +1,145 @@ +import logging +from types import TracebackType +from typing import Any, Dict, List, Optional, Type + +logger = logging.getLogger(__name__) + +_test_token_cache: Dict[str, Any] = {} +_test_user_id_cache: Dict[str, Any] = {} + + +class NuropbContextManager: + """ This class is a context manager that can be used to manage the context transaction relating + to an incoming nuropb service message. when a class instance method is decorated with the + nuropb_context decorator, the context manager is instantiated and injected into the method + as a ctx parameter. + + The nuropb context manager is both a sync and async context manager, it also provides access + to a decorated method to the nuropb service mesh api. Events can be added to the context + manager and will be sent to the service mesh when the context manager exits. If an exception + is raised while the context manager is running, the exception is recorded and the context + transaction is considered to have failed. If any events were added to the context manager, + they are discarded. + """ + _suppress_exceptions: bool | None + _nuropb_payload: Dict[str, Any] | None + _context: Dict[str, Any] + _user_claims: Dict[str, Any] | None + _events: List[Dict[str, Any]] + _exc_type: Type[BaseException] | None + _exec_value: BaseException | None + _exc_tb: TracebackType | None + _done: bool + + def __init__(self, context: Dict[str, Any], suppress_exceptions: Optional[bool] = True): + if context is None: + raise TypeError("context cannot be None") + self._suppress_exceptions = True if suppress_exceptions is None else suppress_exceptions + self._nuropb_payload = None + self._context = context + self._user_claims = None + self._events = [] + + self._exc_type = None + self._exec_value = None + self._exc_tb = None + self._done = False + + @property + def context(self) -> Dict[str, Any]: + return self._context + + @property + def user_claims(self) -> Dict[str, Any] | None: + return self._user_claims + + @user_claims.setter + def user_claims(self, claims: Dict[str, Any] | None) -> None: + if self._user_claims is not None: + raise ValueError("user_claims can only be set once") + self._user_claims = claims + + @property + def events(self) -> List[Dict[str, Any]]: + return self._events + + @property + def error(self) -> Dict[str, Any] | None: + if self._exc_type is None: + return None + return { + "error": self._exc_type.__name__, + "description": str(self._exec_value), + } + + def add_event(self, event: Dict[str, Any]) -> None: + """ Add an event to the context manager. The event will be sent to the service mesh when + the context manager exits successfully. + + Event format: + { + "topic": "test_topic", + "event": "test_event_payload", + "context": {} + } + + :param event: + :return: + """ + self._events.append(event) + + def _handle_context_exit( + self, + exc_type: Type[BaseException] | None, + exc_value: BaseException | None, + exc_tb: TracebackType | None + ) -> bool: + """ This method is for customising the behaviour when a context manager exits. It has the + same signature as __exit__ or __aexit__. + + If an exception was raised while the context manager was running, the exception information + is recorded and content transaction is considered to have failed. If any events were added + to the context manager, they are discarded. + """ + if self._done: + raise RuntimeError("Context manager has already exited") + self._done = True + self._exc_type = exc_type + self._exec_value = exc_value + self._exc_tb = exc_tb + + if exc_type is not None: + self._events = [] + + return self._suppress_exceptions + + """ + **** Context Manager sync and async methods **** + """ + def __enter__(self): + """ This method is called when entering a context manager with a with statement + """ + if self._done: + raise RuntimeError("Context manager has already exited") + return self + + async def __aenter__(self): + return self.__enter__() + + def __exit__( + self, + exc_type: Type[BaseException] | None, + exc_value: BaseException | None, + exc_tb: TracebackType | None + ) -> bool | None: + return self._handle_context_exit(exc_type, exc_value, exc_tb) + + async def __aexit__( + self, + exc_type: Type[BaseException] | None, + exc_value: BaseException | None, + exc_tb: TracebackType | None + ) -> bool | None: + return self.__exit__(exc_type, exc_value, exc_tb) + + diff --git a/src/nuropb/contexts/decorators.py b/src/nuropb/contexts/decorators.py new file mode 100644 index 0000000..3763e8b --- /dev/null +++ b/src/nuropb/contexts/decorators.py @@ -0,0 +1,72 @@ +import inspect +from typing import Optional, Any +from functools import wraps + +from nuropb.contexts.context_manager import NuropbContextManager + + +def nuropb_context( + original_method=None, + *, + context_parameter: Optional[str] = "ctx", + suppress_exceptions: Optional[bool] = False, + authorise_key: Optional[str] = None, + authorise_func: Optional[callable] = None, +) -> Any: + """ This decorator function injects a NuropbContext instance into a method that has ctx:NuropbContext + as an argument. The ctx parameter of the decorated method is hidden from the method's signature visible + on the service mesh. + + The name of the ctx parameter can be changed by passing a string to the context_parameter argument. + + The caller of class_instance.method(ctx=ctx) can either pass a NuropbContext instance or a dict. If a dict + is passed, a NuropbContext instance will be created from the dict. + + *NOTE* This decorator is only for with class methods, using with functions will have unexpected + results and is likely to raise a TypeException + + :param original_method: the method to be decorated + :param context_parameter: str + :param suppress_exceptions: bool + :param authorise_key: str + :param authorise_func: callable(token: str) -> dict + :return: a decorated method + """ + context_parameter = "ctx" if context_parameter is None else context_parameter + suppress_exceptions = False if suppress_exceptions is None else suppress_exceptions + + def decorator(method): + if context_parameter not in inspect.signature(method).parameters: + raise TypeError( + f"method {method.__name__} does not have {context_parameter} as an argument" + ) + + @wraps(method) + def wrapper(*args, **kwargs): + """ validate calling arguments + """ + if len(args) < 2 or not isinstance(args[1], (NuropbContextManager, dict)): + raise TypeError( + f"The @nuropb_context, expects a {context_parameter}: NuropbContextManager " + f"or dict, as the first argument in calling instance.method" + ) + context = args[1] + if isinstance(context, NuropbContextManager): + ctx = context + else: + ctx = NuropbContextManager( + context=context, + suppress_exceptions=suppress_exceptions, + ) + if authorise_key is not None and authorise_func is not None: + ctx.user_claims = authorise_func(ctx.context[authorise_key]) + kwargs[context_parameter] = ctx + return method(*args[1:], **kwargs) + + return wrapper + + if original_method: + return decorator(original_method) + else: + return decorator + diff --git a/src/nuropb/contexts/permissions.py b/src/nuropb/contexts/permissions.py new file mode 100644 index 0000000..5d10e66 --- /dev/null +++ b/src/nuropb/contexts/permissions.py @@ -0,0 +1,18 @@ +from typing import Dict, Any + + +def authorise_from_token(token) -> Dict[str, Any] | None: + """ Receives a JWT bearer token and returns the user claims if the token is valid + :param token: + :return: claims: Dict[str, Any] or None + """ + raise NotImplementedError() + + +def authorise_from_user(credentials: Dict[str, Any]) -> Dict[str, Any] | None: + """ Receives a users credentials and returns the user claims if the credentials are valid + + :param credentials: Dict[str, Any] + :return: claims: Dict[str, Any] or None + """ + raise NotImplementedError() diff --git a/tests/test_context_authorise_decorator.py b/tests/test_context_authorise_decorator.py new file mode 100644 index 0000000..81490bb --- /dev/null +++ b/tests/test_context_authorise_decorator.py @@ -0,0 +1,58 @@ +import logging +import pytest +from typing import Dict, Any +import datetime + +from nuropb.contexts.context_manager import NuropbContextManager +from nuropb.contexts.decorators import nuropb_context + +logger = logging.getLogger(__name__) + + +def authorise_token(token: str) -> Dict[str, Any]: + _ = token + return { + "user_id": "test_user_id", + "expiry": datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=1), + "scope": "openid profile", + "roles": ["test_role1", "test_role2"], + } + + +class AuthoriseServiceClass: + _service_name: str + + @nuropb_context(authorise_key="Authorization", authorise_func=authorise_token) + def hello_requires_auth(self, ctx: NuropbContextManager, param1: str) -> Dict[str, Any]: + _ = self + claims = ctx.user_claims + return { + "context": ctx.context['user_id'], + "claims": claims, + } + + +@pytest.fixture(scope="function") +def context(): + return { + "user_id": "test_user_id", + "session_id": "test_session_id", + "Authorization": "Bearer: authorisation_token", + } + + +@pytest.fixture(scope="function") +def instance(): + return AuthoriseServiceClass() + + +def test_declaring_decorator_on_class_method(instance, context): + """ Using the decorator on a class method with no ctx parameter should raise a TypeError + """ + ctx = NuropbContextManager(context=context, suppress_exceptions=False) + + result = instance.hello_requires_auth(ctx, param1="test_param1") + assert isinstance(result["claims"], dict) + assert result["claims"]["user_id"] == "test_user_id" + assert result["claims"]["scope"] == "openid profile" + assert result["claims"]["roles"] == ["test_role1", "test_role2"] diff --git a/tests/test_context_manager.py b/tests/test_context_manager.py new file mode 100644 index 0000000..dfc3c1d --- /dev/null +++ b/tests/test_context_manager.py @@ -0,0 +1,85 @@ +import logging +import pytest +from typing import Any, Dict +import asyncio + +from nuropb.contexts.context_manager import NuropbContextManager + + +logger = logging.getLogger(__name__) + + +@pytest.fixture(scope="function") +def context(): + return { + "user_id": "test_user_id", + "session_id": "test_session_id", + } + + +@pytest.fixture(scope="function") +def ctx(context): + return NuropbContextManager(context=context) + + +def test_initialise_context_manager(context): + """ Test that we can initialise a context manager with a context dict + """ + ctx = NuropbContextManager(context=context) + assert ctx.context == context + assert ctx.events == [] + + with pytest.raises(TypeError): + ctx = NuropbContextManager(context=None) + + +def test_cm_add_events(context): + ctx = NuropbContextManager(context=context) + ctx.add_event({ + "topic": "test_topic", + "event": "test_event_payload", + "context": context + }) + compare_event = { + "topic": "test_topic", + "event": "test_event_payload", + "context": context + } + assert ctx.events == [compare_event] + + +def test_cm_with_happy_path(ctx): + with ctx as context: + context.add_event({ + "topic": "test_topic", + "event": "test_event_payload", + "context": context + }) + assert len(ctx.events) == 1 + assert ctx.error is None + + +def test_cm_with_happy_error(ctx): + with ctx as context: + 1 / 0 + + assert len(ctx.events) == 0 + + +def test_cm_with_suppress_exception_false(context): + ctx = NuropbContextManager(context=context, suppress_exceptions=False) + with pytest.raises(ZeroDivisionError): + with ctx as context: + 1 / 0 + + +@pytest.mark.asyncio +async def test_cm_with_suppress_exception_false_async(): + ctx = NuropbContextManager(context={}, suppress_exceptions=False) + with pytest.raises(ZeroDivisionError): + async with ctx as context: + 1 / 0 + assert ctx.error["error"] == "ZeroDivisionError" + assert ctx.error["description"] == "division by zero" + + diff --git a/tests/test_context_manager_decorator.py b/tests/test_context_manager_decorator.py new file mode 100644 index 0000000..158b17e --- /dev/null +++ b/tests/test_context_manager_decorator.py @@ -0,0 +1,120 @@ +import logging +import pytest +from typing import Dict, Any + +from nuropb.contexts.context_manager import NuropbContextManager +from nuropb.contexts.decorators import nuropb_context + +logger = logging.getLogger(__name__) + + +class TestServiceClass: + _service_name: str + + def hello_no_context(self, param1: str) -> str: # pragma: no cover + _ = self + return f"hello {param1}" + + def hello_with_context(self, ctx: Dict[str, Any], param1: str) -> str: + _ = self + return f"from {ctx['user_id']}, hello {param1}" + + @nuropb_context + def hello_with_context_decorator(self, ctx: NuropbContextManager, param1: str) -> str: + _ = self + return f"from {ctx.context['user_id']}, hello {param1}" + + +@pytest.fixture(scope="function") +def context(): + return { + "user_id": "test_user_id", + "session_id": "test_session_id", + "Authorisation": "Bearer: authorisation_token", + } + + +@pytest.fixture(scope="function") +def instance(): + return TestServiceClass() + + +def test_declaring_decorator_on_class_method(): + """ Using the decorator on a class method with no ctx parameter should raise a TypeError + """ + with pytest.raises(TypeError): + class ServiceClass: + _service_name: str + + @nuropb_context + def hello_with_context(self, param1: str) -> str: + _ = self, param1 # pragma: no cover + + """ the same test when specifying an alternate the ctx parameter name must also raise a TypeError + """ + with pytest.raises(TypeError): + class ServiceClass: + _service_name: str + + @nuropb_context(context_parameter="context") + def hello_with_context(self, ctx, param1: str) -> str: # pragma: no cover + _ = self, ctx, param1 + + +def test_nuropb_with_context_and_decorator_no_injection(context, instance): + + params = { + "param1": "world", + } + with pytest.raises(TypeError): + result = instance.hello_with_context_decorator(**params) + + +def test_nuropb_context(context, instance): + + params = { + "param1": "world", + } + ctx = NuropbContextManager( + context=context + ) + with pytest.raises(TypeError): + result = instance.hello_no_context(ctx, **params) + + """ NOTE: the caller of instance.method() is responsible for passing the context + which can either be a dictionary or a NuropbContextManager instance. if a dictionary + is passed, a NuropbContextManager instance will be created from the dictionary and + passed to the method. + """ + + """ Test with NuropbContextManager context injection and method has no decorator + """ + with pytest.raises(TypeError): + result = instance.hello_with_context(ctx, **params) + + """ Test with dictionary context injection and method has no decorator and params contains + a ctx parameter as would be expected. + """ + params = { + "param1": "world", + "ctx": {"key": "value"}, + } + ctx = context + with pytest.raises(TypeError): + result = instance.hello_with_context(ctx, **params) + + +def test_nuropb_context_decorator(context, instance): + params = { + "param1": "world", + } + ctx = context + result = instance.hello_with_context_decorator(ctx, **params) + assert result == "from test_user_id, hello world" + + ctx = NuropbContextManager( + context=context + ) + result = instance.hello_with_context_decorator(ctx, **params) + assert result == "from test_user_id, hello world" + From 1d78e6117f6ac1af5a931d3bc74a4374540af67f Mon Sep 17 00:00:00 2001 From: Robert Betts Date: Mon, 18 Sep 2023 01:13:40 +0100 Subject: [PATCH 04/10] New encodings framework - Improved JSON serialisation - Payload Encryption --- examples/service_example.py | 2 +- src/nuropb/contexts/context_manager.py | 45 ++--- src/nuropb/contexts/decorators.py | 27 ++- src/nuropb/contexts/permissions.py | 18 -- src/nuropb/encodings/README.md | 55 ++++++ src/nuropb/encodings/__init__.py | 0 src/nuropb/encodings/encryption.py | 172 ++++++++++++++++++ src/nuropb/encodings/json_serialisation.py | 128 +++++++++++++ src/nuropb/encodings/serializor.py | 62 +++++++ src/nuropb/interface.py | 8 +- src/nuropb/rmq_api.py | 5 +- src/nuropb/rmq_transport.py | 40 +--- src/nuropb/service_handlers.py | 13 +- .../test_encoding_encrypt_decrypt.py | 103 +++++++++++ tests/encodings/test_json_serialisation.py | 58 ++++++ tests/test_context_authorise_decorator.py | 12 +- tests/test_context_manager.py | 23 +-- tests/test_context_manager_decorator.py | 20 +- tests/test_service_container.py | 7 +- 19 files changed, 666 insertions(+), 132 deletions(-) delete mode 100644 src/nuropb/contexts/permissions.py create mode 100644 src/nuropb/encodings/README.md create mode 100644 src/nuropb/encodings/__init__.py create mode 100644 src/nuropb/encodings/encryption.py create mode 100644 src/nuropb/encodings/json_serialisation.py create mode 100644 src/nuropb/encodings/serializor.py create mode 100644 tests/encodings/test_encoding_encrypt_decrypt.py create mode 100644 tests/encodings/test_json_serialisation.py diff --git a/examples/service_example.py b/examples/service_example.py index cce9900..5e1a471 100644 --- a/examples/service_example.py +++ b/examples/service_example.py @@ -39,7 +39,7 @@ def test_method(self, **kwargs) -> str: events: List[EventType] = [ { "topic": "test-event", - "payload": { + "encoded_payload": { "event_key": "event_value", }, "target": [], diff --git a/src/nuropb/contexts/context_manager.py b/src/nuropb/contexts/context_manager.py index 8726a2d..d5478ab 100644 --- a/src/nuropb/contexts/context_manager.py +++ b/src/nuropb/contexts/context_manager.py @@ -9,7 +9,7 @@ class NuropbContextManager: - """ This class is a context manager that can be used to manage the context transaction relating + """This class is a context manager that can be used to manage the context transaction relating to an incoming nuropb service message. when a class instance method is decorated with the nuropb_context decorator, the context manager is instantiated and injected into the method as a ctx parameter. @@ -21,6 +21,7 @@ class NuropbContextManager: transaction is considered to have failed. If any events were added to the context manager, they are discarded. """ + _suppress_exceptions: bool | None _nuropb_payload: Dict[str, Any] | None _context: Dict[str, Any] @@ -31,10 +32,14 @@ class NuropbContextManager: _exc_tb: TracebackType | None _done: bool - def __init__(self, context: Dict[str, Any], suppress_exceptions: Optional[bool] = True): + def __init__( + self, context: Dict[str, Any], suppress_exceptions: Optional[bool] = True + ): if context is None: raise TypeError("context cannot be None") - self._suppress_exceptions = True if suppress_exceptions is None else suppress_exceptions + self._suppress_exceptions = ( + True if suppress_exceptions is None else suppress_exceptions + ) self._nuropb_payload = None self._context = context self._user_claims = None @@ -73,7 +78,7 @@ def error(self) -> Dict[str, Any] | None: } def add_event(self, event: Dict[str, Any]) -> None: - """ Add an event to the context manager. The event will be sent to the service mesh when + """Add an event to the context manager. The event will be sent to the service mesh when the context manager exits successfully. Event format: @@ -89,12 +94,12 @@ def add_event(self, event: Dict[str, Any]) -> None: self._events.append(event) def _handle_context_exit( - self, - exc_type: Type[BaseException] | None, - exc_value: BaseException | None, - exc_tb: TracebackType | None + self, + exc_type: Type[BaseException] | None, + exc_value: BaseException | None, + exc_tb: TracebackType | None, ) -> bool: - """ This method is for customising the behaviour when a context manager exits. It has the + """This method is for customising the behaviour when a context manager exits. It has the same signature as __exit__ or __aexit__. If an exception was raised while the context manager was running, the exception information @@ -116,9 +121,9 @@ def _handle_context_exit( """ **** Context Manager sync and async methods **** """ + def __enter__(self): - """ This method is called when entering a context manager with a with statement - """ + """This method is called when entering a context manager with a with statement""" if self._done: raise RuntimeError("Context manager has already exited") return self @@ -127,19 +132,17 @@ async def __aenter__(self): return self.__enter__() def __exit__( - self, - exc_type: Type[BaseException] | None, - exc_value: BaseException | None, - exc_tb: TracebackType | None + self, + exc_type: Type[BaseException] | None, + exc_value: BaseException | None, + exc_tb: TracebackType | None, ) -> bool | None: return self._handle_context_exit(exc_type, exc_value, exc_tb) async def __aexit__( - self, - exc_type: Type[BaseException] | None, - exc_value: BaseException | None, - exc_tb: TracebackType | None + self, + exc_type: Type[BaseException] | None, + exc_value: BaseException | None, + exc_tb: TracebackType | None, ) -> bool | None: return self.__exit__(exc_type, exc_value, exc_tb) - - diff --git a/src/nuropb/contexts/decorators.py b/src/nuropb/contexts/decorators.py index 3763e8b..ac35b2d 100644 --- a/src/nuropb/contexts/decorators.py +++ b/src/nuropb/contexts/decorators.py @@ -5,15 +5,23 @@ from nuropb.contexts.context_manager import NuropbContextManager +def method_has_nuropb_context(method: callable) -> bool: + """This function checks if a method has been decorated with @nuropb_context + :param method: callable + :return: bool + """ + return getattr(method, "__nuropb_context__", False) + + def nuropb_context( - original_method=None, - *, - context_parameter: Optional[str] = "ctx", - suppress_exceptions: Optional[bool] = False, - authorise_key: Optional[str] = None, - authorise_func: Optional[callable] = None, + original_method=None, + *, + context_parameter: Optional[str] = "ctx", + suppress_exceptions: Optional[bool] = False, + authorise_key: Optional[str] = None, + authorise_func: Optional[callable] = None, ) -> Any: - """ This decorator function injects a NuropbContext instance into a method that has ctx:NuropbContext + """This decorator function injects a NuropbContext instance into a method that has ctx:NuropbContext as an argument. The ctx parameter of the decorated method is hidden from the method's signature visible on the service mesh. @@ -43,8 +51,7 @@ def decorator(method): @wraps(method) def wrapper(*args, **kwargs): - """ validate calling arguments - """ + """validate calling arguments""" if len(args) < 2 or not isinstance(args[1], (NuropbContextManager, dict)): raise TypeError( f"The @nuropb_context, expects a {context_parameter}: NuropbContextManager " @@ -63,10 +70,10 @@ def wrapper(*args, **kwargs): kwargs[context_parameter] = ctx return method(*args[1:], **kwargs) + wrapper.__nuropb_context__ = True return wrapper if original_method: return decorator(original_method) else: return decorator - diff --git a/src/nuropb/contexts/permissions.py b/src/nuropb/contexts/permissions.py deleted file mode 100644 index 5d10e66..0000000 --- a/src/nuropb/contexts/permissions.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import Dict, Any - - -def authorise_from_token(token) -> Dict[str, Any] | None: - """ Receives a JWT bearer token and returns the user claims if the token is valid - :param token: - :return: claims: Dict[str, Any] or None - """ - raise NotImplementedError() - - -def authorise_from_user(credentials: Dict[str, Any]) -> Dict[str, Any] | None: - """ Receives a users credentials and returns the user claims if the credentials are valid - - :param credentials: Dict[str, Any] - :return: claims: Dict[str, Any] or None - """ - raise NotImplementedError() diff --git a/src/nuropb/encodings/README.md b/src/nuropb/encodings/README.md new file mode 100644 index 0000000..8c53e58 --- /dev/null +++ b/src/nuropb/encodings/README.md @@ -0,0 +1,55 @@ +# NuroPb Protocol Encodings +This directory contains the encoding definitions for the NuroPb protocol. + +Out of the box NuroPb transmits JSON encoded messages payloads over the AMQP protocol. There are plans to +support other encodings such as Protocol Buffers and m a y b e Avro. + +## JSON +The JSON encoding is the default encoding for NuroPb. It is the most human-readable encoding and is the +easiest to debug. It is also the most verbose encoding and is not the most efficient encoding. Take a look +src/nuropb/interface.py for information on the structure of messages as this translates directly to the +JSON encoding. + +## Encrypted Payloads +This is the first extension to the NuroPb payload serialisation ahead of the introduction of Protocol Buffers +other serialisation formats. After the JSON or other serialised payload produces a byte stream, this byte stream +is then encrypted. + +It is good practice to transmit any network data over TLS, however this only ensures a level of data protection +over the network. As the payload is passed through Gateways, Message Brokers and other network infrastructure, +it is possible for the payload to be intercepted, read or modifies, but most commonly it can be logged. This +is entirely out of the control of the NuroPb protocol and the application owner or developer. + +In exceeding the requirements of Data Privacy and Data Protection, it is important to ensure that the +visibility and integrity of data payload can be accounted for at all times. This is the primary purpose of the +Encrypted Payload Extension. + +The Encrypted Payload Extension encrypts the payload uses hybrid asymmetric and symmetric keys to produce +encryption level data obfuscation of the payload in transit. + +### High Level Overview + +When a Nuropb service is started, it retrieves or generates a public and private key pair. The public key is then +shared with all other services on the service mesh. A message sender will encrypy the message payload with a +Fernet symmetric key. The symmetric key is then encrypted with the public key of the receiver and is sent with +the message payload. The message receiver will then decrypt the symmetric key with its private key and then decrypt +the message payload with the decrypted symmetric key. + +A response message is then encrypted with the same symmetric key and sent back to the sender. On receipt of the +response, the original sender will then decrypt the response message with the symmetric key used to encrypt the +original message payload. + +### Encryption Algorithms + +The encryption algorithm uses RSA 2048 bit private and public keys. The RSA algorithm is used to encrypt the symmetric +key after it was used to encrypt the message payload. The Fernet algorithm is used for the symmetric key generation +and payload encryption. The recipe for this approach is prompted by an article from Igor Filatov on Medium.com. The +article can be found here: https://medium.com/@igorfilatov/hybrid-encryption-in-python-3e408c73970c + + + + + + + + diff --git a/src/nuropb/encodings/__init__.py b/src/nuropb/encodings/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/nuropb/encodings/encryption.py b/src/nuropb/encodings/encryption.py new file mode 100644 index 0000000..8f1f7cb --- /dev/null +++ b/src/nuropb/encodings/encryption.py @@ -0,0 +1,172 @@ +from typing import Dict, Optional +from base64 import b64encode, b64decode + +from cryptography.fernet import Fernet +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.asymmetric import padding + + +def encrypt_payload(payload: str | bytes, key: str | bytes) -> bytes: + """Encrypt a encoded_payload with a key + :param payload: str | bytes + :param key: str + :return: bytes + """ + f = Fernet(key=key, backend=default_backend()) + if isinstance(payload, str): + payload = payload.encode() + enc_payload = f.encrypt(payload) + return b64encode(enc_payload) + + +def decrypt_payload(encrypted_payload: str | bytes, key: str | bytes) -> bytes: + """Decrypt a encoded_payload with a key + :param encrypted_payload: str | bytes + :param key: str + :return: bytes + """ + f = Fernet(key=key, backend=default_backend()) + payload = f.decrypt(b64decode(encrypted_payload)) + return payload + + +def encrypt_key(symmetric_key: bytes, public_key: rsa.RSAPublicKey) -> bytes: + """Encrypt a symmetric key with an RSA public key + :param symmetric_key: bytes + :param public_key: rsa.RSAPublicKey + :return: bytes, encrypted symmetric key + """ + encrypted_symmetric_key = public_key.encrypt( + symmetric_key, + padding.OAEP( + mgf=padding.MGF1(algorithm=hashes.SHA256()), + algorithm=hashes.SHA256(), + label=None, + ), + ) + return b64encode(encrypted_symmetric_key) + + +def decrypt_key( + encrypted_symmetric_key: bytes, private_key: rsa.RSAPrivateKey +) -> bytes: + """Decrypt a symmetric key encrypted with an RSA private key + :param encrypted_symmetric_key: bytes + :param private_key: rsa.RSAPrivateKey + :return: bytes, decrypted symmetric key + """ + symmetric_key = private_key.decrypt( + b64decode(encrypted_symmetric_key), + padding.OAEP( + mgf=padding.MGF1(algorithm=hashes.SHA256()), + algorithm=hashes.SHA256(), + label=None, + ), + ) + return symmetric_key + + +class Encryptor: + _service_name: str | None + _private_key: rsa.RSAPrivateKey | None + _service_public_keys: Dict[str, rsa.RSAPublicKey] + _correlation_id_symmetric_keys: Dict[str, bytes] + + def __init__( + self, + service_name: Optional[str] = None, + private_key: Optional[rsa.RSAPrivateKey] = None, + ): + self._service_name = service_name + self._private_key = private_key + self._service_public_keys = {} + self._correlation_id_symmetric_keys = {} + + @classmethod + def new_symmetric_key(cls) -> bytes: + """Generate a new symmetric key + :return: bytes + """ + return Fernet.generate_key() + + def add_service_public_key(self, service_name: str, public_key: rsa.RSAPublicKey): + """Add a public key for a service + :param service_name: str + :param public_key: rsa.RSAPublicKey + """ + self._service_public_keys[service_name] = public_key + + def encrypt_payload( + self, + payload: bytes, + correlation_id: str, + service_name: Optional[str | None] = None, + ) -> bytes: + """Encrypt a payload with a symmetric key + + Modes: + 1. Sending a payload to a service (Encrypt) + 4. Sending a response payload from a service (Encrypt) + + :param payload: + :param correlation_id: + :param service_name: + :return: bytes + """ + + """ When service name is provided, it indicates mode 1, else mode 4 + """ + if service_name is not None and service_name not in self._service_public_keys: + raise ValueError( + f"Service public key not found for service: {service_name}" + ) # pragma: no cover + + if service_name is None: + # Mode 4, get public key from the private key + public_key = self._private_key.public_key() + else: + # Mode 1, get public key from the destination service's public key + public_key = self._service_public_keys.get(service_name, None) + + if correlation_id not in self._correlation_id_symmetric_keys: + # Mode 1, generate a new symmetric key and store it for this correlation_id + key = self.new_symmetric_key() + self._correlation_id_symmetric_keys[correlation_id] = key + else: + # Mode 4, use the original received symmetric key to encrypt key + # pop it from the dict as it is not required again + key = self._correlation_id_symmetric_keys.pop(correlation_id, None) + + encrypted_key = encrypt_key(key, public_key) + encrypted_payload = encrypt_payload(payload=payload, key=key) + + return b".".join([encrypted_key, encrypted_payload]) + + def decrypt_payload(self, payload: bytes, correlation_id: str) -> bytes: + """Encrypt a payload with a symmetric key + + Modes: + 2. Receiving a response payload from a service (Decrypt) + 3. Receiving a rpc payload (Decrypt) + + :param payload: + :param correlation_id: + :return: + """ + encrypted_key, encrypted_payload = payload.split(b".", 1) + if correlation_id not in self._correlation_id_symmetric_keys: + """Mode 3, use public key from the private key to decrypt key""" + key = decrypt_key(encrypted_key, self._private_key) + # remember the key for this correlation_id to encrypt the response + self._correlation_id_symmetric_keys[correlation_id] = key + else: + """Mode 2, use the original symmetric key to decrypt key + * pop it from the dict as it is not used again + """ + key = self._correlation_id_symmetric_keys.pop(correlation_id) + + decrypted_payload = decrypt_payload(encrypted_payload, key) + + return decrypted_payload diff --git a/src/nuropb/encodings/json_serialisation.py b/src/nuropb/encodings/json_serialisation.py new file mode 100644 index 0000000..9e60488 --- /dev/null +++ b/src/nuropb/encodings/json_serialisation.py @@ -0,0 +1,128 @@ +from typing import Any, Dict, Optional +import json +import datetime +from decimal import Decimal +import dataclasses + + +def to_json_compatible(obj: Any, recursive: bool = True, max_depth: int = 4) -> Any: + """Returns a json compatible value for obj, if obj is not a native json type, then + return a string representation. + + datetime.datetime: isoformat() + "Z". if there's timezone info, the datetime is + converted to utc. if there is no timezone info, the datetime is assumed to be utc. + + :param obj: Any + :param recursive: bool, whether to recursively convert obj to json compatible types + :param max_depth: int, the maximum depth to recurse + :return: str, or other json compatible type + """ + if max_depth < 0: + return obj # pragma: no cover + + if isinstance(obj, datetime.datetime): + if obj.tzinfo is None: + obj = obj.replace(tzinfo=datetime.timezone.utc) # assume and set to UTC + elif obj.tzinfo != datetime.timezone.utc: + obj = obj.astimezone(datetime.timezone.utc) # convert to UTC + json_string = f"{obj.isoformat()}Z" + return json_string + + if isinstance(obj, (datetime.date, datetime.time)): + json_string = obj.isoformat() + return json_string + + if isinstance(obj, datetime.timedelta): + json_string = str(obj) + return json_string + + if isinstance(obj, Decimal): + return "{0:f}".format(obj) + + if dataclasses.is_dataclass(obj): + dataclass_dict = dataclasses.asdict(obj) + return dataclass_dict + + if isinstance(obj, dict) and recursive: + return { + k: to_json_compatible(v, recursive=recursive, max_depth=max_depth - 1) + for k, v in obj.items() + } + + if isinstance(obj, (list, tuple)) and recursive: + return [ + to_json_compatible(v, recursive=recursive, max_depth=max_depth - 1) + for v in obj + ] + + if isinstance(obj, set) and recursive: + return [ + to_json_compatible(v, recursive=recursive, max_depth=max_depth - 1) + for v in obj + ] + + return obj + + +class NuropbEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, datetime.datetime): + json_string = f"{obj.isoformat()}Z" + return json_string + + if isinstance(obj, (datetime.date, datetime.time)): + json_string = f"{obj.isoformat()}Z" + return json_string + + if isinstance(obj, datetime.timedelta): + json_string = str(obj) + return json_string + + if isinstance(obj, Decimal): + return "{0:f}".format(obj) + + if dataclasses.is_dataclass(obj): + dataclass_dict = dataclasses.asdict(obj) + return dataclass_dict + + try: + return super().default(obj) + except TypeError: + return str(obj) + + +def to_json(obj: Any) -> str: + """Returns a json string representation of the input object, if not a native json type""" + return json.dumps(obj, cls=NuropbEncoder) + + +class JsonSerializor(object): + """Serializes and deserializes nuropb payloads to and from JSON format.""" + + _encryption_keys: Dict[str, Any] + """ encryption keys related to a given correlation_id + """ + + def __init__(self): + """Initializes a new JsonSerializor instance.""" + self._encryption_keys = {} + + def encode(self, payload: Any) -> str: + """Encodes a nuropb encoded_payload to JSON. + + :param payload: Any, The encoded_payload to encode. + :return: str, The JSON-encoded encoded_payload. + """ + _ = self + json_payload = to_json(payload) + return json_payload + + def decode(self, json_payload: str) -> Any: + """Decodes a JSON-encoded nuropb encoded_payload. + + :param json_payload: str, The JSON-encoded encoded_payload to decode. + :return: Any, The decoded encoded_payload. + """ + _ = self + payload = json.loads(json_payload) + return payload diff --git a/src/nuropb/encodings/serializor.py b/src/nuropb/encodings/serializor.py new file mode 100644 index 0000000..241d7f8 --- /dev/null +++ b/src/nuropb/encodings/serializor.py @@ -0,0 +1,62 @@ +from typing import Optional + +from nuropb.encodings.json_serialisation import JsonSerializor +from nuropb.interface import PayloadDict + + +SerializorTypes = JsonSerializor + + +def get_serializor(payload_type: str = "json") -> SerializorTypes: + """Returns a serializor object for the specified encoded_payload type + :param payload_type: "json" + :return: a serializor object + """ + if payload_type != "json": + raise ValueError(f"payload_type {payload_type} is not supported") + + return JsonSerializor() + + +def encode_payload( + payload: PayloadDict, + payload_type: str = "json", +) -> bytes: + """ + :param payload: + :param payload_type: "json" + :return: a json bytes string imputed encoded_payload + """ + if payload_type != "json": + raise ValueError(f"payload_type {payload_type} is not supported") + + return ( + get_serializor( + payload_type=payload_type, + ) + .encode(payload) + .encode() + ) + + +def decode_payload( + encoded_payload: bytes, + payload_type: str = "json", +) -> PayloadDict: + """ + :param encoded_payload: + :param payload_type: "json" + :return: PayloadDict + """ + if payload_type != "json": + raise ValueError(f"payload_type {payload_type} is not supported") + + payload = get_serializor( + payload_type=payload_type, + ).decode(encoded_payload.decode()) + if not isinstance(payload, dict): + raise ValueError( + f"Decoded payload is not a dictionary: {type(payload).__name__}" + ) + + return payload diff --git a/src/nuropb/interface.py b/src/nuropb/interface.py index 993cbdf..89ea89e 100644 --- a/src/nuropb/interface.py +++ b/src/nuropb/interface.py @@ -53,7 +53,7 @@ class ErrorDescriptionType(TypedDict): class EventType(TypedDict): - """For compatibility with better futureproof serialisation support, Any payload type is + """For compatibility with better futureproof serialisation support, Any encoded_payload type is supported.It is encouraged to use a json compatible key/value Type e.g. Dict[str, Any] :target: is currently provided here as an aid for the implementation, there are use cases @@ -154,7 +154,7 @@ class ResponsePayloadDict(TypedDict): class TransportServicePayload(TypedDict): - """Type[TransportServicePayload]: represents valid service instruction payload. + """Type[TransportServicePayload]: represents valid service instruction encoded_payload. Depending on the transport implementation, there wire encoding and serialization may be different, and some of the fields may be in the body or header of the message. """ @@ -169,7 +169,7 @@ class TransportServicePayload(TypedDict): class TransportRespondPayload(TypedDict): """Type[TransportRespondPayload]: represents valid service response message, - valid nuropb payload types are ResponsePayloadDict, and EventPayloadDict + valid nuropb encoded_payload types are ResponsePayloadDict, and EventPayloadDict """ nuropb_protocol: str # nuropb defined and validated @@ -388,7 +388,7 @@ class NuropbCallAgain(NuropbException): class NuropbSuccess(NuropbException): """NuropbSuccessError: when this exception is raised, the transport layer will ACK the message - and return a success response if service payload is a 'request'. This is useful when the request + and return a success response if service encoded_payload is a 'request'. This is useful when the request is a command or event and is executed asynchronously. There are some use cases where the service may want to return a success response irrespective diff --git a/src/nuropb/rmq_api.py b/src/nuropb/rmq_api.py index d9e708e..0a41e71 100644 --- a/src/nuropb/rmq_api.py +++ b/src/nuropb/rmq_api.py @@ -17,7 +17,8 @@ NUROPB_PROTOCOL_VERSION, CommandPayloadDict, ) -from nuropb.rmq_transport import RMQTransport, encode_payload +from nuropb.rmq_transport import RMQTransport +from nuropb.encodings.serializor import encode_payload from nuropb.service_handlers import execute_request, handle_execution_result logger = logging.getLogger(__name__) @@ -331,7 +332,7 @@ async def request( "error": { "error": f"{type(e).__name__}", "description": f"Error sending request message: {e}", - } + }, } response: PayloadDict | None = None try: diff --git a/src/nuropb/rmq_transport.py b/src/nuropb/rmq_transport.py index 01db66d..9bbde87 100644 --- a/src/nuropb/rmq_transport.py +++ b/src/nuropb/rmq_transport.py @@ -1,4 +1,3 @@ -import json import logging import functools from typing import List, Set, Optional, Any, Dict, Awaitable, cast, Literal, TypedDict @@ -13,6 +12,7 @@ import pika.spec from pika.frame import Method +from nuropb.encodings.serializor import encode_payload, decode_payload from nuropb.interface import ( PayloadDict, NuropbTransportError, @@ -69,44 +69,6 @@ class RabbitMQConfiguration(TypedDict): """ -def encode_payload(payload: PayloadDict, payload_type: str = "json") -> bytes: - """ - :param payload: - :param payload_type: - Currently only support json - :return: a byte string encoded json from imputed message dict - """ - if payload_type != "json": - raise ValueError(f"payload_type {payload_type} is not supported") - - return json.dumps(payload).encode() - - -def decode_payload( - payload: bytes, - payload_type: str = "json", - updates: Optional[Dict[str, Any]] = None, -) -> Dict[str, Any]: - """ - :param payload: - :param payload_type: - Currently only support json - :param updates: - Optional dict to update the decoded payload with - :return: convert bytes to a Python Dict - """ - if payload_type != "json": - raise ValueError(f"payload_type {payload_type} is not supported") - - decoded_payload: Any = json.loads(payload) - if not isinstance(decoded_payload, dict): - raise ValueError(f"payload is not a dict: {decoded_payload}") - if updates is not None: - decoded_payload.update(updates) - - return decoded_payload - - def decode_rmq_body( method: pika.spec.Basic.Deliver, properties: pika.spec.BasicProperties, body: bytes ) -> TransportServicePayload: diff --git a/src/nuropb/service_handlers.py b/src/nuropb/service_handlers.py index fb949eb..7f7aece 100644 --- a/src/nuropb/service_handlers.py +++ b/src/nuropb/service_handlers.py @@ -17,6 +17,7 @@ from tornado.concurrent import is_future import pika.spec +from nuropb.contexts.decorators import method_has_nuropb_context from nuropb.interface import ( ResponsePayloadDict, NuropbHandlingError, @@ -189,7 +190,7 @@ def create_transport_responses_from_exceptions( event_payload.update( { "topic": event["topic"], - "event": event["payload"], + "event": event["encoded_payload"], "target": event["target"], } ) @@ -359,7 +360,7 @@ def execute_request( params = payload["params"] """ TODO: think about how to pass the context to the service executing the method - # context = payload["context"] + # context = encoded_payload["context"] """ if ( @@ -375,7 +376,13 @@ def execute_request( ) try: - result = getattr(service_instance, method_name)(**params) + service_instance_method = getattr(service_instance, method_name) + if method_has_nuropb_context(service_instance_method): + result = service_instance_method( + service_message["nuropb_payload"]["context"], **params + ) + else: + result = getattr(service_instance, method_name)(**params) except NuropbException as err: if verbose: logger.exception(err) diff --git a/tests/encodings/test_encoding_encrypt_decrypt.py b/tests/encodings/test_encoding_encrypt_decrypt.py new file mode 100644 index 0000000..4bd3ed9 --- /dev/null +++ b/tests/encodings/test_encoding_encrypt_decrypt.py @@ -0,0 +1,103 @@ +from uuid import uuid4 +from cryptography.fernet import Fernet +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.backends import default_backend + +from nuropb.encodings.encryption import ( + encrypt_payload, + decrypt_payload, + encrypt_key, + decrypt_key, + Encryptor, +) +from nuropb.encodings.serializor import encode_payload, decode_payload +from nuropb.interface import RequestPayloadDict, ResponsePayloadDict + + +def test_symmetric_encryption(): + """Test symmetric encryption and decryption""" + key = Fernet.generate_key() + payload = "Hello World" + encrypted_payload = encrypt_payload(payload, key) + print(encrypted_payload) + decrypted_payload = decrypt_payload(encrypted_payload, key) + assert payload == decrypted_payload.decode() + + +def test_asymmetric_encryption(): + """Test asymmetric encryption and decryption""" + private_key = rsa.generate_private_key( + public_exponent=65537, key_size=2048, backend=default_backend() + ) + public_key = private_key.public_key() + key = Fernet.generate_key() + encrypted_key = encrypt_key(key, public_key) + decrypted_key = decrypt_key(encrypted_key, private_key) + print(encrypted_key) + assert key == decrypted_key + + +def test_encrypted_payload_exchange(): + correlation_id = uuid4().hex + trace_id = uuid4().hex + request_payload: RequestPayloadDict = { + "tag": "request", + "context": {}, + "correlation_id": correlation_id, + "trace_id": trace_id, + "service": "test_service", + "method": "test_async_method", + "params": {"param1": "value1"}, + "reply_to": "", + } + service_name = "test_service" + service_private_key = rsa.generate_private_key( + public_exponent=65537, key_size=2048, backend=default_backend() + ) + service_encryptor = Encryptor(service_name, service_private_key) + client_encryptor = Encryptor() + client_encryptor.add_service_public_key( + service_name=service_name, public_key=service_private_key.public_key() + ) + request_payload_body = encode_payload(request_payload, "json") + encrypted_request_payload = client_encryptor.encrypt_payload( + payload=request_payload_body, + correlation_id=correlation_id, + service_name=service_name, + ) + print(encrypted_request_payload) + + decrypted_request_payload = service_encryptor.decrypt_payload( + payload=encrypted_request_payload, correlation_id=correlation_id + ) + print(decrypted_request_payload) + payload = decode_payload(decrypted_request_payload, "json") + print(payload) + assert payload["service"] == "test_service" + assert payload["method"] == "test_async_method" + assert payload["params"]["param1"] == "value1" + + response_payload: ResponsePayloadDict = { + "tag": "response", + "context": {}, + "correlation_id": correlation_id, + "trace_id": trace_id, + "result": "Hello World", + "error": None, + "warning": None, + } + response_payload_body = encode_payload(response_payload, "json") + encrypted_response_payload = service_encryptor.encrypt_payload( + payload=response_payload_body, + correlation_id=correlation_id, + ) + print(encrypted_response_payload) + + decrypted_response_payload = client_encryptor.decrypt_payload( + payload=encrypted_response_payload, + correlation_id=correlation_id, + ) + print(decrypted_response_payload) + payload = decode_payload(decrypted_response_payload, "json") + print(payload) + assert payload["result"] == "Hello World" diff --git a/tests/encodings/test_json_serialisation.py b/tests/encodings/test_json_serialisation.py new file mode 100644 index 0000000..c075d43 --- /dev/null +++ b/tests/encodings/test_json_serialisation.py @@ -0,0 +1,58 @@ +import datetime +import json +from decimal import Decimal +import pytest + +from nuropb.encodings.json_serialisation import to_json_compatible +from nuropb.encodings.serializor import encode_payload, decode_payload + + +@pytest.fixture(scope="function") +def generic_payload(): + return { + "int": 1, + "float": 1.0, + "decimal": Decimal("1.00000000000001"), + "str": "string", + "bool": True, + "list": [1, 2, 3], + "dict": {"a": 1, "b": 2}, + "none": None, + "date": datetime.date(2020, 1, 1), + "datetime": datetime.datetime(2020, 1, 1, 0, 0, 0), + "utc_datetime": datetime.datetime( + 2020, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc + ), + "utc_datetime_no_tz": datetime.datetime.utcnow(), + "time": datetime.time(0, 0, 0), + "timedelta": datetime.timedelta(days=1), + "mixt_list": [1, "string", True, None, datetime.date(2020, 1, 1)], + "mixed_dict": { + "int": 1, + "float": 1.0, + "decimal": Decimal("1.00000000000001"), + "str": "string", + "bool": True, + "list": [1, 2, 3], + "dict": {"a": 1, "b": 2}, + "none": None, + "date": datetime.date(2020, 1, 1), + "datetime": datetime.datetime(2020, 1, 1, 0, 0, 0), + "set": {1, 2, Decimal(3)}, + }, + } + + +def test_to_json_compatible(generic_payload): + safe_obj = to_json_compatible(generic_payload) + result = json.dumps(safe_obj) + obj = json.loads(result) + assert len(obj) == len(generic_payload) + + +def test_json_encode_decode_python_types(generic_payload): + """Test encoding and decoding of python types.""" + encoded_payload = encode_payload(generic_payload) + decoded_payload = decode_payload(encoded_payload) + + # assert decoded_payload == generic_payload diff --git a/tests/test_context_authorise_decorator.py b/tests/test_context_authorise_decorator.py index 81490bb..2462a4f 100644 --- a/tests/test_context_authorise_decorator.py +++ b/tests/test_context_authorise_decorator.py @@ -13,7 +13,8 @@ def authorise_token(token: str) -> Dict[str, Any]: _ = token return { "user_id": "test_user_id", - "expiry": datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=1), + "expiry": datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(days=1), "scope": "openid profile", "roles": ["test_role1", "test_role2"], } @@ -23,11 +24,13 @@ class AuthoriseServiceClass: _service_name: str @nuropb_context(authorise_key="Authorization", authorise_func=authorise_token) - def hello_requires_auth(self, ctx: NuropbContextManager, param1: str) -> Dict[str, Any]: + def hello_requires_auth( + self, ctx: NuropbContextManager, param1: str + ) -> Dict[str, Any]: _ = self claims = ctx.user_claims return { - "context": ctx.context['user_id'], + "context": ctx.context["user_id"], "claims": claims, } @@ -47,8 +50,7 @@ def instance(): def test_declaring_decorator_on_class_method(instance, context): - """ Using the decorator on a class method with no ctx parameter should raise a TypeError - """ + """Using the decorator on a class method with no ctx parameter should raise a TypeError""" ctx = NuropbContextManager(context=context, suppress_exceptions=False) result = instance.hello_requires_auth(ctx, param1="test_param1") diff --git a/tests/test_context_manager.py b/tests/test_context_manager.py index dfc3c1d..8a0258e 100644 --- a/tests/test_context_manager.py +++ b/tests/test_context_manager.py @@ -23,8 +23,7 @@ def ctx(context): def test_initialise_context_manager(context): - """ Test that we can initialise a context manager with a context dict - """ + """Test that we can initialise a context manager with a context dict""" ctx = NuropbContextManager(context=context) assert ctx.context == context assert ctx.events == [] @@ -35,26 +34,22 @@ def test_initialise_context_manager(context): def test_cm_add_events(context): ctx = NuropbContextManager(context=context) - ctx.add_event({ - "topic": "test_topic", - "event": "test_event_payload", - "context": context - }) + ctx.add_event( + {"topic": "test_topic", "event": "test_event_payload", "context": context} + ) compare_event = { "topic": "test_topic", "event": "test_event_payload", - "context": context + "context": context, } assert ctx.events == [compare_event] def test_cm_with_happy_path(ctx): with ctx as context: - context.add_event({ - "topic": "test_topic", - "event": "test_event_payload", - "context": context - }) + context.add_event( + {"topic": "test_topic", "event": "test_event_payload", "context": context} + ) assert len(ctx.events) == 1 assert ctx.error is None @@ -81,5 +76,3 @@ async def test_cm_with_suppress_exception_false_async(): 1 / 0 assert ctx.error["error"] == "ZeroDivisionError" assert ctx.error["description"] == "division by zero" - - diff --git a/tests/test_context_manager_decorator.py b/tests/test_context_manager_decorator.py index 158b17e..f0acf6a 100644 --- a/tests/test_context_manager_decorator.py +++ b/tests/test_context_manager_decorator.py @@ -20,7 +20,9 @@ def hello_with_context(self, ctx: Dict[str, Any], param1: str) -> str: return f"from {ctx['user_id']}, hello {param1}" @nuropb_context - def hello_with_context_decorator(self, ctx: NuropbContextManager, param1: str) -> str: + def hello_with_context_decorator( + self, ctx: NuropbContextManager, param1: str + ) -> str: _ = self return f"from {ctx.context['user_id']}, hello {param1}" @@ -40,9 +42,9 @@ def instance(): def test_declaring_decorator_on_class_method(): - """ Using the decorator on a class method with no ctx parameter should raise a TypeError - """ + """Using the decorator on a class method with no ctx parameter should raise a TypeError""" with pytest.raises(TypeError): + class ServiceClass: _service_name: str @@ -53,6 +55,7 @@ def hello_with_context(self, param1: str) -> str: """ the same test when specifying an alternate the ctx parameter name must also raise a TypeError """ with pytest.raises(TypeError): + class ServiceClass: _service_name: str @@ -62,7 +65,6 @@ def hello_with_context(self, ctx, param1: str) -> str: # pragma: no cover def test_nuropb_with_context_and_decorator_no_injection(context, instance): - params = { "param1": "world", } @@ -71,13 +73,10 @@ def test_nuropb_with_context_and_decorator_no_injection(context, instance): def test_nuropb_context(context, instance): - params = { "param1": "world", } - ctx = NuropbContextManager( - context=context - ) + ctx = NuropbContextManager(context=context) with pytest.raises(TypeError): result = instance.hello_no_context(ctx, **params) @@ -112,9 +111,6 @@ def test_nuropb_context_decorator(context, instance): result = instance.hello_with_context_decorator(ctx, **params) assert result == "from test_user_id, hello world" - ctx = NuropbContextManager( - context=context - ) + ctx = NuropbContextManager(context=context) result = instance.hello_with_context_decorator(ctx, **params) assert result == "from test_user_id, hello world" - diff --git a/tests/test_service_container.py b/tests/test_service_container.py index 11d6ced..198219b 100644 --- a/tests/test_service_container.py +++ b/tests/test_service_container.py @@ -8,6 +8,7 @@ logger = logging.getLogger() + # @pytest.mark.skip @pytest.mark.asyncio async def test_rmq_api_client_mode(test_settings, test_rmq_url, test_api_url): @@ -37,8 +38,11 @@ async def test_rmq_api_client_mode(test_settings, test_rmq_url, test_api_url): # must resolved the testing issue on github actions # await container.start() + @pytest.mark.asyncio -async def test_rmq_api_service_mode(test_settings, test_rmq_url, test_api_url, service_instance): +async def test_rmq_api_service_mode( + test_settings, test_rmq_url, test_api_url, service_instance +): instance_id = uuid4().hex transport_settings = dict( dl_exchange=test_settings["dl_exchange"], @@ -68,7 +72,6 @@ async def test_rmq_api_service_mode(test_settings, test_rmq_url, test_api_url, s # await container.start() - @pytest.mark.asyncio async def test_rmq_api_service_mode_no_etcd(test_settings, test_rmq_url, test_api_url): instance_id = uuid4().hex From e8096995d923322cc306705569cd5841297243cf Mon Sep 17 00:00:00 2001 From: Robert Betts Date: Mon, 18 Sep 2023 17:17:14 +0100 Subject: [PATCH 05/10] Service describe publish_to_mesh Decorator --- README.md | 2 +- examples/client.py | 2 + examples/service_example.py | 8 +- pyproject.toml | 2 +- src/nuropb/contexts/README.md | 113 +++++++++++ src/nuropb/contexts/context_manager.py | 6 + ...rators.py => context_manager_decorator.py} | 22 ++- src/nuropb/contexts/describe.py | 178 ++++++++++++++++++ src/nuropb/encodings/json_serialisation.py | 22 ++- src/nuropb/encodings/serializor.py | 1 - src/nuropb/rmq_api.py | 61 ++++++ src/nuropb/service_handlers.py | 42 +++-- .../test_context_authorise_decorator.py | 6 +- tests/{ => contexts}/test_context_manager.py | 0 .../test_context_manager_decorator.py | 2 +- tests/contexts/test_describe.py | 173 +++++++++++++++++ tests/encodings/test_json_serialisation.py | 28 +++ 17 files changed, 635 insertions(+), 33 deletions(-) create mode 100644 src/nuropb/contexts/README.md rename src/nuropb/contexts/{decorators.py => context_manager_decorator.py} (81%) create mode 100644 src/nuropb/contexts/describe.py rename tests/{ => contexts}/test_context_authorise_decorator.py (87%) rename tests/{ => contexts}/test_context_manager.py (100%) rename tests/{ => contexts}/test_context_manager_decorator.py (98%) create mode 100644 tests/contexts/test_describe.py diff --git a/README.md b/README.md index e6ba25a..e88c9ef 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # NuroPb -## A Distributed Event Driven Service Mesh +## The neural plumbing for an Asynchronous, Distributed, Event Driven Service Mesh [![codecov](https://codecov.io/gh/robertbetts/nuropb/branch/main/graph/badge.svg?token=DVSBZY794D)](https://codecov.io/gh/robertbetts/nuropb) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) diff --git a/examples/client.py b/examples/client.py index 1b73ce6..1b84b74 100644 --- a/examples/client.py +++ b/examples/client.py @@ -64,6 +64,8 @@ async def main(): ) await api.connect() + sanbox_describe = await api.describe_service("sandbox") + total_seconds = 0 total_sample_count = 0 diff --git a/examples/service_example.py b/examples/service_example.py index 5e1a471..c3f4e76 100644 --- a/examples/service_example.py +++ b/examples/service_example.py @@ -1,7 +1,9 @@ import logging from typing import List +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.backends import default_backend - +from nuropb.contexts.describe import publish_to_mesh from nuropb.interface import NuropbException, NuropbSuccess, NuropbCallAgain, EventType @@ -10,6 +12,9 @@ class ServiceExample: _service_name: str + _private_key = rsa.generate_private_key( + public_exponent=65537, key_size=2048, backend=default_backend() + ) _instance_id: str _method_call_count: int @@ -30,6 +35,7 @@ def _handle_event_( _ = target, context, trace_id logger.debug(f"Received event {topic}:{event}") + @publish_to_mesh(requires_encryption=True) def test_method(self, **kwargs) -> str: self._method_call_count += 1 diff --git a/pyproject.toml b/pyproject.toml index d3073d7..9f0b43d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "nuropb" -version = "0.1.2" +version = "0.1.3" description = "NuroPb - A Distributed Event Driven Service Mesh" authors = ["Robert Betts "] readme = "README.md" diff --git a/src/nuropb/contexts/README.md b/src/nuropb/contexts/README.md new file mode 100644 index 0000000..0287e1b --- /dev/null +++ b/src/nuropb/contexts/README.md @@ -0,0 +1,113 @@ +# Service Context Management and Mesh Publication Features + +## Context Manager and Decorator +When a service instance method is decorated with @nuropb_context, a NuropbContextManager instance will +be injected into the method, and usually into an argument named ctx. This context manager is callable +with the context `with` statement. Once the context manager has exited, it is considered done and +immutable. + +### @nuropb_context +```python +def nuropb_context( + original_method=None, + *, + context_parameter: Optional[str] = "ctx", + suppress_exceptions: Optional[bool] = False, + authorise_key: Optional[str] = None, + authorise_func: Optional[callable] = None, +) -> Any: +``` +This decorator function injects a NuropbContext instance into a method that has ctx:NuropbContext +as an argument. The ctx parameter of the decorated method is hidden from the method's signature +visible on the service mesh. + +The name of the ctx parameter can be changed by entering and alternative name using the +context_parameter argument. + +Any caller of class_instance.method(ctx=ctx) can either pass a NuropbContext instance or a dict. +If a dict is passed, a NuropbContext instance will be created from the dict. + +*NOTE* This decorator is intended only for class methods, using it with functions will have +unexpected results and is likely to result in either and compile or runtime exception. + +:param original_method: reserved for the @decorator logic +:param context_parameter: str, (ctx) alternative context argument name +:param suppress_exceptions: bool, (True), if False then exceptions will be raised during with ctx as ...: +:param authorise_func: callable(token: str) -> dict +:param authorise_key: str, +:return: a decorated method + +## Describe +The published methods of a service are made available to all participants on a service mesh. The +specification follows this example: +```python +service_api_spec = { + "name": "order_management_service", + "service_name": "order_management_service", + "published_events": [], + "service_model": [], + "methods": [ + { + "name": "create_order", + "description": ( + "Create an order for a given security, quantity, side and order_type. Order price input is dependant " + "on order_type and if a date is not given the current date is used, dates prior to today will be " + "rejected by the API." + ), + "parameters": { + "type": "object", + "properties": { + "account": { + "type": "string", + "description": "The account into which the executed trade will be booked", + }, + "security": { + "type": "string", + "description": "the security to be traded", + }, + "status": { + "type": "string", + "enum": ["Open", "Filled", "Cancelled"], + "description": "The status of the order.", + }, + "quantity": { + "type": "integer", + "description": "the quantity to be traded", + }, + "side": { + "type": "string", + "enum": ["Buy", "Sell"], + "description": "The side of the order", + }, + "order_type": { + "type": "string", + "enum": ["Market", "Limit", "Stop"], + "description": "The order type", + }, + "price": { + "type": "number", + "description": "The price of the order", + }, + "time_in_force": { + "type": "string", + "enum": ["Day", "GTC", "FOK", "IOC"], + "description": "The time in force of the order", + }, + "stop_price": { + "type": "number", + "description": "The stop price of the order", + }, + "order_date": { + "type": "string", + "format": "date", + "description": "The date of the order", + }, + }, + "required": ["account", "security", "quantity", "side"] + } + } + ] +} + +``` + diff --git a/src/nuropb/contexts/context_manager.py b/src/nuropb/contexts/context_manager.py index d5478ab..f7b1f6f 100644 --- a/src/nuropb/contexts/context_manager.py +++ b/src/nuropb/contexts/context_manager.py @@ -30,6 +30,7 @@ class NuropbContextManager: _exc_type: Type[BaseException] | None _exec_value: BaseException | None _exc_tb: TracebackType | None + _started: bool _done: bool def __init__( @@ -48,6 +49,7 @@ def __init__( self._exc_type = None self._exec_value = None self._exc_tb = None + self._started = False self._done = False @property @@ -126,6 +128,10 @@ def __enter__(self): """This method is called when entering a context manager with a with statement""" if self._done: raise RuntimeError("Context manager has already exited") + if self._started: + raise RuntimeError("Context manager has already entered") + self._started = True + return self async def __aenter__(self): diff --git a/src/nuropb/contexts/decorators.py b/src/nuropb/contexts/context_manager_decorator.py similarity index 81% rename from src/nuropb/contexts/decorators.py rename to src/nuropb/contexts/context_manager_decorator.py index ac35b2d..e5d6440 100644 --- a/src/nuropb/contexts/decorators.py +++ b/src/nuropb/contexts/context_manager_decorator.py @@ -18,8 +18,6 @@ def nuropb_context( *, context_parameter: Optional[str] = "ctx", suppress_exceptions: Optional[bool] = False, - authorise_key: Optional[str] = None, - authorise_func: Optional[callable] = None, ) -> Any: """This decorator function injects a NuropbContext instance into a method that has ctx:NuropbContext as an argument. The ctx parameter of the decorated method is hidden from the method's signature visible @@ -33,11 +31,17 @@ def nuropb_context( *NOTE* This decorator is only for with class methods, using with functions will have unexpected results and is likely to raise a TypeException + As illustrated by the example below, @nuropb_context must always be on top of @publish_to_mesh when + both decorators are used. + + @nuropb_context + @publish_to_mesh(context_token_key="Authorization", authorise_func=authorise_token) + def hello_requires_auth(...) + + :param original_method: the method to be decorated :param context_parameter: str :param suppress_exceptions: bool - :param authorise_key: str - :param authorise_func: callable(token: str) -> dict :return: a decorated method """ context_parameter = "ctx" if context_parameter is None else context_parameter @@ -65,12 +69,18 @@ def wrapper(*args, **kwargs): context=context, suppress_exceptions=suppress_exceptions, ) - if authorise_key is not None and authorise_func is not None: - ctx.user_claims = authorise_func(ctx.context[authorise_key]) + + authorise_func = getattr(wrapper, "__nuropb_authorise_func__", None) + if authorise_func is not None: + context_token_key = getattr(wrapper, "__nuropb_context_token_key__") + ctx.user_claims = authorise_func(ctx.context[context_token_key]) + kwargs[context_parameter] = ctx return method(*args[1:], **kwargs) + wrapper.__nuropb_context__ = True + wrapper.__nuropb_context_arg__ = "ctx" return wrapper if original_method: diff --git a/src/nuropb/contexts/describe.py b/src/nuropb/contexts/describe.py new file mode 100644 index 0000000..4de1363 --- /dev/null +++ b/src/nuropb/contexts/describe.py @@ -0,0 +1,178 @@ +import logging +import inspect +from typing import Optional, Any, get_origin, get_type_hints, get_args +from functools import wraps +from cryptography.hazmat.primitives import serialization + + +logger = logging.getLogger(__name__) + + +def method_visible_on_mesh(method: callable) -> bool: + """This function checks if a method has been decorated with @publish_to_mesh + :param method: callable + :return: bool + """ + return getattr(method, "__nuropb_mesh_visible__", False) + + +def publish_to_mesh( + original_method=None, + *, + hide_method: Optional[bool] = False, + authorise_func: Optional[callable] = None, + context_token_key: Optional[str] = "Authorization", + requires_encryption: Optional[bool] = False, + description: Optional[str] = None, +) -> Any: + """ Decorator to expose class methods to the service mesh + + When a service instance is connected to the a service mesh via the service mesh client, all + the standard public methods of the service instance is available to the service mesh. Methods + that start with underscore will always remain hidden. methods that are explicitly marked as + hidden by publish_to_mesh will also not be published. + + When and authorise_func is specified, this function will be called with the contents of + context[context_token_key]. if the token validation is unsuccessful, then a failed authorisation + exception is raised. If successful then ctx.user_claims is populated with claims attached to the + token. + + When requires_encryption is True, the service mesh will encrypt the payload of the service message + request and response. It is the responsibility of the process making the request to ensure that + it has the target service's public key. + + As illustrated by the example below, @nuropb_context must always be on top of @publish_to_mesh when + both decorators are used. + + @nuropb_context + @publish_to_mesh(context_token_key="Authorization", authorise_func=authorise_token) + def hello_requires_auth(...) + + + :param original_method: + :param hide_method: + :param authorise_func: + :param context_token_key: + :param requires_encryption: + :param description: str, if present then override the methods doc string + :return: + """ + hide_method = False if hide_method is None else hide_method + context_token_key = "Authorization" if context_token_key is None else context_token_key + if authorise_func is not None and not callable(authorise_func): + raise TypeError("Authorise function must be callable") + requires_encryption = False if requires_encryption is None else requires_encryption + + def decorator(method): + + @wraps(method) + def wrapper(*args, **kwargs): + return method(*args, **kwargs) + + wrapper.__nuropb_mesh_hidden__ = hide_method + wrapper.__nuropb_context_token_key__ = context_token_key + wrapper.__nuropb_authorise_func__ = authorise_func + wrapper.__nuropb_requires_encryption__ = requires_encryption + if description: + wrapper.__nuropb_description__ = description + return wrapper + + if original_method: + return decorator(original_method) + else: + return decorator + + +def describe_service(class_instance): + """ Returns a description of the class methods that will be exposed to the service mesh + """ + if class_instance is None: + logger.warning("No service class base has been input") + return None + else: + service_name = getattr(class_instance, "_service_name", None) + service_description = getattr(class_instance, "__doc__", None) + service_description = service_description.strip() if service_description else service_description + service_version = getattr(class_instance, "_version", None) + methods = [] + service_has_encrypted_methods = False + + for name, method in inspect.getmembers(class_instance): + + """ all private methods are excluded, regardless if one has been decorated with @publish_to_mesh + """ + if name[0] == '_' or not callable(method): + continue + + if getattr(method, "__nuropb_mesh_hidden__", False): + continue + + requires_encryption = getattr(method, "__nuropb_requires_encryption__", False) + if requires_encryption and not service_has_encrypted_methods: + service_has_encrypted_methods = True + + ctx_arg_name = getattr(method, "__nuropb_context_arg__", "ctx") + method_signature = inspect.signature(method) + required = [] + + def map_annotation(arg_props): + annotation = arg_props.annotation + label = "" + if not (annotation == inspect._empty): + origin = get_origin(annotation) + if origin is not None: + args = [a for a in get_args(annotation) if a.__name__ not in ("NoneType",)] + if len(args) == 1: + label = args[0].__name__ + + else: + label = annotation.__name__ + return label + + def map_default(arg_props): + default = arg_props.default + if default == inspect._empty: + required.append(arg_props.name) + return "" + else: + return default + + def map_argument(arg_props): + return (arg_props.name, { + "type": map_annotation(arg_props), + "description": "", + "default": map_default(arg_props), + }) + + properties = [map_argument(p) for n, p in method_signature.parameters.items() if n not in ("self", "cls", ctx_arg_name)] + + method_spec = { + "description": getattr(method, "__nuropb_description__", inspect.getdoc(method)), + "requires_encryption": requires_encryption, + "parameters": { + "type": "object", + "properties": properties, + "required": required, + } + } + methods.append((name, method_spec)) + + service_info = { + "service_name": service_name, + "service_version": service_version, + "description": service_description, + "encrypted_methods": service_has_encrypted_methods, + "methods": dict(methods), + } + + if service_has_encrypted_methods: + private_key = service_name = getattr(class_instance, "_private_key", None) + if private_key is None: + raise ValueError(f"Service {service_name} has encrypted methods but no private key has been set") + + service_info["public_key"] = private_key.public_key().public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo + ) + + return service_info diff --git a/src/nuropb/encodings/json_serialisation.py b/src/nuropb/encodings/json_serialisation.py index 9e60488..c8693c7 100644 --- a/src/nuropb/encodings/json_serialisation.py +++ b/src/nuropb/encodings/json_serialisation.py @@ -1,3 +1,6 @@ +""" This module provides entire nuropb package with json serialisation logic and features +# TODO: Re-check the serialised datetime, date, time and timedelta formats. Look for standards. +""" from typing import Any, Dict, Optional import json import datetime @@ -9,6 +12,13 @@ def to_json_compatible(obj: Any, recursive: bool = True, max_depth: int = 4) -> """Returns a json compatible value for obj, if obj is not a native json type, then return a string representation. + *NOTE 1* This function must be kept in step with the custom json encoder, + NuropbEncoder, below. Next, read NOTE 2. + + *NOTE 2* This function does not exactly follow the structure of the custom json encoder, + NuropbEncoder, below. Difference is that the json library implements its own object + traversal logic. In this function it's required to be done explicitly. + datetime.datetime: isoformat() + "Z". if there's timezone info, the datetime is converted to utc. if there is no timezone info, the datetime is assumed to be utc. @@ -40,8 +50,8 @@ def to_json_compatible(obj: Any, recursive: bool = True, max_depth: int = 4) -> return "{0:f}".format(obj) if dataclasses.is_dataclass(obj): - dataclass_dict = dataclasses.asdict(obj) - return dataclass_dict + v = dataclasses.asdict(obj) + return to_json_compatible(v, recursive=recursive, max_depth=max_depth - 1) if isinstance(obj, dict) and recursive: return { @@ -65,6 +75,14 @@ def to_json_compatible(obj: Any, recursive: bool = True, max_depth: int = 4) -> class NuropbEncoder(json.JSONEncoder): + """ + *NOTE 1* This class must be kept in step with the function to_json_compatible, above. + Next, read NOTE 2. + + *NOTE 2* This class does not exactly follow the structure of the function, + to_json_compatible, above. Difference is that the json library implements its own + object traversal logic. In the function it's required to be done explicitly. + """ def default(self, obj): if isinstance(obj, datetime.datetime): json_string = f"{obj.isoformat()}Z" diff --git a/src/nuropb/encodings/serializor.py b/src/nuropb/encodings/serializor.py index 241d7f8..ada239a 100644 --- a/src/nuropb/encodings/serializor.py +++ b/src/nuropb/encodings/serializor.py @@ -58,5 +58,4 @@ def decode_payload( raise ValueError( f"Decoded payload is not a dictionary: {type(payload).__name__}" ) - return payload diff --git a/src/nuropb/rmq_api.py b/src/nuropb/rmq_api.py index 0a41e71..1b60f0a 100644 --- a/src/nuropb/rmq_api.py +++ b/src/nuropb/rmq_api.py @@ -2,6 +2,8 @@ from typing import Dict, Optional, Any, Union, cast from uuid import uuid4 from asyncio import Future +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.backends import default_backend from nuropb.interface import ( NuropbInterface, @@ -37,6 +39,8 @@ class RMQAPI(NuropbInterface): _service_instance: object | None _default_ttl: int _client_only: bool + _service_discovery: Dict[str, Any] + _service_public_keys: Dict[str, Any] def __init__( self, @@ -70,6 +74,13 @@ def __init__( self._service_name = service_name + self._service_discovery = {} + """ A dictionary of service_name: service_info + """ + self._service_public_keys = {} + """ A dictionary of service_name: public_key + """ + if not self._client_only and service_instance is None: raise ValueError( "A service instance must be provided when starting in service mode" @@ -274,6 +285,10 @@ async def request( -------- ResponsePayloadDict | Any: representing the response from the requested service with any exceptions raised """ + if method != "nuropb_describe": + service_public_key = await self.check_method_for_encryption(service, method) + else: + service_public_key = None correlation_id = uuid4().hex ttl = self._default_ttl if ttl is None else ttl @@ -515,3 +530,49 @@ def publish_event( properties=properties, mandatory=False, ) + + async def describe_service(self, service_name): + """describe_service: returns the service description for the given service_name + :param service_name: str + :return: dict + """ + result = await self.request( + service=service_name, + method="nuropb_describe", + params={}, + context={}, + ttl=60 * 1000, # 1 minute + trace_id=uuid4().hex, + ) + return result + + async def check_method_for_encryption(self, service_name: str, method_name: str): + """check_method_for_encryption: Queries the service discovery cache, if an entry for the service_name + does not exist, then the service discovery is queried directly. + + if encryption is required for the method called, then reference the service discovery cache + for the service private key required to encrypt the request. + + :param service_name: str + :param method_name: str + :return: bool + """ + if service_name not in self._service_discovery: + self._service_discovery[service_name] = await self.describe_service(service_name) + text_public_key = self._service_discovery[service_name].get("public_key", None) + if text_public_key: + self._service_public_keys[service_name] = serialization.load_pem_public_key( + data=text_public_key, + backend=default_backend(), + ) + + service_info = self._service_discovery[service_name] + method_info = service_info["methods"].get(method_name, None) + if method_info is None: + raise ValueError(f"Method {method_name} not found on service {service_name}") + if method_info.get("requires_encryption", False): + return self._service_public_keys.get(service_name, None) + + + + diff --git a/src/nuropb/service_handlers.py b/src/nuropb/service_handlers.py index 7f7aece..a334211 100644 --- a/src/nuropb/service_handlers.py +++ b/src/nuropb/service_handlers.py @@ -17,7 +17,8 @@ from tornado.concurrent import is_future import pika.spec -from nuropb.contexts.decorators import method_has_nuropb_context +from nuropb.contexts.context_manager_decorator import method_has_nuropb_context +from nuropb.contexts.describe import describe_service from nuropb.interface import ( ResponsePayloadDict, NuropbHandlingError, @@ -363,26 +364,31 @@ def execute_request( # context = encoded_payload["context"] """ - if ( - method_name.startswith("_") - or not hasattr(service_instance, method_name) - or not callable(getattr(service_instance, method_name)) - ): - raise NuropbHandlingError( - description="Unknown method {}".format(method_name), - lifecycle="service-handle", - payload=payload, - exception=None, - ) + if method_name != "nuropb_describe": + if ( + method_name.startswith("_") + or not hasattr(service_instance, method_name) + or not callable(getattr(service_instance, method_name)) + ): + raise NuropbHandlingError( + description="Unknown method {}".format(method_name), + lifecycle="service-handle", + payload=payload, + exception=None, + ) try: - service_instance_method = getattr(service_instance, method_name) - if method_has_nuropb_context(service_instance_method): - result = service_instance_method( - service_message["nuropb_payload"]["context"], **params - ) + if method_name == "nuropb_describe": + result = describe_service(service_instance) else: - result = getattr(service_instance, method_name)(**params) + service_instance_method = getattr(service_instance, method_name) + if method_has_nuropb_context(service_instance_method): + result = service_instance_method( + service_message["nuropb_payload"]["context"], **params + ) + else: + result = getattr(service_instance, method_name)(**params) + except NuropbException as err: if verbose: logger.exception(err) diff --git a/tests/test_context_authorise_decorator.py b/tests/contexts/test_context_authorise_decorator.py similarity index 87% rename from tests/test_context_authorise_decorator.py rename to tests/contexts/test_context_authorise_decorator.py index 2462a4f..70e8c8c 100644 --- a/tests/test_context_authorise_decorator.py +++ b/tests/contexts/test_context_authorise_decorator.py @@ -4,7 +4,8 @@ import datetime from nuropb.contexts.context_manager import NuropbContextManager -from nuropb.contexts.decorators import nuropb_context +from nuropb.contexts.context_manager_decorator import nuropb_context +from nuropb.contexts.describe import publish_to_mesh logger = logging.getLogger(__name__) @@ -23,7 +24,8 @@ def authorise_token(token: str) -> Dict[str, Any]: class AuthoriseServiceClass: _service_name: str - @nuropb_context(authorise_key="Authorization", authorise_func=authorise_token) + @nuropb_context + @publish_to_mesh(context_token_key="Authorization", authorise_func=authorise_token) def hello_requires_auth( self, ctx: NuropbContextManager, param1: str ) -> Dict[str, Any]: diff --git a/tests/test_context_manager.py b/tests/contexts/test_context_manager.py similarity index 100% rename from tests/test_context_manager.py rename to tests/contexts/test_context_manager.py diff --git a/tests/test_context_manager_decorator.py b/tests/contexts/test_context_manager_decorator.py similarity index 98% rename from tests/test_context_manager_decorator.py rename to tests/contexts/test_context_manager_decorator.py index f0acf6a..2160602 100644 --- a/tests/test_context_manager_decorator.py +++ b/tests/contexts/test_context_manager_decorator.py @@ -3,7 +3,7 @@ from typing import Dict, Any from nuropb.contexts.context_manager import NuropbContextManager -from nuropb.contexts.decorators import nuropb_context +from nuropb.contexts.context_manager_decorator import nuropb_context logger = logging.getLogger(__name__) diff --git a/tests/contexts/test_describe.py b/tests/contexts/test_describe.py new file mode 100644 index 0000000..4faca0f --- /dev/null +++ b/tests/contexts/test_describe.py @@ -0,0 +1,173 @@ +import datetime +from typing import List, Optional +from uuid import uuid4 +from dataclasses import dataclass +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.backends import default_backend + +from nuropb.contexts.context_manager import NuropbContextManager +from nuropb.contexts.context_manager_decorator import nuropb_context +from nuropb.contexts.describe import describe_service, publish_to_mesh + +service_describe = service_api_spec = { + "name": "order_management_service", + "service_name": "order_management_service", + "published_events": [], + "service_model": [], + "methods": [ + { + "name": "create_order", + "description": ( + "Create an order for a given security, quantity, side and order_type. Order price input is dependant " + "on order_type and if a date is not given the current date is used, dates prior to today will be " + "rejected by the API." + ), + "parameters": { + "type": "object", + "properties": { + "account": { + "type": "string", + "description": "The account into which the executed trade will be booked", + }, + "security": { + "type": "string", + "description": "the security to be traded", + }, + "status": { + "type": "string", + "enum": ["Open", "Filled", "Cancelled"], + "description": "The status of the order.", + }, + "quantity": { + "type": "integer", + "description": "the quantity to be traded", + }, + "side": { + "type": "string", + "enum": ["Buy", "Sell"], + "description": "The side of the order", + }, + "order_type": { + "type": "string", + "enum": ["Market", "Limit", "Stop"], + "description": "The order type", + }, + "price": { + "type": "number", + "description": "The price of the order", + }, + "time_in_force": { + "type": "string", + "enum": ["Day", "GTC", "FOK", "IOC"], + "description": "The time in force of the order", + }, + "stop_price": { + "type": "number", + "description": "The stop price of the order", + }, + "order_date": { + "type": "string", + "format": "date", + "description": "The date of the order", + }, + }, + "required": ["account", "security", "quantity", "side"] + } + } + ] +} + + +@dataclass +class Order: + account: str + security: str + quantity: int + side: str + + order_id: str = uuid4().hex + order_date: datetime.datetime = datetime.datetime.now(datetime.timezone.utc) + status: str = "Open" + order_type: str = "Market" + time_in_force: str = "Day" + + executed_time: Optional[datetime] = None + account_id: Optional[str] = None + security_id: Optional[str] = None + price: Optional[float] = None + stop_price: Optional[float] = None + + +class OrderManagementService: + """ + Some useful documentation to describe the characteristic of the service and its purpose + """ + _service_name = "oms_v2" + _version = "2.0.1" + _private_key = rsa.generate_private_key( + public_exponent=65537, key_size=2048, backend=default_backend() + ) + + @publish_to_mesh(requires_encryption=True) + @nuropb_context + async def get_orders( + self, + ctx, + order_date: datetime.datetime, + account: Optional[str] = None, + status: Optional[str] = "", + security: Optional[str] = None, + side: Optional[str] = None + ) -> List[Order]: + return [] + + @publish_to_mesh + @nuropb_context + async def create_order( + self, + ctx, + order: Order) -> Order: + new_order = Order(account="ABC1234", + security="SSE.L", + quantity=1000, + side="sell") + return new_order + + +class Service: + """ Some useful documentation to describe the characteristic of the service and its purpose + """ + service_name = "describe_service" + + def hello(self, _param1: str, param2: str = "value2") -> str: + _ = self, _param1, param2 + return "hello world" + + @nuropb_context + async def do_some_transaction( + self, + ctx: NuropbContextManager, + order_no: str, + order_date: datetime.datetime, + order_amount: float + ) -> str: + """ Some useful documentation for this method + + :param ctx: + :param order_no: + :param order_date: + :param order_amount: + :return: + """ + _ = self, ctx, order_no, order_date, order_amount + return "transaction successful" + + def _private_method(self, param1, param2): + _ = self, param1, param2 + return "private method" + + +def test_instance_describe(): + service_instance = OrderManagementService() + result = describe_service(service_instance) + assert 1 == 1 diff --git a/tests/encodings/test_json_serialisation.py b/tests/encodings/test_json_serialisation.py index c075d43..46b777f 100644 --- a/tests/encodings/test_json_serialisation.py +++ b/tests/encodings/test_json_serialisation.py @@ -1,12 +1,36 @@ import datetime import json from decimal import Decimal +from dataclasses import dataclass +from uuid import uuid4 +from typing import Optional + import pytest from nuropb.encodings.json_serialisation import to_json_compatible from nuropb.encodings.serializor import encode_payload, decode_payload +@dataclass +class Order: + account: str + security: str + quantity: int + side: str + + order_id: str = uuid4().hex + order_date: datetime.datetime = datetime.datetime.now(datetime.timezone.utc) + status: str = "Open" + order_type: str = "Market" + time_in_force: str = "Day" + + executed_time: Optional[datetime] = None + account_id: Optional[str] = None + security_id: Optional[str] = None + price: Optional[float] = None + stop_price: Optional[float] = None + + @pytest.fixture(scope="function") def generic_payload(): return { @@ -40,6 +64,10 @@ def generic_payload(): "datetime": datetime.datetime(2020, 1, 1, 0, 0, 0), "set": {1, 2, Decimal(3)}, }, + "dataclass": Order(account="ABC1234", + security="SSE.L", + quantity=1000, + side="sell") } From 64f592be10c1ab014e3677934ae017db8b13359a Mon Sep 17 00:00:00 2001 From: Robert Betts Date: Mon, 18 Sep 2023 23:02:47 +0100 Subject: [PATCH 06/10] Big refactor to transport message sending. reduced complexity and did prep work for plugging in encryption Some mypy type linting --- src/nuropb/contexts/context_manager.py | 6 +- .../contexts/context_manager_decorator.py | 23 +- src/nuropb/contexts/describe.py | 108 ++++--- src/nuropb/encodings/encryption.py | 10 +- src/nuropb/encodings/json_serialisation.py | 7 +- src/nuropb/encodings/serializor.py | 2 - src/nuropb/interface.py | 80 +---- src/nuropb/rmq_api.py | 293 +++++++----------- src/nuropb/rmq_lib.py | 11 +- src/nuropb/rmq_transport.py | 218 +++++-------- src/nuropb/service_handlers.py | 15 +- .../test_encoding_encrypt_decrypt.py | 1 - tests/test_api_service_request.py | 54 ++-- tests/test_client.py | 1 - tests/test_handlers.py | 2 - tests/test_nuropb_interface.py | 1 - 16 files changed, 325 insertions(+), 507 deletions(-) diff --git a/src/nuropb/contexts/context_manager.py b/src/nuropb/contexts/context_manager.py index f7b1f6f..e2a3075 100644 --- a/src/nuropb/contexts/context_manager.py +++ b/src/nuropb/contexts/context_manager.py @@ -22,7 +22,7 @@ class NuropbContextManager: they are discarded. """ - _suppress_exceptions: bool | None + _suppress_exceptions: bool _nuropb_payload: Dict[str, Any] | None _context: Dict[str, Any] _user_claims: Dict[str, Any] | None @@ -124,7 +124,7 @@ def _handle_context_exit( **** Context Manager sync and async methods **** """ - def __enter__(self): + def __enter__(self) -> Any: """This method is called when entering a context manager with a with statement""" if self._done: raise RuntimeError("Context manager has already exited") @@ -134,7 +134,7 @@ def __enter__(self): return self - async def __aenter__(self): + async def __aenter__(self) -> Any: return self.__enter__() def __exit__( diff --git a/src/nuropb/contexts/context_manager_decorator.py b/src/nuropb/contexts/context_manager_decorator.py index e5d6440..b2292c1 100644 --- a/src/nuropb/contexts/context_manager_decorator.py +++ b/src/nuropb/contexts/context_manager_decorator.py @@ -1,23 +1,23 @@ import inspect -from typing import Optional, Any +from typing import Optional, Any, Callable from functools import wraps from nuropb.contexts.context_manager import NuropbContextManager -def method_has_nuropb_context(method: callable) -> bool: +def method_has_nuropb_context(method: Callable[..., Any]) -> bool: """This function checks if a method has been decorated with @nuropb_context - :param method: callable + :param method: Callable :return: bool """ return getattr(method, "__nuropb_context__", False) def nuropb_context( - original_method=None, + original_method: Optional[Callable[..., Any]] = None, *, - context_parameter: Optional[str] = "ctx", - suppress_exceptions: Optional[bool] = False, + context_parameter: str = "ctx", + suppress_exceptions: bool = False, ) -> Any: """This decorator function injects a NuropbContext instance into a method that has ctx:NuropbContext as an argument. The ctx parameter of the decorated method is hidden from the method's signature visible @@ -44,17 +44,17 @@ def hello_requires_auth(...) :param suppress_exceptions: bool :return: a decorated method """ - context_parameter = "ctx" if context_parameter is None else context_parameter + context_parameter = context_parameter or "ctx" suppress_exceptions = False if suppress_exceptions is None else suppress_exceptions - def decorator(method): + def decorator(method: Callable[..., Any]) -> Callable[..., Any]: if context_parameter not in inspect.signature(method).parameters: raise TypeError( f"method {method.__name__} does not have {context_parameter} as an argument" ) @wraps(method) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Any: """validate calling arguments""" if len(args) < 2 or not isinstance(args[1], (NuropbContextManager, dict)): raise TypeError( @@ -78,9 +78,8 @@ def wrapper(*args, **kwargs): kwargs[context_parameter] = ctx return method(*args[1:], **kwargs) - - wrapper.__nuropb_context__ = True - wrapper.__nuropb_context_arg__ = "ctx" + setattr(wrapper, "__nuropb_context__", True) + setattr(wrapper, "__nuropb_context_arg__", context_parameter) return wrapper if original_method: diff --git a/src/nuropb/contexts/describe.py b/src/nuropb/contexts/describe.py index 4de1363..eccbeae 100644 --- a/src/nuropb/contexts/describe.py +++ b/src/nuropb/contexts/describe.py @@ -1,14 +1,15 @@ import logging import inspect -from typing import Optional, Any, get_origin, get_type_hints, get_args +from typing import Optional, Any, get_origin, get_args, Callable, Dict, Tuple from functools import wraps from cryptography.hazmat.primitives import serialization logger = logging.getLogger(__name__) +AuthoriseFunc = Callable[[str], Dict[str, Any]] -def method_visible_on_mesh(method: callable) -> bool: +def method_visible_on_mesh(method: Callable[..., Any]) -> bool: """This function checks if a method has been decorated with @publish_to_mesh :param method: callable :return: bool @@ -17,17 +18,17 @@ def method_visible_on_mesh(method: callable) -> bool: def publish_to_mesh( - original_method=None, - *, - hide_method: Optional[bool] = False, - authorise_func: Optional[callable] = None, - context_token_key: Optional[str] = "Authorization", - requires_encryption: Optional[bool] = False, - description: Optional[str] = None, + original_method: Optional[Callable[..., Any]] = None, + *, + hide_method: Optional[bool] = False, + authorise_func: Optional[AuthoriseFunc] = None, + context_token_key: Optional[str] = "Authorization", + requires_encryption: Optional[bool] = False, + description: Optional[str] = None, ) -> Any: - """ Decorator to expose class methods to the service mesh + """Decorator to expose class methods to the service mesh - When a service instance is connected to the a service mesh via the service mesh client, all + When a service instance is connected to a service mesh via the service mesh client, all the standard public methods of the service instance is available to the service mesh. Methods that start with underscore will always remain hidden. methods that are explicitly marked as hidden by publish_to_mesh will also not be published. @@ -58,23 +59,24 @@ def hello_requires_auth(...) :return: """ hide_method = False if hide_method is None else hide_method - context_token_key = "Authorization" if context_token_key is None else context_token_key + context_token_key = ( + "Authorization" if context_token_key is None else context_token_key + ) if authorise_func is not None and not callable(authorise_func): raise TypeError("Authorise function must be callable") requires_encryption = False if requires_encryption is None else requires_encryption - def decorator(method): - + def decorator(method: Callable[..., Any]) -> Callable[..., Any]: @wraps(method) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Any: return method(*args, **kwargs) - wrapper.__nuropb_mesh_hidden__ = hide_method - wrapper.__nuropb_context_token_key__ = context_token_key - wrapper.__nuropb_authorise_func__ = authorise_func - wrapper.__nuropb_requires_encryption__ = requires_encryption + setattr(wrapper, "__nuropb_mesh_hidden__", hide_method) + setattr(wrapper, "__nuropb_context_token_key__", context_token_key) + setattr(wrapper, "__nuropb_authorise_func__", authorise_func) + setattr(wrapper, "__nuropb_requires_encryption__", requires_encryption) if description: - wrapper.__nuropb_description__ = description + setattr(wrapper, "__nuropb_description__", description) return wrapper if original_method: @@ -83,31 +85,32 @@ def wrapper(*args, **kwargs): return decorator -def describe_service(class_instance): - """ Returns a description of the class methods that will be exposed to the service mesh - """ +def describe_service(class_instance: object) -> Dict[str, Any] | None: + """Returns a description of the class methods that will be exposed to the service mesh""" if class_instance is None: logger.warning("No service class base has been input") return None else: service_name = getattr(class_instance, "_service_name", None) service_description = getattr(class_instance, "__doc__", None) - service_description = service_description.strip() if service_description else service_description + service_description = ( + service_description.strip() if service_description else service_description + ) service_version = getattr(class_instance, "_version", None) methods = [] service_has_encrypted_methods = False for name, method in inspect.getmembers(class_instance): - - """ all private methods are excluded, regardless if one has been decorated with @publish_to_mesh - """ - if name[0] == '_' or not callable(method): + """all private methods are excluded, regardless if one has been decorated with @publish_to_mesh""" + if name[0] == "_" or not callable(method): continue if getattr(method, "__nuropb_mesh_hidden__", False): continue - requires_encryption = getattr(method, "__nuropb_requires_encryption__", False) + requires_encryption = getattr( + method, "__nuropb_requires_encryption__", False + ) if requires_encryption and not service_has_encrypted_methods: service_has_encrypted_methods = True @@ -115,13 +118,17 @@ def describe_service(class_instance): method_signature = inspect.signature(method) required = [] - def map_annotation(arg_props): + def map_annotation(arg_props: Any) -> str: annotation = arg_props.annotation label = "" - if not (annotation == inspect._empty): + if annotation != inspect._empty: origin = get_origin(annotation) if origin is not None: - args = [a for a in get_args(annotation) if a.__name__ not in ("NoneType",)] + args = [ + a + for a in get_args(annotation) + if a.__name__ not in ("NoneType",) + ] if len(args) == 1: label = args[0].__name__ @@ -129,7 +136,7 @@ def map_annotation(arg_props): label = annotation.__name__ return label - def map_default(arg_props): + def map_default(arg_props: Any) -> Any: default = arg_props.default if default == inspect._empty: required.append(arg_props.name) @@ -137,23 +144,32 @@ def map_default(arg_props): else: return default - def map_argument(arg_props): - return (arg_props.name, { - "type": map_annotation(arg_props), - "description": "", - "default": map_default(arg_props), - }) - - properties = [map_argument(p) for n, p in method_signature.parameters.items() if n not in ("self", "cls", ctx_arg_name)] + def map_argument(arg_props: Any) -> Tuple[str, Dict[str, Any]]: + return ( + arg_props.name, + { + "type": map_annotation(arg_props), + "description": "", + "default": map_default(arg_props), + }, + ) + + properties = [ + map_argument(p) + for n, p in method_signature.parameters.items() + if n not in ("self", "cls", ctx_arg_name) + ] method_spec = { - "description": getattr(method, "__nuropb_description__", inspect.getdoc(method)), + "description": getattr( + method, "__nuropb_description__", inspect.getdoc(method) + ), "requires_encryption": requires_encryption, "parameters": { "type": "object", "properties": properties, "required": required, - } + }, } methods.append((name, method_spec)) @@ -168,11 +184,13 @@ def map_argument(arg_props): if service_has_encrypted_methods: private_key = service_name = getattr(class_instance, "_private_key", None) if private_key is None: - raise ValueError(f"Service {service_name} has encrypted methods but no private key has been set") + raise ValueError( + f"Service {service_name} has encrypted methods but no private key has been set" + ) service_info["public_key"] = private_key.public_key().public_bytes( encoding=serialization.Encoding.PEM, - format=serialization.PublicFormat.SubjectPublicKeyInfo + format=serialization.PublicFormat.SubjectPublicKeyInfo, ) return service_info diff --git a/src/nuropb/encodings/encryption.py b/src/nuropb/encodings/encryption.py index 8f1f7cb..d434852 100644 --- a/src/nuropb/encodings/encryption.py +++ b/src/nuropb/encodings/encryption.py @@ -9,7 +9,7 @@ def encrypt_payload(payload: str | bytes, key: str | bytes) -> bytes: - """Encrypt a encoded_payload with a key + """Encrypt encoded_payload with a key :param payload: str | bytes :param key: str :return: bytes @@ -22,7 +22,7 @@ def encrypt_payload(payload: str | bytes, key: str | bytes) -> bytes: def decrypt_payload(encrypted_payload: str | bytes, key: str | bytes) -> bytes: - """Decrypt a encoded_payload with a key + """Decrypt encoded_payload with a key :param encrypted_payload: str | bytes :param key: str :return: bytes @@ -91,7 +91,7 @@ def new_symmetric_key(cls) -> bytes: """ return Fernet.generate_key() - def add_service_public_key(self, service_name: str, public_key: rsa.RSAPublicKey): + def add_service_public_key(self, service_name: str, public_key: rsa.RSAPublicKey) -> None: """Add a public key for a service :param service_name: str :param public_key: rsa.RSAPublicKey @@ -128,7 +128,7 @@ def encrypt_payload( public_key = self._private_key.public_key() else: # Mode 1, get public key from the destination service's public key - public_key = self._service_public_keys.get(service_name, None) + public_key = self._service_public_keys[service_name] if correlation_id not in self._correlation_id_symmetric_keys: # Mode 1, generate a new symmetric key and store it for this correlation_id @@ -137,7 +137,7 @@ def encrypt_payload( else: # Mode 4, use the original received symmetric key to encrypt key # pop it from the dict as it is not required again - key = self._correlation_id_symmetric_keys.pop(correlation_id, None) + key = self._correlation_id_symmetric_keys.pop(correlation_id) encrypted_key = encrypt_key(key, public_key) encrypted_payload = encrypt_payload(payload=payload, key=key) diff --git a/src/nuropb/encodings/json_serialisation.py b/src/nuropb/encodings/json_serialisation.py index c8693c7..5f92e73 100644 --- a/src/nuropb/encodings/json_serialisation.py +++ b/src/nuropb/encodings/json_serialisation.py @@ -1,7 +1,7 @@ """ This module provides entire nuropb package with json serialisation logic and features # TODO: Re-check the serialised datetime, date, time and timedelta formats. Look for standards. """ -from typing import Any, Dict, Optional +from typing import Any, Dict import json import datetime from decimal import Decimal @@ -83,7 +83,8 @@ class NuropbEncoder(json.JSONEncoder): to_json_compatible, above. Difference is that the json library implements its own object traversal logic. In the function it's required to be done explicitly. """ - def default(self, obj): + + def default(self, obj: Any) -> Any: if isinstance(obj, datetime.datetime): json_string = f"{obj.isoformat()}Z" return json_string @@ -121,7 +122,7 @@ class JsonSerializor(object): """ encryption keys related to a given correlation_id """ - def __init__(self): + def __init__(self) -> None: """Initializes a new JsonSerializor instance.""" self._encryption_keys = {} diff --git a/src/nuropb/encodings/serializor.py b/src/nuropb/encodings/serializor.py index ada239a..0351b9a 100644 --- a/src/nuropb/encodings/serializor.py +++ b/src/nuropb/encodings/serializor.py @@ -1,5 +1,3 @@ -from typing import Optional - from nuropb.encodings.json_serialisation import JsonSerializor from nuropb.interface import PayloadDict diff --git a/src/nuropb/interface.py b/src/nuropb/interface.py index 89ea89e..735fed0 100644 --- a/src/nuropb/interface.py +++ b/src/nuropb/interface.py @@ -88,7 +88,6 @@ class RequestPayloadDict(TypedDict): service: str method: str params: Dict[str, Any] - reply_to: str class CommandPayloadDict(TypedDict): @@ -143,6 +142,7 @@ class ResponsePayloadDict(TypedDict): result: Any error: Optional[Dict[str, Any]] warning: Optional[str] + reply_to: str PayloadDict = Union[ @@ -218,21 +218,19 @@ class TransportRespondPayload(TypedDict): class NuropbException(Exception): """NuropbException: represents a base exception for all exceptions raised by the nuropb API although the input parameters are optional, it is recommended that the message is set to a - meaningful value and the nuropb_lifecycle and nuropb_message are set to the values that were - present when the exception was raised. + meaningful value and the nuropb_message is set to the values that were present when the + exception was raised. """ description: str - lifecycle: NuropbLifecycleState | None - payload: PayloadDict | TransportServicePayload | TransportRespondPayload | None - exception: Exception | BaseException | None + payload: PayloadDict | TransportServicePayload | TransportRespondPayload | Dict[str, Any] | None + exception: BaseException | None def __init__( self, description: Optional[str] = None, - lifecycle: Optional[NuropbLifecycleState] = None, payload: Optional[PayloadDict] = None, - exception: Optional[Exception] = None, + exception: Optional[BaseException] = None, ): if description is None: description = ( @@ -242,11 +240,10 @@ def __init__( ) super().__init__(description) self.description = description - self.lifecycle = lifecycle self.payload = payload self.exception = exception - def to_dict(self): + def to_dict(self) -> Dict[str, Any]: underlying_exception = str(self.exception) if self.exception else str(self) description = self.description if self.description else underlying_exception return { @@ -256,57 +253,30 @@ def to_dict(self): class NuropbTimeoutError(NuropbException): - """NuropbTimeoutError: represents an error that occurred when a timeout was reached. - - The handling of this error will depend on the message type, context and where in the lifecycle - of the message the timeout occurred. - """ - - pass + """NuropbTimeoutError: represents an error that occurred when a timeout was reached.""" class NuropbTransportError(NuropbException): - """NuropbTransportError: represents an error that inside the plumbing. - - The handling of this error will depend on the message type, context and where in the lifecycle - of the message the timeout occurred. - """ - - pass + """NuropbTransportError: represents an error that inside the plumbing.""" class NuropbMessageError(NuropbException): """NuropbMessageError: represents an error that occurred during the encoding or decoding of a message. - - The handling of this error will depend on the message type, context and where in the lifecycle - of the message the timeout occurred. """ - pass - class NuropbHandlingError(NuropbException): """NuropbHandlingError: represents an error that occurred during the execution or fulfilment of a request or command. An error response is returned to the requester. - - The handling of this error will depend on the message type, context and where in the lifecycle - of the message the timeout occurred. """ - pass - class NuropbDeprecatedError(NuropbHandlingError): """NuropbDeprecatedError: represents an error that occurred during the execution or fulfilment of a request, command or event topic that has been marked deprecated. - - An error response is returned to the requester ONLY for requests and commands. - Events will be rejected with a NACK with requeue=False. """ - pass - class NuropbValidationError(NuropbException): """NuropbValidationError: represents an error that occurred during the validation of a @@ -316,8 +286,6 @@ class NuropbValidationError(NuropbException): Events will be rejected with a NACK with requeue=False. """ - pass - class NuropbAuthenticationError(NuropbException): """NuropbAuthenticationError: when this exception is raised, the transport layer will ACK the @@ -326,16 +294,11 @@ class NuropbAuthenticationError(NuropbException): This exception occurs whe the identity of the requester can not be validated. for example an unknown, invalid or expired user identifier or auth token. - The handling of this error will depend on the message type, context and where in the lifecycle - of the message the timeout occurred. - In most cases, the requester will not be able to recover from this error and will need provide valid credentials and retry the request. The approach to this retry outside the scope of the nuropb API. """ - pass - class NuropbAuthorizationError(NuropbException): """NuropbAuthorizationError: when this exception is raised, the transport layer will ACK the @@ -344,9 +307,6 @@ class NuropbAuthorizationError(NuropbException): This exception occurs whe the requester does not have the required privileges to perform the requested action of either a request or command. - The handling of this error will depend on the message type, context and where in the lifecycle - of the message the timeout occurred. - In most cases, the requester will not be able to recover from this error and will need provide valid credentials and retry the request. The approach to this retry outside the scope of the nuropb API. @@ -411,13 +371,11 @@ def __init__( self, result: Any, description: Optional[str] = None, - payload: ResponsePayloadDict = None, - lifecycle: Optional[NuropbLifecycleState] = None, + payload: Optional[ResponsePayloadDict] = None, events: Optional[List[EventType]] = None, ): super().__init__( description=description, - lifecycle=lifecycle, payload=payload, exception=None, ) @@ -485,14 +443,6 @@ def receive_transport_message( For failures service messages are handled, other than for events, a response including details of the error is returned to the flow originator. - The Exception type raised during the message handling influences the lifecycle flow: - Some of these could be: - - NuropbTimeoutError - - NuropbHandlingError - - NuropbAuthenticationError - - NuropbCallAgain - - NuropbSuccess - :param service_message: TransportServicePayload :param message_complete_callback: MessageCompleteFunction :param metadata: Dict[str, Any] - metric gathering information @@ -519,12 +469,12 @@ async def request( :param context: additional information that represent the context in which the request is executed. The must be easily serializable to JSON. :param ttl: the time to live of the request in milliseconds. After this time and dependent on the - lifecycle state and underlying transport, it will not be consumed by the target service and + state and underlying transport, it will not be consumed by the target service and should be assumed by the requester to have failed with an undetermined state. :param trace_id: an identifier to trace the request over the network (e.g. uuid4 hex string) :param rpc_response: if True (default), the actual response of the RPC call is returned and where - there was an error during the lifecycle, this is raised as an exception. - Where rpc_response is a ResponsePayloadDict, is returned. + there was an error, that is raised as an exception. Where rpc_response is a ResponsePayloadDict, + it is returned. :return: ResponsePayloadDict """ @@ -536,7 +486,6 @@ def command( method: str, params: Dict[str, Any], context: Dict[str, Any], - wait_for_ack: bool = False, ttl: Optional[int] = None, trace_id: Optional[str] = None, ) -> None: @@ -550,9 +499,6 @@ def command( :param params: the method arguments, these must be easily serializable to JSON :param context: additional information that represent the context in which the request is executed. The must be easily serializable to JSON. - :param wait_for_ack: if True, the command will wait for an acknowledgement from the transport layer that the - target has received the command. If False, the request will return immediately after the request is - delivered to the transport layer. :param ttl: the time to live of the request in milliseconds. After this time and dependent on the underlying transport, it will not be consumed by the target or diff --git a/src/nuropb/rmq_api.py b/src/nuropb/rmq_api.py index 1b60f0a..0354f2a 100644 --- a/src/nuropb/rmq_api.py +++ b/src/nuropb/rmq_api.py @@ -4,11 +4,11 @@ from asyncio import Future from cryptography.hazmat.primitives import serialization from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa from nuropb.interface import ( NuropbInterface, NuropbMessageError, - PayloadDict, ResponsePayloadDict, RequestPayloadDict, EventPayloadDict, @@ -16,11 +16,10 @@ ResultFutureResponsePayload, TransportServicePayload, MessageCompleteFunction, - NUROPB_PROTOCOL_VERSION, CommandPayloadDict, + NuropbException, ) from nuropb.rmq_transport import RMQTransport -from nuropb.encodings.serializor import encode_payload from nuropb.service_handlers import execute_request, handle_execution_result logger = logging.getLogger(__name__) @@ -65,20 +64,24 @@ def __init__( self._instance_id = instance_id if instance_id is not None else uuid4().hex if service_name is None: + """Configure for client only mode""" self._client_only = True self._connection_name = f"{vhost}-client-{instance_id}" service_name = f"{vhost}-client" else: + """Configure for service mode""" self._client_only = False self._connection_name = f"{vhost}-{service_name}-{instance_id}" self._service_name = service_name + """ Is also a label for the api whether in client or service mode. + """ self._service_discovery = {} - """ A dictionary of service_name: service_info + """ A dictionary of service_name: service mesh discovered service_info """ self._service_public_keys = {} - """ A dictionary of service_name: public_key + """ A dictionary of service_name: public_key for when encryption is required """ if not self._client_only and service_instance is None: @@ -86,15 +89,19 @@ def __init__( "A service instance must be provided when starting in service mode" ) # pragma: no cover self._service_instance = service_instance + """ the class instance that will be shared on the service mesh + """ transport_settings = {} if transport_settings is None else transport_settings if self._client_only: transport_settings["client_only"] = True - default_ttl = transport_settings.get("default_ttl", None) - self._response_futures = {} + + default_ttl = transport_settings.get("default_ttl", None) self._default_ttl = 60 * 60 * 1000 if default_ttl is None else default_ttl + """ default time to live or timeout service mesh interaction + """ self._api_connected = False @@ -151,8 +158,6 @@ async def connect(self) -> None: """ if not self.connected: await self._transport.start() - # task = asyncio.create_task(self.keep_loop_active()) - # task.add_done_callback(lambda _: None) else: logger.warning("Already connected") @@ -168,9 +173,8 @@ def receive_transport_message( message_complete_callback: MessageCompleteFunction, metadata: Dict[str, Any], ) -> None: - """receive_transport_message: handles a messages received from the transport layer - for processing by the service instance. Both incoming service messages and response - messages pass through this method. + """receive_transport_message: handles a messages received from the transport layer. Both + incoming service messages and response messages pass through this method. :return: None """ @@ -208,11 +212,12 @@ def receive_transport_message( """ The logic below is only relevant for incoming service messages """ if self._service_instance is None: - error_description = f"No service instance configured to handle the {service_message['nuropb_type']} instruction" + error_description = ( + f"No service instance configured to handle the {service_message['nuropb_type']} instruction" + ) logger.warning(error_description) response = NuropbHandlingError( description=error_description, - lifecycle="service-handle", payload=service_message["nuropb_payload"], ) handle_execution_result(service_message, response, MessageCompleteFunction) @@ -240,70 +245,35 @@ async def request( trace_id: Optional[str] = None, rpc_response: bool = True, ) -> Union[ResponsePayloadDict, Any]: - """Make a request for a method on service and wait until the response is received. The - request message uses the 'message expiry' configured on the underlying transport. - - expiry is the time in milliseconds that the message will be kept on the queue before being moved - to the dead letter queue. If None, then the message expiry configured on the transport is used. - - # TODO: Look into returning a dead letter exception for timeout or other errors that result - in the message being returned to dead letter queue. - - Parameters: - ---------- - service: str - The routing key on the rpc exchange to direct the request to the desired service request queue. - - method: str - The name of the api call / method on the service - - params: dict - The method input parameters - - context: dict + """Makes a rpc request for a method on a service mesh service and waits until the response is + received. + + :param service: str, The routing key on the rpc exchange to direct the request to the desired + service request queue. + :param method: str, the name of the api call / method on the service + :param params: dict, The method input parameters + :param context: dict The context of the request. This is used to pass information to the service manager and is not used by the transport. Example content includes: - user_id: str # a unique user identifier or token of the user that made the request - correlation_id: str # a unique identifier of the request used to correlate the response to the - # request or trace the request over the network (e.g. a uuid4 hex string) + # request or trace the request over the network (e.g. uuid4 hex string) - service: str - method: str - - ttl: int optional + :param ttl: int optional expiry is the time in milliseconds that the message will be kept on the queue before being moved to the dead letter queue. If None, then the message expiry configured on the transport is used. - - trace_id: str optional - an identifier to trace the request over the network (e.g. a uuid4 hex string) - - rpc_response: bool optional - if True (default), the actual response of the RPC call is returned and where there was an error - during the lifecycle, this is raised as an exception. + :param trace_id: str optional + an identifier to trace the request over the network (e.g. uuid4 hex string) + :param rpc_response: bool optional + if True (default), the actual response of the RPC call is returned and where there was an error, + that is raised as an exception. Where rpc_response is a ResponsePayloadDict, is returned. - - Returns: - -------- - ResponsePayloadDict | Any: representing the response from the requested service with any exceptions raised + :return ResponsePayloadDict | Any: representing the response from the requested service with any + exceptions raised """ - if method != "nuropb_describe": - service_public_key = await self.check_method_for_encryption(service, method) - else: - service_public_key = None - correlation_id = uuid4().hex ttl = self._default_ttl if ttl is None else ttl - properties = dict( - content_type="application/json", - correlation_id=correlation_id, - reply_to=self._transport.response_queue, - headers={ - "nuropb_protocol": NUROPB_PROTOCOL_VERSION, - "nuropb_type": "request", - "trace_id": trace_id, - }, - expiration=f"{ttl}", - ) - context["rmq_correlation_id"] = correlation_id message: RequestPayloadDict = { "tag": "request", "service": service, @@ -312,11 +282,7 @@ async def request( "correlation_id": correlation_id, "context": context, "trace_id": trace_id, - "reply_to": self._transport.response_queue, } - body = encode_payload(message, "json") - routing_key = service - response_future: ResultFutureResponsePayload = Future() self._response_futures[correlation_id] = response_future @@ -325,66 +291,63 @@ async def request( f"Sending request message:\n" f"correlation_id: {correlation_id}\n" f"trace_id: {trace_id}\n" - f"exchange: {self._transport.rpc_exchange}\n" - f"routing_key: {routing_key}\n" f"service: {service}\n" f"method: {method}\n" ) try: self._transport.send_message( - exchange=self._transport.rpc_exchange, - routing_key=routing_key, - body=body, - properties=properties, - mandatory=True, + payload=message, + expiry=ttl, + priority=None, + encoding="json", + publickey=None, ) - except Exception as e: + except Exception as err: if rpc_response is False: return { "tag": "response", "context": context, + "correlation_id": correlation_id, + "trace_id": trace_id, "result": None, "error": { - "error": f"{type(e).__name__}", - "description": f"Error sending request message: {e}", + "error": f"{type(err).__name__}", + "description": f"Error sending request message: {err}", }, } - response: PayloadDict | None = None + else: + raise err + try: response = await response_future - except Exception as err: - error_message = ( - f"Error while waiting for response to complete." - f"correlation_id: {correlation_id}, trace_id: {trace_id}, error:{err}" - ) - logger.exception(error_message) - raise NuropbMessageError( - description=error_message, - lifecycle="client-handle", - payload=response, - exception=err, - ) - - if response["tag"] != "response": - """This logic condition is prevented in the transport layer""" - raise NuropbMessageError( - description=f"Unexpected response message type: {response['tag']}", - lifecycle="client-handle", - payload=response, - exception=None, - ) - - if not rpc_response: - return response - elif response["error"]: - raise NuropbMessageError( - description=f"RPC service error: {response['error']}", - lifecycle="client-handle", - payload=response, - exception=None, - ) - else: - return response["result"] + if rpc_response is True and response["error"] is not None: + raise NuropbMessageError( + description=response["error"]["description"], + payload=response, + ) + elif rpc_response is True: + return response["result"] + else: + return response + except BaseException as err: + if rpc_response is True: + raise err + else: + if not isinstance(err, NuropbException): + error = { + "error": f"{type(err).__name__}", + "description": f"Error waiting for response: {err}", + } + else: + error = err.to_dict() + return { + "tag": "response", + "context": context, + "correlation_id": correlation_id, + "trace_id": trace_id, + "result": None, + "error": error, + } def command( self, @@ -392,23 +355,17 @@ def command( method: str, params: Dict[str, Any], context: Dict[str, Any], - wait_for_ack: bool = False, ttl: Optional[int] = None, trace_id: Optional[str] = None, ) -> None: - """command: sends a command to the target service. It is up to the implementation to manage message - idempotency and message delivery guarantees. - - any response from the target service is ignored. + """command: sends a command to the target service. I.e. a targeted event. response is not expected + and ignored. :param service: the service name :param method: the method name :param params: the method arguments, these must be easily serializable to JSON :param context: additional information that represent the context in which the request is executed. The must be easily serializable to JSON. - :param wait_for_ack: if True, the command will wait for an acknowledgement from the transport layer that the - target has received the command. If False, the request will return immediately after the request is - delivered to the transport layer. :param ttl: the time to live of the request in milliseconds. After this time and dependent on the underlying transport, it will not be consumed by the target or @@ -419,17 +376,6 @@ def command( """ correlation_id = uuid4().hex ttl = self._default_ttl if ttl is None else ttl - properties = dict( - content_type="application/json", - correlation_id=correlation_id, - headers={ - "nuropb_protocol": NUROPB_PROTOCOL_VERSION, - "nuropb_type": "command", - "trace_id": trace_id, - }, - expiration=f"{ttl}", - ) - context["rmq_correlation_id"] = correlation_id message: CommandPayloadDict = { "tag": "command", "service": service, @@ -439,25 +385,17 @@ def command( "context": context, "trace_id": trace_id, } - body = encode_payload(message, "json") - routing_key = service # mandatory means that if it doesn't get routed to a queue then it will be returned vi self._on_message_returned logger.debug( f"Sending command message:\n" f"correlation_id: {correlation_id}\n" f"trace_id: {trace_id}\n" - f"exchange: {self._transport.rpc_exchange}\n" - f"routing_key: {routing_key}\n" f"service: {service}\n" f"method: {method}\n" ) self._transport.send_message( - exchange=self._transport.rpc_exchange, - routing_key=routing_key, - body=body, - properties=properties, - mandatory=True, + payload=message, expiry=ttl, priority=None, encoding="json", publickey=None ) def publish_event( @@ -467,44 +405,21 @@ def publish_event( context: Dict[str, Any], trace_id: Optional[str] = None, ) -> None: - """Broadcasts an event with the given 'topic'. + """Broadcasts an event with the given topic. - Parameters: - ---------- - topic: str - The routing key on the events exchange - - event: json-encodable Python Dict. - - context: dict - The context around gent generation, example content includes: + :param topic: str, The routing key on the events exchange + :param event: json-encodable Python Dict. + :param context: dict, The context around gent generation, example content includes: - user_id: str # a unique user identifier or token of the user that made the request - correlation_id: str # a unique identifier of the request used to correlate the response # to the request # or trace the request over the network (e.g. an uuid4 hex string) - service: str - method: str - - ttl: int optional - expiry is the time in milliseconds that the message will be kept on the queue before being moved - to the dead letter queue. If None, then the message expiry configured on the transport is used. - defaulted to 0 (no expiry) for events - - trace_id: str optional + :param trace_id: str optional an identifier to trace the request over the network (e.g. an uuid4 hex string) - """ correlation_id = uuid4().hex - properties = dict( - content_type="application/json", - correlation_id=correlation_id, - headers={ - "nuropb_protocol": NUROPB_PROTOCOL_VERSION, - "nuropb_type": "event", - "trace_id": trace_id, - }, - ) - context["rmq_correlation_id"] = correlation_id message: EventPayloadDict = { "tag": "event", "topic": topic, @@ -514,24 +429,19 @@ def publish_event( "correlation_id": correlation_id, "target": None, } - body = encode_payload(message, "json") - routing_key = topic logger.debug( - "Sending event message: (%s - %s) (%s - %s)", + "Sending event message: (%s - %s)", correlation_id, trace_id, - self._transport.events_exchange, - routing_key, ) self._transport.send_message( - exchange=self._transport.events_exchange, - routing_key=routing_key, - body=body, - properties=properties, - mandatory=False, + payload=message, + encoding="json", + priority=None, + publickey=None, ) - async def describe_service(self, service_name): + async def describe_service(self, service_name: str) -> Dict[str, Any] | None: """describe_service: returns the service description for the given service_name :param service_name: str :return: dict @@ -546,7 +456,7 @@ async def describe_service(self, service_name): ) return result - async def check_method_for_encryption(self, service_name: str, method_name: str): + async def check_method_for_encryption(self, service_name: str, method_name: str) -> rsa.RSAPublicKey | None: """check_method_for_encryption: Queries the service discovery cache, if an entry for the service_name does not exist, then the service discovery is queried directly. @@ -558,10 +468,16 @@ async def check_method_for_encryption(self, service_name: str, method_name: str) :return: bool """ if service_name not in self._service_discovery: - self._service_discovery[service_name] = await self.describe_service(service_name) - text_public_key = self._service_discovery[service_name].get("public_key", None) + self._service_discovery[service_name] = await self.describe_service( + service_name + ) + text_public_key = self._service_discovery[service_name].get( + "public_key", None + ) if text_public_key: - self._service_public_keys[service_name] = serialization.load_pem_public_key( + self._service_public_keys[ + service_name + ] = serialization.load_pem_public_key( data=text_public_key, backend=default_backend(), ) @@ -569,10 +485,11 @@ async def check_method_for_encryption(self, service_name: str, method_name: str) service_info = self._service_discovery[service_name] method_info = service_info["methods"].get(method_name, None) if method_info is None: - raise ValueError(f"Method {method_name} not found on service {service_name}") + raise ValueError( + f"Method {method_name} not found on service {service_name}" + ) if method_info.get("requires_encryption", False): return self._service_public_keys.get(service_name, None) - - + return None diff --git a/src/nuropb/rmq_lib.py b/src/nuropb/rmq_lib.py index 84dc345..24ef08b 100644 --- a/src/nuropb/rmq_lib.py +++ b/src/nuropb/rmq_lib.py @@ -9,7 +9,7 @@ import pika from pika.channel import Channel -from nuropb.interface import PayloadDict, NuropbTransportError, NuropbLifecycleState +from nuropb.interface import PayloadDict, NuropbTransportError logger = logging.getLogger(__name__) @@ -242,16 +242,13 @@ def nack_message( properties: pika.spec.BasicProperties, mesg: PayloadDict | None, error: Exception | None = None, - lifecycle: NuropbLifecycleState | None = None, ) -> None: """nack_message: nack the message and requeue it, there was likely a recoverable problem with this instance while processing the message """ if channel is None or not channel.is_open: - lifecycle = "service-handle" if lifecycle is None else lifecycle raise NuropbTransportError( description="Unable to nack and requeue message, RMQ channel closed", - lifecycle=lifecycle, payload=mesg, exception=error, ) @@ -267,14 +264,11 @@ def reject_message( properties: pika.spec.BasicProperties, mesg: PayloadDict | None, error: Exception | None = None, - lifecycle: NuropbLifecycleState | None = None, ) -> None: """reject_message: If the message is not a request, then reject the message and move on""" if channel is None or not channel.is_open: - lifecycle = "service-handle" if lifecycle is None else lifecycle raise NuropbTransportError( description="unable to reject message, RMQ channel closed", - lifecycle=lifecycle, payload=mesg, exception=error, ) @@ -290,14 +284,11 @@ def ack_message( properties: pika.spec.BasicProperties, mesg: PayloadDict | None, error: Exception | None = None, - lifecycle: NuropbLifecycleState | None = None, ) -> None: """ack_message: ack the message""" if channel is None or not channel.is_open: - lifecycle = "service-handle" if lifecycle is None else lifecycle raise NuropbTransportError( description="Unable to ack message, RMQ channel closed", - lifecycle=lifecycle, payload=mesg, exception=error, ) diff --git a/src/nuropb/rmq_transport.py b/src/nuropb/rmq_transport.py index 9bbde87..d0b7930 100644 --- a/src/nuropb/rmq_transport.py +++ b/src/nuropb/rmq_transport.py @@ -1,8 +1,9 @@ import logging import functools -from typing import List, Set, Optional, Any, Dict, Awaitable, cast, Literal, TypedDict +from typing import List, Set, Optional, Any, Dict, Awaitable, Literal, TypedDict import asyncio import time +from cryptography.hazmat.primitives.asymmetric import rsa import pika from pika import connection @@ -14,9 +15,7 @@ from nuropb.encodings.serializor import encode_payload, decode_payload from nuropb.interface import ( - PayloadDict, NuropbTransportError, - NuropbLifecycleState, TransportServicePayload, NUROPB_PROTOCOL_VERSIONS_SUPPORTED, NUROPB_MESSAGE_TYPES, @@ -55,7 +54,7 @@ class RabbitMQConfiguration(TypedDict): logger = logging.getLogger(__name__) -connection.PRODUCT = "NuroPb Distributed RPC-Event Library" +connection.PRODUCT = "NuroPb Distributed RPC-Event Service Mesh Library" """ TODO: configure the RMQ client connection attributes in the pika client properties. See related TODO below in this module """ @@ -139,7 +138,6 @@ class RMQTransport: _is_leader: bool _is_rabbitmq_configured: bool - _ioloop: asyncio.AbstractEventLoop | None _connection: AsyncioConnection | None _channel: Channel | None _consumer_tags: Set[Any] @@ -166,7 +164,6 @@ def __init__( prefetch_count: Optional[int] = None, default_ttl: Optional[int] = None, client_only: Optional[bool] = None, - ioloop: Optional[asyncio.AbstractEventLoop] = None, ): # NOSONAR """Create a new instance of the consumer class, passing in the AMQP URL used to connect to RabbitMQ. @@ -193,7 +190,6 @@ def __init__( self._was_consuming = False self._consuming = False - self._ioloop = ioloop self._connection = None self._channel = None self._consumer_tags = set() @@ -233,13 +229,6 @@ def __init__( self._connected_future = None self._disconnected_future = None - @property - def ioloop(self) -> asyncio.AbstractEventLoop: - """ioloop: returns the asyncio event loop""" - if self._ioloop is None: - self._ioloop = asyncio.get_event_loop() - return self._ioloop - @property def service_name(self) -> str: return self._service_name @@ -416,26 +405,12 @@ def connect(self) -> Awaitable[bool]: logger.info("Connecting to %s", obfuscate_credentials(self._amqp_url)) self._connected_future = asyncio.Future() - client_properties = { - "service_name": self._service_name, - "instance_id": self._instance_id, - "client_only": self._client_only, - "nuropb_protocol": NUROPB_PROTOCOL_VERSION, - "nuropb_version": NUROPB_VERSION, - } conn = AsyncioConnection( parameters=pika.URLParameters(self._amqp_url), on_open_callback=self.on_connection_open, on_open_error_callback=self.on_connection_open_error, on_close_callback=self.on_connection_closed, - custom_ioloop=self.ioloop, - ) - # TODO: overwrite the pika client properties with our own, see top of module too - conn._client_properties.update( - { - "product": "NuroPb Distributed RPC-Event Library", - } ) self._connection = conn self._connecting = True @@ -563,7 +538,7 @@ def on_channel_closed(self, channel: Channel, reason: Exception) -> None: # investigate reasons and methods automatically reopen the channel. # until a solution is found it will be important to monitor for this condition - def declare_service_queue(self) -> None: + def declare_service_queue(self, frame: pika.frame.Method) -> None: """Refresh the request queue on RabbitMQ by invoking the Queue.Declare RPC command. When it is complete, the on_service_queue_declareok method will be invoked by pika. @@ -588,7 +563,7 @@ def declare_service_queue(self) -> None: else: logger.info("Client only, not declaring request queue") # Check that passing None in here is OK - self.on_bindok(None, userdata=self._response_queue) + self.on_bindok(frame, userdata=self._response_queue) def on_service_queue_declareok( self, frame: pika.frame.Method, _userdata: str @@ -651,7 +626,7 @@ def on_response_queue_declareok( """ _ = frame logger.info("Response queue declared ok: %s", _userdata) - self.declare_service_queue() + self.declare_service_queue(frame) def on_bindok(self, _frame: pika.frame.Method, userdata: str) -> None: """Invoked by pika when the Queue.Bind method has completed. At this @@ -661,11 +636,6 @@ def on_bindok(self, _frame: pika.frame.Method, userdata: str) -> None: :param str|unicode userdata: Extra user data (queue name) """ logger.info("Response queue bound ok: %s", userdata) - """This method sets up the consumer prefetch to only be delivered - one message at a time. The consumer must acknowledge this message - before RabbitMQ will deliver another one. You should experiment - with different prefetch values to achieve desired performance. - """ if self._channel is None: raise RuntimeError("RMQ transport channel is not open") @@ -680,14 +650,6 @@ def on_basic_qos_ok(self, _frame: pika.frame.Method) -> None: :param pika.frame.Method _frame: The Basic.QosOk response frame - This method sets up the consumer by first calling - add_on_cancel_callback so that the object is notified if RabbitMQ - cancels the consumer. It then issues the Basic.Consume RPC command - which returns the consumer tag that is used to uniquely identify the - consumer with RabbitMQ. We keep the value to use it when we want to - cancel consuming. The on_service_message method is passed in as a callback pika - will invoke when a message is fully received. - """ logger.info("QOS set to: %d", self._prefetch_count) logger.info("Configure message consumption") @@ -826,50 +788,71 @@ def on_message_returned( def send_message( self, - exchange: str, - routing_key: str, - body: bytes, - properties: Dict[str, Any], - mandatory: Optional[bool] = None, + payload: Dict[str, Any], + expiry: Optional[int] = None, + priority: Optional[int] = None, + encoding: str = "json", + publickey: Optional[rsa.RSAPrivateKey] = None, ) -> None: """Send a message to over the RabbitMQ Transport - TODO: Consider the handling if the channel that's closed. Wait and retry on a new channel? + TODO: Consider the alternative handling if the channel that's closed. + - Wait and retry on a new channel? - setup a retry queue? - should there be a high water mark for the number of retries? - should new messages not be consumed until the channel is re-established and retry queue drained? - :param str exchange: The exchange to publish to - :param str routing_key: The routing key to publish with - :param bytes body: The message body - :param Dict[str, Any] properties: The message properties - :param bool mandatory: The mandatory flag, defaults to True + :param Dict[str, Any] payload: The message contents + :param expiry: The message expiry in milliseconds + :param priority: The message priority + :param encoding: The encoding of the message + :param publickey: The public key to use for encryption, belongs to the target service """ - mandatory = True if mandatory is None else mandatory - headers = properties.setdefault("headers", {}) - headers.update( - { - "nuropb_protocol": NUROPB_PROTOCOL_VERSION, - } - ) - properties["headers"] = headers - if self._channel is None or self._channel.is_closed: - lifecycle: NuropbLifecycleState = "client-send" - if properties.get("headers", {}).get("nuropb_type", "") == "response": - lifecycle = "service-reply" - - payload: PayloadDict | None = None - if properties.get("content_type", "") == "application/json": - payload = cast(PayloadDict, decode_payload(body, "json")) - - raise NuropbTransportError( - description="RMQ channel closed, send message", - lifecycle=lifecycle, - payload=payload, - exception=None, - ) + _ = publickey + + mandatory = False + exchange = "" + reply_to = "" + + if payload["tag"] == "event": + exchange = self._events_exchange + routing_key = payload["topic"] + elif payload["tag"] in ("request", "command"): + mandatory = True + exchange = self._rpc_exchange + routing_key = payload["service"] + reply_to = self._response_queue + elif payload["tag"] == "response": + routing_key = payload["reply_to"] + else: + raise ValueError(f"Unknown payload type {payload['tag']}") + if encoding == "json": + body = encode_payload(payload, "json") else: + raise ValueError(f"unsupported encoding {encoding}") + + properties = dict( + content_type="application/json", + correlation_id=payload["correlation_id"], + reply_to=reply_to, + headers={ + "nuropb_protocol": NUROPB_PROTOCOL_VERSION, + "nuropb_version": NUROPB_VERSION, + "nuropb_type": payload["tag"], + "trace_id": payload["trace_id"], + }, + ) + if expiry: + properties["expiration"] = str(expiry) + if priority: + properties["priority"] = priority + + if not (self._channel is None or self._channel.is_closed): + """Check that the channel is in a valid state before sending the response + # TODO: Consider blocking any new messages until the channel is re-established + and the message is sent? Especially when it is a response message. + """ basic_properties = pika.BasicProperties(**properties) self._channel.basic_publish( exchange=exchange, @@ -879,6 +862,13 @@ def send_message( mandatory=mandatory, ) + else: + raise NuropbTransportError( + description="RMQ channel closed", + payload=payload, + exception=None, + ) + @classmethod def acknowledge_service_message( cls, @@ -925,50 +915,6 @@ def acknowledge_service_message( else: raise ValueError(f"Invalid action {action}") - def send_response_messages( - self, reply_to: str, message: TransportRespondPayload - ) -> None: - """Send response to request messages and also event messages. These messages can be over - new connections and channels. Any exceptions need to be contained and handled within the - scope of this method. - """ - correlation_id = "unknown" - trace_id = "unknown" - try: - nuropb_type = message["nuropb_type"] - correlation_id = message["correlation_id"] or "unknown" - trace_id = message["trace_id"] or "unknown" - if nuropb_type == "response": - routing_key = reply_to - exchange = "" - ttl = "" - else: - routing_key = "" - exchange = self._events_exchange - ttl = str(message.get("ttl") or self._default_ttl) - - body = encode_payload(message["nuropb_payload"], "json") - properties = { - "content_type": "application/json", - "correlation_id": message["correlation_id"], - "headers": { - "nuropb_type": nuropb_type, - "nuropb_protocol": NUROPB_PROTOCOL_VERSION, - "trace_id": trace_id, - }, - } - if ttl: - properties["expiration"] = ttl - - self.send_message( - exchange, routing_key, body, properties=properties, mandatory=True - ) - except Exception as error: - logger.critical( - f"Error sending message in response to request correlation_id: {correlation_id}, trace_id: {trace_id}" - ) - logger.exception(f"Failed to send response message: Error: {error}") - @classmethod def metadata_metrics(cls, metadata: Dict[str, Any]) -> None: """Invoked by the transport after a service message has been processed. @@ -981,6 +927,8 @@ def metadata_metrics(cls, metadata: Dict[str, Any]) -> None: :param metadata: information to drive message processing metrics :return: None """ + metadata["end_time"] = time.time() + metadata["duration"] = metadata["end_time"] - metadata["start_time"] logger.debug(f"metadata log: {metadata}") def on_service_message_complete( @@ -1015,6 +963,8 @@ def on_service_message_complete( delivery_tag = basic_deliver.delivery_tag redelivered = basic_deliver.redelivered reply_to = properties.reply_to + trace_id = properties.headers.get("trace_id", "unknown") + correlation_id = properties.correlation_id if redelivered is True and acknowledgement == "nack": if verbose: @@ -1023,8 +973,6 @@ def on_service_message_complete( ) acknowledgement = "reject" if properties.headers.get("nuropb_type", "") == "request": - trace_id = properties.headers.get("trace_id", "unknown") - correlation_id = properties.correlation_id exception = NuropbCallAgainReject( description=( f"Rejecting second call again request for trace_id: {trace_id}" @@ -1044,18 +992,24 @@ def on_service_message_complete( ) for respond_message in response_messages: - reply_to = reply_to or "" - self.send_response_messages(reply_to, respond_message) + respond_payload = respond_message["nuropb_payload"] + if respond_payload["tag"] == "response": + respond_payload["reply_to"] = reply_to + respond_payload["correlation_id"] = correlation_id + respond_payload["trace_id"] = trace_id + + self.send_message( + payload=respond_payload, + priority=None, + encoding="json", + publickey=None, + ) """ NOTE - METADATA: keep this dictionary in sync with across all these methods: - on_service_message, on_service_message_complete - on_response_message, on_response_message_complete - metadata_metrics """ - private_metadata["end_time"] = time.time() - private_metadata["duration"] = ( - private_metadata["end_time"] - private_metadata["start_time"] - ) private_metadata["acknowledgement"] = acknowledgement private_metadata["message_count"] = len(response_messages) private_metadata["flow_complete"] = True @@ -1288,10 +1242,6 @@ def on_response_message_complete( - on_response_message, on_response_message_complete - metadata_metrics """ - private_metadata["end_time"] = time.time() - private_metadata["duration"] = ( - private_metadata["end_time"] - private_metadata["start_time"] - ) private_metadata["acknowledgement"] = acknowledgement private_metadata["message_count"] = 0 private_metadata["flow_complete"] = True diff --git a/src/nuropb/service_handlers.py b/src/nuropb/service_handlers.py index a334211..38de8ac 100644 --- a/src/nuropb/service_handlers.py +++ b/src/nuropb/service_handlers.py @@ -12,7 +12,7 @@ """ import asyncio import logging -from typing import Any, Tuple, List, Awaitable +from typing import Any, Tuple, List, Awaitable, Dict from tornado.concurrent import is_future import pika.spec @@ -38,7 +38,7 @@ verbose = False -def error_dict_from_exception(exception: Exception | BaseException) -> dict[str, str]: +def error_dict_from_exception(exception: Exception | BaseException) -> Dict[str, str]: """Creates an error dict from an exception :param exception: :return: @@ -71,7 +71,7 @@ def create_transport_response_from_rmq_decode_exception( acknowledgement: AcknowledgeAction = "reject" transport_responses: List[TransportRespondPayload] = [] - context = {} + context: Dict[str, Any] = {} correlation_id = properties.correlation_id trace_id = properties.headers.get("trace_id", "unknown") @@ -83,6 +83,7 @@ def create_transport_response_from_rmq_decode_exception( result=None, error=error_dict_from_exception(exception=exception), warning=None, + reply_to="", ) transport_responses.append( TransportRespondPayload( @@ -122,6 +123,7 @@ def create_transport_responses_from_exceptions( result=None, error=None, warning=None, + reply_to="", ) event_template = EventPayloadDict( tag="event", @@ -191,7 +193,7 @@ def create_transport_responses_from_exceptions( event_payload.update( { "topic": event["topic"], - "event": event["encoded_payload"], + "event": event["payload"], "target": event["target"], } ) @@ -293,6 +295,7 @@ def handle_execution_result( trace_id=service_message["trace_id"], context=service_message["nuropb_payload"]["context"], warning=None, + reply_to="", ) responses.append( TransportRespondPayload( @@ -327,7 +330,6 @@ def execute_request( if service_message["nuropb_type"] not in ("request", "command", "event"): raise NuropbHandlingError( description=f"Service execution not support for message type {service_message['nuropb_type']}", - lifecycle="service-handle", payload=service_message["nuropb_payload"], exception=None, ) @@ -350,7 +352,6 @@ def execute_request( else: raise NuropbHandlingError( description=f"error calling instance._event_handler for topic: {topic}", - lifecycle="service-handle", payload=payload, exception=None, ) @@ -372,7 +373,6 @@ def execute_request( ): raise NuropbHandlingError( description="Unknown method {}".format(method_name), - lifecycle="service-handle", payload=payload, exception=None, ) @@ -398,7 +398,6 @@ def execute_request( logger.exception(err) raise NuropbException( description=f"Runtime exception calling {service_name}.{method_name}:{err}", - lifecycle="service-handle", payload=payload, exception=err, ) diff --git a/tests/encodings/test_encoding_encrypt_decrypt.py b/tests/encodings/test_encoding_encrypt_decrypt.py index 4bd3ed9..58663f8 100644 --- a/tests/encodings/test_encoding_encrypt_decrypt.py +++ b/tests/encodings/test_encoding_encrypt_decrypt.py @@ -48,7 +48,6 @@ def test_encrypted_payload_exchange(): "service": "test_service", "method": "test_async_method", "params": {"param1": "value1"}, - "reply_to": "", } service_name = "test_service" service_private_key = rsa.generate_private_key( diff --git a/tests/test_api_service_request.py b/tests/test_api_service_request.py index b70adcf..b40a132 100644 --- a/tests/test_api_service_request.py +++ b/tests/test_api_service_request.py @@ -2,7 +2,7 @@ from uuid import uuid4 import logging -from nuropb.interface import NuropbMessageError, NuropbCallAgainReject +from nuropb.interface import NuropbException, NuropbMessageError, NuropbCallAgainReject from nuropb.rmq_api import RMQAPI logging.getLogger("pika").setLevel(logging.WARNING) @@ -50,14 +50,15 @@ async def test_request_response_fail(test_settings, test_rmq_url, service_instan await client_api.connect() assert client_api.connected is True service = "test_service" - method = "test_method_fail" + method = "test_method_DOES_NOT_EXIST" params = {"param1": "value1"} context = {"context1": "value1"} ttl = 60 * 30 * 1000 trace_id = uuid4().hex - logging.info(f"Requesting {service}.{method}") - with pytest.raises(NuropbMessageError) as error: - await client_api.request( + logger.info(f"Requesting {service}.{method}") + # with pytest.raises(NuropbException) as error: + try: + result = await client_api.request( service=service, method=method, params=params, @@ -65,10 +66,13 @@ async def test_request_response_fail(test_settings, test_rmq_url, service_instan ttl=ttl, trace_id=trace_id, ) - logging.info(f"response: {error}") - assert ( - error.value.payload["error"]["description"] == "Unknown method test_method_fail" - ) + logger.info(f"result: {result}") + except Exception as error: + logger.info(f"response: {error}") + + # assert ( + # error.value.payload["error"]["description"] == "Unknown method test_method_fail" + # ) method = "test_method" rpc_response = await client_api.request( @@ -80,7 +84,7 @@ async def test_request_response_fail(test_settings, test_rmq_url, service_instan trace_id=trace_id, rpc_response=False, ) - logging.info(f"response: {rpc_response}") + logger.info(f"response: {rpc_response}") assert rpc_response["result"] == f"response from {service}.{method}" await client_api.disconnect() assert client_api.connected is False @@ -110,7 +114,7 @@ async def test_request_response_pass(test_settings, test_rmq_url, service_instan ) await service_api.connect() assert service_api.connected is True - logging.info("SERVICE API CONNECTED") + logger.info("SERVICE API CONNECTED") instance_id = uuid4().hex client_transport_settings = dict( @@ -127,14 +131,14 @@ async def test_request_response_pass(test_settings, test_rmq_url, service_instan ) await client_api.connect() assert client_api.connected is True - logging.info("CLIENT CONNECTED") + logger.info("CLIENT CONNECTED") service = "test_service" method = "test_method" params = {"param1": "value1"} context = {"context1": "value1"} ttl = 60 * 5 * 1000 trace_id = uuid4().hex - logging.info(f"Requesting {service}.{method}") + logger.info(f"Requesting {service}.{method}") rpc_response = await client_api.request( service=service, method=method, @@ -144,7 +148,7 @@ async def test_request_response_pass(test_settings, test_rmq_url, service_instan trace_id=trace_id, rpc_response=False, ) - logging.info(f"response: {rpc_response}") + logger.info(f"response: {rpc_response}") assert rpc_response["result"] == f"response from {service}.{method}" await client_api.disconnect() assert client_api.connected is False @@ -174,7 +178,7 @@ async def test_request_response_success(test_settings, test_rmq_url, service_ins ) await service_api.connect() assert service_api.connected is True - logging.info("SERVICE API CONNECTED") + logger.info("SERVICE API CONNECTED") instance_id = uuid4().hex client_transport_settings = dict( @@ -191,14 +195,14 @@ async def test_request_response_success(test_settings, test_rmq_url, service_ins ) await client_api.connect() assert client_api.connected is True - logging.info("CLIENT CONNECTED") + logger.info("CLIENT CONNECTED") service = "test_service" method = "test_success_error" params = {"param1": "value1"} context = {"context1": "value1"} ttl = 60 * 5 * 1000 trace_id = uuid4().hex - logging.info(f"Requesting {service}.{method}") + logger.info(f"Requesting {service}.{method}") rpc_response = await client_api.request( service=service, method=method, @@ -208,7 +212,7 @@ async def test_request_response_success(test_settings, test_rmq_url, service_ins trace_id=trace_id, rpc_response=False, ) - logging.info(f"response: {rpc_response}") + logger.info(f"response: {rpc_response}") assert rpc_response["result"] == f"response from {service}.{method}" await client_api.disconnect() assert client_api.connected is False @@ -240,7 +244,7 @@ async def test_request_response_call_again( ) await service_api.connect() assert service_api.connected is True - logging.info("SERVICE API CONNECTED") + logger.info("SERVICE API CONNECTED") instance_id = uuid4().hex client_transport_settings = dict( @@ -257,14 +261,14 @@ async def test_request_response_call_again( ) await client_api.connect() assert client_api.connected is True - logging.info("CLIENT CONNECTED") + logger.info("CLIENT CONNECTED") trace_id = uuid4().hex service = "test_service" method = "test_call_again_error" params = {"trace_id": trace_id} context = {"context1": "value1"} ttl = 60 * 5 * 1000 - logging.info(f"Requesting {service}.{method}") + logger.info(f"Requesting {service}.{method}") rpc_response = await client_api.request( service=service, method=method, @@ -273,7 +277,7 @@ async def test_request_response_call_again( ttl=ttl, trace_id=trace_id, ) - logging.info(f"response: {rpc_response}") + logger.info(f"response: {rpc_response}") assert rpc_response["success"] == f"response from {service}.{method}" assert rpc_response["trace_id"] == trace_id await client_api.disconnect() @@ -306,7 +310,7 @@ async def test_request_response_call_again_loop_fail( ) await service_api.connect() assert service_api.connected is True - logging.info("SERVICE API CONNECTED") + logger.info("SERVICE API CONNECTED") instance_id = uuid4().hex client_transport_settings = dict( @@ -323,14 +327,14 @@ async def test_request_response_call_again_loop_fail( ) await client_api.connect() assert client_api.connected is True - logging.info("CLIENT CONNECTED") + logger.info("CLIENT CONNECTED") trace_id = uuid4().hex service = "test_service" method = "test_call_again_loop" params = {"trace_id": trace_id} context = {"context1": "value1"} ttl = 60 * 5 * 1000 - logging.info(f"Requesting {service}.{method}") + logger.info(f"Requesting {service}.{method}") with pytest.raises(NuropbMessageError): await client_api.request( service=service, diff --git a/tests/test_client.py b/tests/test_client.py index c9c3de7..497d620 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -42,7 +42,6 @@ async def test_request_response_pass(test_settings, test_rmq_url, service_instan context=context, ttl=ttl, trace_id=trace_id, - rpc_response=False, ) await client_api.disconnect() assert client_api.connected is False diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 9e4dd21..b522109 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -26,7 +26,6 @@ def test_sync_handler_call(service_instance): "service": "test_service", "method": "test_method", "params": {"param1": "value1"}, - "reply_to": "", } transport_request = TransportServicePayload( nuropb_protocol=NUROPB_PROTOCOL_VERSION, @@ -57,7 +56,6 @@ async def test_async_handler_call_step_one(service_instance): "service": "test_service", "method": "test_async_method", "params": {"param1": "value1"}, - "reply_to": "", } transport_request = TransportServicePayload( nuropb_protocol=NUROPB_PROTOCOL_VERSION, diff --git a/tests/test_nuropb_interface.py b/tests/test_nuropb_interface.py index 83070ec..2e7d3ef 100644 --- a/tests/test_nuropb_interface.py +++ b/tests/test_nuropb_interface.py @@ -130,7 +130,6 @@ async def request( correlation_id=uuid4().hex, trace_id=trace_id, context=context, - reply_to="", ) def acknowledge_function(action: AcknowledgeAction) -> None: From 5a9a1ee37596ad07cd57cf3599ef289d0c2de8c8 Mon Sep 17 00:00:00 2001 From: Robert Betts Date: Mon, 18 Sep 2023 23:07:14 +0100 Subject: [PATCH 07/10] Code linting --- src/nuropb/contexts/context_manager_decorator.py | 2 +- src/nuropb/contexts/describe.py | 4 ++-- src/nuropb/encodings/json_serialisation.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/nuropb/contexts/context_manager_decorator.py b/src/nuropb/contexts/context_manager_decorator.py index b2292c1..bc128c3 100644 --- a/src/nuropb/contexts/context_manager_decorator.py +++ b/src/nuropb/contexts/context_manager_decorator.py @@ -79,7 +79,7 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: return method(*args[1:], **kwargs) setattr(wrapper, "__nuropb_context__", True) - setattr(wrapper, "__nuropb_context_arg__", context_parameter) + setattr(wrapper, "__nuropb_context_arg__", context_parameter) return wrapper if original_method: diff --git a/src/nuropb/contexts/describe.py b/src/nuropb/contexts/describe.py index eccbeae..c41163b 100644 --- a/src/nuropb/contexts/describe.py +++ b/src/nuropb/contexts/describe.py @@ -71,9 +71,9 @@ def decorator(method: Callable[..., Any]) -> Callable[..., Any]: def wrapper(*args: Any, **kwargs: Any) -> Any: return method(*args, **kwargs) - setattr(wrapper, "__nuropb_mesh_hidden__", hide_method) + setattr(wrapper, "__nuropb_mesh_hidden__", hide_method) setattr(wrapper, "__nuropb_context_token_key__", context_token_key) - setattr(wrapper, "__nuropb_authorise_func__", authorise_func) + setattr(wrapper, "__nuropb_authorise_func__", authorise_func) setattr(wrapper, "__nuropb_requires_encryption__", requires_encryption) if description: setattr(wrapper, "__nuropb_description__", description) diff --git a/src/nuropb/encodings/json_serialisation.py b/src/nuropb/encodings/json_serialisation.py index 5f92e73..f81537c 100644 --- a/src/nuropb/encodings/json_serialisation.py +++ b/src/nuropb/encodings/json_serialisation.py @@ -35,6 +35,7 @@ def to_json_compatible(obj: Any, recursive: bool = True, max_depth: int = 4) -> obj = obj.replace(tzinfo=datetime.timezone.utc) # assume and set to UTC elif obj.tzinfo != datetime.timezone.utc: obj = obj.astimezone(datetime.timezone.utc) # convert to UTC + json_string = f"{obj.isoformat()}Z" return json_string From 7d6799a4c519aba5206083ad734038bf2507f0a2 Mon Sep 17 00:00:00 2001 From: Robert Betts Date: Tue, 19 Sep 2023 23:49:32 +0100 Subject: [PATCH 08/10] Plugged in in-flight payload encryption --- examples/client.py | 39 +++-- examples/server.py | 30 ++++ examples/service_example.py | 23 ++- .../contexts/context_manager_decorator.py | 19 ++- src/nuropb/contexts/describe.py | 5 +- src/nuropb/encodings/encryption.py | 14 +- src/nuropb/encodings/json_serialisation.py | 12 +- src/nuropb/encodings/serializor.py | 17 ++- src/nuropb/interface.py | 8 +- src/nuropb/rmq_api.py | 96 +++++++----- src/nuropb/rmq_transport.py | 92 ++++++++--- src/nuropb/service_handlers.py | 42 +++-- src/nuropb/testing/stubs.py | 38 +++++ tests/conftest.py | 43 ++++++ .../test_context_manager_decorator.py | 23 ++- tests/contexts/test_describe.py | 33 +++- tests/mesh/test_service_discover.py | 144 ++++++++++++++++++ 17 files changed, 542 insertions(+), 136 deletions(-) create mode 100644 tests/mesh/test_service_discover.py diff --git a/examples/client.py b/examples/client.py index 1b84b74..be0472c 100644 --- a/examples/client.py +++ b/examples/client.py @@ -11,11 +11,13 @@ async def make_request(api: RMQAPI): service = "sandbox" - method = "test_method" + # method = "test_method" + method = "test_encrypt_method" params = {"param1": "value1"} context = {"context1": "value1"} ttl = 60 * 30 * 1000 trace_id = uuid4().hex + encrypted = await api.requires_encryption(service, method) response = await api.request( service=service, method=method, @@ -23,6 +25,7 @@ async def make_request(api: RMQAPI): context=context, ttl=ttl, trace_id=trace_id, + encrypted=encrypted, ) return response == f"response from {service}.{method}" @@ -34,6 +37,7 @@ async def make_command(api: RMQAPI): context = {"context1": "value1"} ttl = 60 * 30 * 1000 trace_id = uuid4().hex + encrypted = await api.requires_encryption(service, method) api.command( service=service, method=method, @@ -41,6 +45,7 @@ async def make_command(api: RMQAPI): context=context, ttl=ttl, trace_id=trace_id, + encrypted=encrypted, ) @@ -61,18 +66,24 @@ async def main(): amqp_url = "amqp://guest:guest@127.0.0.1:5672/sandbox" api = RMQAPI( amqp_url=amqp_url, + transport_settings={ + "prefetch_count": 1, + } ) await api.connect() - sanbox_describe = await api.describe_service("sandbox") - total_seconds = 0 total_sample_count = 0 - batch_size = 500 + batch_size = 10000 number_of_batches = 5 ioloop = asyncio.get_event_loop() + service = "sandbox" + method = "test_method" + encrypted = await api.requires_encryption(service, method) + logger.info(f"encryption is : {encrypted}") + for _ in range(number_of_batches): start_time = datetime.datetime.utcnow() logger.info(f"Starting: {batch_size} at {start_time}") @@ -84,15 +95,15 @@ async def main(): loop_batch_size += batch_size # logger.info(f"Request complete: {result[0]}") - tasks = [ioloop.create_task(make_command(api)) for _ in range(batch_size)] - logger.info("Waiting for command tasks to complete") - await asyncio.wait(tasks) - loop_batch_size += batch_size + # tasks = [ioloop.create_task(make_command(api)) for _ in range(batch_size)] + # logger.info("Waiting for command tasks to complete") + # await asyncio.wait(tasks) + # loop_batch_size += batch_size - tasks = [ioloop.create_task(publish_event(api)) for _ in range(batch_size)] - logger.info("Waiting for publish tasks to complete") - await asyncio.wait(tasks) - loop_batch_size += batch_size + # tasks = [ioloop.create_task(publish_event(api)) for _ in range(batch_size)] + # logger.info("Waiting for publish tasks to complete") + # await asyncio.wait(tasks) + # loop_batch_size += batch_size end_time = datetime.datetime.utcnow() time_taken = end_time - start_time @@ -106,8 +117,8 @@ async def main(): total_seconds, total_sample_count / total_seconds, ) - fut = asyncio.Future() - await fut + # fut = asyncio.Future() + # await fut logging.info("Client Done") diff --git a/examples/server.py b/examples/server.py index db47293..f514e8f 100644 --- a/examples/server.py +++ b/examples/server.py @@ -1,6 +1,10 @@ import logging import asyncio from uuid import uuid4 +import os +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.backends import default_backend from nuropb.rmq_api import RMQAPI from nuropb.service_runner import ServiceContainer @@ -15,14 +19,40 @@ async def main(): service_name = "sandbox" instance_id = uuid4().hex + """ load private_key and create one if it done not exist + """ + primary_key_filename = "key.pem" + private_key = None + if os.path.exists(primary_key_filename): + with open(primary_key_filename, "rb") as key_file: + private_key = serialization.load_pem_private_key( + data=key_file.read(), + backend=default_backend(), + password=None, + ) + if private_key is None: + private_key = rsa.generate_private_key( + public_exponent=65537, key_size=2048, backend=default_backend() + ) + primary_key_data: bytes = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + with open(primary_key_filename, "wt") as f: + f.write(primary_key_data.decode("utf-8")) + + transport_settings = dict( rpc_bindings=[service_name], event_bindings=[], + prefetch_count=1, ) service_example = ServiceExample( service_name=service_name, instance_id=instance_id, + private_key=private_key, ) api = RMQAPI( diff --git a/examples/service_example.py b/examples/service_example.py index c3f4e76..274bf5d 100644 --- a/examples/service_example.py +++ b/examples/service_example.py @@ -12,15 +12,14 @@ class ServiceExample: _service_name: str - _private_key = rsa.generate_private_key( - public_exponent=65537, key_size=2048, backend=default_backend() - ) _instance_id: str + _private_key: rsa.RSAPrivateKey _method_call_count: int - def __init__(self, service_name: str, instance_id: str): + def __init__(self, service_name: str, instance_id: str, private_key: rsa.RSAPrivateKey): self._service_name = service_name self._instance_id = instance_id + self._private_key = private_key self._method_call_count = 0 @classmethod @@ -35,17 +34,27 @@ def _handle_event_( _ = target, context, trace_id logger.debug(f"Received event {topic}:{event}") - @publish_to_mesh(requires_encryption=True) + @publish_to_mesh(requires_encryption=False) def test_method(self, **kwargs) -> str: self._method_call_count += 1 - success_result = f"response from {self._service_name}.test_method" + return success_result + + @publish_to_mesh(requires_encryption=True) + def test_encrypt_method(self, **kwargs) -> str: + self._method_call_count += 1 + success_result = f"response from {self._service_name}.test_encrypt_method" + return success_result + + def test_exception_method(self, **kwargs) -> str: + self._method_call_count += 1 + success_result = f"response from {self._service_name}.test_exception_method" if self._method_call_count % 400 == 0: events: List[EventType] = [ { "topic": "test-event", - "encoded_payload": { + "payload": { "event_key": "event_value", }, "target": [], diff --git a/src/nuropb/contexts/context_manager_decorator.py b/src/nuropb/contexts/context_manager_decorator.py index bc128c3..f411191 100644 --- a/src/nuropb/contexts/context_manager_decorator.py +++ b/src/nuropb/contexts/context_manager_decorator.py @@ -5,7 +5,7 @@ from nuropb.contexts.context_manager import NuropbContextManager -def method_has_nuropb_context(method: Callable[..., Any]) -> bool: +def method_requires_nuropb_context(method: Callable[..., Any]) -> bool: """This function checks if a method has been decorated with @nuropb_context :param method: Callable :return: bool @@ -61,22 +61,25 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: f"The @nuropb_context, expects a {context_parameter}: NuropbContextManager " f"or dict, as the first argument in calling instance.method" ) - context = args[1] - if isinstance(context, NuropbContextManager): - ctx = context - else: + method_args = list(args) + ctx = method_args[1] + if not isinstance(ctx, NuropbContextManager): + """ Replace dictionary context with NuropbContextManager instance + """ ctx = NuropbContextManager( - context=context, + context=method_args[1], suppress_exceptions=suppress_exceptions, ) + method_args[1] = ctx authorise_func = getattr(wrapper, "__nuropb_authorise_func__", None) if authorise_func is not None: context_token_key = getattr(wrapper, "__nuropb_context_token_key__") ctx.user_claims = authorise_func(ctx.context[context_token_key]) - kwargs[context_parameter] = ctx - return method(*args[1:], **kwargs) + # kwargs[context_parameter] = ctx + # return method(*args[1:], **kwargs) + return method(*method_args, **kwargs) setattr(wrapper, "__nuropb_context__", True) setattr(wrapper, "__nuropb_context_arg__", context_parameter) diff --git a/src/nuropb/contexts/describe.py b/src/nuropb/contexts/describe.py index c41163b..b9a8008 100644 --- a/src/nuropb/contexts/describe.py +++ b/src/nuropb/contexts/describe.py @@ -9,6 +9,7 @@ AuthoriseFunc = Callable[[str], Dict[str, Any]] + def method_visible_on_mesh(method: Callable[..., Any]) -> bool: """This function checks if a method has been decorated with @publish_to_mesh :param method: callable @@ -111,7 +112,7 @@ def describe_service(class_instance: object) -> Dict[str, Any] | None: requires_encryption = getattr( method, "__nuropb_requires_encryption__", False ) - if requires_encryption and not service_has_encrypted_methods: + if requires_encryption: service_has_encrypted_methods = True ctx_arg_name = getattr(method, "__nuropb_context_arg__", "ctx") @@ -191,6 +192,6 @@ def map_argument(arg_props: Any) -> Tuple[str, Dict[str, Any]]: service_info["public_key"] = private_key.public_key().public_bytes( encoding=serialization.Encoding.PEM, format=serialization.PublicFormat.SubjectPublicKeyInfo, - ) + ).decode("ascii") return service_info diff --git a/src/nuropb/encodings/encryption.py b/src/nuropb/encodings/encryption.py index d434852..a6dbc7b 100644 --- a/src/nuropb/encodings/encryption.py +++ b/src/nuropb/encodings/encryption.py @@ -9,7 +9,7 @@ def encrypt_payload(payload: str | bytes, key: str | bytes) -> bytes: - """Encrypt encoded_payload with a key + """Encrypt encoded payload with a key :param payload: str | bytes :param key: str :return: bytes @@ -22,13 +22,14 @@ def encrypt_payload(payload: str | bytes, key: str | bytes) -> bytes: def decrypt_payload(encrypted_payload: str | bytes, key: str | bytes) -> bytes: - """Decrypt encoded_payload with a key + """Decrypt encoded payload with a key :param encrypted_payload: str | bytes :param key: str :return: bytes """ f = Fernet(key=key, backend=default_backend()) - payload = f.decrypt(b64decode(encrypted_payload)) + payload_to_decrypt = b64decode(encrypted_payload) + payload = f.decrypt(payload_to_decrypt) return payload @@ -98,6 +99,13 @@ def add_service_public_key(self, service_name: str, public_key: rsa.RSAPublicKey """ self._service_public_keys[service_name] = public_key + def get_service_public_key(self, service_name: str) -> rsa.RSAPublicKey: + """Get a public key for a service + :param service_name: str + :return: rsa.RSAPublicKey + """ + return self._service_public_keys.get[service_name] + def encrypt_payload( self, payload: bytes, diff --git a/src/nuropb/encodings/json_serialisation.py b/src/nuropb/encodings/json_serialisation.py index f81537c..c8330d9 100644 --- a/src/nuropb/encodings/json_serialisation.py +++ b/src/nuropb/encodings/json_serialisation.py @@ -128,20 +128,20 @@ def __init__(self) -> None: self._encryption_keys = {} def encode(self, payload: Any) -> str: - """Encodes a nuropb encoded_payload to JSON. + """Encodes a nuropb encoded payload to JSON. - :param payload: Any, The encoded_payload to encode. - :return: str, The JSON-encoded encoded_payload. + :param payload: Any, The encoded payload to encode. + :return: str, The JSON-encoded encoded payload. """ _ = self json_payload = to_json(payload) return json_payload def decode(self, json_payload: str) -> Any: - """Decodes a JSON-encoded nuropb encoded_payload. + """Decodes a JSON-encoded nuropb encoded payload. - :param json_payload: str, The JSON-encoded encoded_payload to decode. - :return: Any, The decoded encoded_payload. + :param json_payload: str, The JSON-encoded encoded payload to decode. + :return: Any, The decoded encoded payload. """ _ = self payload = json.loads(json_payload) diff --git a/src/nuropb/encodings/serializor.py b/src/nuropb/encodings/serializor.py index 0351b9a..fcc83a9 100644 --- a/src/nuropb/encodings/serializor.py +++ b/src/nuropb/encodings/serializor.py @@ -1,3 +1,5 @@ +from cryptography.hazmat.primitives.asymmetric import rsa + from nuropb.encodings.json_serialisation import JsonSerializor from nuropb.interface import PayloadDict @@ -6,7 +8,7 @@ def get_serializor(payload_type: str = "json") -> SerializorTypes: - """Returns a serializor object for the specified encoded_payload type + """Returns a serializor object for the specified encoded payload type :param payload_type: "json" :return: a serializor object """ @@ -17,13 +19,16 @@ def get_serializor(payload_type: str = "json") -> SerializorTypes: def encode_payload( - payload: PayloadDict, - payload_type: str = "json", + payload: PayloadDict, + payload_type: str = "json", + public_key: rsa.RSAPublicKey = None ) -> bytes: """ + :param public_key: :param payload: :param payload_type: "json" - :return: a json bytes string imputed encoded_payload + :param public_key: rsa.PublicKey + :return: a json bytes string imputed encoded payload """ if payload_type != "json": raise ValueError(f"payload_type {payload_type} is not supported") @@ -32,8 +37,8 @@ def encode_payload( get_serializor( payload_type=payload_type, ) - .encode(payload) - .encode() + .encode(payload) # to json + .encode() # to bytes ) diff --git a/src/nuropb/interface.py b/src/nuropb/interface.py index 735fed0..8d61713 100644 --- a/src/nuropb/interface.py +++ b/src/nuropb/interface.py @@ -53,7 +53,7 @@ class ErrorDescriptionType(TypedDict): class EventType(TypedDict): - """For compatibility with better futureproof serialisation support, Any encoded_payload type is + """For compatibility with better futureproof serialisation support, Any encoded payload type is supported.It is encouraged to use a json compatible key/value Type e.g. Dict[str, Any] :target: is currently provided here as an aid for the implementation, there are use cases @@ -154,7 +154,7 @@ class ResponsePayloadDict(TypedDict): class TransportServicePayload(TypedDict): - """Type[TransportServicePayload]: represents valid service instruction encoded_payload. + """Type[TransportServicePayload]: represents valid service instruction encoded payload. Depending on the transport implementation, there wire encoding and serialization may be different, and some of the fields may be in the body or header of the message. """ @@ -169,7 +169,7 @@ class TransportServicePayload(TypedDict): class TransportRespondPayload(TypedDict): """Type[TransportRespondPayload]: represents valid service response message, - valid nuropb encoded_payload types are ResponsePayloadDict, and EventPayloadDict + valid nuropb encoded payload types are ResponsePayloadDict, and EventPayloadDict """ nuropb_protocol: str # nuropb defined and validated @@ -348,7 +348,7 @@ class NuropbCallAgain(NuropbException): class NuropbSuccess(NuropbException): """NuropbSuccessError: when this exception is raised, the transport layer will ACK the message - and return a success response if service encoded_payload is a 'request'. This is useful when the request + and return a success response if service encoded payload is a 'request'. This is useful when the request is a command or event and is executed asynchronously. There are some use cases where the service may want to return a success response irrespective diff --git a/src/nuropb/rmq_api.py b/src/nuropb/rmq_api.py index 0354f2a..8e7798f 100644 --- a/src/nuropb/rmq_api.py +++ b/src/nuropb/rmq_api.py @@ -4,8 +4,8 @@ from asyncio import Future from cryptography.hazmat.primitives import serialization from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import rsa +from nuropb.encodings.encryption import Encryptor from nuropb.interface import ( NuropbInterface, NuropbMessageError, @@ -38,6 +38,7 @@ class RMQAPI(NuropbInterface): _service_instance: object | None _default_ttl: int _client_only: bool + _encryptor: Encryptor _service_discovery: Dict[str, Any] _service_public_keys: Dict[str, Any] @@ -68,10 +69,15 @@ def __init__( self._client_only = True self._connection_name = f"{vhost}-client-{instance_id}" service_name = f"{vhost}-client" + self._encryptor = Encryptor() else: """Configure for service mode""" self._client_only = False self._connection_name = f"{vhost}-{service_name}-{instance_id}" + self._encryptor = Encryptor( + service_name=service_name, + private_key=getattr(service_instance, "_private_key", None) + ) self._service_name = service_name """ Is also a label for the api whether in client or service mode. @@ -118,6 +124,7 @@ def __init__( "message_callback": self.receive_transport_message, "rpc_exchange": rpc_exchange, "events_exchange": events_exchange, + "encryptor": self._encryptor, } ) self._transport = RMQTransport(**transport_settings) @@ -244,6 +251,7 @@ async def request( ttl: Optional[int] = None, trace_id: Optional[str] = None, rpc_response: bool = True, + encrypted: bool = False, ) -> Union[ResponsePayloadDict, Any]: """Makes a rpc request for a method on a service mesh service and waits until the response is received. @@ -269,6 +277,8 @@ async def request( if True (default), the actual response of the RPC call is returned and where there was an error, that is raised as an exception. Where rpc_response is a ResponsePayloadDict, is returned. + :param encrypted: bool + if True then the message will be encrypted in transit :return ResponsePayloadDict | Any: representing the response from the requested service with any exceptions raised """ @@ -300,7 +310,7 @@ async def request( expiry=ttl, priority=None, encoding="json", - publickey=None, + encrypted=encrypted, ) except Exception as err: if rpc_response is False: @@ -357,6 +367,7 @@ def command( context: Dict[str, Any], ttl: Optional[int] = None, trace_id: Optional[str] = None, + encrypted: bool = False, ) -> None: """command: sends a command to the target service. I.e. a targeted event. response is not expected and ignored. @@ -371,7 +382,7 @@ def command( or assumed by the requester to have failed with an undetermined state. :param trace_id: an identifier to trace the request over the network (e.g. uuid4 hex string) - + :param encrypted: bool, if True then the message will be encrypted in transit :return: None """ correlation_id = uuid4().hex @@ -395,7 +406,11 @@ def command( f"method: {method}\n" ) self._transport.send_message( - payload=message, expiry=ttl, priority=None, encoding="json", publickey=None + payload=message, + expiry=ttl, + priority=None, + encoding="json", + encrypted=encrypted, ) def publish_event( @@ -404,6 +419,7 @@ def publish_event( event: Dict[str, Any], context: Dict[str, Any], trace_id: Optional[str] = None, + encrypted: bool = False, ) -> None: """Broadcasts an event with the given topic. @@ -418,6 +434,7 @@ def publish_event( - method: str :param trace_id: str optional an identifier to trace the request over the network (e.g. an uuid4 hex string) + :param encrypted: bool, if True then the message will be encrypted in transit """ correlation_id = uuid4().hex message: EventPayloadDict = { @@ -436,17 +453,23 @@ def publish_event( ) self._transport.send_message( payload=message, - encoding="json", priority=None, - publickey=None, + encoding="json", + encrypted=encrypted, ) - async def describe_service(self, service_name: str) -> Dict[str, Any] | None: - """describe_service: returns the service description for the given service_name + async def describe_service(self, service_name: str, refresh: bool = False) -> Dict[str, Any] | None: + """describe_service: returns the service information for the given service_name, + if it is not already cached or refresh is try then the service discovery is queried directly. + :param service_name: str + :param refresh: bool :return: dict """ - result = await self.request( + if service_name in self._service_discovery or refresh: + return self._service_discovery[service_name] + + service_info = await self.request( service=service_name, method="nuropb_describe", params={}, @@ -454,42 +477,41 @@ async def describe_service(self, service_name: str) -> Dict[str, Any] | None: ttl=60 * 1000, # 1 minute trace_id=uuid4().hex, ) - return result - - async def check_method_for_encryption(self, service_name: str, method_name: str) -> rsa.RSAPublicKey | None: - """check_method_for_encryption: Queries the service discovery cache, if an entry for the service_name - does not exist, then the service discovery is queried directly. - - if encryption is required for the method called, then reference the service discovery cache - for the service private key required to encrypt the request. + if not isinstance(service_info, dict): + raise ValueError(f"Invalid service_info returned for service {service_name}") + else: + self._service_discovery[service_name] = service_info + try: + text_public_key = service_info.get( + "public_key", None + ) + if text_public_key: + public_key = serialization.load_pem_public_key( + data=text_public_key.encode("ascii"), + backend=default_backend(), + ) + self._encryptor.add_service_public_key( + service_name=service_name, public_key=public_key + ) + except Exception as err: + logger.error(f"error loading the public key for {service_name}: {err}") + finally: + return service_info + + async def requires_encryption(self, service_name: str, method_name: str) -> bool: + """requires_encryption: Queries the service discovery information for the service_name + and returns True if encryption is required else False. + none of encryption is not required. :param service_name: str :param method_name: str :return: bool """ - if service_name not in self._service_discovery: - self._service_discovery[service_name] = await self.describe_service( - service_name - ) - text_public_key = self._service_discovery[service_name].get( - "public_key", None - ) - if text_public_key: - self._service_public_keys[ - service_name - ] = serialization.load_pem_public_key( - data=text_public_key, - backend=default_backend(), - ) - - service_info = self._service_discovery[service_name] + service_info = await self.describe_service(service_name) method_info = service_info["methods"].get(method_name, None) if method_info is None: raise ValueError( f"Method {method_name} not found on service {service_name}" ) - if method_info.get("requires_encryption", False): - return self._service_public_keys.get(service_name, None) - - return None + return method_info.get("requires_encryption", False) diff --git a/src/nuropb/rmq_transport.py b/src/nuropb/rmq_transport.py index d0b7930..2a3eade 100644 --- a/src/nuropb/rmq_transport.py +++ b/src/nuropb/rmq_transport.py @@ -13,6 +13,7 @@ import pika.spec from pika.frame import Method +from nuropb.encodings.encryption import Encryptor from nuropb.encodings.serializor import encode_payload, decode_payload from nuropb.interface import ( NuropbTransportError, @@ -32,9 +33,10 @@ create_virtual_host, configure_nuropb_rmq, ) +from nuropb import service_handlers from nuropb.service_handlers import ( create_transport_response_from_rmq_decode_exception, - error_dict_from_exception, + error_dict_from_exception ) from nuropb.utils import obfuscate_credentials @@ -63,7 +65,15 @@ class RabbitMQConfiguration(TypedDict): """ The wait when shutting down consumers before closing the connection """ -verbose = False +_verbose = False +@property +def verbose() -> bool: + return _verbose +@verbose.setter +def verbose(value: bool) -> None: + global _verbose + _verbose = value + service_handlers.verbose = value """ Set to True to enable module verbose logging """ @@ -131,6 +141,7 @@ class RMQTransport: _default_ttl: int _client_only: bool _message_callback: MessageCallbackFunction + _encryptor: Encryptor | None _connected_future: Any _disconnected_future: Any @@ -164,6 +175,7 @@ def __init__( prefetch_count: Optional[int] = None, default_ttl: Optional[int] = None, client_only: Optional[bool] = None, + encryptor: Optional[Encryptor] = None, ): # NOSONAR """Create a new instance of the consumer class, passing in the AMQP URL used to connect to RabbitMQ. @@ -183,6 +195,8 @@ def __init__( :param int prefetch_count: The number of messages to prefetch defaults to 1, unlimited is 0. Experiment with larger values for higher throughput in your user case. :param int default_ttl: The default time to live for messages in milliseconds, defaults to 12 hours. + :param bool client_only: + :param Encryptor encryptor: The encryptor to use for encrypting and decrypting messages """ self._connected = False self._closing = False @@ -203,6 +217,8 @@ def __init__( rpc_bindings = [] event_bindings = [] + self._encryptor = encryptor + # Experiment with larger values for higher throughput. self._service_name = service_name self._instance_id = instance_id @@ -726,6 +742,8 @@ def on_message_returned( trace_id = properties.headers.get("trace_id", "unknown") nuropb_type = properties.headers.get("nuropb_type", "unknown") nuropb_version = properties.headers.get("nuropb_version", "unknown") + encrypted = properties.headers.get("encrypted", "") == "yes" + logger.warning( f"Could not route {nuropb_type} message to service {method.routing_key} " f"correlation_id: {correlation_id} " @@ -735,6 +753,11 @@ def on_message_returned( """ End the awaiting request future with a NuropbNotDeliveredError """ if nuropb_type == "request": + if encrypted and self._encryptor: + body = self._encryptor.decrypt_payload( + payload=body, + correlation_id=correlation_id, + ) nuropb_payload = decode_payload(body, "json") request_method = nuropb_payload["method"] nuropb_payload["tag"] = nuropb_type @@ -787,12 +810,9 @@ def on_message_returned( ) def send_message( - self, - payload: Dict[str, Any], - expiry: Optional[int] = None, - priority: Optional[int] = None, - encoding: str = "json", - publickey: Optional[rsa.RSAPrivateKey] = None, + self, payload: Dict[str, Any], expiry: Optional[int] = None, + priority: Optional[int] = None, encoding: str = "json", + encrypted: bool = False, ) -> None: """Send a message to over the RabbitMQ Transport @@ -806,10 +826,8 @@ def send_message( :param expiry: The message expiry in milliseconds :param priority: The message priority :param encoding: The encoding of the message - :param publickey: The public key to use for encryption, belongs to the target service + :param encrypted: True if the message is to be encrypted """ - _ = publickey - mandatory = False exchange = "" reply_to = "" @@ -828,10 +846,29 @@ def send_message( raise ValueError(f"Unknown payload type {payload['tag']}") if encoding == "json": - body = encode_payload(payload, "json") + body = encode_payload( + payload=payload, + payload_type="json", + ) else: raise ValueError(f"unsupported encoding {encoding}") + """ now encrypt the payload if public_key is not None, update RMQ header + to indicate encrypted payload. + """ + if encrypted and self._encryptor and payload["tag"] in ("request", "command", "response"): + encrypted = "yes" + to_service = None if payload["tag"] == "response" else payload["service"] + """ only outgoing response and command messages require the target service name + """ + wire_body = self._encryptor.encrypt_payload( + payload=body, + correlation_id=payload["correlation_id"], + service_name=to_service, + ) + else: + wire_body = body + properties = dict( content_type="application/json", correlation_id=payload["correlation_id"], @@ -841,6 +878,7 @@ def send_message( "nuropb_version": NUROPB_VERSION, "nuropb_type": payload["tag"], "trace_id": payload["trace_id"], + "encrypted": "yes" if encrypted else "", }, ) if expiry: @@ -857,7 +895,7 @@ def send_message( self._channel.basic_publish( exchange=exchange, routing_key=routing_key, - body=body, + body=wire_body, properties=basic_properties, mandatory=mandatory, ) @@ -965,6 +1003,7 @@ def on_service_message_complete( reply_to = properties.reply_to trace_id = properties.headers.get("trace_id", "unknown") correlation_id = properties.correlation_id + encrypted = properties.headers.get("encrypted", "") == "yes" if redelivered is True and acknowledgement == "nack": if verbose: @@ -1002,8 +1041,7 @@ def on_service_message_complete( payload=respond_payload, priority=None, encoding="json", - publickey=None, - ) + encrypted=encrypted) """ NOTE - METADATA: keep this dictionary in sync with across all these methods: - on_service_message, on_service_message_complete @@ -1087,6 +1125,7 @@ def on_service_message( """ nuropb_type = properties.headers.get("nuropb_type", "") nuropb_version = properties.headers.get("nuropb_version", "") + encrypted = properties.headers.get("encrypted", "") == "yes" if not verbose: logger.debug(f"service message received: '{nuropb_type}'") else: @@ -1101,12 +1140,13 @@ def on_service_message( content_type: {properties.content_type} nuropb_type: {nuropb_type} nuropb_version: {nuropb_version} +encrypted: {encrypted} """ ) """ Handle for the unknown message type and reply to the originator if possible """ nuropb_version = properties.headers.get("nuropb_version", "") - + correlation_id = properties.correlation_id """ NOTE - METADATA: keep this dictionary in sync with across all these methods: - on_service_message, on_service_message_complete - on_response_message, on_response_message_complete @@ -1120,8 +1160,9 @@ def on_service_message( "client_only": self._client_only, "nuropb_type": nuropb_type, "nuropb_version": nuropb_version, - "correlation_id": properties.correlation_id, + "correlation_id": correlation_id, "trace_id": properties.headers.get("trace_id", "unknown"), + "encrypted": encrypted, } message_complete_callback = functools.partial( @@ -1135,6 +1176,11 @@ def on_service_message( """ Decode service message """ try: + if encrypted and self._encryptor: + body = self._encryptor.decrypt_payload( + payload=body, + correlation_id=correlation_id, + ) service_message = decode_rmq_body(basic_deliver, properties, body) except Exception as error: """Exceptions caught here are treated as permanent failures, ack the message and send @@ -1267,8 +1313,11 @@ def on_response_message( :param pika.spec.BasicProperties properties: properties :param bytes body: The message body """ + correlation_id = properties.correlation_id nuropb_type = properties.headers.get("nuropb_type", "") nuropb_version = properties.headers.get("nuropb_version", "") + encrypted = properties.headers.get("encrypted", "") == "yes" + if not verbose: logger.debug(f"response message received: '{nuropb_type}'") else: @@ -1280,11 +1329,12 @@ def on_response_message( f"instance_id: {self._instance_id}\n" f"exchange: {basic_deliver.exchange}\n" f"routing_key: {basic_deliver.routing_key}\n" - f"correlation_id: {properties.correlation_id}\n" + f"correlation_id: {correlation_id}\n" f"trace_id: {properties.headers.get('trace_id', '')}\n" f"content_type: {properties.content_type}\n" f"nuropb_type: {nuropb_type}\n" f"nuropb_version: {nuropb_version}\n" + f"encrypted: {encrypted}\n" ) ) try: @@ -1303,6 +1353,7 @@ def on_response_message( "nuropb_version": nuropb_version, "correlation_id": properties.correlation_id, "trace_id": properties.headers.get("trace_id", "unknown"), + "encrypted": encrypted, } message_complete_callback = functools.partial( self.on_service_message_complete, @@ -1311,6 +1362,11 @@ def on_response_message( properties.reply_to, metadata, ) + if encrypted and self._encryptor: + body = self._encryptor.decrypt_payload( + payload=body, + correlation_id=correlation_id, + ) message = decode_rmq_body(basic_deliver, properties, body) self._message_callback(message, message_complete_callback, metadata) except Exception as err: diff --git a/src/nuropb/service_handlers.py b/src/nuropb/service_handlers.py index 38de8ac..9e92ddb 100644 --- a/src/nuropb/service_handlers.py +++ b/src/nuropb/service_handlers.py @@ -17,7 +17,7 @@ from tornado.concurrent import is_future import pika.spec -from nuropb.contexts.context_manager_decorator import method_has_nuropb_context +from nuropb.contexts.context_manager_decorator import method_requires_nuropb_context from nuropb.contexts.describe import describe_service from nuropb.interface import ( ResponsePayloadDict, @@ -35,7 +35,17 @@ ) logger = logging.getLogger(__name__) -verbose = False + +_verbose = False +@property +def verbose() -> bool: + return _verbose +@verbose.setter +def verbose(value: bool) -> None: + global _verbose + _verbose = value +""" Set to True to enable module verbose logging +""" def error_dict_from_exception(exception: Exception | BaseException) -> Dict[str, str]: @@ -362,27 +372,26 @@ def execute_request( params = payload["params"] """ TODO: think about how to pass the context to the service executing the method - # context = encoded_payload["context"] + # context = payload["context"] """ - if method_name != "nuropb_describe": - if ( - method_name.startswith("_") - or not hasattr(service_instance, method_name) - or not callable(getattr(service_instance, method_name)) - ): - raise NuropbHandlingError( - description="Unknown method {}".format(method_name), - payload=payload, - exception=None, - ) + if method_name != "nuropb_describe" and ( + method_name.startswith("_") + or not hasattr(service_instance, method_name) + or not callable(getattr(service_instance, method_name)) + ): + raise NuropbHandlingError( + description="Unknown method {}".format(method_name), + payload=payload, + exception=None, + ) try: if method_name == "nuropb_describe": result = describe_service(service_instance) else: service_instance_method = getattr(service_instance, method_name) - if method_has_nuropb_context(service_instance_method): + if method_requires_nuropb_context(service_instance_method): result = service_instance_method( service_message["nuropb_payload"]["context"], **params ) @@ -396,8 +405,9 @@ def execute_request( except Exception as err: if verbose: logger.exception(err) + error = f"{type(err).__name__}: {err}" raise NuropbException( - description=f"Runtime exception calling {service_name}.{method_name}:{err}", + description=f"Runtime exception calling {service_name}.{method_name}: {error}", payload=payload, exception=err, ) diff --git a/src/nuropb/testing/stubs.py b/src/nuropb/testing/stubs.py index 621b1fb..99deffb 100644 --- a/src/nuropb/testing/stubs.py +++ b/src/nuropb/testing/stubs.py @@ -1,19 +1,39 @@ import logging from typing import Any, Dict +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from nuropb.contexts.context_manager import NuropbContextManager +from nuropb.contexts.context_manager_decorator import nuropb_context +from nuropb.contexts.describe import publish_to_mesh from nuropb.interface import NuropbCallAgain, NuropbSuccess logger = logging.getLogger(__name__) +def get_claims_from_token(bearer_token: str) -> Dict[str, Any] | None: + """This is a stub for the authorise_func that is used in the tests""" + _ = bearer_token + return { + "sub": "test_user", + "user_id": "test_user", + "scope": "openid, profile", + "roles": "user, admin", + } + + class ServiceExample: _service_name: str _instance_id: str + _private_key: rsa.RSAPrivateKey _method_call_count: int def __init__(self, service_name: str, instance_id: str): self._service_name = service_name self._instance_id = instance_id + self._private_key = private_key = rsa.generate_private_key( + public_exponent=65537, key_size=2048, backend=default_backend() + ) self._method_call_count = 0 self.raise_call_again_error = True @@ -35,6 +55,24 @@ def test_success_error(self, **kwargs: Any) -> None: result=success_result, ) + @nuropb_context + @publish_to_mesh(authorise_func=get_claims_from_token) + def test_requires_user_claims(self, ctx, **kwargs: Any) -> Any: + assert isinstance(self, ServiceExample) + assert isinstance(ctx, NuropbContextManager) + self._method_call_count += 1 + logger.debug(f"test_requires_user_claims: {kwargs}") + return ctx.user_claims + + @nuropb_context + @publish_to_mesh(authorise_func=get_claims_from_token) + def test_requires_encryption(self, ctx, **kwargs: Any) -> Any: + assert isinstance(self, ServiceExample) + assert isinstance(ctx, NuropbContextManager) + self._method_call_count += 1 + logger.debug(f"test_requires_encryption: {kwargs}") + return "Result that's to be encrypted in transit" + def test_call_again_error(self, **kwargs: Any) -> Dict[str, Any]: self._method_call_count += 1 logger.debug(f"test_call_again_error: {kwargs}") diff --git a/tests/conftest.py b/tests/conftest.py index 111d7bb..cb9462c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,7 @@ import pytest +from nuropb.rmq_api import RMQAPI from nuropb.rmq_lib import ( build_amqp_url, build_rmq_api_url, @@ -184,3 +185,45 @@ def service_instance(): service_name="test_service", instance_id=uuid4().hex, ) + + +@pytest.fixture(scope="function") +def test_mesh_service(test_settings, test_rmq_url, service_instance): + service_name = test_settings["service_name"] + instance_id = uuid4().hex + transport_settings = dict( + dl_exchange=test_settings["dl_exchange"], + rpc_bindings=test_settings["rpc_bindings"], + event_bindings=test_settings["event_bindings"], + prefetch_count=test_settings["prefetch_count"], + default_ttl=test_settings["default_ttl"], + ) + service_api = RMQAPI( + service_name=service_name, + instance_id=instance_id, + service_instance=service_instance, + amqp_url=test_rmq_url, + rpc_exchange="test_rpc_exchange", + events_exchange="test_events_exchange", + transport_settings=transport_settings, + ) + yield service_api + + +@pytest.fixture(scope="function") +def test_mesh_client(test_rmq_url, test_settings, test_mesh_service): + instance_id = uuid4().hex + settings = test_mesh_service.transport.rmq_configuration + client_transport_settings = dict( + dl_exchange=settings["dl_exchange"], + prefetch_count=test_settings["prefetch_count"], + default_ttl=test_settings["default_ttl"], + ) + client_api = RMQAPI( + instance_id=instance_id, + amqp_url=test_rmq_url, + rpc_exchange=settings["rpc_exchange"], + events_exchange=settings["events_exchange"], + transport_settings=client_transport_settings, + ) + yield client_api diff --git a/tests/contexts/test_context_manager_decorator.py b/tests/contexts/test_context_manager_decorator.py index 2160602..9b185a1 100644 --- a/tests/contexts/test_context_manager_decorator.py +++ b/tests/contexts/test_context_manager_decorator.py @@ -12,18 +12,21 @@ class TestServiceClass: _service_name: str def hello_no_context(self, param1: str) -> str: # pragma: no cover - _ = self + assert isinstance(self, TestServiceClass) return f"hello {param1}" - def hello_with_context(self, ctx: Dict[str, Any], param1: str) -> str: - _ = self + def hello_with_context_param(self, ctx: Dict[str, Any], param1: str) -> str: + assert isinstance(self, TestServiceClass) + if not isinstance(ctx, dict): + raise TypeError("ctx parameter must be a dictionary") return f"from {ctx['user_id']}, hello {param1}" @nuropb_context def hello_with_context_decorator( self, ctx: NuropbContextManager, param1: str ) -> str: - _ = self + assert isinstance(self, TestServiceClass) + assert isinstance(ctx, NuropbContextManager) return f"from {ctx.context['user_id']}, hello {param1}" @@ -89,18 +92,24 @@ def test_nuropb_context(context, instance): """ Test with NuropbContextManager context injection and method has no decorator """ with pytest.raises(TypeError): - result = instance.hello_with_context(ctx, **params) + result = instance.hello_with_context_param(ctx, **params) """ Test with dictionary context injection and method has no decorator and params contains a ctx parameter as would be expected. """ params = { "param1": "world", - "ctx": {"key": "value"}, + "ctx": {"user_id": "test_user_id"}, } + result = instance.hello_with_context_param(context, param1="world") + assert result == "from test_user_id, hello world" + + result = instance.hello_with_context_param(**params) + assert result == "from test_user_id, hello world" + ctx = context with pytest.raises(TypeError): - result = instance.hello_with_context(ctx, **params) + result = instance.hello_with_context_param(ctx, **params) def test_nuropb_context_decorator(context, instance): diff --git a/tests/contexts/test_describe.py b/tests/contexts/test_describe.py index 4faca0f..5ef0be2 100644 --- a/tests/contexts/test_describe.py +++ b/tests/contexts/test_describe.py @@ -108,31 +108,44 @@ class OrderManagementService: public_exponent=65537, key_size=2048, backend=default_backend() ) - @publish_to_mesh(requires_encryption=True) @nuropb_context + @publish_to_mesh(requires_encryption=True) async def get_orders( self, - ctx, + ctx: NuropbContextManager, order_date: datetime.datetime, account: Optional[str] = None, status: Optional[str] = "", security: Optional[str] = None, side: Optional[str] = None ) -> List[Order]: + _ = order_date, account, status, security, side + assert isinstance(self, OrderManagementService) + assert isinstance(ctx, NuropbContextManager) return [] - @publish_to_mesh @nuropb_context - async def create_order( - self, - ctx, - order: Order) -> Order: + @publish_to_mesh + async def create_order(self, ctx: NuropbContextManager) -> Order: + assert isinstance(self, OrderManagementService) + assert isinstance(ctx, NuropbContextManager) new_order = Order(account="ABC1234", security="SSE.L", quantity=1000, side="sell") return new_order + @nuropb_context + @publish_to_mesh(hide_method=True) + async def internal_method(self, ctx: NuropbContextManager) -> str: + assert isinstance(self, OrderManagementService) + assert isinstance(ctx, NuropbContextManager) + return "OK" + + async def undecorated_method(self) -> str: + assert isinstance(self, OrderManagementService) + return "OK" + class Service: """ Some useful documentation to describe the characteristic of the service and its purpose @@ -170,4 +183,8 @@ def _private_method(self, param1, param2): def test_instance_describe(): service_instance = OrderManagementService() result = describe_service(service_instance) - assert 1 == 1 + assert result["description"] == service_instance.__doc__.strip() + assert len(result["methods"]) == 3 + assert len(result["methods"]["create_order"]) == 3 + assert result["methods"]["get_orders"]["requires_encryption"] is True + diff --git a/tests/mesh/test_service_discover.py b/tests/mesh/test_service_discover.py new file mode 100644 index 0000000..279ff96 --- /dev/null +++ b/tests/mesh/test_service_discover.py @@ -0,0 +1,144 @@ +import logging +from pprint import pformat + +import pytest + +from nuropb import rmq_transport + +logger = logging.getLogger(__name__) + + +@pytest.mark.asyncio +async def test_requires_user_token(test_mesh_client, test_mesh_service): + + await test_mesh_service.connect() + assert test_mesh_service.connected is True + logger.info("SERVICE API CONNECTED") + + await test_mesh_client.connect() + assert test_mesh_client.connected is True + logger.info("CLIENT CONNECTED") + + service = test_mesh_service.service_name + method = "test_requires_user_claims" + params = {"param1": "value1"} + context = {"Authorization": "my_jwt_token"} + logger.info(f"Requesting {service}.{method}") + rpc_response = await test_mesh_service.request( + service=service, + method=method, + params=params, + context=context, + rpc_response=False, + ) + logger.info(f"response: {pformat(rpc_response)}") + + await test_mesh_client.disconnect() + await test_mesh_service.disconnect() + + assert rpc_response["error"] is None + assert rpc_response["result"]["user_id"] == "test_user" + assert rpc_response["result"]["scope"] == "openid, profile" + assert rpc_response["result"]["roles"] == "user, admin" + assert rpc_response["result"]["sub"] == "test_user" + + +@pytest.mark.asyncio +async def test_mesh_service_describe(test_mesh_client, test_mesh_service): + """ Call the describe function for a service on the mesh. This should return a dictionary + describing the service and its methods. + """ + + await test_mesh_service.connect() + assert test_mesh_service.connected is True + logger.info("SERVICE API CONNECTED") + + await test_mesh_client.connect() + assert test_mesh_client.connected is True + logger.info("CLIENT CONNECTED") + + service = test_mesh_service.service_name + method = "nuropb_describe" + params = {} + context = {} + logger.info(f"Requesting {service}.{method}") + rpc_response = await test_mesh_service.request( + service=service, + method=method, + params=params, + context=context, + rpc_response=False, + ) + logger.info(f"response: {pformat(rpc_response)}") + + +@pytest.mark.asyncio +async def test_mesh_service_describe(test_mesh_client, test_mesh_service): + """ user the service mesh api helper function to call the describe function for a service on the mesh. + Test that service metta information is cached in the mesh client. + """ + + await test_mesh_service.connect() + assert test_mesh_service.connected is True + logger.info("SERVICE API CONNECTED") + + await test_mesh_client.connect() + assert test_mesh_client.connected is True + logger.info("CLIENT CONNECTED") + + service_name = test_mesh_service.service_name + logger.info(f"test_mesh_service.describe_service('{service_name}')") + service_info = await test_mesh_client.describe_service( + service_name=service_name, + ) + logger.info(f"response: {pformat(service_info)}") + assert isinstance(service_info, dict) + + service = test_mesh_service.service_name + method = "test_requires_encryption" + public_key = await test_mesh_client.requires_encryption( + service_name=service, + method_name=method + ) + params = {} + context = {} + logger.info(f"Requesting {service}.{method}") + rpc_response = await test_mesh_service.request( + service=service, + method=method, + params=params, + context=context, + rpc_response=False, + ) + logger.info(f"response: {pformat(rpc_response)}") + + +@pytest.mark.asyncio +async def test_mesh_service_encrypt(test_mesh_client, test_mesh_service): + """ user the service mesh api helper function to call the describe function for a service on the mesh. + Test that service metta information is cached in the mesh client. + """ + + await test_mesh_service.connect() + await test_mesh_client.connect() + rmq_transport.verbose = True + + service = "test_service" + method = "test_requires_encryption" + logger.info(f"Requesting encrypted transport for request {service}.{method}") + + encrypted = await test_mesh_client.requires_encryption(service, method) + params = {} + context = { + "Authorization": "Bearer: user_token" + } + rpc_response = await test_mesh_service.request( + service=service, + method=method, + params=params, + context=context, + rpc_response=False, + encrypted=encrypted, + ) + logger.info(f"response: {pformat(rpc_response)}") + From 85dfa42475c5eb1e1b2590df3ec28581e9df93b0 Mon Sep 17 00:00:00 2001 From: Robert Betts Date: Wed, 20 Sep 2023 00:16:56 +0100 Subject: [PATCH 09/10] Code Linting Bumping version to 0.1.3 protocol version to 0.1.1 --- poetry.lock | 362 +++++++++--------- .../contexts/context_manager_decorator.py | 3 +- src/nuropb/contexts/describe.py | 12 +- src/nuropb/encodings/encryption.py | 4 +- src/nuropb/encodings/serializor.py | 6 +- src/nuropb/interface.py | 10 +- src/nuropb/rmq_api.py | 19 +- src/nuropb/rmq_transport.py | 26 +- src/nuropb/service_handlers.py | 14 +- 9 files changed, 240 insertions(+), 216 deletions(-) diff --git a/poetry.lock b/poetry.lock index e5dda37..396308b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -153,33 +153,33 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte [[package]] name = "black" -version = "23.7.0" +version = "23.9.1" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" files = [ - {file = "black-23.7.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:5c4bc552ab52f6c1c506ccae05681fab58c3f72d59ae6e6639e8885e94fe2587"}, - {file = "black-23.7.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:552513d5cd5694590d7ef6f46e1767a4df9af168d449ff767b13b084c020e63f"}, - {file = "black-23.7.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:86cee259349b4448adb4ef9b204bb4467aae74a386bce85d56ba4f5dc0da27be"}, - {file = "black-23.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:501387a9edcb75d7ae8a4412bb8749900386eaef258f1aefab18adddea1936bc"}, - {file = "black-23.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:fb074d8b213749fa1d077d630db0d5f8cc3b2ae63587ad4116e8a436e9bbe995"}, - {file = "black-23.7.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:b5b0ee6d96b345a8b420100b7d71ebfdd19fab5e8301aff48ec270042cd40ac2"}, - {file = "black-23.7.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:893695a76b140881531062d48476ebe4a48f5d1e9388177e175d76234ca247cd"}, - {file = "black-23.7.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:c333286dc3ddca6fdff74670b911cccedacb4ef0a60b34e491b8a67c833b343a"}, - {file = "black-23.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:831d8f54c3a8c8cf55f64d0422ee875eecac26f5f649fb6c1df65316b67c8926"}, - {file = "black-23.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:7f3bf2dec7d541b4619b8ce526bda74a6b0bffc480a163fed32eb8b3c9aed8ad"}, - {file = "black-23.7.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:f9062af71c59c004cd519e2fb8f5d25d39e46d3af011b41ab43b9c74e27e236f"}, - {file = "black-23.7.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:01ede61aac8c154b55f35301fac3e730baf0c9cf8120f65a9cd61a81cfb4a0c3"}, - {file = "black-23.7.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:327a8c2550ddc573b51e2c352adb88143464bb9d92c10416feb86b0f5aee5ff6"}, - {file = "black-23.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d1c6022b86f83b632d06f2b02774134def5d4d4f1dac8bef16d90cda18ba28a"}, - {file = "black-23.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:27eb7a0c71604d5de083757fbdb245b1a4fae60e9596514c6ec497eb63f95320"}, - {file = "black-23.7.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:8417dbd2f57b5701492cd46edcecc4f9208dc75529bcf76c514864e48da867d9"}, - {file = "black-23.7.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:47e56d83aad53ca140da0af87678fb38e44fd6bc0af71eebab2d1f59b1acf1d3"}, - {file = "black-23.7.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:25cc308838fe71f7065df53aedd20327969d05671bac95b38fdf37ebe70ac087"}, - {file = "black-23.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:642496b675095d423f9b8448243336f8ec71c9d4d57ec17bf795b67f08132a91"}, - {file = "black-23.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:ad0014efc7acf0bd745792bd0d8857413652979200ab924fbf239062adc12491"}, - {file = "black-23.7.0-py3-none-any.whl", hash = "sha256:9fd59d418c60c0348505f2ddf9609c1e1de8e7493eab96198fc89d9f865e7a96"}, - {file = "black-23.7.0.tar.gz", hash = "sha256:022a582720b0d9480ed82576c920a8c1dde97cc38ff11d8d8859b3bd6ca9eedb"}, + {file = "black-23.9.1-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:d6bc09188020c9ac2555a498949401ab35bb6bf76d4e0f8ee251694664df6301"}, + {file = "black-23.9.1-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:13ef033794029b85dfea8032c9d3b92b42b526f1ff4bf13b2182ce4e917f5100"}, + {file = "black-23.9.1-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:75a2dc41b183d4872d3a500d2b9c9016e67ed95738a3624f4751a0cb4818fe71"}, + {file = "black-23.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13a2e4a93bb8ca74a749b6974925c27219bb3df4d42fc45e948a5d9feb5122b7"}, + {file = "black-23.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:adc3e4442eef57f99b5590b245a328aad19c99552e0bdc7f0b04db6656debd80"}, + {file = "black-23.9.1-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:8431445bf62d2a914b541da7ab3e2b4f3bc052d2ccbf157ebad18ea126efb91f"}, + {file = "black-23.9.1-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:8fc1ddcf83f996247505db6b715294eba56ea9372e107fd54963c7553f2b6dfe"}, + {file = "black-23.9.1-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:7d30ec46de88091e4316b17ae58bbbfc12b2de05e069030f6b747dfc649ad186"}, + {file = "black-23.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:031e8c69f3d3b09e1aa471a926a1eeb0b9071f80b17689a655f7885ac9325a6f"}, + {file = "black-23.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:538efb451cd50f43aba394e9ec7ad55a37598faae3348d723b59ea8e91616300"}, + {file = "black-23.9.1-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:638619a559280de0c2aa4d76f504891c9860bb8fa214267358f0a20f27c12948"}, + {file = "black-23.9.1-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:a732b82747235e0542c03bf352c126052c0fbc458d8a239a94701175b17d4855"}, + {file = "black-23.9.1-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:cf3a4d00e4cdb6734b64bf23cd4341421e8953615cba6b3670453737a72ec204"}, + {file = "black-23.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf99f3de8b3273a8317681d8194ea222f10e0133a24a7548c73ce44ea1679377"}, + {file = "black-23.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:14f04c990259576acd093871e7e9b14918eb28f1866f91968ff5524293f9c573"}, + {file = "black-23.9.1-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:c619f063c2d68f19b2d7270f4cf3192cb81c9ec5bc5ba02df91471d0b88c4c5c"}, + {file = "black-23.9.1-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:6a3b50e4b93f43b34a9d3ef00d9b6728b4a722c997c99ab09102fd5efdb88325"}, + {file = "black-23.9.1-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:c46767e8df1b7beefb0899c4a95fb43058fa8500b6db144f4ff3ca38eb2f6393"}, + {file = "black-23.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50254ebfa56aa46a9fdd5d651f9637485068a1adf42270148cd101cdf56e0ad9"}, + {file = "black-23.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:403397c033adbc45c2bd41747da1f7fc7eaa44efbee256b53842470d4ac5a70f"}, + {file = "black-23.9.1-py3-none-any.whl", hash = "sha256:6ccd59584cc834b6d127628713e4b6b968e5f79572da66284532525a042549f9"}, + {file = "black-23.9.1.tar.gz", hash = "sha256:24b6b3ff5c6d9ea08a8888f6977eae858e1f340d7260cf56d70a49823236b62d"}, ] [package.dependencies] @@ -189,7 +189,7 @@ packaging = ">=22.0" pathspec = ">=0.9.0" platformdirs = ">=2" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] @@ -454,63 +454,63 @@ files = [ [[package]] name = "coverage" -version = "7.3.0" +version = "7.3.1" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db76a1bcb51f02b2007adacbed4c88b6dee75342c37b05d1822815eed19edee5"}, - {file = "coverage-7.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c02cfa6c36144ab334d556989406837336c1d05215a9bdf44c0bc1d1ac1cb637"}, - {file = "coverage-7.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:477c9430ad5d1b80b07f3c12f7120eef40bfbf849e9e7859e53b9c93b922d2af"}, - {file = "coverage-7.3.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce2ee86ca75f9f96072295c5ebb4ef2a43cecf2870b0ca5e7a1cbdd929cf67e1"}, - {file = "coverage-7.3.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68d8a0426b49c053013e631c0cdc09b952d857efa8f68121746b339912d27a12"}, - {file = "coverage-7.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b3eb0c93e2ea6445b2173da48cb548364f8f65bf68f3d090404080d338e3a689"}, - {file = "coverage-7.3.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:90b6e2f0f66750c5a1178ffa9370dec6c508a8ca5265c42fbad3ccac210a7977"}, - {file = "coverage-7.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:96d7d761aea65b291a98c84e1250cd57b5b51726821a6f2f8df65db89363be51"}, - {file = "coverage-7.3.0-cp310-cp310-win32.whl", hash = "sha256:63c5b8ecbc3b3d5eb3a9d873dec60afc0cd5ff9d9f1c75981d8c31cfe4df8527"}, - {file = "coverage-7.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:97c44f4ee13bce914272589b6b41165bbb650e48fdb7bd5493a38bde8de730a1"}, - {file = "coverage-7.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:74c160285f2dfe0acf0f72d425f3e970b21b6de04157fc65adc9fd07ee44177f"}, - {file = "coverage-7.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b543302a3707245d454fc49b8ecd2c2d5982b50eb63f3535244fd79a4be0c99d"}, - {file = "coverage-7.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad0f87826c4ebd3ef484502e79b39614e9c03a5d1510cfb623f4a4a051edc6fd"}, - {file = "coverage-7.3.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:13c6cbbd5f31211d8fdb477f0f7b03438591bdd077054076eec362cf2207b4a7"}, - {file = "coverage-7.3.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fac440c43e9b479d1241fe9d768645e7ccec3fb65dc3a5f6e90675e75c3f3e3a"}, - {file = "coverage-7.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3c9834d5e3df9d2aba0275c9f67989c590e05732439b3318fa37a725dff51e74"}, - {file = "coverage-7.3.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4c8e31cf29b60859876474034a83f59a14381af50cbe8a9dbaadbf70adc4b214"}, - {file = "coverage-7.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7a9baf8e230f9621f8e1d00c580394a0aa328fdac0df2b3f8384387c44083c0f"}, - {file = "coverage-7.3.0-cp311-cp311-win32.whl", hash = "sha256:ccc51713b5581e12f93ccb9c5e39e8b5d4b16776d584c0f5e9e4e63381356482"}, - {file = "coverage-7.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:887665f00ea4e488501ba755a0e3c2cfd6278e846ada3185f42d391ef95e7e70"}, - {file = "coverage-7.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d000a739f9feed900381605a12a61f7aaced6beae832719ae0d15058a1e81c1b"}, - {file = "coverage-7.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:59777652e245bb1e300e620ce2bef0d341945842e4eb888c23a7f1d9e143c446"}, - {file = "coverage-7.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9737bc49a9255d78da085fa04f628a310c2332b187cd49b958b0e494c125071"}, - {file = "coverage-7.3.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5247bab12f84a1d608213b96b8af0cbb30d090d705b6663ad794c2f2a5e5b9fe"}, - {file = "coverage-7.3.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2ac9a1de294773b9fa77447ab7e529cf4fe3910f6a0832816e5f3d538cfea9a"}, - {file = "coverage-7.3.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:85b7335c22455ec12444cec0d600533a238d6439d8d709d545158c1208483873"}, - {file = "coverage-7.3.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:36ce5d43a072a036f287029a55b5c6a0e9bd73db58961a273b6dc11a2c6eb9c2"}, - {file = "coverage-7.3.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:211a4576e984f96d9fce61766ffaed0115d5dab1419e4f63d6992b480c2bd60b"}, - {file = "coverage-7.3.0-cp312-cp312-win32.whl", hash = "sha256:56afbf41fa4a7b27f6635bc4289050ac3ab7951b8a821bca46f5b024500e6321"}, - {file = "coverage-7.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f297e0c1ae55300ff688568b04ff26b01c13dfbf4c9d2b7d0cb688ac60df479"}, - {file = "coverage-7.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac0dec90e7de0087d3d95fa0533e1d2d722dcc008bc7b60e1143402a04c117c1"}, - {file = "coverage-7.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:438856d3f8f1e27f8e79b5410ae56650732a0dcfa94e756df88c7e2d24851fcd"}, - {file = "coverage-7.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1084393c6bda8875c05e04fce5cfe1301a425f758eb012f010eab586f1f3905e"}, - {file = "coverage-7.3.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49ab200acf891e3dde19e5aa4b0f35d12d8b4bd805dc0be8792270c71bd56c54"}, - {file = "coverage-7.3.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a67e6bbe756ed458646e1ef2b0778591ed4d1fcd4b146fc3ba2feb1a7afd4254"}, - {file = "coverage-7.3.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8f39c49faf5344af36042b293ce05c0d9004270d811c7080610b3e713251c9b0"}, - {file = "coverage-7.3.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7df91fb24c2edaabec4e0eee512ff3bc6ec20eb8dccac2e77001c1fe516c0c84"}, - {file = "coverage-7.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:34f9f0763d5fa3035a315b69b428fe9c34d4fc2f615262d6be3d3bf3882fb985"}, - {file = "coverage-7.3.0-cp38-cp38-win32.whl", hash = "sha256:bac329371d4c0d456e8d5f38a9b0816b446581b5f278474e416ea0c68c47dcd9"}, - {file = "coverage-7.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:b859128a093f135b556b4765658d5d2e758e1fae3e7cc2f8c10f26fe7005e543"}, - {file = "coverage-7.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fc0ed8d310afe013db1eedd37176d0839dc66c96bcfcce8f6607a73ffea2d6ba"}, - {file = "coverage-7.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61260ec93f99f2c2d93d264b564ba912bec502f679793c56f678ba5251f0393"}, - {file = "coverage-7.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97af9554a799bd7c58c0179cc8dbf14aa7ab50e1fd5fa73f90b9b7215874ba28"}, - {file = "coverage-7.3.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3558e5b574d62f9c46b76120a5c7c16c4612dc2644c3d48a9f4064a705eaee95"}, - {file = "coverage-7.3.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37d5576d35fcb765fca05654f66aa71e2808d4237d026e64ac8b397ffa66a56a"}, - {file = "coverage-7.3.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:07ea61bcb179f8f05ffd804d2732b09d23a1238642bf7e51dad62082b5019b34"}, - {file = "coverage-7.3.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:80501d1b2270d7e8daf1b64b895745c3e234289e00d5f0e30923e706f110334e"}, - {file = "coverage-7.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4eddd3153d02204f22aef0825409091a91bf2a20bce06fe0f638f5c19a85de54"}, - {file = "coverage-7.3.0-cp39-cp39-win32.whl", hash = "sha256:2d22172f938455c156e9af2612650f26cceea47dc86ca048fa4e0b2d21646ad3"}, - {file = "coverage-7.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:60f64e2007c9144375dd0f480a54d6070f00bb1a28f65c408370544091c9bc9e"}, - {file = "coverage-7.3.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:5492a6ce3bdb15c6ad66cb68a0244854d9917478877a25671d70378bdc8562d0"}, - {file = "coverage-7.3.0.tar.gz", hash = "sha256:49dbb19cdcafc130f597d9e04a29d0a032ceedf729e41b181f51cd170e6ee865"}, + {file = "coverage-7.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cd0f7429ecfd1ff597389907045ff209c8fdb5b013d38cfa7c60728cb484b6e3"}, + {file = "coverage-7.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:966f10df9b2b2115da87f50f6a248e313c72a668248be1b9060ce935c871f276"}, + {file = "coverage-7.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0575c37e207bb9b98b6cf72fdaaa18ac909fb3d153083400c2d48e2e6d28bd8e"}, + {file = "coverage-7.3.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:245c5a99254e83875c7fed8b8b2536f040997a9b76ac4c1da5bff398c06e860f"}, + {file = "coverage-7.3.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c96dd7798d83b960afc6c1feb9e5af537fc4908852ef025600374ff1a017392"}, + {file = "coverage-7.3.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:de30c1aa80f30af0f6b2058a91505ea6e36d6535d437520067f525f7df123887"}, + {file = "coverage-7.3.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:50dd1e2dd13dbbd856ffef69196781edff26c800a74f070d3b3e3389cab2600d"}, + {file = "coverage-7.3.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b9c0c19f70d30219113b18fe07e372b244fb2a773d4afde29d5a2f7930765136"}, + {file = "coverage-7.3.1-cp310-cp310-win32.whl", hash = "sha256:770f143980cc16eb601ccfd571846e89a5fe4c03b4193f2e485268f224ab602f"}, + {file = "coverage-7.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:cdd088c00c39a27cfa5329349cc763a48761fdc785879220d54eb785c8a38520"}, + {file = "coverage-7.3.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:74bb470399dc1989b535cb41f5ca7ab2af561e40def22d7e188e0a445e7639e3"}, + {file = "coverage-7.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:025ded371f1ca280c035d91b43252adbb04d2aea4c7105252d3cbc227f03b375"}, + {file = "coverage-7.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6191b3a6ad3e09b6cfd75b45c6aeeffe7e3b0ad46b268345d159b8df8d835f9"}, + {file = "coverage-7.3.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7eb0b188f30e41ddd659a529e385470aa6782f3b412f860ce22b2491c89b8593"}, + {file = "coverage-7.3.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75c8f0df9dfd8ff745bccff75867d63ef336e57cc22b2908ee725cc552689ec8"}, + {file = "coverage-7.3.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7eb3cd48d54b9bd0e73026dedce44773214064be93611deab0b6a43158c3d5a0"}, + {file = "coverage-7.3.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ac3c5b7e75acac31e490b7851595212ed951889918d398b7afa12736c85e13ce"}, + {file = "coverage-7.3.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5b4ee7080878077af0afa7238df1b967f00dc10763f6e1b66f5cced4abebb0a3"}, + {file = "coverage-7.3.1-cp311-cp311-win32.whl", hash = "sha256:229c0dd2ccf956bf5aeede7e3131ca48b65beacde2029f0361b54bf93d36f45a"}, + {file = "coverage-7.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:c6f55d38818ca9596dc9019eae19a47410d5322408140d9a0076001a3dcb938c"}, + {file = "coverage-7.3.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5289490dd1c3bb86de4730a92261ae66ea8d44b79ed3cc26464f4c2cde581fbc"}, + {file = "coverage-7.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ca833941ec701fda15414be400c3259479bfde7ae6d806b69e63b3dc423b1832"}, + {file = "coverage-7.3.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd694e19c031733e446c8024dedd12a00cda87e1c10bd7b8539a87963685e969"}, + {file = "coverage-7.3.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aab8e9464c00da5cb9c536150b7fbcd8850d376d1151741dd0d16dfe1ba4fd26"}, + {file = "coverage-7.3.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87d38444efffd5b056fcc026c1e8d862191881143c3aa80bb11fcf9dca9ae204"}, + {file = "coverage-7.3.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:8a07b692129b8a14ad7a37941a3029c291254feb7a4237f245cfae2de78de037"}, + {file = "coverage-7.3.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:2829c65c8faaf55b868ed7af3c7477b76b1c6ebeee99a28f59a2cb5907a45760"}, + {file = "coverage-7.3.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1f111a7d85658ea52ffad7084088277135ec5f368457275fc57f11cebb15607f"}, + {file = "coverage-7.3.1-cp312-cp312-win32.whl", hash = "sha256:c397c70cd20f6df7d2a52283857af622d5f23300c4ca8e5bd8c7a543825baa5a"}, + {file = "coverage-7.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:5ae4c6da8b3d123500f9525b50bf0168023313963e0e2e814badf9000dd6ef92"}, + {file = "coverage-7.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ca70466ca3a17460e8fc9cea7123c8cbef5ada4be3140a1ef8f7b63f2f37108f"}, + {file = "coverage-7.3.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f2781fd3cabc28278dc982a352f50c81c09a1a500cc2086dc4249853ea96b981"}, + {file = "coverage-7.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6407424621f40205bbe6325686417e5e552f6b2dba3535dd1f90afc88a61d465"}, + {file = "coverage-7.3.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:04312b036580ec505f2b77cbbdfb15137d5efdfade09156961f5277149f5e344"}, + {file = "coverage-7.3.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac9ad38204887349853d7c313f53a7b1c210ce138c73859e925bc4e5d8fc18e7"}, + {file = "coverage-7.3.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:53669b79f3d599da95a0afbef039ac0fadbb236532feb042c534fbb81b1a4e40"}, + {file = "coverage-7.3.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:614f1f98b84eb256e4f35e726bfe5ca82349f8dfa576faabf8a49ca09e630086"}, + {file = "coverage-7.3.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f1a317fdf5c122ad642db8a97964733ab7c3cf6009e1a8ae8821089993f175ff"}, + {file = "coverage-7.3.1-cp38-cp38-win32.whl", hash = "sha256:defbbb51121189722420a208957e26e49809feafca6afeef325df66c39c4fdb3"}, + {file = "coverage-7.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:f4f456590eefb6e1b3c9ea6328c1e9fa0f1006e7481179d749b3376fc793478e"}, + {file = "coverage-7.3.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f12d8b11a54f32688b165fd1a788c408f927b0960984b899be7e4c190ae758f1"}, + {file = "coverage-7.3.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f09195dda68d94a53123883de75bb97b0e35f5f6f9f3aa5bf6e496da718f0cb6"}, + {file = "coverage-7.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6601a60318f9c3945be6ea0f2a80571f4299b6801716f8a6e4846892737ebe4"}, + {file = "coverage-7.3.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07d156269718670d00a3b06db2288b48527fc5f36859425ff7cec07c6b367745"}, + {file = "coverage-7.3.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:636a8ac0b044cfeccae76a36f3b18264edcc810a76a49884b96dd744613ec0b7"}, + {file = "coverage-7.3.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5d991e13ad2ed3aced177f524e4d670f304c8233edad3210e02c465351f785a0"}, + {file = "coverage-7.3.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:586649ada7cf139445da386ab6f8ef00e6172f11a939fc3b2b7e7c9082052fa0"}, + {file = "coverage-7.3.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4aba512a15a3e1e4fdbfed2f5392ec221434a614cc68100ca99dcad7af29f3f8"}, + {file = "coverage-7.3.1-cp39-cp39-win32.whl", hash = "sha256:6bc6f3f4692d806831c136c5acad5ccedd0262aa44c087c46b7101c77e139140"}, + {file = "coverage-7.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:553d7094cb27db58ea91332e8b5681bac107e7242c23f7629ab1316ee73c4981"}, + {file = "coverage-7.3.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:220eb51f5fb38dfdb7e5d54284ca4d0cd70ddac047d750111a68ab1798945194"}, + {file = "coverage-7.3.1.tar.gz", hash = "sha256:6cb7fe1581deb67b782c153136541e20901aa312ceedaf1467dcb35255787952"}, ] [package.dependencies] @@ -532,34 +532,34 @@ files = [ [[package]] name = "cryptography" -version = "41.0.3" +version = "41.0.4" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507"}, - {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116"}, - {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c"}, - {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae"}, - {file = "cryptography-41.0.3-cp37-abi3-win32.whl", hash = "sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306"}, - {file = "cryptography-41.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4"}, - {file = "cryptography-41.0.3.tar.gz", hash = "sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34"}, + {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839"}, + {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143"}, + {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397"}, + {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860"}, + {file = "cryptography-41.0.4-cp37-abi3-win32.whl", hash = "sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd"}, + {file = "cryptography-41.0.4-cp37-abi3-win_amd64.whl", hash = "sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311"}, + {file = "cryptography-41.0.4.tar.gz", hash = "sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a"}, ] [package.dependencies] @@ -588,67 +588,67 @@ files = [ [[package]] name = "dulwich" -version = "0.21.5" +version = "0.21.6" description = "Python Git Library" optional = false python-versions = ">=3.7" files = [ - {file = "dulwich-0.21.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8864719bc176cdd27847332a2059127e2f7bab7db2ff99a999873cb7fff54116"}, - {file = "dulwich-0.21.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3800cdc17d144c1f7e114972293bd6c46688f5bcc2c9228ed0537ded72394082"}, - {file = "dulwich-0.21.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e2f676bfed8146966fe934ee734969d7d81548fbd250a8308582973670a9dab1"}, - {file = "dulwich-0.21.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4db330fb59fe3b9d253bdf0e49a521739db83689520c4921ab1c5242aaf77b82"}, - {file = "dulwich-0.21.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e8f6d4f4f4d01dd1d3c968e486d4cd77f96f772da7265941bc506de0944ddb9"}, - {file = "dulwich-0.21.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1cc0c9ba19ac1b2372598802bc9201a9c45e5d6f1f7a80ec40deeb10acc4e9ae"}, - {file = "dulwich-0.21.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:61e10242b5a7a82faa8996b2c76239cfb633620b02cdd2946e8af6e7eb31d651"}, - {file = "dulwich-0.21.5-cp310-cp310-win32.whl", hash = "sha256:7f357639b56146a396f48e5e0bc9bbaca3d6d51c8340bd825299272b588fff5f"}, - {file = "dulwich-0.21.5-cp310-cp310-win_amd64.whl", hash = "sha256:891d5c73e2b66d05dbb502e44f027dc0dbbd8f6198bc90dae348152e69d0befc"}, - {file = "dulwich-0.21.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:45d6198e804b539708b73a003419e48fb42ff2c3c6dd93f63f3b134dff6dd259"}, - {file = "dulwich-0.21.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c2a565d4e704d7f784cdf9637097141f6d47129c8fffc2fac699d57cb075a169"}, - {file = "dulwich-0.21.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:823091d6b6a1ea07dc4839c9752198fb39193213d103ac189c7669736be2eaff"}, - {file = "dulwich-0.21.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2c9931b657f2206abec0964ec2355ee2c1e04d05f8864e823ffa23c548c4548"}, - {file = "dulwich-0.21.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7dc358c2ee727322a09b7c6da43d47a1026049dbd3ad8d612eddca1f9074b298"}, - {file = "dulwich-0.21.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6155ab7388ee01c670f7c5d8003d4e133eebebc7085a856c007989f0ba921b36"}, - {file = "dulwich-0.21.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a605e10d72f90a39ea2e634fbfd80f866fc4df29a02ea6db52ae92e5fd4a2003"}, - {file = "dulwich-0.21.5-cp311-cp311-win32.whl", hash = "sha256:daa607370722c3dce99a0022397c141caefb5ed32032a4f72506f4817ea6405b"}, - {file = "dulwich-0.21.5-cp311-cp311-win_amd64.whl", hash = "sha256:5e56b2c1911c344527edb2bf1a4356e2fb7e086b1ba309666e1e5c2224cdca8a"}, - {file = "dulwich-0.21.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:85d3401d08b1ec78c7d58ae987c4bb7b768a438f3daa74aeb8372bebc7fb16fa"}, - {file = "dulwich-0.21.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90479608e49db93d8c9e4323bc0ec5496678b535446e29d8fd67dc5bbb5d51bf"}, - {file = "dulwich-0.21.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9a6bf99f57bcac4c77fc60a58f1b322c91cc4d8c65dc341f76bf402622f89cb"}, - {file = "dulwich-0.21.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3e68b162af2aae995355e7920f89d50d72b53d56021e5ac0a546d493b17cbf7e"}, - {file = "dulwich-0.21.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0ab86d6d42e385bf3438e70f3c9b16de68018bd88929379e3484c0ef7990bd3c"}, - {file = "dulwich-0.21.5-cp37-cp37m-win32.whl", hash = "sha256:f2eeca6d61366cf5ee8aef45bed4245a67d4c0f0d731dc2383eabb80fa695683"}, - {file = "dulwich-0.21.5-cp37-cp37m-win_amd64.whl", hash = "sha256:1b20a3656b48c941d49c536824e1e5278a695560e8de1a83b53a630143c4552e"}, - {file = "dulwich-0.21.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3932b5e17503b265a85f1eda77ede647681c3bab53bc9572955b6b282abd26ea"}, - {file = "dulwich-0.21.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6616132d219234580de88ceb85dd51480dc43b1bdc05887214b8dd9cfd4a9d40"}, - {file = "dulwich-0.21.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:eaf6c7fb6b13495c19c9aace88821c2ade3c8c55b4e216cd7cc55d3e3807d7fa"}, - {file = "dulwich-0.21.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be12a46f73023970125808a4a78f610c055373096c1ecea3280edee41613eba8"}, - {file = "dulwich-0.21.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baecef0d8b9199822c7912876a03a1af17833f6c0d461efb62decebd45897e49"}, - {file = "dulwich-0.21.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:82f632afb9c7c341a875d46aaa3e6c5e586c7a64ce36c9544fa400f7e4f29754"}, - {file = "dulwich-0.21.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82cdf482f8f51fcc965ffad66180b54a9abaea9b1e985a32e1acbfedf6e0e363"}, - {file = "dulwich-0.21.5-cp38-cp38-win32.whl", hash = "sha256:c8ded43dc0bd2e65420eb01e778034be5ca7f72e397a839167eda7dcb87c4248"}, - {file = "dulwich-0.21.5-cp38-cp38-win_amd64.whl", hash = "sha256:2aba0fdad2a19bd5bb3aad6882580cb33359c67b48412ccd4cfccd932012b35e"}, - {file = "dulwich-0.21.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:fd4ad079758514375f11469e081723ba8831ce4eaa1a64b41f06a3a866d5ac34"}, - {file = "dulwich-0.21.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7fe62685bf356bfb4d0738f84a3fcf0d1fc9e11fee152e488a20b8c66a52429e"}, - {file = "dulwich-0.21.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:aae448da7d80306dda4fc46292fed7efaa466294571ab3448be16714305076f1"}, - {file = "dulwich-0.21.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b24cb1fad0525dba4872e9381bc576ea2a6dcdf06b0ed98f8e953e3b1d719b89"}, - {file = "dulwich-0.21.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e39b7c2c9bda6acae83b25054650a8bb7e373e886e2334721d384e1479bf04b"}, - {file = "dulwich-0.21.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26456dba39d1209fca17187db06967130e27eeecad2b3c2bbbe63467b0bf09d6"}, - {file = "dulwich-0.21.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:281310644e02e3aa6d76bcaffe2063b9031213c4916b5f1a6e68c25bdecfaba4"}, - {file = "dulwich-0.21.5-cp39-cp39-win32.whl", hash = "sha256:4814ca3209dabe0fe7719e9545fbdad7f8bb250c5a225964fe2a31069940c4cf"}, - {file = "dulwich-0.21.5-cp39-cp39-win_amd64.whl", hash = "sha256:c922a4573267486be0ef85216f2da103fb38075b8465dc0e90457843884e4860"}, - {file = "dulwich-0.21.5-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e52b20c4368171b7d32bd3ab0f1d2402e76ad4f2ea915ff9aa73bc9fa2b54d6d"}, - {file = "dulwich-0.21.5-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aeb736d777ee21f2117a90fc453ee181aa7eedb9e255b5ef07c51733f3fe5cb6"}, - {file = "dulwich-0.21.5-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e8a79c1ed7166f32ad21974fa98d11bf6fd74e94a47e754c777c320e01257c6"}, - {file = "dulwich-0.21.5-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:b943517e30bd651fbc275a892bb96774f3893d95fe5a4dedd84496a98eaaa8ab"}, - {file = "dulwich-0.21.5-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:32493a456358a3a6c15bbda07106fc3d4cc50834ee18bc7717968d18be59b223"}, - {file = "dulwich-0.21.5-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0aa44b812d978fc22a04531f5090c3c369d5facd03fa6e0501d460a661800c7f"}, - {file = "dulwich-0.21.5-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f46bcb6777e5f9f4af24a2bd029e88b77316269d24ce66be590e546a0d8f7b7"}, - {file = "dulwich-0.21.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:a917fd3b4493db3716da2260f16f6b18f68d46fbe491d851d154fc0c2d984ae4"}, - {file = "dulwich-0.21.5-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:684c52cff867d10c75a7238151ca307582b3d251bbcd6db9e9cffbc998ef804e"}, - {file = "dulwich-0.21.5-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9019189d7a8f7394df6a22cd5b484238c5776e42282ad5d6d6c626b4c5f43597"}, - {file = "dulwich-0.21.5-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:494024f74c2eef9988adb4352b3651ac1b6c0466176ec62b69d3d3672167ba68"}, - {file = "dulwich-0.21.5-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:f9b6ac1b1c67fc6083c42b7b6cd3b211292c8a6517216c733caf23e8b103ab6d"}, - {file = "dulwich-0.21.5.tar.gz", hash = "sha256:70955e4e249ddda6e34a4636b90f74e931e558f993b17c52570fa6144b993103"}, + {file = "dulwich-0.21.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7f89bee4c97372e8aaf8ffaf5899f1bcd5184b5306d7eaf68738c1101ceba10e"}, + {file = "dulwich-0.21.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:847bb52562a211b596453a602e75739350c86d7edb846b5b1c46896a5c86b9bb"}, + {file = "dulwich-0.21.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4e09d0b4e985b371aa6728773781b19298d361a00772e20f98522868cf7edc6f"}, + {file = "dulwich-0.21.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8dfb50b3915e223a97f50fbac0dbc298d5fffeaac004eeeb3d552c57fe38416f"}, + {file = "dulwich-0.21.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a64eca1601e79c16df78afe08da9ac9497b934cbc5765990ca7d89a4b87453d9"}, + {file = "dulwich-0.21.6-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1fedd924763a5d640348db43a267a394aa80d551228ad45708e0b0cc2130bb62"}, + {file = "dulwich-0.21.6-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:edc21c3784dd9d9b85abd9fe53f81a884e2cdcc4e5e09ada17287420d64cfd46"}, + {file = "dulwich-0.21.6-cp310-cp310-win32.whl", hash = "sha256:daa3584beabfcf0da76df57535a23c80ff6d8ccde6ddbd23bdc79d317a0e20a7"}, + {file = "dulwich-0.21.6-cp310-cp310-win_amd64.whl", hash = "sha256:40623cc39a3f1634663d22d87f86e2e406cc8ff17ae7a3edc7fcf963c288992f"}, + {file = "dulwich-0.21.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e8ed878553f0b76facbb620b455fafa0943162fe8e386920717781e490444efa"}, + {file = "dulwich-0.21.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a89b19f4960e759915dbc23a4dd0abc067b55d8d65e9df50961b73091b87b81a"}, + {file = "dulwich-0.21.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:28acbd08d6b38720d99cc01da9dd307a2e0585e00436c95bcac6357b9a9a6f76"}, + {file = "dulwich-0.21.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c2f2683e0598f7c7071ef08a0822f062d8744549a0d45f2c156741033b7e3d7d"}, + {file = "dulwich-0.21.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54342cf96fe8a44648505c65f23d18889595762003a168d67d7263df66143bd2"}, + {file = "dulwich-0.21.6-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2a3fc071e5b14f164191286f7ffc02f60fe8b439d01fad0832697cc08c2237dd"}, + {file = "dulwich-0.21.6-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:32d7acfe3fe2ce4502446d8f7a5ab34cfd24c9ff8961e60337638410906a8fbb"}, + {file = "dulwich-0.21.6-cp311-cp311-win32.whl", hash = "sha256:5e58171a5d70f7910f73d25ff82a058edff09a4c1c3bd1de0dc6b1fbc9a42c3e"}, + {file = "dulwich-0.21.6-cp311-cp311-win_amd64.whl", hash = "sha256:ceabe8f96edfb9183034a860f5dc77586700b517457032867b64a03c44e5cf96"}, + {file = "dulwich-0.21.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4fdc2f081bc3e9e120079c2cea4be213e3f127335aca7c0ab0c19fe791270caa"}, + {file = "dulwich-0.21.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fe957564108f74325d0d042d85e0c67ef470921ca92b6e7d330c7c49a3b9c1d"}, + {file = "dulwich-0.21.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2912c8a845c8ccbc79d068a89db7172e355adeb84eb31f062cd3a406d528b30"}, + {file = "dulwich-0.21.6-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:81e237a6b1b20c79ef62ca19a8fb231f5519bab874b9a1c2acf9c05edcabd600"}, + {file = "dulwich-0.21.6-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:513d045e74307eeb31592255c38f37042c9aa68ce845a167943018ab5138b0e3"}, + {file = "dulwich-0.21.6-cp37-cp37m-win32.whl", hash = "sha256:e1ac882afa890ef993b8502647e6c6d2b3977ce56e3fe80058ce64607cbc7107"}, + {file = "dulwich-0.21.6-cp37-cp37m-win_amd64.whl", hash = "sha256:5d2ccf3d355850674f75655154a6519bf1f1664176c670109fa7041019b286f9"}, + {file = "dulwich-0.21.6-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:28c9724a167c84a83fc6238e0781f4702b5fe8c53ede31604525fb1a9d1833f4"}, + {file = "dulwich-0.21.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c816be529680659b6a19798287b4ec6de49040f58160d40b1b2934fd6c28e93f"}, + {file = "dulwich-0.21.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b0545f0fa9444a0eb84977d08e302e3f55fd7c34a0466ec28bedc3c839b2fc1f"}, + {file = "dulwich-0.21.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b1682e8e826471ea3c22b8521435e93799e3db8ad05dd3c8f9b1aaacfa78147"}, + {file = "dulwich-0.21.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24ad45928a65f39ea0f451f9989b7aaedba9893d48c3189b544a70c6a1043f71"}, + {file = "dulwich-0.21.6-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b1c9e55233f19cd19c484f607cd90ab578ac50ebfef607f77e3b35c2b6049470"}, + {file = "dulwich-0.21.6-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:18697b58e0fc5972de68b529b08ac9ddda3f39af27bcf3f6999635ed3da7ef68"}, + {file = "dulwich-0.21.6-cp38-cp38-win32.whl", hash = "sha256:22798e9ba59e32b8faff5d9067e2b5a308f6b0fba9b1e1e928571ad278e7b36c"}, + {file = "dulwich-0.21.6-cp38-cp38-win_amd64.whl", hash = "sha256:6c91e1ed20d3d9a6aaaed9e75adae37272b3fcbcc72bab1eb09574806da88563"}, + {file = "dulwich-0.21.6-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8b84450766a3b151c3676fec3e3ed76304e52a84d5d69ade0f34fff2782c1b41"}, + {file = "dulwich-0.21.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a3da632648ee27b64bb5b285a3a94fddf297a596891cca12ac0df43c4f59448f"}, + {file = "dulwich-0.21.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cef50c0a19f322b7150248b8fa0862ce1652dec657e340c4020573721e85f215"}, + {file = "dulwich-0.21.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ac20dfcfd6057efb8499158d23f2c059f933aefa381e192100e6d8bc25d562"}, + {file = "dulwich-0.21.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81d10aa50c0a9a6dd495990c639358e3a3bbff39e17ff302179be6e93b573da7"}, + {file = "dulwich-0.21.6-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a9b52a08d49731375662936d05a12c4a64a6fe0ce257111f62638e475fb5d26d"}, + {file = "dulwich-0.21.6-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ed2f1f638b9adfba862719693b371ffe5d58e94d552ace9a23dea0fb0db6f468"}, + {file = "dulwich-0.21.6-cp39-cp39-win32.whl", hash = "sha256:bf90f2f9328a82778cf85ab696e4a7926918c3f315c75fc432ba31346bfa89b7"}, + {file = "dulwich-0.21.6-cp39-cp39-win_amd64.whl", hash = "sha256:e0dee3840c3c72e1d60c8f87a7a715d8eac023b9e1b80199d97790f7a1c60d9c"}, + {file = "dulwich-0.21.6-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:32d3a35caad6879d04711b358b861142440a543f5f4e02df67b13cbcd57f84a6"}, + {file = "dulwich-0.21.6-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c04df87098053b7767b46fc04b7943d75443f91c73560ca50157cdc22e27a5d3"}, + {file = "dulwich-0.21.6-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e07f145c7b0d82a9f77d157f493a61900e913d1c1f8b1f40d07d919ffb0929a4"}, + {file = "dulwich-0.21.6-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:008ff08629ab16d3638a9f36cfc6f5bd74b4d594657f2dc1583d8d3201794571"}, + {file = "dulwich-0.21.6-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:bf469cd5076623c2aad69d01ce9d5392fcb38a5faef91abe1501be733453e37d"}, + {file = "dulwich-0.21.6-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6592ef2d16ac61a27022647cf64a048f5be6e0a6ab2ebc7322bfbe24fb2b971b"}, + {file = "dulwich-0.21.6-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99577b2b37f64bc87280079245fb2963494c345d7db355173ecec7ab3d64b949"}, + {file = "dulwich-0.21.6-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:d7cd9fb896c65e4c28cb9332f2be192817805978dd8dc299681c4fe83c631158"}, + {file = "dulwich-0.21.6-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d9002094198e57e88fe77412d3aa64dd05978046ae725a16123ba621a7704628"}, + {file = "dulwich-0.21.6-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9b6f8a16f32190aa88c37ef013858b3e01964774bc983900bd0d74ecb6576e6"}, + {file = "dulwich-0.21.6-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eee8aba4dec4d0a52737a8a141f3456229c87dcfd7961f8115786a27b6ebefed"}, + {file = "dulwich-0.21.6-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a780e2a0ff208c4f218e72eff8d13f9aff485ff9a6f3066c22abe4ec8cec7dcd"}, + {file = "dulwich-0.21.6.tar.gz", hash = "sha256:30fbe87e8b51f3813c131e2841c86d007434d160bd16db586b40d47f31dd05b0"}, ] [package.dependencies] @@ -696,21 +696,19 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.12.3" +version = "3.12.4" description = "A platform independent file lock." optional = false python-versions = ">=3.8" files = [ - {file = "filelock-3.12.3-py3-none-any.whl", hash = "sha256:f067e40ccc40f2b48395a80fcbd4728262fab54e232e090a4063ab804179efeb"}, - {file = "filelock-3.12.3.tar.gz", hash = "sha256:0ecc1dd2ec4672a10c8550a8182f1bd0c0a5088470ecd5a125e45f49472fac3d"}, + {file = "filelock-3.12.4-py3-none-any.whl", hash = "sha256:08c21d87ded6e2b9da6728c3dff51baf1dcecf973b768ef35bcbc3447edb9ad4"}, + {file = "filelock-3.12.4.tar.gz", hash = "sha256:2e6f249f1f3654291606e046b09f1fd5eac39b360664c27f5aad072012f8bcbd"}, ] -[package.dependencies] -typing-extensions = {version = ">=4.7.1", markers = "python_version < \"3.11\""} - [package.extras] docs = ["furo (>=2023.7.26)", "sphinx (>=7.1.2)", "sphinx-autodoc-typehints (>=1.24)"] testing = ["covdefaults (>=2.3)", "coverage (>=7.3)", "diff-cover (>=7.7)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-timeout (>=2.1)"] +typing = ["typing-extensions (>=4.7.1)"] [[package]] name = "frozenlist" @@ -1383,13 +1381,13 @@ files = [ [[package]] name = "pytest" -version = "7.4.0" +version = "7.4.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, - {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, + {file = "pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002"}, + {file = "pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"}, ] [package.dependencies] @@ -1696,13 +1694,13 @@ files = [ [[package]] name = "trove-classifiers" -version = "2023.8.7" +version = "2023.9.19" description = "Canonical source for classifiers on PyPI (pypi.org)." optional = false python-versions = "*" files = [ - {file = "trove-classifiers-2023.8.7.tar.gz", hash = "sha256:c9f2a0a85d545e5362e967e4f069f56fddfd91215e22ffa48c66fb283521319a"}, - {file = "trove_classifiers-2023.8.7-py3-none-any.whl", hash = "sha256:a676626a31286130d56de2ea1232484df97c567eb429d56cfcb0637e681ecf09"}, + {file = "trove-classifiers-2023.9.19.tar.gz", hash = "sha256:3e700af445c802f251ce2b741ee78d2e5dfa5ab8115b933b89ca631b414691c9"}, + {file = "trove_classifiers-2023.9.19-py3-none-any.whl", hash = "sha256:55460364fe248294386d4dfa5d16544ec930493ecc6bd1db07a0d50afb37018e"}, ] [[package]] @@ -1732,13 +1730,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.7.1" -description = "Backported and Experimental Type Hints for Python 3.7+" +version = "4.8.0" +description = "Backported and Experimental Type Hints for Python 3.8+" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, - {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, + {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, + {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, ] [[package]] @@ -1760,13 +1758,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "virtualenv" -version = "20.24.4" +version = "20.24.5" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.24.4-py3-none-any.whl", hash = "sha256:29c70bb9b88510f6414ac3e55c8b413a1f96239b6b789ca123437d5e892190cb"}, - {file = "virtualenv-20.24.4.tar.gz", hash = "sha256:772b05bfda7ed3b8ecd16021ca9716273ad9f4467c801f27e83ac73430246dca"}, + {file = "virtualenv-20.24.5-py3-none-any.whl", hash = "sha256:b80039f280f4919c77b30f1c23294ae357c4c8701042086e3fc005963e4e537b"}, + {file = "virtualenv-20.24.5.tar.gz", hash = "sha256:e8361967f6da6fbdf1426483bfe9fca8287c242ac0bc30429905721cefbff752"}, ] [package.dependencies] @@ -1951,17 +1949,17 @@ multidict = ">=4.0" [[package]] name = "zipp" -version = "3.16.2" +version = "3.17.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.16.2-py3-none-any.whl", hash = "sha256:679e51dd4403591b2d6838a48de3d283f3d188412a9782faadf845f298736ba0"}, - {file = "zipp-3.16.2.tar.gz", hash = "sha256:ebc15946aa78bd63458992fc81ec3b6f7b1e92d51c35e6de1c3804e73b799147"}, + {file = "zipp-3.17.0-py3-none-any.whl", hash = "sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31"}, + {file = "zipp-3.17.0.tar.gz", hash = "sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy (>=0.9.1)", "pytest-ruff"] [metadata] diff --git a/src/nuropb/contexts/context_manager_decorator.py b/src/nuropb/contexts/context_manager_decorator.py index f411191..9def715 100644 --- a/src/nuropb/contexts/context_manager_decorator.py +++ b/src/nuropb/contexts/context_manager_decorator.py @@ -64,8 +64,7 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: method_args = list(args) ctx = method_args[1] if not isinstance(ctx, NuropbContextManager): - """ Replace dictionary context with NuropbContextManager instance - """ + """Replace dictionary context with NuropbContextManager instance""" ctx = NuropbContextManager( context=method_args[1], suppress_exceptions=suppress_exceptions, diff --git a/src/nuropb/contexts/describe.py b/src/nuropb/contexts/describe.py index b9a8008..cdf97b6 100644 --- a/src/nuropb/contexts/describe.py +++ b/src/nuropb/contexts/describe.py @@ -189,9 +189,13 @@ def map_argument(arg_props: Any) -> Tuple[str, Dict[str, Any]]: f"Service {service_name} has encrypted methods but no private key has been set" ) - service_info["public_key"] = private_key.public_key().public_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PublicFormat.SubjectPublicKeyInfo, - ).decode("ascii") + service_info["public_key"] = ( + private_key.public_key() + .public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + .decode("ascii") + ) return service_info diff --git a/src/nuropb/encodings/encryption.py b/src/nuropb/encodings/encryption.py index a6dbc7b..f7a1e2b 100644 --- a/src/nuropb/encodings/encryption.py +++ b/src/nuropb/encodings/encryption.py @@ -92,7 +92,9 @@ def new_symmetric_key(cls) -> bytes: """ return Fernet.generate_key() - def add_service_public_key(self, service_name: str, public_key: rsa.RSAPublicKey) -> None: + def add_service_public_key( + self, service_name: str, public_key: rsa.RSAPublicKey + ) -> None: """Add a public key for a service :param service_name: str :param public_key: rsa.RSAPublicKey diff --git a/src/nuropb/encodings/serializor.py b/src/nuropb/encodings/serializor.py index fcc83a9..ae3f653 100644 --- a/src/nuropb/encodings/serializor.py +++ b/src/nuropb/encodings/serializor.py @@ -19,9 +19,9 @@ def get_serializor(payload_type: str = "json") -> SerializorTypes: def encode_payload( - payload: PayloadDict, - payload_type: str = "json", - public_key: rsa.RSAPublicKey = None + payload: PayloadDict, + payload_type: str = "json", + public_key: rsa.RSAPublicKey = None, ) -> bytes: """ :param public_key: diff --git a/src/nuropb/interface.py b/src/nuropb/interface.py index 8d61713..b70a81f 100644 --- a/src/nuropb/interface.py +++ b/src/nuropb/interface.py @@ -15,9 +15,9 @@ logger = logging.getLogger(__name__) -NUROPB_VERSION = "0.1.2" -NUROPB_PROTOCOL_VERSION = "0.1.0" -NUROPB_PROTOCOL_VERSIONS_SUPPORTED = ("0.1.0",) +NUROPB_VERSION = "0.1.3" +NUROPB_PROTOCOL_VERSION = "0.1.1" +NUROPB_PROTOCOL_VERSIONS_SUPPORTED = ("0.1.1",) NUROPB_MESSAGE_TYPES = ( "request", "response", @@ -223,7 +223,9 @@ class NuropbException(Exception): """ description: str - payload: PayloadDict | TransportServicePayload | TransportRespondPayload | Dict[str, Any] | None + payload: PayloadDict | TransportServicePayload | TransportRespondPayload | Dict[ + str, Any + ] | None exception: BaseException | None def __init__( diff --git a/src/nuropb/rmq_api.py b/src/nuropb/rmq_api.py index 8e7798f..c541ae9 100644 --- a/src/nuropb/rmq_api.py +++ b/src/nuropb/rmq_api.py @@ -76,7 +76,7 @@ def __init__( self._connection_name = f"{vhost}-{service_name}-{instance_id}" self._encryptor = Encryptor( service_name=service_name, - private_key=getattr(service_instance, "_private_key", None) + private_key=getattr(service_instance, "_private_key", None), ) self._service_name = service_name @@ -219,9 +219,7 @@ def receive_transport_message( """ The logic below is only relevant for incoming service messages """ if self._service_instance is None: - error_description = ( - f"No service instance configured to handle the {service_message['nuropb_type']} instruction" - ) + error_description = f"No service instance configured to handle the {service_message['nuropb_type']} instruction" logger.warning(error_description) response = NuropbHandlingError( description=error_description, @@ -458,7 +456,9 @@ def publish_event( encrypted=encrypted, ) - async def describe_service(self, service_name: str, refresh: bool = False) -> Dict[str, Any] | None: + async def describe_service( + self, service_name: str, refresh: bool = False + ) -> Dict[str, Any] | None: """describe_service: returns the service information for the given service_name, if it is not already cached or refresh is try then the service discovery is queried directly. @@ -478,13 +478,13 @@ async def describe_service(self, service_name: str, refresh: bool = False) -> Di trace_id=uuid4().hex, ) if not isinstance(service_info, dict): - raise ValueError(f"Invalid service_info returned for service {service_name}") + raise ValueError( + f"Invalid service_info returned for service {service_name}" + ) else: self._service_discovery[service_name] = service_info try: - text_public_key = service_info.get( - "public_key", None - ) + text_public_key = service_info.get("public_key", None) if text_public_key: public_key = serialization.load_pem_public_key( data=text_public_key.encode("ascii"), @@ -514,4 +514,3 @@ async def requires_encryption(self, service_name: str, method_name: str) -> bool f"Method {method_name} not found on service {service_name}" ) return method_info.get("requires_encryption", False) - diff --git a/src/nuropb/rmq_transport.py b/src/nuropb/rmq_transport.py index 2a3eade..1a3a25f 100644 --- a/src/nuropb/rmq_transport.py +++ b/src/nuropb/rmq_transport.py @@ -36,7 +36,7 @@ from nuropb import service_handlers from nuropb.service_handlers import ( create_transport_response_from_rmq_decode_exception, - error_dict_from_exception + error_dict_from_exception, ) from nuropb.utils import obfuscate_credentials @@ -66,14 +66,20 @@ class RabbitMQConfiguration(TypedDict): """ _verbose = False + + @property def verbose() -> bool: return _verbose + + @verbose.setter def verbose(value: bool) -> None: global _verbose _verbose = value service_handlers.verbose = value + + """ Set to True to enable module verbose logging """ @@ -810,9 +816,12 @@ def on_message_returned( ) def send_message( - self, payload: Dict[str, Any], expiry: Optional[int] = None, - priority: Optional[int] = None, encoding: str = "json", - encrypted: bool = False, + self, + payload: Dict[str, Any], + expiry: Optional[int] = None, + priority: Optional[int] = None, + encoding: str = "json", + encrypted: bool = False, ) -> None: """Send a message to over the RabbitMQ Transport @@ -856,7 +865,11 @@ def send_message( """ now encrypt the payload if public_key is not None, update RMQ header to indicate encrypted payload. """ - if encrypted and self._encryptor and payload["tag"] in ("request", "command", "response"): + if ( + encrypted + and self._encryptor + and payload["tag"] in ("request", "command", "response") + ): encrypted = "yes" to_service = None if payload["tag"] == "response" else payload["service"] """ only outgoing response and command messages require the target service name @@ -1041,7 +1054,8 @@ def on_service_message_complete( payload=respond_payload, priority=None, encoding="json", - encrypted=encrypted) + encrypted=encrypted, + ) """ NOTE - METADATA: keep this dictionary in sync with across all these methods: - on_service_message, on_service_message_complete diff --git a/src/nuropb/service_handlers.py b/src/nuropb/service_handlers.py index 9e92ddb..2f3ec7d 100644 --- a/src/nuropb/service_handlers.py +++ b/src/nuropb/service_handlers.py @@ -37,13 +37,19 @@ logger = logging.getLogger(__name__) _verbose = False + + @property def verbose() -> bool: return _verbose + + @verbose.setter def verbose(value: bool) -> None: global _verbose _verbose = value + + """ Set to True to enable module verbose logging """ @@ -376,10 +382,10 @@ def execute_request( """ if method_name != "nuropb_describe" and ( - method_name.startswith("_") - or not hasattr(service_instance, method_name) - or not callable(getattr(service_instance, method_name)) - ): + method_name.startswith("_") + or not hasattr(service_instance, method_name) + or not callable(getattr(service_instance, method_name)) + ): raise NuropbHandlingError( description="Unknown method {}".format(method_name), payload=payload, From 80dc7500ba114b724ac7a5f958a0dc012813f33d Mon Sep 17 00:00:00 2001 From: Robert Betts Date: Wed, 20 Sep 2023 00:19:01 +0100 Subject: [PATCH 10/10] Code Linting Bumping version to 0.1.3 protocol version to 0.1.1 --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9f0b43d..72bea25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,6 @@ keywords = ["python", "asynchrous", "api", "event", "rpc", "distributed", "edd", classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.10", - "License :: OSI Approved :: Apache-2.0 license", "Operating System :: OS Independent" ] packages = [