Add ZstdSerializer, change Key to include "domain"

RevenueCat · Mar 5, 2024 · 1d2ab94 · 1d2ab94
1 parent a0f1bb3
commit 1d2ab94
Show file tree

Hide file tree

Showing 21 changed files with 752 additions and 194 deletions.
diff --git a/.flake8 b/.flake8
@@ -1,10 +1,10 @@
 [flake8]
-ignore=E203,W503
+ignore=E203,W503,E704,E701
 max-line-length = 88
 select = B,B9,BLK,C,E,F,I,S,W
 max-complexity = 10
 application-import-names = meta_memcache,tests
 import-order-style = cryptography
 per-file-ignores =
     __init__.py:F401
-    tests/*:S101,S403
+    tests/*:S101,S403
diff --git a/README.md b/README.md
@@ -97,14 +97,26 @@ will be gone or present, since they are stored in the same server). Note this is
 also risky, if you place all keys of a user in the same server, and the server
 goes down, the user life will be miserable.
 
-### Unicode keys:
-Unicode keys are supported, the keys will be hashed according to Meta commands
+### Custom domains:
+You can add a domain to keys. This domain can be used for custom per-domain
+metrics like hit ratios or to control serialization of the values.
+```python:
+Key("key:1:2", domain="example")
+```
+For example the ZstdSerializer allows to configure different dictionaries by
+domain, so you can compress more efficiently data of different domains.
+
+### Unicode/binary keys:
+Both unicode and binary keys are supported, the keys will be hashed/encoded according to Meta commands
 [binary encoded keys](https://github.com/memcached/memcached/wiki/MetaCommands#binary-encoded-keys)
 specification.
 
-To use this, mark the key as unicode:
+Using binary keys can have benefits, saving space in memory. While over the wire the key
+is transmited b64 encoded, the memcache server will use the byte representation, so it will
+not have the 1/4 overhead of b64 encoding.
+
 ```python:
-Key("🍺", unicode=True)
+Key("🍺")
 ```
 
 ### Large keys:

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -14,7 +14,8 @@ packages = [{include = "meta_memcache", from="src"}]
 python = "^3.8"
 uhashring = "^2.1"
 marisa-trie = "^1.0.0"
-meta-memcache-socket = "0.1.1"
+meta-memcache-socket = "0.1.3"
+zstandard = "^0.22.0"
 
 [tool.poetry.group.extras.dependencies]
 prometheus-client = "^0.17.1"
@@ -27,7 +28,7 @@ testpaths = [
 
 [tool.isort]
 profile = "black"
-known_third_party = ["uhashring", "pytest", "pytest_mock", "marisa-trie"]
+known_third_party = ["uhashring", "pytest", "pytest_mock", "marisa-trie", "zstandard"]
 
 [tool.coverage.paths]
 source = ["src", "*/site-packages"]

diff --git a/src/meta_memcache/base/base_serializer.py b/src/meta_memcache/base/base_serializer.py
@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from typing import Any, NamedTuple
 
-from meta_memcache.protocol import Blob
+from meta_memcache.protocol import Blob, Key
 
 
 class EncodedValue(NamedTuple):
@@ -13,10 +13,9 @@ class BaseSerializer(ABC):
     @abstractmethod
     def serialize(
         self,
+        key: Key,
         value: Any,
-    ) -> EncodedValue:
-        ...
+    ) -> EncodedValue: ...
 
     @abstractmethod
-    def unserialize(self, data: Blob, encoding_id: int) -> Any:
-        ...
+    def unserialize(self, data: Blob, encoding_id: int) -> Any: ...
diff --git a/src/meta_memcache/commands/high_level_commands.py b/src/meta_memcache/commands/high_level_commands.py
@@ -56,22 +56,19 @@ def _get(
         lease_policy: Optional[LeasePolicy] = None,
         recache_policy: Optional[RecachePolicy] = None,
         return_cas_token: bool = False,
-    ) -> Optional[Value]:
-        ...  # pragma: no cover
+    ) -> Optional[Value]: ...  # pragma: no cover
 
     def _process_get_result(
         self, key: Union[Key, str], result: ReadResponse
-    ) -> Optional[Value]:
-        ...  # pragma: no cover
+    ) -> Optional[Value]: ...  # pragma: no cover
 
     def _get_typed_value(
         self,
         key: Union[Key, str],
         value: Any,
         cls: Type[T],
         error_on_type_mismatch: bool = False,
-    ) -> Optional[T]:
-        ...  # pragma: no cover
+    ) -> Optional[T]: ...  # pragma: no cover
 
     def _get_delta_flags(
         self,
@@ -80,8 +77,7 @@ def _get_delta_flags(
         no_reply: bool = False,
         cas_token: Optional[int] = None,
         return_value: bool = False,
-    ) -> RequestFlags:
-        ...  # pragma: no cover
+    ) -> RequestFlags: ...  # pragma: no cover
 
 
 class HighLevelCommandsMixin:

diff --git a/src/meta_memcache/connection/providers.py b/src/meta_memcache/connection/providers.py
@@ -9,11 +9,9 @@
 
 
 class ConnectionPoolProvider(Protocol):
-    def get_pool(self, key: Key) -> ConnectionPool:
-        ...  # pragma: no cover
+    def get_pool(self, key: Key) -> ConnectionPool: ...  # pragma: no cover
 
-    def get_counters(self) -> Dict[ServerAddress, PoolCounters]:
-        ...  # pragma: no cover
+    def get_counters(self) -> Dict[ServerAddress, PoolCounters]: ...  # pragma: no cover
 
 
 class HostConnectionPoolProvider:

diff --git a/src/meta_memcache/errors.py b/src/meta_memcache/errors.py
@@ -8,5 +8,4 @@ def __init__(self, server: str, message: str) -> None:
         super().__init__(message)
 
 
-class ServerMarkedDownError(MemcacheServerError):
-    ...
+class ServerMarkedDownError(MemcacheServerError): ...
diff --git a/src/meta_memcache/executors/default.py b/src/meta_memcache/executors/default.py
@@ -68,10 +68,11 @@ def _build_cmd(
 
     def _prepare_serialized_value_and_flags(
         self,
+        key: Key,
         value: ValueContainer,
         flags: Optional[RequestFlags],
     ) -> Tuple[Optional[bytes], RequestFlags]:
-        encoded_value = self._serializer.serialize(value.value)
+        encoded_value = self._serializer.serialize(key, value.value)
         flags = flags if flags is not None else RequestFlags()
         flags.client_flag = encoded_value.encoding_id
         return encoded_value.data, flags
@@ -106,7 +107,7 @@ def exec_on_pool(
         cmd_value, flags = (
             (None, flags)
             if value is None
-            else self._prepare_serialized_value_and_flags(value, flags)
+            else self._prepare_serialized_value_and_flags(key, value, flags)
         )
         try:
             conn = pool.pop_connection()
@@ -159,7 +160,7 @@ def exec_multi_on_pool(  # noqa: C901
                     cmd_value, flags = (
                         (None, flags)
                         if value is None
-                        else self._prepare_serialized_value_and_flags(value, flags)
+                        else self._prepare_serialized_value_and_flags(key, value, flags)
                     )
 
                     self._conn_send_cmd(

diff --git a/src/meta_memcache/interfaces/cache_api.py b/src/meta_memcache/interfaces/cache_api.py
@@ -7,12 +7,11 @@
 
 class CacheApi(CommandsProtocol, Protocol):
     @property
-    def on_write_failure(self) -> WriteFailureEvent:
-        ...  # pragma: no cover
+    def on_write_failure(self) -> WriteFailureEvent: ...  # pragma: no cover
 
     @on_write_failure.setter
-    def on_write_failure(self, value: WriteFailureEvent) -> None:
-        ...  # pragma: no cover
+    def on_write_failure(
+        self, value: WriteFailureEvent
+    ) -> None: ...  # pragma: no cover
 
-    def get_counters(self) -> Dict[ServerAddress, PoolCounters]:
-        ...  # pragma: no cover
+    def get_counters(self) -> Dict[ServerAddress, PoolCounters]: ...  # pragma: no cover
diff --git a/src/meta_memcache/interfaces/executor.py b/src/meta_memcache/interfaces/executor.py
@@ -47,9 +47,9 @@ def exec_multi_on_pool(
         ...  # pragma: no cover
 
     @property
-    def on_write_failure(self) -> WriteFailureEvent:
-        ...  # pragma: no cover
+    def on_write_failure(self) -> WriteFailureEvent: ...  # pragma: no cover
 
     @on_write_failure.setter
-    def on_write_failure(self, value: WriteFailureEvent) -> None:
-        ...  # pragma: no cover
+    def on_write_failure(
+        self, value: WriteFailureEvent
+    ) -> None: ...  # pragma: no cover
diff --git a/src/meta_memcache/interfaces/high_level_commands.py b/src/meta_memcache/interfaces/high_level_commands.py
@@ -16,8 +16,7 @@ def set(
         cas_token: Optional[int] = None,
         stale_policy: Optional[StalePolicy] = None,
         set_mode: SetMode = SetMode.SET,
-    ) -> bool:
-        ...  # pragma: no cover
+    ) -> bool: ...  # pragma: no cover
 
     def refill(
         self,
@@ -76,59 +75,52 @@ def touch(
         key: Union[Key, str],
         ttl: int,
         no_reply: bool = False,
-    ) -> bool:
-        ...  # pragma: no cover
+    ) -> bool: ...  # pragma: no cover
 
     def get_or_lease(
         self,
         key: Union[Key, str],
         lease_policy: LeasePolicy,
         touch_ttl: Optional[int] = None,
         recache_policy: Optional[RecachePolicy] = None,
-    ) -> Optional[Any]:
-        ...  # pragma: no cover
+    ) -> Optional[Any]: ...  # pragma: no cover
 
     def get_or_lease_cas(
         self,
         key: Union[Key, str],
         lease_policy: LeasePolicy,
         touch_ttl: Optional[int] = None,
         recache_policy: Optional[RecachePolicy] = None,
-    ) -> Tuple[Optional[Any], Optional[int]]:
-        ...  # pragma: no cover
+    ) -> Tuple[Optional[Any], Optional[int]]: ...  # pragma: no cover
 
     def get(
         self,
         key: Union[Key, str],
         touch_ttl: Optional[int] = None,
         recache_policy: Optional[RecachePolicy] = None,
-    ) -> Optional[Any]:
-        ...  # pragma: no cover
+    ) -> Optional[Any]: ...  # pragma: no cover
 
     def multi_get(
         self,
         keys: Iterable[Union[Key, str]],
         touch_ttl: Optional[int] = None,
         recache_policy: Optional[RecachePolicy] = None,
-    ) -> Dict[Key, Optional[Any]]:
-        ...  # pragma: no cover
+    ) -> Dict[Key, Optional[Any]]: ...  # pragma: no cover
 
     def _multi_get(
         self,
         keys: Iterable[Union[Key, str]],
         touch_ttl: Optional[int] = None,
         recache_policy: Optional[RecachePolicy] = None,
         return_cas_token: bool = False,
-    ) -> Dict[Key, Optional[Value]]:
-        ...  # pragma: no cover
+    ) -> Dict[Key, Optional[Value]]: ...  # pragma: no cover
 
     def get_cas(
         self,
         key: Union[Key, str],
         touch_ttl: Optional[int] = None,
         recache_policy: Optional[RecachePolicy] = None,
-    ) -> Tuple[Optional[Any], Optional[int]]:
-        ...  # pragma: no cover
+    ) -> Tuple[Optional[Any], Optional[int]]: ...  # pragma: no cover
 
     def _get(
         self,
@@ -137,8 +129,7 @@ def _get(
         lease_policy: Optional[LeasePolicy] = None,
         recache_policy: Optional[RecachePolicy] = None,
         return_cas_token: bool = False,
-    ) -> Optional[Value]:
-        ...  # pragma: no cover
+    ) -> Optional[Value]: ...  # pragma: no cover
 
     def get_typed(
         self,
@@ -147,8 +138,7 @@ def get_typed(
         touch_ttl: Optional[int] = None,
         recache_policy: Optional[RecachePolicy] = None,
         error_on_type_mismatch: bool = False,
-    ) -> Optional[T]:
-        ...  # pragma: no cover
+    ) -> Optional[T]: ...  # pragma: no cover
 
     def get_cas_typed(
         self,
@@ -157,8 +147,7 @@ def get_cas_typed(
         touch_ttl: Optional[int] = None,
         recache_policy: Optional[RecachePolicy] = None,
         error_on_type_mismatch: bool = False,
-    ) -> Tuple[Optional[T], Optional[int]]:
-        ...  # pragma: no cover
+    ) -> Tuple[Optional[T], Optional[int]]: ...  # pragma: no cover
 
     def delta(
         self,
@@ -167,8 +156,7 @@ def delta(
         refresh_ttl: Optional[int] = None,
         no_reply: bool = False,
         cas_token: Optional[int] = None,
-    ) -> bool:
-        ...  # pragma: no cover
+    ) -> bool: ...  # pragma: no cover
 
     def delta_initialize(
         self,
@@ -179,17 +167,15 @@ def delta_initialize(
         refresh_ttl: Optional[int] = None,
         no_reply: bool = False,
         cas_token: Optional[int] = None,
-    ) -> bool:
-        ...  # pragma: no cover
+    ) -> bool: ...  # pragma: no cover
 
     def delta_and_get(
         self,
         key: Union[Key, str],
         delta: int,
         refresh_ttl: Optional[int] = None,
         cas_token: Optional[int] = None,
-    ) -> Optional[int]:
-        ...  # pragma: no cover
+    ) -> Optional[int]: ...  # pragma: no cover
 
     def delta_initialize_and_get(
         self,
@@ -199,5 +185,4 @@ def delta_initialize_and_get(
         initial_ttl: int,
         refresh_ttl: Optional[int] = None,
         cas_token: Optional[int] = None,
-    ) -> Optional[int]:
-        ...  # pragma: no cover
+    ) -> Optional[int]: ...  # pragma: no cover
diff --git a/src/meta_memcache/interfaces/meta_commands.py b/src/meta_memcache/interfaces/meta_commands.py
@@ -15,16 +15,14 @@ def meta_multiget(
         keys: List[Key],
         flags: Optional[RequestFlags] = None,
         failure_handling: FailureHandling = DEFAULT_FAILURE_HANDLING,
-    ) -> Dict[Key, ReadResponse]:
-        ...  # pragma: no cover
+    ) -> Dict[Key, ReadResponse]: ...  # pragma: no cover
 
     def meta_get(
         self,
         key: Key,
         flags: Optional[RequestFlags] = None,
         failure_handling: FailureHandling = DEFAULT_FAILURE_HANDLING,
-    ) -> ReadResponse:
-        ...  # pragma: no cover
+    ) -> ReadResponse: ...  # pragma: no cover
 
     def meta_set(
         self,
@@ -33,21 +31,18 @@ def meta_set(
         ttl: int,
         flags: Optional[RequestFlags] = None,
         failure_handling: FailureHandling = DEFAULT_FAILURE_HANDLING,
-    ) -> WriteResponse:
-        ...  # pragma: no cover
+    ) -> WriteResponse: ...  # pragma: no cover
 
     def meta_delete(
         self,
         key: Key,
         flags: Optional[RequestFlags] = None,
         failure_handling: FailureHandling = DEFAULT_FAILURE_HANDLING,
-    ) -> WriteResponse:
-        ...  # pragma: no cover
+    ) -> WriteResponse: ...  # pragma: no cover
 
     def meta_arithmetic(
         self,
         key: Key,
         flags: Optional[RequestFlags] = None,
         failure_handling: FailureHandling = DEFAULT_FAILURE_HANDLING,
-    ) -> WriteResponse:
-        ...  # pragma: no cover
+    ) -> WriteResponse: ...  # pragma: no cover