From da3663a52fc0d75c59404fab81820a59f191985b Mon Sep 17 00:00:00 2001 From: Matthew Galbraith Date: Fri, 13 Dec 2024 07:14:07 -0600 Subject: [PATCH 1/2] Add DopplerConfiguration to access Doppler secrets as a Configuration. See: https://www.doppler.com/ Update pyproject to includes doppler in cloud feature. Update README to include DopplerConfiguration. Add python 3.13 to supported environments. --- README.md | 9 ++ pyproject.toml | 8 +- src/config/contrib/doppler.py | 184 ++++++++++++++++++++++++++++ tests/contrib/test_doppler.py | 223 ++++++++++++++++++++++++++++++++++ 4 files changed, 422 insertions(+), 2 deletions(-) create mode 100644 src/config/contrib/doppler.py create mode 100644 tests/contrib/test_doppler.py diff --git a/README.md b/README.md index cc152aa..0a07121 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ The `python-configuration` library supports the following configuration formats - AWS Secrets Manager credentials: requires `boto3` - GCP Secret Manager credentials: requires `google-cloud-secret-manager` - Hashicorp Vault credentials: requires `hvac` + - Doppler secrets: requires `doppler` ## Installing @@ -376,6 +377,14 @@ The `config.contrib` package contains extra implementations of the `Configuratio pip install python-configuration[vault] ``` +* `DopplerConfiguration` in `config.contrib.doppler`, produces a + `Configuration`-compatible instance connected to a Doppler project config. To install + the needed dependencies execute: + + ```shell + pip install python-configuration[doppler] + ``` + ## Features * Load multiple configuration types diff --git a/pyproject.toml b/pyproject.toml index 564858e..876cd5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ Source = "https://github.com/tr11/python-configuration" # cloud aws = ["boto3>=1.28.20"] azure = ["azure-keyvault>=4.2.0", "azure-identity"] +doppler = ["doppler-sdk>=1.3.0"] gcp = ["google-cloud-secret-manager>=2.16.3"] vault = ["hvac>=1.1.1"] # file formats @@ -37,7 +38,7 @@ yaml = ["pyyaml>=6.0"] # utilities validation = ["jsonschema>=4.21.1"] # groups -cloud = ["python-configuration[aws,azure,gcp,vault]"] +cloud = ["python-configuration[aws,azure,doppler,gcp,vault]"] file-formats = ["python-configuration[toml,yaml]"] [tool.hatch.version] @@ -91,7 +92,7 @@ description = "Testing Environment to run all\nthe tests across different\nPytho template = "test" [[tool.hatch.envs.testing.matrix]] -python = ["3.8", "3.9", "3.10", "3.11", "3.12"] +python = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] [tool.hatch.envs.dev] template = "test" @@ -149,6 +150,9 @@ module = [ 'jsonschema', 'jsonschema.exceptions', 'azure.identity', + 'dopplersdk', + 'dopplersdk.models', + 'dopplersdk.services.secrets', ] ignore_missing_imports = true diff --git a/src/config/contrib/doppler.py b/src/config/contrib/doppler.py new file mode 100644 index 0000000..a693c17 --- /dev/null +++ b/src/config/contrib/doppler.py @@ -0,0 +1,184 @@ +r""" +Configuration object connected to Doppler config. + +Copyright (c) 2024 Matthew Galbraith + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +# ruff: noqa: I001 + +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Iterable, List, Union, Optional + +from dopplersdk.models import SecretsGetResponse, SecretsListResponse +from dopplersdk.services.secrets import Secrets + +from .. import Configuration + + +class DopplerConfiguration(Configuration): + r"""Configuration class for use with Doppler, https://www.doppler.com/.""" + _doppler_vars: List[str] = [ + "DOPPLER_CONFIG", + "DOPPLER_ENVIRONMENT", + "DOPPLER_PROJECT", + ] + + def __init__( + self, + access_token: str, + project: str, + config: str, + secrets: Optional[Union[str, Iterable[str], None]] = None, + cache_expiration: Optional[Union[float, int]] = 5 * 60, + suppress_exceptions: bool = True, + **kwargs: Dict[str, Any], + ): + r""" + Construct a Configuration subclass for Doppler. + + Parameters: + access_token: User or Service token to access Doppler. + project: Doppler project name. Unused if access_token is a service token. + config: Doppler config name. Unused if access_token is a service token. + secrets: Either a CSV string or a list of secret names to include. If None, all + secrets for the config will be included. + cache_expiration: Cache expiration (in seconds) + suppress_exceptions: Suppress exceptions raised by calls to the Doppler service. + """ + # Doppler-specific fields + self._doppler_client = DopplerConfiguration._get_doppler_secrets_client( + access_token=access_token, + ) + self._doppler_project = project + self._doppler_config_name = config + self._requested_secrets: List[str] + if isinstance(secrets, str): + self._requested_secrets = secrets.split(",") + else: + self._requested_secrets = list(secrets or []) + self._suppress_exceptions = suppress_exceptions + + # Cache fields + self._cache_duration: Optional[timedelta] = None + self._cache_expiration: Optional[datetime] = None + if cache_expiration: + self._cache_duration = timedelta(seconds=float(cache_expiration)) + self._cache: Optional[Dict[str, Any]] = None + + # call the super init with config_ == {} since we've implemented + # _config as a property + super().__init__({}, **kwargs) # type: ignore[arg-type] + + # perform initial read of the Doppler config and cache the results + self.reload() + + @staticmethod + def _get_doppler_secrets_client(access_token: str) -> Secrets: # type: ignore[misc] + r"""Wrap Doppler Secrets client constructor to enable mock testing.""" + return Secrets(access_token=access_token) # pragma: no cover + + def _reset_expiration(self) -> None: + if self._cache_duration: + expire_at = datetime.now(tz=timezone.utc) + self._cache_duration + self._cache_expiration = expire_at + + def _is_cache_expired(self) -> bool: + if self._cache_expiration is None: + return False + return datetime.now(tz=timezone.utc) >= self._cache_expiration + + def __repr__(self) -> str: + r"""Construct repr value from class name and configuration values.""" + class_name = self.__class__.__name__ + project_name = self._doppler_project or "NONE" + config_name = self._doppler_config_name or "NONE" + return f"<{class_name}: {project_name} | {config_name}>" + + def _get_doppler_keys(self) -> Iterable[str]: + r"""List all keys available in the Doppler config.""" + parameters: Dict[str, Any] = { + "config": self._doppler_config_name, + "project": self._doppler_project, + } + if self._requested_secrets: + parameters["secrets"] = ",".join(self._requested_secrets) + response: SecretsListResponse = self._doppler_client.list( + **parameters, + ) + _keys: List[str] = [] + for secret_name in response.secrets: + if ( + secret_name in self._requested_secrets + or len(self._requested_secrets or []) == 0 + ): + _keys.append(secret_name) + return _keys + + def _get_doppler_value(self, item: str) -> Any: + r"""Get single value from the Doppler config.""" + value: Any = None + try: + response: SecretsGetResponse = self._doppler_client.get( + name=item, + project=self._doppler_project, + config=self._doppler_config_name, + ) + value = response.value.get("computed") + except Exception: # pragma: no cover + if not self._suppress_exceptions: + raise + + return value + + def _get_doppler_config_values(self) -> Dict[str, Any]: + r"""Get all values from Doppler config.""" + config_values: Dict[str, Any] = {} + try: + for key in self._get_doppler_keys(): + config_values[key] = self._get_doppler_value(key) + except TypeError: # pragma: no cover + pass + except Exception: # pragma: no cover + if not self._suppress_exceptions: + raise + return config_values + + @property + def _config(self) -> Dict[str, Any]: # type: ignore[misc] + r"""Override Configuration._config to enable cache management.""" + if self._cache is None or self._is_cache_expired(): + self.reload() + return (self._cache or {}).copy() + + @_config.setter + def _config(self, value: Any) -> None: + r"""Ignore attempts to set _config to override related base-class behaviors.""" + return + + def reload(self) -> None: + r"""Remove cached values and requery the Doppler service.""" + config_values = self._get_doppler_config_values() + self._reset_expiration() + self._cache = self._flatten_dict(config_values) + + def as_dict(self) -> Dict[str, Any]: + r"""Return a copy of internal the dictionary.""" + return self._config.copy() diff --git a/tests/contrib/test_doppler.py b/tests/contrib/test_doppler.py new file mode 100644 index 0000000..1807ceb --- /dev/null +++ b/tests/contrib/test_doppler.py @@ -0,0 +1,223 @@ +r""" +Tests for Doppler secret service integration. + +Copyright (c) 2024 Matthew Galbraith + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +# ruff: noqa: D103, D107, I001 + +from datetime import timedelta +from time import sleep +from typing import List, Dict, Any, Optional, Set, Union +from unittest.mock import MagicMock + +import pytest + +try: + import dopplersdk as doppler + from config.contrib.doppler import DopplerConfiguration +except ImportError: # pragma: no cover + doppler = None + +ACCESS_TOKEN = "dp.st.dev.MockTokenForUnitTestsXXXXXXXXXXXXXXXXXXXXXX" + +DOPPLER_PROJECT_CONFIG: Dict[str, str] = { + "DOPPLER_PROJECT": "doppler-testing-mock", + "DOPPLER_ENVIRONMENT": "dev", + "DOPPLER_CONFIG": "dev_local", +} + +ENV_DATA: Dict[str, Any] = { + "VALUE_1": "A simple string value.", + "ANOTHER_VALUE": "Something else.", +} + +REQUESTED_SECRETS: List[str] = [ + "VALUE_1", +] + +SECRET_LISTS: Dict[str, Optional[Union[str, List[str]]]] = { + "none": None, + "string": "VALUE_1", + "string-list": ["VALUE_1"], + "string-list-multi": ["VALUE_1", "ANOTHER_VALUE"], + "string-csv": "VALUE_1,ANOTHER_VALUE", +} + +class DopplerSecretServiceMock: + r"""Mock the Doppler `Service` class.""" + class SecretsListResponseMock: + r"""Mock the response type from call to list secrets.""" + def __init__(self, data: dict): + self._data = data + @property + def secrets(self) -> List[str]: # noqa: D102 + return list(self._data.keys()) if self._data else [] + + class SecretsGetResponseMock: + r"""Mock the response type from call to get a secret.""" + def __init__(self, value: Any): + self._value = value + @property + def value(self) -> Dict[str, str]: # noqa: D102 + return { + "computed": self._value, + } + + def __init__(self) -> None: + self._project: str = DOPPLER_PROJECT_CONFIG["DOPPLER_PROJECT"] + self._env: str = DOPPLER_PROJECT_CONFIG["DOPPLER_ENVIRONMENT"] + self._config: str = DOPPLER_PROJECT_CONFIG["DOPPLER_CONFIG"] + self._data: Dict[str, Any] = DOPPLER_PROJECT_CONFIG.copy() + self._data.update(ENV_DATA) + + def get(self, name: str, project: str, config: str) -> SecretsGetResponseMock: # noqa: D102 + return DopplerSecretServiceMock.SecretsGetResponseMock(self._data.get(name)) + + def list( # noqa: D102 + self, + project: str, + config: str, + secrets: Optional[str] = None, + ) -> SecretsListResponseMock: + if secrets is None: + return DopplerSecretServiceMock.SecretsListResponseMock(self._data) + key_filter: Set[str] = set(secrets.split(",")) + key_filter.update(DOPPLER_PROJECT_CONFIG.keys()) + filtered_data: Dict[str, Any] = { + k: v + for k, v in self._data.items() + if k in key_filter + } + return DopplerSecretServiceMock.SecretsListResponseMock( + filtered_data, + ) + +@pytest.mark.skipif("doppler is None") +def apply_doppler_config_mocks() -> None: + mock_class = DopplerConfiguration + def mock_get_doppler_secrets_client(access_token: str) -> DopplerSecretServiceMock: + return DopplerSecretServiceMock() + mock_class._get_doppler_secrets_client = mock_get_doppler_secrets_client # type: ignore[method-assign] + +@pytest.mark.skipif("doppler is None") +@pytest.mark.parametrize("secrets", SECRET_LISTS.values()) +def test_constructor(secrets: Optional[Union[str, List[str]]]) -> None: + apply_doppler_config_mocks() + + parameters: Dict[str, Any] = {} + parameters.update({"secrets": secrets}) + instance: DopplerConfiguration = DopplerConfiguration( + access_token=ACCESS_TOKEN, + project=DOPPLER_PROJECT_CONFIG["DOPPLER_PROJECT"], + config=DOPPLER_PROJECT_CONFIG["DOPPLER_CONFIG"], + **parameters, + ) + assert isinstance(instance, DopplerConfiguration) + + # if secrets is None: + # assert len(instance.as_dict()) > 0 + + def check_moar_things() -> None: + requested_keys: Set[str] = set(instance._requested_secrets) + env_keys: Set[str] = set(ENV_DATA.keys()) + doppler_keys: Set[str] = set(DOPPLER_PROJECT_CONFIG.keys()) + all_keys = env_keys.union(doppler_keys) + keys: Set[str] = set(instance.keys()) # type: ignore[arg-type] + + all_values: Dict[str, Any] = ENV_DATA.copy() + all_values.update(DOPPLER_PROJECT_CONFIG.copy()) + selected_values: Dict[str, Any] = { + k: v + for (k, v) in all_values.items() + if ( + len(requested_keys) == 0 + or k in requested_keys + ) + } + + # Check for expected keys + if len(requested_keys) == 0: + assert keys == all_keys + else: + assert len(keys.symmetric_difference(requested_keys)) == 0 + + # check for correct dictionary values + dc_dict: Dict[str, Any] = instance.as_dict() + if len(requested_keys) == 0: + # All keys from the env and doppler vars should be present in the dict + assert dc_dict == all_values + else: + assert dc_dict == selected_values + + check_moar_things() + + +@pytest.fixture +@pytest.mark.skipif("doppler is None") +def doppler_config() -> "DopplerConfiguration": # type: ignore[misc] + apply_doppler_config_mocks() + + dc: DopplerConfiguration = DopplerConfiguration( + access_token=ACCESS_TOKEN, + project=DOPPLER_PROJECT_CONFIG["DOPPLER_PROJECT"], + config=DOPPLER_PROJECT_CONFIG["DOPPLER_CONFIG"], + secrets=REQUESTED_SECRETS, + ) + dc._doppler_client = DopplerSecretServiceMock() + yield dc + +@pytest.mark.skipif("doppler is None") +def test_cache_expiration(doppler_config) -> None: # type: ignore[misc, no-untyped-def] + variable_key_name: str = set(ENV_DATA.keys()).pop() + + # set a short expiration so we don't wait too long for the test + short_expiration_time: float = .1 + doppler_config._cache_duration = timedelta(seconds=short_expiration_time) + doppler_config.reload() + + doppler_config._get_doppler_config_values = MagicMock("_get_doppler_config_values") + + doppler_config.get(variable_key_name) + doppler_config._get_doppler_config_values.assert_not_called() + + sleep(short_expiration_time) + doppler_config.get(variable_key_name) + doppler_config._get_doppler_config_values.assert_called() + +@pytest.mark.skipif("doppler is None") +def test_expiration_duration_none() -> None: + apply_doppler_config_mocks() + dc: DopplerConfiguration = DopplerConfiguration( + access_token=ACCESS_TOKEN, + project=DOPPLER_PROJECT_CONFIG["DOPPLER_PROJECT"], + config=DOPPLER_PROJECT_CONFIG["DOPPLER_CONFIG"], + secrets=None, + cache_expiration=None, + ) + variable_key_name: str = set(ENV_DATA.keys()).pop() + variable_value: str = str(dc.get(variable_key_name, "")) + assert variable_value == ENV_DATA.get(variable_key_name) + +@pytest.mark.skipif("doppler is None") +def test_repr(doppler_config) -> None: # type: ignore[misc, no-untyped-def] + expected_repr = r"""""" + assert doppler_config.__repr__() == expected_repr From 5a7c661fc05265ed3e432f28f2b1c81122e6a066 Mon Sep 17 00:00:00 2001 From: Matthew Galbraith Date: Mon, 6 Jan 2025 09:59:24 -0600 Subject: [PATCH 2/2] Enable raising exceptions from Doppler service by default. --- pyproject.toml | 1 + src/config/contrib/doppler.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 876cd5e..57c402e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] diff --git a/src/config/contrib/doppler.py b/src/config/contrib/doppler.py index a693c17..84044b7 100644 --- a/src/config/contrib/doppler.py +++ b/src/config/contrib/doppler.py @@ -48,7 +48,7 @@ def __init__( config: str, secrets: Optional[Union[str, Iterable[str], None]] = None, cache_expiration: Optional[Union[float, int]] = 5 * 60, - suppress_exceptions: bool = True, + suppress_exceptions: bool = False, **kwargs: Dict[str, Any], ): r""" @@ -56,8 +56,8 @@ def __init__( Parameters: access_token: User or Service token to access Doppler. - project: Doppler project name. Unused if access_token is a service token. - config: Doppler config name. Unused if access_token is a service token. + project: Doppler project name. + config: Doppler config name. secrets: Either a CSV string or a list of secret names to include. If None, all secrets for the config will be included. cache_expiration: Cache expiration (in seconds)