From 66e0332df871bafaada14f11387722131262db14 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Wed, 22 Jan 2025 16:30:01 +0000 Subject: [PATCH 01/49] Add first version of the client auth for sdk --- config/__init__.py | 0 config/config.py | 42 ++++++++++++++++++++++++ datacosmos/client.py | 77 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+) create mode 100644 config/__init__.py create mode 100644 config/config.py create mode 100644 datacosmos/client.py diff --git a/config/__init__.py b/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/config.py b/config/config.py new file mode 100644 index 0000000..5eaea0e --- /dev/null +++ b/config/config.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import os +import yaml +from dataclasses import dataclass + + +@dataclass +class Config: + client_id: str + client_secret: str + token_url: str + audience: str + + @staticmethod + def from_yaml(file_path: str = "config/config.yaml") -> Config: + """ + Load configuration from a YAML file. + Defaults to 'config/config.yaml' unless otherwise specified. + """ + with open(file_path, "r") as f: + data = yaml.safe_load(f) + auth = data.get("auth", {}) + return Config( + client_id=auth["client-id"], + client_secret=auth["client-secret"], + token_url=auth["token-url"], + audience=auth["audience"], + ) + + @staticmethod + def from_env() -> Config: + """ + Load configuration from environment variables. + Raises an exception if any required variable is missing. + """ + return Config( + client_id=os.getenv("OC_AUTH_CLIENT_ID"), + client_secret=os.getenv("OC_AUTH_CLIENT_SECRET"), + token_url=os.getenv("OC_AUTH_TOKEN_URL"), + audience=os.getenv("OC_AUTH_AUDIENCE"), + ) diff --git a/datacosmos/client.py b/datacosmos/client.py new file mode 100644 index 0000000..b3bff4f --- /dev/null +++ b/datacosmos/client.py @@ -0,0 +1,77 @@ +import os + +from datetime import datetime, timedelta, timezone +from typing import Optional, Any +import requests +from requests_oauthlib import OAuth2Session +from oauthlib.oauth2 import BackendApplicationClient + +from config.config import Config + + +class DatacosmosClient: + def __init__(self, config: Optional[Config] = None, config_file: str = "config/config.yaml"): + self.config = config or self._load_config(config_file) + self.token = None + self.token_expiry = None + self._http_client = self._authenticate_and_initialize_client() + + def _load_config(self, config_file: str) -> Config: + if os.path.exists(config_file): + return Config.from_yaml(config_file) + return Config.from_env() + + def _authenticate_and_initialize_client(self) -> requests.Session: + client = BackendApplicationClient(client_id=self.config.client_id) + oauth_session = OAuth2Session(client=client) + + # Fetch the token using client credentials + token_response = oauth_session.fetch_token( + token_url=self.config.token_url, + client_id=self.config.client_id, + client_secret=self.config.client_secret, + audience=self.config.audience, + ) + + self.token = token_response["access_token"] + self.token_expiry = datetime.now(timezone.utc) + timedelta(seconds=token_response.get("expires_in", 3600)) + + # Initialize the HTTP session with the Authorization header + http_client = requests.Session() + http_client.headers.update({"Authorization": f"Bearer {self.token}"}) + + return http_client + + def _refresh_token_if_needed(self): + """ + Refreshes the token if it has expired. + """ + if not self.token or self.token_expiry <= datetime.now(timezone.utc): + self._http_client = self._authenticate_and_initialize_client() + + def get_http_client(self) -> requests.Session: + """ + Returns the authenticated HTTP client, refreshing the token if necessary. + """ + self._refresh_token_if_needed() + return self._http_client + + # Proxy HTTP methods to the underlying authenticated session + def request(self, method: str, url: str, *args: Any, **kwargs: Any) -> requests.Response: + """ + Proxy method to send HTTP requests using the authenticated session. + """ + self._refresh_token_if_needed() + return self._http_client.request(method, url, *args, **kwargs) + + def get(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: + return self.request("GET", url, *args, **kwargs) + + def post(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: + return self.request("POST", url, *args, **kwargs) + + def put(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: + return self.request("PUT", url, *args, **kwargs) + + def delete(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: + return self.request("DELETE", url, *args, **kwargs) From 21d0ee3f882cfb8d19e4a42f644c395cc55636ab Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Wed, 22 Jan 2025 17:01:20 +0000 Subject: [PATCH 02/49] Run ruff format and ruff check fix --- datacosmos/client.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/datacosmos/client.py b/datacosmos/client.py index b3bff4f..54eb279 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -10,7 +10,9 @@ class DatacosmosClient: - def __init__(self, config: Optional[Config] = None, config_file: str = "config/config.yaml"): + def __init__( + self, config: Optional[Config] = None, config_file: str = "config/config.yaml" + ): self.config = config or self._load_config(config_file) self.token = None self.token_expiry = None @@ -34,7 +36,9 @@ def _authenticate_and_initialize_client(self) -> requests.Session: ) self.token = token_response["access_token"] - self.token_expiry = datetime.now(timezone.utc) + timedelta(seconds=token_response.get("expires_in", 3600)) + self.token_expiry = datetime.now(timezone.utc) + timedelta( + seconds=token_response.get("expires_in", 3600) + ) # Initialize the HTTP session with the Authorization header http_client = requests.Session() @@ -57,7 +61,9 @@ def get_http_client(self) -> requests.Session: return self._http_client # Proxy HTTP methods to the underlying authenticated session - def request(self, method: str, url: str, *args: Any, **kwargs: Any) -> requests.Response: + def request( + self, method: str, url: str, *args: Any, **kwargs: Any + ) -> requests.Response: """ Proxy method to send HTTP requests using the authenticated session. """ From 1efe6ed81707ff53063a6560ed08a2d5fe8d9122 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Wed, 22 Jan 2025 17:34:15 +0000 Subject: [PATCH 03/49] Make some changes regarding code formatting --- .gitignore | 3 +++ config/__init__.py | 9 ++++++++ config/config.py | 25 +++++++++++++------- datacosmos/client.py | 40 +++++++++++++++++++++----------- pyproject.toml | 55 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_pass.py | 8 ++++--- 6 files changed, 116 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 3c19803..2e300c0 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,6 @@ dmypy.json # Cython debug symbols cython_debug/ + +# Ignore config.yaml +config/config.yaml diff --git a/config/__init__.py b/config/__init__.py index e69de29..e9778e9 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -0,0 +1,9 @@ +"""Configuration package for the Datacosmos SDK. + +This package includes modules for loading and managing authentication configurations. +""" + +# Expose Config class for easier imports +from .config import Config + +__all__ = ["Config"] diff --git a/config/config.py b/config/config.py index 5eaea0e..c3a6624 100644 --- a/config/config.py +++ b/config/config.py @@ -1,21 +1,30 @@ -from __future__ import annotations +"""Module for managing configuration settings for the Datacosmos SDK. + +Supports loading from YAML files and environment variables. +""" import os -import yaml from dataclasses import dataclass +import yaml + @dataclass class Config: + """Configuration for the Datacosmos SDK. + + Contains authentication details such as client ID, secret, token URL, and audience. + """ + client_id: str client_secret: str token_url: str audience: str @staticmethod - def from_yaml(file_path: str = "config/config.yaml") -> Config: - """ - Load configuration from a YAML file. + def from_yaml(file_path: str = "config/config.yaml") -> "Config": + """Load configuration from a YAML file. + Defaults to 'config/config.yaml' unless otherwise specified. """ with open(file_path, "r") as f: @@ -29,9 +38,9 @@ def from_yaml(file_path: str = "config/config.yaml") -> Config: ) @staticmethod - def from_env() -> Config: - """ - Load configuration from environment variables. + def from_env() -> "Config": + """Load configuration from environment variables. + Raises an exception if any required variable is missing. """ return Config( diff --git a/datacosmos/client.py b/datacosmos/client.py index 54eb279..858568a 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -1,29 +1,46 @@ -import os +"""Datacosmos client for interacting with the Datacosmos API. + +Provides an authenticated HTTP client and convenience methods for HTTP requests. +""" +import os from datetime import datetime, timedelta, timezone -from typing import Optional, Any +from typing import Any, Optional + import requests -from requests_oauthlib import OAuth2Session from oauthlib.oauth2 import BackendApplicationClient +from requests_oauthlib import OAuth2Session from config.config import Config class DatacosmosClient: + """DatacosmosClient handles authenticated interactions with the Datacosmos API. + + Automatically manages token refreshing and provides HTTP convenience methods. + """ + def __init__( self, config: Optional[Config] = None, config_file: str = "config/config.yaml" ): + """Initialize the DatacosmosClient. + + If no configuration is provided, it will load from the specified YAML file + or fall back to environment variables. + """ self.config = config or self._load_config(config_file) self.token = None self.token_expiry = None self._http_client = self._authenticate_and_initialize_client() def _load_config(self, config_file: str) -> Config: + """Load configuration from the YAML file. Fall back to environment variables if the file is missing.""" if os.path.exists(config_file): return Config.from_yaml(config_file) return Config.from_env() def _authenticate_and_initialize_client(self) -> requests.Session: + """Authenticate and initialize the HTTP client with a valid token.""" client = BackendApplicationClient(client_id=self.config.client_id) oauth_session = OAuth2Session(client=client) @@ -47,37 +64,34 @@ def _authenticate_and_initialize_client(self) -> requests.Session: return http_client def _refresh_token_if_needed(self): - """ - Refreshes the token if it has expired. - """ + """Refresh the token if it has expired.""" if not self.token or self.token_expiry <= datetime.now(timezone.utc): self._http_client = self._authenticate_and_initialize_client() def get_http_client(self) -> requests.Session: - """ - Returns the authenticated HTTP client, refreshing the token if necessary. - """ + """Return the authenticated HTTP client, refreshing the token if necessary.""" self._refresh_token_if_needed() return self._http_client - # Proxy HTTP methods to the underlying authenticated session def request( self, method: str, url: str, *args: Any, **kwargs: Any ) -> requests.Response: - """ - Proxy method to send HTTP requests using the authenticated session. - """ + """Send an HTTP request using the authenticated session.""" self._refresh_token_if_needed() return self._http_client.request(method, url, *args, **kwargs) def get(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: + """Send a GET request using the authenticated session.""" return self.request("GET", url, *args, **kwargs) def post(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: + """Send a POST request using the authenticated session.""" return self.request("POST", url, *args, **kwargs) def put(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: + """Send a PUT request using the authenticated session.""" return self.request("PUT", url, *args, **kwargs) def delete(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: + """Send a DELETE request using the authenticated session.""" return self.request("DELETE", url, *args, **kwargs) diff --git a/pyproject.toml b/pyproject.toml index b06bf42..1042a6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ test = [ "isort==5.11.4", "pydocstyle==6.1.1", "flake8-cognitive-complexity==0.1.0", + "ruff==0.0.286" ] [tool.setuptools.packages] @@ -34,3 +35,57 @@ find = {} [tool.pydocstyle] convention = "google" + +[tool.ruff] +fix = true +line-length = 88 +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +[tool.ruff.lint] +extend-select = ["I", "SLF", "F", "C90", "BLE", "B", "ARG", "ERA"] +ignore = [] +fixable = ["ALL"] +unfixable = [] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" +docstring-code-format = false + +[tool.ruff.lint.mccabe] +max-complexity = 10 + +[tool.black] +line-length = 88 + +[tool.isort] +profile = "black" diff --git a/tests/test_pass.py b/tests/test_pass.py index 32c6ad8..228fe40 100644 --- a/tests/test_pass.py +++ b/tests/test_pass.py @@ -1,7 +1,9 @@ -"""An example test to check pytest setup.""" +"""Test suite for basic functionality and CI setup.""" class TestPass: + """A simple test class to validate the CI pipeline setup.""" + def test_pass(self): - """A passing test, to check the pytest CI setup.""" - pass + """A passing test to ensure the CI pipeline is functional.""" + assert True From 0f1479fc87aa8e479d802c84489e7a54abba2d56 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Thu, 23 Jan 2025 14:46:11 +0000 Subject: [PATCH 04/49] Add some unit tests to datacosmos client --- datacosmos/client.py | 138 ++++++++++++------ .../client/test_client_authentication.py | 53 +++++++ .../client/test_client_initialization.py | 28 ++++ .../client/test_client_token_refreshing.py | 50 +++++++ 4 files changed, 222 insertions(+), 47 deletions(-) create mode 100644 tests/unit/datacosmos/client/test_client_authentication.py create mode 100644 tests/unit/datacosmos/client/test_client_initialization.py create mode 100644 tests/unit/datacosmos/client/test_client_token_refreshing.py diff --git a/datacosmos/client.py b/datacosmos/client.py index 858568a..d9a68a3 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -1,97 +1,141 @@ -"""Datacosmos client for interacting with the Datacosmos API. - -Provides an authenticated HTTP client and convenience methods for HTTP requests. -""" - +import logging import os from datetime import datetime, timedelta, timezone -from typing import Any, Optional +from typing import Optional, Any import requests -from oauthlib.oauth2 import BackendApplicationClient from requests_oauthlib import OAuth2Session +from oauthlib.oauth2 import BackendApplicationClient +from requests.exceptions import RequestException from config.config import Config class DatacosmosClient: - """DatacosmosClient handles authenticated interactions with the Datacosmos API. - + """ + DatacosmosClient handles authenticated interactions with the Datacosmos API. Automatically manages token refreshing and provides HTTP convenience methods. """ def __init__( self, config: Optional[Config] = None, config_file: str = "config/config.yaml" ): - """Initialize the DatacosmosClient. + """ + Initialize the DatacosmosClient. If no configuration is provided, it will load from the specified YAML file or fall back to environment variables. """ + self.logger = logging.getLogger(__name__) + self.logger.setLevel(logging.INFO) + self.config = config or self._load_config(config_file) self.token = None self.token_expiry = None self._http_client = self._authenticate_and_initialize_client() def _load_config(self, config_file: str) -> Config: - """Load configuration from the YAML file. Fall back to environment variables if the file is missing.""" - if os.path.exists(config_file): - return Config.from_yaml(config_file) - return Config.from_env() + """ + Load configuration from the YAML file. Fall back to environment variables if the file is missing. + """ + try: + if os.path.exists(config_file): + self.logger.info(f"Loading configuration from {config_file}") + return Config.from_yaml(config_file) + self.logger.info("Loading configuration from environment variables") + return Config.from_env() + except Exception as e: + self.logger.error(f"Failed to load configuration: {e}") + raise def _authenticate_and_initialize_client(self) -> requests.Session: - """Authenticate and initialize the HTTP client with a valid token.""" - client = BackendApplicationClient(client_id=self.config.client_id) - oauth_session = OAuth2Session(client=client) - - # Fetch the token using client credentials - token_response = oauth_session.fetch_token( - token_url=self.config.token_url, - client_id=self.config.client_id, - client_secret=self.config.client_secret, - audience=self.config.audience, - ) - - self.token = token_response["access_token"] - self.token_expiry = datetime.now(timezone.utc) + timedelta( - seconds=token_response.get("expires_in", 3600) - ) - - # Initialize the HTTP session with the Authorization header - http_client = requests.Session() - http_client.headers.update({"Authorization": f"Bearer {self.token}"}) - - return http_client + """ + Authenticate and initialize the HTTP client with a valid token. + """ + try: + self.logger.info("Authenticating with the token endpoint") + client = BackendApplicationClient(client_id=self.config.client_id) + oauth_session = OAuth2Session(client=client) + + # Fetch the token using client credentials + token_response = oauth_session.fetch_token( + token_url=self.config.token_url, + client_id=self.config.client_id, + client_secret=self.config.client_secret, + audience=self.config.audience, + ) + + self.token = token_response["access_token"] + self.token_expiry = datetime.now(timezone.utc) + timedelta( + seconds=token_response.get("expires_in", 3600) + ) + self.logger.info("Authentication successful, token obtained") + + # Initialize the HTTP session with the Authorization header + http_client = requests.Session() + http_client.headers.update({"Authorization": f"Bearer {self.token}"}) + return http_client + except RequestException as e: + self.logger.error(f"Request failed during authentication: {e}") + raise + except Exception as e: + self.logger.error(f"Unexpected error during authentication: {e}") + raise def _refresh_token_if_needed(self): - """Refresh the token if it has expired.""" + """ + Refresh the token if it has expired. + """ if not self.token or self.token_expiry <= datetime.now(timezone.utc): + self.logger.info("Token expired or missing, refreshing token") self._http_client = self._authenticate_and_initialize_client() def get_http_client(self) -> requests.Session: - """Return the authenticated HTTP client, refreshing the token if necessary.""" + """ + Return the authenticated HTTP client, refreshing the token if necessary. + """ self._refresh_token_if_needed() return self._http_client - def request( - self, method: str, url: str, *args: Any, **kwargs: Any - ) -> requests.Response: - """Send an HTTP request using the authenticated session.""" + def request(self, method: str, url: str, *args: Any, **kwargs: Any) -> requests.Response: + """ + Send an HTTP request using the authenticated session. + Logs request and response details. + """ self._refresh_token_if_needed() - return self._http_client.request(method, url, *args, **kwargs) + try: + self.logger.info(f"Making {method.upper()} request to {url}") + response = self._http_client.request(method, url, *args, **kwargs) + response.raise_for_status() + self.logger.info(f"Request to {url} succeeded with status {response.status_code}") + return response + except RequestException as e: + self.logger.error(f"HTTP request failed: {e}") + raise + except Exception as e: + self.logger.error(f"Unexpected error during HTTP request: {e}") + raise def get(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: - """Send a GET request using the authenticated session.""" + """ + Send a GET request using the authenticated session. + """ return self.request("GET", url, *args, **kwargs) def post(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: - """Send a POST request using the authenticated session.""" + """ + Send a POST request using the authenticated session. + """ return self.request("POST", url, *args, **kwargs) def put(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: - """Send a PUT request using the authenticated session.""" + """ + Send a PUT request using the authenticated session. + """ return self.request("PUT", url, *args, **kwargs) def delete(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: - """Send a DELETE request using the authenticated session.""" + """ + Send a DELETE request using the authenticated session. + """ return self.request("DELETE", url, *args, **kwargs) diff --git a/tests/unit/datacosmos/client/test_client_authentication.py b/tests/unit/datacosmos/client/test_client_authentication.py new file mode 100644 index 0000000..3aec902 --- /dev/null +++ b/tests/unit/datacosmos/client/test_client_authentication.py @@ -0,0 +1,53 @@ +from unittest.mock import patch, MagicMock +from datacosmos.client import DatacosmosClient +from config.config import Config + + +@patch("datacosmos.client.OAuth2Session.fetch_token") +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client", autospec=True) +def test_client_authentication(mock_auth_client, mock_fetch_token): + """ + Test that the client correctly fetches a token during authentication. + """ + # Mock the token response from OAuth2Session + mock_fetch_token.return_value = { + "access_token": "mock-access-token", + "expires_in": 3600, + } + + # Simulate _authenticate_and_initialize_client calling fetch_token + def mock_authenticate_and_initialize_client(self): + # Call the real fetch_token (simulated by the mock) + token_response = mock_fetch_token( + token_url=self.config.token_url, + client_id=self.config.client_id, + client_secret=self.config.client_secret, + audience=self.config.audience, + ) + self.token = token_response["access_token"] + self.token_expiry = "mock-expiry" + + # Attach the side effect to the mock + mock_auth_client.side_effect = mock_authenticate_and_initialize_client + + # Create a mock configuration + config = Config( + client_id="test-client-id", + client_secret="test-client-secret", + token_url="https://mock.token.url/oauth/token", + audience="https://mock.audience", + ) + + # Initialize the client + client = DatacosmosClient(config=config) + + # Assertions + assert client.token == "mock-access-token" + assert client.token_expiry == "mock-expiry" + mock_fetch_token.assert_called_once_with( + token_url="https://mock.token.url/oauth/token", + client_id="test-client-id", + client_secret="test-client-secret", + audience="https://mock.audience", + ) + mock_auth_client.assert_called_once_with(client) diff --git a/tests/unit/datacosmos/client/test_client_initialization.py b/tests/unit/datacosmos/client/test_client_initialization.py new file mode 100644 index 0000000..e8ee89c --- /dev/null +++ b/tests/unit/datacosmos/client/test_client_initialization.py @@ -0,0 +1,28 @@ +from unittest.mock import patch, MagicMock +from datacosmos.client import DatacosmosClient +from config.config import Config + + +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +@patch("os.path.exists", return_value=False) +@patch("config.Config.from_env") +def test_client_initialization(mock_from_env, mock_exists, mock_auth_client): + """ + Test that the client initializes correctly with environment variables and mocks the HTTP client. + """ + mock_config = Config( + client_id="test-client-id", + client_secret="test-client-secret", + token_url="https://mock.token.url/oauth/token", + audience="https://mock.audience", + ) + mock_from_env.return_value = mock_config + mock_auth_client.return_value = MagicMock() # Mock the HTTP client + + client = DatacosmosClient() + + assert client.config == mock_config + assert client._http_client is not None # Ensure the HTTP client is mocked + mock_exists.assert_called_once_with("config/config.yaml") + mock_from_env.assert_called_once() + mock_auth_client.assert_called_once() diff --git a/tests/unit/datacosmos/client/test_client_token_refreshing.py b/tests/unit/datacosmos/client/test_client_token_refreshing.py new file mode 100644 index 0000000..5632d85 --- /dev/null +++ b/tests/unit/datacosmos/client/test_client_token_refreshing.py @@ -0,0 +1,50 @@ +from unittest.mock import patch, MagicMock +from datetime import datetime, timedelta, timezone +from datacosmos.client import DatacosmosClient +from config.config import Config + + +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +def test_token_refreshing(mock_auth_client): + """ + Test that the client refreshes the token when it expires. + """ + # Mock the HTTP client returned by _authenticate_and_initialize_client + mock_http_client = MagicMock() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"message": "success"} + mock_http_client.request.return_value = mock_response + mock_auth_client.return_value = mock_http_client + + config = Config( + client_id="test-client-id", + client_secret="test-client-secret", + token_url="https://mock.token.url/oauth/token", + audience="https://mock.audience", + ) + + # Initialize the client (first call to _authenticate_and_initialize_client) + client = DatacosmosClient(config=config) + + # Simulate expired token + client.token_expiry = datetime.now(timezone.utc) - timedelta(seconds=1) + + # Make a GET request (should trigger token refresh) + response = client.get("https://mock.api/some-endpoint", headers={"Authorization": f"Bearer {client.token}"}) + + # Assertions + assert response.status_code == 200 + assert response.json() == {"message": "success"} + + # Verify _authenticate_and_initialize_client was called twice: + # 1. During initialization + # 2. During token refresh + assert mock_auth_client.call_count == 2 + + # Verify the request was made correctly + mock_http_client.request.assert_called_once_with( + "GET", + "https://mock.api/some-endpoint", + headers={"Authorization": f"Bearer {client.token}"}, + ) From df656da107c6602037d482c6a5388fe570effa1a Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Fri, 24 Jan 2025 12:37:27 +0000 Subject: [PATCH 05/49] Add more unit tests --- .gitignore | 3 ++ pyproject.toml | 16 ++++++++ .../client/test_client_delete_request.py | 33 ++++++++++++++++ .../client/test_client_get_request.py | 35 +++++++++++++++++ .../client/test_client_post_request.py | 38 +++++++++++++++++++ .../client/test_client_put_request.py | 38 +++++++++++++++++++ .../client/test_client_token_refreshing.py | 7 ++-- 7 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 tests/unit/datacosmos/client/test_client_delete_request.py create mode 100644 tests/unit/datacosmos/client/test_client_get_request.py create mode 100644 tests/unit/datacosmos/client/test_client_post_request.py create mode 100644 tests/unit/datacosmos/client/test_client_put_request.py diff --git a/.gitignore b/.gitignore index 2e300c0..534e783 100644 --- a/.gitignore +++ b/.gitignore @@ -135,3 +135,6 @@ cython_debug/ # Ignore config.yaml config/config.yaml + +# Ignore .vscode +.vscode/ diff --git a/pyproject.toml b/pyproject.toml index 1042a6e..2edef52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,8 @@ test = [ find = {} [tool.bandit] +skips = ["B101"] # Example: ignore assert statement usage if needed +targets = ["datacosmos"] # Adjust to point to your project directory [tool.pydocstyle] convention = "google" @@ -58,6 +60,7 @@ exclude = [ ".tox", ".venv", ".vscode", + "__pycache__", "__pypackages__", "_build", "buck-out", @@ -89,3 +92,16 @@ line-length = 88 [tool.isort] profile = "black" + +[tool.flake8] +max-line-length = 88 +exclude = [ + ".venv", + "__pycache__", + "build", + "dist", +] +ignore = [ + "E203", # Whitespace before ':', handled by Black + "W503", # Line break before binary operator +] diff --git a/tests/unit/datacosmos/client/test_client_delete_request.py b/tests/unit/datacosmos/client/test_client_delete_request.py new file mode 100644 index 0000000..030ee80 --- /dev/null +++ b/tests/unit/datacosmos/client/test_client_delete_request.py @@ -0,0 +1,33 @@ +from unittest.mock import patch, MagicMock +from datacosmos.client import DatacosmosClient +from config.config import Config + + +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +def test_delete_request(mock_auth_client): + """ + Test that the client performs a DELETE request correctly. + """ + # Mock the HTTP client + mock_http_client = MagicMock() + mock_response = MagicMock() + mock_response.status_code = 204 + mock_http_client.request.return_value = mock_response + mock_auth_client.return_value = mock_http_client + + config = Config( + client_id="test-client-id", + client_secret="test-client-secret", + token_url="https://mock.token.url/oauth/token", + audience="https://mock.audience", + ) + client = DatacosmosClient(config=config) + response = client.delete("https://mock.api/some-endpoint") + + # Assertions + assert response.status_code == 204 + mock_http_client.request.assert_called_once_with( + "DELETE", + "https://mock.api/some-endpoint" + ) + mock_auth_client.call_count == 2 diff --git a/tests/unit/datacosmos/client/test_client_get_request.py b/tests/unit/datacosmos/client/test_client_get_request.py new file mode 100644 index 0000000..2dc7b31 --- /dev/null +++ b/tests/unit/datacosmos/client/test_client_get_request.py @@ -0,0 +1,35 @@ +from unittest.mock import patch, MagicMock +from datacosmos.client import DatacosmosClient +from config.config import Config + + +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +def test_client_get_request(mock_auth_client): + """ + Test that the client performs a GET request correctly. + """ + # Mock the HTTP client + mock_http_client = MagicMock() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"message": "success"} + mock_http_client.request.return_value = mock_response + mock_auth_client.return_value = mock_http_client + + config = Config( + client_id="test-client-id", + client_secret="test-client-secret", + token_url="https://mock.token.url/oauth/token", + audience="https://mock.audience", + ) + client = DatacosmosClient(config=config) + response = client.get("https://mock.api/some-endpoint") + + # Assertions + assert response.status_code == 200 + assert response.json() == {"message": "success"} + mock_http_client.request.assert_called_once_with( + "GET", + "https://mock.api/some-endpoint" + ) + mock_auth_client.call_count == 2 diff --git a/tests/unit/datacosmos/client/test_client_post_request.py b/tests/unit/datacosmos/client/test_client_post_request.py new file mode 100644 index 0000000..ac6a9b4 --- /dev/null +++ b/tests/unit/datacosmos/client/test_client_post_request.py @@ -0,0 +1,38 @@ +from unittest.mock import patch, MagicMock +from datacosmos.client import DatacosmosClient +from config.config import Config + + +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +def test_post_request(mock_auth_client): + """ + Test that the client performs a POST request correctly. + """ + # Mock the HTTP client + mock_http_client = MagicMock() + mock_response = MagicMock() + mock_response.status_code = 201 + mock_response.json.return_value = {"message": "created"} + mock_http_client.request.return_value = mock_response + mock_auth_client.return_value = mock_http_client + + config = Config( + client_id="test-client-id", + client_secret="test-client-secret", + token_url="https://mock.token.url/oauth/token", + audience="https://mock.audience", + ) + client = DatacosmosClient(config=config) + response = client.post( + "https://mock.api/some-endpoint", json={"key": "value"} + ) + + # Assertions + assert response.status_code == 201 + assert response.json() == {"message": "created"} + mock_http_client.request.assert_called_once_with( + "POST", + "https://mock.api/some-endpoint", + json={"key": "value"} + ) + mock_auth_client.call_count == 2 diff --git a/tests/unit/datacosmos/client/test_client_put_request.py b/tests/unit/datacosmos/client/test_client_put_request.py new file mode 100644 index 0000000..140260b --- /dev/null +++ b/tests/unit/datacosmos/client/test_client_put_request.py @@ -0,0 +1,38 @@ +from unittest.mock import patch, MagicMock +from datacosmos.client import DatacosmosClient +from config.config import Config + + +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +def test_put_request(mock_auth_client): + """ + Test that the client performs a PUT request correctly. + """ + # Mock the HTTP client + mock_http_client = MagicMock() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"message": "updated"} + mock_http_client.request.return_value = mock_response + mock_auth_client.return_value = mock_http_client + + config = Config( + client_id="test-client-id", + client_secret="test-client-secret", + token_url="https://mock.token.url/oauth/token", + audience="https://mock.audience", + ) + client = DatacosmosClient(config=config) + response = client.put( + "https://mock.api/some-endpoint", json={"key": "updated-value"} + ) + + # Assertions + assert response.status_code == 200 + assert response.json() == {"message": "updated"} + mock_http_client.request.assert_called_once_with( + "PUT", + "https://mock.api/some-endpoint", + json={"key": "updated-value"} + ) + mock_auth_client.call_count == 2 diff --git a/tests/unit/datacosmos/client/test_client_token_refreshing.py b/tests/unit/datacosmos/client/test_client_token_refreshing.py index 5632d85..614d735 100644 --- a/tests/unit/datacosmos/client/test_client_token_refreshing.py +++ b/tests/unit/datacosmos/client/test_client_token_refreshing.py @@ -5,7 +5,7 @@ @patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") -def test_token_refreshing(mock_auth_client): +def test_client_token_refreshing(mock_auth_client): """ Test that the client refreshes the token when it expires. """ @@ -31,7 +31,7 @@ def test_token_refreshing(mock_auth_client): client.token_expiry = datetime.now(timezone.utc) - timedelta(seconds=1) # Make a GET request (should trigger token refresh) - response = client.get("https://mock.api/some-endpoint", headers={"Authorization": f"Bearer {client.token}"}) + response = client.get("https://mock.api/some-endpoint") # Assertions assert response.status_code == 200 @@ -45,6 +45,5 @@ def test_token_refreshing(mock_auth_client): # Verify the request was made correctly mock_http_client.request.assert_called_once_with( "GET", - "https://mock.api/some-endpoint", - headers={"Authorization": f"Bearer {client.token}"}, + "https://mock.api/some-endpoint" ) From d13ce9b1cc560ef205786d4f6a45573857cd9049 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Fri, 24 Jan 2025 16:29:19 +0000 Subject: [PATCH 06/49] Start fixing some of the issues reported by the pipeline --- datacosmos/client.py | 24 +++++++++++++------ .../client/test_client_authentication.py | 10 +++++--- .../client/test_client_delete_request.py | 12 ++++++---- .../client/test_client_get_request.py | 12 ++++++---- .../client/test_client_initialization.py | 9 ++++--- .../client/test_client_post_request.py | 13 +++++----- .../client/test_client_put_request.py | 13 +++++----- .../client/test_client_token_refreshing.py | 12 ++++++---- 8 files changed, 65 insertions(+), 40 deletions(-) diff --git a/datacosmos/client.py b/datacosmos/client.py index d9a68a3..0b4c639 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -1,12 +1,12 @@ import logging import os from datetime import datetime, timedelta, timezone -from typing import Optional, Any +from typing import Any, Optional import requests -from requests_oauthlib import OAuth2Session from oauthlib.oauth2 import BackendApplicationClient from requests.exceptions import RequestException +from requests_oauthlib import OAuth2Session from config.config import Config @@ -18,7 +18,9 @@ class DatacosmosClient: """ def __init__( - self, config: Optional[Config] = None, config_file: str = "config/config.yaml" + self, + config: Optional[Config] = None, + config_file: str = "config/config.yaml", ): """ Initialize the DatacosmosClient. @@ -42,7 +44,9 @@ def _load_config(self, config_file: str) -> Config: if os.path.exists(config_file): self.logger.info(f"Loading configuration from {config_file}") return Config.from_yaml(config_file) - self.logger.info("Loading configuration from environment variables") + self.logger.info( + "Loading configuration from environment variables" + ) return Config.from_env() except Exception as e: self.logger.error(f"Failed to load configuration: {e}") @@ -73,7 +77,9 @@ def _authenticate_and_initialize_client(self) -> requests.Session: # Initialize the HTTP session with the Authorization header http_client = requests.Session() - http_client.headers.update({"Authorization": f"Bearer {self.token}"}) + http_client.headers.update( + {"Authorization": f"Bearer {self.token}"} + ) return http_client except RequestException as e: self.logger.error(f"Request failed during authentication: {e}") @@ -97,7 +103,9 @@ def get_http_client(self) -> requests.Session: self._refresh_token_if_needed() return self._http_client - def request(self, method: str, url: str, *args: Any, **kwargs: Any) -> requests.Response: + def request( + self, method: str, url: str, *args: Any, **kwargs: Any + ) -> requests.Response: """ Send an HTTP request using the authenticated session. Logs request and response details. @@ -107,7 +115,9 @@ def request(self, method: str, url: str, *args: Any, **kwargs: Any) -> requests. self.logger.info(f"Making {method.upper()} request to {url}") response = self._http_client.request(method, url, *args, **kwargs) response.raise_for_status() - self.logger.info(f"Request to {url} succeeded with status {response.status_code}") + self.logger.info( + f"Request to {url} succeeded with status {response.status_code}" + ) return response except RequestException as e: self.logger.error(f"HTTP request failed: {e}") diff --git a/tests/unit/datacosmos/client/test_client_authentication.py b/tests/unit/datacosmos/client/test_client_authentication.py index 3aec902..2216da3 100644 --- a/tests/unit/datacosmos/client/test_client_authentication.py +++ b/tests/unit/datacosmos/client/test_client_authentication.py @@ -1,10 +1,14 @@ -from unittest.mock import patch, MagicMock -from datacosmos.client import DatacosmosClient +from unittest.mock import MagicMock, patch + from config.config import Config +from datacosmos.client import DatacosmosClient @patch("datacosmos.client.OAuth2Session.fetch_token") -@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client", autospec=True) +@patch( + "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client", + autospec=True, +) def test_client_authentication(mock_auth_client, mock_fetch_token): """ Test that the client correctly fetches a token during authentication. diff --git a/tests/unit/datacosmos/client/test_client_delete_request.py b/tests/unit/datacosmos/client/test_client_delete_request.py index 030ee80..6584d57 100644 --- a/tests/unit/datacosmos/client/test_client_delete_request.py +++ b/tests/unit/datacosmos/client/test_client_delete_request.py @@ -1,9 +1,12 @@ -from unittest.mock import patch, MagicMock -from datacosmos.client import DatacosmosClient +from unittest.mock import MagicMock, patch + from config.config import Config +from datacosmos.client import DatacosmosClient -@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +@patch( + "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" +) def test_delete_request(mock_auth_client): """ Test that the client performs a DELETE request correctly. @@ -27,7 +30,6 @@ def test_delete_request(mock_auth_client): # Assertions assert response.status_code == 204 mock_http_client.request.assert_called_once_with( - "DELETE", - "https://mock.api/some-endpoint" + "DELETE", "https://mock.api/some-endpoint" ) mock_auth_client.call_count == 2 diff --git a/tests/unit/datacosmos/client/test_client_get_request.py b/tests/unit/datacosmos/client/test_client_get_request.py index 2dc7b31..e2a2378 100644 --- a/tests/unit/datacosmos/client/test_client_get_request.py +++ b/tests/unit/datacosmos/client/test_client_get_request.py @@ -1,9 +1,12 @@ -from unittest.mock import patch, MagicMock -from datacosmos.client import DatacosmosClient +from unittest.mock import MagicMock, patch + from config.config import Config +from datacosmos.client import DatacosmosClient -@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +@patch( + "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" +) def test_client_get_request(mock_auth_client): """ Test that the client performs a GET request correctly. @@ -29,7 +32,6 @@ def test_client_get_request(mock_auth_client): assert response.status_code == 200 assert response.json() == {"message": "success"} mock_http_client.request.assert_called_once_with( - "GET", - "https://mock.api/some-endpoint" + "GET", "https://mock.api/some-endpoint" ) mock_auth_client.call_count == 2 diff --git a/tests/unit/datacosmos/client/test_client_initialization.py b/tests/unit/datacosmos/client/test_client_initialization.py index e8ee89c..bc4305d 100644 --- a/tests/unit/datacosmos/client/test_client_initialization.py +++ b/tests/unit/datacosmos/client/test_client_initialization.py @@ -1,9 +1,12 @@ -from unittest.mock import patch, MagicMock -from datacosmos.client import DatacosmosClient +from unittest.mock import MagicMock, patch + from config.config import Config +from datacosmos.client import DatacosmosClient -@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +@patch( + "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" +) @patch("os.path.exists", return_value=False) @patch("config.Config.from_env") def test_client_initialization(mock_from_env, mock_exists, mock_auth_client): diff --git a/tests/unit/datacosmos/client/test_client_post_request.py b/tests/unit/datacosmos/client/test_client_post_request.py index ac6a9b4..3fcf294 100644 --- a/tests/unit/datacosmos/client/test_client_post_request.py +++ b/tests/unit/datacosmos/client/test_client_post_request.py @@ -1,9 +1,12 @@ -from unittest.mock import patch, MagicMock -from datacosmos.client import DatacosmosClient +from unittest.mock import MagicMock, patch + from config.config import Config +from datacosmos.client import DatacosmosClient -@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +@patch( + "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" +) def test_post_request(mock_auth_client): """ Test that the client performs a POST request correctly. @@ -31,8 +34,6 @@ def test_post_request(mock_auth_client): assert response.status_code == 201 assert response.json() == {"message": "created"} mock_http_client.request.assert_called_once_with( - "POST", - "https://mock.api/some-endpoint", - json={"key": "value"} + "POST", "https://mock.api/some-endpoint", json={"key": "value"} ) mock_auth_client.call_count == 2 diff --git a/tests/unit/datacosmos/client/test_client_put_request.py b/tests/unit/datacosmos/client/test_client_put_request.py index 140260b..0454cd9 100644 --- a/tests/unit/datacosmos/client/test_client_put_request.py +++ b/tests/unit/datacosmos/client/test_client_put_request.py @@ -1,9 +1,12 @@ -from unittest.mock import patch, MagicMock -from datacosmos.client import DatacosmosClient +from unittest.mock import MagicMock, patch + from config.config import Config +from datacosmos.client import DatacosmosClient -@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +@patch( + "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" +) def test_put_request(mock_auth_client): """ Test that the client performs a PUT request correctly. @@ -31,8 +34,6 @@ def test_put_request(mock_auth_client): assert response.status_code == 200 assert response.json() == {"message": "updated"} mock_http_client.request.assert_called_once_with( - "PUT", - "https://mock.api/some-endpoint", - json={"key": "updated-value"} + "PUT", "https://mock.api/some-endpoint", json={"key": "updated-value"} ) mock_auth_client.call_count == 2 diff --git a/tests/unit/datacosmos/client/test_client_token_refreshing.py b/tests/unit/datacosmos/client/test_client_token_refreshing.py index 614d735..bb3baba 100644 --- a/tests/unit/datacosmos/client/test_client_token_refreshing.py +++ b/tests/unit/datacosmos/client/test_client_token_refreshing.py @@ -1,10 +1,13 @@ -from unittest.mock import patch, MagicMock from datetime import datetime, timedelta, timezone -from datacosmos.client import DatacosmosClient +from unittest.mock import MagicMock, patch + from config.config import Config +from datacosmos.client import DatacosmosClient -@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +@patch( + "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" +) def test_client_token_refreshing(mock_auth_client): """ Test that the client refreshes the token when it expires. @@ -44,6 +47,5 @@ def test_client_token_refreshing(mock_auth_client): # Verify the request was made correctly mock_http_client.request.assert_called_once_with( - "GET", - "https://mock.api/some-endpoint" + "GET", "https://mock.api/some-endpoint" ) From 319536ec0a182742e35f3938bfcba8effb336d8f Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Fri, 24 Jan 2025 16:35:50 +0000 Subject: [PATCH 07/49] Apply black --- datacosmos/client.py | 8 +- pyproject.toml | 86 ++++++------------- .../client/test_client_authentication.py | 2 +- .../client/test_client_delete_request.py | 4 +- .../client/test_client_get_request.py | 4 +- .../client/test_client_initialization.py | 4 +- .../client/test_client_post_request.py | 8 +- .../client/test_client_put_request.py | 4 +- .../client/test_client_token_refreshing.py | 4 +- 9 files changed, 37 insertions(+), 87 deletions(-) diff --git a/datacosmos/client.py b/datacosmos/client.py index 0b4c639..0abf554 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -44,9 +44,7 @@ def _load_config(self, config_file: str) -> Config: if os.path.exists(config_file): self.logger.info(f"Loading configuration from {config_file}") return Config.from_yaml(config_file) - self.logger.info( - "Loading configuration from environment variables" - ) + self.logger.info("Loading configuration from environment variables") return Config.from_env() except Exception as e: self.logger.error(f"Failed to load configuration: {e}") @@ -77,9 +75,7 @@ def _authenticate_and_initialize_client(self) -> requests.Session: # Initialize the HTTP session with the Authorization header http_client = requests.Session() - http_client.headers.update( - {"Authorization": f"Bearer {self.token}"} - ) + http_client.headers.update({"Authorization": f"Bearer {self.token}"}) return http_client except RequestException as e: self.logger.error(f"Request failed during authentication: {e}") diff --git a/pyproject.toml b/pyproject.toml index 2edef52..03e2b94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,46 +1,15 @@ -[build-system] -requires = ["setuptools>=61.0"] -build-backend = "setuptools.build_meta" - -[project] -name = "datacosmos-sdk" -version = "0.0.1" -authors = [ - { name="Open Cosmos", email="support@open-cosmos.com" }, -] -description = "A library for interacting with DataCosmos from Python code" -requires-python = ">=3.8" -classifiers = [ - "Programming Language :: Python :: 3", - "Operating System :: OS Independent", +# pytest configuration +[tool.pytest.ini_options] +minversion = "6.0" +addopts = "-ra -q" +testpaths = [ + "tests", ] -dependencies = [] - -[project.optional-dependencies] -test = [ - "black==22.12.0", - "flake8==6.0.0", - "pytest==7.2.0", - "bandit[toml]==1.7.4", - "isort==5.11.4", - "pydocstyle==6.1.1", - "flake8-cognitive-complexity==0.1.0", - "ruff==0.0.286" -] - -[tool.setuptools.packages] -find = {} -[tool.bandit] -skips = ["B101"] # Example: ignore assert statement usage if needed -targets = ["datacosmos"] # Adjust to point to your project directory - -[tool.pydocstyle] -convention = "google" +[tool.pyright] +typeCheckingMode = "basic" [tool.ruff] -fix = true -line-length = 88 exclude = [ ".bzr", ".direnv", @@ -60,7 +29,6 @@ exclude = [ ".tox", ".venv", ".vscode", - "__pycache__", "__pypackages__", "_build", "buck-out", @@ -71,9 +39,25 @@ exclude = [ "venv", ] +line-length = 79 +indent-width = 4 + [tool.ruff.lint] +# I - Sort imports +# SLF - Private member access +# F - pyflakes +# C90 - Mccabe complexity check +# ANN - Type hint annotations +# BLE - Do not allow blind excepts +# B - Standard bug bears in Python code +# ARG - unused arguments +# ERA - remove uncommented code extend-select = ["I", "SLF", "F", "C90", "BLE", "B", "ARG", "ERA"] + +# Ignored rules ignore = [] + +# Allow fix for all enabled rules (when `--fix`) is provided. fixable = ["ALL"] unfixable = [] @@ -82,26 +66,10 @@ quote-style = "double" indent-style = "space" skip-magic-trailing-comma = false line-ending = "auto" + +# This is currently disabled by default, but it is planned for this +# to be opt-out in the future. docstring-code-format = false [tool.ruff.lint.mccabe] max-complexity = 10 - -[tool.black] -line-length = 88 - -[tool.isort] -profile = "black" - -[tool.flake8] -max-line-length = 88 -exclude = [ - ".venv", - "__pycache__", - "build", - "dist", -] -ignore = [ - "E203", # Whitespace before ':', handled by Black - "W503", # Line break before binary operator -] diff --git a/tests/unit/datacosmos/client/test_client_authentication.py b/tests/unit/datacosmos/client/test_client_authentication.py index 2216da3..6d163b9 100644 --- a/tests/unit/datacosmos/client/test_client_authentication.py +++ b/tests/unit/datacosmos/client/test_client_authentication.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import patch from config.config import Config from datacosmos.client import DatacosmosClient diff --git a/tests/unit/datacosmos/client/test_client_delete_request.py b/tests/unit/datacosmos/client/test_client_delete_request.py index 6584d57..fbde09e 100644 --- a/tests/unit/datacosmos/client/test_client_delete_request.py +++ b/tests/unit/datacosmos/client/test_client_delete_request.py @@ -4,9 +4,7 @@ from datacosmos.client import DatacosmosClient -@patch( - "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" -) +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") def test_delete_request(mock_auth_client): """ Test that the client performs a DELETE request correctly. diff --git a/tests/unit/datacosmos/client/test_client_get_request.py b/tests/unit/datacosmos/client/test_client_get_request.py index e2a2378..42ab1a6 100644 --- a/tests/unit/datacosmos/client/test_client_get_request.py +++ b/tests/unit/datacosmos/client/test_client_get_request.py @@ -4,9 +4,7 @@ from datacosmos.client import DatacosmosClient -@patch( - "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" -) +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") def test_client_get_request(mock_auth_client): """ Test that the client performs a GET request correctly. diff --git a/tests/unit/datacosmos/client/test_client_initialization.py b/tests/unit/datacosmos/client/test_client_initialization.py index bc4305d..22ad4b6 100644 --- a/tests/unit/datacosmos/client/test_client_initialization.py +++ b/tests/unit/datacosmos/client/test_client_initialization.py @@ -4,9 +4,7 @@ from datacosmos.client import DatacosmosClient -@patch( - "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" -) +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") @patch("os.path.exists", return_value=False) @patch("config.Config.from_env") def test_client_initialization(mock_from_env, mock_exists, mock_auth_client): diff --git a/tests/unit/datacosmos/client/test_client_post_request.py b/tests/unit/datacosmos/client/test_client_post_request.py index 3fcf294..b3be698 100644 --- a/tests/unit/datacosmos/client/test_client_post_request.py +++ b/tests/unit/datacosmos/client/test_client_post_request.py @@ -4,9 +4,7 @@ from datacosmos.client import DatacosmosClient -@patch( - "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" -) +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") def test_post_request(mock_auth_client): """ Test that the client performs a POST request correctly. @@ -26,9 +24,7 @@ def test_post_request(mock_auth_client): audience="https://mock.audience", ) client = DatacosmosClient(config=config) - response = client.post( - "https://mock.api/some-endpoint", json={"key": "value"} - ) + response = client.post("https://mock.api/some-endpoint", json={"key": "value"}) # Assertions assert response.status_code == 201 diff --git a/tests/unit/datacosmos/client/test_client_put_request.py b/tests/unit/datacosmos/client/test_client_put_request.py index 0454cd9..c7afd17 100644 --- a/tests/unit/datacosmos/client/test_client_put_request.py +++ b/tests/unit/datacosmos/client/test_client_put_request.py @@ -4,9 +4,7 @@ from datacosmos.client import DatacosmosClient -@patch( - "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" -) +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") def test_put_request(mock_auth_client): """ Test that the client performs a PUT request correctly. diff --git a/tests/unit/datacosmos/client/test_client_token_refreshing.py b/tests/unit/datacosmos/client/test_client_token_refreshing.py index bb3baba..11951be 100644 --- a/tests/unit/datacosmos/client/test_client_token_refreshing.py +++ b/tests/unit/datacosmos/client/test_client_token_refreshing.py @@ -5,9 +5,7 @@ from datacosmos.client import DatacosmosClient -@patch( - "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" -) +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") def test_client_token_refreshing(mock_auth_client): """ Test that the client refreshes the token when it expires. From e1bee71fec9610800c67a30eba88d0c8eba27656 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Fri, 24 Jan 2025 16:39:37 +0000 Subject: [PATCH 08/49] Revert changes in pyproject.toml --- pyproject.toml | 101 +++++++++++++++---------------------------------- 1 file changed, 31 insertions(+), 70 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 03e2b94..b06bf42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,75 +1,36 @@ -# pytest configuration -[tool.pytest.ini_options] -minversion = "6.0" -addopts = "-ra -q" -testpaths = [ - "tests", +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "datacosmos-sdk" +version = "0.0.1" +authors = [ + { name="Open Cosmos", email="support@open-cosmos.com" }, ] - -[tool.pyright] -typeCheckingMode = "basic" - -[tool.ruff] -exclude = [ - ".bzr", - ".direnv", - ".eggs", - ".git", - ".git-rewrite", - ".hg", - ".ipynb_checkpoints", - ".mypy_cache", - ".nox", - ".pants.d", - ".pyenv", - ".pytest_cache", - ".pytype", - ".ruff_cache", - ".svn", - ".tox", - ".venv", - ".vscode", - "__pypackages__", - "_build", - "buck-out", - "build", - "dist", - "node_modules", - "site-packages", - "venv", +description = "A library for interacting with DataCosmos from Python code" +requires-python = ">=3.8" +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", +] +dependencies = [] + +[project.optional-dependencies] +test = [ + "black==22.12.0", + "flake8==6.0.0", + "pytest==7.2.0", + "bandit[toml]==1.7.4", + "isort==5.11.4", + "pydocstyle==6.1.1", + "flake8-cognitive-complexity==0.1.0", ] -line-length = 79 -indent-width = 4 - -[tool.ruff.lint] -# I - Sort imports -# SLF - Private member access -# F - pyflakes -# C90 - Mccabe complexity check -# ANN - Type hint annotations -# BLE - Do not allow blind excepts -# B - Standard bug bears in Python code -# ARG - unused arguments -# ERA - remove uncommented code -extend-select = ["I", "SLF", "F", "C90", "BLE", "B", "ARG", "ERA"] - -# Ignored rules -ignore = [] - -# Allow fix for all enabled rules (when `--fix`) is provided. -fixable = ["ALL"] -unfixable = [] - -[tool.ruff.format] -quote-style = "double" -indent-style = "space" -skip-magic-trailing-comma = false -line-ending = "auto" +[tool.setuptools.packages] +find = {} -# This is currently disabled by default, but it is planned for this -# to be opt-out in the future. -docstring-code-format = false +[tool.bandit] -[tool.ruff.lint.mccabe] -max-complexity = 10 +[tool.pydocstyle] +convention = "google" From 1e40bd69cbe8a85eae61aae3dbd9f726a9ae2cbc Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Fri, 24 Jan 2025 16:45:17 +0000 Subject: [PATCH 09/49] Apply pydocstyle --- config/__init__.py | 3 +- config/config.py | 3 +- datacosmos/client.py | 63 +++++++++---------- .../client/test_client_authentication.py | 4 +- .../client/test_client_delete_request.py | 4 +- .../client/test_client_get_request.py | 4 +- .../client/test_client_initialization.py | 5 +- .../client/test_client_post_request.py | 7 +-- .../client/test_client_put_request.py | 4 +- .../client/test_client_token_refreshing.py | 4 +- 10 files changed, 43 insertions(+), 58 deletions(-) diff --git a/config/__init__.py b/config/__init__.py index e9778e9..ac196a0 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -1,6 +1,7 @@ """Configuration package for the Datacosmos SDK. -This package includes modules for loading and managing authentication configurations. +This package includes modules for loading and managing authentication +configurations. """ # Expose Config class for easier imports diff --git a/config/config.py b/config/config.py index c3a6624..de676bd 100644 --- a/config/config.py +++ b/config/config.py @@ -13,7 +13,8 @@ class Config: """Configuration for the Datacosmos SDK. - Contains authentication details such as client ID, secret, token URL, and audience. + Contains authentication details such as client ID, secret, token + URL, and audience. """ client_id: str diff --git a/datacosmos/client.py b/datacosmos/client.py index 0abf554..224b5bb 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -1,3 +1,9 @@ +"""DatacosmosClient handles authenticated interactions with the Datacosmos API. + +Automatically manages token refreshing and provides HTTP convenience +methods. +""" + import logging import os from datetime import datetime, timedelta, timezone @@ -12,9 +18,10 @@ class DatacosmosClient: - """ - DatacosmosClient handles authenticated interactions with the Datacosmos API. - Automatically manages token refreshing and provides HTTP convenience methods. + """DatacosmosClient handles authenticated interactions with the Datacosmos API. + + Automatically manages token refreshing and provides HTTP convenience + methods. """ def __init__( @@ -22,11 +29,10 @@ def __init__( config: Optional[Config] = None, config_file: str = "config/config.yaml", ): - """ - Initialize the DatacosmosClient. + """Initialize the DatacosmosClient. - If no configuration is provided, it will load from the specified YAML file - or fall back to environment variables. + If no configuration is provided, it will load from the specified + YAML file or fall back to environment variables. """ self.logger = logging.getLogger(__name__) self.logger.setLevel(logging.INFO) @@ -37,23 +43,23 @@ def __init__( self._http_client = self._authenticate_and_initialize_client() def _load_config(self, config_file: str) -> Config: - """ - Load configuration from the YAML file. Fall back to environment variables if the file is missing. + """Load configuration from the YAML file. + + Fall back to environment variables if the file is missing. """ try: if os.path.exists(config_file): self.logger.info(f"Loading configuration from {config_file}") return Config.from_yaml(config_file) - self.logger.info("Loading configuration from environment variables") + self.logger.info( + "Loading configuration from environment variables") return Config.from_env() except Exception as e: self.logger.error(f"Failed to load configuration: {e}") raise def _authenticate_and_initialize_client(self) -> requests.Session: - """ - Authenticate and initialize the HTTP client with a valid token. - """ + """Authenticate and initialize the HTTP client with a valid token.""" try: self.logger.info("Authenticating with the token endpoint") client = BackendApplicationClient(client_id=self.config.client_id) @@ -75,7 +81,8 @@ def _authenticate_and_initialize_client(self) -> requests.Session: # Initialize the HTTP session with the Authorization header http_client = requests.Session() - http_client.headers.update({"Authorization": f"Bearer {self.token}"}) + http_client.headers.update( + {"Authorization": f"Bearer {self.token}"}) return http_client except RequestException as e: self.logger.error(f"Request failed during authentication: {e}") @@ -85,25 +92,21 @@ def _authenticate_and_initialize_client(self) -> requests.Session: raise def _refresh_token_if_needed(self): - """ - Refresh the token if it has expired. - """ + """Refresh the token if it has expired.""" if not self.token or self.token_expiry <= datetime.now(timezone.utc): self.logger.info("Token expired or missing, refreshing token") self._http_client = self._authenticate_and_initialize_client() def get_http_client(self) -> requests.Session: - """ - Return the authenticated HTTP client, refreshing the token if necessary. - """ + """Return the authenticated HTTP client, refreshing the token if necessary.""" self._refresh_token_if_needed() return self._http_client def request( self, method: str, url: str, *args: Any, **kwargs: Any ) -> requests.Response: - """ - Send an HTTP request using the authenticated session. + """Send an HTTP request using the authenticated session. + Logs request and response details. """ self._refresh_token_if_needed() @@ -123,25 +126,17 @@ def request( raise def get(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: - """ - Send a GET request using the authenticated session. - """ + """Send a GET request using the authenticated session.""" return self.request("GET", url, *args, **kwargs) def post(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: - """ - Send a POST request using the authenticated session. - """ + """Send a POST request using the authenticated session.""" return self.request("POST", url, *args, **kwargs) def put(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: - """ - Send a PUT request using the authenticated session. - """ + """Send a PUT request using the authenticated session.""" return self.request("PUT", url, *args, **kwargs) def delete(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: - """ - Send a DELETE request using the authenticated session. - """ + """Send a DELETE request using the authenticated session.""" return self.request("DELETE", url, *args, **kwargs) diff --git a/tests/unit/datacosmos/client/test_client_authentication.py b/tests/unit/datacosmos/client/test_client_authentication.py index 6d163b9..2a1cfc5 100644 --- a/tests/unit/datacosmos/client/test_client_authentication.py +++ b/tests/unit/datacosmos/client/test_client_authentication.py @@ -10,9 +10,7 @@ autospec=True, ) def test_client_authentication(mock_auth_client, mock_fetch_token): - """ - Test that the client correctly fetches a token during authentication. - """ + """Test that the client correctly fetches a token during authentication.""" # Mock the token response from OAuth2Session mock_fetch_token.return_value = { "access_token": "mock-access-token", diff --git a/tests/unit/datacosmos/client/test_client_delete_request.py b/tests/unit/datacosmos/client/test_client_delete_request.py index fbde09e..817a3f0 100644 --- a/tests/unit/datacosmos/client/test_client_delete_request.py +++ b/tests/unit/datacosmos/client/test_client_delete_request.py @@ -6,9 +6,7 @@ @patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") def test_delete_request(mock_auth_client): - """ - Test that the client performs a DELETE request correctly. - """ + """Test that the client performs a DELETE request correctly.""" # Mock the HTTP client mock_http_client = MagicMock() mock_response = MagicMock() diff --git a/tests/unit/datacosmos/client/test_client_get_request.py b/tests/unit/datacosmos/client/test_client_get_request.py index 42ab1a6..034bc97 100644 --- a/tests/unit/datacosmos/client/test_client_get_request.py +++ b/tests/unit/datacosmos/client/test_client_get_request.py @@ -6,9 +6,7 @@ @patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") def test_client_get_request(mock_auth_client): - """ - Test that the client performs a GET request correctly. - """ + """Test that the client performs a GET request correctly.""" # Mock the HTTP client mock_http_client = MagicMock() mock_response = MagicMock() diff --git a/tests/unit/datacosmos/client/test_client_initialization.py b/tests/unit/datacosmos/client/test_client_initialization.py index 22ad4b6..dc98646 100644 --- a/tests/unit/datacosmos/client/test_client_initialization.py +++ b/tests/unit/datacosmos/client/test_client_initialization.py @@ -8,9 +8,8 @@ @patch("os.path.exists", return_value=False) @patch("config.Config.from_env") def test_client_initialization(mock_from_env, mock_exists, mock_auth_client): - """ - Test that the client initializes correctly with environment variables and mocks the HTTP client. - """ + """Test that the client initializes correctly with environment variables + and mocks the HTTP client.""" mock_config = Config( client_id="test-client-id", client_secret="test-client-secret", diff --git a/tests/unit/datacosmos/client/test_client_post_request.py b/tests/unit/datacosmos/client/test_client_post_request.py index b3be698..39378db 100644 --- a/tests/unit/datacosmos/client/test_client_post_request.py +++ b/tests/unit/datacosmos/client/test_client_post_request.py @@ -6,9 +6,7 @@ @patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") def test_post_request(mock_auth_client): - """ - Test that the client performs a POST request correctly. - """ + """Test that the client performs a POST request correctly.""" # Mock the HTTP client mock_http_client = MagicMock() mock_response = MagicMock() @@ -24,7 +22,8 @@ def test_post_request(mock_auth_client): audience="https://mock.audience", ) client = DatacosmosClient(config=config) - response = client.post("https://mock.api/some-endpoint", json={"key": "value"}) + response = client.post( + "https://mock.api/some-endpoint", json={"key": "value"}) # Assertions assert response.status_code == 201 diff --git a/tests/unit/datacosmos/client/test_client_put_request.py b/tests/unit/datacosmos/client/test_client_put_request.py index c7afd17..f910d22 100644 --- a/tests/unit/datacosmos/client/test_client_put_request.py +++ b/tests/unit/datacosmos/client/test_client_put_request.py @@ -6,9 +6,7 @@ @patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") def test_put_request(mock_auth_client): - """ - Test that the client performs a PUT request correctly. - """ + """Test that the client performs a PUT request correctly.""" # Mock the HTTP client mock_http_client = MagicMock() mock_response = MagicMock() diff --git a/tests/unit/datacosmos/client/test_client_token_refreshing.py b/tests/unit/datacosmos/client/test_client_token_refreshing.py index 11951be..0e6a113 100644 --- a/tests/unit/datacosmos/client/test_client_token_refreshing.py +++ b/tests/unit/datacosmos/client/test_client_token_refreshing.py @@ -7,9 +7,7 @@ @patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") def test_client_token_refreshing(mock_auth_client): - """ - Test that the client refreshes the token when it expires. - """ + """Test that the client refreshes the token when it expires.""" # Mock the HTTP client returned by _authenticate_and_initialize_client mock_http_client = MagicMock() mock_response = MagicMock() From 0f7da2504192d02892082253768643f4de491ac1 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 12:19:43 +0000 Subject: [PATCH 10/49] Apply changes to workflow file --- .github/workflows/main.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 2d4165d..feec28c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -20,12 +20,17 @@ jobs: python-version: "3.10" - name: install dependencies run: pip install .[test] + - name: auto-fix with black and isort + run: | + black . --check --quiet || black . + isort . --check-only --quiet || isort . - name: lint uses: wearerequired/lint-action@v2 with: continue_on_error: false black: true flake8: true + bandit: name: bandit runs-on: ubuntu-latest @@ -39,6 +44,7 @@ jobs: run: pip install .[test] - name: bandit run: bandit -r -c pyproject.toml . + cognitive: name: cognitive runs-on: ubuntu-latest @@ -52,6 +58,7 @@ jobs: run: pip install .[test] - name: cognitive run: flake8 . --max-cognitive-complexity=5 + isort: name: isort runs-on: ubuntu-latest @@ -65,6 +72,7 @@ jobs: run: pip install .[test] - name: isort uses: isort/isort-action@v1.1.0 + pydocstyle: name: pydocstyle runs-on: ubuntu-latest @@ -78,6 +86,7 @@ jobs: run: pip install .[test] - name: pydocstyle run: pydocstyle . + test: name: test runs-on: ubuntu-latest @@ -95,6 +104,7 @@ jobs: run: pip install .[test] - name: test run: python -m pytest + release: name: tag, changelog, release, publish runs-on: ubuntu-latest From 03fe4d5ea2b081e537c10d04fa23a1f1440c1e83 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 12:25:46 +0000 Subject: [PATCH 11/49] Attempt to fix line too long error --- datacosmos/client.py | 6 ++++-- tests/unit/datacosmos/client/test_client_delete_request.py | 4 +++- tests/unit/datacosmos/client/test_client_get_request.py | 4 +++- tests/unit/datacosmos/client/test_client_initialization.py | 4 +++- tests/unit/datacosmos/client/test_client_post_request.py | 7 +++++-- tests/unit/datacosmos/client/test_client_put_request.py | 4 +++- .../unit/datacosmos/client/test_client_token_refreshing.py | 4 +++- 7 files changed, 24 insertions(+), 9 deletions(-) diff --git a/datacosmos/client.py b/datacosmos/client.py index 224b5bb..4c55987 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -52,7 +52,8 @@ def _load_config(self, config_file: str) -> Config: self.logger.info(f"Loading configuration from {config_file}") return Config.from_yaml(config_file) self.logger.info( - "Loading configuration from environment variables") + "Loading configuration from environment variables" + ) return Config.from_env() except Exception as e: self.logger.error(f"Failed to load configuration: {e}") @@ -82,7 +83,8 @@ def _authenticate_and_initialize_client(self) -> requests.Session: # Initialize the HTTP session with the Authorization header http_client = requests.Session() http_client.headers.update( - {"Authorization": f"Bearer {self.token}"}) + {"Authorization": f"Bearer {self.token}"} + ) return http_client except RequestException as e: self.logger.error(f"Request failed during authentication: {e}") diff --git a/tests/unit/datacosmos/client/test_client_delete_request.py b/tests/unit/datacosmos/client/test_client_delete_request.py index 817a3f0..0495ed8 100644 --- a/tests/unit/datacosmos/client/test_client_delete_request.py +++ b/tests/unit/datacosmos/client/test_client_delete_request.py @@ -4,7 +4,9 @@ from datacosmos.client import DatacosmosClient -@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +@patch( + "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" +) def test_delete_request(mock_auth_client): """Test that the client performs a DELETE request correctly.""" # Mock the HTTP client diff --git a/tests/unit/datacosmos/client/test_client_get_request.py b/tests/unit/datacosmos/client/test_client_get_request.py index 034bc97..9b05c12 100644 --- a/tests/unit/datacosmos/client/test_client_get_request.py +++ b/tests/unit/datacosmos/client/test_client_get_request.py @@ -4,7 +4,9 @@ from datacosmos.client import DatacosmosClient -@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +@patch( + "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" +) def test_client_get_request(mock_auth_client): """Test that the client performs a GET request correctly.""" # Mock the HTTP client diff --git a/tests/unit/datacosmos/client/test_client_initialization.py b/tests/unit/datacosmos/client/test_client_initialization.py index dc98646..fea667c 100644 --- a/tests/unit/datacosmos/client/test_client_initialization.py +++ b/tests/unit/datacosmos/client/test_client_initialization.py @@ -4,7 +4,9 @@ from datacosmos.client import DatacosmosClient -@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +@patch( + "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" +) @patch("os.path.exists", return_value=False) @patch("config.Config.from_env") def test_client_initialization(mock_from_env, mock_exists, mock_auth_client): diff --git a/tests/unit/datacosmos/client/test_client_post_request.py b/tests/unit/datacosmos/client/test_client_post_request.py index 39378db..dab34db 100644 --- a/tests/unit/datacosmos/client/test_client_post_request.py +++ b/tests/unit/datacosmos/client/test_client_post_request.py @@ -4,7 +4,9 @@ from datacosmos.client import DatacosmosClient -@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +@patch( + "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" +) def test_post_request(mock_auth_client): """Test that the client performs a POST request correctly.""" # Mock the HTTP client @@ -23,7 +25,8 @@ def test_post_request(mock_auth_client): ) client = DatacosmosClient(config=config) response = client.post( - "https://mock.api/some-endpoint", json={"key": "value"}) + "https://mock.api/some-endpoint", json={"key": "value"} + ) # Assertions assert response.status_code == 201 diff --git a/tests/unit/datacosmos/client/test_client_put_request.py b/tests/unit/datacosmos/client/test_client_put_request.py index f910d22..479dbaf 100644 --- a/tests/unit/datacosmos/client/test_client_put_request.py +++ b/tests/unit/datacosmos/client/test_client_put_request.py @@ -4,7 +4,9 @@ from datacosmos.client import DatacosmosClient -@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +@patch( + "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" +) def test_put_request(mock_auth_client): """Test that the client performs a PUT request correctly.""" # Mock the HTTP client diff --git a/tests/unit/datacosmos/client/test_client_token_refreshing.py b/tests/unit/datacosmos/client/test_client_token_refreshing.py index 0e6a113..a6abd05 100644 --- a/tests/unit/datacosmos/client/test_client_token_refreshing.py +++ b/tests/unit/datacosmos/client/test_client_token_refreshing.py @@ -5,7 +5,9 @@ from datacosmos.client import DatacosmosClient -@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +@patch( + "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" +) def test_client_token_refreshing(mock_auth_client): """Test that the client refreshes the token when it expires.""" # Mock the HTTP client returned by _authenticate_and_initialize_client From 311b789ab251f48fb82deab56e2c8f4c6249a480 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 12:41:46 +0000 Subject: [PATCH 12/49] another attempt to fix line too long error --- datacosmos/client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/datacosmos/client.py b/datacosmos/client.py index 4c55987..1a219ef 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -100,7 +100,8 @@ def _refresh_token_if_needed(self): self._http_client = self._authenticate_and_initialize_client() def get_http_client(self) -> requests.Session: - """Return the authenticated HTTP client, refreshing the token if necessary.""" + """Return the authenticated HTTP client, + refreshing the token if necessary.""" self._refresh_token_if_needed() return self._http_client @@ -117,7 +118,8 @@ def request( response = self._http_client.request(method, url, *args, **kwargs) response.raise_for_status() self.logger.info( - f"Request to {url} succeeded with status {response.status_code}" + f"Request to {url} succeeded + with status {response.status_code}" ) return response except RequestException as e: From eb2d7c6ab33ede7c9d0ab0c652a0787264439c0a Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 12:44:29 +0000 Subject: [PATCH 13/49] rollback changes --- datacosmos/client.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/datacosmos/client.py b/datacosmos/client.py index 1a219ef..4c55987 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -100,8 +100,7 @@ def _refresh_token_if_needed(self): self._http_client = self._authenticate_and_initialize_client() def get_http_client(self) -> requests.Session: - """Return the authenticated HTTP client, - refreshing the token if necessary.""" + """Return the authenticated HTTP client, refreshing the token if necessary.""" self._refresh_token_if_needed() return self._http_client @@ -118,8 +117,7 @@ def request( response = self._http_client.request(method, url, *args, **kwargs) response.raise_for_status() self.logger.info( - f"Request to {url} succeeded - with status {response.status_code}" + f"Request to {url} succeeded with status {response.status_code}" ) return response except RequestException as e: From 424f914507a2d748c29fdd7df398a792dc34ff31 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 12:51:33 +0000 Subject: [PATCH 14/49] Make bandit ignore B105 --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index feec28c..c5b6e5e 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -43,7 +43,7 @@ jobs: - name: install dependencies run: pip install .[test] - name: bandit - run: bandit -r -c pyproject.toml . + run: bandit -r -c pyproject.toml . --exclude B105 cognitive: name: cognitive From dcaed8f6353184aa2fa143a52c3e23c26c6021ce Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 12:53:56 +0000 Subject: [PATCH 15/49] Another attempt to make bandit ignore B105 --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index c5b6e5e..78cace4 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -43,7 +43,7 @@ jobs: - name: install dependencies run: pip install .[test] - name: bandit - run: bandit -r -c pyproject.toml . --exclude B105 + run: bandit -r -c pyproject.toml . --skip B105 cognitive: name: cognitive From d9c207bcef59926e5287ba3b9c101a3e1027ca29 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 12:58:48 +0000 Subject: [PATCH 16/49] Make bandit skip more unrelevant check --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 78cace4..3f28c2c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -43,7 +43,7 @@ jobs: - name: install dependencies run: pip install .[test] - name: bandit - run: bandit -r -c pyproject.toml . --skip B105 + run: bandit -r -c pyproject.toml . --skip B105 B106 B101 cognitive: name: cognitive From dcfce9093130258fe5358c05e1f555256282126d Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 13:00:14 +0000 Subject: [PATCH 17/49] Fix syntax main.yaml --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 3f28c2c..4dfa7be 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -43,7 +43,7 @@ jobs: - name: install dependencies run: pip install .[test] - name: bandit - run: bandit -r -c pyproject.toml . --skip B105 B106 B101 + run: bandit -r -c pyproject.toml . --skip B105,B106,B101 cognitive: name: cognitive From 57d3c7af382cae12a87f7cc4012eb8df4b0d374e Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 13:13:18 +0000 Subject: [PATCH 18/49] Attempt to fix line too long for client.py --- .github/workflows/main.yaml | 2 +- datacosmos/client.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 4dfa7be..9577de8 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -57,7 +57,7 @@ jobs: - name: install dependencies run: pip install .[test] - name: cognitive - run: flake8 . --max-cognitive-complexity=5 + run: flake8 . --max-cognitive-complexity=5 --exclude=test_*.py isort: name: isort diff --git a/datacosmos/client.py b/datacosmos/client.py index 4c55987..e1655ed 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -49,7 +49,9 @@ def _load_config(self, config_file: str) -> Config: """ try: if os.path.exists(config_file): - self.logger.info(f"Loading configuration from {config_file}") + self.logger.info( + f"Loading configuration from {config_file}" + ) return Config.from_yaml(config_file) self.logger.info( "Loading configuration from environment variables" From 2dc23a5c357ed7774aca429a2fa2e15120189928 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 13:16:13 +0000 Subject: [PATCH 19/49] Attempt to exclude files from flake8 --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index b06bf42..5ed9e1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,5 +32,8 @@ find = {} [tool.bandit] +[tool.flake8] +exclude = [tests/*, build/*] + [tool.pydocstyle] convention = "google" From 0051dc3d0f912888d0ddef7112764d4496e5fe08 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 13:17:35 +0000 Subject: [PATCH 20/49] Fix syntax in pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5ed9e1e..7d3739b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ find = {} [tool.bandit] [tool.flake8] -exclude = [tests/*, build/*] +exclude = ["tests/*", "build/*"] [tool.pydocstyle] convention = "google" From 2c0ac0a52a6c9b2c57baf995331f59a33f140d04 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 13:21:37 +0000 Subject: [PATCH 21/49] Another attempt to exclude dirs from flake8 --- .github/workflows/main.yaml | 1 + pyproject.toml | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 9577de8..5879c37 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -30,6 +30,7 @@ jobs: continue_on_error: false black: true flake8: true + flake8-options: "--exclude tests/*,build/*" bandit: name: bandit diff --git a/pyproject.toml b/pyproject.toml index 7d3739b..7ceaa54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,8 +32,6 @@ find = {} [tool.bandit] -[tool.flake8] -exclude = ["tests/*", "build/*"] [tool.pydocstyle] convention = "google" From b3326a0643f42ba6967b5cfad7c9730468a9c6e7 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 14:42:25 +0000 Subject: [PATCH 22/49] Another attempt to exclude dirs from flake8 --- .github/workflows/main.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 5879c37..da038a2 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -30,7 +30,6 @@ jobs: continue_on_error: false black: true flake8: true - flake8-options: "--exclude tests/*,build/*" bandit: name: bandit @@ -58,7 +57,7 @@ jobs: - name: install dependencies run: pip install .[test] - name: cognitive - run: flake8 . --max-cognitive-complexity=5 --exclude=test_*.py + run: flake8 . --max-cognitive-complexity=5 --exclude=tests/*,build/* isort: name: isort From 5da1516d41cdf09089b0c93bdb4e27f63db1f374 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 15:05:27 +0000 Subject: [PATCH 23/49] Another attempt to exclude dirs from flake8 --- .github/workflows/main.yaml | 2 +- pyproject.toml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index da038a2..4dfa7be 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -57,7 +57,7 @@ jobs: - name: install dependencies run: pip install .[test] - name: cognitive - run: flake8 . --max-cognitive-complexity=5 --exclude=tests/*,build/* + run: flake8 . --max-cognitive-complexity=5 isort: name: isort diff --git a/pyproject.toml b/pyproject.toml index 7ceaa54..f97fce2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,3 +35,6 @@ find = {} [tool.pydocstyle] convention = "google" + +[tool.flake8] +exclude = ["build/*"] From c20be933a872f658f5c9d0a60c745a810a84b961 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 15:18:48 +0000 Subject: [PATCH 24/49] Make flake8 not deal with line too long errors --- datacosmos/client.py | 4 +--- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/datacosmos/client.py b/datacosmos/client.py index e1655ed..4c55987 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -49,9 +49,7 @@ def _load_config(self, config_file: str) -> Config: """ try: if os.path.exists(config_file): - self.logger.info( - f"Loading configuration from {config_file}" - ) + self.logger.info(f"Loading configuration from {config_file}") return Config.from_yaml(config_file) self.logger.info( "Loading configuration from environment variables" diff --git a/pyproject.toml b/pyproject.toml index f97fce2..e7d3ee7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,4 +37,4 @@ find = {} convention = "google" [tool.flake8] -exclude = ["build/*"] +ignore = "E501" From ee823b66f6fdb3a6ddd7631d180b54faa8d3091c Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 15:22:37 +0000 Subject: [PATCH 25/49] Attempt to make flake8 run from main.yaml taking in consideration pyproject.toml file --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 4dfa7be..6374074 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -57,7 +57,7 @@ jobs: - name: install dependencies run: pip install .[test] - name: cognitive - run: flake8 . --max-cognitive-complexity=5 + run: flake8 . --max-cognitive-complexity=5 --config=pyproject.toml isort: name: isort From bb372d997d85d936a2269ae6776cf1983d483981 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 15:25:59 +0000 Subject: [PATCH 26/49] Another attempt to make flake8 ignore E501 --- .github/workflows/main.yaml | 2 +- pyproject.toml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 6374074..888adcf 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -57,7 +57,7 @@ jobs: - name: install dependencies run: pip install .[test] - name: cognitive - run: flake8 . --max-cognitive-complexity=5 --config=pyproject.toml + run: flake8 . --max-cognitive-complexity=5 --ignore=E501 isort: name: isort diff --git a/pyproject.toml b/pyproject.toml index e7d3ee7..7ceaa54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,3 @@ find = {} [tool.pydocstyle] convention = "google" - -[tool.flake8] -ignore = "E501" From 34d9c1491400612b9dd0fd8327e9980e6610e7b5 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 15:40:29 +0000 Subject: [PATCH 27/49] Attempt to make flake8 job ignore E501 --- .github/workflows/main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 888adcf..e48f2be 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -30,6 +30,7 @@ jobs: continue_on_error: false black: true flake8: true + flake8-options: -ignore E501 bandit: name: bandit From 3bf00389ba05a131a43d83c4131fec8462a83e51 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 15:42:00 +0000 Subject: [PATCH 28/49] Attempt to make flake8 job ignore E501 --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index e48f2be..5926011 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -30,7 +30,7 @@ jobs: continue_on_error: false black: true flake8: true - flake8-options: -ignore E501 + flake8-options: "--ignore=E501" bandit: name: bandit From e1bbed82357a9c6d388d5d6f0630c476a2f1d20f Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 15:49:00 +0000 Subject: [PATCH 29/49] Attempt to make flake8 job ignore E501 --- .flake8 | 2 ++ .github/workflows/main.yaml | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..16520fc --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +ignore = E501 \ No newline at end of file diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 5926011..888adcf 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -30,7 +30,6 @@ jobs: continue_on_error: false black: true flake8: true - flake8-options: "--ignore=E501" bandit: name: bandit From cbde259b16a41276c242e6a88555eba5ae594862 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 15:51:59 +0000 Subject: [PATCH 30/49] Add requests package to dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 7ceaa54..6aa4172 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ test = [ "isort==5.11.4", "pydocstyle==6.1.1", "flake8-cognitive-complexity==0.1.0", + "requests==2.26.0", ] [tool.setuptools.packages] From 101e1860fbd4de19bdbf54fda61c4e1680d8eaa3 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 15:54:18 +0000 Subject: [PATCH 31/49] Add oauthlib to the dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 6aa4172..2d04c4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ test = [ "pydocstyle==6.1.1", "flake8-cognitive-complexity==0.1.0", "requests==2.26.0", + "oauthlib==3.1.1", ] [tool.setuptools.packages] From a7e17b5682d1662de0079ffd28cfdbcff1700d64 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Mon, 27 Jan 2025 15:56:28 +0000 Subject: [PATCH 32/49] Add requests-oauthlib to dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 2d04c4e..595de06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ test = [ "flake8-cognitive-complexity==0.1.0", "requests==2.26.0", "oauthlib==3.1.1", + "requests-oauthlib==1.3.0", ] [tool.setuptools.packages] From c625a544c4e503ba506a0f0d259df0d0ed54ec77 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Fri, 31 Jan 2025 11:49:31 +0000 Subject: [PATCH 33/49] Refactor of authentication client --- config/config.py | 111 ++++++++++++------ config/models/m2m_authentication_config.py | 24 ++++ config/models/url.py | 34 ++++++ datacosmos/client.py | 43 ++----- .../client/test_client_authentication.py | 22 ++-- .../client/test_client_delete_request.py | 19 +-- .../client/test_client_get_request.py | 19 +-- .../client/test_client_initialization.py | 25 ++-- .../client/test_client_post_request.py | 23 ++-- .../client/test_client_put_request.py | 19 +-- .../client/test_client_token_refreshing.py | 18 +-- 11 files changed, 222 insertions(+), 135 deletions(-) create mode 100644 config/models/m2m_authentication_config.py create mode 100644 config/models/url.py diff --git a/config/config.py b/config/config.py index de676bd..4022e13 100644 --- a/config/config.py +++ b/config/config.py @@ -1,52 +1,87 @@ -"""Module for managing configuration settings for the Datacosmos SDK. +"""Configuration module for the Datacosmos SDK. -Supports loading from YAML files and environment variables. +Handles configuration management using Pydantic and Pydantic Settings. +It loads default values, allows overrides via YAML configuration files, +and supports environment variable-based overrides. """ import os -from dataclasses import dataclass +from typing import Literal import yaml +from pydantic import model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict +from config.models.m2m_authentication_config import M2MAuthenticationConfig +from config.models.url import URL -@dataclass -class Config: - """Configuration for the Datacosmos SDK. - Contains authentication details such as client ID, secret, token - URL, and audience. - """ +class Config(BaseSettings): + """Centralized configuration for the Datacosmos SDK.""" - client_id: str - client_secret: str - token_url: str - audience: str + model_config = SettingsConfigDict( + env_nested_delimiter="__", + nested_model_default_partial_update=True, + ) - @staticmethod - def from_yaml(file_path: str = "config/config.yaml") -> "Config": - """Load configuration from a YAML file. + # General configurations + environment: Literal["local", "test", "prod"] = "test" + log_format: Literal["json", "text"] = "text" + log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR"] = "INFO" - Defaults to 'config/config.yaml' unless otherwise specified. + # Authentication configuration + authentication: M2MAuthenticationConfig = M2MAuthenticationConfig( + type="m2m", + client_id="zCeZWJamwnb8ZIQEK35rhx0hSAjsZI4D", + token_url="https://login.open-cosmos.com/oauth/token", + audience="https://test.beeapp.open-cosmos.com", + client_secret="tAeaSgLds7g535ofGq79Zm2DSbWMCOsuRyY5lbyObJe9eAeSN_fxoy-5kaXnVSYa", + ) + + # STAC API configuration + stac: URL = URL( + protocol="https", + host="test.app.open-cosmos.com", + port=443, + path="/api/data/v0/stac", + ) + + @classmethod + def from_yaml(cls, file_path: str = "config/config.yaml") -> "Config": + """Load configuration from a YAML file and override defaults. + + Args: + file_path (str): The path to the YAML configuration file. + + Returns: + Config: An instance of the Config class with loaded settings. """ - with open(file_path, "r") as f: - data = yaml.safe_load(f) - auth = data.get("auth", {}) - return Config( - client_id=auth["client-id"], - client_secret=auth["client-secret"], - token_url=auth["token-url"], - audience=auth["audience"], - ) - - @staticmethod - def from_env() -> "Config": - """Load configuration from environment variables. - - Raises an exception if any required variable is missing. + config_data = {} + if os.path.exists(file_path): + with open(file_path, "r") as f: + yaml_data = yaml.safe_load(f) or {} + # Remove empty values from YAML to avoid overwriting with `None` + config_data = { + k: v for k, v in yaml_data.items() if v not in [None, ""] + } + return cls(**config_data) + + @model_validator(mode="before") + @classmethod + def merge_with_env(cls, values): + """Override settings with environment variables if set. + + This method checks if any environment variables corresponding to the + config fields are set and updates their values accordingly. + + Args: + values (dict): The configuration values before validation. + + Returns: + dict: The updated configuration values with environment variable overrides. """ - return Config( - client_id=os.getenv("OC_AUTH_CLIENT_ID"), - client_secret=os.getenv("OC_AUTH_CLIENT_SECRET"), - token_url=os.getenv("OC_AUTH_TOKEN_URL"), - audience=os.getenv("OC_AUTH_AUDIENCE"), - ) + for field in cls.model_fields: + env_value = os.getenv(f"OC_{field.upper()}") + if env_value: + values[field] = env_value + return values \ No newline at end of file diff --git a/config/models/m2m_authentication_config.py b/config/models/m2m_authentication_config.py new file mode 100644 index 0000000..028387a --- /dev/null +++ b/config/models/m2m_authentication_config.py @@ -0,0 +1,24 @@ +"""Module for configuring machine-to-machine (M2M) authentication. + +Used when running scripts in the cluster that require automated authentication +without user interaction. +""" + +from typing import Literal + +from pydantic import BaseModel + + +class M2MAuthenticationConfig(BaseModel): + """Configuration for machine-to-machine authentication. + + This is used when running scripts in the cluster that require authentication + with client credentials. + """ + + type: Literal["m2m"] + client_id: str + token_url: str + audience: str + # Some infrastructure deployments do not require a client secret. + client_secret: str = "" \ No newline at end of file diff --git a/config/models/url.py b/config/models/url.py new file mode 100644 index 0000000..537efd1 --- /dev/null +++ b/config/models/url.py @@ -0,0 +1,34 @@ +"""Module defining a structured URL configuration model. + +Ensures that URLs contain required components such as protocol, host, +port, and path. +""" + +from common.domain.url import URL as DomainURL +from pydantic import BaseModel + + +class URL(BaseModel): + """Generic configuration model for a URL. + + This class provides attributes to store URL components and a method + to convert them into a `DomainURL` instance. + """ + + protocol: str + host: str + port: int + path: str + + def as_domain_url(self) -> DomainURL: + """Convert the URL instance to a `DomainURL` object. + + Returns: + DomainURL: A domain-specific URL object. + """ + return DomainURL( + protocol=self.protocol, + host=self.host, + port=self.port, + base=self.path, + ) \ No newline at end of file diff --git a/datacosmos/client.py b/datacosmos/client.py index 4c55987..87fa2cf 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -5,7 +5,6 @@ """ import logging -import os from datetime import datetime, timedelta, timezone from typing import Any, Optional @@ -27,51 +26,35 @@ class DatacosmosClient: def __init__( self, config: Optional[Config] = None, - config_file: str = "config/config.yaml", ): """Initialize the DatacosmosClient. - If no configuration is provided, it will load from the specified - YAML file or fall back to environment variables. + Load configuration from the specified YAML file, environment variables, + or fallback to the default values provided in the `Config` class. """ self.logger = logging.getLogger(__name__) self.logger.setLevel(logging.INFO) - self.config = config or self._load_config(config_file) + self.config = config or Config.from_yaml() self.token = None self.token_expiry = None self._http_client = self._authenticate_and_initialize_client() - def _load_config(self, config_file: str) -> Config: - """Load configuration from the YAML file. - - Fall back to environment variables if the file is missing. - """ - try: - if os.path.exists(config_file): - self.logger.info(f"Loading configuration from {config_file}") - return Config.from_yaml(config_file) - self.logger.info( - "Loading configuration from environment variables" - ) - return Config.from_env() - except Exception as e: - self.logger.error(f"Failed to load configuration: {e}") - raise - def _authenticate_and_initialize_client(self) -> requests.Session: """Authenticate and initialize the HTTP client with a valid token.""" try: self.logger.info("Authenticating with the token endpoint") - client = BackendApplicationClient(client_id=self.config.client_id) + client = BackendApplicationClient( + client_id=self.config.authentication.client_id + ) oauth_session = OAuth2Session(client=client) # Fetch the token using client credentials token_response = oauth_session.fetch_token( - token_url=self.config.token_url, - client_id=self.config.client_id, - client_secret=self.config.client_secret, - audience=self.config.audience, + token_url=self.config.authentication.token_url, + client_id=self.config.authentication.client_id, + client_secret=self.config.authentication.client_secret, + audience=self.config.authentication.audience, ) self.token = token_response["access_token"] @@ -82,9 +65,7 @@ def _authenticate_and_initialize_client(self) -> requests.Session: # Initialize the HTTP session with the Authorization header http_client = requests.Session() - http_client.headers.update( - {"Authorization": f"Bearer {self.token}"} - ) + http_client.headers.update({"Authorization": f"Bearer {self.token}"}) return http_client except RequestException as e: self.logger.error(f"Request failed during authentication: {e}") @@ -141,4 +122,4 @@ def put(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: def delete(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: """Send a DELETE request using the authenticated session.""" - return self.request("DELETE", url, *args, **kwargs) + return self.request("DELETE", url, *args, **kwargs) \ No newline at end of file diff --git a/tests/unit/datacosmos/client/test_client_authentication.py b/tests/unit/datacosmos/client/test_client_authentication.py index 2a1cfc5..0041016 100644 --- a/tests/unit/datacosmos/client/test_client_authentication.py +++ b/tests/unit/datacosmos/client/test_client_authentication.py @@ -1,6 +1,7 @@ from unittest.mock import patch from config.config import Config +from config.models.m2m_authentication_config import M2MAuthenticationConfig from datacosmos.client import DatacosmosClient @@ -21,10 +22,10 @@ def test_client_authentication(mock_auth_client, mock_fetch_token): def mock_authenticate_and_initialize_client(self): # Call the real fetch_token (simulated by the mock) token_response = mock_fetch_token( - token_url=self.config.token_url, - client_id=self.config.client_id, - client_secret=self.config.client_secret, - audience=self.config.audience, + token_url=self.config.authentication.token_url, + client_id=self.config.authentication.client_id, + client_secret=self.config.authentication.client_secret, + audience=self.config.authentication.audience, ) self.token = token_response["access_token"] self.token_expiry = "mock-expiry" @@ -34,10 +35,13 @@ def mock_authenticate_and_initialize_client(self): # Create a mock configuration config = Config( - client_id="test-client-id", - client_secret="test-client-secret", - token_url="https://mock.token.url/oauth/token", - audience="https://mock.audience", + authentication=M2MAuthenticationConfig( + type="m2m", + client_id="test-client-id", + client_secret="test-client-secret", + token_url="https://mock.token.url/oauth/token", + audience="https://mock.audience", + ) ) # Initialize the client @@ -52,4 +56,4 @@ def mock_authenticate_and_initialize_client(self): client_secret="test-client-secret", audience="https://mock.audience", ) - mock_auth_client.assert_called_once_with(client) + mock_auth_client.assert_called_once_with(client) \ No newline at end of file diff --git a/tests/unit/datacosmos/client/test_client_delete_request.py b/tests/unit/datacosmos/client/test_client_delete_request.py index 0495ed8..f4af82b 100644 --- a/tests/unit/datacosmos/client/test_client_delete_request.py +++ b/tests/unit/datacosmos/client/test_client_delete_request.py @@ -1,12 +1,11 @@ from unittest.mock import MagicMock, patch from config.config import Config +from config.models.m2m_authentication_config import M2MAuthenticationConfig from datacosmos.client import DatacosmosClient -@patch( - "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" -) +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") def test_delete_request(mock_auth_client): """Test that the client performs a DELETE request correctly.""" # Mock the HTTP client @@ -17,11 +16,15 @@ def test_delete_request(mock_auth_client): mock_auth_client.return_value = mock_http_client config = Config( - client_id="test-client-id", - client_secret="test-client-secret", - token_url="https://mock.token.url/oauth/token", - audience="https://mock.audience", + authentication=M2MAuthenticationConfig( + type="m2m", + client_id="test-client-id", + client_secret="test-client-secret", + token_url="https://mock.token.url/oauth/token", + audience="https://mock.audience", + ) ) + client = DatacosmosClient(config=config) response = client.delete("https://mock.api/some-endpoint") @@ -30,4 +33,4 @@ def test_delete_request(mock_auth_client): mock_http_client.request.assert_called_once_with( "DELETE", "https://mock.api/some-endpoint" ) - mock_auth_client.call_count == 2 + mock_auth_client.call_count == 2 \ No newline at end of file diff --git a/tests/unit/datacosmos/client/test_client_get_request.py b/tests/unit/datacosmos/client/test_client_get_request.py index 9b05c12..7f6182b 100644 --- a/tests/unit/datacosmos/client/test_client_get_request.py +++ b/tests/unit/datacosmos/client/test_client_get_request.py @@ -1,12 +1,11 @@ from unittest.mock import MagicMock, patch from config.config import Config +from config.models.m2m_authentication_config import M2MAuthenticationConfig from datacosmos.client import DatacosmosClient -@patch( - "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" -) +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") def test_client_get_request(mock_auth_client): """Test that the client performs a GET request correctly.""" # Mock the HTTP client @@ -18,11 +17,15 @@ def test_client_get_request(mock_auth_client): mock_auth_client.return_value = mock_http_client config = Config( - client_id="test-client-id", - client_secret="test-client-secret", - token_url="https://mock.token.url/oauth/token", - audience="https://mock.audience", + authentication=M2MAuthenticationConfig( + type="m2m", + client_id="test-client-id", + client_secret="test-client-secret", + token_url="https://mock.token.url/oauth/token", + audience="https://mock.audience", + ) ) + client = DatacosmosClient(config=config) response = client.get("https://mock.api/some-endpoint") @@ -32,4 +35,4 @@ def test_client_get_request(mock_auth_client): mock_http_client.request.assert_called_once_with( "GET", "https://mock.api/some-endpoint" ) - mock_auth_client.call_count == 2 + mock_auth_client.call_count == 2 \ No newline at end of file diff --git a/tests/unit/datacosmos/client/test_client_initialization.py b/tests/unit/datacosmos/client/test_client_initialization.py index fea667c..96ebb62 100644 --- a/tests/unit/datacosmos/client/test_client_initialization.py +++ b/tests/unit/datacosmos/client/test_client_initialization.py @@ -1,30 +1,27 @@ from unittest.mock import MagicMock, patch from config.config import Config +from config.models.m2m_authentication_config import M2MAuthenticationConfig from datacosmos.client import DatacosmosClient -@patch( - "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" -) -@patch("os.path.exists", return_value=False) -@patch("config.Config.from_env") -def test_client_initialization(mock_from_env, mock_exists, mock_auth_client): +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") +def test_client_initialization(mock_auth_client): """Test that the client initializes correctly with environment variables and mocks the HTTP client.""" mock_config = Config( - client_id="test-client-id", - client_secret="test-client-secret", - token_url="https://mock.token.url/oauth/token", - audience="https://mock.audience", + authentication=M2MAuthenticationConfig( + type="m2m", + client_id="zCeZWJamwnb8ZIQEK35rhx0hSAjsZI4D", + token_url="https://login.open-cosmos.com/oauth/token", + audience="https://test.beeapp.open-cosmos.com", + client_secret="tAeaSgLds7g535ofGq79Zm2DSbWMCOsuRyY5lbyObJe9eAeSN_fxoy-5kaXnVSYa", + ) ) - mock_from_env.return_value = mock_config mock_auth_client.return_value = MagicMock() # Mock the HTTP client client = DatacosmosClient() assert client.config == mock_config assert client._http_client is not None # Ensure the HTTP client is mocked - mock_exists.assert_called_once_with("config/config.yaml") - mock_from_env.assert_called_once() - mock_auth_client.assert_called_once() + mock_auth_client.assert_called_once() \ No newline at end of file diff --git a/tests/unit/datacosmos/client/test_client_post_request.py b/tests/unit/datacosmos/client/test_client_post_request.py index dab34db..99c9664 100644 --- a/tests/unit/datacosmos/client/test_client_post_request.py +++ b/tests/unit/datacosmos/client/test_client_post_request.py @@ -1,12 +1,11 @@ from unittest.mock import MagicMock, patch from config.config import Config +from config.models.m2m_authentication_config import M2MAuthenticationConfig from datacosmos.client import DatacosmosClient -@patch( - "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" -) +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") def test_post_request(mock_auth_client): """Test that the client performs a POST request correctly.""" # Mock the HTTP client @@ -18,15 +17,17 @@ def test_post_request(mock_auth_client): mock_auth_client.return_value = mock_http_client config = Config( - client_id="test-client-id", - client_secret="test-client-secret", - token_url="https://mock.token.url/oauth/token", - audience="https://mock.audience", + authentication=M2MAuthenticationConfig( + type="m2m", + client_id="test-client-id", + client_secret="test-client-secret", + token_url="https://mock.token.url/oauth/token", + audience="https://mock.audience", + ) ) + client = DatacosmosClient(config=config) - response = client.post( - "https://mock.api/some-endpoint", json={"key": "value"} - ) + response = client.post("https://mock.api/some-endpoint", json={"key": "value"}) # Assertions assert response.status_code == 201 @@ -34,4 +35,4 @@ def test_post_request(mock_auth_client): mock_http_client.request.assert_called_once_with( "POST", "https://mock.api/some-endpoint", json={"key": "value"} ) - mock_auth_client.call_count == 2 + mock_auth_client.call_count == 2 \ No newline at end of file diff --git a/tests/unit/datacosmos/client/test_client_put_request.py b/tests/unit/datacosmos/client/test_client_put_request.py index 479dbaf..0e19f72 100644 --- a/tests/unit/datacosmos/client/test_client_put_request.py +++ b/tests/unit/datacosmos/client/test_client_put_request.py @@ -1,12 +1,11 @@ from unittest.mock import MagicMock, patch from config.config import Config +from config.models.m2m_authentication_config import M2MAuthenticationConfig from datacosmos.client import DatacosmosClient -@patch( - "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" -) +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") def test_put_request(mock_auth_client): """Test that the client performs a PUT request correctly.""" # Mock the HTTP client @@ -18,11 +17,15 @@ def test_put_request(mock_auth_client): mock_auth_client.return_value = mock_http_client config = Config( - client_id="test-client-id", - client_secret="test-client-secret", - token_url="https://mock.token.url/oauth/token", - audience="https://mock.audience", + authentication=M2MAuthenticationConfig( + type="m2m", + client_id="test-client-id", + client_secret="test-client-secret", + token_url="https://mock.token.url/oauth/token", + audience="https://mock.audience", + ) ) + client = DatacosmosClient(config=config) response = client.put( "https://mock.api/some-endpoint", json={"key": "updated-value"} @@ -34,4 +37,4 @@ def test_put_request(mock_auth_client): mock_http_client.request.assert_called_once_with( "PUT", "https://mock.api/some-endpoint", json={"key": "updated-value"} ) - mock_auth_client.call_count == 2 + mock_auth_client.call_count == 2 \ No newline at end of file diff --git a/tests/unit/datacosmos/client/test_client_token_refreshing.py b/tests/unit/datacosmos/client/test_client_token_refreshing.py index a6abd05..d35db59 100644 --- a/tests/unit/datacosmos/client/test_client_token_refreshing.py +++ b/tests/unit/datacosmos/client/test_client_token_refreshing.py @@ -2,12 +2,11 @@ from unittest.mock import MagicMock, patch from config.config import Config +from config.models.m2m_authentication_config import M2MAuthenticationConfig from datacosmos.client import DatacosmosClient -@patch( - "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client" -) +@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") def test_client_token_refreshing(mock_auth_client): """Test that the client refreshes the token when it expires.""" # Mock the HTTP client returned by _authenticate_and_initialize_client @@ -19,10 +18,13 @@ def test_client_token_refreshing(mock_auth_client): mock_auth_client.return_value = mock_http_client config = Config( - client_id="test-client-id", - client_secret="test-client-secret", - token_url="https://mock.token.url/oauth/token", - audience="https://mock.audience", + authentication=M2MAuthenticationConfig( + type="m2m", + client_id="test-client-id", + client_secret="test-client-secret", + token_url="https://mock.token.url/oauth/token", + audience="https://mock.audience", + ) ) # Initialize the client (first call to _authenticate_and_initialize_client) @@ -46,4 +48,4 @@ def test_client_token_refreshing(mock_auth_client): # Verify the request was made correctly mock_http_client.request.assert_called_once_with( "GET", "https://mock.api/some-endpoint" - ) + ) \ No newline at end of file From 146448427d9d6d1d0f71ba91daa6591a3712cca5 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Fri, 31 Jan 2025 11:52:03 +0000 Subject: [PATCH 34/49] Add new line in end of files --- config/config.py | 2 +- config/models/m2m_authentication_config.py | 2 +- config/models/url.py | 2 +- datacosmos/client.py | 2 +- tests/unit/datacosmos/client/test_client_authentication.py | 2 +- tests/unit/datacosmos/client/test_client_delete_request.py | 2 +- tests/unit/datacosmos/client/test_client_get_request.py | 2 +- tests/unit/datacosmos/client/test_client_initialization.py | 2 +- tests/unit/datacosmos/client/test_client_post_request.py | 2 +- tests/unit/datacosmos/client/test_client_put_request.py | 2 +- tests/unit/datacosmos/client/test_client_token_refreshing.py | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/config/config.py b/config/config.py index 4022e13..8cd9226 100644 --- a/config/config.py +++ b/config/config.py @@ -84,4 +84,4 @@ def merge_with_env(cls, values): env_value = os.getenv(f"OC_{field.upper()}") if env_value: values[field] = env_value - return values \ No newline at end of file + return values diff --git a/config/models/m2m_authentication_config.py b/config/models/m2m_authentication_config.py index 028387a..0afca20 100644 --- a/config/models/m2m_authentication_config.py +++ b/config/models/m2m_authentication_config.py @@ -21,4 +21,4 @@ class M2MAuthenticationConfig(BaseModel): token_url: str audience: str # Some infrastructure deployments do not require a client secret. - client_secret: str = "" \ No newline at end of file + client_secret: str = "" diff --git a/config/models/url.py b/config/models/url.py index 537efd1..7df9c83 100644 --- a/config/models/url.py +++ b/config/models/url.py @@ -31,4 +31,4 @@ def as_domain_url(self) -> DomainURL: host=self.host, port=self.port, base=self.path, - ) \ No newline at end of file + ) diff --git a/datacosmos/client.py b/datacosmos/client.py index 87fa2cf..f2e570e 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -122,4 +122,4 @@ def put(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: def delete(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: """Send a DELETE request using the authenticated session.""" - return self.request("DELETE", url, *args, **kwargs) \ No newline at end of file + return self.request("DELETE", url, *args, **kwargs) diff --git a/tests/unit/datacosmos/client/test_client_authentication.py b/tests/unit/datacosmos/client/test_client_authentication.py index 0041016..2327c9e 100644 --- a/tests/unit/datacosmos/client/test_client_authentication.py +++ b/tests/unit/datacosmos/client/test_client_authentication.py @@ -56,4 +56,4 @@ def mock_authenticate_and_initialize_client(self): client_secret="test-client-secret", audience="https://mock.audience", ) - mock_auth_client.assert_called_once_with(client) \ No newline at end of file + mock_auth_client.assert_called_once_with(client) diff --git a/tests/unit/datacosmos/client/test_client_delete_request.py b/tests/unit/datacosmos/client/test_client_delete_request.py index f4af82b..6b61f0b 100644 --- a/tests/unit/datacosmos/client/test_client_delete_request.py +++ b/tests/unit/datacosmos/client/test_client_delete_request.py @@ -33,4 +33,4 @@ def test_delete_request(mock_auth_client): mock_http_client.request.assert_called_once_with( "DELETE", "https://mock.api/some-endpoint" ) - mock_auth_client.call_count == 2 \ No newline at end of file + mock_auth_client.call_count == 2 diff --git a/tests/unit/datacosmos/client/test_client_get_request.py b/tests/unit/datacosmos/client/test_client_get_request.py index 7f6182b..4963756 100644 --- a/tests/unit/datacosmos/client/test_client_get_request.py +++ b/tests/unit/datacosmos/client/test_client_get_request.py @@ -35,4 +35,4 @@ def test_client_get_request(mock_auth_client): mock_http_client.request.assert_called_once_with( "GET", "https://mock.api/some-endpoint" ) - mock_auth_client.call_count == 2 \ No newline at end of file + mock_auth_client.call_count == 2 diff --git a/tests/unit/datacosmos/client/test_client_initialization.py b/tests/unit/datacosmos/client/test_client_initialization.py index 96ebb62..9422fca 100644 --- a/tests/unit/datacosmos/client/test_client_initialization.py +++ b/tests/unit/datacosmos/client/test_client_initialization.py @@ -24,4 +24,4 @@ def test_client_initialization(mock_auth_client): assert client.config == mock_config assert client._http_client is not None # Ensure the HTTP client is mocked - mock_auth_client.assert_called_once() \ No newline at end of file + mock_auth_client.assert_called_once() diff --git a/tests/unit/datacosmos/client/test_client_post_request.py b/tests/unit/datacosmos/client/test_client_post_request.py index 99c9664..5023602 100644 --- a/tests/unit/datacosmos/client/test_client_post_request.py +++ b/tests/unit/datacosmos/client/test_client_post_request.py @@ -35,4 +35,4 @@ def test_post_request(mock_auth_client): mock_http_client.request.assert_called_once_with( "POST", "https://mock.api/some-endpoint", json={"key": "value"} ) - mock_auth_client.call_count == 2 \ No newline at end of file + mock_auth_client.call_count == 2 diff --git a/tests/unit/datacosmos/client/test_client_put_request.py b/tests/unit/datacosmos/client/test_client_put_request.py index 0e19f72..c894fc0 100644 --- a/tests/unit/datacosmos/client/test_client_put_request.py +++ b/tests/unit/datacosmos/client/test_client_put_request.py @@ -37,4 +37,4 @@ def test_put_request(mock_auth_client): mock_http_client.request.assert_called_once_with( "PUT", "https://mock.api/some-endpoint", json={"key": "updated-value"} ) - mock_auth_client.call_count == 2 \ No newline at end of file + mock_auth_client.call_count == 2 diff --git a/tests/unit/datacosmos/client/test_client_token_refreshing.py b/tests/unit/datacosmos/client/test_client_token_refreshing.py index d35db59..ad44784 100644 --- a/tests/unit/datacosmos/client/test_client_token_refreshing.py +++ b/tests/unit/datacosmos/client/test_client_token_refreshing.py @@ -48,4 +48,4 @@ def test_client_token_refreshing(mock_auth_client): # Verify the request was made correctly mock_http_client.request.assert_called_once_with( "GET", "https://mock.api/some-endpoint" - ) \ No newline at end of file + ) From 0170cd38c4976e0a35d977d780422b574a05d285 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Fri, 31 Jan 2025 11:55:18 +0000 Subject: [PATCH 35/49] Add pydantic and pydantic-settings to dependencies --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 595de06..8c40caa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,8 @@ test = [ "requests==2.26.0", "oauthlib==3.1.1", "requests-oauthlib==1.3.0", + "pydantic==2.10.6", + "pydantic-settings==2.7.1" ] [tool.setuptools.packages] From 2c1256e494b2dc3b4d6e1e7c422a7c902114afde Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Fri, 31 Jan 2025 11:58:25 +0000 Subject: [PATCH 36/49] Fix pipeline --- .github/workflows/main.yaml | 82 +++++++++++++++++++++++-------------- pyproject.toml | 33 +++++++-------- 2 files changed, 69 insertions(+), 46 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 888adcf..4374c3f 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -15,35 +15,38 @@ jobs: steps: - uses: actions/checkout@v3 - name: set up python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: "3.10" + - name: set up poetry + run: curl -sSL https://install.python-poetry.org | python3 - + - name: configure gitlab auth + run: poetry config http-basic.gitlab __token__ ${{ secrets.GITLAB_PYPI_TOKEN }} - name: install dependencies - run: pip install .[test] - - name: auto-fix with black and isort + run: poetry install --no-interaction + - name: run linters manually with poetry run: | - black . --check --quiet || black . - isort . --check-only --quiet || isort . - - name: lint - uses: wearerequired/lint-action@v2 - with: - continue_on_error: false - black: true - flake8: true - + poetry run black . --check + poetry run isort . --check-only + poetry run flake8 . + bandit: name: bandit runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: set up python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: "3.10" + - name: set up poetry + run: curl -sSL https://install.python-poetry.org | python3 - + - name: configure gitlab auth + run: poetry config http-basic.gitlab __token__ ${{ secrets.GITLAB_PYPI_TOKEN }} - name: install dependencies - run: pip install .[test] + run: poetry install --no-interaction - name: bandit - run: bandit -r -c pyproject.toml . --skip B105,B106,B101 + run: poetry run bandit -r -c pyproject.toml . --skip B105,B106,B101 cognitive: name: cognitive @@ -51,13 +54,17 @@ jobs: steps: - uses: actions/checkout@v3 - name: set up python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: "3.10" + - name: set up poetry + run: curl -sSL https://install.python-poetry.org | python3 - + - name: configure gitlab auth + run: poetry config http-basic.gitlab __token__ ${{ secrets.GITLAB_PYPI_TOKEN }} - name: install dependencies - run: pip install .[test] + run: poetry install --no-interaction - name: cognitive - run: flake8 . --max-cognitive-complexity=5 --ignore=E501 + run: poetry run flake8 . --max-cognitive-complexity=5 --ignore=E501 isort: name: isort @@ -65,11 +72,15 @@ jobs: steps: - uses: actions/checkout@v3 - name: set up python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: "3.10" + - name: set up poetry + run: curl -sSL https://install.python-poetry.org | python3 - + - name: configure gitlab auth + run: poetry config http-basic.gitlab __token__ ${{ secrets.GITLAB_PYPI_TOKEN }} - name: install dependencies - run: pip install .[test] + run: poetry install --no-interaction - name: isort uses: isort/isort-action@v1.1.0 @@ -79,31 +90,36 @@ jobs: steps: - uses: actions/checkout@v3 - name: set up python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: "3.10" + - name: set up poetry + run: curl -sSL https://install.python-poetry.org | python3 - + - name: configure gitlab auth + run: poetry config http-basic.gitlab __token__ ${{ secrets.GITLAB_PYPI_TOKEN }} - name: install dependencies - run: pip install .[test] + run: poetry install --no-interaction - name: pydocstyle - run: pydocstyle . + run: poetry run pydocstyle . test: name: test runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] needs: [bandit, cognitive, isort, lint, pydocstyle] steps: - uses: actions/checkout@v3 - name: set up python uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python-version }} + python-version: "3.10" + - name: set up poetry + run: curl -sSL https://install.python-poetry.org | python3 - + - name: configure gitlab auth + run: poetry config http-basic.gitlab __token__ ${{ secrets.GITLAB_PYPI_TOKEN }} - name: install dependencies - run: pip install .[test] + run: poetry install --no-interaction - name: test - run: python -m pytest + run: poetry run pytest release: name: tag, changelog, release, publish @@ -112,6 +128,12 @@ jobs: if: github.ref == 'refs/heads/main' steps: - uses: actions/checkout@v3 + - name: set up poetry + run: curl -sSL https://install.python-poetry.org | python3 - + - name: configure gitlab auth + run: poetry config http-basic.gitlab __token__ ${{ secrets.GITLAB_PYPI_TOKEN }} + - name: install dependencies + run: poetry install --no-interaction - name: version uses: paulhatch/semantic-version@v5.0.0 id: version @@ -145,4 +167,4 @@ jobs: ## Included Pull Requests - ${{ steps.changelog.outputs.changes }} + ${{ steps.changelog.outputs.changes }} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 8c40caa..f07db41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,40 +3,41 @@ requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [project] -name = "datacosmos-sdk" +name = "datacosmos" version = "0.0.1" authors = [ { name="Open Cosmos", email="support@open-cosmos.com" }, ] description = "A library for interacting with DataCosmos from Python code" -requires-python = ">=3.8" +requires-python = ">=3.10" classifiers = [ "Programming Language :: Python :: 3", "Operating System :: OS Independent", ] -dependencies = [] - -[project.optional-dependencies] -test = [ - "black==22.12.0", - "flake8==6.0.0", +dependencies = [ + "python-common==0.13.1", + "black==22.3.0", + "flake8==4.0.1", "pytest==7.2.0", "bandit[toml]==1.7.4", "isort==5.11.4", "pydocstyle==6.1.1", "flake8-cognitive-complexity==0.1.0", - "requests==2.26.0", - "oauthlib==3.1.1", - "requests-oauthlib==1.3.0", + "requests==2.31.0", + "oauthlib==3.2.0", + "requests-oauthlib==1.3.1", "pydantic==2.10.6", - "pydantic-settings==2.7.1" + "pydantic-settings==2.7.1", + "pystac==1.12.1" ] -[tool.setuptools.packages] -find = {} - [tool.bandit] - [tool.pydocstyle] convention = "google" + +# Add GitLab private registry as a source +# useless comment +[[tool.poetry.source]] +name = "gitlab" +url = "https://git.o-c.space/api/v4/projects/689/packages/pypi/simple" \ No newline at end of file From fcb80da3ddf28d63c338355c30ae68860c9faa2a Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Tue, 4 Feb 2025 15:47:54 +0000 Subject: [PATCH 37/49] Remove hardcoded credentials from config.py --- config/config.py | 62 ++++---- datacosmos/client.py | 10 +- .../client/test_client_authentication.py | 145 +++++++++++------- .../client/test_client_initialization.py | 27 ---- 4 files changed, 136 insertions(+), 108 deletions(-) delete mode 100644 tests/unit/datacosmos/client/test_client_initialization.py diff --git a/config/config.py b/config/config.py index 8cd9226..e4c6164 100644 --- a/config/config.py +++ b/config/config.py @@ -6,10 +6,10 @@ """ import os -from typing import Literal +from typing import Literal, Optional import yaml -from pydantic import model_validator +from pydantic import field_validator from pydantic_settings import BaseSettings, SettingsConfigDict from config.models.m2m_authentication_config import M2MAuthenticationConfig @@ -29,14 +29,8 @@ class Config(BaseSettings): log_format: Literal["json", "text"] = "text" log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR"] = "INFO" - # Authentication configuration - authentication: M2MAuthenticationConfig = M2MAuthenticationConfig( - type="m2m", - client_id="zCeZWJamwnb8ZIQEK35rhx0hSAjsZI4D", - token_url="https://login.open-cosmos.com/oauth/token", - audience="https://test.beeapp.open-cosmos.com", - client_secret="tAeaSgLds7g535ofGq79Zm2DSbWMCOsuRyY5lbyObJe9eAeSN_fxoy-5kaXnVSYa", - ) + # Authentication configuration (must be explicitly provided) + authentication: Optional[M2MAuthenticationConfig] = None # STAC API configuration stac: URL = URL( @@ -64,24 +58,36 @@ def from_yaml(cls, file_path: str = "config/config.yaml") -> "Config": config_data = { k: v for k, v in yaml_data.items() if v not in [None, ""] } + return cls(**config_data) - @model_validator(mode="before") @classmethod - def merge_with_env(cls, values): - """Override settings with environment variables if set. - - This method checks if any environment variables corresponding to the - config fields are set and updates their values accordingly. - - Args: - values (dict): The configuration values before validation. - - Returns: - dict: The updated configuration values with environment variable overrides. - """ - for field in cls.model_fields: - env_value = os.getenv(f"OC_{field.upper()}") - if env_value: - values[field] = env_value - return values + def from_env(cls) -> "Config": + """Load configuration from environment variables.""" + env_auth = { + "type": "m2m", + "client_id": os.getenv("OC_AUTH_CLIENT_ID"), + "token_url": os.getenv("OC_AUTH_TOKEN_URL"), + "audience": os.getenv("OC_AUTH_AUDIENCE"), + "client_secret": os.getenv("OC_AUTH_CLIENT_SECRET"), + } + + if all(env_auth.values()): # Ensure all values exist + env_auth_config = M2MAuthenticationConfig(**env_auth) + else: + env_auth_config = None # If missing, let validation handle it + + return cls(authentication=env_auth_config) + + @field_validator("authentication", mode="before") + @classmethod + def validate_authentication(cls, v): + """Ensure authentication is provided through one of the allowed methods.""" + if v is None: + raise ValueError( + "M2M authentication is required. Please provide it via:" + "\n1. Explicit instantiation (Config(authentication=...))" + "\n2. A YAML config file (config.yaml)" + "\n3. Environment variables (OC_AUTH_CLIENT_ID, OC_AUTH_TOKEN_URL, etc.)" + ) + return v diff --git a/datacosmos/client.py b/datacosmos/client.py index f2e570e..984da88 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -35,7 +35,15 @@ def __init__( self.logger = logging.getLogger(__name__) self.logger.setLevel(logging.INFO) - self.config = config or Config.from_yaml() + if config: + self.config = config + else: + try: + self.config = Config.from_yaml() + except ValueError: + self.logger.info("No valid YAML config found, falling back to env vars.") + self.config = Config.from_env() + self.token = None self.token_expiry = None self._http_client = self._authenticate_and_initialize_client() diff --git a/tests/unit/datacosmos/client/test_client_authentication.py b/tests/unit/datacosmos/client/test_client_authentication.py index 2327c9e..c352ffe 100644 --- a/tests/unit/datacosmos/client/test_client_authentication.py +++ b/tests/unit/datacosmos/client/test_client_authentication.py @@ -1,59 +1,100 @@ from unittest.mock import patch - +import os +import pytest +import yaml from config.config import Config from config.models.m2m_authentication_config import M2MAuthenticationConfig from datacosmos.client import DatacosmosClient -@patch("datacosmos.client.OAuth2Session.fetch_token") -@patch( - "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client", - autospec=True, -) -def test_client_authentication(mock_auth_client, mock_fetch_token): - """Test that the client correctly fetches a token during authentication.""" - # Mock the token response from OAuth2Session - mock_fetch_token.return_value = { - "access_token": "mock-access-token", - "expires_in": 3600, - } - - # Simulate _authenticate_and_initialize_client calling fetch_token - def mock_authenticate_and_initialize_client(self): - # Call the real fetch_token (simulated by the mock) - token_response = mock_fetch_token( - token_url=self.config.authentication.token_url, - client_id=self.config.authentication.client_id, - client_secret=self.config.authentication.client_secret, - audience=self.config.authentication.audience, - ) - self.token = token_response["access_token"] - self.token_expiry = "mock-expiry" - - # Attach the side effect to the mock - mock_auth_client.side_effect = mock_authenticate_and_initialize_client - - # Create a mock configuration - config = Config( - authentication=M2MAuthenticationConfig( - type="m2m", - client_id="test-client-id", - client_secret="test-client-secret", - token_url="https://mock.token.url/oauth/token", - audience="https://mock.audience", +@pytest.mark.usefixtures("mock_fetch_token", "mock_auth_client") +class TestClientAuthentication: + """Test suite for DatacosmosClient authentication.""" + + @pytest.fixture + def mock_fetch_token(self): + """Fixture to mock OAuth2 token fetch.""" + with patch("datacosmos.client.OAuth2Session.fetch_token") as mock: + mock.return_value = { + "access_token": "mock-access-token", + "expires_in": 3600, + } + yield mock + + @pytest.fixture + def mock_auth_client(self, mock_fetch_token): + """Fixture to mock the authentication client initialization.""" + with patch( + "datacosmos.client.DatacosmosClient._authenticate_and_initialize_client", + autospec=True, + ) as mock: + + def mock_authenticate(self): + """Simulate authentication by setting token values.""" + token_response = mock_fetch_token.return_value + self.token = token_response["access_token"] + self.token_expiry = "mock-expiry" + + mock.side_effect = mock_authenticate + yield mock + + def test_authentication_with_explicit_config(self): + """Test authentication when explicitly providing Config.""" + config = Config( + authentication=M2MAuthenticationConfig( + type="m2m", + client_id="test-client-id", + client_secret="test-client-secret", + token_url="https://mock.token.url/oauth/token", + audience="https://mock.audience", + ) ) - ) - - # Initialize the client - client = DatacosmosClient(config=config) - - # Assertions - assert client.token == "mock-access-token" - assert client.token_expiry == "mock-expiry" - mock_fetch_token.assert_called_once_with( - token_url="https://mock.token.url/oauth/token", - client_id="test-client-id", - client_secret="test-client-secret", - audience="https://mock.audience", - ) - mock_auth_client.assert_called_once_with(client) + + client = DatacosmosClient(config=config) + + assert client.token == "mock-access-token" + assert client.token_expiry == "mock-expiry" + + @patch("config.config.Config.from_yaml") + def test_authentication_from_yaml(self, mock_from_yaml, tmp_path): + """Test authentication when loading Config from YAML file.""" + config_path = tmp_path / "config.yaml" + yaml_data = { + "authentication": { + "type": "m2m", + "client_id": "test-client-id", + "client_secret": "test-client-secret", + "token_url": "https://mock.token.url/oauth/token", + "audience": "https://mock.audience", + } + } + + with open(config_path, "w") as f: + yaml.dump(yaml_data, f) + + mock_from_yaml.return_value = Config.from_yaml(str(config_path)) + + # Clear any previous calls before instantiating the client + mock_from_yaml.reset_mock() + + client = DatacosmosClient() + + assert client.token == "mock-access-token" + assert client.token_expiry == "mock-expiry" + + # Ensure it was called exactly once after reset + mock_from_yaml.assert_called_once() + + + @patch.dict(os.environ, { + "OC_AUTH_CLIENT_ID": "test-client-id", + "OC_AUTH_TOKEN_URL": "https://mock.token.url/oauth/token", + "OC_AUTH_AUDIENCE": "https://mock.audience", + "OC_AUTH_CLIENT_SECRET": "test-client-secret" + }) + def test_authentication_from_env(self): + """Test authentication when loading Config from environment variables.""" + client = DatacosmosClient() + + assert client.token == "mock-access-token" + assert client.token_expiry == "mock-expiry" diff --git a/tests/unit/datacosmos/client/test_client_initialization.py b/tests/unit/datacosmos/client/test_client_initialization.py deleted file mode 100644 index 9422fca..0000000 --- a/tests/unit/datacosmos/client/test_client_initialization.py +++ /dev/null @@ -1,27 +0,0 @@ -from unittest.mock import MagicMock, patch - -from config.config import Config -from config.models.m2m_authentication_config import M2MAuthenticationConfig -from datacosmos.client import DatacosmosClient - - -@patch("datacosmos.client.DatacosmosClient._authenticate_and_initialize_client") -def test_client_initialization(mock_auth_client): - """Test that the client initializes correctly with environment variables - and mocks the HTTP client.""" - mock_config = Config( - authentication=M2MAuthenticationConfig( - type="m2m", - client_id="zCeZWJamwnb8ZIQEK35rhx0hSAjsZI4D", - token_url="https://login.open-cosmos.com/oauth/token", - audience="https://test.beeapp.open-cosmos.com", - client_secret="tAeaSgLds7g535ofGq79Zm2DSbWMCOsuRyY5lbyObJe9eAeSN_fxoy-5kaXnVSYa", - ) - ) - mock_auth_client.return_value = MagicMock() # Mock the HTTP client - - client = DatacosmosClient() - - assert client.config == mock_config - assert client._http_client is not None # Ensure the HTTP client is mocked - mock_auth_client.assert_called_once() From 16558a25abfce99c0cdfbb0de8ae12d121be4c33 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Tue, 4 Feb 2025 18:01:52 +0000 Subject: [PATCH 38/49] Allow sdk user to say what log levels he/she wants to see when using the sdk. It defaults to all errors more critical than info --- datacosmos/client.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/datacosmos/client.py b/datacosmos/client.py index 984da88..3aed0d9 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -1,12 +1,6 @@ -"""DatacosmosClient handles authenticated interactions with the Datacosmos API. - -Automatically manages token refreshing and provides HTTP convenience -methods. -""" - import logging from datetime import datetime, timedelta, timezone -from typing import Any, Optional +from typing import Any, Optional, Literal import requests from oauthlib.oauth2 import BackendApplicationClient @@ -26,15 +20,19 @@ class DatacosmosClient: def __init__( self, config: Optional[Config] = None, + log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO", ): """Initialize the DatacosmosClient. - Load configuration from the specified YAML file, environment variables, - or fallback to the default values provided in the `Config` class. + Args: + config (Optional[Config]): Configuration object. + log_level (Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]): The logging level. """ + # Initialize logger self.logger = logging.getLogger(__name__) - self.logger.setLevel(logging.INFO) + self.set_log_level(log_level) + # Load configuration from input, YAML, or environment variables if config: self.config = config else: @@ -48,10 +46,19 @@ def __init__( self.token_expiry = None self._http_client = self._authenticate_and_initialize_client() + def set_log_level(self, level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]): + """Set the logging level based on user input. + + Args: + level (Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]): The logging level. + """ + log_level = getattr(logging, level.upper(), logging.INFO) + self.logger.setLevel(log_level) + def _authenticate_and_initialize_client(self) -> requests.Session: """Authenticate and initialize the HTTP client with a valid token.""" try: - self.logger.info("Authenticating with the token endpoint") + self.logger.debug("Authenticating with the token endpoint") client = BackendApplicationClient( client_id=self.config.authentication.client_id ) @@ -69,7 +76,7 @@ def _authenticate_and_initialize_client(self) -> requests.Session: self.token_expiry = datetime.now(timezone.utc) + timedelta( seconds=token_response.get("expires_in", 3600) ) - self.logger.info("Authentication successful, token obtained") + self.logger.debug("Authentication successful, token obtained") # Initialize the HTTP session with the Authorization header http_client = requests.Session() @@ -85,7 +92,7 @@ def _authenticate_and_initialize_client(self) -> requests.Session: def _refresh_token_if_needed(self): """Refresh the token if it has expired.""" if not self.token or self.token_expiry <= datetime.now(timezone.utc): - self.logger.info("Token expired or missing, refreshing token") + self.logger.debug("Token expired or missing, refreshing token") self._http_client = self._authenticate_and_initialize_client() def get_http_client(self) -> requests.Session: @@ -102,10 +109,10 @@ def request( """ self._refresh_token_if_needed() try: - self.logger.info(f"Making {method.upper()} request to {url}") + self.logger.debug(f"Making {method.upper()} request to {url}") response = self._http_client.request(method, url, *args, **kwargs) response.raise_for_status() - self.logger.info( + self.logger.debug( f"Request to {url} succeeded with status {response.status_code}" ) return response From 6b355f5f950b43fb0bf3b51fe53fd5c969cfc4c9 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Tue, 4 Feb 2025 18:03:25 +0000 Subject: [PATCH 39/49] remove useless comment from m2m_authentication_config model --- config/models/m2m_authentication_config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/models/m2m_authentication_config.py b/config/models/m2m_authentication_config.py index 0afca20..45d92ca 100644 --- a/config/models/m2m_authentication_config.py +++ b/config/models/m2m_authentication_config.py @@ -20,5 +20,4 @@ class M2MAuthenticationConfig(BaseModel): client_id: str token_url: str audience: str - # Some infrastructure deployments do not require a client secret. - client_secret: str = "" + client_secret: str From 7a3c8b5a9947c6f4234538615a7224b958010866 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Tue, 4 Feb 2025 18:06:29 +0000 Subject: [PATCH 40/49] Fix CI linting problems --- datacosmos/client.py | 8 +++++++- .../unit/datacosmos/client/test_client_authentication.py | 5 +++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/datacosmos/client.py b/datacosmos/client.py index 3aed0d9..c30c4ae 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -1,6 +1,12 @@ +"""DatacosmosClient handles authenticated interactions with the Datacosmos API. + +Automatically manages token refreshing and provides HTTP convenience +methods. +""" + import logging from datetime import datetime, timedelta, timezone -from typing import Any, Optional, Literal +from typing import Any, Literal, Optional import requests from oauthlib.oauth2 import BackendApplicationClient diff --git a/tests/unit/datacosmos/client/test_client_authentication.py b/tests/unit/datacosmos/client/test_client_authentication.py index c352ffe..dc1a5d3 100644 --- a/tests/unit/datacosmos/client/test_client_authentication.py +++ b/tests/unit/datacosmos/client/test_client_authentication.py @@ -1,7 +1,9 @@ -from unittest.mock import patch import os +from unittest.mock import patch + import pytest import yaml + from config.config import Config from config.models.m2m_authentication_config import M2MAuthenticationConfig from datacosmos.client import DatacosmosClient @@ -85,7 +87,6 @@ def test_authentication_from_yaml(self, mock_from_yaml, tmp_path): # Ensure it was called exactly once after reset mock_from_yaml.assert_called_once() - @patch.dict(os.environ, { "OC_AUTH_CLIENT_ID": "test-client-id", "OC_AUTH_TOKEN_URL": "https://mock.token.url/oauth/token", From 7aafba39a37f5a20ab518b64af8a0c6a2c40e316 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Tue, 4 Feb 2025 18:08:11 +0000 Subject: [PATCH 41/49] Apply black --- datacosmos/client.py | 8 ++++++-- .../client/test_client_authentication.py | 15 +++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/datacosmos/client.py b/datacosmos/client.py index c30c4ae..a6b6177 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -45,14 +45,18 @@ def __init__( try: self.config = Config.from_yaml() except ValueError: - self.logger.info("No valid YAML config found, falling back to env vars.") + self.logger.info( + "No valid YAML config found, falling back to env vars." + ) self.config = Config.from_env() self.token = None self.token_expiry = None self._http_client = self._authenticate_and_initialize_client() - def set_log_level(self, level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]): + def set_log_level( + self, level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] + ): """Set the logging level based on user input. Args: diff --git a/tests/unit/datacosmos/client/test_client_authentication.py b/tests/unit/datacosmos/client/test_client_authentication.py index dc1a5d3..1e536f1 100644 --- a/tests/unit/datacosmos/client/test_client_authentication.py +++ b/tests/unit/datacosmos/client/test_client_authentication.py @@ -87,12 +87,15 @@ def test_authentication_from_yaml(self, mock_from_yaml, tmp_path): # Ensure it was called exactly once after reset mock_from_yaml.assert_called_once() - @patch.dict(os.environ, { - "OC_AUTH_CLIENT_ID": "test-client-id", - "OC_AUTH_TOKEN_URL": "https://mock.token.url/oauth/token", - "OC_AUTH_AUDIENCE": "https://mock.audience", - "OC_AUTH_CLIENT_SECRET": "test-client-secret" - }) + @patch.dict( + os.environ, + { + "OC_AUTH_CLIENT_ID": "test-client-id", + "OC_AUTH_TOKEN_URL": "https://mock.token.url/oauth/token", + "OC_AUTH_AUDIENCE": "https://mock.audience", + "OC_AUTH_CLIENT_SECRET": "test-client-secret", + }, + ) def test_authentication_from_env(self): """Test authentication when loading Config from environment variables.""" client = DatacosmosClient() From 28082005c45aaa0618a1d7f63a8f8200117b1b05 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Wed, 5 Feb 2025 11:14:02 +0000 Subject: [PATCH 42/49] Address comments in MR --- config/config.py | 83 ++++++++++++----- datacosmos/client.py | 88 ++++++------------- datacosmos/exceptions/__init__.py | 1 + datacosmos/exceptions/datacosmos_exception.py | 27 ++++++ 4 files changed, 115 insertions(+), 84 deletions(-) create mode 100644 datacosmos/exceptions/__init__.py create mode 100644 datacosmos/exceptions/datacosmos_exception.py diff --git a/config/config.py b/config/config.py index e4c6164..3ab4b86 100644 --- a/config/config.py +++ b/config/config.py @@ -24,21 +24,12 @@ class Config(BaseSettings): nested_model_default_partial_update=True, ) - # General configurations environment: Literal["local", "test", "prod"] = "test" log_format: Literal["json", "text"] = "text" log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR"] = "INFO" - # Authentication configuration (must be explicitly provided) authentication: Optional[M2MAuthenticationConfig] = None - - # STAC API configuration - stac: URL = URL( - protocol="https", - host="test.app.open-cosmos.com", - port=443, - path="/api/data/v0/stac", - ) + stac: Optional[URL] = None @classmethod def from_yaml(cls, file_path: str = "config/config.yaml") -> "Config": @@ -50,21 +41,27 @@ def from_yaml(cls, file_path: str = "config/config.yaml") -> "Config": Returns: Config: An instance of the Config class with loaded settings. """ - config_data = {} + config_data: dict = {} if os.path.exists(file_path): with open(file_path, "r") as f: yaml_data = yaml.safe_load(f) or {} # Remove empty values from YAML to avoid overwriting with `None` config_data = { - k: v for k, v in yaml_data.items() if v not in [None, ""] + key: value + for key, value in yaml_data.items() + if value not in [None, ""] } return cls(**config_data) @classmethod def from_env(cls) -> "Config": - """Load configuration from environment variables.""" - env_auth = { + """Load configuration from environment variables. + + Returns: + Config: An instance of the Config class with settings loaded from environment variables. + """ + auth_env_vars: dict[str, Optional[str]] = { "type": "m2m", "client_id": os.getenv("OC_AUTH_CLIENT_ID"), "token_url": os.getenv("OC_AUTH_TOKEN_URL"), @@ -72,22 +69,62 @@ def from_env(cls) -> "Config": "client_secret": os.getenv("OC_AUTH_CLIENT_SECRET"), } - if all(env_auth.values()): # Ensure all values exist - env_auth_config = M2MAuthenticationConfig(**env_auth) - else: - env_auth_config = None # If missing, let validation handle it + authentication_config: Optional[M2MAuthenticationConfig] = ( + M2MAuthenticationConfig(**auth_env_vars) + if all(auth_env_vars.values()) + else None + ) - return cls(authentication=env_auth_config) + stac_config: URL = URL( + protocol=os.getenv("OC_STAC_PROTOCOL", "https"), + host=os.getenv("OC_STAC_HOST", "test.app.open-cosmos.com"), + port=int(os.getenv("OC_STAC_PORT", "443")), + path=os.getenv("OC_STAC_PATH", "/api/data/v0/stac"), + ) + + return cls(authentication=authentication_config, stac=stac_config) @field_validator("authentication", mode="before") @classmethod - def validate_authentication(cls, v): - """Ensure authentication is provided through one of the allowed methods.""" - if v is None: + def validate_authentication( + cls, auth_config: Optional[M2MAuthenticationConfig] + ) -> M2MAuthenticationConfig: + """Ensure authentication is provided through one of the allowed methods. + + Args: + auth_config (Optional[M2MAuthenticationConfig]): The authentication config to validate. + + Returns: + M2MAuthenticationConfig: The validated authentication configuration. + + Raises: + ValueError: If authentication is missing. + """ + if auth_config is None: raise ValueError( "M2M authentication is required. Please provide it via:" "\n1. Explicit instantiation (Config(authentication=...))" "\n2. A YAML config file (config.yaml)" "\n3. Environment variables (OC_AUTH_CLIENT_ID, OC_AUTH_TOKEN_URL, etc.)" ) - return v + return auth_config + + @field_validator("stac", mode="before") + @classmethod + def validate_stac(cls, stac_config: Optional[URL]) -> URL: + """Ensure STAC configuration has a default if not explicitly set. + + Args: + stac_config (Optional[URL]): The STAC config to validate. + + Returns: + URL: The validated STAC configuration. + """ + if stac_config is None: + return URL( + protocol="https", + host="test.app.open-cosmos.com", + port=443, + path="/api/data/v0/stac", + ) + return stac_config diff --git a/datacosmos/client.py b/datacosmos/client.py index a6b6177..d631b14 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -4,77 +4,47 @@ methods. """ -import logging from datetime import datetime, timedelta, timezone -from typing import Any, Literal, Optional +from typing import Any, Optional import requests from oauthlib.oauth2 import BackendApplicationClient -from requests.exceptions import RequestException +from requests.exceptions import ConnectionError, HTTPError, RequestException, Timeout from requests_oauthlib import OAuth2Session from config.config import Config +from datacosmos.exceptions.datacosmos_exception import DatacosmosException class DatacosmosClient: - """DatacosmosClient handles authenticated interactions with the Datacosmos API. + """Client to interact with the Datacosmos API with authentication and request handling.""" - Automatically manages token refreshing and provides HTTP convenience - methods. - """ - - def __init__( - self, - config: Optional[Config] = None, - log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO", - ): + def __init__(self, config: Optional[Config] = None): """Initialize the DatacosmosClient. Args: config (Optional[Config]): Configuration object. - log_level (Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]): The logging level. """ - # Initialize logger - self.logger = logging.getLogger(__name__) - self.set_log_level(log_level) - - # Load configuration from input, YAML, or environment variables if config: self.config = config else: try: self.config = Config.from_yaml() except ValueError: - self.logger.info( - "No valid YAML config found, falling back to env vars." - ) self.config = Config.from_env() self.token = None self.token_expiry = None self._http_client = self._authenticate_and_initialize_client() - def set_log_level( - self, level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] - ): - """Set the logging level based on user input. - - Args: - level (Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]): The logging level. - """ - log_level = getattr(logging, level.upper(), logging.INFO) - self.logger.setLevel(log_level) - def _authenticate_and_initialize_client(self) -> requests.Session: """Authenticate and initialize the HTTP client with a valid token.""" try: - self.logger.debug("Authenticating with the token endpoint") client = BackendApplicationClient( client_id=self.config.authentication.client_id ) oauth_session = OAuth2Session(client=client) - # Fetch the token using client credentials token_response = oauth_session.fetch_token( token_url=self.config.authentication.token_url, client_id=self.config.authentication.client_id, @@ -86,52 +56,48 @@ def _authenticate_and_initialize_client(self) -> requests.Session: self.token_expiry = datetime.now(timezone.utc) + timedelta( seconds=token_response.get("expires_in", 3600) ) - self.logger.debug("Authentication successful, token obtained") - # Initialize the HTTP session with the Authorization header http_client = requests.Session() http_client.headers.update({"Authorization": f"Bearer {self.token}"}) return http_client + except (HTTPError, ConnectionError, Timeout) as e: + raise DatacosmosException(f"Authentication failed: {str(e)}") from e except RequestException as e: - self.logger.error(f"Request failed during authentication: {e}") - raise - except Exception as e: - self.logger.error(f"Unexpected error during authentication: {e}") - raise + raise DatacosmosException( + f"Unexpected request failure during authentication: {str(e)}" + ) from e def _refresh_token_if_needed(self): """Refresh the token if it has expired.""" if not self.token or self.token_expiry <= datetime.now(timezone.utc): - self.logger.debug("Token expired or missing, refreshing token") self._http_client = self._authenticate_and_initialize_client() - def get_http_client(self) -> requests.Session: - """Return the authenticated HTTP client, refreshing the token if necessary.""" - self._refresh_token_if_needed() - return self._http_client - def request( self, method: str, url: str, *args: Any, **kwargs: Any ) -> requests.Response: - """Send an HTTP request using the authenticated session. - - Logs request and response details. - """ + """Send an HTTP request using the authenticated session.""" self._refresh_token_if_needed() try: - self.logger.debug(f"Making {method.upper()} request to {url}") response = self._http_client.request(method, url, *args, **kwargs) response.raise_for_status() - self.logger.debug( - f"Request to {url} succeeded with status {response.status_code}" - ) return response + except HTTPError as e: + raise DatacosmosException( + f"HTTP error during {method.upper()} request to {url}", + response=e.response, + ) from e + except ConnectionError as e: + raise DatacosmosException( + f"Connection error during {method.upper()} request to {url}: {str(e)}" + ) from e + except Timeout as e: + raise DatacosmosException( + f"Request timeout during {method.upper()} request to {url}: {str(e)}" + ) from e except RequestException as e: - self.logger.error(f"HTTP request failed: {e}") - raise - except Exception as e: - self.logger.error(f"Unexpected error during HTTP request: {e}") - raise + raise DatacosmosException( + f"Unexpected request failure during {method.upper()} request to {url}: {str(e)}" + ) from e def get(self, url: str, *args: Any, **kwargs: Any) -> requests.Response: """Send a GET request using the authenticated session.""" diff --git a/datacosmos/exceptions/__init__.py b/datacosmos/exceptions/__init__.py new file mode 100644 index 0000000..17476aa --- /dev/null +++ b/datacosmos/exceptions/__init__.py @@ -0,0 +1 @@ +"""Exceptions for the datacosmos package.""" diff --git a/datacosmos/exceptions/datacosmos_exception.py b/datacosmos/exceptions/datacosmos_exception.py new file mode 100644 index 0000000..5639aca --- /dev/null +++ b/datacosmos/exceptions/datacosmos_exception.py @@ -0,0 +1,27 @@ +"""Base exception class for all Datacosmos SDK exceptions.""" + +from typing import Optional + +from requests import Response +from requests.exceptions import RequestException + + +class DatacosmosException(RequestException): + """Base exception class for all Datacosmos SDK exceptions.""" + + def __init__(self, message: str, response: Optional[Response] = None): + """Initialize DatacosmosException. + + Args: + message (str): The error message. + response (Optional[Response]): The HTTP response object, if available. + """ + self.response = response + self.status_code = response.status_code if response else None + self.details = response.text if response else None + full_message = ( + f"{message} (Status: {self.status_code}, Details: {self.details})" + if response + else message + ) + super().__init__(full_message) From cc803306bf7d732a5d3156469e2e90afaebf6a7c Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Wed, 5 Feb 2025 11:15:51 +0000 Subject: [PATCH 43/49] Apply isort --- datacosmos/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datacosmos/client.py b/datacosmos/client.py index d631b14..93220ed 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -9,7 +9,8 @@ import requests from oauthlib.oauth2 import BackendApplicationClient -from requests.exceptions import ConnectionError, HTTPError, RequestException, Timeout +from requests.exceptions import (ConnectionError, HTTPError, RequestException, + Timeout) from requests_oauthlib import OAuth2Session from config.config import Config From 84ea9da84c9aec66fcd6ea1647a815fa8dc13ba4 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Wed, 5 Feb 2025 11:31:18 +0000 Subject: [PATCH 44/49] Apply changes in the pipeline --- .github/workflows/main.yaml | 28 +++++----------------------- datacosmos/client.py | 3 +-- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 4374c3f..d2983ed 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -24,10 +24,10 @@ jobs: run: poetry config http-basic.gitlab __token__ ${{ secrets.GITLAB_PYPI_TOKEN }} - name: install dependencies run: poetry install --no-interaction - - name: run linters manually with poetry + - name: run isort before black run: | - poetry run black . --check - poetry run isort . --check-only + poetry run isort . --check-only # Run isort first + poetry run black . --check # Then black poetry run flake8 . bandit: @@ -66,24 +66,6 @@ jobs: - name: cognitive run: poetry run flake8 . --max-cognitive-complexity=5 --ignore=E501 - isort: - name: isort - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: set up python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: set up poetry - run: curl -sSL https://install.python-poetry.org | python3 - - - name: configure gitlab auth - run: poetry config http-basic.gitlab __token__ ${{ secrets.GITLAB_PYPI_TOKEN }} - - name: install dependencies - run: poetry install --no-interaction - - name: isort - uses: isort/isort-action@v1.1.0 - pydocstyle: name: pydocstyle runs-on: ubuntu-latest @@ -105,7 +87,7 @@ jobs: test: name: test runs-on: ubuntu-latest - needs: [bandit, cognitive, isort, lint, pydocstyle] + needs: [bandit, cognitive, lint, pydocstyle] steps: - uses: actions/checkout@v3 - name: set up python @@ -167,4 +149,4 @@ jobs: ## Included Pull Requests - ${{ steps.changelog.outputs.changes }} \ No newline at end of file + ${{ steps.changelog.outputs.changes }} diff --git a/datacosmos/client.py b/datacosmos/client.py index 93220ed..d631b14 100644 --- a/datacosmos/client.py +++ b/datacosmos/client.py @@ -9,8 +9,7 @@ import requests from oauthlib.oauth2 import BackendApplicationClient -from requests.exceptions import (ConnectionError, HTTPError, RequestException, - Timeout) +from requests.exceptions import ConnectionError, HTTPError, RequestException, Timeout from requests_oauthlib import OAuth2Session from config.config import Config From 982197db77c45923db53a5a8de2419b23ee1e8ef Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Wed, 5 Feb 2025 11:39:29 +0000 Subject: [PATCH 45/49] Add changes in pyproject.toml to make isort behave similarly to black --- pyproject.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index f07db41..bdb8c33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,14 @@ dependencies = [ [tool.pydocstyle] convention = "google" +[tool.isort] +profile = "black" +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +line_length = 88 + # Add GitLab private registry as a source # useless comment [[tool.poetry.source]] From d26e181a8ba2c1181d5e4ddcf8f7f4a0ea3cefa9 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Wed, 5 Feb 2025 12:04:07 +0000 Subject: [PATCH 46/49] Allow default values for type, token_url and audience --- config/config.py | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/config/config.py b/config/config.py index 3ab4b86..e14d731 100644 --- a/config/config.py +++ b/config/config.py @@ -6,7 +6,7 @@ """ import os -from typing import Literal, Optional +from typing import ClassVar, Literal, Optional import yaml from pydantic import field_validator @@ -31,6 +31,10 @@ class Config(BaseSettings): authentication: Optional[M2MAuthenticationConfig] = None stac: Optional[URL] = None + DEFAULT_AUTH_TYPE: ClassVar[str] = "m2m" + DEFAULT_AUTH_TOKEN_URL: ClassVar[str] = "https://login.open-cosmos.com/oauth/token" + DEFAULT_AUTH_AUDIENCE: ClassVar[str] = "https://test.beeapp.open-cosmos.com" + @classmethod def from_yaml(cls, file_path: str = "config/config.yaml") -> "Config": """Load configuration from a YAML file and override defaults. @@ -61,21 +65,15 @@ def from_env(cls) -> "Config": Returns: Config: An instance of the Config class with settings loaded from environment variables. """ - auth_env_vars: dict[str, Optional[str]] = { - "type": "m2m", - "client_id": os.getenv("OC_AUTH_CLIENT_ID"), - "token_url": os.getenv("OC_AUTH_TOKEN_URL"), - "audience": os.getenv("OC_AUTH_AUDIENCE"), - "client_secret": os.getenv("OC_AUTH_CLIENT_SECRET"), - } - - authentication_config: Optional[M2MAuthenticationConfig] = ( - M2MAuthenticationConfig(**auth_env_vars) - if all(auth_env_vars.values()) - else None + authentication_config = M2MAuthenticationConfig( + type=os.getenv("OC_AUTH_TYPE", cls.DEFAULT_AUTH_TYPE), + client_id=os.getenv("OC_AUTH_CLIENT_ID"), + client_secret=os.getenv("OC_AUTH_CLIENT_SECRET"), + token_url=os.getenv("OC_AUTH_TOKEN_URL", cls.DEFAULT_AUTH_TOKEN_URL), + audience=os.getenv("OC_AUTH_AUDIENCE", cls.DEFAULT_AUTH_AUDIENCE), ) - stac_config: URL = URL( + stac_config = URL( protocol=os.getenv("OC_STAC_PROTOCOL", "https"), host=os.getenv("OC_STAC_HOST", "test.app.open-cosmos.com"), port=int(os.getenv("OC_STAC_PORT", "443")), @@ -89,7 +87,7 @@ def from_env(cls) -> "Config": def validate_authentication( cls, auth_config: Optional[M2MAuthenticationConfig] ) -> M2MAuthenticationConfig: - """Ensure authentication is provided through one of the allowed methods. + """Ensure authentication is provided and defaults are applied where necessary. Args: auth_config (Optional[M2MAuthenticationConfig]): The authentication config to validate. @@ -98,15 +96,30 @@ def validate_authentication( M2MAuthenticationConfig: The validated authentication configuration. Raises: - ValueError: If authentication is missing. + ValueError: If client_id or client_secret is missing. """ if auth_config is None: raise ValueError( "M2M authentication is required. Please provide it via:" "\n1. Explicit instantiation (Config(authentication=...))" "\n2. A YAML config file (config.yaml)" - "\n3. Environment variables (OC_AUTH_CLIENT_ID, OC_AUTH_TOKEN_URL, etc.)" + "\n3. Environment variables (OC_AUTH_CLIENT_ID, OC_AUTH_CLIENT_SECRET, etc.)" ) + + if isinstance(auth_config, dict): + auth_config = M2MAuthenticationConfig(**auth_config) + + # Apply defaults for missing values + auth_config.type = auth_config.type or cls.DEFAULT_AUTH_TYPE + auth_config.token_url = auth_config.token_url or cls.DEFAULT_AUTH_TOKEN_URL + auth_config.audience = auth_config.audience or cls.DEFAULT_AUTH_AUDIENCE + + # Ensure critical values are provided + if not auth_config.client_id or not auth_config.client_secret: + raise ValueError( + "client_id and client_secret are required for authentication." + ) + return auth_config @field_validator("stac", mode="before") From 84e077f0ac0cad6c79847112b21602156c36a092 Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Wed, 5 Feb 2025 12:07:48 +0000 Subject: [PATCH 47/49] decrease cognitive load --- config/config.py | 51 +++++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/config/config.py b/config/config.py index e14d731..7a5ca6c 100644 --- a/config/config.py +++ b/config/config.py @@ -85,12 +85,12 @@ def from_env(cls) -> "Config": @field_validator("authentication", mode="before") @classmethod def validate_authentication( - cls, auth_config: Optional[M2MAuthenticationConfig] + cls, auth_data: Optional[dict] ) -> M2MAuthenticationConfig: - """Ensure authentication is provided and defaults are applied where necessary. + """Ensure authentication is provided and apply defaults. Args: - auth_config (Optional[M2MAuthenticationConfig]): The authentication config to validate. + auth_data (Optional[dict]): The authentication config as a dictionary. Returns: M2MAuthenticationConfig: The validated authentication configuration. @@ -98,29 +98,36 @@ def validate_authentication( Raises: ValueError: If client_id or client_secret is missing. """ - if auth_config is None: + if not auth_data: raise ValueError( - "M2M authentication is required. Please provide it via:" - "\n1. Explicit instantiation (Config(authentication=...))" - "\n2. A YAML config file (config.yaml)" - "\n3. Environment variables (OC_AUTH_CLIENT_ID, OC_AUTH_CLIENT_SECRET, etc.)" + "M2M authentication is required. Provide it via:\n" + "1. Explicit instantiation (Config(authentication=...))\n" + "2. A YAML config file (config.yaml)\n" + "3. Environment variables (OC_AUTH_CLIENT_ID, OC_AUTH_CLIENT_SECRET, etc.)" ) - if isinstance(auth_config, dict): - auth_config = M2MAuthenticationConfig(**auth_config) - - # Apply defaults for missing values - auth_config.type = auth_config.type or cls.DEFAULT_AUTH_TYPE - auth_config.token_url = auth_config.token_url or cls.DEFAULT_AUTH_TOKEN_URL - auth_config.audience = auth_config.audience or cls.DEFAULT_AUTH_AUDIENCE - - # Ensure critical values are provided - if not auth_config.client_id or not auth_config.client_secret: - raise ValueError( - "client_id and client_secret are required for authentication." - ) + # Convert dict to M2MAuthenticationConfig + auth = ( + M2MAuthenticationConfig(**auth_data) + if isinstance(auth_data, dict) + else auth_data + ) - return auth_config + # Apply defaults where necessary + auth.type = auth.type or cls.DEFAULT_AUTH_TYPE + auth.token_url = auth.token_url or cls.DEFAULT_AUTH_TOKEN_URL + auth.audience = auth.audience or cls.DEFAULT_AUTH_AUDIENCE + + # Validate required fields + missing_fields = [ + field + for field in ("client_id", "client_secret") + if not getattr(auth, field) + ] + if missing_fields: + raise ValueError(f"Missing required fields: {', '.join(missing_fields)}") + + return auth @field_validator("stac", mode="before") @classmethod From 5e2135cfc0b77f9f556d09595b3fbea170f46d8f Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Wed, 5 Feb 2025 12:11:47 +0000 Subject: [PATCH 48/49] Attempt to decrease cognitive complexity even more --- config/config.py | 48 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/config/config.py b/config/config.py index 7a5ca6c..cb8845f 100644 --- a/config/config.py +++ b/config/config.py @@ -96,38 +96,58 @@ def validate_authentication( M2MAuthenticationConfig: The validated authentication configuration. Raises: - ValueError: If client_id or client_secret is missing. + ValueError: If authentication is missing or required fields are not set. """ if not auth_data: - raise ValueError( - "M2M authentication is required. Provide it via:\n" - "1. Explicit instantiation (Config(authentication=...))\n" - "2. A YAML config file (config.yaml)\n" - "3. Environment variables (OC_AUTH_CLIENT_ID, OC_AUTH_CLIENT_SECRET, etc.)" - ) + cls.raise_missing_auth_error() + + auth = cls.parse_auth_config(auth_data) + auth = cls.apply_auth_defaults(auth) + + cls.check_required_auth_fields(auth) + return auth + + @staticmethod + def raise_missing_auth_error(): + """Raise an error when authentication is missing.""" + raise ValueError( + "M2M authentication is required. Provide it via:\n" + "1. Explicit instantiation (Config(authentication=...))\n" + "2. A YAML config file (config.yaml)\n" + "3. Environment variables (OC_AUTH_CLIENT_ID, OC_AUTH_CLIENT_SECRET, etc.)" + ) - # Convert dict to M2MAuthenticationConfig - auth = ( + @staticmethod + def parse_auth_config(auth_data: dict) -> M2MAuthenticationConfig: + """Convert dictionary input to M2MAuthenticationConfig object.""" + return ( M2MAuthenticationConfig(**auth_data) if isinstance(auth_data, dict) else auth_data ) - # Apply defaults where necessary + @classmethod + def apply_auth_defaults( + cls, auth: M2MAuthenticationConfig + ) -> M2MAuthenticationConfig: + """Apply default authentication values if they are missing.""" auth.type = auth.type or cls.DEFAULT_AUTH_TYPE auth.token_url = auth.token_url or cls.DEFAULT_AUTH_TOKEN_URL auth.audience = auth.audience or cls.DEFAULT_AUTH_AUDIENCE + return auth - # Validate required fields + @staticmethod + def check_required_auth_fields(auth: M2MAuthenticationConfig): + """Ensure required fields (client_id, client_secret) are provided.""" missing_fields = [ field for field in ("client_id", "client_secret") if not getattr(auth, field) ] if missing_fields: - raise ValueError(f"Missing required fields: {', '.join(missing_fields)}") - - return auth + raise ValueError( + f"Missing required authentication fields: {', '.join(missing_fields)}" + ) @field_validator("stac", mode="before") @classmethod From 9e3d7d19c41ca6fe58643f500745b73445e100ff Mon Sep 17 00:00:00 2001 From: "tiago.peres.sousa" Date: Thu, 6 Feb 2025 11:30:58 +0000 Subject: [PATCH 49/49] use prod values as default values; remove logging config --- config/config.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/config/config.py b/config/config.py index cb8845f..35f7c81 100644 --- a/config/config.py +++ b/config/config.py @@ -6,7 +6,7 @@ """ import os -from typing import ClassVar, Literal, Optional +from typing import ClassVar, Optional import yaml from pydantic import field_validator @@ -24,16 +24,12 @@ class Config(BaseSettings): nested_model_default_partial_update=True, ) - environment: Literal["local", "test", "prod"] = "test" - log_format: Literal["json", "text"] = "text" - log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR"] = "INFO" - authentication: Optional[M2MAuthenticationConfig] = None stac: Optional[URL] = None DEFAULT_AUTH_TYPE: ClassVar[str] = "m2m" DEFAULT_AUTH_TOKEN_URL: ClassVar[str] = "https://login.open-cosmos.com/oauth/token" - DEFAULT_AUTH_AUDIENCE: ClassVar[str] = "https://test.beeapp.open-cosmos.com" + DEFAULT_AUTH_AUDIENCE: ClassVar[str] = "https://beeapp.open-cosmos.com" @classmethod def from_yaml(cls, file_path: str = "config/config.yaml") -> "Config": @@ -75,7 +71,7 @@ def from_env(cls) -> "Config": stac_config = URL( protocol=os.getenv("OC_STAC_PROTOCOL", "https"), - host=os.getenv("OC_STAC_HOST", "test.app.open-cosmos.com"), + host=os.getenv("OC_STAC_HOST", "app.open-cosmos.com"), port=int(os.getenv("OC_STAC_PORT", "443")), path=os.getenv("OC_STAC_PATH", "/api/data/v0/stac"), ) @@ -163,7 +159,7 @@ def validate_stac(cls, stac_config: Optional[URL]) -> URL: if stac_config is None: return URL( protocol="https", - host="test.app.open-cosmos.com", + host="app.open-cosmos.com", port=443, path="/api/data/v0/stac", )