feat: namespaced caches (#471)

CHANGELOG.rst

@@ -5,6 +5,8 @@ Changelog
 Next
 ====
 
+- Allow finer grain for the requests cache (#487)
+
 Version 1.2.27 - 2024-09-10
 ===========================
 

examples/preset.py

@@ -31,7 +31,7 @@
 
     SQL = """
     SELECT * FROM
-    "https://d90230ca.us1a.app-sdx.preset.io/api/v1/chart/"
+    "https://12345678.us1a.app.preset.io/api/v1/chart/"
     LIMIT 12
     """
     for row in cursor.execute(SQL):

src/shillelagh/adapters/api/github.py

@@ -10,12 +10,12 @@
 from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TypedDict
 
 import jsonpath
-import requests_cache
 
 from shillelagh.adapters.base import Adapter
 from shillelagh.exceptions import ProgrammingError
 from shillelagh.fields import Boolean, DateTime, Field, Integer, String, StringDateTime
 from shillelagh.filters import Equal, Filter
+from shillelagh.lib import get_session
 from shillelagh.typing import RequestedOrder, Row
 
 _logger = logging.getLogger(__name__)
@@ -220,11 +220,10 @@ def __init__(  # pylint: disable=too-many-arguments
         self.resource = resource
         self.access_token = access_token
 
-        # use a cache for the API requests
-        self._session = requests_cache.CachedSession(
+        self._session = get_session(
+            request_headers={},
             cache_name="github_cache",
-            backend="sqlite",
-            expire_after=180,
+            expire_after=timedelta(minutes=3),
         )
 
     def get_columns(self) -> Dict[str, Field]:

src/shillelagh/adapters/api/socrata.py

@@ -7,18 +7,18 @@
 import logging
 import re
 import urllib.parse
+from datetime import timedelta
 from pathlib import Path
 from typing import Any, Dict, Iterator, List, Optional, Tuple, Type, Union
 
-import requests_cache
 from requests import Request
 from typing_extensions import TypedDict
 
 from shillelagh.adapters.base import Adapter
 from shillelagh.exceptions import ImpossibleFilterError, ProgrammingError
 from shillelagh.fields import Field, Order, String, StringDate
 from shillelagh.filters import Equal, Filter, IsNotNull, IsNull, Like, NotEqual, Range
-from shillelagh.lib import SimpleCostModel, build_sql, flatten
+from shillelagh.lib import SimpleCostModel, build_sql, flatten, get_session
 from shillelagh.typing import RequestedOrder, Row
 
 _logger = logging.getLogger(__name__)
@@ -126,10 +126,10 @@ def __init__(self, netloc: str, dataset_id: str, app_token: Optional[str] = None
         self.app_token = app_token
 
         # use a cache for the API requests
-        self._session = requests_cache.CachedSession(
+        self._session = get_session(
+            request_headers={},
             cache_name="socrata_cache",
-            backend="sqlite",
-            expire_after=180,
+            expire_after=timedelta(minutes=3),
         )
 
         self._set_columns()

src/shillelagh/adapters/api/weatherapi.py

@@ -9,12 +9,12 @@
 
 import dateutil.parser
 import dateutil.tz
-import requests_cache
 
 from shillelagh.adapters.base import Adapter
 from shillelagh.exceptions import ImpossibleFilterError
 from shillelagh.fields import DateTime, Float, IntBoolean, Integer, Order, String
 from shillelagh.filters import Filter, Impossible, Operator, Range
+from shillelagh.lib import get_session
 from shillelagh.typing import RequestedOrder, Row
 
 _logger = logging.getLogger(__name__)
@@ -150,10 +150,10 @@ def __init__(self, location: str, api_key: str, window: int = 7):
 
         # use a cache, since the adapter does a lot of similar API requests,
         # and the data should rarely (never?) change
-        self._session = requests_cache.CachedSession(
+        self._session = get_session(
+            request_headers={},
             cache_name="weatherapi_cache",
-            backend="sqlite",
-            expire_after=180,
+            expire_after=timedelta(minutes=3),
         )
 
     def get_cost(

src/shillelagh/lib.py

@@ -611,16 +611,26 @@ def best_index_object_available() -> bool:
     return bool(Version(apsw.apswversion()) >= Version("3.41.0.0"))
 
 
+def create_namespaced_cache_key(cache_name: str) -> str:
+    """
+    Get the cache name with a specific namespace.
+
+    This function does nothing. It can be monkeypatched to add a namespace to the cache,
+    if one is needed -- eg, when using the library in a multi-tenant environment.
+    """
+    return cache_name
+
+
 def get_session(
     request_headers: Dict[str, str],
     cache_name: str,
     expire_after: timedelta = CACHE_EXPIRATION,
-) -> requests_cache.CachedSession:  # E: line too long (81 > 79 characters)
+) -> requests_cache.CachedSession:
     """
     Return a cached session.
     """
     session = requests_cache.CachedSession(
-        cache_name=cache_name,
+        cache_name=create_namespaced_cache_key(cache_name),
         backend="sqlite",
         expire_after=(
             requests_cache.DO_NOT_CACHE

tests/adapters/api/github_test.py

@@ -28,7 +28,7 @@ def test_github(mocker: MockerFixture, requests_mock: Mocker) -> None:
     Test a simple request.
     """
     mocker.patch(
-        "shillelagh.adapters.api.github.requests_cache.CachedSession",
+        "shillelagh.adapters.api.github.get_session",
         return_value=Session(),
     )
 
@@ -204,7 +204,7 @@ def test_github_single_resource(mocker: MockerFixture, requests_mock: Mocker) ->
     Test a request to a single resource.
     """
     mocker.patch(
-        "shillelagh.adapters.api.github.requests_cache.CachedSession",
+        "shillelagh.adapters.api.github.get_session",
         return_value=Session(),
     )
 
@@ -247,7 +247,7 @@ def test_github_single_resource_with_offset(
     Test a request to a single resource.
     """
     mocker.patch(
-        "shillelagh.adapters.api.github.requests_cache.CachedSession",
+        "shillelagh.adapters.api.github.get_session",
         return_value=Session(),
     )
 
@@ -273,7 +273,7 @@ def test_github_rate_limit(mocker: MockerFixture, requests_mock: Mocker) -> None
     Test that the adapter was rate limited by the API.
     """
     mocker.patch(
-        "shillelagh.adapters.api.github.requests_cache.CachedSession",
+        "shillelagh.adapters.api.github.get_session",
         return_value=Session(),
     )
 
@@ -315,7 +315,7 @@ def test_github_auth_token(mocker: MockerFixture, requests_mock: Mocker) -> None
     Test a simple request.
     """
     mocker.patch(
-        "shillelagh.adapters.api.github.requests_cache.CachedSession",
+        "shillelagh.adapters.api.github.get_session",
         return_value=Session(),
     )
 
@@ -358,7 +358,7 @@ def test_get_multiple_resources(mocker: MockerFixture, requests_mock: Mocker) ->
     """
     mocker.patch("shillelagh.adapters.api.github.PAGE_SIZE", new=5)
     mocker.patch(
-        "shillelagh.adapters.api.github.requests_cache.CachedSession",
+        "shillelagh.adapters.api.github.get_session",
         return_value=Session(),
     )
 
@@ -553,7 +553,7 @@ def test_github_missing_field(mocker: MockerFixture, requests_mock: Mocker) -> N
     For example, some issues don't have the ``draft`` field in the response.
     """
     mocker.patch(
-        "shillelagh.adapters.api.github.requests_cache.CachedSession",
+        "shillelagh.adapters.api.github.get_session",
         return_value=Session(),
     )
 
@@ -590,7 +590,7 @@ def test_github_json_field(mocker: MockerFixture, requests_mock: Mocker) -> None
     Test a request when the response has a JSON field.
     """
     mocker.patch(
-        "shillelagh.adapters.api.github.requests_cache.CachedSession",
+        "shillelagh.adapters.api.github.get_session",
         return_value=Session(),
     )
 
@@ -674,7 +674,7 @@ def test_github_participation(mocker: MockerFixture, requests_mock: Mocker) -> N
     Test a request to the participation stats.
     """
     mocker.patch(
-        "shillelagh.adapters.api.github.requests_cache.CachedSession",
+        "shillelagh.adapters.api.github.get_session",
         return_value=Session(),
     )
 

tests/adapters/api/socrata_test.py

@@ -23,7 +23,7 @@ def test_socrata(mocker: MockerFixture, requests_mock: Mocker) -> None:
     Test a simple query.
     """
     mocker.patch(
-        "shillelagh.adapters.api.socrata.requests_cache.CachedSession",
+        "shillelagh.adapters.api.socrata.get_session",
         return_value=Session(),
     )
 
@@ -62,7 +62,7 @@ def test_socrata_app_token_url(mocker: MockerFixture, requests_mock: Mocker) ->
     Test app token being passed via the URL.
     """
     mocker.patch(
-        "shillelagh.adapters.api.socrata.requests_cache.CachedSession",
+        "shillelagh.adapters.api.socrata.get_session",
         return_value=Session(),
     )
 
@@ -96,7 +96,7 @@ def test_socrata_app_token_connection(
     Test app token being passed via the connection instead of the URL.
     """
     mocker.patch(
-        "shillelagh.adapters.api.socrata.requests_cache.CachedSession",
+        "shillelagh.adapters.api.socrata.get_session",
         return_value=Session(),
     )
 
@@ -129,7 +129,7 @@ def test_socrata_no_data(mocker: MockerFixture, requests_mock: Mocker) -> None:
     Test that some queries return no data.
     """
     mocker.patch(
-        "shillelagh.adapters.api.socrata.requests_cache.CachedSession",
+        "shillelagh.adapters.api.socrata.get_session",
         return_value=Session(),
     )
 
@@ -160,7 +160,7 @@ def test_socrata_impossible(mocker: MockerFixture, requests_mock: Mocker) -> Non
     Test that impossible queries return no data.
     """
     mocker.patch(
-        "shillelagh.adapters.api.socrata.requests_cache.CachedSession",
+        "shillelagh.adapters.api.socrata.get_session",
         return_value=Session(),
     )
 
@@ -190,7 +190,7 @@ def test_socrata_invalid_query(mocker: MockerFixture, requests_mock: Mocker) ->
     Test that invalid queries are handled correctly.
     """
     mocker.patch(
-        "shillelagh.adapters.api.socrata.requests_cache.CachedSession",
+        "shillelagh.adapters.api.socrata.get_session",
         return_value=Session(),
     )
 
@@ -258,7 +258,7 @@ def test_get_cost(mocker: MockerFixture) -> None:
     Test ``get_cost``.
     """
     mocker.patch(
-        "shillelagh.adapters.api.socrata.requests_cache.CachedSession",
+        "shillelagh.adapters.api.socrata.get_session",
     )
 
     adapter = SocrataAPI("netloc", "dataset", "XXX")

tests/adapters/api/weatherapi_test.py

@@ -25,7 +25,7 @@ def test_weatherapi(mocker: MockerFixture, requests_mock: Mocker) -> None:
     Test the adapter.
     """
     mocker.patch(
-        "shillelagh.adapters.api.weatherapi.requests_cache.CachedSession",
+        "shillelagh.adapters.api.weatherapi.get_session",
         return_value=Session(),
     )
 
@@ -108,7 +108,7 @@ def test_weatherapi_api_error(mocker: MockerFixture, requests_mock: Mocker) -> N
     Test handling errors in the API.
     """
     mocker.patch(
-        "shillelagh.adapters.api.weatherapi.requests_cache.CachedSession",
+        "shillelagh.adapters.api.weatherapi.get_session",
         return_value=Session(),
     )
 
@@ -532,7 +532,7 @@ def test_dispatch(mocker: MockerFixture, requests_mock: Mocker) -> None:
     Test the dispatcher.
     """
     mocker.patch(
-        "shillelagh.adapters.api.weatherapi.requests_cache.CachedSession",
+        "shillelagh.adapters.api.weatherapi.get_session",
         return_value=Session(),
     )
 
@@ -593,7 +593,7 @@ def test_dispatch_api_key_connection(
     Test passing the key via the adapter kwargs.
     """
     mocker.patch(
-        "shillelagh.adapters.api.weatherapi.requests_cache.CachedSession",
+        "shillelagh.adapters.api.weatherapi.get_session",
         return_value=Session(),
     )
 
@@ -625,7 +625,7 @@ def test_dispatch_impossible(mocker: MockerFixture) -> None:
     any network requests.
     """
     session = mocker.patch(
-        "shillelagh.adapters.api.weatherapi.requests_cache.CachedSession",
+        "shillelagh.adapters.api.weatherapi.get_session",
     )
 
     connection = connect(
@@ -728,7 +728,7 @@ def test_get_cost(mocker: MockerFixture) -> None:
     Test ``get_cost``.
     """
     mocker.patch(
-        "shillelagh.adapters.api.weatherapi.requests_cache.CachedSession",
+        "shillelagh.adapters.api.weatherapi.get_session",
         return_value=Session(),
     )
 
@@ -761,9 +761,7 @@ def test_window(mocker: MockerFixture) -> None:
     """
     Test the default window size of days to fetch data.
     """
-    session = mocker.patch(
-        "shillelagh.adapters.api.weatherapi.requests_cache.CachedSession",
-    )
+    session = mocker.patch("shillelagh.adapters.api.weatherapi.get_session")
     session.return_value.get.return_value.json.return_value = weatherapi_response
 
     adapter = WeatherAPI("location", "XXX")

tests/lib_test.py

@@ -2,6 +2,7 @@
 Tests for shillelagh.lib.
 """
 
+from datetime import timedelta
 from typing import Any, Dict, Iterator, List, Tuple
 
 import pytest
@@ -31,6 +32,7 @@
     escape_string,
     filter_data,
     find_adapter,
+    get_session,
     is_not_null,
     is_null,
     serialize,
@@ -485,3 +487,32 @@ def test_apply_limit_and_offset() -> None:
 
     rows = apply_limit_and_offset(iter(range(10)), offset=2)
     assert list(rows) == [2, 3, 4, 5, 6, 7, 8, 9]
+
+
+def test_get_session(mocker: MockerFixture) -> None:
+    """
+    Test ``get_session``.
+    """
+    requests_cache = mocker.patch("shillelagh.lib.requests_cache")
+
+    get_session({}, "test", timedelta(seconds=10))
+    requests_cache.CachedSession.assert_called_once_with(
+        cache_name="test",
+        backend="sqlite",
+        expire_after=10,
+    )
+
+
+def test_get_session_namespaced(mocker: MockerFixture) -> None:
+    """
+    Test ``get_session`` with a namespaced cache key.
+    """
+    requests_cache = mocker.patch("shillelagh.lib.requests_cache")
+    mocker.patch("shillelagh.lib.create_namespaced_cache_key", return_value="ns")
+
+    get_session({}, "test", timedelta(seconds=10))
+    requests_cache.CachedSession.assert_called_once_with(
+        cache_name="ns",
+        backend="sqlite",
+        expire_after=10,
+    )