From 4c3e515d7ba45fe8e526b794b247c55ba4bac930 Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Thu, 25 Jan 2024 14:53:59 +1100
Subject: [PATCH 01/26] Move keyring into database
---
testproject/home/tests.py | 125 +++++++++++++++++-
wagtailcache/cache.py | 27 ++--
wagtailcache/migrations/0001_initial.py | 48 +++++++
wagtailcache/migrations/__init__.py | 0
wagtailcache/models.py | 86 ++++++++++++
wagtailcache/settings.py | 1 +
.../templates/wagtailcache/index.html | 9 +-
wagtailcache/utils.py | 16 +++
wagtailcache/views.py | 5 +-
9 files changed, 290 insertions(+), 27 deletions(-)
create mode 100644 wagtailcache/migrations/0001_initial.py
create mode 100644 wagtailcache/migrations/__init__.py
create mode 100644 wagtailcache/models.py
create mode 100644 wagtailcache/utils.py
diff --git a/testproject/home/tests.py b/testproject/home/tests.py
index 88fd078..b525918 100644
--- a/testproject/home/tests.py
+++ b/testproject/home/tests.py
@@ -1,3 +1,4 @@
+import datetime
import time
from django.contrib.auth.models import User
@@ -7,6 +8,7 @@
from django.test import modify_settings
from django.test import override_settings
from django.urls import reverse
+from django.utils.timezone import now
from wagtail import hooks
from wagtail.models import PageViewRestriction
@@ -19,6 +21,8 @@
from wagtailcache.cache import CacheControl
from wagtailcache.cache import Status
from wagtailcache.cache import clear_cache
+from wagtailcache.cache import Status
+from wagtailcache.models import KeyringItem
from wagtailcache.settings import wagtailcache_settings
@@ -510,14 +514,32 @@ def test_admin_clearcache(self):
def test_cache_keyring(self):
# Check if keyring is not present
- self.assertEqual(self.cache.get("keyring"), None)
+ self.assertEqual(KeyringItem.objects.count(), 0)
# Get should hit cache.
self.get_miss(self.page_cachedpage.get_url())
+ self.assertEqual(KeyringItem.objects.count(), 1)
# Get first key from keyring
- key = next(iter(self.cache.get("keyring")))
url = "http://%s%s" % ("testserver", self.page_cachedpage.get_url())
+ keyring_item = KeyringItem.objects.active_for_urls(url).first()
# Compare Keys
- self.assertEqual(key, url)
+ self.assertEqual(keyring_item.url, url)
+
+ @override_settings(WAGTAIL_CACHE_BACKEND="one_second")
+ def test_cache_keyring_no_uri_key_duplication(self):
+ # First get to populate keyring
+ self.get_miss(self.page_cachedpage.get_url())
+ # Wait a short time
+ time.sleep(0.5)
+ # Fetch a different page
+ self.get_miss(self.page_wagtailpage.get_url())
+ # Wait until the first page is expired, but not the keyring
+ time.sleep(0.6)
+ # Fetch the first page again
+ self.get_miss(self.page_cachedpage.get_url())
+ # Check the keyring does not contain duplicate uri_keys
+ url = "http://%s%s" % ("testserver", self.page_cachedpage.get_url())
+ keyring = self.cache.get("keyring")
+ self.assertEqual(len(keyring.get(url, [])), 1)
@override_settings(WAGTAIL_CACHE_BACKEND="one_second")
def test_cache_keyring_no_uri_key_duplication(self):
@@ -684,3 +706,100 @@ def test_response_hook_any(self):
self.assertEqual(hook_fns, [hook_any])
# The page should be cached normally due to hook returning garbage.
self.test_page_hit()
+
+ # ---- MODELS --------------------------------------------------------------
+ def test_keyring_update_or_create(self):
+ expiry = now() + datetime.timedelta(hours=1)
+ key = "abc123"
+ url = "https://example.com/"
+
+ KeyringItem.objects.set(
+ expiry=expiry,
+ key=key,
+ url=url,
+ )
+ self.assertEqual(KeyringItem.objects.count(), 1)
+ self.assertEqual(KeyringItem.objects.first().url, url)
+
+ expiry2 = now() + datetime.timedelta(hours=1)
+ KeyringItem.objects.set(
+ expiry=expiry2,
+ key=key,
+ url=url,
+ )
+ self.assertEqual(KeyringItem.objects.count(), 1)
+ self.assertEqual(KeyringItem.objects.first().expiry, expiry2)
+
+ def test_delete_expired(self):
+ """
+ Cache items expire by themselves, so we only need to actively
+ delete database items
+ """
+ expiry1 = now() + datetime.timedelta(seconds=1)
+ expiry2 = now() + datetime.timedelta(seconds=2)
+ used_keys = []
+
+ for exp in [expiry1, expiry2]:
+ key = f"key-{exp}"
+ url = f"https://example.com/{exp}"
+ KeyringItem.objects.set(
+ expiry=exp,
+ key=key,
+ url=url,
+ )
+ # Item should not expire
+ self.cache.set(key, url, 100)
+ used_keys.append(key)
+ self.assertEqual(KeyringItem.objects.count(), 2)
+ time.sleep(1)
+ KeyringItem.objects.clear_expired()
+ self.assertEqual(KeyringItem.objects.count(), 1)
+
+ # Cache items remain
+ for key in used_keys:
+ self.assertTrue(self.cache.get(key))
+
+ @override_settings(WAGTAIL_CACHE_BATCH_SIZE=2)
+ def test_bulk_delete(self):
+ """
+ Bulk delete removes cache items and database items that refer to them
+ """
+ timeout = 10
+ expiry = now() + datetime.timedelta(seconds=timeout)
+ keys = [f"key-{counter}" for counter in range(8)]
+
+ for key in keys:
+ url = "https://example.com/"
+ KeyringItem.objects.set(
+ expiry=expiry,
+ key=key,
+ url=url,
+ )
+ self.cache.set(key, url, timeout)
+
+ KeyringItem.objects.bulk_delete_cache_keys(keys[:4])
+
+ for key in keys[:4]:
+ self.assertFalse(KeyringItem.objects.filter(key=key).exists())
+ self.assertFalse(self.cache.get(key))
+
+ for key in keys[4:]:
+ self.assertTrue(KeyringItem.objects.filter(key=key).exists())
+ self.assertTrue(self.cache.get(key))
+
+ def test_active_for_urls(self):
+ past_expiry = now() - datetime.timedelta(seconds=1)
+ future_expiry = now() + datetime.timedelta(seconds=1)
+ url = "https://example.com"
+
+ KeyringItem.objects.set(
+ expiry=past_expiry,
+ key="key",
+ url=url,
+ )
+ KeyringItem.objects.set(
+ expiry=future_expiry,
+ key="key-2",
+ url=url,
+ )
+ self.assertEqual(KeyringItem.objects.active_for_urls(url).count(), 1)
diff --git a/wagtailcache/cache.py b/wagtailcache/cache.py
index 0d92c75..5a0b519 100644
--- a/wagtailcache/cache.py
+++ b/wagtailcache/cache.py
@@ -3,6 +3,7 @@
"""
import logging
+import datetime
import re
from enum import Enum
from functools import wraps
@@ -24,8 +25,10 @@
from django.utils.cache import learn_cache_key
from django.utils.cache import patch_response_headers
from django.utils.deprecation import MiddlewareMixin
+from django.utils.timezone import now
from wagtail import hooks
+from wagtailcache.models import KeyringItem
from wagtailcache.settings import wagtailcache_settings
@@ -378,25 +381,15 @@ def clear_cache(urls: List[str] = []) -> None:
return
_wagcache = caches[wagtailcache_settings.WAGTAIL_CACHE_BACKEND]
- if urls and "keyring" in _wagcache:
- keyring = _wagcache.get("keyring")
- # Check the provided URL matches a key in our keyring.
- matched_urls = []
- for regex in urls:
- for key in keyring:
- if re.match(regex, key):
- matched_urls.append(key)
- # If it matches, delete each entry from the cache,
- # and delete the URL from the keyring.
- for url in matched_urls:
- entries = keyring.get(url, [])
- for cache_key in entries:
- _wagcache.delete(cache_key)
- del keyring[url]
- # Save the keyring.
- _wagcache.set("keyring", keyring)
+ if urls:
+ active_keys = KeyringItem.objects.active_for_urls(urls).values_list(
+ "key", flat=True
+ )
+ # Delete the keys from the cache and the keyring
+ KeyringItem.objects.bulk_delete_cache_keys(active_keys)
# Clears the entire cache backend used by wagtail-cache.
else:
+ KeyringItem.objects.all().delete()
_wagcache.clear()
diff --git a/wagtailcache/migrations/0001_initial.py b/wagtailcache/migrations/0001_initial.py
new file mode 100644
index 0000000..c29c05f
--- /dev/null
+++ b/wagtailcache/migrations/0001_initial.py
@@ -0,0 +1,48 @@
+# Generated by Django 4.1.9 on 2024-01-25 03:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ initial = True
+
+ dependencies: list[tuple[str, str]] = []
+
+ operations = [
+ migrations.CreateModel(
+ name="KeyringItem",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("expiry", models.DateTimeField()),
+ ("key", models.CharField(max_length=512)),
+ ("url", models.URLField()),
+ ],
+ options={
+ "ordering": ["url"],
+ },
+ ),
+ migrations.AddIndex(
+ model_name="keyringitem",
+ index=models.Index(fields=["expiry"], name="wagtailcach_expiry_b9702b_idx"),
+ ),
+ migrations.AddIndex(
+ model_name="keyringitem",
+ index=models.Index(fields=["key"], name="wagtailcach_key_0c2934_idx"),
+ ),
+ migrations.AddIndex(
+ model_name="keyringitem",
+ index=models.Index(fields=["url"], name="wagtailcach_url_04699f_idx"),
+ ),
+ migrations.AlterUniqueTogether(
+ name="keyringitem",
+ unique_together={("url", "key")},
+ ),
+ ]
diff --git a/wagtailcache/migrations/__init__.py b/wagtailcache/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/wagtailcache/models.py b/wagtailcache/models.py
new file mode 100644
index 0000000..525814a
--- /dev/null
+++ b/wagtailcache/models.py
@@ -0,0 +1,86 @@
+from django.core.cache import caches
+from django.db import models
+from django.db.models import Q
+from django.utils.timezone import now
+
+from wagtailcache.settings import wagtailcache_settings
+from wagtailcache.utils import batched
+
+
+class KeyringItemManager(models.Manager):
+ def __init__(self):
+ super().__init__()
+ self._wagcache = caches[wagtailcache_settings.WAGTAIL_CACHE_BACKEND]
+
+ def set(self, url, key, expiry) -> models.Model:
+ """
+ Create or update a keyring item, clearing expired items too.
+ """
+ item, _ = self.update_or_create(
+ defaults={"expiry": expiry},
+ url=url,
+ key=key,
+ )
+ self.clear_expired()
+ return item
+
+ def bulk_delete_cache_keys(self, keys: list[str]) -> None:
+ """
+ Bulk delete the keys that exist, in batches
+ """
+ existing_keys = self.filter(key__in=keys)
+ # Delete from cache
+ for key_batch in batched(
+ existing_keys.values_list("key", flat=True),
+ wagtailcache_settings.WAGTAIL_CACHE_BATCH_SIZE,
+ ):
+ self._wagcache.delete_many(key_batch)
+ # Delete from database
+ existing_keys.delete()
+
+ def clear_expired(self) -> None:
+ """
+ Clear all items whose expiry has passed.
+ """
+ self.filter(expiry__lt=now()).delete()
+
+ def active(self):
+ return self.filter(expiry__gt=now())
+
+ def active_for_urls(self, urls):
+ if urls is None:
+ urls = []
+ if not isinstance(urls, (list, tuple)):
+ urls = list(urls)
+ qs = self.active()
+ if not urls:
+ return qs
+ filter_set = Q(url__regex=urls[0])
+ for url in urls[1:]:
+ filter_set = filter_set | Q(url__regex=url)
+ return qs.filter(filter_set)
+
+
+class KeyringItem(models.Model):
+ """
+ KeyringItems relate the URL of a page on the site to the key of an item
+ in the cache.
+ """
+
+ expiry = models.DateTimeField()
+ key = models.CharField(max_length=512)
+ url = models.URLField()
+
+ objects = KeyringItemManager()
+
+ class Meta:
+ ordering = ["url"]
+ indexes = [
+ models.Index(fields=["expiry"]),
+ models.Index(fields=["key"]),
+ models.Index(fields=["url"]),
+ ]
+ unique_together = [["url", "key"]]
+
+ def __str__(self):
+ return f"{self.url} -> {self.key} (Expires: {self.expiry})"
diff --git a/wagtailcache/settings.py b/wagtailcache/settings.py
index d56c9a6..7e01140 100644
--- a/wagtailcache/settings.py
+++ b/wagtailcache/settings.py
@@ -10,6 +10,7 @@
class _DefaultSettings:
WAGTAIL_CACHE = True
WAGTAIL_CACHE_BACKEND = "default"
+ WAGTAIL_CACHE_BATCH_SIZE = 100
WAGTAIL_CACHE_HEADER = "X-Wagtail-Cache"
WAGTAIL_CACHE_IGNORE_COOKIES = True
WAGTAIL_CACHE_IGNORE_QS = [
diff --git a/wagtailcache/templates/wagtailcache/index.html b/wagtailcache/templates/wagtailcache/index.html
index 58aaca0..31b53a3 100644
--- a/wagtailcache/templates/wagtailcache/index.html
+++ b/wagtailcache/templates/wagtailcache/index.html
@@ -41,12 +41,13 @@ {% trans "Contents" %}
{% trans "Note that 301/302 redirects and 404s may also be cached." %}
- {% for url, entries in keyring.items %}
+ {% regroup keyring by url as keyring_list %}
+ {% for url in keyring_list %}
-
- {{ url }}
+ {{ url.grouper }}
- {% for entry in entries %}
- - {{ entry }}
+ {% for entry in url.list %}
+ - {{ entry.key }}
{% endfor %}
diff --git a/wagtailcache/utils.py b/wagtailcache/utils.py
new file mode 100644
index 0000000..083da98
--- /dev/null
+++ b/wagtailcache/utils.py
@@ -0,0 +1,16 @@
+from itertools import islice
+from typing import Any
+from typing import Generator
+
+
+# `itertools.batched` is only available from Python3.12
+# This is a simplified version of the equivalent code described
+# in the Python docs.
+# https://docs.python.org/3/library/itertools.html#itertools.batched
+def batched(iterable, batch_size: int) -> Generator[tuple, Any, None]:
+ # batched('ABCDEFG', 3) --> ABC DEF G
+ if batch_size < 1:
+ raise ValueError("n must be at least one")
+ it = iter(iterable)
+ while batch := tuple(islice(it, batch_size)):
+ yield batch
diff --git a/wagtailcache/views.py b/wagtailcache/views.py
index 30b5a70..1275b7b 100644
--- a/wagtailcache/views.py
+++ b/wagtailcache/views.py
@@ -11,7 +11,7 @@
from django.urls import reverse
from wagtailcache.cache import clear_cache
-from wagtailcache.settings import wagtailcache_settings
+from wagtailcache.models import KeyringItem
def index(request):
@@ -19,8 +19,7 @@ def index(request):
The wagtail-cache admin panel.
"""
# Get the keyring to show cache contents.
- _wagcache = caches[wagtailcache_settings.WAGTAIL_CACHE_BACKEND]
- keyring: Dict[str, List[str]] = _wagcache.get("keyring", {})
+ keyring = KeyringItem.objects.active()
return render(
request,
"wagtailcache/index.html",
From 44510fd0a98f947c371169ca1ff051e5ffc855db Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Thu, 25 Jan 2024 16:35:15 +1100
Subject: [PATCH 02/26] Better method name
---
testproject/home/tests.py | 8 +++++---
wagtailcache/cache.py | 6 +++---
wagtailcache/models.py | 2 +-
3 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/testproject/home/tests.py b/testproject/home/tests.py
index b525918..5c1a251 100644
--- a/testproject/home/tests.py
+++ b/testproject/home/tests.py
@@ -520,7 +520,7 @@ def test_cache_keyring(self):
self.assertEqual(KeyringItem.objects.count(), 1)
# Get first key from keyring
url = "http://%s%s" % ("testserver", self.page_cachedpage.get_url())
- keyring_item = KeyringItem.objects.active_for_urls(url).first()
+ keyring_item = KeyringItem.objects.active_for_url_regexes(url).first()
# Compare Keys
self.assertEqual(keyring_item.url, url)
@@ -787,7 +787,7 @@ def test_bulk_delete(self):
self.assertTrue(KeyringItem.objects.filter(key=key).exists())
self.assertTrue(self.cache.get(key))
- def test_active_for_urls(self):
+ def test_active_for_url_regexes(self):
past_expiry = now() - datetime.timedelta(seconds=1)
future_expiry = now() + datetime.timedelta(seconds=1)
url = "https://example.com"
@@ -802,4 +802,6 @@ def test_active_for_urls(self):
key="key-2",
url=url,
)
- self.assertEqual(KeyringItem.objects.active_for_urls(url).count(), 1)
+ self.assertEqual(
+ KeyringItem.objects.active_for_url_regexes(url).count(), 1
+ )
diff --git a/wagtailcache/cache.py b/wagtailcache/cache.py
index 5a0b519..b5ecfd3 100644
--- a/wagtailcache/cache.py
+++ b/wagtailcache/cache.py
@@ -382,9 +382,9 @@ def clear_cache(urls: List[str] = []) -> None:
_wagcache = caches[wagtailcache_settings.WAGTAIL_CACHE_BACKEND]
if urls:
- active_keys = KeyringItem.objects.active_for_urls(urls).values_list(
- "key", flat=True
- )
+ active_keys = KeyringItem.objects.active_for_url_regexes(
+ urls
+ ).values_list("key", flat=True)
# Delete the keys from the cache and the keyring
KeyringItem.objects.bulk_delete_cache_keys(active_keys)
# Clears the entire cache backend used by wagtail-cache.
diff --git a/wagtailcache/models.py b/wagtailcache/models.py
index 525814a..76e9592 100644
--- a/wagtailcache/models.py
+++ b/wagtailcache/models.py
@@ -47,7 +47,7 @@ def clear_expired(self) -> None:
def active(self):
return self.filter(expiry__gt=now())
- def active_for_urls(self, urls):
+ def active_for_url_regexes(self, urls):
if urls is None:
urls = []
if not isinstance(urls, (list, tuple)):
From 05b2c4138fe01ffe43f7fce16608c7a8071a7391 Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Mon, 12 Feb 2024 16:45:56 +1100
Subject: [PATCH 03/26] Prevent Memcached warning
---
testproject/home/tests.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/testproject/home/tests.py b/testproject/home/tests.py
index 5c1a251..d409668 100644
--- a/testproject/home/tests.py
+++ b/testproject/home/tests.py
@@ -740,8 +740,9 @@ def test_delete_expired(self):
used_keys = []
for exp in [expiry1, expiry2]:
- key = f"key-{exp}"
- url = f"https://example.com/{exp}"
+ exp_iso = exp.isoformat()
+ key = f"key-{exp_iso}"
+ url = f"https://example.com/{exp_iso}"
KeyringItem.objects.set(
expiry=exp,
key=key,
From a171092748d8bef2eb5c2f773b0d804c66326ee6 Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Mon, 12 Feb 2024 16:46:42 +1100
Subject: [PATCH 04/26] Ensure active_for_url_regexes has more than one to work
with
---
testproject/home/tests.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/testproject/home/tests.py b/testproject/home/tests.py
index d409668..c66db75 100644
--- a/testproject/home/tests.py
+++ b/testproject/home/tests.py
@@ -803,6 +803,11 @@ def test_active_for_url_regexes(self):
key="key-2",
url=url,
)
+ KeyringItem.objects.set(
+ expiry=future_expiry,
+ key="key-3",
+ url=f"{url}/key-3/",
+ )
self.assertEqual(
- KeyringItem.objects.active_for_url_regexes(url).count(), 1
+ KeyringItem.objects.active_for_url_regexes(url).count(), 2
)
From 9cbf22302df44263dda546205bb36a7d2ad4309e Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Mon, 12 Feb 2024 16:47:02 +1100
Subject: [PATCH 05/26] Better style
---
wagtailcache/models.py | 8 ++++----
wagtailcache/utils.py | 26 +++++++++++++-------------
2 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/wagtailcache/models.py b/wagtailcache/models.py
index 76e9592..7569f06 100644
--- a/wagtailcache/models.py
+++ b/wagtailcache/models.py
@@ -55,10 +55,10 @@ def active_for_url_regexes(self, urls):
qs = self.active()
if not urls:
return qs
- filter_set = Q(url__regex=urls[0])
- for url in urls[1:]:
- filter_set = filter_set | Q(url__regex=url)
- return qs.filter(filter_set)
+ q_objects = Q()
+ for url in urls:
+ q_objects.add(Q(url__regex=url), Q.OR)
+ return qs.filter(q_objects)
class KeyringItem(models.Model):
diff --git a/wagtailcache/utils.py b/wagtailcache/utils.py
index 083da98..c26c26a 100644
--- a/wagtailcache/utils.py
+++ b/wagtailcache/utils.py
@@ -1,16 +1,16 @@
from itertools import islice
-from typing import Any
-from typing import Generator
-# `itertools.batched` is only available from Python3.12
-# This is a simplified version of the equivalent code described
-# in the Python docs.
-# https://docs.python.org/3/library/itertools.html#itertools.batched
-def batched(iterable, batch_size: int) -> Generator[tuple, Any, None]:
- # batched('ABCDEFG', 3) --> ABC DEF G
- if batch_size < 1:
- raise ValueError("n must be at least one")
- it = iter(iterable)
- while batch := tuple(islice(it, batch_size)):
- yield batch
+try:
+ from itertools import batched # type: ignore
+except ImportError:
+ # `itertools.batched` is only available from Python3.12
+ # This is the equivalent code described in the Python docs.
+ # https://docs.python.org/3/library/itertools.html#itertools.batched
+ def batched(iterable, n):
+ # batched('ABCDEFG', 3) --> ABC DEF G
+ if n < 1:
+ raise ValueError("n must be at least one")
+ it = iter(iterable)
+ while batch := tuple(islice(it, n)):
+ yield batch
From c1178f516619492ec3feb1bdb87c425790cc4b9d Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Tue, 13 Feb 2024 15:57:09 +1100
Subject: [PATCH 06/26] Correct typing syntax
---
wagtailcache/migrations/0001_initial.py | 3 ++-
wagtailcache/models.py | 4 +++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/wagtailcache/migrations/0001_initial.py b/wagtailcache/migrations/0001_initial.py
index c29c05f..85bf13a 100644
--- a/wagtailcache/migrations/0001_initial.py
+++ b/wagtailcache/migrations/0001_initial.py
@@ -1,12 +1,13 @@
# Generated by Django 4.1.9 on 2024-01-25 03:33
+from typing import List
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
- dependencies: list[tuple[str, str]] = []
+ dependencies: List[tuple[str, str]] = []
operations = [
migrations.CreateModel(
diff --git a/wagtailcache/models.py b/wagtailcache/models.py
index 7569f06..81eb9a4 100644
--- a/wagtailcache/models.py
+++ b/wagtailcache/models.py
@@ -1,3 +1,5 @@
+from typing import List
+
from django.core.cache import caches
from django.db import models
from django.db.models import Q
@@ -24,7 +26,7 @@ def set(self, url, key, expiry) -> models.Model:
self.clear_expired()
return item
- def bulk_delete_cache_keys(self, keys: list[str]) -> None:
+ def bulk_delete_cache_keys(self, keys: List[str]) -> None:
"""
Bulk delete the keys that exist, in batches
"""
From 1a5a2d68354fb0eeda17c061c0545be58621aeb3 Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Tue, 13 Feb 2024 16:17:46 +1100
Subject: [PATCH 07/26] Additional typing syntax fix
---
wagtailcache/migrations/0001_initial.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/wagtailcache/migrations/0001_initial.py b/wagtailcache/migrations/0001_initial.py
index 85bf13a..49292bc 100644
--- a/wagtailcache/migrations/0001_initial.py
+++ b/wagtailcache/migrations/0001_initial.py
@@ -1,13 +1,13 @@
# Generated by Django 4.1.9 on 2024-01-25 03:33
-from typing import List
+from typing import List, Tuple
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
- dependencies: List[tuple[str, str]] = []
+ dependencies: List[Tuple[str, str]] = []
operations = [
migrations.CreateModel(
From 9bd919c8d719a756bbbe8cfcff8420747c076f52 Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Tue, 13 Feb 2024 17:28:37 +1100
Subject: [PATCH 08/26] Use latest black version
---
wagtailcache/models.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/wagtailcache/models.py b/wagtailcache/models.py
index 81eb9a4..4d77e48 100644
--- a/wagtailcache/models.py
+++ b/wagtailcache/models.py
@@ -14,7 +14,7 @@ def __init__(self):
super().__init__()
self._wagcache = caches[wagtailcache_settings.WAGTAIL_CACHE_BACKEND]
- def set(self, url, key, expiry) -> models.Model:
+ def set(self, url, key, expiry) -> "KeyringItem":
"""
Create or update a keyring item, clearing expired items too.
"""
From 1460249805ab421bbdfeca69034eab432c7cc4cb Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Tue, 13 Feb 2024 20:47:08 +1100
Subject: [PATCH 09/26] Improve test coverage
---
testproject/home/tests.py | 50 +++++++++++++++++++++++++++++++++++++++
wagtailcache/models.py | 4 ++--
2 files changed, 52 insertions(+), 2 deletions(-)
diff --git a/testproject/home/tests.py b/testproject/home/tests.py
index c66db75..30401ea 100644
--- a/testproject/home/tests.py
+++ b/testproject/home/tests.py
@@ -24,6 +24,7 @@
from wagtailcache.cache import Status
from wagtailcache.models import KeyringItem
from wagtailcache.settings import wagtailcache_settings
+from wagtailcache.utils import batched
def hook_true(obj, is_cacheable: bool) -> bool:
@@ -811,3 +812,52 @@ def test_active_for_url_regexes(self):
self.assertEqual(
KeyringItem.objects.active_for_url_regexes(url).count(), 2
)
+
+ def test_active_for_urls_no_regexes(self):
+ past_expiry = now() - datetime.timedelta(seconds=1)
+ future_expiry = now() + datetime.timedelta(seconds=1)
+ url = "https://example.com"
+ url2 = "https://test.example.com"
+
+ KeyringItem.objects.set(
+ expiry=past_expiry,
+ key="key",
+ url=url,
+ )
+ KeyringItem.objects.set(
+ expiry=future_expiry,
+ key="key-2",
+ url=url,
+ )
+ KeyringItem.objects.set(
+ expiry=future_expiry,
+ key="key-3",
+ url=url2,
+ )
+ self.assertEqual(
+ KeyringItem.objects.active_for_url_regexes().count(), 2
+ )
+
+ def test_keyringitem_str(self):
+ future_expiry = datetime.datetime(year=2030, month=1, day=1)
+ url = "https://example.com"
+
+ KeyringItem.objects.set(
+ expiry=future_expiry,
+ key="key-2",
+ url=url,
+ )
+ self.assertEqual(
+ str(KeyringItem.objects.first()),
+ "https://example.com -> key-2 (Expires: 2030-01-01 00:00:00+00:00)",
+ )
+
+ def test_batched(self):
+ self.assertEqual(
+ [batch for batch in batched("ABCDEFG", 3)],
+ [("A", "B", "C"), ("D", "E", "F"), ("G",)],
+ )
+
+ def test_batched_invalid_batch_size(self):
+ with self.assertRaises(ValueError):
+ next(batched("ABCDEFG", 0))
diff --git a/wagtailcache/models.py b/wagtailcache/models.py
index 4d77e48..63a4b81 100644
--- a/wagtailcache/models.py
+++ b/wagtailcache/models.py
@@ -49,11 +49,11 @@ def clear_expired(self) -> None:
def active(self):
return self.filter(expiry__gt=now())
- def active_for_url_regexes(self, urls):
+ def active_for_url_regexes(self, urls=None):
if urls is None:
urls = []
if not isinstance(urls, (list, tuple)):
- urls = list(urls)
+ urls = [urls]
qs = self.active()
if not urls:
return qs
From 5d6080c6467397f8211f8377b50f00f7a5bcd430 Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Tue, 13 Feb 2024 15:37:29 +1100
Subject: [PATCH 10/26] Optionally use _raw_delete for db KeyringItems for
speed on large caches
---
testproject/home/tests.py | 29 +++++++++++++++++++++++++++++
wagtailcache/models.py | 8 ++++++--
wagtailcache/settings.py | 1 +
3 files changed, 36 insertions(+), 2 deletions(-)
diff --git a/testproject/home/tests.py b/testproject/home/tests.py
index 30401ea..b306550 100644
--- a/testproject/home/tests.py
+++ b/testproject/home/tests.py
@@ -789,6 +789,35 @@ def test_bulk_delete(self):
self.assertTrue(KeyringItem.objects.filter(key=key).exists())
self.assertTrue(self.cache.get(key))
+ @override_settings(WAGTAIL_CACHE_USE_RAW_DELETE=True)
+ def test_bulk_delete_raw_delete(self):
+ """
+ You can optionally use Django's `_raw_delete`
+ for speed with many cache keys.
+ """
+ timeout = 10
+ expiry = now() + datetime.timedelta(seconds=timeout)
+ keys = [f"key-{counter}" for counter in range(8)]
+
+ for key in keys:
+ url = "https://example.com/"
+ KeyringItem.objects.set(
+ expiry=expiry,
+ key=key,
+ url=url,
+ )
+ self.cache.set(key, url, timeout)
+
+ KeyringItem.objects.bulk_delete_cache_keys(keys[:4])
+
+ for key in keys[:4]:
+ self.assertFalse(KeyringItem.objects.filter(key=key).exists())
+ self.assertFalse(self.cache.get(key))
+
+ for key in keys[4:]:
+ self.assertTrue(KeyringItem.objects.filter(key=key).exists())
+ self.assertTrue(self.cache.get(key))
+
def test_active_for_url_regexes(self):
past_expiry = now() - datetime.timedelta(seconds=1)
future_expiry = now() + datetime.timedelta(seconds=1)
diff --git a/wagtailcache/models.py b/wagtailcache/models.py
index 63a4b81..e46c8b7 100644
--- a/wagtailcache/models.py
+++ b/wagtailcache/models.py
@@ -37,8 +37,12 @@ def bulk_delete_cache_keys(self, keys: List[str]) -> None:
wagtailcache_settings.WAGTAIL_CACHE_BATCH_SIZE,
):
self._wagcache.delete_many(key_batch)
- # Delete from database
- existing_keys.delete()
+ # Delete from database, optionally use `_raw_delete`
+ # for speed with many cache keys.
+ if wagtailcache_settings.WAGTAIL_CACHE_USE_RAW_DELETE:
+ existing_keys._raw_delete(using=self.db)
+ else:
+ existing_keys.delete()
def clear_expired(self) -> None:
"""
diff --git a/wagtailcache/settings.py b/wagtailcache/settings.py
index 7e01140..3856118 100644
--- a/wagtailcache/settings.py
+++ b/wagtailcache/settings.py
@@ -35,6 +35,7 @@ class _DefaultSettings:
r"^trk_.*$", # Listrak
r"^utm_.*$", # Google Analytics
]
+ WAGTAIL_CACHE_USE_RAW_DELETE = False
def __getattribute__(self, attr: Text):
# First load from Django settings.
From b99a7b9f92798e5449439dda974494591c2dc204 Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Tue, 13 Feb 2024 15:57:09 +1100
Subject: [PATCH 11/26] Correct typing syntax
---
wagtailcache/migrations/0001_initial.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/wagtailcache/migrations/0001_initial.py b/wagtailcache/migrations/0001_initial.py
index 49292bc..85bf13a 100644
--- a/wagtailcache/migrations/0001_initial.py
+++ b/wagtailcache/migrations/0001_initial.py
@@ -1,13 +1,13 @@
# Generated by Django 4.1.9 on 2024-01-25 03:33
-from typing import List, Tuple
+from typing import List
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
- dependencies: List[Tuple[str, str]] = []
+ dependencies: List[tuple[str, str]] = []
operations = [
migrations.CreateModel(
From ace137952620b835d6d9bbf12e39ff099fcd5085 Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Tue, 13 Feb 2024 16:17:46 +1100
Subject: [PATCH 12/26] Additional typing syntax fix
---
wagtailcache/migrations/0001_initial.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/wagtailcache/migrations/0001_initial.py b/wagtailcache/migrations/0001_initial.py
index 85bf13a..49292bc 100644
--- a/wagtailcache/migrations/0001_initial.py
+++ b/wagtailcache/migrations/0001_initial.py
@@ -1,13 +1,13 @@
# Generated by Django 4.1.9 on 2024-01-25 03:33
-from typing import List
+from typing import List, Tuple
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
- dependencies: List[tuple[str, str]] = []
+ dependencies: List[Tuple[str, str]] = []
operations = [
migrations.CreateModel(
From 73140edc9e97a739cadd7d50092729b74474e8c7 Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Wed, 14 Feb 2024 09:03:46 +1100
Subject: [PATCH 13/26] Update docs to include additional setting
---
docs/getting_started/django_settings.rst | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/docs/getting_started/django_settings.rst b/docs/getting_started/django_settings.rst
index 7d68ffe..fd12e49 100644
--- a/docs/getting_started/django_settings.rst
+++ b/docs/getting_started/django_settings.rst
@@ -100,3 +100,23 @@ own unique page in the cache, set this value to ``None`` or ``[]``.
If you feel as though the spammers have won, and want the nuclear option, you
can set this to ``[r".*"]`` which will ignore all querystrings. This is surely
a terrible idea, but it can be done.
+
+
+.. _WAGTAIL_CACHE_USE_RAW_DELETE:
+
+WAGTAIL_CACHE_USE_RAW_DELETE
+----------------------------
+
+.. versionadded:: 2.3.0
+
+ This setting will use Django's ``QuerySet._raw_delete`` method to clear
+ KeyringItems from the database. This is fast but means that signals are not
+ sent during that process. This is OFF by default.
+
+If your cache is large, then there can be many ``KeyringItem`` objects in the
+database. When you publish a Wagtail page that is high in the tree, many
+of those items may be deleted.
+
+If the delete process is too slow, then you can change this setting to use
+Django's ``QuerySet._raw_delete`` method. That runs significantly faster than
+``QuerySet.delete`` but it means that signals are not sent during that process.
From 3d6d7f89a4b91a7448f0f43dc41993405f123767 Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Mon, 25 Mar 2024 09:17:17 +1100
Subject: [PATCH 14/26] Increase URL field length
---
testproject/home/tests.py | 13 ++++++++++++-
.../migrations/0002_increase_url_length.py | 15 +++++++++++++++
wagtailcache/models.py | 2 +-
3 files changed, 28 insertions(+), 2 deletions(-)
create mode 100644 wagtailcache/migrations/0002_increase_url_length.py
diff --git a/testproject/home/tests.py b/testproject/home/tests.py
index b306550..fb96357 100644
--- a/testproject/home/tests.py
+++ b/testproject/home/tests.py
@@ -731,6 +731,18 @@ def test_keyring_update_or_create(self):
self.assertEqual(KeyringItem.objects.count(), 1)
self.assertEqual(KeyringItem.objects.first().expiry, expiry2)
+ def test_keyring_update_or_create__long_url(self):
+ expiry = now() + datetime.timedelta(hours=1)
+ key = "abc123"
+ url = f"https://example.com/?query={ 'a' * 900 }"
+
+ KeyringItem.objects.set(
+ expiry=expiry,
+ key=key,
+ url=url,
+ )
+ self.assertEqual(KeyringItem.objects.count(), 1)
+
def test_delete_expired(self):
"""
Cache items expire by themselves, so we only need to actively
@@ -756,7 +768,6 @@ def test_delete_expired(self):
time.sleep(1)
KeyringItem.objects.clear_expired()
self.assertEqual(KeyringItem.objects.count(), 1)
-
# Cache items remain
for key in used_keys:
self.assertTrue(self.cache.get(key))
diff --git a/wagtailcache/migrations/0002_increase_url_length.py b/wagtailcache/migrations/0002_increase_url_length.py
new file mode 100644
index 0000000..b52bb22
--- /dev/null
+++ b/wagtailcache/migrations/0002_increase_url_length.py
@@ -0,0 +1,15 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("wagtailcache", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="keyringitem",
+ name="url",
+ field=models.URLField(max_length=1000),
+ ),
+ ]
diff --git a/wagtailcache/models.py b/wagtailcache/models.py
index e46c8b7..de684d1 100644
--- a/wagtailcache/models.py
+++ b/wagtailcache/models.py
@@ -75,7 +75,7 @@ class KeyringItem(models.Model):
expiry = models.DateTimeField()
key = models.CharField(max_length=512)
- url = models.URLField()
+ url = models.URLField(max_length=1000)
objects = KeyringItemManager()
From caec651d9728004fea6240518e3103d3a275ff96 Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Mon, 29 Jul 2024 10:29:05 +1000
Subject: [PATCH 15/26] Ruff formatting after rebase
---
testproject/home/tests.py | 18 ------------------
wagtailcache/cache.py | 2 --
wagtailcache/views.py | 4 ----
3 files changed, 24 deletions(-)
diff --git a/testproject/home/tests.py b/testproject/home/tests.py
index fb96357..5b1afa2 100644
--- a/testproject/home/tests.py
+++ b/testproject/home/tests.py
@@ -21,7 +21,6 @@
from wagtailcache.cache import CacheControl
from wagtailcache.cache import Status
from wagtailcache.cache import clear_cache
-from wagtailcache.cache import Status
from wagtailcache.models import KeyringItem
from wagtailcache.settings import wagtailcache_settings
from wagtailcache.utils import batched
@@ -542,23 +541,6 @@ def test_cache_keyring_no_uri_key_duplication(self):
keyring = self.cache.get("keyring")
self.assertEqual(len(keyring.get(url, [])), 1)
- @override_settings(WAGTAIL_CACHE_BACKEND="one_second")
- def test_cache_keyring_no_uri_key_duplication(self):
- # First get to populate keyring
- self.get_miss(self.page_cachedpage.get_url())
- # Wait a short time
- time.sleep(0.5)
- # Fetch a different page
- self.get_miss(self.page_wagtailpage.get_url())
- # Wait until the first page is expired, but not the keyring
- time.sleep(0.6)
- # Fetch the first page again
- self.get_miss(self.page_cachedpage.get_url())
- # Check the keyring does not contain duplicate uri_keys
- url = "http://%s%s" % ("testserver", self.page_cachedpage.get_url())
- keyring = self.cache.get("keyring")
- self.assertEqual(len(keyring.get(url, [])), 1)
-
def test_clear_cache(self):
# First get should miss cache.
self.get_miss(self.page_cachedpage.get_url())
diff --git a/wagtailcache/cache.py b/wagtailcache/cache.py
index b5ecfd3..ca20f76 100644
--- a/wagtailcache/cache.py
+++ b/wagtailcache/cache.py
@@ -3,7 +3,6 @@
"""
import logging
-import datetime
import re
from enum import Enum
from functools import wraps
@@ -25,7 +24,6 @@
from django.utils.cache import learn_cache_key
from django.utils.cache import patch_response_headers
from django.utils.deprecation import MiddlewareMixin
-from django.utils.timezone import now
from wagtail import hooks
from wagtailcache.models import KeyringItem
diff --git a/wagtailcache/views.py b/wagtailcache/views.py
index 1275b7b..f97ac4c 100644
--- a/wagtailcache/views.py
+++ b/wagtailcache/views.py
@@ -2,10 +2,6 @@
Views for the wagtail admin dashboard.
"""
-from typing import Dict
-from typing import List
-
-from django.core.cache import caches
from django.http import HttpResponseRedirect
from django.shortcuts import render
from django.urls import reverse
From 1254d88c8073a8eda0bea870bafcad2414f612e0 Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Mon, 29 Jul 2024 11:23:08 +1000
Subject: [PATCH 16/26] Fix reversion from rebase
---
testproject/home/tests.py | 17 -----------------
wagtailcache/cache.py | 19 ++++++++++---------
2 files changed, 10 insertions(+), 26 deletions(-)
diff --git a/testproject/home/tests.py b/testproject/home/tests.py
index 5b1afa2..5ec298f 100644
--- a/testproject/home/tests.py
+++ b/testproject/home/tests.py
@@ -524,23 +524,6 @@ def test_cache_keyring(self):
# Compare Keys
self.assertEqual(keyring_item.url, url)
- @override_settings(WAGTAIL_CACHE_BACKEND="one_second")
- def test_cache_keyring_no_uri_key_duplication(self):
- # First get to populate keyring
- self.get_miss(self.page_cachedpage.get_url())
- # Wait a short time
- time.sleep(0.5)
- # Fetch a different page
- self.get_miss(self.page_wagtailpage.get_url())
- # Wait until the first page is expired, but not the keyring
- time.sleep(0.6)
- # Fetch the first page again
- self.get_miss(self.page_cachedpage.get_url())
- # Check the keyring does not contain duplicate uri_keys
- url = "http://%s%s" % ("testserver", self.page_cachedpage.get_url())
- keyring = self.cache.get("keyring")
- self.assertEqual(len(keyring.get(url, [])), 1)
-
def test_clear_cache(self):
# First get should miss cache.
self.get_miss(self.page_cachedpage.get_url())
diff --git a/wagtailcache/cache.py b/wagtailcache/cache.py
index ca20f76..1456c04 100644
--- a/wagtailcache/cache.py
+++ b/wagtailcache/cache.py
@@ -2,6 +2,7 @@
Functionality to set, serve from, and clear the cache.
"""
+import datetime
import logging
import re
from enum import Enum
@@ -24,6 +25,7 @@
from django.utils.cache import learn_cache_key
from django.utils.cache import patch_response_headers
from django.utils.deprecation import MiddlewareMixin
+from django.utils.timezone import now
from wagtail import hooks
from wagtailcache.models import KeyringItem
@@ -340,15 +342,14 @@ def process_response(
# (of the chopped request, not the real one).
cr = _chop_querystring(request)
uri = unquote(cr.build_absolute_uri())
- keyring = self._wagcache.get("keyring", {})
- # Get current cache keys belonging to this URI.
- # This should be a list of keys.
- uri_keys: List[str] = keyring.get(uri, [])
- # Append the key to this list if not already present and save.
- if cache_key not in uri_keys:
- uri_keys.append(cache_key)
- keyring[uri] = uri_keys
- self._wagcache.set("keyring", keyring)
+
+ expiry = now() + datetime.timedelta(seconds=timeout)
+ KeyringItem.objects.set(
+ expiry=expiry,
+ key=cache_key,
+ url=uri,
+ )
+
if isinstance(response, SimpleTemplateResponse):
def callback(r):
From dfb02fe9882420b46fe51f6a205632a6d3a46fbf Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Mon, 29 Jul 2024 11:25:40 +1000
Subject: [PATCH 17/26] Requested changes for PR #66
---
testproject/home/tests.py | 12 ++++++++----
wagtailcache/cache.py | 4 +---
wagtailcache/models.py | 18 +++++++-----------
3 files changed, 16 insertions(+), 18 deletions(-)
diff --git a/testproject/home/tests.py b/testproject/home/tests.py
index 5ec298f..5bbba92 100644
--- a/testproject/home/tests.py
+++ b/testproject/home/tests.py
@@ -755,7 +755,9 @@ def test_bulk_delete(self):
)
self.cache.set(key, url, timeout)
- KeyringItem.objects.bulk_delete_cache_keys(keys[:4])
+ KeyringItem.objects.bulk_delete_cache_keys(
+ KeyringItem.objects.filter(key__in=keys[:4])
+ )
for key in keys[:4]:
self.assertFalse(KeyringItem.objects.filter(key=key).exists())
@@ -784,7 +786,9 @@ def test_bulk_delete_raw_delete(self):
)
self.cache.set(key, url, timeout)
- KeyringItem.objects.bulk_delete_cache_keys(keys[:4])
+ KeyringItem.objects.bulk_delete_cache_keys(
+ KeyringItem.objects.filter(key__in=keys[:4])
+ )
for key in keys[:4]:
self.assertFalse(KeyringItem.objects.filter(key=key).exists())
@@ -815,7 +819,7 @@ def test_active_for_url_regexes(self):
url=f"{url}/key-3/",
)
self.assertEqual(
- KeyringItem.objects.active_for_url_regexes(url).count(), 2
+ KeyringItem.objects.active_for_url_regexes([url]).count(), 2
)
def test_active_for_urls_no_regexes(self):
@@ -840,7 +844,7 @@ def test_active_for_urls_no_regexes(self):
url=url2,
)
self.assertEqual(
- KeyringItem.objects.active_for_url_regexes().count(), 2
+ KeyringItem.objects.active_for_url_regexes([]).count(), 2
)
def test_keyringitem_str(self):
diff --git a/wagtailcache/cache.py b/wagtailcache/cache.py
index 1456c04..d59dd98 100644
--- a/wagtailcache/cache.py
+++ b/wagtailcache/cache.py
@@ -381,9 +381,7 @@ def clear_cache(urls: List[str] = []) -> None:
_wagcache = caches[wagtailcache_settings.WAGTAIL_CACHE_BACKEND]
if urls:
- active_keys = KeyringItem.objects.active_for_url_regexes(
- urls
- ).values_list("key", flat=True)
+ active_keys = KeyringItem.objects.active_for_url_regexes(urls)
# Delete the keys from the cache and the keyring
KeyringItem.objects.bulk_delete_cache_keys(active_keys)
# Clears the entire cache backend used by wagtail-cache.
diff --git a/wagtailcache/models.py b/wagtailcache/models.py
index de684d1..341ca0b 100644
--- a/wagtailcache/models.py
+++ b/wagtailcache/models.py
@@ -3,6 +3,7 @@
from django.core.cache import caches
from django.db import models
from django.db.models import Q
+from django.db.models import QuerySet
from django.utils.timezone import now
from wagtailcache.settings import wagtailcache_settings
@@ -26,23 +27,22 @@ def set(self, url, key, expiry) -> "KeyringItem":
self.clear_expired()
return item
- def bulk_delete_cache_keys(self, keys: List[str]) -> None:
+ def bulk_delete_cache_keys(self, keys_qs: QuerySet) -> None:
"""
- Bulk delete the keys that exist, in batches
+ Bulk delete the keys, in batches
"""
- existing_keys = self.filter(key__in=keys)
# Delete from cache
for key_batch in batched(
- existing_keys.values_list("key", flat=True),
+ keys_qs.values_list("key", flat=True),
wagtailcache_settings.WAGTAIL_CACHE_BATCH_SIZE,
):
self._wagcache.delete_many(key_batch)
# Delete from database, optionally use `_raw_delete`
# for speed with many cache keys.
if wagtailcache_settings.WAGTAIL_CACHE_USE_RAW_DELETE:
- existing_keys._raw_delete(using=self.db)
+ keys_qs._raw_delete(using=self.db)
else:
- existing_keys.delete()
+ keys_qs.delete()
def clear_expired(self) -> None:
"""
@@ -53,11 +53,7 @@ def clear_expired(self) -> None:
def active(self):
return self.filter(expiry__gt=now())
- def active_for_url_regexes(self, urls=None):
- if urls is None:
- urls = []
- if not isinstance(urls, (list, tuple)):
- urls = [urls]
+ def active_for_url_regexes(self, urls: List[str]):
qs = self.active()
if not urls:
return qs
From 436a877616152072da942bf075fffc9a253eeaea Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Tue, 30 Jul 2024 14:18:33 +1000
Subject: [PATCH 18/26] Mypy ignore rule
---
wagtailcache/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/wagtailcache/utils.py b/wagtailcache/utils.py
index c26c26a..d3b22e0 100644
--- a/wagtailcache/utils.py
+++ b/wagtailcache/utils.py
@@ -7,7 +7,7 @@
# `itertools.batched` is only available from Python3.12
# This is the equivalent code described in the Python docs.
# https://docs.python.org/3/library/itertools.html#itertools.batched
- def batched(iterable, n):
+ def batched(iterable, n): # type: ignore[no-redef]
# batched('ABCDEFG', 3) --> ABC DEF G
if n < 1:
raise ValueError("n must be at least one")
From ead0a37118e824f141ea5f489661ef9c08a97051 Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Wed, 21 Aug 2024 22:12:29 +1000
Subject: [PATCH 19/26] Support _raw_delete when clearing the whole cache
---
testproject/home/tests.py | 11 +++++++++++
wagtailcache/cache.py | 6 +++---
wagtailcache/models.py | 27 ++++++++++++++++++++-------
3 files changed, 34 insertions(+), 10 deletions(-)
diff --git a/testproject/home/tests.py b/testproject/home/tests.py
index 5bbba92..27300e4 100644
--- a/testproject/home/tests.py
+++ b/testproject/home/tests.py
@@ -534,6 +534,17 @@ def test_clear_cache(self):
# Now the page should miss cache.
self.get_miss(self.page_cachedpage.get_url())
+ @override_settings(WAGTAIL_CACHE_USE_RAW_DELETE=True)
+ def test_clear_cache_raw_delete(self):
+ # First get should miss cache.
+ self.get_miss(self.page_cachedpage.get_url())
+ # Second get should hit cache.
+ self.get_hit(self.page_cachedpage.get_url())
+ # clear all from Cache
+ clear_cache()
+ # Now the page should miss cache.
+ self.get_miss(self.page_cachedpage.get_url())
+
def test_clear_cache_url(self):
u1 = self.page_cachedpage.get_url()
u2 = self.page_cachedpage.get_url() + "?action=pytest"
diff --git a/wagtailcache/cache.py b/wagtailcache/cache.py
index d59dd98..e5086bd 100644
--- a/wagtailcache/cache.py
+++ b/wagtailcache/cache.py
@@ -384,10 +384,10 @@ def clear_cache(urls: List[str] = []) -> None:
active_keys = KeyringItem.objects.active_for_url_regexes(urls)
# Delete the keys from the cache and the keyring
KeyringItem.objects.bulk_delete_cache_keys(active_keys)
- # Clears the entire cache backend used by wagtail-cache.
else:
- KeyringItem.objects.all().delete()
- _wagcache.clear()
+ # Clear the entire cache backend used by wagtail-cache
+ # and the KeyringItems.
+ KeyringItem.objects.bulk_clear_cache()
def cache_page(view_func: Callable[..., HttpResponse]):
diff --git a/wagtailcache/models.py b/wagtailcache/models.py
index 341ca0b..df5ea33 100644
--- a/wagtailcache/models.py
+++ b/wagtailcache/models.py
@@ -27,9 +27,19 @@ def set(self, url, key, expiry) -> "KeyringItem":
self.clear_expired()
return item
+ def _delete_qs(self, keys_qs: QuerySet) -> None:
+ # Delete from database, optionally use `_raw_delete`
+ # for speed with many cache keys.
+ if wagtailcache_settings.WAGTAIL_CACHE_USE_RAW_DELETE:
+ keys_qs.delete()
+ keys_qs._raw_delete(using=self.db)
+ else:
+ keys_qs.delete()
+
def bulk_delete_cache_keys(self, keys_qs: QuerySet) -> None:
"""
- Bulk delete the keys, in batches
+ Bulk delete the keys from the cache in batches, and the
+ KeyringItem instances.
"""
# Delete from cache
for key_batch in batched(
@@ -37,12 +47,15 @@ def bulk_delete_cache_keys(self, keys_qs: QuerySet) -> None:
wagtailcache_settings.WAGTAIL_CACHE_BATCH_SIZE,
):
self._wagcache.delete_many(key_batch)
- # Delete from database, optionally use `_raw_delete`
- # for speed with many cache keys.
- if wagtailcache_settings.WAGTAIL_CACHE_USE_RAW_DELETE:
- keys_qs._raw_delete(using=self.db)
- else:
- keys_qs.delete()
+
+ self._delete_qs(keys_qs)
+
+ def bulk_clear_cache(self):
+ """
+ Clear the whole cache and all KeyringItem instances.
+ """
+ self._wagcache.clear()
+ self._delete_qs(self.all())
def clear_expired(self) -> None:
"""
From 0d2b14f3505c1f55a80c0a92122e5469c76ee9dd Mon Sep 17 00:00:00 2001
From: Jacob Colyvan
Date: Thu, 22 Aug 2024 17:33:26 +1000
Subject: [PATCH 20/26] Add clear_expired_cache_items cmd
---
.../commands/clear_wagtail_expired_cache_items.py | 14 ++++++++++++++
wagtailcache/models.py | 9 ++++++---
wagtailcache/settings.py | 1 +
3 files changed, 21 insertions(+), 3 deletions(-)
create mode 100644 wagtailcache/management/commands/clear_wagtail_expired_cache_items.py
diff --git a/wagtailcache/management/commands/clear_wagtail_expired_cache_items.py b/wagtailcache/management/commands/clear_wagtail_expired_cache_items.py
new file mode 100644
index 0000000..2edbc43
--- /dev/null
+++ b/wagtailcache/management/commands/clear_wagtail_expired_cache_items.py
@@ -0,0 +1,14 @@
+from django.core.management.base import BaseCommand
+from wagtailcache.models import KeyringItem
+
+class Command(BaseCommand):
+ help = 'Clear expired KeyringItems from the database'
+
+ def handle(self, *args, **options):
+ try:
+ cleared_count = KeyringItem.objects.clear_expired()
+ msg = f"Successfully cleared {cleared_count} expired KeyringItems"
+ self.stdout.write(self.style.SUCCESS(msg))
+ except Exception as e:
+ msg = f"Failed to clear expired KeyringItems: {e}"
+ self.stdout.write(self.style.ERROR(msg))
diff --git a/wagtailcache/models.py b/wagtailcache/models.py
index df5ea33..29ee2d5 100644
--- a/wagtailcache/models.py
+++ b/wagtailcache/models.py
@@ -24,7 +24,10 @@ def set(self, url, key, expiry) -> "KeyringItem":
url=url,
key=key,
)
- self.clear_expired()
+
+ if wagtailcache_settings.WAGTAIL_CACHE_CLEAR_EXPIRED_ON_SET:
+ self.clear_expired()
+
return item
def _delete_qs(self, keys_qs: QuerySet) -> None:
@@ -57,11 +60,11 @@ def bulk_clear_cache(self):
self._wagcache.clear()
self._delete_qs(self.all())
- def clear_expired(self) -> None:
+ def clear_expired(self) -> int:
"""
Clear all items whose expiry has passed.
"""
- self.filter(expiry__lt=now()).delete()
+ return self.filter(expiry__lt=now()).delete()[0]
def active(self):
return self.filter(expiry__gt=now())
diff --git a/wagtailcache/settings.py b/wagtailcache/settings.py
index 3856118..f3320e9 100644
--- a/wagtailcache/settings.py
+++ b/wagtailcache/settings.py
@@ -35,6 +35,7 @@ class _DefaultSettings:
r"^trk_.*$", # Listrak
r"^utm_.*$", # Google Analytics
]
+ WAGTAIL_CACHE_CLEAR_EXPIRED_ON_SET = False
WAGTAIL_CACHE_USE_RAW_DELETE = False
def __getattribute__(self, attr: Text):
From 1f8d9373736a3f096b157493568625e5819f16d0 Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Mon, 30 Sep 2024 12:49:14 +1000
Subject: [PATCH 21/26] Documentation for clear expired setting
---
docs/getting_started/django_settings.rst | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/docs/getting_started/django_settings.rst b/docs/getting_started/django_settings.rst
index fd12e49..e736342 100644
--- a/docs/getting_started/django_settings.rst
+++ b/docs/getting_started/django_settings.rst
@@ -102,6 +102,23 @@ can set this to ``[r".*"]`` which will ignore all querystrings. This is surely
a terrible idea, but it can be done.
+.. _WAGTAIL_CACHE_CLEAR_EXPIRED_ON_SET:
+
+WAGTAIL_CACHE_CLEAR_EXPIRED_ON_SET
+----------------------------------
+
+.. versionadded::
+
+ This setting will clear any expired `KeyringItems` as a new item is set,
+ and is OFF by default.
+
+If set to `True`, as a cache item is set the manager will delete any expired
+items from the database. If there are likely to be many expired items in the
+cache, then that might be time-consuming so this setting can be turned off.
+You can use the Django management command `wagtail_cache_clear_expired_items`
+periodically to clear expired items instead.
+
+
.. _WAGTAIL_CACHE_USE_RAW_DELETE:
WAGTAIL_CACHE_USE_RAW_DELETE
From 0c77dfd28669cc6967fbff4ae8cae1c131e79ffe Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Fri, 23 Aug 2024 10:34:50 +1000
Subject: [PATCH 22/26] Add optional jitter to cache timeouts
---
testproject/home/tests.py | 11 +++++++++++
wagtailcache/cache.py | 4 ++++
.../commands/clear_wagtail_expired_cache_items.py | 4 +++-
wagtailcache/settings.py | 1 +
4 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/testproject/home/tests.py b/testproject/home/tests.py
index 27300e4..f796bf2 100644
--- a/testproject/home/tests.py
+++ b/testproject/home/tests.py
@@ -611,6 +611,17 @@ def test_page_error_set(self):
self.head_error(page.get_url())
self.get_error(page.get_url())
+ @override_settings(
+ WAGTAIL_CACHE_BACKEND="one_second",
+ WAGTAIL_CACHE_TIMEOUT_JITTER_FUNC=lambda timeout: timeout * 2,
+ )
+ def test_timeout_jitter(self):
+ # Wagtail-cache should apply jitter to the timeout.
+ url = self.page_cachedpage.get_url()
+ self.client.get(url)
+ time.sleep(1.5)
+ self.get_hit(url)
+
# ---- HOOKS ---------------------------------------------------------------
def test_request_hook_true(self):
diff --git a/wagtailcache/cache.py b/wagtailcache/cache.py
index e5086bd..eff87e9 100644
--- a/wagtailcache/cache.py
+++ b/wagtailcache/cache.py
@@ -332,6 +332,10 @@ def process_response(
timeout = get_max_age(response)
if timeout is None:
timeout = self._wagcache.default_timeout
+ if wagtailcache_settings.WAGTAIL_CACHE_TIMEOUT_JITTER_FUNC:
+ timeout = wagtailcache_settings.WAGTAIL_CACHE_TIMEOUT_JITTER_FUNC(
+ timeout
+ )
patch_response_headers(response, timeout)
if timeout:
try:
diff --git a/wagtailcache/management/commands/clear_wagtail_expired_cache_items.py b/wagtailcache/management/commands/clear_wagtail_expired_cache_items.py
index 2edbc43..1e25821 100644
--- a/wagtailcache/management/commands/clear_wagtail_expired_cache_items.py
+++ b/wagtailcache/management/commands/clear_wagtail_expired_cache_items.py
@@ -1,8 +1,10 @@
from django.core.management.base import BaseCommand
+
from wagtailcache.models import KeyringItem
+
class Command(BaseCommand):
- help = 'Clear expired KeyringItems from the database'
+ help = "Clear expired KeyringItems from the database"
def handle(self, *args, **options):
try:
diff --git a/wagtailcache/settings.py b/wagtailcache/settings.py
index f3320e9..3fbf412 100644
--- a/wagtailcache/settings.py
+++ b/wagtailcache/settings.py
@@ -36,6 +36,7 @@ class _DefaultSettings:
r"^utm_.*$", # Google Analytics
]
WAGTAIL_CACHE_CLEAR_EXPIRED_ON_SET = False
+ WAGTAIL_CACHE_TIMEOUT_JITTER_FUNC = None
WAGTAIL_CACHE_USE_RAW_DELETE = False
def __getattribute__(self, attr: Text):
From cfaf1595b1aa6a37ad26a710ff1cc15e6c0741fe Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Mon, 30 Sep 2024 11:26:43 +1000
Subject: [PATCH 23/26] Ensure KeyringItem is validated
- 'testserver' is not a valid netloc so fails Django's validation.
- tests for very long URLs
---
testproject/home/tests.py | 68 +++++++++++++++++++++++++++------------
wagtailcache/models.py | 13 +++++---
2 files changed, 55 insertions(+), 26 deletions(-)
diff --git a/testproject/home/tests.py b/testproject/home/tests.py
index f796bf2..6c25096 100644
--- a/testproject/home/tests.py
+++ b/testproject/home/tests.py
@@ -39,6 +39,8 @@ def hook_any(obj, is_cacheable: bool):
class WagtailCacheTest(TestCase):
+ client_headers = {"SERVER_NAME": "example.com"}
+
@classmethod
def get_content_type(cls, modelname: str):
ctype, _ = ContentType.objects.get_or_create(
@@ -145,7 +147,7 @@ def head_hit(self, url: str):
"""
HEAD a page and test that it was served from the cache.
"""
- response = self.client.head(url)
+ response = self.client.head(url, **self.client_headers)
self.assertEqual(response.get(self.header_name, None), Status.HIT.value)
return response
@@ -153,7 +155,7 @@ def get_hit(self, url: str):
"""
Gets a page and tests that it was served from the cache.
"""
- response = self.client.get(url)
+ response = self.client.get(url, **self.client_headers)
self.assertEqual(response.get(self.header_name, None), Status.HIT.value)
return response
@@ -161,7 +163,7 @@ def head_miss(self, url: str):
"""
HEAD a page and test that it was not served from the cache.
"""
- response = self.client.head(url)
+ response = self.client.head(url, **self.client_headers)
self.assertEqual(
response.get(self.header_name, None), Status.MISS.value
)
@@ -170,7 +172,7 @@ def get_miss(self, url: str):
"""
Gets a page and tests that it was not served from the cache.
"""
- response = self.client.get(url)
+ response = self.client.get(url, **self.client_headers)
self.assertEqual(
response.get(self.header_name, None), Status.MISS.value
)
@@ -181,7 +183,7 @@ def head_skip(self, url: str):
HEAD a page and test that it was intentionally not served from the
cache.
"""
- response = self.client.head(url)
+ response = self.client.head(url, **self.client_headers)
self.assertEqual(
response.get(self.header_name, None), Status.SKIP.value
)
@@ -195,7 +197,7 @@ def get_skip(self, url: str):
Gets a page and tests that it was intentionally not served from
the cache.
"""
- response = self.client.get(url)
+ response = self.client.get(url, **self.client_headers)
self.assertEqual(
response.get(self.header_name, None), Status.SKIP.value
)
@@ -209,7 +211,7 @@ def get_error(self, url: str):
"""
Gets a page and tests that an error in the cache backend was handled.
"""
- response = self.client.get(url)
+ response = self.client.get(url, **self.client_headers)
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.get(self.header_name, None), Status.ERROR.value
@@ -220,7 +222,7 @@ def head_error(self, url: str):
"""
HEAD a page and tests that an error in the cache backend was handled.
"""
- response = self.client.head(url)
+ response = self.client.head(url, **self.client_headers)
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.get(self.header_name, None), Status.ERROR.value
@@ -232,7 +234,7 @@ def post_skip(self, url: str):
POSTS a page and tests that it was intentionally not served from
the cache.
"""
- response = self.client.post(url)
+ response = self.client.post(url, **self.client_headers)
self.assertEqual(
response.get(self.header_name, None), Status.SKIP.value
)
@@ -292,6 +294,9 @@ def test_querystrings(self):
# A get with both should also hit, since it is the second request.
self.head_hit(page.get_url() + "?valid=0&utm_code=0")
self.get_hit(page.get_url() + "?valid=0&utm_code=0")
+ # A get with a very long querysting should return an error
+ self.head_error(page.get_url() + "?" + "a" * 2000)
+ self.get_error(page.get_url() + "?" + "a" * 2000)
@override_settings(WAGTAIL_CACHE_IGNORE_COOKIES=False)
def test_cookie_page(self):
@@ -384,6 +389,7 @@ def test_page_restricted(self):
"password": "the cybers",
"return_url": self.page_cachedpage_restricted.get_url(),
},
+ **self.client_headers,
)
self.assertRedirects(
response, self.page_cachedpage_restricted.get_url()
@@ -493,7 +499,9 @@ def test_template_response_view_hit(self):
def test_admin(self):
self.client.force_login(self.user)
- response = self.client.get(reverse("wagtailcache:index"))
+ response = self.client.get(
+ reverse("wagtailcache:index"), **self.client_headers
+ )
self.client.logout()
self.assertEqual(response.status_code, 200)
@@ -504,7 +512,9 @@ def test_admin_clearcache(self):
self.get_hit(self.page_cachedpage.get_url())
# Now log in as admin and clear the cache.
self.client.force_login(self.user)
- response = self.client.get(reverse("wagtailcache:clearcache"))
+ response = self.client.get(
+ reverse("wagtailcache:clearcache"), **self.client_headers
+ )
self.client.logout()
self.assertEqual(response.status_code, 302)
# Now the page should miss cache.
@@ -519,7 +529,7 @@ def test_cache_keyring(self):
self.get_miss(self.page_cachedpage.get_url())
self.assertEqual(KeyringItem.objects.count(), 1)
# Get first key from keyring
- url = "http://%s%s" % ("testserver", self.page_cachedpage.get_url())
+ url = "http://%s%s" % ("example.com", self.page_cachedpage.get_url())
keyring_item = KeyringItem.objects.active_for_url_regexes(url).first()
# Compare Keys
self.assertEqual(keyring_item.url, url)
@@ -569,22 +579,30 @@ def test_clear_cache_url(self):
@override_settings(WAGTAIL_CACHE=True)
def test_enable_wagtailcache(self):
# Intentionally enable wagtail-cache, make sure it works.
- response = self.client.get(self.page_cachedpage.get_url())
+ response = self.client.get(
+ self.page_cachedpage.get_url(), **self.client_headers
+ )
self.assertIsNotNone(response.get(self.header_name, None))
@override_settings(WAGTAIL_CACHE=False)
def test_disable_wagtailcache(self):
# Intentionally disable wagtail-cache, make sure it is inactive.
- response = self.client.get(self.page_cachedpage.get_url())
+ response = self.client.get(
+ self.page_cachedpage.get_url(), **self.client_headers
+ )
self.assertIsNone(response.get(self.header_name, None))
@override_settings(WAGTAIL_CACHE_BACKEND="zero")
def test_zero_timeout(self):
# Wagtail-cache should ignore the page when a timeout is zero.
- response = self.client.get(self.page_cachedpage.get_url())
+ response = self.client.get(
+ self.page_cachedpage.get_url(), **self.client_headers
+ )
self.assertIsNone(response.get(self.header_name, None))
# Second should also not cache.
- response = self.client.get(self.page_cachedpage.get_url())
+ response = self.client.get(
+ self.page_cachedpage.get_url(), **self.client_headers
+ )
self.assertIsNone(response.get(self.header_name, None))
# Load admin panel to render the zero timeout.
self.test_admin()
@@ -618,7 +636,7 @@ def test_page_error_set(self):
def test_timeout_jitter(self):
# Wagtail-cache should apply jitter to the timeout.
url = self.page_cachedpage.get_url()
- self.client.get(url)
+ self.client.get(url, **self.client_headers)
time.sleep(1.5)
self.get_hit(url)
@@ -626,11 +644,15 @@ def test_timeout_jitter(self):
def test_request_hook_true(self):
# A POST should never be cached.
- response = self.client.post(reverse("cached_view"))
+ response = self.client.post(
+ reverse("cached_view"), **self.client_headers
+ )
self.assertEqual(
response.get(self.header_name, None), Status.SKIP.value
)
- response = self.client.post(reverse("cached_view"))
+ response = self.client.post(
+ reverse("cached_view"), **self.client_headers
+ )
self.assertEqual(
response.get(self.header_name, None), Status.SKIP.value
)
@@ -642,11 +664,15 @@ def test_request_hook_true(self):
# the response still has the final say in whether or not the response is
# cached. However a simple POST request where the response does not
# forbid caching will in fact get cached!
- response = self.client.post(reverse("cached_view"))
+ response = self.client.post(
+ reverse("cached_view"), **self.client_headers
+ )
self.assertEqual(
response.get(self.header_name, None), Status.MISS.value
)
- response = self.client.post(reverse("cached_view"))
+ response = self.client.post(
+ reverse("cached_view"), **self.client_headers
+ )
self.assertEqual(response.get(self.header_name, None), Status.HIT.value)
def test_request_hook_false(self):
diff --git a/wagtailcache/models.py b/wagtailcache/models.py
index 29ee2d5..c33a138 100644
--- a/wagtailcache/models.py
+++ b/wagtailcache/models.py
@@ -19,11 +19,14 @@ def set(self, url, key, expiry) -> "KeyringItem":
"""
Create or update a keyring item, clearing expired items too.
"""
- item, _ = self.update_or_create(
- defaults={"expiry": expiry},
- url=url,
- key=key,
- )
+ # Ensure `full_clean` is called to validate the model.
+ try:
+ item = self.get(url=url, key=key)
+ item.expiry = expiry
+ except KeyringItem.DoesNotExist:
+ item = KeyringItem(url=url, key=key, expiry=expiry)
+ item.full_clean()
+ item.save()
if wagtailcache_settings.WAGTAIL_CACHE_CLEAR_EXPIRED_ON_SET:
self.clear_expired()
From da573a3bcda5075d253699c1095f35aea83809bb Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Mon, 30 Sep 2024 11:38:01 +1000
Subject: [PATCH 24/26] Add comment
---
testproject/home/tests.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/testproject/home/tests.py b/testproject/home/tests.py
index 6c25096..d594aad 100644
--- a/testproject/home/tests.py
+++ b/testproject/home/tests.py
@@ -39,6 +39,7 @@ def hook_any(obj, is_cacheable: bool):
class WagtailCacheTest(TestCase):
+ # Django's default `testserver` is not a valid domain name.
client_headers = {"SERVER_NAME": "example.com"}
@classmethod
From 58f3f7407bab736b9461f5e75abb5f54eee1e063 Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Wed, 23 Oct 2024 18:51:43 +1100
Subject: [PATCH 25/26] Describe jitter setting
---
docs/getting_started/django_settings.rst | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/docs/getting_started/django_settings.rst b/docs/getting_started/django_settings.rst
index e736342..18db5c5 100644
--- a/docs/getting_started/django_settings.rst
+++ b/docs/getting_started/django_settings.rst
@@ -137,3 +137,27 @@ of those items may be deleted.
If the delete process is too slow, then you can change this setting to use
Django's ``QuerySet._raw_delete`` method. That runs significantly faster than
``QuerySet.delete`` but it means that signals are not sent during that process.
+
+
+WAGTAIL_CACHE_TIMEOUT_JITTER_FUNC
+---------------------------------
+
+.. versionadded::
+
+ An optional function that will be called to adjust the cache timeout each
+ time a cache item is set. Set to None by default.
+
+This can be used to add a random jitter to the cache timeout to avoid cache
+stampedes.
+
+The function should take the timeout as an argument and return a new
+timeout. For example, to add a random jitter of up to 10% to the timeout:
+
+.. code-block:: python
+
+ import random
+
+ def jitter_timeout(timeout):
+ return timeout * random.uniform(0.9, 1.1)
+
+ WAGTAIL_CACHE_TIMEOUT_JITTER_FUNC = jitter_timeout
From 2386b305961c1ce29070fd653d51d91d54d77c48 Mon Sep 17 00:00:00 2001
From: Alastair Weakley
Date: Thu, 24 Oct 2024 11:12:29 +1100
Subject: [PATCH 26/26] URLs with GET parameters can be very long
---
testproject/home/tests.py | 8 +++++---
.../migrations/0003_alter_keyringitem_url.py | 17 +++++++++++++++++
wagtailcache/models.py | 2 +-
3 files changed, 23 insertions(+), 4 deletions(-)
create mode 100644 wagtailcache/migrations/0003_alter_keyringitem_url.py
diff --git a/testproject/home/tests.py b/testproject/home/tests.py
index d594aad..b608e8e 100644
--- a/testproject/home/tests.py
+++ b/testproject/home/tests.py
@@ -295,9 +295,11 @@ def test_querystrings(self):
# A get with both should also hit, since it is the second request.
self.head_hit(page.get_url() + "?valid=0&utm_code=0")
self.get_hit(page.get_url() + "?valid=0&utm_code=0")
- # A get with a very long querysting should return an error
- self.head_error(page.get_url() + "?" + "a" * 2000)
- self.get_error(page.get_url() + "?" + "a" * 2000)
+ # A get with a very long querysting should be cached.
+ self.head_miss(page.get_url() + "?" + "a" * 2000)
+ self.get_miss(page.get_url() + "?" + "a" * 2000)
+ self.head_hit(page.get_url() + "?" + "a" * 2000)
+ self.get_hit(page.get_url() + "?" + "a" * 2000)
@override_settings(WAGTAIL_CACHE_IGNORE_COOKIES=False)
def test_cookie_page(self):
diff --git a/wagtailcache/migrations/0003_alter_keyringitem_url.py b/wagtailcache/migrations/0003_alter_keyringitem_url.py
new file mode 100644
index 0000000..2432e4f
--- /dev/null
+++ b/wagtailcache/migrations/0003_alter_keyringitem_url.py
@@ -0,0 +1,17 @@
+# Generated by Django 4.2.15 on 2024-10-24 00:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("wagtailcache", "0002_increase_url_length"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="keyringitem",
+ name="url",
+ field=models.TextField(),
+ ),
+ ]
diff --git a/wagtailcache/models.py b/wagtailcache/models.py
index c33a138..1ca689d 100644
--- a/wagtailcache/models.py
+++ b/wagtailcache/models.py
@@ -90,7 +90,7 @@ class KeyringItem(models.Model):
expiry = models.DateTimeField()
key = models.CharField(max_length=512)
- url = models.URLField(max_length=1000)
+ url = models.TextField()
objects = KeyringItemManager()