From 54d95541cf223995fe7005e405dec28c971cb33b Mon Sep 17 00:00:00 2001 From: Dirk Doesburg Date: Mon, 30 Oct 2023 19:07:12 +0100 Subject: [PATCH] Use Djagno 4.2 storages API This solves a problem where a new S3Storage object was made for each file, which led to terrible performance from loading the cloudfront key. --- .../0041_alter_membergroup_photo.py | 22 +++++---- website/activemembers/models.py | 4 +- .../migrations/0013_alter_slide_content.py | 21 +++++---- .../migrations/0017_alter_slide_content.py | 4 +- website/announcements/models.py | 4 +- .../migrations/0041_alter_profile_photo.py | 21 +++++---- website/members/models/profile.py | 4 +- .../0008_alter_merchandiseitem_image.py | 17 +++---- website/merchandise/models.py | 4 +- ...logo_alter_partner_site_header_and_more.py | 45 ++++++++++++------- .../migrations/0026_partner_alternate_logo.py | 17 ++++--- website/partners/models.py | 12 ++--- website/thaliawebsite/settings.py | 25 ++++++++--- website/thaliawebsite/storage/backend.py | 6 +-- .../commands/remove_unused_media.py | 6 +-- 15 files changed, 127 insertions(+), 85 deletions(-) diff --git a/website/activemembers/migrations/0041_alter_membergroup_photo.py b/website/activemembers/migrations/0041_alter_membergroup_photo.py index 3f4c0927a..e68fd33c2 100644 --- a/website/activemembers/migrations/0041_alter_membergroup_photo.py +++ b/website/activemembers/migrations/0041_alter_membergroup_photo.py @@ -3,11 +3,11 @@ from django.db import migrations, models -import thaliawebsite.storage.backend +from django.core.files.storage import storages def forwards_func(apps, schema_editor): - MemberGroup = apps.get_model('activemembers', 'MemberGroup') + MemberGroup = apps.get_model("activemembers", "MemberGroup") existing_images = [] @@ -26,23 +26,29 @@ def forwards_func(apps, schema_editor): def reverse_func(apps, schema_editor): - MemberGroup = apps.get_model('activemembers', 'MemberGroup') + MemberGroup = apps.get_model("activemembers", "MemberGroup") for item in MemberGroup.objects.filter(photo__isnull=False): item.photo.name = f"public/{item.photo.name}" item.save() -class Migration(migrations.Migration): +class Migration(migrations.Migration): dependencies = [ - ('activemembers', '0040_remove_multilang_field'), + ("activemembers", "0040_remove_multilang_field"), ] operations = [ migrations.AlterField( - model_name='membergroup', - name='photo', - field=models.ImageField(blank=True, null=True, storage=thaliawebsite.storage.backend.get_public_storage, upload_to='committeephotos/', verbose_name='Image'), + model_name="membergroup", + name="photo", + field=models.ImageField( + blank=True, + null=True, + storage=storages["public"], + upload_to="committeephotos/", + verbose_name="Image", + ), ), migrations.RunPython(forwards_func, reverse_func), ] diff --git a/website/activemembers/models.py b/website/activemembers/models.py index 14e52238b..27440dfd5 100644 --- a/website/activemembers/models.py +++ b/website/activemembers/models.py @@ -6,6 +6,7 @@ from django.contrib.admin import display as admin_display from django.contrib.auth.models import Permission from django.core.exceptions import NON_FIELD_ERRORS, ValidationError +from django.core.files.storage import storages from django.core.validators import MinValueValidator from django.db import models from django.urls import reverse @@ -15,7 +16,6 @@ from thumbnails.fields import ImageField from tinymce.models import HTMLField -from thaliawebsite.storage.backend import get_public_storage from utils.media.services import get_upload_to_function from utils.snippets import overlaps @@ -43,7 +43,7 @@ class MemberGroup(models.Model): verbose_name=_("Image"), resize_source_to="source", upload_to=get_upload_to_function("committeephotos"), - storage=get_public_storage, + storage=storages["public"], null=True, blank=True, ) diff --git a/website/announcements/migrations/0013_alter_slide_content.py b/website/announcements/migrations/0013_alter_slide_content.py index 1a3c4533d..7c51405a0 100644 --- a/website/announcements/migrations/0013_alter_slide_content.py +++ b/website/announcements/migrations/0013_alter_slide_content.py @@ -3,11 +3,11 @@ import announcements.models from django.db import migrations, models -import thaliawebsite.storage.backend +from django.core.files.storage import storages def forwards_func(apps, schema_editor): - Slide = apps.get_model('announcements', 'Slide') + Slide = apps.get_model("announcements", "Slide") existing_images = [] @@ -26,7 +26,7 @@ def forwards_func(apps, schema_editor): def reverse_func(apps, schema_editor): - Slide = apps.get_model('announcements', 'Slide') + Slide = apps.get_model("announcements", "Slide") for item in Slide.objects.filter(content__isnull=False): item.photo.name = f"public/{item.content.name}" @@ -34,16 +34,21 @@ def reverse_func(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ - ('announcements', '0012_auto_20201209_1704'), + ("announcements", "0012_auto_20201209_1704"), ] operations = [ migrations.AlterField( - model_name='slide', - name='content', - field=models.FileField(help_text='The content of the slide; what image to display.', storage=thaliawebsite.storage.backend.get_public_storage, upload_to='announcements/slides/', validators=[announcements.models.validate_image], verbose_name='Content'), + model_name="slide", + name="content", + field=models.FileField( + help_text="The content of the slide; what image to display.", + storage=storages["public"], + upload_to="announcements/slides/", + validators=[announcements.models.validate_image], + verbose_name="Content", + ), ), migrations.RunPython(forwards_func, reverse_func), ] diff --git a/website/announcements/migrations/0017_alter_slide_content.py b/website/announcements/migrations/0017_alter_slide_content.py index 0914bbf54..13b027496 100644 --- a/website/announcements/migrations/0017_alter_slide_content.py +++ b/website/announcements/migrations/0017_alter_slide_content.py @@ -3,7 +3,7 @@ import announcements.models from django.db import migrations, models import functools -import thaliawebsite.storage.backend +from django.core.files.storage import storages import utils.media.services @@ -18,7 +18,7 @@ class Migration(migrations.Migration): name="content", field=models.FileField( help_text="The content of the slide; what image to display.", - storage=thaliawebsite.storage.backend.get_public_storage, + storage=storages["public"], upload_to=functools.partial( utils.media.services._generic_upload_to, *(), diff --git a/website/announcements/models.py b/website/announcements/models.py index 9bb2eb127..3076e995e 100644 --- a/website/announcements/models.py +++ b/website/announcements/models.py @@ -1,4 +1,5 @@ """The models defined by the announcement package.""" +from django.core.files.storage import storages from django.core.validators import ( FileExtensionValidator, get_available_image_extensions, @@ -11,7 +12,6 @@ from tinymce.models import HTMLField -from thaliawebsite.storage.backend import get_public_storage from utils.media.services import get_upload_to_function @@ -155,7 +155,7 @@ class Slide(models.Model): help_text=_("The content of the slide; what image to display."), blank=False, upload_to=get_upload_to_function("announcements/slides"), - storage=get_public_storage, + storage=storages["public"], validators=[validate_image], ) diff --git a/website/members/migrations/0041_alter_profile_photo.py b/website/members/migrations/0041_alter_profile_photo.py index 05198849d..e4926e22d 100644 --- a/website/members/migrations/0041_alter_profile_photo.py +++ b/website/members/migrations/0041_alter_profile_photo.py @@ -3,11 +3,11 @@ from django.db import migrations, models import members.models.profile -import thaliawebsite.storage.backend +from django.core.files.storage import storages def forwards_func(apps, schema_editor): - Profile = apps.get_model('members', 'Profile') + Profile = apps.get_model("members", "Profile") existing_images = [] @@ -26,7 +26,7 @@ def forwards_func(apps, schema_editor): def reverse_func(apps, schema_editor): - Profile = apps.get_model('members', 'Profile') + Profile = apps.get_model("members", "Profile") for item in Profile.objects.filter(photo__isnull=False): item.photo.name = f"public/{item.photo.name}" @@ -34,16 +34,21 @@ def reverse_func(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ - ('members', '0040_remove_profile_auto_renew'), + ("members", "0040_remove_profile_auto_renew"), ] operations = [ migrations.AlterField( - model_name='profile', - name='photo', - field=models.ImageField(blank=True, null=True, storage=thaliawebsite.storage.backend.get_public_storage, upload_to=members.models.profile._profile_image_path, verbose_name='Photo'), + model_name="profile", + name="photo", + field=models.ImageField( + blank=True, + null=True, + storage=storages["public"], + upload_to=members.models.profile._profile_image_path, + verbose_name="Photo", + ), ), migrations.RunPython(forwards_func, reverse_func), ] diff --git a/website/members/models/profile.py b/website/members/models/profile.py index 0d9b289b8..046362082 100644 --- a/website/members/models/profile.py +++ b/website/members/models/profile.py @@ -3,13 +3,13 @@ from django.conf import settings from django.core import validators from django.core.exceptions import ValidationError +from django.core.files.storage import storages from django.db import models from django.utils import timezone from django.utils.translation import gettext_lazy as _ from thumbnails.fields import ImageField -from thaliawebsite.storage.backend import get_public_storage from utils import countries from utils.media.services import get_upload_to_function @@ -208,7 +208,7 @@ class Profile(models.Model): verbose_name=_("Photo"), resize_source_to="source", upload_to=_profile_image_path, - storage=get_public_storage, + storage=storages["public"], null=True, blank=True, ) diff --git a/website/merchandise/migrations/0008_alter_merchandiseitem_image.py b/website/merchandise/migrations/0008_alter_merchandiseitem_image.py index 6ac507a7d..476504e6e 100644 --- a/website/merchandise/migrations/0008_alter_merchandiseitem_image.py +++ b/website/merchandise/migrations/0008_alter_merchandiseitem_image.py @@ -3,11 +3,11 @@ from django.db import migrations, models -import thaliawebsite.storage.backend +from django.core.files.storage import storages def forwards_func(apps, schema_editor): - MerchandiseItem = apps.get_model('merchandise', 'MerchandiseItem') + MerchandiseItem = apps.get_model("merchandise", "MerchandiseItem") existing_images = [] @@ -26,7 +26,7 @@ def forwards_func(apps, schema_editor): def reverse_func(apps, schema_editor): - MerchandiseItem = apps.get_model('merchandise', 'MerchandiseItem') + MerchandiseItem = apps.get_model("merchandise", "MerchandiseItem") for item in MerchandiseItem.objects.filter(image__isnull=False): item.image.name = f"public/{item.image.name}" @@ -34,16 +34,17 @@ def reverse_func(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ - ('merchandise', '0007_alter_merchandiseitem_price'), + ("merchandise", "0007_alter_merchandiseitem_price"), ] operations = [ migrations.AlterField( - model_name='merchandiseitem', - name='image', - field=models.ImageField(storage=thaliawebsite.storage.backend.get_public_storage, upload_to='merchandise'), + model_name="merchandiseitem", + name="image", + field=models.ImageField( + storage=storages["public"], upload_to="merchandise" + ), ), migrations.RunPython(forwards_func, reverse_func), ] diff --git a/website/merchandise/models.py b/website/merchandise/models.py index bd7ce388b..cc0e0e16b 100644 --- a/website/merchandise/models.py +++ b/website/merchandise/models.py @@ -1,9 +1,9 @@ """Models for the merchandise database tables.""" +from django.core.files.storage import storages from django.db import models from thumbnails.fields import ImageField -from thaliawebsite.storage.backend import get_public_storage from utils.media.services import get_upload_to_function _merchandise_photo_upload_to = get_upload_to_function("merchandise") @@ -30,7 +30,7 @@ class MerchandiseItem(models.Model): #: Image of the merchandise item image = ImageField( upload_to=_merchandise_photo_upload_to, - storage=get_public_storage, + storage=storages["public"], resize_source_to="source", ) diff --git a/website/partners/migrations/0024_alter_partner_logo_alter_partner_site_header_and_more.py b/website/partners/migrations/0024_alter_partner_logo_alter_partner_site_header_and_more.py index 836d3f0ab..3ec3cfb5f 100644 --- a/website/partners/migrations/0024_alter_partner_logo_alter_partner_site_header_and_more.py +++ b/website/partners/migrations/0024_alter_partner_logo_alter_partner_site_header_and_more.py @@ -1,32 +1,47 @@ # Generated by Django 4.0.4 on 2022-05-27 07:44 from django.db import migrations, models -import thaliawebsite.storage.backend +from django.core.files.storage import storages -class Migration(migrations.Migration): +class Migration(migrations.Migration): dependencies = [ - ('partners', '0023_delete_partnerevent'), + ("partners", "0023_delete_partnerevent"), ] operations = [ migrations.AlterField( - model_name='partner', - name='logo', - field=models.ImageField(storage=thaliawebsite.storage.backend.get_public_storage, upload_to='partners/logos/'), + model_name="partner", + name="logo", + field=models.ImageField( + storage=storages["public"], upload_to="partners/logos/" + ), ), migrations.AlterField( - model_name='partner', - name='site_header', - field=models.ImageField(blank=True, null=True, storage=thaliawebsite.storage.backend.get_public_storage, upload_to='partners/headers/'), + model_name="partner", + name="site_header", + field=models.ImageField( + blank=True, + null=True, + storage=storages["public"], + upload_to="partners/headers/", + ), ), migrations.AlterField( - model_name='partnerimage', - name='image', - field=models.ImageField(storage=thaliawebsite.storage.backend.get_public_storage, upload_to='partners/images/'), + model_name="partnerimage", + name="image", + field=models.ImageField( + storage=storages["public"], upload_to="partners/images/" + ), ), migrations.AlterField( - model_name='vacancy', - name='company_logo', - field=models.ImageField(blank=True, null=True, storage=thaliawebsite.storage.backend.get_public_storage, upload_to='partners/vacancy-logos/', verbose_name='company logo'), + model_name="vacancy", + name="company_logo", + field=models.ImageField( + blank=True, + null=True, + storage=storages["public"], + upload_to="partners/vacancy-logos/", + verbose_name="company logo", + ), ), ] diff --git a/website/partners/migrations/0026_partner_alternate_logo.py b/website/partners/migrations/0026_partner_alternate_logo.py index d46f78e6f..e6895740f 100644 --- a/website/partners/migrations/0026_partner_alternate_logo.py +++ b/website/partners/migrations/0026_partner_alternate_logo.py @@ -1,19 +1,24 @@ # Generated by Django 4.0.7 on 2022-10-05 18:11 from django.db import migrations, models -import thaliawebsite.storage.backend +from django.core.files.storage import storages class Migration(migrations.Migration): - dependencies = [ - ('partners', '0025_migrate_public_folders'), + ("partners", "0025_migrate_public_folders"), ] operations = [ migrations.AddField( - model_name='partner', - name='alternate_logo', - field=models.ImageField(blank=True, help_text='If set, this logo will be shown on the frontpage banner. Please use files with proper transparency.', null=True, storage=thaliawebsite.storage.backend.get_public_storage, upload_to='partners/logos/'), + model_name="partner", + name="alternate_logo", + field=models.ImageField( + blank=True, + help_text="If set, this logo will be shown on the frontpage banner. Please use files with proper transparency.", + null=True, + storage=storages["public"], + upload_to="partners/logos/", + ), ), ] diff --git a/website/partners/models.py b/website/partners/models.py index 3c6a1f6cd..61cb299a3 100644 --- a/website/partners/models.py +++ b/website/partners/models.py @@ -1,4 +1,5 @@ from django.core.exceptions import ValidationError +from django.core.files.storage import storages from django.core.validators import RegexValidator, URLValidator from django.db import models from django.urls import reverse @@ -8,7 +9,6 @@ from thumbnails.fields import ImageField from tinymce.models import HTMLField -from thaliawebsite.storage.backend import get_public_storage from utils import countries from utils.media.services import get_upload_to_function @@ -27,13 +27,13 @@ class Partner(models.Model): logo = ImageField( upload_to=get_upload_to_function("partners/logos/"), resize_source_to="source_png", - storage=get_public_storage, + storage=storages["public"], ) alternate_logo = ImageField( upload_to=get_upload_to_function("partners/logos/"), resize_source_to="source_png", - storage=get_public_storage, + storage=storages["public"], blank=True, null=True, help_text=_( @@ -44,7 +44,7 @@ class Partner(models.Model): site_header = ImageField( upload_to=get_upload_to_function("partners/headers/"), resize_source_to="source_png", - storage=get_public_storage, + storage=storages["public"], null=True, blank=True, ) @@ -144,7 +144,7 @@ class PartnerImage(models.Model): image = ImageField( upload_to="partners/images/", resize_source_to="source_png", - storage=get_public_storage, + storage=storages["public"], ) def __init__(self, *args, **kwargs): @@ -218,7 +218,7 @@ class Vacancy(models.Model): _("company logo"), upload_to="partners/vacancy-logos/", resize_source_to="source_png", - storage=get_public_storage, + storage=storages["public"], null=True, blank=True, ) diff --git a/website/thaliawebsite/settings.py b/website/thaliawebsite/settings.py index 94230d832..6be42eefe 100644 --- a/website/thaliawebsite/settings.py +++ b/website/thaliawebsite/settings.py @@ -285,25 +285,30 @@ def from_env( AWS_CLOUDFRONT_KEY_ID = os.environ.get("AWS_CLOUDFRONT_KEY_ID", None) AWS_S3_CUSTOM_DOMAIN = os.environ.get("AWS_CLOUDFRONT_DOMAIN", None) - STATICFILES_STORAGE = "thaliawebsite.storage.backend.StaticS3Storage" + _STATICFILES_STORAGE = "thaliawebsite.storage.backend.StaticS3Storage" STATIC_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/static/" - DEFAULT_FILE_STORAGE = "thaliawebsite.storage.backend.PrivateS3Storage" + _DEFAULT_FILE_STORAGE = "thaliawebsite.storage.backend.PrivateS3Storage" - PUBLIC_FILE_STORAGE = "thaliawebsite.storage.backend.PublicS3Storage" + _PUBLIC_FILE_STORAGE = "thaliawebsite.storage.backend.PublicS3Storage" PUBLIC_MEDIA_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/" else: - STATICFILES_STORAGE = setting( + _STATICFILES_STORAGE = setting( development="django.contrib.staticfiles.storage.StaticFilesStorage", production="django.contrib.staticfiles.storage.ManifestStaticFilesStorage", ) STATIC_URL = "/static/" - DEFAULT_FILE_STORAGE = "thaliawebsite.storage.backend.PrivateFileSystemStorage" + _DEFAULT_FILE_STORAGE = "thaliawebsite.storage.backend.PrivateFileSystemStorage" - PUBLIC_FILE_STORAGE = "thaliawebsite.storage.backend.PublicFileSystemStorage" + _PUBLIC_FILE_STORAGE = "thaliawebsite.storage.backend.PublicFileSystemStorage" PUBLIC_MEDIA_URL = "/media/public/" +STORAGES = { + "default": {"BACKEND": _DEFAULT_FILE_STORAGE}, + "public": {"BACKEND": _PUBLIC_FILE_STORAGE}, + "staticfiles": {"BACKEND": _STATICFILES_STORAGE}, +} # https://docs.djangoproject.com/en/dev/ref/settings/#conn-max-age CONN_MAX_AGE = int(from_env("CONN_MAX_AGE", development="0", production="60")) @@ -879,6 +884,10 @@ def from_env( # See https://github.com/jrief/django-sass-processor SASS_PROCESSOR_INCLUDE_FILE_PATTERN = r"^.+\.scss$" +# django-sass-processor does not use the Django 4.2 `storages` API yet, +# but we can simply give it the path as we would with the new API. +SASS_PROCESSOR_STORAGE = _STATICFILES_STORAGE + # See utils/model/signals.py for explanation SUSPEND_SIGNALS = False @@ -897,7 +906,9 @@ def from_env( THUMBNAILS = { "METADATA": THUMBNAILS_METADATA, "STORAGE": { - "BACKEND": DEFAULT_FILE_STORAGE, + # django-thumbs does not use the Django 4.2 `storages` API yet, + # but we can simply give it the path as we would with the new API. + "BACKEND": _DEFAULT_FILE_STORAGE, }, "SIZES": { "small": { diff --git a/website/thaliawebsite/storage/backend.py b/website/thaliawebsite/storage/backend.py index 477de1be3..8230402a6 100644 --- a/website/thaliawebsite/storage/backend.py +++ b/website/thaliawebsite/storage/backend.py @@ -2,15 +2,11 @@ from django.conf import settings from django.core import signing -from django.core.files.storage import FileSystemStorage, get_storage_class +from django.core.files.storage import FileSystemStorage from storages.backends.s3boto3 import S3Boto3Storage, S3ManifestStaticStorage -def get_public_storage(): - return get_storage_class(settings.PUBLIC_FILE_STORAGE)() - - class S3RenameMixin: def rename(self, old_name, new_name): self.bucket.Object(new_name).copy_from( diff --git a/website/utils/management/commands/remove_unused_media.py b/website/utils/management/commands/remove_unused_media.py index 60bfaf255..1f5e2b898 100644 --- a/website/utils/management/commands/remove_unused_media.py +++ b/website/utils/management/commands/remove_unused_media.py @@ -2,14 +2,12 @@ from django.apps import apps from django.conf import settings -from django.core.files.storage import DefaultStorage +from django.core.files.storage import DefaultStorage, storages from django.core.management.base import BaseCommand from django.core.validators import EMPTY_VALUES from django.db import models from django.utils import timezone -from thaliawebsite.storage.backend import get_public_storage - def get_file_fields(): """Get all FileFields of all models.""" @@ -152,7 +150,7 @@ def handle(self, *args, **options): minimum_file_age=options.get("minimum_file_age"), ) - public_storage = get_public_storage() + public_storage = storages["public"] unused_public_media = get_unused_media( public_storage, minimum_file_age=options.get("minimum_file_age"),