chore: update Django to 4.2 LTS (#2911)

* chore: update Django to 4.2 LTS

* fix: update_or_create failed on encrypted fields

* chore: add protocol in CSRF_TRUSTED_ORIGINS

* fix: failing tests

* chore: remove obsolete USE_L10N setting

* chore: update readme

.env.benefit-backend.example

@@ -60,7 +60,7 @@ AZURE_CONTAINER=
 MEDIA_ROOT=/app/var/media
 
 CSRF_COOKIE_NAME=yjdhcsrftoken
-CSRF_TRUSTED_ORIGINS="localhost:3000,localhost:3100"
+CSRF_TRUSTED_ORIGINS="https://localhost:3000,https://localhost:3100"
 
 YRTTI_BASE_URL=https://yrtti-integration-test.agw.arodevtest.hel.fi/api
 YRTTI_AUTH_PASSWORD=
@@ -119,3 +119,5 @@ ENABLE_CLAMAV=1
 CLAMAV_URL=http://localhost:8080/api/v1
 AHJO_REQUEST_TIMEOUT=60
 ENABLE_AHJO_AUTOMATION=0
+#For Django 4.2 compatibility
+DJANGO_4_CSRF_TRUSTED_ORIGINS="https://localhost:3000,https://localhost:3100"

backend/README.md

@@ -4,6 +4,8 @@ Refer to [top level README](https://github.com/City-of-Helsinki/yjdh/blob/main/R
 
 [kesaseteli/README.md](https://github.com/City-of-Helsinki/yjdh/blob/main/backend/kesaseteli/README.md) or [benefit/README.md](https://github.com/City-of-Helsinki/yjdh/blob/main/backend/benefit/README.md) contain more information about the local setup. This applies to TET pretty well.
 
+Django updated to version 4.2 during May 2024, as security support for 3.2 ends 30.4.2024. Django 4.2 will stop receiving security updates 30.4.2026.
+
 ## Authentication in YJDH
 
 The authentication setups used by YJDH projects are described in this document. The emphasis is on

backend/benefit/README.md

@@ -115,6 +115,8 @@ application using the applicant UI.
 1. Install `pip-tools`:
 
    - `pip install pip-tools`
+   - `pip install --upgrade pip-tools`
+
 
 2. Add new packages to `requirements.in` or `requirements-dev.in`
 

backend/benefit/applications/api/v1/serializers/application.py

@@ -1316,16 +1316,16 @@ def create(self, validated_data):
         return application
 
     def _update_or_create_employee(self, application, employee_data):
-        employee, was_created = Employee.objects.update_or_create(
-            application=application, defaults=employee_data
-        )
-        user = self.get_logged_in_user()
-
-        if was_created:
-            audit_log_operation = Operation.CREATE
-        else:
+        try:
+            employee = Employee.objects.get(application=application)
+            for key, value in employee_data.items():
+                setattr(employee, key, value)
+            employee.save()
             audit_log_operation = Operation.UPDATE
-
+        except Employee.DoesNotExist:
+            employee = Employee.objects.create(application=application, **employee_data)
+            audit_log_operation = Operation.CREATE
+        user = self.get_logged_in_user()
         audit_logging.log(
             user,
             "",

backend/benefit/helsinkibenefit/settings.py

@@ -48,6 +48,7 @@
     CORS_ALLOWED_ORIGINS=(list, []),
     CORS_ALLOW_ALL_ORIGINS=(bool, False),
     CSRF_COOKIE_DOMAIN=(str, "localhost"),
+    DJANGO_4_CSRF_TRUSTED_ORIGINS=(list, []),
     CSRF_TRUSTED_ORIGINS=(list, ["localhost:3000", "localhost:3100"]),
     CSRF_COOKIE_NAME=(str, "yjdhcsrftoken"),
     YTJ_BASE_URL=(str, "https://avoindata.prh.fi"),
@@ -238,7 +239,6 @@
 
 TIME_ZONE = "Europe/Helsinki"
 USE_I18N = True
-USE_L10N = True
 USE_TZ = True
 DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
 
@@ -323,7 +323,10 @@
 CORS_ALLOWED_ORIGINS = env.list("CORS_ALLOWED_ORIGINS")
 CORS_ALLOW_ALL_ORIGINS = env.bool("CORS_ALLOW_ALL_ORIGINS")
 CSRF_COOKIE_DOMAIN = env.str("CSRF_COOKIE_DOMAIN")
-CSRF_TRUSTED_ORIGINS = env.list("CSRF_TRUSTED_ORIGINS")
+django_4_csrf_origins = env.list("DJANGO_4_CSRF_TRUSTED_ORIGINS")
+CSRF_TRUSTED_ORIGINS = (
+    django_4_csrf_origins if django_4_csrf_origins else env.list("CSRF_TRUSTED_ORIGINS")
+)
 CSRF_COOKIE_NAME = env.str("CSRF_COOKIE_NAME")
 CSRF_COOKIE_SECURE = True
 CSRF_USE_SESSIONS = True
@@ -481,8 +484,14 @@
 FIELD_ENCRYPTION_KEYS = [ENCRYPTION_KEY]
 
 # Django storages
-DEFAULT_FILE_STORAGE = env("DEFAULT_FILE_STORAGE")
-
+STORAGES = {
+    "default": {
+        "BACKEND": env("DEFAULT_FILE_STORAGE"),
+    },
+    "staticfiles": {
+        "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
+    },
+}
 AZURE_ACCOUNT_NAME = env("AZURE_ACCOUNT_NAME")
 AZURE_ACCOUNT_KEY = env("AZURE_ACCOUNT_KEY")
 AZURE_CONTAINER = env("AZURE_CONTAINER")

backend/benefit/requirements.in

@@ -12,7 +12,7 @@ django-sql-utils
 django-storages[azure]
 django_auth_adfs
 djangorestframework
-django~=3.2.23
+django~=4.2.11
 drf-nested-routers
 drf-spectacular
 elasticsearch~=7.17 # TODO: remove and update to 8.x when elasticsearch servers are updated to elasticsearch 8

backend/benefit/requirements.txt

@@ -41,7 +41,7 @@ defusedxml==0.7.1
     #   pysaml2
 deprecation==2.1.0
     # via django-helusers
-django==3.2.23
+django==4.2.11
     # via
     #   -r requirements.in
     #   django-auth-adfs
@@ -193,7 +193,6 @@ python-stdnum==1.19
 pytz==2022.7.1
     # via
     #   babel
-    #   django
     #   djangorestframework
     #   pysaml2
 pyyaml==6.0

backend/shared/shared/oidc/tests/conftest.py

@@ -1,15 +1,30 @@
 import pytest
 from django.contrib.sessions.middleware import SessionMiddleware
+from django.http import HttpResponse
 from django.test import RequestFactory
 
 from shared.common.tests.conftest import *  # noqa
 
 
+@pytest.fixture
+def mock_request():
+    state = "test"
+    code = "test"
+    factory = RequestFactory()
+
+    return factory.get("/", {"code": code, "state": state})
+
+
+@pytest.fixture
+def get_response(mock_request):
+    return HttpResponse()
+
+
 @pytest.fixture()
 def session_request():
     factory = RequestFactory()
     request = factory.get("/")
-    middleware = SessionMiddleware()
+    middleware = SessionMiddleware(get_response)
     middleware.process_request(request)
     request.session.save()
 

backend/shared/shared/oidc/tests/test_auth_backend.py

@@ -8,7 +8,7 @@
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.contrib.sessions.middleware import SessionMiddleware
-from django.test import override_settings, RequestFactory
+from django.test import override_settings
 from django.utils import timezone
 
 from shared.common.tests.conftest import store_tokens_in_session
@@ -78,6 +78,8 @@ def test_authenticate(
     requests_mock,
     oidc_save_personally_identifiable_info: Optional[bool],
     expect_personal_info_removal: bool,
+    mock_request,
+    get_response,
 ):
     set_setting_to_value_or_del_with_none(
         "OIDC_SAVE_PERSONALLY_IDENTIFIABLE_INFO",
@@ -88,11 +90,8 @@ def test_authenticate(
     matcher = re.compile(re.escape(settings.OIDC_OP_USER_ENDPOINT))
     requests_mock.get(matcher, json=_test_claims)
 
-    state = "test"
-    code = "test"
-    factory = RequestFactory()
-    request = factory.get("/", {"code": code, "state": state})
-    middleware = SessionMiddleware()
+    request = mock_request
+    middleware = SessionMiddleware(get_response)
     middleware.process_request(request)
     request.session.save()