feat: HL 1476 talpa improvements (#3371)

* fix: handle talpa lists on success and failure

* feat: refactor talpa callback

* feat: enable/disable talpa callbacks

backend/benefit/applications/api/v1/application_batch_views.py

@@ -1,5 +1,7 @@
 import logging
 
+from django.conf import settings
+from django.contrib.auth.models import AnonymousUser
 from django.db import transaction
 from django.forms import ValidationError
 from django.http import HttpResponse
@@ -34,6 +36,9 @@
 from applications.services.applications_csv_report import ApplicationsCsvService
 from common.authentications import RobotBasicAuthentication
 from common.permissions import BFIsHandler
+from common.utils import get_request_ip_address
+from shared.audit_log import audit_logging
+from shared.audit_log.enums import Operation
 from shared.audit_log.viewsets import AuditLoggingModelViewSet
 
 LOGGER = logging.getLogger(__name__)
@@ -218,22 +223,37 @@ def talpa_export_batch(self, request, *args, **kwargs) -> HttpResponse:
         response["Content-Disposition"] = "attachment; filename={filename}.csv".format(
             filename=file_name
         )
-        # for easier testing in the test environment do not update the batches as sent_to_talpa
-        # remove this when TALPA integration is ready for production
-        if not skip_update:
-            try:
-                # Update all approved batches to SENT_TO_TALPA status in a single query
-                approved_batches.update(status=ApplicationBatchStatus.SENT_TO_TALPA)
-                # Update all applications in the approved batches to SUCCESSFULLY_SENT_TO_TALPA status and archived=True
-                for a in applications:
-                    a.talpa_status = ApplicationTalpaStatus.SUCCESSFULLY_SENT_TO_TALPA
-                    a.archived = True
-                    a.save()
-
-            except Exception as e:
-                LOGGER.error(
-                    f"An error occurred while updating batches after Talpa csv download: {e}"
-                )
+        if settings.TALPA_CALLBACK_ENABLED is False:
+            # for easier testing in the test environment do not update the batches as sent_to_talpa
+            # remove this when TALPA integration is ready for production
+            if not skip_update:
+                ip_address = get_request_ip_address(request)
+                try:
+                    # Update all approved batches to SENT_TO_TALPA status in a single query
+                    approved_batches.update(status=ApplicationBatchStatus.SENT_TO_TALPA)
+                    # Update all applications in the approved batches to
+                    # SUCCESSFULLY_SENT_TO_TALPA status and archived=True
+                    for a in applications:
+                        a.talpa_status = (
+                            ApplicationTalpaStatus.SUCCESSFULLY_SENT_TO_TALPA
+                        )
+                        a.archived = True
+                        a.save()
+
+                        audit_logging.log(
+                            AnonymousUser,
+                            "",
+                            Operation.READ,
+                            a,
+                            ip_address=ip_address,
+                            additional_information="application csv data was downloaded by TALPA\
+    and it was marked as archived",
+                        )
+
+                except Exception as e:
+                    LOGGER.error(
+                        f"An error occurred while updating batches after Talpa csv download: {e}"
+                    )
         return response
 
     @action(methods=["PATCH"], detail=False)

backend/benefit/applications/api/v1/talpa_integration_views.py

@@ -1,6 +1,7 @@
 import logging
 from typing import List, Union
 
+from django.conf import settings
 from django.contrib.auth.models import AnonymousUser
 from django.db import transaction
 from rest_framework import status
@@ -10,7 +11,7 @@
 
 from applications.api.v1.serializers.talpa_callback import TalpaCallbackSerializer
 from applications.enums import ApplicationBatchStatus, ApplicationTalpaStatus
-from applications.models import Application
+from applications.models import Application, ApplicationBatch
 from common.authentications import RobotBasicAuthentication
 from common.utils import get_request_ip_address
 from shared.audit_log import audit_logging
@@ -24,6 +25,13 @@ class TalpaCallbackView(APIView):
     permission_classes = [AllowAny]
 
     def post(self, request, *args, **kwargs):
+        if settings.TALPA_CALLBACK_ENABLED is False:
+            LOGGER.warning("Talpa callback is disabled")
+            return Response(
+                {"message": "Talpa callback is disabled"},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
         serializer = TalpaCallbackSerializer(data=request.data)
 
         if serializer.is_valid():
@@ -33,60 +41,79 @@ def post(self, request, *args, **kwargs):
             return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
 
     def process_callback(self, data, request):
-        if data["status"] == "Success":
+        if data["status"] in ["Success", "Failure"]:
+            ip_address = get_request_ip_address(request)
             self._handle_successful_applications(
-                data["successful_applications"], get_request_ip_address(request)
+                data["successful_applications"],
+                ip_address,
             )
-            self._handle_failed_applications(data["failed_applications"])
+            self._handle_failed_applications(data["failed_applications"], ip_address)
         else:
-            LOGGER.error(f"Received a talpa callback with status: {data['status']}")
+            LOGGER.error(
+                f"Received a talpa callback with unknown status: {data['status']}"
+            )
 
     def _get_applications(self, application_numbers) -> Union[List[Application], None]:
         applications = Application.objects.filter(
             application_number__in=application_numbers
         )
         if not applications.exists() and application_numbers:
-            LOGGER.error(f"No applications found with numbers: {application_numbers}")
+            LOGGER.error(
+                f"No applications found with numbers: {application_numbers} for update after TALPA download"
+            )
             return None
         return applications
 
     def _handle_successful_applications(
         self, application_numbers: list, ip_address: str
     ):
-        """Add audit log entries for applications which were processed successfully by TALPA"""
-        successful_applications = self._get_applications(application_numbers)
-        if successful_applications:
-            successful_applications.update(
-                talpa_status=ApplicationTalpaStatus.SUCCESSFULLY_SENT_TO_TALPA,
-                archived=True,
-            )
-
-            for application in successful_applications:
-                audit_logging.log(
-                    AnonymousUser,
-                    "",
-                    Operation.READ,
-                    application,
-                    ip_address=ip_address,
-                    additional_information="application was read succesfully by TALPA",
-                )
+        applications = self._get_applications(application_numbers)
+        self.update_application_and_related_batch(
+            applications,
+            ip_address,
+            ApplicationTalpaStatus.SUCCESSFULLY_SENT_TO_TALPA,
+            ApplicationBatchStatus.SENT_TO_TALPA,
+            "application was read succesfully by TALPA and archived",
+            is_archived=True,
+        )
 
     @transaction.atomic
-    def _handle_failed_applications(self, application_numbers: list):
+    def _handle_failed_applications(self, application_numbers: list, ip_address: str):
         """Update applications and related batch which could not be processed with status REJECTED_BY_TALPA"""
         applications = self._get_applications(application_numbers)
-        if applications:
-            try:
-                batch = applications.first().batch
-                if batch:
-                    batch.status = ApplicationBatchStatus.REJECTED_BY_TALPA
-                    batch.save()
-                    applications.update(
-                        talpa_status=ApplicationTalpaStatus.REJECTED_BY_TALPA
-                    )
-                else:
-                    LOGGER.error(
-                        f"No batch associated with applications: {applications.values_list('id', flat=True)}"
-                    )
-            except Exception as e:
-                LOGGER.error(f"Error updating batch and applications: {str(e)}")
+        self.update_application_and_related_batch(
+            applications,
+            ip_address,
+            ApplicationTalpaStatus.REJECTED_BY_TALPA,
+            ApplicationBatchStatus.REJECTED_BY_TALPA,
+            "application was rejected by TALPA",
+        )
+
+    @staticmethod
+    def update_application_and_related_batch(
+        applications: List[Application],
+        ip_address: str,
+        application_talpa_status: ApplicationTalpaStatus,
+        batch_status: ApplicationBatchStatus,
+        log_message: str,
+        is_archived: bool = False,
+    ):
+        """Update applications and related batch with given statuses and log the event"""
+        applications.update(
+            talpa_status=application_talpa_status,
+            archived=is_archived,
+        )
+
+        batch_ids = applications.values_list("batch_id", flat=True).distinct()
+        ApplicationBatch.objects.filter(id__in=batch_ids).update(status=batch_status)
+
+        for application in applications:
+            """Add audit log entries for applications which were processed by TALPA"""
+            audit_logging.log(
+                AnonymousUser,
+                "",
+                Operation.READ,
+                application,
+                ip_address=ip_address,
+                additional_information=log_message,
+            )

backend/benefit/applications/tests/test_talpa_integration.py

@@ -125,8 +125,38 @@ def test_write_talpa_csv_file(
         assert "äöÄÖtest" in contents
 
 
+def test_talpa_callback_is_disabled(
+    talpa_client,
+    settings,
+    decided_application,
+):
+    settings.TALPA_CALLBACK_ENABLED = False
+
+    url = reverse(
+        "talpa_callback_url",
+    )
+
+    payload = {
+        "status": "Success",
+        "successful_applications": [decided_application.application_number],
+        "failed_applications": [],
+    }
+
+    response = talpa_client.post(url, data=payload)
+
+    assert response.status_code == 400
+    assert response.data == {"message": "Talpa callback is disabled"}
+
+
 @pytest.mark.django_db
-def test_talpa_callback_success(talpa_client, decided_application):
+def test_talpa_callback_success(
+    talpa_client, decided_application, application_batch, settings
+):
+    settings.TALPA_CALLBACK_ENABLED = True
+
+    decided_application.batch = application_batch
+    decided_application.save()
+
     url = reverse(
         "talpa_callback_url",
     )
@@ -155,13 +185,16 @@ def test_talpa_callback_success(talpa_client, decided_application):
         == ApplicationTalpaStatus.SUCCESSFULLY_SENT_TO_TALPA
     )
 
+    assert decided_application.batch.status == ApplicationBatchStatus.SENT_TO_TALPA
+
     assert decided_application.archived is True
 
 
 @pytest.mark.django_db
 def test_talpa_callback_rejected_application(
-    talpa_client, decided_application, application_batch
+    talpa_client, decided_application, application_batch, settings
 ):
+    settings.TALPA_CALLBACK_ENABLED = True
     decided_application.batch = application_batch
     decided_application.save()
 
@@ -181,6 +214,7 @@ def test_talpa_callback_rejected_application(
     assert response.data == {"message": "Callback received"}
 
     decided_application.refresh_from_db()
+    decided_application.archived = False
 
     assert decided_application.talpa_status == ApplicationTalpaStatus.REJECTED_BY_TALPA
     assert decided_application.batch.status == ApplicationBatchStatus.REJECTED_BY_TALPA

backend/benefit/helsinkibenefit/settings.py

@@ -175,6 +175,7 @@
     ENABLE_CLAMAV=(bool, False),
     CLAMAV_URL=(str, ""),
     ENABLE_AHJO_AUTOMATION=(bool, False),
+    TALPA_CALLBACK_ENABLED=(bool, False),
 )
 if os.path.exists(env_file):
     env.read_env(env_file)
@@ -560,3 +561,4 @@
 CLAMAV_URL = env.str("CLAMAV_URL")
 ENABLE_AHJO_AUTOMATION = env.bool("ENABLE_AHJO_AUTOMATION")
 DEFAULT_SYSTEM_EMAIL = env.str("DEFAULT_SYSTEM_EMAIL")
+TALPA_CALLBACK_ENABLED = env.bool("TALPA_CALLBACK_ENABLED")