MVJ-470 Avoid PicklingError in email reports (#753)

* Avoid PicklingError

Passing complex objects to django-q async_task results in a
PicklingError during serialization which prevents report creation.
This change avoids the problem by only passing module-level functions
and regular Python data structures to async_task and between the
asynchronous function and the hook.

* Add custom timeout to rent forecast report

Without async_task_timeout property, async task would use the default
timeout, which is currently too short at 90 seconds, because task
is constantly being killed by Qcluster.

* Add custom timeout to lease statistic report

This is just in case, because async email reports appear to sometimes
get eternally stuck in the queues if the worker is killed by a timeout.

* Add type hinting to excel.py

leasing/report/excel.py

@@ -1,4 +1,5 @@
 from enum import Enum
+from typing import TypeAlias
 
 from xlsxwriter.utility import xl_range, xl_rowcol_to_cell
 
@@ -12,61 +13,76 @@ class FormatType(Enum):
     AREA = "area"
 
 
-class ExcelRow:
-    def __init__(self, cells=None):
-        self.cells = []
-
-        if cells is not None:
-            self.cells.extend(cells)
-
-
 class ExcelCell:
-    def __init__(self, column, value=None, format_type=None):
+    def __init__(
+        self,
+        column: int,
+        value: str | None = None,
+        format_type: FormatType | None = None,
+    ):
         self.column = column
         self.value = value
         self.format_type = format_type
-        self.row = None
-        self.first_data_row_num = None
+        self.row: int | None = None
+        self.first_data_row_num: int | None = None
 
-    def get_value(self):
+    def get_value(self) -> str | None:
         return self.value
 
-    def get_format_type(self):
+    def get_format_type(self) -> FormatType | None:
         return self.format_type
 
-    def set_row(self, row_num):
+    def set_row(self, row_num: int) -> None:
         self.row = row_num
 
-    def set_first_data_row_num(self, row_num):
+    def set_first_data_row_num(self, row_num: int) -> None:
         self.first_data_row_num = row_num
 
 
+class ExcelRow:
+    def __init__(self, cells: list[ExcelCell] | None = None):
+        self.cells: list[ExcelCell] = []
+
+        if cells is not None:
+            self.cells.extend(cells)
+
+
 class PreviousRowsSumCell(ExcelCell):
-    def __init__(self, column, count, format_type=FormatType.BOLD):
+    def __init__(
+        self, column: int, count: int, format_type: FormatType | None = FormatType.BOLD
+    ):
         super().__init__(column, format_type=format_type)
 
         self.count = count
 
-    def get_value(self):
+    def get_value(self) -> str:
         return "=SUM({}:{})".format(
             xl_rowcol_to_cell(self.row - self.count, self.column),
             xl_rowcol_to_cell(self.row - 1, self.column),
         )
 
 
+TargetRange: TypeAlias = tuple[int, int, int, int]
+
+
 class SumCell(ExcelCell):
-    def __init__(self, column, format_type=FormatType.BOLD, target_ranges=None):
+    def __init__(
+        self,
+        column: int,
+        format_type=FormatType.BOLD,
+        target_ranges: list[TargetRange] | None = None,
+    ):
         super().__init__(column, format_type=format_type)
 
         if target_ranges:
             self.target_ranges = target_ranges
         else:
             self.target_ranges = []
 
-    def add_target_range(self, range):
+    def add_target_range(self, range: TargetRange):
         self.target_ranges.append(range)
 
-    def get_value(self):
+    def get_value(self) -> str:
         return "=SUM({})".format(
             ",".join(
                 [

leasing/report/forms.py

@@ -5,7 +5,15 @@ class ReportFormBase(forms.Form):
     """Dynamic form that initializes its fields from `input_fields` parameter"""
 
     def __init__(self, *args, **kwargs):
-        input_fields = kwargs.pop("input_fields")
+        """
+        args is expected to contain the query parameters, which in turn contains
+        the report settings when the report was requested.
+
+        kwargs is expected to contain a key "input_fields", whose value is a
+        dictionary containing the names of the query parameters and the form
+        field objects they reference.
+        """
+        input_fields: dict[str, forms.Field] = kwargs.pop("input_fields")
         super().__init__(*args, **kwargs)
 
         for field_name, field in input_fields.items():

leasing/report/invoice/invoice_payments.py

@@ -1,5 +1,6 @@
 from django import forms
 from django.utils.translation import gettext_lazy as _
+from rest_framework.request import Request
 from rest_framework.response import Response
 
 from leasing.models import ServiceUnit
@@ -65,8 +66,9 @@ def get_data(self, input_data):
 
         return qs
 
-    def get_response(self, request):
-        report_data = self.get_data(self.get_input_data(request))
+    def get_response(self, request: Request) -> Response:
+        input_data = self.get_input_data(request.query_params)
+        report_data = self.get_data(input_data)
         serialized_report_data = self.serialize_data(report_data)
 
         if request.accepted_renderer.format != "xlsx":

leasing/report/invoice/invoices_in_period.py

@@ -1,6 +1,7 @@
 from django import forms
 from django.utils.translation import gettext_lazy as _
 from enumfields.drf import EnumField
+from rest_framework.request import Request
 from rest_framework.response import Response
 
 from leasing.enums import InvoiceState
@@ -119,8 +120,9 @@ def get_data(self, input_data):
 
         return qs
 
-    def get_response(self, request):
-        report_data = self.get_data(self.get_input_data(request))
+    def get_response(self, request: Request) -> Response:
+        input_data = self.get_input_data(request.query_params)
+        report_data = self.get_data(input_data)
         serialized_report_data = self.serialize_data(report_data)
 
         if request.accepted_renderer.format != "xlsx":

leasing/report/invoice/invoicing_review.py

@@ -12,6 +12,7 @@
 from django.utils.translation import gettext_lazy, pgettext_lazy
 from enumfields import Enum
 from enumfields.drf import EnumField
+from rest_framework.request import Request
 from rest_framework.response import Response
 
 from leasing.models import ReceivableType, ServiceUnit
@@ -610,8 +611,9 @@ def get_data(self, input_data):
 
         return result
 
-    def get_response(self, request):
-        report_data = self.get_data(self.get_input_data(request))
+    def get_response(self, request: Request) -> Response:
+        input_data = self.get_input_data(request.query_params)
+        report_data = self.get_data(input_data)
         serialized_report_data = self.serialize_data(report_data)
 
         if request.accepted_renderer.format != "xlsx":

leasing/report/invoice/open_invoices_report.py

@@ -3,6 +3,7 @@
 
 from django import forms
 from django.utils.translation import gettext_lazy as _
+from rest_framework.request import Request
 from rest_framework.response import Response
 
 from leasing.enums import InvoiceState
@@ -91,8 +92,9 @@ def get_data(self, input_data):
 
         return qs
 
-    def get_response(self, request):
-        report_data = self.get_data(self.get_input_data(request))
+    def get_response(self, request: Request) -> Response:
+        input_data = self.get_input_data(request.query_params)
+        report_data = self.get_data(input_data)
         serialized_report_data = self.serialize_data(report_data)
 
         if request.accepted_renderer.format != "xlsx":

leasing/report/lease/extra_city_rent.py

@@ -6,6 +6,7 @@
 from django import forms
 from django.db.models import Q
 from django.utils.translation import gettext_lazy as _
+from rest_framework.request import Request
 from rest_framework.response import Response
 
 from leasing.enums import TenantContactType
@@ -229,8 +230,9 @@ def get_data(self, input_data):
 
         return aggregated_data
 
-    def get_response(self, request):
-        report_data = self.get_data(self.get_input_data(request))
+    def get_response(self, request: Request) -> Response:
+        input_data = self.get_input_data(request.query_params)
+        report_data = self.get_data(input_data)
 
         if request.accepted_renderer.format != "xlsx":
             serialized_report_data = self.serialize_data(report_data)

leasing/report/lease/lease_count_report.py

@@ -3,6 +3,7 @@
 from django.db.models.aggregates import Count
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
+from rest_framework.request import Request
 from rest_framework.response import Response
 
 from leasing.models import Lease, ServiceUnit
@@ -45,8 +46,9 @@ def get_data(self, input_data):
 
         return qs
 
-    def get_response(self, request):
-        report_data = self.get_data(self.get_input_data(request))
+    def get_response(self, request: Request) -> Response:
+        input_data = self.get_input_data(request.query_params)
+        report_data = self.get_data(input_data)
         serialized_report_data = self.serialize_data(report_data)
 
         if request.accepted_renderer.format != "xlsx":

leasing/report/lease/lease_statistic_report.py

@@ -4,7 +4,7 @@
 from functools import lru_cache
 
 from django import forms
-from django.db.models import Q
+from django.db.models import Q, QuerySet
 from django.utils import formats
 from django.utils.translation import gettext_lazy as _
 from enumfields.drf import EnumField
@@ -371,8 +371,9 @@ class LeaseStatisticReport(AsyncReportBase):
             "width": 20,
         },
     }
+    async_task_timeout = 60 * 30  # 30 minutes
 
-    def get_data(self, input_data):
+    def get_data(self, input_data) -> QuerySet[Lease]:
         qs = Lease.objects.select_related(
             "identifier__type",
             "identifier__district",
@@ -413,9 +414,3 @@ def get_data(self, input_data):
             )
 
         return qs
-
-    def generate_report(self, user, input_data):
-        report_data = self.get_data(input_data)
-        serialized_report_data = self.serialize_data(report_data)
-
-        return self.data_as_excel(serialized_report_data)

leasing/report/lease/lease_statistic_report2.py

@@ -3,7 +3,7 @@
 from decimal import ROUND_HALF_UP, Decimal
 
 from django import forms
-from django.db.models import Q
+from django.db.models import Q, QuerySet
 from django.utils import formats
 from django.utils.translation import gettext_lazy as _
 from enumfields.drf import EnumField
@@ -397,7 +397,7 @@ class LeaseStatisticReport2(AsyncReportBase):
     }
     async_task_timeout = 60 * 30  # 30 minutes
 
-    def get_data(self, input_data):
+    def get_data(self, input_data) -> QuerySet[Lease]:
         qs = Lease.objects.select_related(
             "identifier__type",
             "identifier__district",
@@ -446,9 +446,3 @@ def get_data(self, input_data):
             )
 
         return qs
-
-    def generate_report(self, user, input_data):
-        report_data = self.get_data(input_data)
-        serialized_report_data = self.serialize_data(report_data)
-
-        return self.data_as_excel(serialized_report_data)

leasing/report/lease/rent_forecast.py

@@ -56,6 +56,7 @@ class RentForecastReport(AsyncReportBase):
         "year": {"label": _("Year")},
         "rent": {"label": _("Rent"), "format": "money", "width": 13},
     }
+    async_task_timeout = 60 * 30  # 30 minutes
 
     def get_data(self, input_data):  # NOQA C901
         start_date = datetime.date(year=input_data["start_year"], month=1, day=1)