aug v2

vespadb/observations/admin.py

@@ -152,7 +152,6 @@ class ObservationAdmin(gis_admin.GISModelAdmin):
         "wn_validation_status",
         "wn_admin_notes",
         "observer_name",
-        "observer_email",
         "observer_phone_number",
         "created_datetime",
         "created_by",

vespadb/observations/helpers.py

@@ -1,9 +1,13 @@
 """Observation helpers."""
 
+import time
+from collections.abc import Callable
 from datetime import datetime
+from typing import Any, TypeVar
 
 import pytz
 from dateutil import parser
+from django.db.utils import OperationalError
 
 # List of accepted datetime formats
 DATETIME_FORMATS = [
@@ -17,6 +21,8 @@
     "%Y-%m-%d",
 ]
 
+T = TypeVar("T")
+
 
 def parse_and_convert_to_utc(datetime_str: str) -> datetime:
     """
@@ -54,3 +60,18 @@ def parse_and_convert_to_cet(datetime_str: str) -> datetime:
     """
     cet_tz = pytz.timezone("Europe/Brussels")
     return parser.parse(datetime_str).astimezone(cet_tz)
+
+
+def retry_with_backoff(func: Callable[..., T], *args: Any, max_retries: int = 3, backoff_in_seconds: int = 2) -> Any:
+    """Retry mechanism for retrying a function with a backoff strategy."""
+    attempt = 0
+    while attempt < max_retries:
+        try:
+            return func(*args)  # Call the function with the provided arguments
+        except OperationalError:
+            attempt += 1
+            if attempt < max_retries:
+                time.sleep(backoff_in_seconds)  # Wait before retrying
+            else:
+                raise
+    return None

vespadb/observations/migrations/0028_alter_observation_eradication_duration.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-10-15 14:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('observations', '0027_alter_observation_eradication_product_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='observation',
+            name='eradication_duration',
+            field=models.PositiveIntegerField(blank=True, help_text='Duration of the eradication in minutes', null=True),
+        ),
+    ]

vespadb/observations/models.py

@@ -298,7 +298,7 @@ class Observation(models.Model):
     eradicator_name = models.CharField(
         max_length=255, blank=True, null=True, help_text="Name of the person who eradicated the nest"
     )
-    eradication_duration = models.IntegerField(
+    eradication_duration = models.PositiveIntegerField(
         blank=True, null=True, help_text="Duration of the eradication in minutes"
     )
     eradication_persons = models.IntegerField(

vespadb/observations/views.py

@@ -16,7 +16,7 @@
 from django.db import transaction
 from django.db.models import CharField, OuterRef, QuerySet, Subquery, Value
 from django.db.models.functions import Coalesce
-from django.db.utils import IntegrityError
+from django.db.utils import IntegrityError, OperationalError
 from django.http import HttpResponse, JsonResponse
 from django.utils.decorators import method_decorator
 from django.utils.timezone import now
@@ -40,7 +40,7 @@
 
 from vespadb.observations.cache import invalidate_geojson_cache, invalidate_observation_cache
 from vespadb.observations.filters import ObservationFilter
-from vespadb.observations.helpers import parse_and_convert_to_utc
+from vespadb.observations.helpers import parse_and_convert_to_utc, retry_with_backoff
 from vespadb.observations.models import Municipality, Observation, Province
 from vespadb.observations.serializers import (
     MunicipalitySerializer,
@@ -633,39 +633,49 @@ def save_observations(self, valid_data: list[dict[str, Any]]) -> Response:
     @method_decorator(ratelimit(key="ip", rate="60/m", method="GET", block=True))
     @action(detail=False, methods=["get"], permission_classes=[AllowAny])
     def export(self, request: Request) -> Response:
-        """Export observations data in the specified format."""
+        """Export observations data in the specified format with retry mechanism."""
         user = request.user
         export_format = request.query_params.get("export_format", "json").lower()
         queryset = self.filter_queryset(self.get_queryset())
         serializer_class = self.get_serializer_class()
         serializer_context = self.get_serializer_context()
 
-        paginator = Paginator(queryset, 1000)
+        paginator = Paginator(queryset, 1000)  # Limiting to smaller batches
         serialized_data = []
         errors = []
 
         for page_number in paginator.page_range:
             page = paginator.page(page_number)
             try:
+                # Try serializing the entire page first
                 serializer = serializer_class(page, many=True, context=serializer_context)
                 serialized_data.extend(serializer.data)
             except ValidationError:
                 logger.exception(f"Validation error in page {page_number}, processing individually.")
                 for obj in page.object_list:
-                    serializer = serializer_class(obj, context=serializer_context)
                     try:
-                        serializer.is_valid(raise_exception=True)
-                        serialized_data.append(serializer.data)
+                        # Retry the serialization of each object individually with backoff
+                        serializer_data = retry_with_backoff(
+                            lambda obj=obj: serializer_class(obj, context=serializer_context).data
+                        )
+                        serialized_data.append(serializer_data.data)
                     except ValidationError as e:
                         errors.append({
                             "id": obj.id,
                             "error": str(e),
-                            "data": serializer.data,
+                            "data": serializer_data.data if serializer_data else None,
+                        })
+                    except OperationalError as e:
+                        logger.exception(f"Database error for object {obj.id}: {e}")
+                        errors.append({
+                            "id": obj.id,
+                            "error": "Database connection error",
                         })
 
         if errors:
             logger.error(f"Errors during export: {errors}")
 
+        # Handle the final export based on the requested format
         if export_format == "json":
             response_data = {
                 "data": serialized_data,
@@ -682,6 +692,7 @@ def export(self, request: Request) -> Response:
                 writer.writeheader()
                 writer.writerows(serialized_data)
             return response
+
         return Response({"error": "Unsupported format specified."}, status=status.HTTP_400_BAD_REQUEST)