Update farm ingestor (#163)

django_project/gap/ingestor/farm.py

@@ -6,6 +6,7 @@
 """
 
 import pandas as pd
+from django.contrib.gis.geos import Point
 
 from gap.ingestor.base import BaseIngestor
 from gap.ingestor.exceptions import (
@@ -50,6 +51,10 @@ def __init__(self, session: IngestorSession, working_dir: str = '/tmp'):
         except FarmGroup.DoesNotExist:
             raise Exception('Farm group does not exist')
 
+    def is_not_empty(self, value):
+        """Check data is empty."""
+        return value and f'{value}' != 'nan'
+
     def _run(self):
         """Run the ingestor."""
         df = pd.read_excel(
@@ -66,18 +71,41 @@ def _run(self):
         for idx, row in enumerate(data):
             try:
                 farm_id = row[Keys.FARM_ID]
-                phone_number = row[Keys.PHONE_NUMBER]
-                geometry = dms_string_to_point(row[Keys.GEOMETRY])
-                crop, _ = Crop.objects.get_or_create(name=row[Keys.CROP])
-                category, _ = FarmCategory.objects.get_or_create(
-                    name=row[Keys.CATEGORY]
-                )
-                rsvp_status, _ = FarmRSVPStatus.objects.get_or_create(
-                    name=row[Keys.RSVP]
-                )
-                village, _ = Village.objects.get_or_create(
-                    name=row[Keys.VILLAGE_NAME]
-                )
+
+                phone_number = None
+                if self.is_not_empty(row[Keys.PHONE_NUMBER]):
+                    phone_number = row[Keys.PHONE_NUMBER]
+
+                try:
+                    geometry = dms_string_to_point(row[Keys.GEOMETRY])
+                except ValueError:
+                    try:
+                        coords = [
+                            float(coord) for coord in
+                            row[Keys.GEOMETRY].split(',')
+                        ]
+                        geometry = Point(coords[1], coords[0])
+                    except ValueError:
+                        raise Exception('Invalid latitude, longitude format')
+
+                crop = None
+                if self.is_not_empty(row[Keys.CROP]):
+                    crop, _ = Crop.objects.get_or_create(name=row[Keys.CROP])
+                category = None
+                if self.is_not_empty(row[Keys.CATEGORY]):
+                    category, _ = FarmCategory.objects.get_or_create(
+                        name=row[Keys.CATEGORY]
+                    )
+                rsvp_status = None
+                if self.is_not_empty(row[Keys.RSVP]):
+                    rsvp_status, _ = FarmRSVPStatus.objects.get_or_create(
+                        name=row[Keys.RSVP]
+                    )
+                village = None
+                if self.is_not_empty(row[Keys.VILLAGE_NAME]):
+                    village, _ = Village.objects.get_or_create(
+                        name=row[Keys.VILLAGE_NAME]
+                    )
                 farm, _ = Farm.objects.update_or_create(
                     unique_id=farm_id,
                     geometry=geometry,

django_project/gap/migrations/0024_alter_farm_category_alter_farm_rsvp_status.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.2.7 on 2024-09-27 10:13
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gap', '0023_alter_collectorsession_options_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='farm',
+            name='category',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='gap.farmcategory'),
+        ),
+        migrations.AlterField(
+            model_name='farm',
+            name='rsvp_status',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='gap.farmrsvpstatus'),
+        ),
+    ]

django_project/gap/models/farm.py

@@ -43,10 +43,10 @@ class Farm(models.Model):
         srid=4326
     )
     rsvp_status = models.ForeignKey(
-        FarmRSVPStatus, on_delete=models.CASCADE
+        FarmRSVPStatus, on_delete=models.CASCADE, null=True, blank=True
     )
     category = models.ForeignKey(
-        FarmCategory, on_delete=models.CASCADE
+        FarmCategory, on_delete=models.CASCADE, null=True, blank=True
     )
     crop = models.ForeignKey(
         'gap.Crop', on_delete=models.SET_NULL, null=True, blank=True

django_project/gap/tests/ingestor/test_farm.py

@@ -118,7 +118,7 @@ def test_error_coordinate(self):
         session.run()
         session.delete()
         self.assertEqual(
-            session.notes, "Row 3 : Invalid dms format"
+            session.notes, "Row 3 : Invalid latitude, longitude format"
         )
         self.assertEqual(session.status, IngestorSessionStatus.FAILED)