diff --git a/django_project/gap/ingestor/farm.py b/django_project/gap/ingestor/farm.py index cbf172b..9618a26 100644 --- a/django_project/gap/ingestor/farm.py +++ b/django_project/gap/ingestor/farm.py @@ -6,6 +6,7 @@ """ import pandas as pd +from django.contrib.gis.geos import Point from gap.ingestor.base import BaseIngestor from gap.ingestor.exceptions import ( @@ -50,6 +51,10 @@ def __init__(self, session: IngestorSession, working_dir: str = '/tmp'): except FarmGroup.DoesNotExist: raise Exception('Farm group does not exist') + def is_not_empty(self, value): + """Check data is empty.""" + return value and f'{value}' != 'nan' + def _run(self): """Run the ingestor.""" df = pd.read_excel( @@ -66,18 +71,41 @@ def _run(self): for idx, row in enumerate(data): try: farm_id = row[Keys.FARM_ID] - phone_number = row[Keys.PHONE_NUMBER] - geometry = dms_string_to_point(row[Keys.GEOMETRY]) - crop, _ = Crop.objects.get_or_create(name=row[Keys.CROP]) - category, _ = FarmCategory.objects.get_or_create( - name=row[Keys.CATEGORY] - ) - rsvp_status, _ = FarmRSVPStatus.objects.get_or_create( - name=row[Keys.RSVP] - ) - village, _ = Village.objects.get_or_create( - name=row[Keys.VILLAGE_NAME] - ) + + phone_number = None + if self.is_not_empty(row[Keys.PHONE_NUMBER]): + phone_number = row[Keys.PHONE_NUMBER] + + try: + geometry = dms_string_to_point(row[Keys.GEOMETRY]) + except ValueError: + try: + coords = [ + float(coord) for coord in + row[Keys.GEOMETRY].split(',') + ] + geometry = Point(coords[1], coords[0]) + except ValueError: + raise Exception('Invalid latitude, longitude format') + + crop = None + if self.is_not_empty(row[Keys.CROP]): + crop, _ = Crop.objects.get_or_create(name=row[Keys.CROP]) + category = None + if self.is_not_empty(row[Keys.CATEGORY]): + category, _ = FarmCategory.objects.get_or_create( + name=row[Keys.CATEGORY] + ) + rsvp_status = None + if self.is_not_empty(row[Keys.RSVP]): + rsvp_status, _ = FarmRSVPStatus.objects.get_or_create( + name=row[Keys.RSVP] + ) + village = None + if self.is_not_empty(row[Keys.VILLAGE_NAME]): + village, _ = Village.objects.get_or_create( + name=row[Keys.VILLAGE_NAME] + ) farm, _ = Farm.objects.update_or_create( unique_id=farm_id, geometry=geometry, diff --git a/django_project/gap/migrations/0024_alter_farm_category_alter_farm_rsvp_status.py b/django_project/gap/migrations/0024_alter_farm_category_alter_farm_rsvp_status.py new file mode 100644 index 0000000..c78a735 --- /dev/null +++ b/django_project/gap/migrations/0024_alter_farm_category_alter_farm_rsvp_status.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.7 on 2024-09-27 10:13 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('gap', '0023_alter_collectorsession_options_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='farm', + name='category', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='gap.farmcategory'), + ), + migrations.AlterField( + model_name='farm', + name='rsvp_status', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='gap.farmrsvpstatus'), + ), + ] diff --git a/django_project/gap/models/farm.py b/django_project/gap/models/farm.py index 5597181..b412c18 100644 --- a/django_project/gap/models/farm.py +++ b/django_project/gap/models/farm.py @@ -43,10 +43,10 @@ class Farm(models.Model): srid=4326 ) rsvp_status = models.ForeignKey( - FarmRSVPStatus, on_delete=models.CASCADE + FarmRSVPStatus, on_delete=models.CASCADE, null=True, blank=True ) category = models.ForeignKey( - FarmCategory, on_delete=models.CASCADE + FarmCategory, on_delete=models.CASCADE, null=True, blank=True ) crop = models.ForeignKey( 'gap.Crop', on_delete=models.SET_NULL, null=True, blank=True diff --git a/django_project/gap/tests/ingestor/data/farms/Correct.xlsx b/django_project/gap/tests/ingestor/data/farms/Correct.xlsx index 5d037bf..96ad854 100644 Binary files a/django_project/gap/tests/ingestor/data/farms/Correct.xlsx and b/django_project/gap/tests/ingestor/data/farms/Correct.xlsx differ diff --git a/django_project/gap/tests/ingestor/test_farm.py b/django_project/gap/tests/ingestor/test_farm.py index 1b86857..c77790b 100644 --- a/django_project/gap/tests/ingestor/test_farm.py +++ b/django_project/gap/tests/ingestor/test_farm.py @@ -118,7 +118,7 @@ def test_error_coordinate(self): session.run() session.delete() self.assertEqual( - session.notes, "Row 3 : Invalid dms format" + session.notes, "Row 3 : Invalid latitude, longitude format" ) self.assertEqual(session.status, IngestorSessionStatus.FAILED)