Add solar_radiation to tio ingestor (#189)

* add solar_radiation to tio ingestor * add additional ingestor config in preference * fix test
kartoza · Oct 11, 2024 · d723eeb · d723eeb
1 parent 776fc15
commit d723eeb
Show file tree

Hide file tree

Showing 9 changed files with 132 additions and 33 deletions.
diff --git a/django_project/gap/admin/preferences.py b/django_project/gap/admin/preferences.py
@@ -52,8 +52,9 @@ class PreferencesAdmin(admin.ModelAdmin):
             }
         ),
         (
-            'Dask Config', {
+            'Ingestor Config', {
                 'fields': (
+                    'ingestor_config',
                     'dask_threads_num',
                 )
             }

diff --git a/django_project/gap/fixtures/8.dataset_attribute.json b/django_project/gap/fixtures/8.dataset_attribute.json
@@ -1043,5 +1043,27 @@
       "source_unit": 8,
       "ensembles": false
     }
+  },
+  {
+    "model": "gap.datasetattribute",
+    "pk": 96,
+    "fields": {
+      "dataset": 6,
+      "attribute": 6,
+      "source": "solarGHI",
+      "source_unit": 7,
+      "ensembles": false
+    }
+  },
+  {
+    "model": "gap.datasetattribute",
+    "pk": 97,
+    "fields": {
+      "dataset": 4,
+      "attribute": 6,
+      "source": "solar_radiation",
+      "source_unit": 7,
+      "ensembles": false
+    }
   }
 ]
diff --git a/django_project/gap/ingestor/base.py b/django_project/gap/ingestor/base.py
@@ -72,7 +72,17 @@ def get_config(self, name: str, default_value = None):
 
 
 class BaseZarrIngestor(BaseIngestor):
-    """Base Ingestor class for Zarr product."""
+    """Base Ingestor class for Zarr product.
+
+    Available config for ingestor:
+    datasourcefile_id: Id of existing DataSourceFile
+    datasourcefile_zarr_exists: Indicates that Zarr exists on s3,
+        or create new one
+    datasourcefile_name: set the zarr name if creating new zarr
+    remove_temp_file: remove temporary from the collector
+    use_latest_datasource: Default to True,
+        always use the latest DataSourceFile
+    """
 
     default_zarr_name = f'{uuid.uuid4()}.zarr'
 

diff --git a/django_project/gap/ingestor/tio_shortterm.py b/django_project/gap/ingestor/tio_shortterm.py
@@ -182,7 +182,8 @@ class TioShortTermIngestor(BaseZarrIngestor):
         'precipitation_probability',
         'humidity_maximum',
         'humidity_minimum',
-        'wind_speed_avg'
+        'wind_speed_avg',
+        'solar_radiation'
     ]
 
     def __init__(self, session: IngestorSession, working_dir: str = '/tmp'):

diff --git a/django_project/gap/migrations/0030_preferences_ingestor_config.py b/django_project/gap/migrations/0030_preferences_ingestor_config.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.7 on 2024-10-11 11:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gap', '0029_datasourcefilecache_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='preferences',
+            name='ingestor_config',
+            field=models.JSONField(blank=True, default=dict, help_text='Dict of ProviderName and AdditionalConfig; AdditionalConfig will be passed to the Ingestor Session.', null=True),
+        ),
+    ]
diff --git a/django_project/gap/models/preferences.py b/django_project/gap/models/preferences.py
@@ -109,6 +109,17 @@ class Preferences(SingletonModel):
         )
     )
 
+    # ingestor config
+    ingestor_config = models.JSONField(
+        default=dict,
+        blank=True,
+        null=True,
+        help_text=(
+            'Dict of ProviderName and AdditionalConfig; '
+            'AdditionalConfig will be passed to the Ingestor Session.'
+        )
+    )
+
     class Meta:  # noqa: D106
         verbose_name_plural = "preferences"
 

diff --git a/django_project/gap/tasks/collector.py b/django_project/gap/tasks/collector.py
@@ -8,12 +8,12 @@
 from celery.utils.log import get_task_logger
 
 from core.celery import app
-from gap.models.dataset import (
+from gap.models import (
+    Preferences,
+    Provider,
     Dataset,
     DatasetStore,
-    DataSourceFile
-)
-from gap.models.ingestor import (
+    DataSourceFile,
     IngestorType,
     IngestorSession,
     CollectorSession
@@ -48,6 +48,18 @@ def run_cbam_collector_session():
         )
 
 
+def _get_ingestor_config_from_preferences(provider: Provider) -> dict:
+    """Retrieve additional config for a provider.
+
+    :param provider: provider
+    :type provider: Provider
+    :return: additional config for Ingestor
+    :rtype: dict
+    """
+    config = Preferences.load().ingestor_config
+    return config.get(provider.name, {})
+
+
 def _do_run_zarr_collector(
         dataset: Dataset, collector_session: CollectorSession,
         ingestor_type):
@@ -67,18 +79,25 @@ def _do_run_zarr_collector(
     collector_session.refresh_from_db()
     total_file = collector_session.dataset_files.count()
     if total_file > 0:
-        # find latest DataSourceFile
-        data_source = DataSourceFile.objects.filter(
-            dataset=dataset,
-            format=DatasetStore.ZARR,
-            is_latest=True
-        ).last()
         additional_conf = {}
-        if data_source:
-            additional_conf = {
-                'datasourcefile_id': data_source.id,
-                'datasourcefile_zarr_exists': True
-            }
+        config = _get_ingestor_config_from_preferences(dataset.provider)
+
+        use_latest_datasource = config.get('use_latest_datasource', True)
+        if use_latest_datasource:
+            # find latest DataSourceFile
+            data_source = DataSourceFile.objects.filter(
+                dataset=dataset,
+                format=DatasetStore.ZARR,
+                is_latest=True
+            ).last()
+            if data_source:
+                additional_conf = {
+                    'datasourcefile_id': data_source.id,
+                    'datasourcefile_zarr_exists': True
+                }
+        additional_conf.update(config)
+
+        # create session and trigger the task
         session = IngestorSession.objects.create(
             ingestor_type=ingestor_type,
             trigger_task=False,

diff --git a/django_project/gap/tests/ingestor/test_tio_shortterm_collector.py b/django_project/gap/tests/ingestor/test_tio_shortterm_collector.py
@@ -136,6 +136,7 @@ def test_collector_one_grid(self, mock_timezone):
             self.assertEqual(len(zip_file.filelist), 1)
             _file = zip_file.open(f'grid-{grid.id}.json')
             _data = json.loads(_file.read().decode('utf-8'))
+            print(_data)
             self.assertEqual(
                 _data,
                 {
@@ -157,7 +158,8 @@ def test_collector_one_grid(self, mock_timezone):
                                 'precipitation_probability': 0,
                                 'humidity_maximum': 84,
                                 'humidity_minimum': 69,
-                                'wind_speed_avg': 4.77
+                                'wind_speed_avg': 4.77,
+                                'solar_radiation': None
                             }
                         },
                         {
@@ -170,7 +172,8 @@ def test_collector_one_grid(self, mock_timezone):
                                 'precipitation_probability': 5,
                                 'humidity_maximum': 80,
                                 'humidity_minimum': 71,
-                                'wind_speed_avg': 4.35
+                                'wind_speed_avg': 4.35,
+                                'solar_radiation': None
                             }
                         },
                         {
@@ -183,7 +186,8 @@ def test_collector_one_grid(self, mock_timezone):
                                 'precipitation_probability': 5,
                                 'humidity_maximum': 79,
                                 'humidity_minimum': 73,
-                                'wind_speed_avg': 5.58
+                                'wind_speed_avg': 5.58,
+                                'solar_radiation': None
                             }
                         },
                         {
@@ -196,7 +200,8 @@ def test_collector_one_grid(self, mock_timezone):
                                 'precipitation_probability': 5,
                                 'humidity_maximum': 78,
                                 'humidity_minimum': 72,
-                                'wind_speed_avg': 5.74
+                                'wind_speed_avg': 5.74,
+                                'solar_radiation': None
                             }
                         },
                         {
@@ -209,7 +214,8 @@ def test_collector_one_grid(self, mock_timezone):
                                 'precipitation_probability': 5,
                                 'humidity_maximum': 76,
                                 'humidity_minimum': 70,
-                                'wind_speed_avg': 5.09
+                                'wind_speed_avg': 5.09,
+                                'solar_radiation': None
                             }
                         },
                         {
@@ -222,7 +228,8 @@ def test_collector_one_grid(self, mock_timezone):
                                 'precipitation_probability': 5,
                                 'humidity_maximum': 76,
                                 'humidity_minimum': 72,
-                                'wind_speed_avg': 4.01
+                                'wind_speed_avg': 4.01,
+                                'solar_radiation': None
                             }
                         },
                         {
@@ -235,7 +242,8 @@ def test_collector_one_grid(self, mock_timezone):
                                 'precipitation_probability': 0,
                                 'humidity_maximum': 76,
                                 'humidity_minimum': 70,
-                                'wind_speed_avg': 3.82
+                                'wind_speed_avg': 3.82,
+                                'solar_radiation': None
                             }
                         },
                         {
@@ -248,7 +256,8 @@ def test_collector_one_grid(self, mock_timezone):
                                 'precipitation_probability': 5,
                                 'humidity_maximum': 78,
                                 'humidity_minimum': 72,
-                                'wind_speed_avg': 4.12
+                                'wind_speed_avg': 4.12,
+                                'solar_radiation': None
                             }
                         },
                         {
@@ -261,7 +270,8 @@ def test_collector_one_grid(self, mock_timezone):
                                 'precipitation_probability': 5,
                                 'humidity_maximum': 80,
                                 'humidity_minimum': 74,
-                                'wind_speed_avg': 5.29
+                                'wind_speed_avg': 5.29,
+                                'solar_radiation': None
                             }
                         },
                         {
@@ -274,7 +284,8 @@ def test_collector_one_grid(self, mock_timezone):
                                 'precipitation_probability': 5,
                                 'humidity_maximum': 80,
                                 'humidity_minimum': 73,
-                                'wind_speed_avg': 4.96
+                                'wind_speed_avg': 4.96,
+                                'solar_radiation': None
                             }
                         },
                         {
@@ -287,7 +298,8 @@ def test_collector_one_grid(self, mock_timezone):
                                 'precipitation_probability': 5,
                                 'humidity_maximum': 77,
                                 'humidity_minimum': 68,
-                                'wind_speed_avg': 4.1
+                                'wind_speed_avg': 4.1,
+                                'solar_radiation': None
                             }
                         },
                         {
@@ -300,7 +312,8 @@ def test_collector_one_grid(self, mock_timezone):
                                 'precipitation_probability': 5,
                                 'humidity_maximum': 78,
                                 'humidity_minimum': 70,
-                                'wind_speed_avg': 4.42
+                                'wind_speed_avg': 4.42,
+                                'solar_radiation': None
                             }
                         },
                         {
@@ -313,7 +326,8 @@ def test_collector_one_grid(self, mock_timezone):
                                 'precipitation_probability': 5,
                                 'humidity_maximum': 78,
                                 'humidity_minimum': 72,
-                                'wind_speed_avg': 4.52
+                                'wind_speed_avg': 4.52,
+                                'solar_radiation': None
                             }
                         },
                         {
@@ -326,7 +340,8 @@ def test_collector_one_grid(self, mock_timezone):
                                 'precipitation_probability': 5,
                                 'humidity_maximum': 78,
                                 'humidity_minimum': 72,
-                                'wind_speed_avg': 4.74
+                                'wind_speed_avg': 4.74,
+                                'solar_radiation': None
                             }
                         },
                         {
@@ -339,7 +354,8 @@ def test_collector_one_grid(self, mock_timezone):
                                 'precipitation_probability': 5,
                                 'humidity_maximum': 77.83,
                                 'humidity_minimum': 72.77,
-                                'wind_speed_avg': 3.17
+                                'wind_speed_avg': 3.17,
+                                'solar_radiation': None
                             }
                         }
                     ]

diff --git a/docs/src/developer/api/guide/measurment.md b/docs/src/developer/api/guide/measurment.md
@@ -86,6 +86,7 @@ TomorrowNow provides access to the data through a RESTful API, available at http
 | Short-term Forecast | Humidity Maximum | | % | humidity_maximum |
 | Short-term Forecast | Humidity Minimum | | % | humidity_minimum |
 | Short-term Forecast | Wind Speed Average | | m/s | wind_speed_avg |
+| Short-term Forecast | Solar radiation | | Wh/m2 | solar_radiation |
 | **Historical Reanalysis** |
 | Historical Reanalysis | Min Total Temperature | Minimum temperature (0000:2300) | °C | min_temperature |
 | Historical Reanalysis | Min Day Temperature | Minimum day-time temperature (0600:1800) | °C | min_day_temperature |