[Version 1.40.7] Update importer and fix RT query (#271)

* Update importer

* Fix related table query for uppercase field

* Bump version 1.40.7

django_project/frontend/src/app/mui.scss

@@ -653,7 +653,7 @@
     position: absolute;
     height: 100%;
     right: 0 !important;
-    top: 0;
+    top: 50%;
     width: 25px;
     display: flex;
     align-items: center;

django_project/frontend/src/pages/Admin/Importer/LogData/index.jsx

@@ -80,7 +80,7 @@ export default function ImporterLogData() {
     indicators: defaultFilters.indicators ? splitParams(defaultFilters.indicators) : [],
     datasets: defaultFilters.datasets ? splitParams(defaultFilters.datasets, false) : [],
     levels: defaultFilters.levels ? splitParams(defaultFilters.levels) : [],
-    status: defaultFilters.status ? splitParams(defaultFilters.status) : [],
+    status: defaultFilters.status ? splitParams(defaultFilters.status, false) : [],
     geographies: defaultFilters.geographies ? splitParams(defaultFilters.geographies) : [],
     fromTime: defaultFilters.fromTime ? defaultFilters.fromTime : null,
     toTime: defaultFilters.toTime ? defaultFilters.toTime : null,
@@ -208,7 +208,6 @@ export default function ImporterLogData() {
         const isDate = isValueDate(key, columnDetail)
         columnsByDict[key] = {
           renderCell: (params) => {
-            console.log(params.row)
             const value = params.row.data[params.field]
             const note = params.row.note
             if (note && note[key]) {

django_project/frontend/src/pages/Admin/Importer/LogDetail/index.jsx

@@ -114,6 +114,15 @@ export function ImporterDetailSection({ inputData }) {
               <div>{data.attributes.indicator_data_type === 'By Value' ? 'Selected indicator' : 'Data-Driven Indicator Column'}</div>
             </div>
           </Grid>
+          {
+            data.attributes.indicator_data_type !== 'By Value' ?
+              <Grid item xs={3}>
+                <div className='DetailSection'>
+                  <div>Indicator Column</div>
+                  <div>{data.attributes.indicator_data_field}</div>
+                </div>
+              </Grid> : null
+          }
           <Grid item xs={3}>
             <div className='DetailSection'>
               <div>Indicator Names</div>
@@ -449,7 +458,7 @@ export default function ImporterLogDetail() {
                   </div>
                   <div className='DetailButtonButton'>
                     <div className='Separator'/>
-                    <a href={urls.api.dataView}>
+                    <a href={urls.api.dataView+'?status=Warning and Error'}>
                       <ThemeButton
                         variant="primary">
                         See the data

django_project/frontend/src/utils/queryExtraction.jsx

@@ -124,7 +124,9 @@ export const INIT_DATA = {
 }
 
 export function spacedField(field) {
-  if (!field.includes('`') && field.includes(' ')) {
+  if (!field.includes('`') && (
+    field.includes(' ') || field.includes('-') || field[0] === field[0].toUpperCase())
+  ) {
     field = '`' + field + '`'
   }
   return field
@@ -188,7 +190,8 @@ export function returnWhere(where, ignoreActive, ids, sameGroupOperator = true,
         }
       }
     case TYPE.EXPRESSION:
-      let field = where.field.includes(' ') ? `"${where.field}"`.replaceAll('""', '"') : where.field
+      const needQuote = where.field.includes(' ') || (where.field[0] === where.field[0].toUpperCase())
+      let field = needQuote ? `"${where.field}"`.replaceAll('""', '"') : where.field
       const fieldSplit = field.split('.')
       // We put all geometry_x as geometry_layer
       if (changeGeometryId && fieldSplit[0].includes('geometry_')) {

django_project/geosight/importer/importers/base/indicator_value.py

@@ -417,6 +417,9 @@ def _save_log_data_to_model(self, log_data: ImporterLogData):
             # Skip if the value is empty
             if data['value'] in [None, '']:
                 return
+            # Skip if the geo_code is empty
+            if data['geo_code'] in [None, '']:
+                return
 
             indicator.save_value(
                 datetime.fromtimestamp(data['date_time']),

django_project/geosight/importer/importers/base/indicator_value_long_format.py

@@ -152,17 +152,12 @@ def process_data_from_records(self):
 
             if not indicator:
                 note['indicator_name'] = 'Indicator does not found'
-            if not geo_code:
-                note['value'] = 'Administrative code is empty'
             if not date_time:
                 note['date_time'] = 'date_time is empty'
             if date_time_error:
                 note['date_time'] = date_time_error
 
             # ----------------------------------------
-            # Check the geocode
-            data[code_type] = data['geo_code']
-
             # Check admin level
             try:
                 admin_level = self.get_admin_level_from_data(record)
@@ -172,18 +167,24 @@ def process_data_from_records(self):
             except ValueError:
                 note['admin_level'] = 'Admin level is not integer'
 
-            entity, error = self.get_entity(
-                data, self.importer.admin_code_type, auto_fetch=False
-            )
-            if entity:
-                data['geo_code'] = entity.geom_id
-                data['admin_level'] = entity.admin_level
+            # ----------------------------------------
+            # Check the geocode
+            if not geo_code:
+                note['warning'] = 'Administrative code is empty'
             else:
-                if error == 'This code does not exist.':
-                    checked_geocode.append(data['geo_code'])
-                    invalid_check_idx.append(idx)
+                data[code_type] = geo_code
+                entity, error = self.get_entity(
+                    data, self.importer.admin_code_type, auto_fetch=False
+                )
+                if entity:
+                    data['geo_code'] = entity.geom_id
+                    data['admin_level'] = entity.admin_level
                 else:
-                    note[code_type] = error
+                    if error == 'This code does not exist.':
+                        checked_geocode.append(data['geo_code'])
+                        invalid_check_idx.append(idx)
+                    else:
+                        note[code_type] = error
             # ----------------------------------------
 
             # Save to clean records
@@ -192,7 +193,12 @@ def process_data_from_records(self):
             idx += 1
 
             # If there is note, failed
-            if len(note.keys()):
+            note_keys = list(note.keys())
+            try:
+                note_keys.remove('warning')
+            except ValueError:
+                pass
+            if len(note_keys):
                 success = False
 
         # Checking missing geocode
@@ -214,7 +220,14 @@ def process_data_from_records(self):
                         clean_records[idx]['geo_code'] = entity.geom_id
                         clean_records[idx]['admin_level'] = entity.admin_level
                     except (IndexError, KeyError):
-                        notes[idx][code_type] = 'This code does not exist.'
-                        success = False
+                        try:
+                            if record['geo_code']:
+                                notes[idx][
+                                    'geo_code'
+                                ] = 'This code does not exist.'
+                                success = False
+                        except KeyError:
+                            notes[idx][code_type] = 'This code does not exist.'
+                            success = False
 
         return clean_records, notes, success

django_project/geosight/importer/importers/base/indicator_value_wide_format.py

@@ -93,17 +93,12 @@ def process_data_from_records(self):
                             self.get_indicator(data)
                         except Indicator.DoesNotExist:
                             note['indicator_id'] = 'Indicator does not exist'
-                    if not geo_code:
-                        note['geo_code'] = 'administrative code is empty'
                     if not date_time:
                         note['date_time'] = 'date_time is empty'
                     if date_time_error:
                         note['date_time'] = date_time_error
 
                     # ----------------------------------------
-                    # Check the geocode
-                    data[code_type] = data['geo_code']
-
                     # Check admin level
                     try:
                         admin_level = self.get_admin_level_from_data(record)
@@ -113,18 +108,25 @@ def process_data_from_records(self):
                     except ValueError:
                         note['admin_level'] = 'Admin level is not integer'
 
-                    entity, error = self.get_entity(
-                        data, self.importer.admin_code_type, auto_fetch=False
-                    )
-                    if entity:
-                        data['geo_code'] = entity.geom_id
-                        data['admin_level'] = entity.admin_level
+                    # ----------------------------------------
+                    # Check the geocode
+                    if not geo_code:
+                        note['warning'] = 'Administrative code is empty'
                     else:
-                        if error == 'This code does not exist.':
-                            checked_geocode.append(data['geo_code'])
-                            invalid_check_idx.append(idx)
+                        data[code_type] = data['geo_code']
+                        entity, error = self.get_entity(
+                            data, self.importer.admin_code_type,
+                            auto_fetch=False
+                        )
+                        if entity:
+                            data['geo_code'] = entity.geom_id
+                            data['admin_level'] = entity.admin_level
                         else:
-                            note[code_type] = error
+                            if error == 'This code does not exist.':
+                                checked_geocode.append(data['geo_code'])
+                                invalid_check_idx.append(idx)
+                            else:
+                                note[code_type] = error
                     # ----------------------------------------
 
                     # Save to clean records
@@ -147,15 +149,22 @@ def process_data_from_records(self):
                 else:
                     try:
                         geo_code = record['geo_code']
-                        codes = results[geo_code]
+                        codes = results[f'{geo_code}']
                         entity = codes[len(codes) - 1]
                         entity = self.importer.reference_layer.save_entity(
                             entity
                         )
                         clean_records[idx]['geo_code'] = entity.geom_id
                         clean_records[idx]['admin_level'] = entity.admin_level
                     except (IndexError, KeyError):
-                        notes[idx][code_type] = 'This code does not exist.'
-                        success = False
+                        try:
+                            if record['geo_code']:
+                                notes[idx][
+                                    'geo_code'
+                                ] = 'This code does not exist.'
+                                success = False
+                        except KeyError:
+                            notes[idx][code_type] = 'This code does not exist.'
+                            success = False
 
         return clean_records, notes, success

django_project/geosight/importer/models/log.py

@@ -168,8 +168,9 @@ def run(self):
             id__in=self.target_ids
         )
         importer = self.log.importer.importer(self.log)
-        for log_data in log_datas:
-            importer._save_log_data_to_model(log_data)
-            self.saved_ids.append(log_data.id)
-            self.save()
+        for idx, log_data in enumerate(log_datas):
+            if log_data.id not in self.saved_ids:
+                importer._save_log_data_to_model(log_data)
+                self.saved_ids.append(log_data.id)
+                self.save()
         self.delete()

django_project/geosight/importer/restart_functions.py

@@ -0,0 +1,27 @@
+# coding=utf-8
+"""
+GeoSight is UNICEF's geospatial web-based business intelligence platform.
+
+Contact : [email protected]
+
+.. note:: This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation; either version 3 of the License, or
+    (at your option) any later version.
+
+"""
+__author__ = '[email protected]'
+__date__ = '28/04/2024'
+__copyright__ = ('Copyright 2023, Unicef')
+
+from geosight.importer.models.log import ImporterLogDataSaveProgress
+from geosight.importer.tasks import run_save_log_data
+
+
+class RestartFunctions:
+    """Class that contains some functions for restart."""
+
+    def restart_log_sata_save_progress(self):
+        """Restart harvester by removing running status and logs."""
+        for progress in ImporterLogDataSaveProgress.objects.filter(done=False):
+            run_save_log_data.delay(progress.id)

django_project/initialize.py

@@ -14,7 +14,6 @@
 __date__ = '13/06/2023'
 __copyright__ = ('Copyright 2023, Unicef')
 
-import ast
 import os
 import shutil
 import time
@@ -128,3 +127,18 @@
         print("Version is different, remove all")
     except Exception:
         pass
+
+#########################################################
+# 6. Restart backgrounds functions
+#########################################################
+
+try:
+    from geosight.importer.restart_functions import RestartFunctions
+
+    print("-----------------------------------------------------")
+    print("6. Restart backgrounds functions")
+
+    RestartFunctions().restart_log_sata_save_progress()
+except Exception as e:
+    print(f'{e}')
+    pass

django_project/version/version.txt

@@ -1 +1 @@
-1.40.6
+1.40.7