add migration, prepare telemetry data calculation

DOSPORTAL/forms.py

@@ -101,7 +101,7 @@ def __init__(self,*args, user=None, **kwargs):
     class Meta:
         model = Record
         exclude = ("time_end", "measurement", "log_original_filename", "metadata", "duration", "record_duration", "author", 'data_file',
-                   "created", "detector")
+                   "metadata_file", "created", "detector")
 
 
 class DetectorEditForm(forms.ModelForm):

DOSPORTAL/migrations/0017_record_metadata_file.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.11 on 2024-04-05 16:38
+
+import DOSPORTAL.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('DOSPORTAL', '0016_alter_record_time_start'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='record',
+            name='metadata_file',
+            field=models.FileField(blank=True, help_text='Processed metadata file', null=True, upload_to=DOSPORTAL.models.Record.user_directory_path_data, verbose_name='Metadata file'),
+        ),
+    ]

DOSPORTAL/models.py

@@ -507,9 +507,9 @@ def user_directory_path(instance, filename):
         print("USER FILENAME", filename)
         return "user_records/record_{0}".format(instance.pk)
 
-    def user_directory_path_data(instance, filename):
-        print("USER FILENAME", filename)
-        return "user_records/record_{0}.pk".format(instance.pk)
+    def user_directory_path_data(instance, extension='pk'):
+        print("USER FILENAME", extension)
+        return "user_records/record_{0}.{1}".format(instance.pk, extension)
 
     log_file = models.FileField(
         verbose_name=_("File log"),
@@ -538,6 +538,15 @@ def user_directory_path_data(instance, filename):
         blank=True
     )
 
+
+    metadata_file = models.FileField(
+        verbose_name=_("Metadata file"),
+        help_text=_("Processed metadata file"),
+        upload_to=user_directory_path_data,
+        null=True,
+        blank=True
+    )
+
     time_tracked = models.BooleanField(
         verbose_name = _("Is time tracked?"),
         default = False,

DOSPORTAL/signals.py

@@ -84,21 +84,34 @@ def save_record(sender, instance, created = None, **kwargs):
                         'eeprom': parts[3].strip()
                     }
 
-        df_spectrum = pd.read_csv(instance.log_file.path, sep = ',', header = None, names=range(max_size))
+        df_log = pd.read_csv(instance.log_file.path, sep = ',', header = None, names=range(max_size))
 
-        df_spectrum = df_spectrum [df_spectrum[0] == '$HIST'] 
+        data_types = df_log[0].unique().tolist()
+
+        df_spectrum = df_log [df_log[0] == '$HIST'] 
         df_spectrum = df_spectrum.drop(columns=[0, 1, 3, 4, 5, 6, 7, 8])
 
         new_columns = ['time'] + list(range(df_spectrum.shape[1] - 1))
         df_spectrum.columns = new_columns
 
         df_spectrum['time'] = df_spectrum['time'].astype(float)
         duration = df_spectrum['time'].max() - df_spectrum['time'].min()
+
+        metadata['log_info'] = {}
+        metadata['log_info']['internat_time_min'] = df_spectrum['time'].min()
+        metadata['log_info']['internat_time_max'] = df_spectrum['time'].max()
+        metadata['log_info']['log_duration'] = float(duration)
+        metadata['log_info']['spectral_count'] = df_spectrum.shape[0]
+        metadata['log_info']['types'] = data_types
+
+        df_spectrum['time'] = df_spectrum['time'] - df_spectrum['time'].min()
         instance.record_duration = datetime.timedelta(seconds=float(duration))
 
         new_name = instance.user_directory_path_data('pk')
         df_spectrum.to_pickle('data/media/'+new_name)
 
+        del df_spectrum
+
         instance.data_file.name = new_name
 
         try:
@@ -110,6 +123,46 @@ def save_record(sender, instance, created = None, **kwargs):
         except Exception as e:
             print(e)
 
+        df_metadata = pd.DataFrame()
+
+        try:
+            for index, row in df_log.iterrows():
+                first_column_value = row[0]
+                row_as_list = row.tolist()[2:]
+
+                match first_column_value:
+                    case '$BAT':
+                        keys = ['time', 'voltage', 'current', 'capacity_remaining', 'capacity_full', 'temperature']
+                        bat = { k:float(v) for (k,v) in zip(keys, row_as_list[0:len(keys)])}
+                        bat['current'] /= 1000.0
+                        bat['voltage'] /= 1000.0
+                        df_metadata = pd.concat([df_metadata, pd.DataFrame([bat])], ignore_index=True)
+                        del bat
+                    case '$ENV':
+                        keys = ['time', 'temperature_0', 'humidity_0', 'temperature_1', 'humidity_1', 'temperature_2', 'pressure_3']
+                        env = { k:float(v) for (k,v) in zip(keys, row_as_list[0:len(keys)])}
+                        df_metadata = pd.concat([df_metadata, pd.DataFrame([env])], ignore_index=True)
+                        del env
+                    case '$HIST':
+                        pass
+                    case _:
+                        print('Unknown row', first_column_value)
+
+            print(df_metadata)
+
+            df_metadata = df_metadata.sort_values(by=['time']).reset_index(drop=True)
+            df_metadata['time'] -= metadata['log_info']['internat_time_min']
+            new_name = instance.user_directory_path_data('metadata.pk')
+            instance.metadata_file.name = new_name
+            print("NAME", new_name)
+            print(instance.metadata_file)
+            df_metadata.to_pickle('data/media/'+new_name)
+
+
+        except Exception as e:
+            print(e)
+
+
         print(instance.data_file)