From 523145e2a648f46482870b1e6a5f9529f9350f42 Mon Sep 17 00:00:00 2001 From: paulapreuss Date: Thu, 31 Oct 2024 14:45:14 +0100 Subject: [PATCH 01/13] Add new Timeseries as ForeignKey and rename old field --- app/projects/forms.py | 7 +++--- ...t_timeseries_asset_input_timeseries_old.py | 16 +++++++++++++ .../migrations/0021_asset_input_timeseries.py | 23 +++++++++++++++++++ app/projects/models/base_models.py | 5 ++-- 4 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 app/projects/migrations/0020_rename_input_timeseries_asset_input_timeseries_old.py create mode 100644 app/projects/migrations/0021_asset_input_timeseries.py diff --git a/app/projects/forms.py b/app/projects/forms.py index f51504953..411402029 100644 --- a/app/projects/forms.py +++ b/app/projects/forms.py @@ -810,11 +810,11 @@ def is_input_timeseries_empty(self): else: return True - def clean_input_timeseries(self): + def clean_input_timeseries_old(self): """Override built-in Form method which is called upon form validation""" try: input_timeseries_values = [] - timeseries_file = self.files.get("input_timeseries", None) + timeseries_file = self.files.get("input_timeseries_file", None) # read the timeseries from file if any if timeseries_file is not None: input_timeseries_values = parse_input_timeseries(timeseries_file) @@ -949,8 +949,7 @@ class Meta: "lifetime": forms.NumberInput( attrs={"placeholder": "e.g. 10 years", "min": "0", "step": "1"} ), - # TODO: Try changing this to FileInput - "input_timeseries": forms.FileInput( + "input_timeseries_old": forms.FileInput( attrs={ "onchange": "plot_file_trace(obj=this.files, plot_id='timeseries_trace')" } diff --git a/app/projects/migrations/0020_rename_input_timeseries_asset_input_timeseries_old.py b/app/projects/migrations/0020_rename_input_timeseries_asset_input_timeseries_old.py new file mode 100644 index 000000000..bb1437c01 --- /dev/null +++ b/app/projects/migrations/0020_rename_input_timeseries_asset_input_timeseries_old.py @@ -0,0 +1,16 @@ +# Generated by Django 4.2.4 on 2024-10-10 14:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [("projects", "0019_timeseries")] + + operations = [ + migrations.RenameField( + model_name="asset", + old_name="input_timeseries", + new_name="input_timeseries_old", + ) + ] diff --git a/app/projects/migrations/0021_asset_input_timeseries.py b/app/projects/migrations/0021_asset_input_timeseries.py new file mode 100644 index 000000000..3d6cce2b4 --- /dev/null +++ b/app/projects/migrations/0021_asset_input_timeseries.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.4 on 2024-10-10 14:38 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("projects", "0020_rename_input_timeseries_asset_input_timeseries_old") + ] + + operations = [ + migrations.AddField( + model_name="asset", + name="input_timeseries", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="projects.timeseries", + ), + ) + ] diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index 94213cf6f..d7fee0751 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -435,9 +435,10 @@ def save(self, *args, **kwargs): lifetime = models.IntegerField( null=True, blank=False, validators=[MinValueValidator(0)] ) - input_timeseries = models.TextField( + input_timeseries_old = models.TextField( null=True, blank=False ) # , validators=[validate_timeseries]) + input_timeseries = models.ForeignKey(Timeseries, on_delete=models.CASCADE, null=True, blank=False) crate = models.FloatField( null=True, blank=False, default=1, validators=[MinValueValidator(0.0)] ) @@ -513,7 +514,7 @@ def get_field_value(self, field_name): "efficiency_multiple", "energy_price", "feedin_tariff", - "input_timeseries", + "input_timeseries_old", ): try: answer = float(answer) From 1734b5327c41f4a564184f2ad82a6416b35abdb2 Mon Sep 17 00:00:00 2001 From: paulapreuss Date: Thu, 31 Oct 2024 14:48:25 +0100 Subject: [PATCH 02/13] Rename DateTime fields and compute from scenario --- ...ame_end_time_timeseries_end_date_and_more.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 app/projects/migrations/0022_rename_end_time_timeseries_end_date_and_more.py diff --git a/app/projects/migrations/0022_rename_end_time_timeseries_end_date_and_more.py b/app/projects/migrations/0022_rename_end_time_timeseries_end_date_and_more.py new file mode 100644 index 000000000..ac47990d3 --- /dev/null +++ b/app/projects/migrations/0022_rename_end_time_timeseries_end_date_and_more.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.4 on 2024-10-14 15:19 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [("projects", "0021_asset_input_timeseries")] + + operations = [ + migrations.RenameField( + model_name="timeseries", old_name="end_time", new_name="end_date" + ), + migrations.RenameField( + model_name="timeseries", old_name="start_time", new_name="start_date" + ), + ] From 4b93416c1c788b24f4157d9411f66e321b845fdb Mon Sep 17 00:00:00 2001 From: paulapreuss Date: Thu, 31 Oct 2024 15:47:33 +0100 Subject: [PATCH 03/13] Rename DateTime attributes and set them from scenario Also replace ts_type with MVS type instead of scalar/vector --- app/projects/models/base_models.py | 35 +++++++++++++++++++----------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index d7fee0751..db70c5f92 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -314,9 +314,9 @@ class Timeseries(models.Model): settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True ) # TODO check that if both a user and scenario are provided the scenario belongs to the user - # scenario = models.ForeignKey( - # Scenario, on_delete=models.CASCADE, null=True, blank=True - # ) + scenario = models.ForeignKey( + Scenario, on_delete=models.CASCADE, null=True, blank=True + ) ts_type = models.CharField(max_length=12, choices=MVS_TYPE, blank=True, null=True) open_source = models.BooleanField( @@ -324,19 +324,18 @@ class Timeseries(models.Model): ) # get this from the scenario - # TODO rename with _date instead of _time - start_time = models.DateTimeField(blank=True, default=None, null=True) - end_time = models.DateTimeField(blank=True, default=None, null=True) + start_date = models.DateTimeField(blank=True, default=None, null=True) + end_date = models.DateTimeField(blank=True, default=None, null=True) time_step = models.IntegerField( blank=True, default=None, null=True, validators=[MinValueValidator(1)] ) + def __str__(self): + return f"{self.name} ({self.pk})" + def save(self, *args, **kwargs): - n = len(self.values) - if n == 1: - self.ts_type = "scalar" - elif n > 1: - self.ts_type = "vector" + # set time attributes + self.set_date_attributes_from_scenario() super().save(*args, **kwargs) @property @@ -350,8 +349,18 @@ def get_values(self): def compute_time_attribute_from_timestamps(self, timestamps): pass - def compute_end_time_from_duration(self, duration): - pass + def compute_end_date_from_duration(self): + duration = self.scenario.evaluated_period + total_duration = timedelta(hours=self.time_step) * duration + end_time = self.start_date + total_duration + return end_time + + def set_date_attributes_from_scenario(self): + if self.scenario is not None: + self.start_date = self.scenario.start_date + self.time_step = self.scenario.time_step + self.end_date = self.compute_end_date_from_duration() + class AssetType(models.Model): From 60811dd222e043fcadf6a1e80b206911d2eae539 Mon Sep 17 00:00:00 2001 From: paulapreuss Date: Thu, 31 Oct 2024 15:49:37 +0100 Subject: [PATCH 04/13] Set TimeseriesField and create instances from input --- app/dashboard/views.py | 2 +- app/projects/dtos.py | 2 +- app/projects/forms.py | 45 ++++++++++++++++++++--- app/projects/helpers.py | 19 ++-------- app/projects/scenario_topology_helpers.py | 8 +++- app/projects/views.py | 2 +- 6 files changed, 54 insertions(+), 24 deletions(-) diff --git a/app/dashboard/views.py b/app/dashboard/views.py index 6e1a2d3ce..29c23add1 100644 --- a/app/dashboard/views.py +++ b/app/dashboard/views.py @@ -755,7 +755,7 @@ def view_asset_parameters(request, scen_id, asset_type_name, asset_uuid): proj_id=scenario.project.id, ) input_timeseries_data = ( - existing_asset.input_timeseries if existing_asset.input_timeseries else "" + existing_asset.input_timeseries.values if existing_asset.input_timeseries else "" ) context.update( diff --git a/app/projects/dtos.py b/app/projects/dtos.py index cb39b0685..8ab30fa86 100644 --- a/app/projects/dtos.py +++ b/app/projects/dtos.py @@ -672,7 +672,7 @@ def to_timeseries_data(model_obj, field_name): value_type = ValueType.objects.filter(type=field_name).first() unit = value_type.unit if value_type is not None else None value_list = ( - json.loads(getattr(model_obj, field_name)) + getattr(model_obj, field_name).values if getattr(model_obj, field_name) is not None else None ) diff --git a/app/projects/forms.py b/app/projects/forms.py index 411402029..71955d351 100644 --- a/app/projects/forms.py +++ b/app/projects/forms.py @@ -4,6 +4,7 @@ import json import io import csv +from django.db.models import Q from openpyxl import load_workbook import numpy as np @@ -33,6 +34,7 @@ PARAMETERS, DualNumberField, parse_input_timeseries, + TimeseriesField, ) @@ -664,16 +666,19 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # which fields exists in the form are decided upon AssetType saved in the db - asset_type = AssetType.objects.get(asset_type=self.asset_type_name) + self.asset_type = AssetType.objects.get(asset_type=self.asset_type_name) + + # remove the fields not needed for the AssetType [ self.fields.pop(field) for field in list(self.fields) - if field not in asset_type.visible_fields + if field not in self.asset_type.visible_fields ] self.timestamps = None if self.existing_asset is not None: self.timestamps = self.existing_asset.timestamps + self.user = self.existing_asset.scenario.project.user elif scenario_id is not None: qs = Scenario.objects.filter(id=scenario_id) if qs.exists(): @@ -688,6 +693,14 @@ def __init__(self, *args, **kwargs): currency = qs.values_list("economic_data__currency", flat=True).get() currency = CURRENCY_SYMBOLS[currency] # TODO use mapping to display currency symbol + self.user = qs.get().user + + # set the custom timeseries field for timeseries + # the qs_ts selects timeseries of the corresponding MVS type that either belong to the user or are open source + if "input_timeseries" in self.fields: + self.fields["input_timeseries"] = TimeseriesField(qs_ts=Timeseries.objects.filter( + Q(ts_type=self.asset_type.mvs_type) & (Q(open_source=True) | Q(user=self.user)), + )) self.fields["inputs"] = forms.CharField( widget=forms.HiddenInput(), required=False @@ -799,7 +812,7 @@ def __init__(self, *args, **kwargs): ) if ":unit:" in self.fields[field].label: self.fields[field].label = self.fields[field].label.replace( - ":unit:", asset_type.unit + ":unit:", self.asset_type.unit ) """ ----------------------------------------------------- """ @@ -899,8 +912,32 @@ def clean(self): self.timeseries_same_as_timestamps(feedin_tariff, "feedin_tariff") self.timeseries_same_as_timestamps(energy_price, "energy_price") + if "input_timeseries" in cleaned_data: + ts_data = json.loads(cleaned_data["input_timeseries"]) + input_method = ts_data["input_method"]["type"] + if input_method == "upload" or input_method == "manual": + # replace the dict with a new timeseries instance + cleaned_data["input_timeseries"] = self.create_timeseries_from_input( + ts_data) + if input_method == "select": + # return the timeseries instance + timeseries_id = ts_data["input_method"]["extra_info"] + cleaned_data["input_timeseries"] = Timeseries.objects.get(id=timeseries_id) + return cleaned_data + def create_timeseries_from_input(self, input_timeseries): + timeseries_name = input_timeseries["input_method"]["extra_info"] + timeseries_values = input_timeseries["values"] + ts_instance = Timeseries.objects.create(user=self.user, + name=timeseries_name, + ts_type=self.asset_type.mvs_type, + values=timeseries_values, + open_source=False + ) + + return ts_instance + def timeseries_same_as_timestamps(self, ts, param): if isinstance(ts, np.ndarray): ts = np.squeeze(ts).tolist() @@ -954,8 +991,6 @@ class Meta: "onchange": "plot_file_trace(obj=this.files, plot_id='timeseries_trace')" } ), - # 'input_timeseries': forms.Textarea(attrs={'placeholder': 'e.g. [4,3,2,5,3,...]', - # 'style': 'font-weight:400; font-size:13px;'}), "crate": forms.NumberInput( attrs={ "placeholder": "factor of total capacity (kWh), e.g. 0.7", diff --git a/app/projects/helpers.py b/app/projects/helpers.py index cd08667f6..c492a6537 100644 --- a/app/projects/helpers.py +++ b/app/projects/helpers.py @@ -355,19 +355,8 @@ def use_required_attribute(self, initial): return False def decompress(self, value): - - answer = [self.default, "", None] - if value is not None: - value = value.replace("'", '"') - value = json.loads(value) - input_method = value["input_method"]["type"] - ts_values = value["values"] - if len(ts_values) == 1: - ts_values = ts_values[0] - if input_method == "select": - answer = [ts_values, value["input_method"]["extra_info"], None] - else: - answer = [ts_values, "", None] + # TODO update handling here - value corresponds to the pk of the timeseries + answer = ["", "", value] return answer @@ -427,13 +416,13 @@ def clean(self, values): self.set_widget_error() raise ValidationError( _( - "Please provide either a number within %(boundaries) s or upload a timeseries from a file" + "Please provide either a number within %(boundaries) s, select a timeseries or upload a timeseries from a file" ), code="required", params={"boundaries": self.boundaries}, ) else: - input_dict = dict(type="manuel") + input_dict = dict(type="manual") self.check_boundaries(answer) return json.dumps(dict(values=answer, input_method=input_dict)) diff --git a/app/projects/scenario_topology_helpers.py b/app/projects/scenario_topology_helpers.py index 575a9bcb8..08926a495 100644 --- a/app/projects/scenario_topology_helpers.py +++ b/app/projects/scenario_topology_helpers.py @@ -113,7 +113,13 @@ def track_asset_changes(scenario, param, form, existing_asset, new_value=None): old_value = pi.old_value if pi.parameter_type == "vector": old_value = (old_value, None) - if new_value == form.fields[pi.name].clean(old_value): + + if pi.name == "input_timeseries": + new_value = str(new_value) + else: + old_value = form.fields[pi.name].clean(old_value) + + if new_value == old_value: pi.delete() else: qs_param.update(new_value=new_value) diff --git a/app/projects/views.py b/app/projects/views.py index 37dda979e..d032bf5a8 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -1597,7 +1597,7 @@ def get_asset_create_form(request, scen_id=0, asset_type_name="", asset_uuid=Non proj_id=scenario.project.id, ) input_timeseries_data = ( - existing_asset.input_timeseries + existing_asset.input_timeseries.values if existing_asset.input_timeseries else "" ) From 525a701fb637afceedd3bc9d197ad030e3adff4e Mon Sep 17 00:00:00 2001 From: paulapreuss Date: Thu, 31 Oct 2024 15:51:38 +0100 Subject: [PATCH 05/13] Create Timeseries instances from existing data --- .../0023_migrate_timeseries_to_model.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 app/projects/migrations/0023_migrate_timeseries_to_model.py diff --git a/app/projects/migrations/0023_migrate_timeseries_to_model.py b/app/projects/migrations/0023_migrate_timeseries_to_model.py new file mode 100644 index 000000000..0b267e756 --- /dev/null +++ b/app/projects/migrations/0023_migrate_timeseries_to_model.py @@ -0,0 +1,82 @@ +from django.db import migrations +from django.db.models import Q +import json +from datetime import timedelta + + +def convert_timeseries_to_model(apps, schema_editor): + """ + Forward migration: Convert timeseries_old to Timeseries instance + """ + # Get historical models + Asset = apps.get_model('projects', 'Asset') + Timeseries = apps.get_model('projects', 'Timeseries') + db_alias = schema_editor.connection.alias + + # Iterate through all assets with timeseries_old data + for asset in Asset.objects.using(db_alias).exclude(Q(input_timeseries_old__isnull=True) | Q(input_timeseries_old=[])): + try: + # Calculate end time from asset start date and duration + duration = asset.scenario.evaluated_period + total_duration = timedelta(hours=asset.scenario.time_step) * duration + end_date = asset.scenario.start_date + total_duration + + # Create new Timeseries instance + timeseries = Timeseries.objects.using(db_alias).create( + name=f"{asset.name}_migration", + user=asset.scenario.project.user, + scenario=asset.scenario, + values=json.loads(asset.input_timeseries_old), + ts_type=asset.asset_type.mvs_type, + open_source=False, + start_date=asset.scenario.start_date, + time_step=asset.scenario.time_step, + end_date=end_date + ) + + # Update asset to point to new timeseries + asset.input_timeseries = timeseries + asset.save() + + except Exception as e: + print(f"Error migrating asset {asset.id} timeseries: {str(e)}") + raise e + + +def reverse_timeseries_conversion(apps, schema_editor): + """ + Reverse migration: Delete created Timeseries instances and restore old data + """ + Asset = apps.get_model('projects', 'Asset') + Timeseries = apps.get_model('projects', 'Timeseries') + db_alias = schema_editor.connection.alias + + try: + # Find all timeseries created by this migration + migration_timeseries = Timeseries.objects.using(db_alias).filter(name__contains="_migration") + + # Update assets to remove reference to timeseries + Asset.objects.using(db_alias).filter(input_timeseries__in=migration_timeseries).update( + input_timeseries=None + ) + + # Delete the timeseries instances + migration_timeseries.delete() + + except Exception as e: + print(f"Error deleting migrated timeseries: {str(e)}") + raise e + + +class Migration(migrations.Migration): + + dependencies = [("projects", "0022_rename_end_time_timeseries_end_date_and_more")] + + operations = [ + # Run the timeseries migration + migrations.RunPython( + convert_timeseries_to_model, + reverse_timeseries_conversion + ), + ] + From 037d90ac0cc4a7aaa6beaf744a8bf44d9a2a8323 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 6 Feb 2025 17:14:25 +0100 Subject: [PATCH 06/13] Move ts template one block higher, set constant for widget types --- app/projects/forms.py | 10 ++++-- app/projects/helpers.py | 12 ++++--- app/templates/asset/timeseries_input.html | 39 ++++++++++++++++++++-- app/templates/asset/ts_input_template.html | 33 ------------------ 4 files changed, 52 insertions(+), 42 deletions(-) delete mode 100644 app/templates/asset/ts_input_template.html diff --git a/app/projects/forms.py b/app/projects/forms.py index 71955d351..adf190998 100644 --- a/app/projects/forms.py +++ b/app/projects/forms.py @@ -35,6 +35,9 @@ DualNumberField, parse_input_timeseries, TimeseriesField, + TS_SELECT_TYPE, + TS_UPLOAD_TYPE, + TS_MANUAL_TYPE, ) @@ -915,11 +918,12 @@ def clean(self): if "input_timeseries" in cleaned_data: ts_data = json.loads(cleaned_data["input_timeseries"]) input_method = ts_data["input_method"]["type"] - if input_method == "upload" or input_method == "manual": + if input_method == TS_UPLOAD_TYPE or input_method == TS_MANUAL_TYPE: # replace the dict with a new timeseries instance cleaned_data["input_timeseries"] = self.create_timeseries_from_input( - ts_data) - if input_method == "select": + ts_data + ) + if input_method == TS_SELECT_TYPE: # return the timeseries instance timeseries_id = ts_data["input_method"]["extra_info"] cleaned_data["input_timeseries"] = Timeseries.objects.get(id=timeseries_id) diff --git a/app/projects/helpers.py b/app/projects/helpers.py index c492a6537..607d50dd0 100644 --- a/app/projects/helpers.py +++ b/app/projects/helpers.py @@ -13,6 +13,10 @@ from projects.constants import MAP_MVS_EPA from dashboard.helpers import KPIFinder +TS_SELECT_TYPE = "select" +TS_UPLOAD_TYPE = "upload" +TS_MANUAL_TYPE = "manual" +TS_INPUT_TYPES = (TS_MANUAL_TYPE, TS_SELECT_TYPE, TS_UPLOAD_TYPE) PARAMETERS = {} if os.path.exists(staticfiles_storage.path("MVS_parameters_list.csv")) is True: @@ -386,17 +390,17 @@ def __init__( super().__init__(fields=fields, require_all_fields=False, **kwargs) def clean(self, values): - """If a file is provided it will be considered over the scalar""" + """If a file is provided it will be considered over the other fields""" scalar_value, timeseries_id, timeseries_file = values if timeseries_file is not None: input_timeseries_values = parse_input_timeseries(timeseries_file) answer = input_timeseries_values - input_dict = dict(type="upload", extra_info=timeseries_file.name) + input_dict = dict(type=TS_UPLOAD_TYPE, extra_info=timeseries_file.name) elif timeseries_id != "": ts = Timeseries.objects.get(id=timeseries_id) answer = ts.get_values - input_dict = dict(type="select", extra_info=timeseries_id) + input_dict = dict(type=TS_SELECT_TYPE, extra_info=timeseries_id) else: if scalar_value is None: scalar_value = "" @@ -422,7 +426,7 @@ def clean(self, values): params={"boundaries": self.boundaries}, ) else: - input_dict = dict(type="manual") + input_dict = dict(type=TS_MANUAL_TYPE) self.check_boundaries(answer) return json.dumps(dict(values=answer, input_method=input_dict)) diff --git a/app/templates/asset/timeseries_input.html b/app/templates/asset/timeseries_input.html index a4bc596bc..1f63df574 100644 --- a/app/templates/asset/timeseries_input.html +++ b/app/templates/asset/timeseries_input.html @@ -1,2 +1,37 @@ -{% include "asset/ts_input_template.html" with user="user_one" %} - +{% load i18n %} + +
+ {% spaceless %}{% for widget in widget.subwidgets %} + {{ widget.id }} + {% if 'scalar' in widget.name %} + +
+ {% include widget.template_name %} +
+ {% elif 'select' in widget.name %} +
+ {% include widget.template_name %} +
+ {% else %} +
+ {% include widget.template_name %} +
+ {% endif %} + {% endfor %}{% endspaceless %} +
+
+{% url 'get_timeseries' as ts_url %} +{{ ts_url|json_script:"tsUrl" }} \ No newline at end of file diff --git a/app/templates/asset/ts_input_template.html b/app/templates/asset/ts_input_template.html deleted file mode 100644 index bfb01f14f..000000000 --- a/app/templates/asset/ts_input_template.html +++ /dev/null @@ -1,33 +0,0 @@ -{% load i18n %} - -
- {% spaceless %}{% for widget in widget.subwidgets %} - {{ widget.id }} - {% if 'scalar' in widget.name %} - -
- {% include widget.template_name %} -
- {% elif 'select' in widget.name %} -
- {% include widget.template_name %} -
- {% else %} -
- {% include widget.template_name %} -
- {% endif %} - {% endfor %}{% endspaceless %} -
-
From 5459d6983ce2c90290eb4978ae6f9b144ba559a2 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 6 Feb 2025 17:16:24 +0100 Subject: [PATCH 07/13] Add a view to fetch timeseries data (first draft) --- app/projects/urls.py | 2 ++ app/projects/views.py | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/app/projects/urls.py b/app/projects/urls.py index 3af6f7ee4..b6cc41da9 100644 --- a/app/projects/urls.py +++ b/app/projects/urls.py @@ -135,6 +135,8 @@ # path('scenario/upload/', LoadScenarioFromFileView.as_view(), name='scenario_upload'), # Timeseries Model path("upload/timeseries", upload_timeseries, name="upload_timeseries"), + path("get/timeseries", get_timeseries, name="get_timeseries"), + path("get/timeseries/", get_timeseries, name="get_timeseries"), # Grid Model (Assets Creation) re_path( r"^asset/get_form/(?P\d+)/(?P\w+)?(/(?P[0-9a-f-]+))?$", diff --git a/app/projects/views.py b/app/projects/views.py index d032bf5a8..c73025a4f 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -1509,6 +1509,19 @@ def upload_timeseries(request): ts.user = request.user +@json_view +@login_required +@require_http_methods(["GET"]) +def get_timeseries(request, ts_id=None): + if request.method == "GET": + # TODO prevent user to get it is no access rights + # ts.user = request.user + if ts_id is not None: + ts = Timeseries.objects.get(id=ts_id) + # import pdb;pdb.set_trace() + return JsonResponse({"values": ts.get_values}) + + # region Asset From 80305348d76e87d0a770f244edd99e1dfa1e14ef Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 6 Feb 2025 17:17:46 +0100 Subject: [PATCH 08/13] Split file loading function according to file extension (python side) --- app/projects/helpers.py | 59 ++++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/app/projects/helpers.py b/app/projects/helpers.py index 607d50dd0..68d37e3ba 100644 --- a/app/projects/helpers.py +++ b/app/projects/helpers.py @@ -516,39 +516,62 @@ def parse_csv_timeseries(file_str): return timeseries_values -def parse_input_timeseries(timeseries_file): - if timeseries_file.name.endswith("xls") or timeseries_file.name.endswith("xlsx"): - wb = load_workbook(filename=timeseries_file) - worksheet = wb.active - timeseries_values = [] - n_col = worksheet.max_column +def parse_xlsx_timeseries(file_buffer): + wb = load_workbook(filename=file_buffer) + worksheet = wb.active + timeseries_values = [] + n_col = worksheet.max_column - col_idx = 0 + col_idx = 0 - if n_col > 1: - col_idx = 1 + if n_col > 1: + col_idx = 1 - for j in range(0, worksheet.max_row): - try: - timeseries_values.append( - float(worksheet.cell(row=j + 1, column=col_idx + 1).value) - ) - except ValueError: - pass + for j in range(0, worksheet.max_row): + try: + timeseries_values.append( + float(worksheet.cell(row=j + 1, column=col_idx + 1).value) + ) + except ValueError: + pass + # TODO add error message if this fails + return timeseries_values + +def parse_json_timeseries(file_str): + # TODO add error message if this fails + timeseries_values = json.loads(file_str) + # TODO add error message if this is not a timeseries + return timeseries_values + + +def parse_input_timeseries(timeseries_file): + """Parse timeseries input file provided by user for .xlsx, .csv, .txt and .json file formats + + Parameters + ---------- + timeseries_file file handle return by forms.FileInput clean method + + Returns + ------- + The values of the timeseries + + """ + if timeseries_file.name.endswith("xls") or timeseries_file.name.endswith("xlsx"): + timeseries_values = parse_xlsx_timeseries(file_buffer=timeseries_file) else: timeseries_file_str = timeseries_file.read().decode("utf-8-sig") if timeseries_file_str != "": if timeseries_file.name.endswith("json"): - timeseries_values = json.loads(timeseries_file_str) + timeseries_values = parse_json_timeseries(timeseries_file_str) elif timeseries_file.name.endswith("csv"): timeseries_values = parse_csv_timeseries(timeseries_file_str) elif timeseries_file.name.endswith("txt"): nlines = timeseries_file_str.count("\n") + 1 if nlines == 1: - timeseries_values = json.loads(timeseries_file_str) + timeseries_values = parse_json_timeseries(timeseries_file_str) else: timeseries_values = parse_csv_timeseries(timeseries_file_str) else: From bade6d22d5c74a7cec656ff4b4c98d53a5749d70 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 6 Feb 2025 17:18:38 +0100 Subject: [PATCH 09/13] Update TimeseriesInputWidget decompress method --- app/projects/helpers.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/projects/helpers.py b/app/projects/helpers.py index 68d37e3ba..576f13a64 100644 --- a/app/projects/helpers.py +++ b/app/projects/helpers.py @@ -1,4 +1,5 @@ import json +import logging import os import io import csv @@ -323,6 +324,7 @@ class TimeseriesInputWidget(forms.MultiWidget): template_name = "asset/timeseries_input.html" class Media: + # TODO: currently not loading the content as not within head js = [JSPlotlyLib(), JSD3Lib(), "js/traceplot.js"] def __init__(self, select_widget, **kwargs): @@ -359,9 +361,10 @@ def use_required_attribute(self, initial): return False def decompress(self, value): - # TODO update handling here - value corresponds to the pk of the timeseries - answer = ["", "", value] - + if not isinstance(value, int): + logging.error("The value of timeseries index is not an integer") + if Timeseries.objects.filter(id=value).exists(): + answer = [value, value, ""] return answer From 1a1ae8b1e05eb3d2010db19b59a099c4552e2ed0 Mon Sep 17 00:00:00 2001 From: "pierre-francois.duc" Date: Thu, 6 Feb 2025 17:19:50 +0100 Subject: [PATCH 10/13] Split TimeseriesInput and DualInput js logic, delete unused functions --- app/projects/helpers.py | 7 +- app/static/js/grid_model_topology.js | 8 +- app/static/js/traceplot.js | 88 ++++++++++++++++------ app/templates/asset/asset_create_form.html | 12 ++- app/templates/scenario/scenario_step2.html | 1 + 5 files changed, 82 insertions(+), 34 deletions(-) diff --git a/app/projects/helpers.py b/app/projects/helpers.py index 576f13a64..8fce65c76 100644 --- a/app/projects/helpers.py +++ b/app/projects/helpers.py @@ -335,21 +335,22 @@ def __init__(self, select_widget, **kwargs): select_widget.attrs.update( { "class": "form-select", - "onchange": f"selectExistingTimeseries(obj=this.value, param_name='{self.param_name}')", + "onchange": f"changeTimeseriesSelectValue(obj=this.value, param_name='{self.param_name}')", + "onload": f"changeTimeseriesSelectValue(obj=this.value, param_name='{self.param_name}')", } ) widgets = { "scalar": forms.TextInput( attrs={ "class": "form-control", - "onchange": f"plotDualInputTrace(obj=this.value, param_name='{self.param_name}')", + "onchange": f"changeTimeseriesManualValue(obj=this.value, param_name='{self.param_name}')", } ), "select": select_widget, "file": forms.FileInput( attrs={ "class": "form-control", - "onchange": f"uploadDualInputTrace(obj=this.files, param_name='{self.param_name}')", + "onchange": f"changeTimeseriesUploadValue(obj=this.files, param_name='{self.param_name}')", } ), } diff --git a/app/static/js/grid_model_topology.js b/app/static/js/grid_model_topology.js index 759646b56..ae49d2f50 100644 --- a/app/static/js/grid_model_topology.js +++ b/app/static/js/grid_model_topology.js @@ -102,16 +102,17 @@ async function addNodeToDrawFlow(name, pos_x, pos_y, nodeInputs = 1, nodeOutputs return createNodeObject(name, nodeInputs, nodeOutputs, nodeData, pos_x, pos_y); } +// TODO potentially remove this function function updateInputTimeseries(){ //connected to the templates/asset/asset_create_form.html content - ts_data_div = document.getElementById("input_timeseries_data"); + /*ts_data_div = document.getElementById("input_timeseries_data"); if(ts_data_div){ var ts_data = JSON.parse(ts_data_div.querySelector("textarea").value); var ts_data = ts_data.map(String); var ts_idx = [...Array(ts_data.length).keys()]; ts_idx = ts_idx.map(String); makePlotly( ts_idx, ts_data, plot_id="timeseries_trace") - } + }*/ } // find out the name of the other nodes the given node is connected to @@ -327,7 +328,8 @@ $("#guiModal").on('shown.bs.modal', function (event) { var formDiv = document.getElementsByClassName("form-group"); var plotDiv = null; - var plotDivIds = ["flow_trace", "timeseries_trace", "soc_traces"]; + //TODO get rid of maybe soc_traces + var plotDivIds = ["flow_trace", "soc_traces"]; for(i=0;i myarray.push([el])) + processData(myarray); + graphDOM.style.display = "block"; + } + else{ + graphDOM.style.display = "none"; + // reset file in memory if the user inputs a scalar after uploading a file + var fileID = "id_" + param_name + "_2"; + var file_input = document.getElementById(fileID); + file_input.value = ""; + }; + } -var PLOT_ID = ""; /* Plot update of textinput field of DualInput field */ -function plotDualInputTrace(obj, param_name="", select=false){ - - if(select == true) { - // TODO in a different function, load the data from DB and then plot this data - var valueID = "id_" + param_name + "_0"; - var value_input = document.getElementById(valueID); - value_input.value = obj; - } +function plotDualInputTrace(obj, param_name=""){ // TODO get the timeseries timestamps (if exists) from a hidden safejs div with the django tag method jsObj = JSON.parse(obj); @@ -112,8 +154,6 @@ function plotDualInputTrace(obj, param_name="", select=false){ graphDOM.style.display = "block"; } else{ - graphDOM.style.display = "none"; - // reset file in memory if the user inputs a scalar after uploading a file var fileID = "id_" + param_name + "_1"; var file_input = document.getElementById(fileID); file_input.value = ""; diff --git a/app/templates/asset/asset_create_form.html b/app/templates/asset/asset_create_form.html index 8ba9a6841..c82a3ab9d 100644 --- a/app/templates/asset/asset_create_form.html +++ b/app/templates/asset/asset_create_form.html @@ -3,6 +3,8 @@ {% load crispy_forms_tags %} {% load custom_filters %} +{{ form.media }} + @@ -78,9 +80,11 @@

{% translate "Technical parameters" %}

{{ form|get_field:"input_timeseries"|as_crispy_field }}
- -
- + {% if input_timeseries_timestamps %}