From 9607392b43715ab0e981915e99abfcdb6df15662 Mon Sep 17 00:00:00 2001 From: Ross Perry Date: Fri, 29 Sep 2023 08:37:56 -0600 Subject: [PATCH 1/3] eeej small files --- docker-compose.dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 897163600f..2928c20150 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -22,6 +22,7 @@ services: environment: - DJANGO_SETTINGS_MODULE=config.settings.docker_dev - DJANGO_LOG_LEVEL + - EEEJ_LOAD_SMALL_TEST_DATASET=True volumes: - ref_seed_media:/seed/media - ./seed/:/seed/seed/ From 86c7fd98906a8eca0aa9b9345a75d752cf7ffa20 Mon Sep 17 00:00:00 2001 From: Ross Perry Date: Mon, 8 Jan 2024 15:42:44 -0700 Subject: [PATCH 2/3] bulk import duplicate date error handling --- seed/serializers/meter_readings.py | 13 +++++++++- seed/tests/test_meter_views.py | 41 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/seed/serializers/meter_readings.py b/seed/serializers/meter_readings.py index b4f9befcd9..d1d9476542 100644 --- a/seed/serializers/meter_readings.py +++ b/seed/serializers/meter_readings.py @@ -9,6 +9,7 @@ import dateutil.parser from django.db import connection from django.utils.timezone import make_aware +from django.core.exceptions import ValidationError from psycopg2.extras import execute_values from pytz import timezone from rest_framework import serializers @@ -53,6 +54,17 @@ def create(self, validated_data) -> list[MeterReading]: updated_readings = list(map(lambda result: MeterReading(**{field: result[i] for i, field in enumerate(meter_fields)}), results)) return updated_readings + + def validate(self, data): + # duplicate start and end date pairs will cause sql errors + date_pairs = set() + for datum in data: + date_pair = (datum.get('start_time'), datum.get('end_time')) + if date_pair in date_pairs: + raise ValidationError('Error: Each reading must have a unique combination of start_time end end_time.') + date_pairs.add(date_pair) + + return data class MeterReadingSerializer(serializers.ModelSerializer): @@ -95,7 +107,6 @@ def create(self, validated_data) -> MeterReading: # Convert tuple to MeterReading for response updated_reading = MeterReading(**{field: result[i] for i, field in enumerate(meter_fields)}) - return updated_reading def to_representation(self, obj): diff --git a/seed/tests/test_meter_views.py b/seed/tests/test_meter_views.py index ff97419c64..91cfe7a99d 100644 --- a/seed/tests/test_meter_views.py +++ b/seed/tests/test_meter_views.py @@ -466,6 +466,47 @@ def test_bulk_import(self): self.assertEqual(response.status_code, 200) self.assertEqual(len(response.json()), 3) + def test_bulk_import_duplicate_dates(self): + """Bulk Meter Readings with duplicate start_time and end_time pairs will be rejected to avoid sql errors""" + + property_view = self.property_view_factory.get_property_view() + url = reverse('api:v3:property-meters-list', kwargs={'property_pk': property_view.id}) + + payload = { + 'type': 'Electric', + 'source': 'Manual Entry', + 'source_id': '1234567890', + } + + response = self.client.post(url, data=json.dumps(payload), content_type='application/json') + meter_pk = response.json()['id'] + + url = reverse('api:v3:property-meter-readings-list', kwargs={'property_pk': property_view.id, 'meter_pk': meter_pk}) + + # prepare the data in bulk format + reading1 = { + "start_time": "2022-01-05 05:00:00", + "end_time": "2022-01-05 06:00:00", + "reading": 10, + "source_unit": "Wh (Watt-hours)", + # conversion factor is required and is the conversion from the source unit to kBTU (1 Wh = 0.00341 kBtu) + "conversion_factor": 0.00341, + } + reading2 = dict(reading1) + reading2["end_time"] = "2022-01-05 07:00:00" + payload = [reading1, reading2] + + response = self.client.post(url, data=json.dumps(payload), content_type='application/json') + self.assertEqual(response.status_code, 201) + self.assertEqual(response.json()[0]['reading'], 10) + self.assertEqual(response.json()[1]['reading'], 10) + + # Duplicate start and end times will be rejected + payload = [reading1, reading1] + response = self.client.post(url, data=json.dumps(payload), content_type='application/json') + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json()['non_field_errors'], ['Error: Each reading must have a unique combination of start_time end end_time.']) + def test_delete_meter_readings(self): # would be nice nice to make a factory out of the meter / meter reading requests property_view = self.property_view_factory.get_property_view() From 6509934edb9a4ea98f989815777429046db18b9a Mon Sep 17 00:00:00 2001 From: Ross Perry Date: Mon, 8 Jan 2024 15:47:29 -0700 Subject: [PATCH 3/3] precommit --- seed/serializers/meter_readings.py | 6 +++--- seed/tests/test_meter_views.py | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/seed/serializers/meter_readings.py b/seed/serializers/meter_readings.py index d1d9476542..730e820db0 100644 --- a/seed/serializers/meter_readings.py +++ b/seed/serializers/meter_readings.py @@ -7,9 +7,9 @@ from typing import Tuple import dateutil.parser +from django.core.exceptions import ValidationError from django.db import connection from django.utils.timezone import make_aware -from django.core.exceptions import ValidationError from psycopg2.extras import execute_values from pytz import timezone from rest_framework import serializers @@ -54,14 +54,14 @@ def create(self, validated_data) -> list[MeterReading]: updated_readings = list(map(lambda result: MeterReading(**{field: result[i] for i, field in enumerate(meter_fields)}), results)) return updated_readings - + def validate(self, data): # duplicate start and end date pairs will cause sql errors date_pairs = set() for datum in data: date_pair = (datum.get('start_time'), datum.get('end_time')) if date_pair in date_pairs: - raise ValidationError('Error: Each reading must have a unique combination of start_time end end_time.') + raise ValidationError('Error: Each reading must have a unique combination of start_time end end_time.') date_pairs.add(date_pair) return data diff --git a/seed/tests/test_meter_views.py b/seed/tests/test_meter_views.py index 91cfe7a99d..89ba8d6c79 100644 --- a/seed/tests/test_meter_views.py +++ b/seed/tests/test_meter_views.py @@ -485,13 +485,13 @@ def test_bulk_import_duplicate_dates(self): # prepare the data in bulk format reading1 = { - "start_time": "2022-01-05 05:00:00", - "end_time": "2022-01-05 06:00:00", - "reading": 10, - "source_unit": "Wh (Watt-hours)", - # conversion factor is required and is the conversion from the source unit to kBTU (1 Wh = 0.00341 kBtu) - "conversion_factor": 0.00341, - } + "start_time": "2022-01-05 05:00:00", + "end_time": "2022-01-05 06:00:00", + "reading": 10, + "source_unit": "Wh (Watt-hours)", + # conversion factor is required and is the conversion from the source unit to kBTU (1 Wh = 0.00341 kBtu) + "conversion_factor": 0.00341, + } reading2 = dict(reading1) reading2["end_time"] = "2022-01-05 07:00:00" payload = [reading1, reading2]