From 4a60eb25ade31468b42fdae29ecfe7877b7f8497 Mon Sep 17 00:00:00 2001 From: Ross Perry Date: Fri, 12 Jan 2024 11:24:08 -0700 Subject: [PATCH 1/6] Add bulk meter reading error handling for duplicate date-pairs (#4467) * eeej small files * bulk import duplicate date error handling * precommit --------- Co-authored-by: Alex Swindler --- seed/serializers/meter_readings.py | 13 +++++++++- seed/tests/test_meter_views.py | 41 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/seed/serializers/meter_readings.py b/seed/serializers/meter_readings.py index b4f9befcd9..730e820db0 100644 --- a/seed/serializers/meter_readings.py +++ b/seed/serializers/meter_readings.py @@ -7,6 +7,7 @@ from typing import Tuple import dateutil.parser +from django.core.exceptions import ValidationError from django.db import connection from django.utils.timezone import make_aware from psycopg2.extras import execute_values @@ -54,6 +55,17 @@ def create(self, validated_data) -> list[MeterReading]: return updated_readings + def validate(self, data): + # duplicate start and end date pairs will cause sql errors + date_pairs = set() + for datum in data: + date_pair = (datum.get('start_time'), datum.get('end_time')) + if date_pair in date_pairs: + raise ValidationError('Error: Each reading must have a unique combination of start_time end end_time.') + date_pairs.add(date_pair) + + return data + class MeterReadingSerializer(serializers.ModelSerializer): class Meta: @@ -95,7 +107,6 @@ def create(self, validated_data) -> MeterReading: # Convert tuple to MeterReading for response updated_reading = MeterReading(**{field: result[i] for i, field in enumerate(meter_fields)}) - return updated_reading def to_representation(self, obj): diff --git a/seed/tests/test_meter_views.py b/seed/tests/test_meter_views.py index ff97419c64..89ba8d6c79 100644 --- a/seed/tests/test_meter_views.py +++ b/seed/tests/test_meter_views.py @@ -466,6 +466,47 @@ def test_bulk_import(self): self.assertEqual(response.status_code, 200) self.assertEqual(len(response.json()), 3) + def test_bulk_import_duplicate_dates(self): + """Bulk Meter Readings with duplicate start_time and end_time pairs will be rejected to avoid sql errors""" + + property_view = self.property_view_factory.get_property_view() + url = reverse('api:v3:property-meters-list', kwargs={'property_pk': property_view.id}) + + payload = { + 'type': 'Electric', + 'source': 'Manual Entry', + 'source_id': '1234567890', + } + + response = self.client.post(url, data=json.dumps(payload), content_type='application/json') + meter_pk = response.json()['id'] + + url = reverse('api:v3:property-meter-readings-list', kwargs={'property_pk': property_view.id, 'meter_pk': meter_pk}) + + # prepare the data in bulk format + reading1 = { + "start_time": "2022-01-05 05:00:00", + "end_time": "2022-01-05 06:00:00", + "reading": 10, + "source_unit": "Wh (Watt-hours)", + # conversion factor is required and is the conversion from the source unit to kBTU (1 Wh = 0.00341 kBtu) + "conversion_factor": 0.00341, + } + reading2 = dict(reading1) + reading2["end_time"] = "2022-01-05 07:00:00" + payload = [reading1, reading2] + + response = self.client.post(url, data=json.dumps(payload), content_type='application/json') + self.assertEqual(response.status_code, 201) + self.assertEqual(response.json()[0]['reading'], 10) + self.assertEqual(response.json()[1]['reading'], 10) + + # Duplicate start and end times will be rejected + payload = [reading1, reading1] + response = self.client.post(url, data=json.dumps(payload), content_type='application/json') + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json()['non_field_errors'], ['Error: Each reading must have a unique combination of start_time end end_time.']) + def test_delete_meter_readings(self): # would be nice nice to make a factory out of the meter / meter reading requests property_view = self.property_view_factory.get_property_view() From 5c10968c2ccf8d53d831ac35a8fade9149ebe4c2 Mon Sep 17 00:00:00 2001 From: Hannah Eslinger Date: Fri, 12 Jan 2024 12:50:05 -0700 Subject: [PATCH 2/6] Retrieve all map data at once (#4469) * Retrieve all data at once * de-list results * remove console log --------- Co-authored-by: kflemin <2205659+kflemin@users.noreply.github.com> --- .../controllers/inventory_map_controller.js | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/seed/static/seed/js/controllers/inventory_map_controller.js b/seed/static/seed/js/controllers/inventory_map_controller.js index bbfa869081..3b17c9e9b0 100644 --- a/seed/static/seed/js/controllers/inventory_map_controller.js +++ b/seed/static/seed/js/controllers/inventory_map_controller.js @@ -44,17 +44,24 @@ angular.module('BE.seed.controller.inventory_map', []).controller('inventory_map }; const chunk = 250; - const fetchRecords = (fn, page = 1) => fn(page, chunk, undefined, undefined).then((data) => { - $scope.progress = { - current: data.pagination.end, - total: data.pagination.total, - percent: Math.round((data.pagination.end / data.pagination.total) * 100) - }; - if (data.pagination.has_next) { - return fetchRecords(fn, page + 1).then((newData) => data.results.concat(newData)); - } - return data.results; - }); + const fetchRecords = async (fn) => { + pagination = await fn(1, chunk, undefined, undefined).then(data => data.pagination); + + $scope.progress = {current: 0, total: pagination.total, percent:0}; + + page_numbers = [...Array(pagination.num_pages).keys()] + page_promises = page_numbers.map(page => { + return fn(page, chunk, undefined, undefined).then(data => { + num_data = data.pagination.end - data.pagination.start + 1; + $scope.progress.current += num_data; + $scope.progress.percent += Math.round((num_data / data.pagination.total) * 100) + return data.results + }) + }) + + return Promise.all(page_promises).then(pages => [].concat(...pages)) + } + $scope.progress = {}; const loadingModal = $uibModal.open({ From ab24aed6a5fe5edad9a1b9693c139174bd303453 Mon Sep 17 00:00:00 2001 From: Hannah Eslinger Date: Fri, 12 Jan 2024 16:31:00 -0700 Subject: [PATCH 3/6] Change unique name contraint on Derived column so it s within inventory type (#4472) Fix #4343 Co-authored-by: Katherine Fleming <2205659+kflemin@users.noreply.github.com> --- seed/migrations/0211_auto_20240109_1348.py | 21 +++++++++++++++++++++ seed/models/derived_columns.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 seed/migrations/0211_auto_20240109_1348.py diff --git a/seed/migrations/0211_auto_20240109_1348.py b/seed/migrations/0211_auto_20240109_1348.py new file mode 100644 index 0000000000..ad0266c223 --- /dev/null +++ b/seed/migrations/0211_auto_20240109_1348.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.23 on 2024-01-09 21:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('seed', '0210_natural_sort'), + ] + + operations = [ + migrations.RemoveConstraint( + model_name='derivedcolumn', + name='unique_name_for_organization', + ), + migrations.AddConstraint( + model_name='derivedcolumn', + constraint=models.UniqueConstraint(fields=('organization', 'name', 'inventory_type'), name='unique_name_for_organization'), + ), + ] diff --git a/seed/models/derived_columns.py b/seed/models/derived_columns.py index 86c92f729c..598ffa6d67 100644 --- a/seed/models/derived_columns.py +++ b/seed/models/derived_columns.py @@ -202,7 +202,7 @@ class DerivedColumn(models.Model): class Meta: constraints = [ models.UniqueConstraint( - fields=['organization', 'name'], name='unique_name_for_organization' + fields=['organization', 'name', 'inventory_type'], name='unique_name_for_organization' ) ] From a75aa382081271d0f34b7ec3e74a31a65f2ad528 Mon Sep 17 00:00:00 2001 From: Hannah Eslinger Date: Tue, 16 Jan 2024 10:54:21 -0700 Subject: [PATCH 4/6] Allow omitted fields on the mapping page to be unfilled (#4471) Fix #4361 Co-authored-by: Katherine Fleming <2205659+kflemin@users.noreply.github.com> --- .../seed/js/controllers/mapping_controller.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/seed/static/seed/js/controllers/mapping_controller.js b/seed/static/seed/js/controllers/mapping_controller.js index 1abd25c03f..d87f75573a 100644 --- a/seed/static/seed/js/controllers/mapping_controller.js +++ b/seed/static/seed/js/controllers/mapping_controller.js @@ -598,15 +598,24 @@ angular.module('BE.seed.controller.mapping', []).controller('mapping_controller' /** * empty_units_present: used to disable or enable the 'Map Your Data' button if any units are empty */ - $scope.empty_units_present = () => Boolean(_.find($scope.mappings, (field) => field.suggestion_table_name === 'PropertyState' && field.from_units === null && ($scope.is_area_column(field) || $scope.is_eui_column(field)))); + $scope.empty_units_present = () => { + return $scope.mappings.some((field) => ( + !field.isOmitted && + field.suggestion_table_name === 'PropertyState' && + field.from_units === null && + ($scope.is_area_column(field) || $scope.is_eui_column(field)) + )) + }; /** * empty_fields_present: used to disable or enable the 'show & review * mappings' button. No warning associated as users "aren't done" listing their mapping settings. */ const suggestions_not_provided_yet = () => { - const no_suggestion_value = Boolean(_.find($scope.mappings, { suggestion: undefined })); - const no_suggestion_table_name = Boolean(_.find($scope.mappings, { suggestion_table_name: undefined })); + const non_omitted_mappings = $scope.mappings.filter(m => !m.isOmitted) + const no_suggestion_value = Boolean(_.find(non_omitted_mappings, { suggestion: undefined })); + const no_suggestion_table_name = Boolean(_.find(non_omitted_mappings, { suggestion_table_name: undefined })); + return no_suggestion_value || no_suggestion_table_name; }; From 9777ea1ac111674373518d6c1d8caeb8bd30481a Mon Sep 17 00:00:00 2001 From: Nicholas Long <1907354+nllong@users.noreply.github.com> Date: Tue, 16 Jan 2024 14:14:10 -0700 Subject: [PATCH 5/6] Update developer documentation dependencies (#4485) update sphinx version and related files --- .cspell.json | 2 ++ .gitignore | 8 ++++---- bin/protractor_start_server.sh | 4 +--- docs/source/conf.py | 6 ++++-- docs/source/docker.rst | 2 +- docs/source/kubernetes_deployment.rst | 8 ++++---- docs/source/migrations.rst | 4 +--- docs/source/modules/seed.rst | 8 -------- docs/source/modules/seed.utils.rst | 16 ---------------- requirements/test.txt | 25 ++++++++++++------------- tox.ini | 3 ++- 11 files changed, 31 insertions(+), 55 deletions(-) diff --git a/.cspell.json b/.cspell.json index 398795018a..f74492eb55 100644 --- a/.cspell.json +++ b/.cspell.json @@ -25,6 +25,7 @@ "auditlog", "auth", "autogenerated", + "automodule", "aws", "AWS", "backend", @@ -54,6 +55,7 @@ "casted", "cb", "CEJST", + "celerybeat", "cfg", "changelog", "checkbox", diff --git a/.gitignore b/.gitignore index 166547d8dc..a670635509 100644 --- a/.gitignore +++ b/.gitignore @@ -1,17 +1,18 @@ -.DS_Store .cache +.coverage +.cspell.txt +.DS_Store .env .hypothesis .idea .project .pydevproject .python-version -.coverage .vscode coverage.protractor.json -.vscode *~ *~$* +*.ipynb *.pyc *.swp *.swo @@ -25,7 +26,6 @@ pkgs/newrelic-1\.6\.0\.13\.tar\.gz chromedriver.log ipython_input_log_history .ipython_input_log_history -*.ipynb **/ipython_input_log_history web_root/csvs/* web_root/uploads/* diff --git a/bin/protractor_start_server.sh b/bin/protractor_start_server.sh index c576cb64d4..3c25e1fb7e 100755 --- a/bin/protractor_start_server.sh +++ b/bin/protractor_start_server.sh @@ -18,8 +18,6 @@ echo "run e2e tests" ./node_modules/protractor/bin/protractor seed/static/seed/tests/protractor-tests/protractorConfigCoverage.js # echo "install coverall merge stuffs" # gem install coveralls-lcov -# pip install coveralls-merge # echo "run lcov to coveralls json" # coveralls-lcov -v -n protractorReports/lcov.info > coverage.protractor.json -# echo "merge and post coveralls" -# coveralls-merge coverage.protractor.json +# echo "upload to coveralls" diff --git a/docs/source/conf.py b/docs/source/conf.py index ac5d5c046f..57768e6b56 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -56,7 +56,9 @@ templates_path = ['_templates'] # Location of word list. -spelling_word_list_filename = '../../.cspell.json' +# convert the spelling list to a text file and save +open('../../.cspell.txt', 'w').write('\n'.join(json.load(open('../../.cspell.json'))['words'])) +spelling_word_list_filename = '../../.cspell.txt' # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: @@ -91,7 +93,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: diff --git a/docs/source/docker.rst b/docs/source/docker.rst index 60db652e50..a0208c345b 100644 --- a/docs/source/docker.rst +++ b/docs/source/docker.rst @@ -110,7 +110,7 @@ Ubuntu server 18.04 or newer with a m5ad.xlarge (if using in Production instance Deploying with Docker -^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^ The preferred way to deploy with Docker is using docker swarm and docker stack. Look at the `deploy.sh script`_ for implementation details. diff --git a/docs/source/kubernetes_deployment.rst b/docs/source/kubernetes_deployment.rst index a67dc9ab44..3f70fdc154 100644 --- a/docs/source/kubernetes_deployment.rst +++ b/docs/source/kubernetes_deployment.rst @@ -59,14 +59,14 @@ Helm Helm organizes all of your Kubernetes deployment, service, and volume yml files into "charts" that can be deployed, managed, and published with simple commands. To install Helm: -* `Windows eksctl https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl-on-windows`_ +* `Windows eksctl `_ * Mac (with Homebrew) :code:`brew install helm` EKS Control (AWS Specific) ^^^^^^^^^^^^^^^^^^^^^^^^^^ EKSCtl is a command line tool to manage Elastic Kubernetes clusters on AWS. If not using AWS, then disregard this section. -* `Windows `_ +* `Windows eksctl config `_ * Mac (with Homebrew) :code:`brew install eksctl` To launch a cluster on using EKSCts, run the following command in the terminal (assuming adequate permissions for the user). Also make sure to replace items in the `<>` brackets. @@ -161,7 +161,7 @@ This chart contains the deployment specification for the Celery container to con value: # must match db-postgres-deployment.yaml and web-celery-deployment.yaml bsyncr-deployment.yaml -************************** +********************** This chart contains the deployment specification for the bsyncr analysis server. Request a NOAA token from `this website `_. .. code-block:: yaml @@ -239,7 +239,7 @@ The command below will restart the pods and re-pull the docker images. Other Resources --------------- -Common kubectl actions can be found `here `_ +Common kubectl actions can be found `on the kubernetes website `_ .. _AWS: https://docs.aws.amazon.com/eks/latest/userguide/create-cluster.html diff --git a/docs/source/migrations.rst b/docs/source/migrations.rst index cd321bd2ba..624a23c329 100644 --- a/docs/source/migrations.rst +++ b/docs/source/migrations.rst @@ -223,9 +223,7 @@ Max OSX Version 2.5.2 ------------- -- There are no manual migrations that are needed. The `./manage.py migrate` command may take awhile -to run since the migration requires the recalculation of all the normalized addresses to parse -bldg correct and to cast the result as a string and not a bytestring. +- There are no manual migrations that are needed. The `./manage.py migrate` command may take awhile to run since the migration requires the recalculation of all the normalized addresses to parse bldg correct and to cast the result as a string and not a bytestring. Version 2.5.1 ------------- diff --git a/docs/source/modules/seed.rst b/docs/source/modules/seed.rst index a8c3166a76..771c0324c2 100644 --- a/docs/source/modules/seed.rst +++ b/docs/source/modules/seed.rst @@ -80,14 +80,6 @@ Token Generator :undoc-members: :show-inheritance: -URLs ----- - -.. automodule:: seed.urls - :members: - :undoc-members: - :show-inheritance: - Utils ----- diff --git a/docs/source/modules/seed.utils.rst b/docs/source/modules/seed.utils.rst index 813f849f94..27fd9b6cf7 100644 --- a/docs/source/modules/seed.utils.rst +++ b/docs/source/modules/seed.utils.rst @@ -20,22 +20,6 @@ Buildings :undoc-members: :show-inheritance: -Constants ---------- - -.. automodule:: seed.utils.constants - :members: - :undoc-members: - :show-inheritance: - -Mappings --------- - -.. automodule:: seed.utils.mapping - :members: - :undoc-members: - :show-inheritance: - Organizations ------------- diff --git a/requirements/test.txt b/requirements/test.txt index 191964657e..7808659ee7 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -11,29 +11,28 @@ psutil==5.6.7 # python testing Faker==0.9.3 mock==2.0.0 -coveralls-merge==0.0.3 vcrpy==4.2.1 -pytest==7.2.0 -pytest-django==4.5.2 +pytest==7.4.4 +pytest-django==4.7.0 # Lock urllib3 to v1 until vcrpy supports it urllib3<2 # static code analysis -flake8==3.8.1 -pycodestyle==2.6.0 -pre-commit==2.19.0 +flake8==7.0.0 +pycodestyle==2.11.1 +pre-commit==3.6.0 # documentation and spelling -Sphinx==4.0.2 -sphinxcontrib-spelling==4.3.0 -sphinx_rtd_theme==0.4.3 -docutils==0.17.1 +Sphinx==7.2.6 +sphinxcontrib-spelling==8.0.0 +sphinx_rtd_theme==2.0.0 +docutils==0.20.1 # property-based testing -hypothesis==6.12.0 +hypothesis==6.94.0 # For running the server -uWSGI==2.0.22; sys_platform != "win32" +uWSGI==2.0.23; sys_platform != "win32" # static type checking -mypy==1.0.0 +mypy==1.8.0 diff --git a/tox.ini b/tox.ini index e8cf581650..25cc8bfbc4 100644 --- a/tox.ini +++ b/tox.ini @@ -44,6 +44,8 @@ changedir=docs deps= -r{toxinidir}/requirements/test.txt commands= + # After we fix doc build links/issues, then add the -W flag + ; make spelling SPHINXOPTS='-W --keep-going' make spelling sphinx-build -b html -d {envtmpdir}/doctrees {toxinidir}/docs/source {envtmpdir}/html whitelist_externals= @@ -76,7 +78,6 @@ passenv= TRAVIS_BUILD_NUMBER MAPQUEST_API_KEY whitelist_externals= - ; coveralls-merge cp npm From 418e9c04698e97d60e0bb622d92474d9816f8a0f Mon Sep 17 00:00:00 2001 From: Katherine Fleming <2205659+kflemin@users.noreply.github.com> Date: Tue, 16 Jan 2024 15:09:29 -0700 Subject: [PATCH 6/6] Add JSON response to GET Audit Template submission (#4477) * add JSON response support to audit template submission GET * isort --------- Co-authored-by: Nicholas Long <1907354+nllong@users.noreply.github.com> Co-authored-by: Alex Swindler --- seed/audit_template/audit_template.py | 11 +++++------ seed/views/v3/audit_template.py | 19 +++++++++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/seed/audit_template/audit_template.py b/seed/audit_template/audit_template.py index 80a97cb96e..b822640877 100644 --- a/seed/audit_template/audit_template.py +++ b/seed/audit_template/audit_template.py @@ -59,24 +59,23 @@ def get_submission(self, audit_template_submission_id: int, report_format: str = Args: audit_template_submission_id (int): value of the "Submission ID" as seen on Audit Template - report_format (str, optional): Report format, either `xml` or `pdf`. Defaults to 'pdf'. + report_format (str, optional): Report format, either `json`, `xml`, or `pdf`. Defaults to 'pdf'. Returns: requests.response: Result from Audit Template website """ - # supporting 'PDF' and 'XML' formats only for now + # supporting 'JSON', PDF', and 'XML' formats only for now token, message = self.get_api_token() if not token: return None, message # validate format - if report_format.lower() not in ['xml', 'pdf']: + if report_format.lower() not in ['json', 'xml', 'pdf']: report_format = 'pdf' # set headers - headers = {'accept': 'application/pdf'} - if report_format.lower() == 'xml': - headers = {'accept': 'application/xml'} + accept_type = 'application/' + report_format.lower() + headers = {'accept': accept_type} url = f'{self.API_URL}/rp/submissions/{audit_template_submission_id}.{report_format}?token={token}' try: diff --git a/seed/views/v3/audit_template.py b/seed/views/v3/audit_template.py index 1dfb398d10..20082ea92c 100644 --- a/seed/views/v3/audit_template.py +++ b/seed/views/v3/audit_template.py @@ -4,6 +4,8 @@ SEED Platform (TM), Copyright (c) Alliance for Sustainable Energy, LLC, and other contributors. See also https://github.com/seed-platform/seed/main/LICENSE.md """ +import json + from django.http import HttpResponse, JsonResponse from drf_yasg.utils import swagger_auto_schema from rest_framework import viewsets @@ -37,7 +39,7 @@ def get_submission(self, request, pk): default_report_format = 'pdf' report_format = request.query_params.get('report_format', default_report_format) - valid_file_formats = ['xml', 'pdf'] + valid_file_formats = ['json', 'xml', 'pdf'] if report_format.lower() not in valid_file_formats: message = f"The report_format specified is invalid. Must be one of: {valid_file_formats}." return JsonResponse({ @@ -49,18 +51,23 @@ def get_submission(self, request, pk): at = AuditTemplate(self.get_organization(self.request)) response, message = at.get_submission(pk, report_format) + # error if response is None: return JsonResponse({ 'success': False, 'message': message }, status=400) + # json + if report_format.lower() == 'json': + return JsonResponse(json.loads(response.content)) + # xml if report_format.lower() == 'xml': return HttpResponse(response.text) - else: - response2 = HttpResponse(response.content) - response2.headers["Content-Type"] = 'application/pdf' - response2.headers["Content-Disposition"] = f'attachment; filename="at_submission_{pk}.pdf"' - return response2 + # pdf + response2 = HttpResponse(response.content) + response2.headers["Content-Type"] = 'application/pdf' + response2.headers["Content-Disposition"] = f'attachment; filename="at_submission_{pk}.pdf"' + return response2 @swagger_auto_schema(manual_parameters=[AutoSchemaHelper.query_org_id_field()]) @has_perm_class('can_view_data')