From af1ac6abd092c96df7ab5face25206ec33169c01 Mon Sep 17 00:00:00 2001 From: Ross Perry Date: Fri, 9 Aug 2024 15:22:11 -0600 Subject: [PATCH] Add cross cycle data quality checks (#4695) * Initial cross-cycle data quality changes * create default cross cycle rules * update propertyview label name * rule span * merge develop * add target to rule * change cross-cycle to goal and add action * order * update goalnote passed_check based on rule * update goalnote passed_check based on rule * add and remove labels based on cross cycle data quality checks * specify goal and baseline_view when applying cc labels * add default labels, update rule model fields * cross cycle rules functional * labels limited to goal, or none goal * update test * checking default single cycle * results for range and not null * frontend results * precommit * precommit * fix eui inverse * add help text * prevent new goal rules * precommit * lint * test hard coded goal rules * fix frontend test * update migration to remove old constraint * fix tests * precommit * remove consoles * label width bugfix * help text if not passing checks * clarifying text * default goal labels to show in list * lint * modify migration * show all labels * qaqc * precommit * lint * temp disable wui rules * lint * remove WUI data type - indroduced in different pr * precommit * update rule count * migration order * update migration update migration precommit * move spinner utility earlier --------- Co-authored-by: Alex Swindler Co-authored-by: Ross Perry Co-authored-by: Ross Perry Co-authored-by: Katherine Fleming <2205659+kflemin@users.noreply.github.com> --- seed/api/v3/urls.py | 2 + seed/data_importer/tasks.py | 45 ++- .../0222_cross_cycle_data_quality.py | 214 +++++++++++++ seed/models/columns.py | 26 +- seed/models/data_quality.py | 291 +++++++++++++++++- seed/models/models.py | 10 +- seed/models/properties.py | 20 +- seed/serializers/property_view_labels.py | 21 ++ seed/serializers/rules.py | 6 +- .../data_quality_admin_controller.js | 86 +++--- .../data_quality_modal_controller.js | 5 + .../js/controllers/delete_modal_controller.js | 1 - .../controllers/inventory_list_controller.js | 2 +- .../portfolio_summary_controller.js | 74 +++-- seed/static/seed/js/seed.js | 18 +- .../seed/js/services/data_quality_service.js | 9 +- seed/static/seed/js/services/label_service.js | 7 + seed/static/seed/partials/accounts.html | 2 +- seed/static/seed/partials/accounts_nav.html | 2 +- .../seed/partials/data_quality_admin.html | 127 +++++++- .../seed/partials/portfolio_summary.html | 22 +- seed/static/seed/scss/style.scss | 6 +- .../data_quality_admin_controller.spec.js | 2 +- seed/templates/seed/_header.html | 2 +- seed/test_helpers/fake.py | 55 ++++ .../test_data_quality_check_rules_views.py | 11 +- seed/tests/test_data_quality_checks.py | 186 ++++++++++- seed/tests/test_labels_api_views.py | 1 + seed/tests/test_property_view_labels.py | 95 ++++++ seed/utils/goals.py | 48 ++- seed/utils/labels.py | 5 +- seed/utils/merge.py | 2 +- seed/views/v3/data_quality_checks.py | 7 +- seed/views/v3/label_inventories.py | 2 +- seed/views/v3/property_view_labels.py | 46 +++ 35 files changed, 1315 insertions(+), 143 deletions(-) create mode 100644 seed/migrations/0222_cross_cycle_data_quality.py create mode 100644 seed/serializers/property_view_labels.py create mode 100644 seed/tests/test_property_view_labels.py create mode 100644 seed/views/v3/property_view_labels.py diff --git a/seed/api/v3/urls.py b/seed/api/v3/urls.py index 0f584807c8..3c8b554ebf 100644 --- a/seed/api/v3/urls.py +++ b/seed/api/v3/urls.py @@ -55,6 +55,7 @@ from seed.views.v3.properties import PropertyViewSet from seed.views.v3.property_measures import PropertyMeasureViewSet from seed.views.v3.property_scenarios import PropertyScenarioViewSet +from seed.views.v3.property_view_labels import PropertyViewLabelViewSet from seed.views.v3.property_views import PropertyViewViewSet from seed.views.v3.public import PublicCycleViewSet, PublicOrganizationViewSet from seed.views.v3.salesforce_configs import SalesforceConfigViewSet @@ -102,6 +103,7 @@ api_v3_router.register(r"postoffice_email", PostOfficeEmailViewSet, basename="postoffice_email") api_v3_router.register(r"progress", ProgressViewSet, basename="progress") api_v3_router.register(r"properties", PropertyViewSet, basename="properties") +api_v3_router.register(r"property_view_labels", PropertyViewLabelViewSet, basename="property_view_labels") api_v3_router.register(r"property_views", PropertyViewViewSet, basename="property_views") api_v3_router.register(r"salesforce_configs", SalesforceConfigViewSet, basename="salesforce_configs") api_v3_router.register(r"salesforce_mappings", SalesforceMappingViewSet, basename="salesforce_mappings") diff --git a/seed/data_importer/tasks.py b/seed/data_importer/tasks.py index 1ff3cc5369..38ce8a5f46 100644 --- a/seed/data_importer/tasks.py +++ b/seed/data_importer/tasks.py @@ -72,6 +72,7 @@ ColumnMapping, Cycle, DataLogger, + Goal, Meter, PropertyAuditLog, PropertyState, @@ -86,6 +87,7 @@ from seed.models.data_quality import DataQualityCheck, Rule from seed.utils.buildings import get_source_type from seed.utils.geocode import MapQuestAPIKeyError, create_geocoded_additional_columns, geocode_buildings +from seed.utils.goals import get_state_pairs from seed.utils.match import update_sub_progress_total from seed.utils.ubid import decode_unique_ids @@ -95,17 +97,26 @@ @shared_task(ignore_result=True) -def check_data_chunk(model, ids, dq_id): +def check_data_chunk(org_id, model, ids, dq_id, goal_id=None): + try: + organization = Organization.objects.get(id=org_id) + super_organization = organization.get_parent() + except Organization.DoesNotExist: + return + if model == "PropertyState": qs = PropertyState.objects.filter(id__in=ids) elif model == "TaxLotState": qs = TaxLotState.objects.filter(id__in=ids) - else: - qs = None - organization = qs.first().organization - super_organization = organization.get_parent() + elif model == "Property" and goal_id: + # return a list of dicts with property, basseline_state, and current_state + state_pairs = get_state_pairs(ids, goal_id) + d = DataQualityCheck.retrieve(super_organization.id) - d.check_data(model, qs.iterator()) + if not goal_id: + d.check_data(model, qs.iterator()) + else: + d.check_data_cross_cycle(goal_id, state_pairs) d.save_to_cache(dq_id, organization.id) @@ -122,13 +133,14 @@ def finish_checking(progress_key): return progress_data.result() -def do_checks(org_id, propertystate_ids, taxlotstate_ids, import_file_id=None): +def do_checks(org_id, propertystate_ids, taxlotstate_ids, goal_id, import_file_id=None): """ Run the dq checks on the data :param org_id: :param propertystate_ids: :param taxlotstate_ids: + :param goal_id: :param import_file_id: int, if present, find the data to check by the import file id :return: """ @@ -151,7 +163,7 @@ def do_checks(org_id, propertystate_ids, taxlotstate_ids, import_file_id=None): .values_list("id", flat=True) ) - tasks = _data_quality_check_create_tasks(org_id, propertystate_ids, taxlotstate_ids, dq_id) + tasks = _data_quality_check_create_tasks(org_id, propertystate_ids, taxlotstate_ids, goal_id, dq_id) progress_data.total = len(tasks) progress_data.save() if tasks: @@ -589,7 +601,7 @@ def _map_data_create_tasks(import_file_id, progress_key): return tasks -def _data_quality_check_create_tasks(org_id, property_state_ids, taxlot_state_ids, dq_id): +def _data_quality_check_create_tasks(org_id, property_state_ids, taxlot_state_ids, goal_id, dq_id): """ Entry point into running data quality checks. @@ -612,12 +624,23 @@ def _data_quality_check_create_tasks(org_id, property_state_ids, taxlot_state_id if property_state_ids: id_chunks = [list(chunk) for chunk in batch(property_state_ids, 100)] for ids in id_chunks: - tasks.append(check_data_chunk.s("PropertyState", ids, dq_id)) + tasks.append(check_data_chunk.s(org_id, "PropertyState", ids, dq_id)) if taxlot_state_ids: id_chunks_tl = [list(chunk) for chunk in batch(taxlot_state_ids, 100)] for ids in id_chunks_tl: - tasks.append(check_data_chunk.s("TaxLotState", ids, dq_id)) + tasks.append(check_data_chunk.s(org_id, "TaxLotState", ids, dq_id)) + + if goal_id: + # If goal_id is passed, treat as a cross cycle data quality check. + try: + goal = Goal.objects.get(id=goal_id) + property_ids = goal.properties().values_list("id", flat=True) + id_chunks = [list(chunk) for chunk in batch(property_ids, 100)] + for ids in id_chunks: + tasks.append(check_data_chunk.s(org_id, "Property", ids, dq_id, goal.id)) + except Goal.DoesNotExist: + pass return tasks diff --git a/seed/migrations/0222_cross_cycle_data_quality.py b/seed/migrations/0222_cross_cycle_data_quality.py new file mode 100644 index 0000000000..de62e890ce --- /dev/null +++ b/seed/migrations/0222_cross_cycle_data_quality.py @@ -0,0 +1,214 @@ +# Generated by Django 3.2.20 on 2023-10-24 21:54 + +import django.db.models.deletion +from django.db import connection, migrations, models + + +def forwards(apps, schema_editor): + Organization = apps.get_model("orgs", "Organization") + Rule = apps.get_model("seed", "Rule") + DataQualityCheck = apps.get_model("seed", "DataQualityCheck") + Label = apps.get_model("seed", "StatusLabel") + + # Populate the default labels for goal rules. + NEW_DEFAULT_LABELS = [ + "High EUI % Change", + "Low EUI % Change", + "High Area", + "Low Area", + "High Area % Change", + "Low Area % Change", + ] + + # Populate the default data quality goal rules if they do not already exist. + TYPE_AREA = 4 + TYPE_EUI = 5 + RULE_TYPE_DEFAULT = 0 + SEVERITY_ERROR = 0 + RULE_NOT_NULL = "not_null" + RULE_RANGE = "range" + + NEW_DEFAULT_RULES = [ + { + "table_name": "Goal", + "name": "High EUI % Change", + "field": "eui", + "data_type": TYPE_EUI, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_RANGE, + "max": 40, + "cross_cycle": True, + }, + { + "table_name": "Goal", + "name": "Low EUI % Change", + "field": "eui", + "data_type": TYPE_EUI, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_RANGE, + "min": -40, + "cross_cycle": True, + }, + { + "table_name": "Goal", + "name": "High Area % Change", + "field": "area", + "data_type": TYPE_AREA, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_RANGE, + "max": 5, + "cross_cycle": True, + }, + { + "table_name": "Goal", + "name": "Low Area % Change", + "field": "area", + "data_type": TYPE_AREA, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_RANGE, + "min": -5, + "cross_cycle": True, + }, + { + "table_name": "Goal", + "name": "High EUI", + "field": "eui", + "data_type": TYPE_EUI, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_RANGE, + "max": 1000, + }, + { + "table_name": "Goal", + "name": "Low EUI", + "field": "eui", + "data_type": TYPE_EUI, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_RANGE, + "min": 40, + }, + { + "table_name": "Goal", + "name": "High Area", + "field": "area", + "data_type": TYPE_AREA, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_RANGE, + "max": 1000000, + }, + { + "table_name": "Goal", + "name": "Low Area", + "field": "area", + "data_type": TYPE_AREA, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_RANGE, + "min": 1000, + }, + { + "table_name": "Goal", + "name": "Missing EUI", + "field": "eui", + "data_type": TYPE_EUI, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_NOT_NULL, + }, + { + "table_name": "Goal", + "name": "Missing Area", + "field": "area", + "data_type": TYPE_AREA, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_NOT_NULL, + }, + ] + + for org in Organization.objects.all(): + for label in NEW_DEFAULT_LABELS: + Label.objects.get_or_create(name=label, super_organization=org, defaults={"color": "blue"}) + for dqc in DataQualityCheck.objects.all(): + for rule in NEW_DEFAULT_RULES: + Rule.objects.get_or_create(**rule, data_quality_check=dqc) + + +def remove_unique_constraint(apps, schema_editor): + # The auto generated unique constraint for PropertyViewLabels is PropertyView_id and StatusLabel_id. + # The constraint needs to be updated to include PropertyView_id, StatusLabel_id, and Goal.id + PropertyViewLabel = apps.get_model("seed", "PropertyViewLabel") + table_name = PropertyViewLabel._meta.db_table + + # Get the original unique constraint + constraints = connection.introspection.get_constraints(connection.cursor(), table_name) + constraint_names = [ + name + for name, details in constraints.items() + if details.get("unique") and set(details.get("columns")) == {"propertyview_id", "statuslabel_id"} + ] + # Remove the constraint + if constraint_names: + with connection.cursor() as cursor: + cursor.execute(f"ALTER TABLE {table_name} DROP CONSTRAINT {constraint_names[0]};") + # A new unique constraint will be created in the following operation + + +class Migration(migrations.Migration): + dependencies = [ + ("seed", "0221_audittemplateconfig"), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + database_operations=[ + migrations.RunSQL( + sql="ALTER TABLE seed_propertyview_labels RENAME TO seed_propertyviewlabel", + reverse_sql="ALTER TABLE seed_propertyviewlabel RENAME TO seed_propertyview_labels", + ), + ], + state_operations=[ + migrations.CreateModel( + name="PropertyViewLabel", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("propertyview", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="seed.propertyview")), + ("statuslabel", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="seed.statuslabel")), + ], + ), + migrations.AlterField( + model_name="propertyview", + name="labels", + field=models.ManyToManyField(through="seed.PropertyViewLabel", to="seed.StatusLabel"), + ), + ], + ), + migrations.AddField( + model_name="propertyviewlabel", + name="goal", + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to="seed.goal"), + ), + migrations.AlterField( + model_name="rule", + name="field", + field=models.CharField(blank=True, max_length=200, null=True), + ), + migrations.AddField( + model_name="rule", + name="cross_cycle", + field=models.BooleanField(default=False), + ), + migrations.RunPython(remove_unique_constraint), + migrations.AddConstraint( + model_name="propertyviewlabel", + constraint=models.UniqueConstraint(fields=("propertyview", "statuslabel", "goal"), name="unique_propertyview_statuslabel_goal"), + ), + migrations.RunPython(forwards), + ] diff --git a/seed/models/columns.py b/seed/models/columns.py index ca2ed13241..0bb76220e4 100644 --- a/seed/models/columns.py +++ b/seed/models/columns.py @@ -458,7 +458,7 @@ class Column(models.Model): "display_name": "Gross Floor Area", "column_description": "Gross Floor Area", "data_type": "area", - # 'type': 'number', + # "type": "number", }, { "column_name": "use_description", @@ -473,7 +473,7 @@ class Column(models.Model): "display_name": "ENERGY STAR Score", "column_description": "ENERGY STAR Score", "data_type": "integer", - # 'type': 'number', + # "type": "number", }, { "column_name": "property_notes", @@ -523,7 +523,7 @@ class Column(models.Model): "display_name": "Building Count", "column_description": "Building Count", "data_type": "integer", - # 'type': 'number', + # "type": "number", }, { "column_name": "year_built", @@ -531,7 +531,7 @@ class Column(models.Model): "display_name": "Year Built", "column_description": "Year Built", "data_type": "integer", - # 'type': 'number', + # "type": "number", }, { "column_name": "recent_sale_date", @@ -548,7 +548,7 @@ class Column(models.Model): "display_name": "Conditioned Floor Area", "column_description": "Conditioned Floor Area", "data_type": "area", - # 'type': 'number', + # "type": "number", # 'dbField': True, }, { @@ -557,7 +557,7 @@ class Column(models.Model): "display_name": "Occupied Floor Area", "column_description": "Occupied Floor Area", "data_type": "area", - # 'type': 'number', + # "type": "number", }, { "column_name": "owner_address", @@ -611,7 +611,7 @@ class Column(models.Model): "display_name": "Site EUI", "column_description": "Site EUI", "data_type": "eui", - # 'type': 'number', + # "type": "number", }, { "column_name": "site_eui_weather_normalized", @@ -619,7 +619,7 @@ class Column(models.Model): "display_name": "Site EUI Weather Normalized", "column_description": "Site EUI Weather Normalized", "data_type": "eui", - # 'type': 'number', + # "type": "number", }, { "column_name": "site_eui_modeled", @@ -627,7 +627,7 @@ class Column(models.Model): "display_name": "Site EUI Modeled", "column_description": "Site EUI Modeled", "data_type": "eui", - # 'type': 'number', + # "type": "number", }, { "column_name": "source_eui", @@ -635,7 +635,7 @@ class Column(models.Model): "display_name": "Source EUI", "column_description": "Source EUI", "data_type": "eui", - # 'type': 'number', + # "type": "number", }, { "column_name": "source_eui_weather_normalized", @@ -643,7 +643,7 @@ class Column(models.Model): "display_name": "Source EUI Weather Normalized", "column_description": "Source EUI Weather Normalized", "data_type": "eui", - # 'type': 'number', + # "type": "number", }, { "column_name": "source_eui_modeled", @@ -651,7 +651,7 @@ class Column(models.Model): "display_name": "Source EUI Modeled", "column_description": "Source EUI Modeled", "data_type": "eui", - # 'type': 'number', + # "type": "number", }, { "column_name": "energy_alerts", @@ -680,7 +680,7 @@ class Column(models.Model): "display_name": "Number Properties", "column_description": "Number Properties", "data_type": "integer", - # 'type': 'number', + # "type": "number", }, { "column_name": "block_number", diff --git a/seed/models/data_quality.py b/seed/models/data_quality.py index 68c85ad0eb..a761cbf47e 100644 --- a/seed/models/data_quality.py +++ b/seed/models/data_quality.py @@ -7,6 +7,7 @@ import json import logging import re +from collections import defaultdict from datetime import date, datetime from random import randint @@ -19,9 +20,10 @@ from quantityfield.units import ureg from seed.lib.superperms.orgs.models import Organization -from seed.models import Column, DerivedColumn, PropertyView, StatusLabel, TaxLotView, obj_to_dict +from seed.models import Column, DerivedColumn, GoalNote, PropertyView, PropertyViewLabel, StatusLabel, TaxLotView, obj_to_dict from seed.serializers.pint import pretty_units from seed.utils.cache import get_cache_raw, set_cache_raw +from seed.utils.goals import get_area_value, get_eui_value, percentage_difference from seed.utils.time import convert_datestr _log = logging.getLogger(__name__) @@ -332,14 +334,117 @@ class Rule(models.Model): "severity": SEVERITY_ERROR, "condition": RULE_RANGE, }, + { + "table_name": "Goal", + "name": "High EUI % Change", + "field": "eui", + "data_type": TYPE_EUI, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_RANGE, + "max": 40, + "cross_cycle": True, + }, + { + "table_name": "Goal", + "name": "Low EUI % Change", + "field": "eui", + "data_type": TYPE_EUI, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_RANGE, + "min": -40, + "cross_cycle": True, + }, + { + "table_name": "Goal", + "name": "High Area % Change", + "field": "area", + "data_type": TYPE_AREA, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_RANGE, + "max": 5, + "cross_cycle": True, + }, + { + "table_name": "Goal", + "name": "Low Area % Change", + "field": "area", + "data_type": TYPE_AREA, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_RANGE, + "min": -5, + "cross_cycle": True, + }, + { + "table_name": "Goal", + "name": "High EUI", + "field": "eui", + "data_type": TYPE_EUI, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_RANGE, + "max": 1000, + }, + { + "table_name": "Goal", + "name": "Low EUI", + "field": "eui", + "data_type": TYPE_EUI, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_RANGE, + "min": 40, + }, + { + "table_name": "Goal", + "name": "High Area", + "field": "area", + "data_type": TYPE_AREA, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_RANGE, + "max": 1000000, + }, + { + "table_name": "Goal", + "name": "Low Area", + "field": "area", + "data_type": TYPE_AREA, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_RANGE, + "min": 1000, + }, + { + "table_name": "Goal", + "name": "Missing EUI", + "field": "eui", + "data_type": TYPE_EUI, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_NOT_NULL, + }, + { + "table_name": "Goal", + "name": "Missing Area", + "field": "area", + "data_type": TYPE_AREA, + "rule_type": RULE_TYPE_DEFAULT, + "severity": SEVERITY_ERROR, + "condition": RULE_NOT_NULL, + }, ] name = models.CharField(max_length=255, blank=True) + cross_cycle = models.BooleanField(default=False) description = models.CharField(max_length=1000, blank=True) data_quality_check = models.ForeignKey("DataQualityCheck", on_delete=models.CASCADE, related_name="rules", null=True) status_label = models.ForeignKey(StatusLabel, on_delete=models.DO_NOTHING, null=True) table_name = models.CharField(max_length=200, default="PropertyState", blank=True) - field = models.CharField(max_length=200) + field = models.CharField(max_length=200, null=True, blank=True) # If for_derived_column is True, `Rule.field` is a derived column name # If False, `Rule.field` is a *State field or extra_data key for_derived_column = models.BooleanField(default=False) @@ -646,16 +751,51 @@ def check_data(self, record_type, rows): for row in rows: # Initialize the ID if it does not exist yet. Add in the other # fields that are of interest to the GUI - if row.id not in self.results: - self.results[row.id] = {} - for field in fields: - self.results[row.id][field] = getattr(row, field) - self.results[row.id]["data_quality_results"] = [] + self.init_result(record_type, row, fields) # Run the checks self._check(rules, row, derived_columns_by_name) # Prune the results will remove any entries that have zero data_quality_results + self.prune_results() + + def check_data_cross_cycle(self, goal_id, state_pairs): + """ + Send in data as properties and their goal state pairs. Update GoalNote.passed_checks with data quality check return + :param goal: + :param state_pairs: [{property: Property, baseline: PropertyState, current: PropertyState}, {}, ...] + """ + rules = self.rules.filter(enabled=True, table_name="Goal") + goal_notes = GoalNote.objects.filter(goal=goal_id) + goal_notes = {note.property.id: note for note in goal_notes} + goal_notes_to_update = [] + fields = self.get_fieldnames("PropertyState") + for row in state_pairs: + for cycle_key in ["baseline", "current"]: + self.init_result("PropertyState", row[cycle_key], fields) + + goal_note = self._check_cross_cycle(rules, row, goal_notes) + if goal_note: + goal_notes_to_update.append(goal_note) + + self.prune_results() + + GoalNote.objects.bulk_update(goal_notes_to_update, ["passed_checks"]) + + def init_result(self, record_type, row, fields): + # Initialize the ID if it does not exist yet. Add in the other + # fields that are of interest to the GUI + if row and row.id not in self.results: + self.results[row.id] = {} + for field in fields: + self.results[row.id][field] = getattr(row, field) + view = row.taxlotview_set.first() if record_type == "TaxLotState" else row.propertyview_set.first() + if view: + self.results[row.id]["cycle"] = view.cycle.name + self.results[row.id]["data_quality_results"] = [] + + def prune_results(self): + # prune_results will remove any entries that have zero data_quality_results for k, v in self.results.copy().items(): if not v["data_quality_results"]: del self.results[k] @@ -681,7 +821,7 @@ def _check(self, rules, row, derived_columns_by_name): # check if the row has any rules applied to it model_labels = {"linked_id": None, "label_ids": []} if row.__class__.__name__ == "PropertyState": - label = apps.get_model("seed", "PropertyView_labels") + label = apps.get_model("seed", "PropertyViewLabel") if PropertyView.objects.filter(state=row).exists(): model_labels["linked_id"] = PropertyView.objects.get(state=row).id model_labels["label_ids"] = list( @@ -793,6 +933,108 @@ def _check(self, rules, row, derived_columns_by_name): if not label_applied and rule.status_label_id in model_labels["label_ids"]: self.remove_status_label(label, rule, linked_id) + def _check_cross_cycle(self, rules, row, goal_notes): + """ + Check if property percent changes are within tolerance. Update GoalNote.passed_checks with result + :param rules: list, rules to run from database objects + :param row: { property: Property, baseline: PropertyState, current PropertyState } + :goal_notes: dictionary of { property_id: GoalNote, ... } + """ + apply_labels = {"baseline": defaultdict(list), "current": defaultdict(list)} + results = [] + property_id = row["property"].id + goal_note = goal_notes.get(property_id) + if not goal_note or not row["current"]: + # NEED TO MAKE SURE MISSING DATA IS APPLIED + return + goal = goal_note.goal + baseline_view = row["baseline"].propertyview_set.first() if row["baseline"] else None + current_view = row["current"].propertyview_set.first() if row["current"] else None + + def check_range(): + return (rule.min is None or value > rule.min) and (rule.max is None or value < rule.max) + + def append_to_apply_labels(): + if rule.status_label: + apply_labels[cycle_key][rule.status_label.id].append(result) + + for rule in rules: + result = None + data_type = rule.DATA_TYPES[rule.data_type][1] + baseline = self.get_value(row, data_type, goal, "baseline") + current = self.get_value(row, data_type, goal, "current") + # EUI is inverese as a drop in EUI is an improvement + cycle_values = [baseline, current] if data_type == "eui" else [current, baseline] + + if rule.cross_cycle: + cycle_key = "current" + value = percentage_difference(*cycle_values) + if value is None: + continue + if rule.condition == Rule.RULE_RANGE: + result = check_range() + results.append(result) + append_to_apply_labels() + if not result: + self.add_result_range_error(row["current"].id, rule, data_type, value) + self.update_status_label(PropertyViewLabel, rule, current_view.id, row["current"].id) + + # other rule condition types + else: + logging.error(">>> OTHER") + + else: # Within Cycle + for cycle_key in ["baseline", "current"]: + state = row["baseline"] if cycle_key == "baseline" else row["current"] + view = baseline_view if cycle_key == "baseline" else current_view + if not state: + return + value = baseline if cycle_key == "baseline" else current + if rule.condition == rule.RULE_RANGE: + if value: + result = check_range() + results.append(result) + append_to_apply_labels() + if not result: + self.add_result_range_error(state.id, rule, data_type, value) + self.update_status_label(PropertyViewLabel, rule, view.id, state.id) + + elif rule.condition == rule.RULE_NOT_NULL: + result = value is not None + results.append(result) + append_to_apply_labels() + if not result: + self.add_result_is_null(state.id, rule, data_type, value) + self.update_status_label(PropertyViewLabel, rule, view.id, state.id) + + # other rule condition types. + else: + logging.error(">>> OTHER") + + goal_note.passed_checks = all(results) + + # if there are multiple rules with the same label, determine if they are all passing to add or remove the label + for cycle_key in ["baseline", "current"]: + view = baseline_view if cycle_key == "baseline" else current_view + + for id, results in apply_labels[cycle_key].items(): + label = StatusLabel.objects.get(id=id) + property_view_label = PropertyViewLabel.objects.filter(propertyview=view, statuslabel=label, goal=goal) + if all(results): + property_view_label.delete() + elif not property_view_label: + PropertyViewLabel.objects.create(propertyview=view, statuslabel=label, goal=goal) + + return goal_note + + def get_value(self, property_obj, data_type, goal, cycle_key): + if not property_obj[cycle_key]: + return None + if data_type == "area": + return get_area_value(property_obj[cycle_key], goal) + elif data_type == "eui": + return get_eui_value(property_obj[cycle_key], goal) + def save_to_cache(self, identifier, organization_id): """ Save the results to the cache database. The data in the cache are @@ -823,7 +1065,22 @@ def initialize_rules(self): :return: None """ for rule in Rule.DEFAULT_RULES: - self.rules.add(Rule.objects.create(**rule)) + rule_instance = Rule.objects.create(**rule) + self.rules.add(rule_instance) + self.add_goal_rule_labels(rule_instance) + + def add_goal_rule_labels(self, rule): + # Add labels to default portfolio/goal rules + if rule: + org = rule.data_quality_check.organization + try: + if "Missing" in rule.name: + rule.status_label = StatusLabel.objects.get(name="Missing Data", super_organization=org) + else: + rule.status_label = StatusLabel.objects.get(name=rule.name, super_organization=org) + rule.save() + except StatusLabel.DoesNotExist: + pass def remove_all_rules(self): """ @@ -928,6 +1185,20 @@ def add_result_max_error(self, row_id, rule, display_name, value, rule_max): } ) + def add_result_range_error(self, row_id, rule, display_name, value): + self.results[row_id]["data_quality_results"].append( + { + "field": rule.field, + "formatted_field": display_name, + "value": value, + "table_name": rule.table_name, + "message": f"{display_name} out of range", + "detailed_message": f"{display_name} [ {value} ] outside range {rule.min} to {rule.max}", + "severity": rule.get_severity_display(), + "condition": rule.condition, + } + ) + def add_result_comparison_error(self, row_id, rule, display_name, value, rule_check): self.results[row_id]["data_quality_results"].append( { @@ -1054,7 +1325,7 @@ def update_status_label(self, label_class, rule, linked_id, row_id, add_to_resul f"Label with super_organization_id={label_org_id} cannot be applied to a record with parent " f"organization_id={property_parent_org_id}." ) - else: + elif rule.table_name == "TaxLotState": taxlot_parent_org_id = TaxLotView.objects.get(pk=linked_id).taxlot.organization.get_parent().id if taxlot_parent_org_id == label_org_id: label_class.objects.get_or_create(taxlotview_id=linked_id, statuslabel_id=rule.status_label_id) diff --git a/seed/models/models.py b/seed/models/models.py index f745f76831..9b836d028a 100644 --- a/seed/models/models.py +++ b/seed/models/models.py @@ -118,11 +118,17 @@ class StatusLabel(TimeStampedModel): "Update Bldg Info", "Call", "Email", - "High EUI", - "Low EUI", "Exempted", "Extension", "Change of Ownership", + "High EUI", + "Low EUI", + "High EUI % Change", + "Low EUI % Change", + "High Area", + "Low Area", + "High Area % Change", + "Low Area % Change", ] class Meta: diff --git a/seed/models/properties.py b/seed/models/properties.py index 1d2a7396e8..978744352c 100644 --- a/seed/models/properties.py +++ b/seed/models/properties.py @@ -13,7 +13,7 @@ from django.contrib.gis.db import models as geomodels from django.core.exceptions import ValidationError from django.db import IntegrityError, models, transaction -from django.db.models import Case, Value, When +from django.db.models import Case, UniqueConstraint, Value, When from django.db.models.signals import m2m_changed, post_save, pre_delete, pre_save from django.dispatch import receiver from django.forms.models import model_to_dict @@ -22,8 +22,6 @@ from quantityfield.units import ureg from seed.data_importer.models import ImportFile - -# from seed.utils.cprofile import cprofile from seed.lib.mcm.cleaners import date_cleaner from seed.lib.superperms.orgs.models import AccessLevelInstance, Organization from seed.models.cycles import Cycle @@ -78,7 +76,7 @@ class Meta: verbose_name_plural = "properties" def __str__(self): - return "Property - %s" % (self.pk) + return f"Property - {self.pk}" def copy_meters(self, source_property_id, source_persists=True): """ @@ -881,7 +879,6 @@ class PropertyView(models.Model): A PropertyView contains a reference to a property (which should not change) and to a cycle (time period), and a state (characteristics). - """ # different property views can be associated with each other (2012, 2013) @@ -889,12 +886,12 @@ class PropertyView(models.Model): cycle = models.ForeignKey(Cycle, on_delete=models.PROTECT) state = models.ForeignKey(PropertyState, on_delete=models.CASCADE) - labels = models.ManyToManyField(StatusLabel) + labels = models.ManyToManyField(StatusLabel, through="PropertyViewLabel", through_fields=("propertyview", "statuslabel")) # notes has a relationship here -- PropertyViews have notes, not the state, and not the property. def __str__(self): - return "Property View - %s" % self.pk + return f"Property View - {self.pk}" class Meta: unique_together = ( @@ -958,6 +955,15 @@ def post_save_property_view(sender, **kwargs): kwargs["instance"].property.save() +class PropertyViewLabel(models.Model): + propertyview = models.ForeignKey(PropertyView, on_delete=models.CASCADE) + statuslabel = models.ForeignKey(StatusLabel, on_delete=models.CASCADE) + goal = models.ForeignKey("seed.Goal", on_delete=models.CASCADE, null=True) + + class Meta: + constraints = [UniqueConstraint(fields=["propertyview", "statuslabel", "goal"], name="unique_propertyview_statuslabel_goal")] + + class PropertyAuditLog(models.Model): organization = models.ForeignKey(Organization, on_delete=models.CASCADE) parent1 = models.ForeignKey( diff --git a/seed/serializers/property_view_labels.py b/seed/serializers/property_view_labels.py new file mode 100644 index 0000000000..d84abf1554 --- /dev/null +++ b/seed/serializers/property_view_labels.py @@ -0,0 +1,21 @@ +""" +SEED Platform (TM), Copyright (c) Alliance for Sustainable Energy, LLC, and other contributors. +See also https://github.com/SEED-platform/seed/blob/main/LICENSE.md +""" + +from rest_framework import serializers + +from seed.models import PropertyViewLabel + + +class PropertyViewLabelSerializer(serializers.ModelSerializer): + class Meta: + model = PropertyViewLabel + fields = "__all__" + + def to_representation(self, obj): + result = super().to_representation(obj) + result["show_in_list"] = obj.statuslabel.show_in_list + result["color"] = obj.statuslabel.color + result["name"] = obj.statuslabel.name + return result diff --git a/seed/serializers/rules.py b/seed/serializers/rules.py index 701a93d8fd..eabce8ae9a 100644 --- a/seed/serializers/rules.py +++ b/seed/serializers/rules.py @@ -17,9 +17,11 @@ class Meta: model = Rule fields = [ "condition", + "cross_cycle", "data_type", "enabled", "field", + "for_derived_column", "id", "max", "min", @@ -31,7 +33,6 @@ class Meta: "table_name", "text_match", "units", - "for_derived_column", ] def create(self, validated_data): @@ -78,6 +79,9 @@ def validate(self, data): # Rule is new severity_is_valid = False label_is_not_associated = False + if data.get("table_name") == "Goal": + # prevent new Goal type rules from being created + raise serializers.ValidationError({"message": "Creating new Goal Rules has been disabled"}) else: severity_is_valid = self.instance.severity == Rule.SEVERITY_VALID label_is_not_associated = self.instance.status_label is None diff --git a/seed/static/seed/js/controllers/data_quality_admin_controller.js b/seed/static/seed/js/controllers/data_quality_admin_controller.js index eec74f4d04..e88bebd69a 100644 --- a/seed/static/seed/js/controllers/data_quality_admin_controller.js +++ b/seed/static/seed/js/controllers/data_quality_admin_controller.js @@ -48,7 +48,7 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit naturalSort, $translate ) { - $scope.inventory_type = $stateParams.inventory_type; + $scope.rule_type = $stateParams.rule_type; $scope.org = organization_payload.organization; $scope.auth = auth_payload.auth; $scope.ruleGroups = {}; @@ -56,6 +56,7 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit $scope.state = $state.current; $scope.rule_count_property = 0; $scope.rule_count_taxlot = 0; + $scope.rule_count_cross_cycle = 0; $scope.conditions = [ { id: 'required', label: 'Required' }, @@ -117,12 +118,18 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit { id: 'ft**2', label: 'square feet' }, { id: 'm**2', label: 'square metres' }, { id: 'kBtu/ft**2/year', label: 'kBtu/sq. ft./year' }, + { id: 'gal/ft**2/year', label: 'gal/sq. ft./year' }, { id: 'GJ/m**2/year', label: 'GJ/m²/year' }, { id: 'MJ/m**2/year', label: 'MJ/m²/year' }, { id: 'kWh/m**2/year', label: 'kWh/m²/year' }, { id: 'kBtu/m**2/year', label: 'kBtu/m²/year' } ]; + $scope.cross_cycle_options = [ + { bool: true, label: '% Change aross cycle' }, + { bool: false, label: 'Within cycle' } + ]; + $scope.columns = _.map(angular.copy(columns.filter((col) => !col.derived_column)), (col) => { if (!_.find(used_columns, ['id', col.id])) { col.group = 'Not Mapped'; @@ -147,19 +154,20 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit const loadRules = (rules_payload) => { const ruleGroups = { properties: {}, - taxlots: {} + taxlots: {}, + goals: {} }; - let inventory_type; + let rule_type; _.forEach(rules_payload, (rule) => { if (rule.table_name === 'PropertyState') { - inventory_type = 'properties'; + rule_type = 'properties'; } else if (rule.table_name === 'TaxLotState') { - inventory_type = 'taxlots'; + rule_type = 'taxlots'; } else { - inventory_type = rule.table_name; + rule_type = 'goals'; } - if (!_.has(ruleGroups[inventory_type], rule.field)) ruleGroups[inventory_type][rule.field] = []; + if (!_.has(ruleGroups[rule_type], rule.field)) ruleGroups[rule_type][rule.field] = []; const row = rule; if (row.data_type === $scope.data_type_keys.date) { if (row.min) row.min = moment(row.min, 'YYYYMMDD').toDate(); @@ -171,18 +179,22 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit row.label = match; } } - ruleGroups[inventory_type][rule.field].push(row); + ruleGroups[rule_type][rule.field].push(row); }); $scope.ruleGroups = ruleGroups; $scope.rule_count_property = 0; $scope.rule_count_taxlot = 0; + $scope.rule_count_cross_cycle = 0; _.map($scope.ruleGroups.properties, (rule) => { $scope.rule_count_property += rule.length; }); _.map($scope.ruleGroups.taxlots, (rule) => { $scope.rule_count_taxlot += rule.length; }); + _.map($scope.ruleGroups.goals, (rule) => { + $scope.rule_count_cross_cycle += rule.length; + }); }; loadRules(data_quality_rules_payload); @@ -224,7 +236,7 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit const rules = []; const misconfigured_rules = []; $scope.duplicate_rule_keys = []; - _.forEach($scope.ruleGroups, (ruleGroups, inventory_type) => { + _.forEach($scope.ruleGroups, (ruleGroups, rule_type) => { _.forEach(ruleGroups, (ruleGroup) => { const duplicate_rules = _.groupBy( ruleGroup, @@ -244,6 +256,7 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit if (rule.field === null) return; const column = _.find($scope.columns, { column_name: rule.field }) || {}; + const rule_type_lookup = { properties: 'PropertyState', taxlots: 'TaxLotState', goals: 'Goal' }; const r = { enabled: rule.enabled, condition: rule.condition, @@ -255,10 +268,11 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit not_null: rule.not_null, min: rule.min, max: rule.max, - table_name: inventory_type === 'properties' ? 'PropertyState' : 'TaxLotState', + table_name: rule_type_lookup[rule_type], text_match: rule.text_match, severity: rule.severity, units: rule.units, + target: rule.target, status_label: null, for_derived_column: !!column.is_derived }; @@ -451,7 +465,7 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit }; // perform checks on load - _.forEach($scope.ruleGroups[$scope.inventory_type], (group, group_name) => { + _.forEach($scope.ruleGroups[$scope.rule_type], (group, group_name) => { $scope.validate_data_types(group, group_name); $scope.validate_conditions(group, group_name); }); @@ -475,7 +489,7 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit } const group_name = rule.field; - const group = $scope.ruleGroups[$scope.inventory_type][group_name]; + const group = $scope.ruleGroups[$scope.rule_type][group_name]; $scope.validate_conditions(group, group_name); $scope.validate_data_types(group, group_name); }; @@ -509,33 +523,33 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit rule.data_type = newDataType; // move rule to appropriate spot in ruleGroups. - if (!_.has($scope.ruleGroups[$scope.inventory_type], rule.field)) { - $scope.ruleGroups[$scope.inventory_type][rule.field] = []; + if (!_.has($scope.ruleGroups[$scope.rule_type], rule.field)) { + $scope.ruleGroups[$scope.rule_type][rule.field] = []; } else { // Rules already exist for the new field name, so match the data_type, required, and not_null columns - const existingRule = _.first($scope.ruleGroups[$scope.inventory_type][rule.field]); + const existingRule = _.first($scope.ruleGroups[$scope.rule_type][rule.field]); rule.data_type = existingRule.data_type; rule.required = existingRule.required; rule.not_null = existingRule.not_null; } - $scope.ruleGroups[$scope.inventory_type][rule.field].push(rule); + $scope.ruleGroups[$scope.rule_type][rule.field].push(rule); // remove old rule. - if ($scope.ruleGroups[$scope.inventory_type][oldField].length === 1) delete $scope.ruleGroups[$scope.inventory_type][oldField]; - else $scope.ruleGroups[$scope.inventory_type][oldField].splice(index, 1); + if ($scope.ruleGroups[$scope.rule_type][oldField].length === 1) delete $scope.ruleGroups[$scope.rule_type][oldField]; + else $scope.ruleGroups[$scope.rule_type][oldField].splice(index, 1); rule.autofocus = true; const group_name = rule.field; - const group = $scope.ruleGroups[$scope.inventory_type][group_name]; + const group = $scope.ruleGroups[$scope.rule_type][group_name]; $scope.validate_data_types(group, group_name); $scope.validate_conditions(group, group_name); - $scope.validate_data_types($scope.ruleGroups[$scope.inventory_type][oldField], oldField); - $scope.validate_conditions($scope.ruleGroups[$scope.inventory_type][oldField], oldField); + $scope.validate_data_types($scope.ruleGroups[$scope.rule_type][oldField], oldField); + $scope.validate_conditions($scope.ruleGroups[$scope.rule_type][oldField], oldField); }; // Keep field types consistent for identical fields $scope.change_data_type = (rule, oldValue) => { const { data_type } = rule; - const rule_group = $scope.ruleGroups[$scope.inventory_type][rule.field]; + const rule_group = $scope.ruleGroups[$scope.rule_type][rule.field]; _.forEach(rule_group, (currentRule) => { currentRule.text_match = null; @@ -558,10 +572,10 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit // create a new rule. $scope.create_new_rule = () => { const field = null; - if (!_.has($scope.ruleGroups[$scope.inventory_type], field)) { - $scope.ruleGroups[$scope.inventory_type][field] = []; + if (!_.has($scope.ruleGroups[$scope.rule_type], field)) { + $scope.ruleGroups[$scope.rule_type][field] = []; } - $scope.ruleGroups[$scope.inventory_type][field].push({ + $scope.ruleGroups[$scope.rule_type][field].push({ enabled: true, condition: '', field, @@ -581,8 +595,9 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit autofocus: true }); $scope.rules_changed(); - if ($scope.inventory_type === 'properties') $scope.rule_count_property += 1; - else $scope.rule_count_taxlot += 1; + if ($scope.rule_type === 'properties') $scope.rule_count_property += 1; + else if ($scope.rule_type === 'taxlots') $scope.rule_count_taxlot += 1; + else $scope.rule_count_cross_cycle += 1; }; // create label and assign to that rule @@ -608,12 +623,13 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit // set rule as deleted. $scope.delete_rule = (rule, index) => { - if ($scope.ruleGroups[$scope.inventory_type][rule.field].length === 1) { - delete $scope.ruleGroups[$scope.inventory_type][rule.field]; - } else $scope.ruleGroups[$scope.inventory_type][rule.field].splice(index, 1); + if ($scope.ruleGroups[$scope.rule_type][rule.field].length === 1) { + delete $scope.ruleGroups[$scope.rule_type][rule.field]; + } else $scope.ruleGroups[$scope.rule_type][rule.field].splice(index, 1); $scope.rules_changed(); - if ($scope.inventory_type === 'properties') $scope.rule_count_property -= 1; - else $scope.rule_count_taxlot -= 1; + if ($scope.rule_type === 'properties') $scope.rule_count_property -= 1; + else if ($scope.rule_type === 'taxlots') $scope.rule_count_taxlot -= 1; + else $scope.rule_count_cross_cycle -= 1; }; const displayNames = {}; @@ -623,7 +639,7 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit }); $scope.sortedRuleGroups = () => { - const sortedKeys = _.keys($scope.ruleGroups[$scope.inventory_type]).sort((a, b) => naturalSort(displayNames[a], displayNames[b])); + const sortedKeys = _.keys($scope.ruleGroups[$scope.rule_type]).sort((a, b) => naturalSort(displayNames[a], displayNames[b])); const nullKey = _.remove(sortedKeys, (key) => key === 'null'); // Put created unassigned rows first @@ -634,7 +650,7 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit $scope.rules_changed(); const allEnabled = $scope.allEnabled(); - _.forEach($scope.ruleGroups[$scope.inventory_type], (ruleGroup) => { + _.forEach($scope.ruleGroups[$scope.rule_type], (ruleGroup) => { _.forEach(ruleGroup, (rule) => { rule.enabled = !allEnabled; }); @@ -644,7 +660,7 @@ angular.module('SEED.controller.data_quality_admin', []).controller('data_qualit $scope.allEnabled = () => { let total = 0; const enabled = _.reduce( - $scope.ruleGroups[$scope.inventory_type], + $scope.ruleGroups[$scope.rule_type], (result, ruleGroup) => { total += ruleGroup.length; return result + _.filter(ruleGroup, 'enabled').length; diff --git a/seed/static/seed/js/controllers/data_quality_modal_controller.js b/seed/static/seed/js/controllers/data_quality_modal_controller.js index aff55f10a7..1eab6a6c00 100644 --- a/seed/static/seed/js/controllers/data_quality_modal_controller.js +++ b/seed/static/seed/js/controllers/data_quality_modal_controller.js @@ -39,6 +39,11 @@ angular.module('SEED.controller.data_quality_modal', []).controller('data_qualit sortable: false, title: 'Table' }, + { + sort_column: 'cycle', + sortable: true, + title: 'Cycle' + }, { sort_column: 'address_line_1', sortable: true, diff --git a/seed/static/seed/js/controllers/delete_modal_controller.js b/seed/static/seed/js/controllers/delete_modal_controller.js index 6e54becbfe..8cb08d4899 100644 --- a/seed/static/seed/js/controllers/delete_modal_controller.js +++ b/seed/static/seed/js/controllers/delete_modal_controller.js @@ -43,7 +43,6 @@ angular.module('SEED.controller.delete_modal', []).controller('delete_modal_cont $scope.delete_state = 'success'; }) .catch((resp) => { - console.log('resp', resp); $scope.delete_state = 'fail'; if (resp.status === 422) { $scope.error = resp.data.message; diff --git a/seed/static/seed/js/controllers/inventory_list_controller.js b/seed/static/seed/js/controllers/inventory_list_controller.js index eef75728b7..8d2090c22e 100644 --- a/seed/static/seed/js/controllers/inventory_list_controller.js +++ b/seed/static/seed/js/controllers/inventory_list_controller.js @@ -749,7 +749,7 @@ angular.module('SEED.controller.inventory_list', []).controller('inventory_list_ const property_view_ids = $scope.inventory_type === 'properties' ? selectedViewIds : []; const taxlot_view_ids = $scope.inventory_type === 'taxlots' ? selectedViewIds : []; - data_quality_service.start_data_quality_checks(property_view_ids, taxlot_view_ids).then((response) => { + data_quality_service.start_data_quality_checks(property_view_ids, taxlot_view_ids, null).then((response) => { data_quality_service .data_quality_checks_status(response.progress_key) .then((result) => { diff --git a/seed/static/seed/js/controllers/portfolio_summary_controller.js b/seed/static/seed/js/controllers/portfolio_summary_controller.js index 1e06e8b45b..9b6740af16 100644 --- a/seed/static/seed/js/controllers/portfolio_summary_controller.js +++ b/seed/static/seed/js/controllers/portfolio_summary_controller.js @@ -11,9 +11,11 @@ angular.module('SEED.controller.portfolio_summary', []) '$window', 'urls', 'ah_service', + 'data_quality_service', 'inventory_service', 'label_service', 'goal_service', + 'Notification', 'cycles', 'organization_payload', 'access_level_tree', @@ -31,9 +33,11 @@ angular.module('SEED.controller.portfolio_summary', []) $window, urls, ah_service, + data_quality_service, inventory_service, label_service, goal_service, + Notification, cycles, organization_payload, access_level_tree, @@ -142,8 +146,15 @@ angular.module('SEED.controller.portfolio_summary', []) } }; + $scope.toggle_help = (bool) => { + $scope.show_help = bool; + _.delay($scope.updateHeight, 150); + }; + const get_goal_stats = (summary) => { const passing_sqft = summary.current ? summary.current.total_sqft : null; + // show help text if less than {50}% of properties are passing checks + $scope.show_help = summary.total_passing <= summary.total_properties * 0.5; $scope.goal_stats = [ { name: 'Commitment (Sq. Ft)', value: $scope.goal.commitment_sqft }, { name: 'Shared (Sq. Ft)', value: summary.shared_sqft }, @@ -284,11 +295,11 @@ angular.module('SEED.controller.portfolio_summary', []) $scope.max_label_width = 750; $scope.get_label_column_width = (labels_col, key) => { - if (!$scope.show_full_labels[key]) { + const renderContainer = document.body.getElementsByClassName('ui-grid-render-container-body')[1]; + if (!$scope.show_full_labels[key] || !renderContainer) { return 31; } let maxWidth = 0; - const renderContainer = document.body.getElementsByClassName('ui-grid-render-container-body')[1]; const col = $scope.gridApi.grid.getColumn(labels_col); const cells = renderContainer.querySelectorAll(`.${uiGridConstants.COL_CLASS_PREFIX}${col.uid} .ui-grid-cell-contents`); Array.prototype.forEach.call(cells, (cell) => { @@ -318,17 +329,9 @@ angular.module('SEED.controller.portfolio_summary', []) }, 0); }; - // retrieve labels for cycle + // retrieve labels, key = 'baseline' or 'current' const get_labels = (key) => { - const cycle = key === 'baseline' ? $scope.goal.baseline_cycle : $scope.goal.current_cycle; - - label_service.get_labels('properties', undefined, cycle).then((current_labels) => { - const labels = _.filter(current_labels, (label) => !_.isEmpty(label.is_applied)); - - // load saved label filter - // const ids = inventory_service.loadSelectedLabels('grid.properties.labels'); - // $scope.selected_labels = _.filter(labels, (label) => _.includes(ids, label.id)); - + label_service.get_property_view_labels_by_goal($scope.organization.id, $scope.goal.id, key).then((labels) => { if (key === 'baseline') { $scope.baseline_labels = labels; $scope.build_labels(key, $scope.baseline_labels); @@ -338,6 +341,7 @@ angular.module('SEED.controller.portfolio_summary', []) } }); }; + const get_all_labels = () => { get_labels('baseline'); get_labels('current'); @@ -349,16 +353,11 @@ angular.module('SEED.controller.portfolio_summary', []) $scope.show_labels_by_inventory_id[key] = {}; for (const n in labels) { const label = labels[n]; - if (label.show_in_list) { - for (const m in label.is_applied) { - const id = label.is_applied[m]; - const property_id = $scope.property_lookup[id]; - if (!$scope.show_labels_by_inventory_id[key][property_id]) { - $scope.show_labels_by_inventory_id[key][property_id] = []; - } - $scope.show_labels_by_inventory_id[key][property_id].push(label); - } + const property_id = $scope.property_lookup[label.propertyview]; + if (!$scope.show_labels_by_inventory_id[key][property_id]) { + $scope.show_labels_by_inventory_id[key][property_id] = []; } + $scope.show_labels_by_inventory_id[key][property_id].push(label); } }; @@ -897,6 +896,7 @@ angular.module('SEED.controller.portfolio_summary', []) }; const set_grid_options = (result) => { + $scope.show_full_labels = { baseline: false, current: false }; $scope.selected_ids = []; $scope.data = format_properties(result); spinner_utility.hide(); @@ -1113,4 +1113,36 @@ angular.module('SEED.controller.portfolio_summary', []) } }; }; + + $scope.run_data_quality_check = () => { + spinner_utility.show(); + data_quality_service.start_data_quality_checks([], [], $scope.goal.id) + .then((response) => { + data_quality_service.data_quality_checks_status(response.progress_key) + .then((result) => { + data_quality_service.get_data_quality_results($scope.organization.id, result.unique_id) + .then((dq_result) => { + $uibModal.open({ + templateUrl: `${urls.static_url}seed/partials/data_quality_modal.html`, + controller: 'data_quality_modal_controller', + size: 'lg', + resolve: { + dataQualityResults: () => dq_result, + name: () => null, + uploaded: () => null, + run_id: () => result.unique_id, + orgId: () => $scope.organization.id + } + }); + spinner_utility.hide(); + load_summary(); + load_inventory(); + }); + }); + }) + .catch(() => { + spinner_utility.hide(); + Notification.erorr('Unexpected Error'); + }); + }; }]); diff --git a/seed/static/seed/js/seed.js b/seed/static/seed/js/seed.js index f5e7923fbd..58d583c780 100644 --- a/seed/static/seed/js/seed.js +++ b/seed/static/seed/js/seed.js @@ -1488,7 +1488,7 @@ }) .state({ name: 'organization_data_quality', - url: '/accounts/{organization_id:int}/data_quality/{inventory_type:properties|taxlots}', + url: '/accounts/{organization_id:int}/data_quality/{rule_type:properties|taxlots|goals}', templateUrl: `${static_url}seed/partials/data_quality_admin.html`, controller: 'data_quality_admin_controller', resolve: { @@ -1498,7 +1498,7 @@ 'naturalSort', ($stateParams, inventory_service, naturalSort) => { const { organization_id } = $stateParams; - if ($stateParams.inventory_type === 'properties') { + if ($stateParams.rule_type === 'properties' || $stateParams.rule_type === 'goals') { return inventory_service.get_property_columns_for_org(organization_id).then((columns) => { columns = _.reject(columns, 'related'); columns = _.map(columns, (col) => _.omit(col, ['pinnedLeft', 'related'])); @@ -1506,12 +1506,14 @@ return columns; }); } - return inventory_service.get_taxlot_columns_for_org(organization_id).then((columns) => { - columns = _.reject(columns, 'related'); - columns = _.map(columns, (col) => _.omit(col, ['pinnedLeft', 'related'])); - columns.sort((a, b) => naturalSort(a.displayName, b.displayName)); - return columns; - }); + if ($stateParams.rule_type === 'taxlots') { + return inventory_service.get_taxlot_columns_for_org(organization_id).then((columns) => { + columns = _.reject(columns, 'related'); + columns = _.map(columns, (col) => _.omit(col, ['pinnedLeft', 'related'])); + columns.sort((a, b) => naturalSort(a.displayName, b.displayName)); + return columns; + }); + } } ], used_columns: [ diff --git a/seed/static/seed/js/services/data_quality_service.js b/seed/static/seed/js/services/data_quality_service.js index 8328ee656e..1c5a9bb262 100644 --- a/seed/static/seed/js/services/data_quality_service.js +++ b/seed/static/seed/js/services/data_quality_service.js @@ -61,13 +61,14 @@ angular.module('SEED.service.data_quality', []).factory('data_quality_service', data_quality_factory.start_data_quality_checks_for_import_file = (org_id, import_file_id) => $http.post(`/api/v3/import_files/${import_file_id}/start_data_quality_checks/?organization_id=${org_id}`).then((response) => response.data); - data_quality_factory.start_data_quality_checks = (property_view_ids, taxlot_view_ids) => data_quality_factory - .start_data_quality_checks_for_org(user_service.get_organization().id, property_view_ids, taxlot_view_ids); + data_quality_factory.start_data_quality_checks = (property_view_ids, taxlot_view_ids, goal_id) => data_quality_factory + .start_data_quality_checks_for_org(user_service.get_organization().id, property_view_ids, taxlot_view_ids, goal_id); - data_quality_factory.start_data_quality_checks_for_org = (org_id, property_view_ids, taxlot_view_ids) => $http + data_quality_factory.start_data_quality_checks_for_org = (org_id, property_view_ids, taxlot_view_ids, goal_id) => $http .post(`/api/v3/data_quality_checks/${org_id}/start/`, { property_view_ids, - taxlot_view_ids + taxlot_view_ids, + goal_id }) .then((response) => response.data); diff --git a/seed/static/seed/js/services/label_service.js b/seed/static/seed/js/services/label_service.js index b8ac112038..8cbd5a2041 100644 --- a/seed/static/seed/js/services/label_service.js +++ b/seed/static/seed/js/services/label_service.js @@ -281,6 +281,12 @@ angular.module('SEED.service.label', []).factory('label_service', [ } ]; + const get_property_view_labels_by_goal = (org_id, goal_id, cycle) => $http.get( + '/api/v3/property_view_labels/list_by_goal/', + { params: { organization_id: org_id, goal_id, cycle } } + ) + .then(map_labels); + return { get_labels, get_labels_for_org, @@ -293,6 +299,7 @@ angular.module('SEED.service.label', []).factory('label_service', [ update_property_labels, update_taxlot_labels, get_available_colors, + get_property_view_labels_by_goal, lookup_label }; } diff --git a/seed/static/seed/partials/accounts.html b/seed/static/seed/partials/accounts.html index 16504708c6..20fb94ead8 100644 --- a/seed/static/seed/partials/accounts.html +++ b/seed/static/seed/partials/accounts.html @@ -36,7 +36,7 @@

{$:: 'Organizations I Manage' | translate >{$:: 'Column Settings' | translate $} {$:: 'Cycles' | translate $} - {$:: 'Data Quality' | translate $} Derived Columns diff --git a/seed/static/seed/partials/accounts_nav.html b/seed/static/seed/partials/accounts_nav.html index 1076d089fc..e1c1603ff9 100644 --- a/seed/static/seed/partials/accounts_nav.html +++ b/seed/static/seed/partials/accounts_nav.html @@ -15,7 +15,7 @@ > Cycles Modifying Data Quality Rules

assign or remove a label if the condition is not met.
Reset All Rules: delete all rules and reinitialize the default set of rules.

- +
Fields that have a "Must Contain" or "Must Not Contain" Condition Check rule cannot have a "Range" Condition Check rule. @@ -50,16 +50,17 @@

Modifying Data Quality Rules

-
+
@@ -78,7 +79,7 @@

Modifying Data Quality Rules

- + @@ -87,7 +88,7 @@

Modifying Data Quality Rules

class="form-control input-sm" ng-model="rule.condition" ng-options="condition.id as condition.label for condition in ::conditions" - ng-change="rules_changed(rule); change_condition(rule, ruleGroups[inventory_type][field], field)" + ng-change="rules_changed(rule); change_condition(rule, ruleGroups[rule_type][field], field)" ng-class="{'border-red red': invalid_conditions.includes(field), 'error-bg': misconfigured_fields_ref.condition.includes(rule.$$hashKey)}" > @@ -209,6 +210,116 @@

Modifying Data Quality Rules
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + Rule TypeData TypeCondition CheckMinimumMaximumSeverity LevelLabelDelete
+ + + + + + + + + + + + + + + + +
+ {$ rule.label.name | translate $} + + + +
+ + + + + + + 		+ +
+
diff --git a/seed/static/seed/partials/portfolio_summary.html b/seed/static/seed/partials/portfolio_summary.html index 9f1939909c..9bad2e4abf 100644 --- a/seed/static/seed/partials/portfolio_summary.html +++ b/seed/static/seed/partials/portfolio_summary.html @@ -56,7 +56,24 @@

Portfolio Summary

-

Portfolio Summary

+
+

Portfolio Summary

+ +
+
+

Unexpected Portfolio Summary Calculations?

+
+ Run "Data Quality Check" from the Actions dropdown to auto populate the "Passed Checks" column.

+ Portfolio Summary calculations only include properties that have "Passed Checks" and are not "New Build or Acquired" (see far right columns below).
+
+ Data Quality Checks can be configured in + Data Quality Settings . +
+
+ +

{$:: 'Loading Summary Data...' | translate $}

@@ -93,6 +110,9 @@

{$:: 'Loading Summary Data...' | translate $}

Edit Fields for Selected Edit Fields for Selected +
  • + Data Quality Check +
  • {$ show_access_level_instances ? 'Hide': 'Show' $} Access Levels {$ show_access_level_instances ? 'Hide': 'Show' $} Access Levels diff --git a/seed/static/seed/scss/style.scss b/seed/static/seed/scss/style.scss index 41fa1762af..130153b483 100755 --- a/seed/static/seed/scss/style.scss +++ b/seed/static/seed/scss/style.scss @@ -5154,7 +5154,7 @@ ul.r-list { margin: 15px; .goal-select-label { - margin: auto; + margin-top: auto; } .goal-selection { @@ -5231,6 +5231,10 @@ ul.r-list { margin: 0; padding-bottom: 10px; } + + .portfolio-summary-help-text { + width: fit-content; + } } } diff --git a/seed/static/seed/tests/data_quality_admin_controller.spec.js b/seed/static/seed/tests/data_quality_admin_controller.spec.js index 334b50377c..581bd30c8a 100644 --- a/seed/static/seed/tests/data_quality_admin_controller.spec.js +++ b/seed/static/seed/tests/data_quality_admin_controller.spec.js @@ -91,7 +91,7 @@ describe('controller: data_quality_admin_controller', () => { } }; data_quality_admin_controller_scope.ruleGroups = ruleGroups; - data_quality_admin_controller_scope.inventory_type = 'properties'; + data_quality_admin_controller_scope.rule_type = 'properties'; // act data_quality_admin_controller_scope.change_field(ruleGroups.properties.address_line_1[0], 'address_line_1', 0); data_quality_admin_controller_scope.change_data_type(ruleGroups.properties.address_line_1[0], 'string'); diff --git a/seed/templates/seed/_header.html b/seed/templates/seed/_header.html index f26c460981..c42b8181f8 100644 --- a/seed/templates/seed/_header.html +++ b/seed/templates/seed/_header.html @@ -55,7 +55,7 @@