-
Notifications
You must be signed in to change notification settings - Fork 54
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add cross cycle data quality checks (#4695)
* Initial cross-cycle data quality changes * create default cross cycle rules * update propertyview label name * rule span * merge develop * add target to rule * change cross-cycle to goal and add action * order * update goalnote passed_check based on rule * update goalnote passed_check based on rule * add and remove labels based on cross cycle data quality checks * specify goal and baseline_view when applying cc labels * add default labels, update rule model fields * cross cycle rules functional * labels limited to goal, or none goal * update test * checking default single cycle * results for range and not null * frontend results * precommit * precommit * fix eui inverse * add help text * prevent new goal rules * precommit * lint * test hard coded goal rules * fix frontend test * update migration to remove old constraint * fix tests * precommit * remove consoles * label width bugfix * help text if not passing checks * clarifying text * default goal labels to show in list * lint * modify migration * show all labels * qaqc * precommit * lint * temp disable wui rules * lint * remove WUI data type - indroduced in different pr * precommit * update rule count * migration order * update migration update migration precommit * move spinner utility earlier --------- Co-authored-by: Alex Swindler <[email protected]> Co-authored-by: Ross Perry <[email protected]> Co-authored-by: Ross Perry <[email protected]> Co-authored-by: Katherine Fleming <[email protected]>
- Loading branch information
5 people
authored
Aug 9, 2024
1 parent
d19f19e
commit af1ac6a
Showing
35 changed files
with
1,315 additions
and
143 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,214 @@ | ||
| # Generated by Django 3.2.20 on 2023-10-24 21:54 | ||
|
|
||
| import django.db.models.deletion | ||
| from django.db import connection, migrations, models | ||
|
|
||
|
|
||
| def forwards(apps, schema_editor): | ||
| Organization = apps.get_model("orgs", "Organization") | ||
| Rule = apps.get_model("seed", "Rule") | ||
| DataQualityCheck = apps.get_model("seed", "DataQualityCheck") | ||
| Label = apps.get_model("seed", "StatusLabel") | ||
|
|
||
| # Populate the default labels for goal rules. | ||
| NEW_DEFAULT_LABELS = [ | ||
| "High EUI % Change", | ||
| "Low EUI % Change", | ||
| "High Area", | ||
| "Low Area", | ||
| "High Area % Change", | ||
| "Low Area % Change", | ||
| ] | ||
|
|
||
| # Populate the default data quality goal rules if they do not already exist. | ||
| TYPE_AREA = 4 | ||
| TYPE_EUI = 5 | ||
| RULE_TYPE_DEFAULT = 0 | ||
| SEVERITY_ERROR = 0 | ||
| RULE_NOT_NULL = "not_null" | ||
| RULE_RANGE = "range" | ||
|
|
||
| NEW_DEFAULT_RULES = [ | ||
| { | ||
| "table_name": "Goal", | ||
| "name": "High EUI % Change", | ||
| "field": "eui", | ||
| "data_type": TYPE_EUI, | ||
| "rule_type": RULE_TYPE_DEFAULT, | ||
| "severity": SEVERITY_ERROR, | ||
| "condition": RULE_RANGE, | ||
| "max": 40, | ||
| "cross_cycle": True, | ||
| }, | ||
| { | ||
| "table_name": "Goal", | ||
| "name": "Low EUI % Change", | ||
| "field": "eui", | ||
| "data_type": TYPE_EUI, | ||
| "rule_type": RULE_TYPE_DEFAULT, | ||
| "severity": SEVERITY_ERROR, | ||
| "condition": RULE_RANGE, | ||
| "min": -40, | ||
| "cross_cycle": True, | ||
| }, | ||
| { | ||
| "table_name": "Goal", | ||
| "name": "High Area % Change", | ||
| "field": "area", | ||
| "data_type": TYPE_AREA, | ||
| "rule_type": RULE_TYPE_DEFAULT, | ||
| "severity": SEVERITY_ERROR, | ||
| "condition": RULE_RANGE, | ||
| "max": 5, | ||
| "cross_cycle": True, | ||
| }, | ||
| { | ||
| "table_name": "Goal", | ||
| "name": "Low Area % Change", | ||
| "field": "area", | ||
| "data_type": TYPE_AREA, | ||
| "rule_type": RULE_TYPE_DEFAULT, | ||
| "severity": SEVERITY_ERROR, | ||
| "condition": RULE_RANGE, | ||
| "min": -5, | ||
| "cross_cycle": True, | ||
| }, | ||
| { | ||
| "table_name": "Goal", | ||
| "name": "High EUI", | ||
| "field": "eui", | ||
| "data_type": TYPE_EUI, | ||
| "rule_type": RULE_TYPE_DEFAULT, | ||
| "severity": SEVERITY_ERROR, | ||
| "condition": RULE_RANGE, | ||
| "max": 1000, | ||
| }, | ||
| { | ||
| "table_name": "Goal", | ||
| "name": "Low EUI", | ||
| "field": "eui", | ||
| "data_type": TYPE_EUI, | ||
| "rule_type": RULE_TYPE_DEFAULT, | ||
| "severity": SEVERITY_ERROR, | ||
| "condition": RULE_RANGE, | ||
| "min": 40, | ||
| }, | ||
| { | ||
| "table_name": "Goal", | ||
| "name": "High Area", | ||
| "field": "area", | ||
| "data_type": TYPE_AREA, | ||
| "rule_type": RULE_TYPE_DEFAULT, | ||
| "severity": SEVERITY_ERROR, | ||
| "condition": RULE_RANGE, | ||
| "max": 1000000, | ||
| }, | ||
| { | ||
| "table_name": "Goal", | ||
| "name": "Low Area", | ||
| "field": "area", | ||
| "data_type": TYPE_AREA, | ||
| "rule_type": RULE_TYPE_DEFAULT, | ||
| "severity": SEVERITY_ERROR, | ||
| "condition": RULE_RANGE, | ||
| "min": 1000, | ||
| }, | ||
| { | ||
| "table_name": "Goal", | ||
| "name": "Missing EUI", | ||
| "field": "eui", | ||
| "data_type": TYPE_EUI, | ||
| "rule_type": RULE_TYPE_DEFAULT, | ||
| "severity": SEVERITY_ERROR, | ||
| "condition": RULE_NOT_NULL, | ||
| }, | ||
| { | ||
| "table_name": "Goal", | ||
| "name": "Missing Area", | ||
| "field": "area", | ||
| "data_type": TYPE_AREA, | ||
| "rule_type": RULE_TYPE_DEFAULT, | ||
| "severity": SEVERITY_ERROR, | ||
| "condition": RULE_NOT_NULL, | ||
| }, | ||
| ] | ||
|
|
||
| for org in Organization.objects.all(): | ||
| for label in NEW_DEFAULT_LABELS: | ||
| Label.objects.get_or_create(name=label, super_organization=org, defaults={"color": "blue"}) | ||
| for dqc in DataQualityCheck.objects.all(): | ||
| for rule in NEW_DEFAULT_RULES: | ||
| Rule.objects.get_or_create(**rule, data_quality_check=dqc) | ||
|
|
||
|
|
||
| def remove_unique_constraint(apps, schema_editor): | ||
| # The auto generated unique constraint for PropertyViewLabels is PropertyView_id and StatusLabel_id. | ||
| # The constraint needs to be updated to include PropertyView_id, StatusLabel_id, and Goal.id | ||
| PropertyViewLabel = apps.get_model("seed", "PropertyViewLabel") | ||
| table_name = PropertyViewLabel._meta.db_table | ||
|
|
||
| # Get the original unique constraint | ||
| constraints = connection.introspection.get_constraints(connection.cursor(), table_name) | ||
| constraint_names = [ | ||
| name | ||
| for name, details in constraints.items() | ||
| if details.get("unique") and set(details.get("columns")) == {"propertyview_id", "statuslabel_id"} | ||
| ] | ||
| # Remove the constraint | ||
| if constraint_names: | ||
| with connection.cursor() as cursor: | ||
| cursor.execute(f"ALTER TABLE {table_name} DROP CONSTRAINT {constraint_names[0]};") | ||
| # A new unique constraint will be created in the following operation | ||
|
|
||
|
|
||
| class Migration(migrations.Migration): | ||
| dependencies = [ | ||
| ("seed", "0221_audittemplateconfig"), | ||
| ] | ||
|
|
||
| operations = [ | ||
| migrations.SeparateDatabaseAndState( | ||
| database_operations=[ | ||
| migrations.RunSQL( | ||
| sql="ALTER TABLE seed_propertyview_labels RENAME TO seed_propertyviewlabel", | ||
| reverse_sql="ALTER TABLE seed_propertyviewlabel RENAME TO seed_propertyview_labels", | ||
| ), | ||
| ], | ||
| state_operations=[ | ||
| migrations.CreateModel( | ||
| name="PropertyViewLabel", | ||
| fields=[ | ||
| ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), | ||
| ("propertyview", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="seed.propertyview")), | ||
| ("statuslabel", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="seed.statuslabel")), | ||
| ], | ||
| ), | ||
| migrations.AlterField( | ||
| model_name="propertyview", | ||
| name="labels", | ||
| field=models.ManyToManyField(through="seed.PropertyViewLabel", to="seed.StatusLabel"), | ||
| ), | ||
| ], | ||
| ), | ||
| migrations.AddField( | ||
| model_name="propertyviewlabel", | ||
| name="goal", | ||
| field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to="seed.goal"), | ||
| ), | ||
| migrations.AlterField( | ||
| model_name="rule", | ||
| name="field", | ||
| field=models.CharField(blank=True, max_length=200, null=True), | ||
| ), | ||
| migrations.AddField( | ||
| model_name="rule", | ||
| name="cross_cycle", | ||
| field=models.BooleanField(default=False), | ||
| ), | ||
| migrations.RunPython(remove_unique_constraint), | ||
| migrations.AddConstraint( | ||
| model_name="propertyviewlabel", | ||
| constraint=models.UniqueConstraint(fields=("propertyview", "statuslabel", "goal"), name="unique_propertyview_statuslabel_goal"), | ||
| ), | ||
| migrations.RunPython(forwards), | ||
| ] |
Oops, something went wrong.