Squash Accountability Hierarchy Migrations (#4560)

* Reject import file if it contains braces (#4537) reject import file if it contains braces * Update default dates on data quality rules (#4538) * new default dates for dq rules * make year dynamic --------- Co-authored-by: Katherine Fleming <[email protected]> * Apply unit conversion to extra data (#4496) * eeej small files * option 1 * precommit * revert * apply optional units to serialized properties * precommit * import unit lookup and apply to mapping * precommit * formatting * taxlots * precommit --------- Co-authored-by: Katherine Fleming <[email protected]> * Fix unit conversion for non numeric values (#4555) * guard against none * guard against all exceptions * more robust type checking * db patterns * Fixed several issues with tests --------- Co-authored-by: Alex Swindler <[email protected]> * Update GitHub Actions for Node v20 (#4550) * squash AH migrations * Fixed precommit issues * Added missing run_python code to AH migrations * Improve AH migration performance * Additional performance improvements --------- Co-authored-by: Ross Perry <[email protected]> Co-authored-by: Nicholas Long <[email protected]> Co-authored-by: Alex Swindler <[email protected]>
SEED-platform · Mar 9, 2024 · 6867836 · 6867836
1 parent a3bf8ae
commit 6867836
Show file tree

Hide file tree

Showing 22 changed files with 311 additions and 294 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -21,11 +21,11 @@ jobs:
       matrix:
         test_env: [django, functional, api]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Cache Docker layers
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/.buildx-cache
           # using `-v3` in key to clear old cache due to errors
@@ -53,7 +53,7 @@ jobs:
           docker exec --env DJANGO_LOG_LEVEL seed_web ./manage.py migrate
           docker exec --env DJANGO_LOG_LEVEL seed_web ./manage.py create_default_user [email protected] --password=demo123
           docker exec --env DJANGO_LOG_LEVEL seed_web /bin/bash -c 'echo "y" | ./manage.py make_superuser --user [email protected]'
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
       - name: Install dependencies
         run: |
           npm install
@@ -109,8 +109,8 @@ jobs:
       matrix:
         tox_env: [docs, precommit, mypy]
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.9"
       - name: Install deps

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -17,13 +17,13 @@ jobs:
   publish:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Cache Docker layers
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/.buildx-cache
           # using `v2` in key to clear old cache due to errors

diff --git a/...019_importrecord_access_level_instance.py → ...grations/0019_accountability_hierarchy.py b/...019_importrecord_access_level_instance.py → ...grations/0019_accountability_hierarchy.py
@@ -1,5 +1,4 @@
-
-# Generated by Django 3.2.18 on 2023-05-26 18:30
+# Generated by Django 3.2.23 on 2024-03-08 06:05
 
 import django.db.models.deletion
 from django.db import migrations, models, transaction
@@ -10,23 +9,20 @@ def set_import_records_ali(apps, schema_editor):
     ImportRecord = apps.get_model('data_importer', 'ImportRecord')
     AccessLevelInstance = apps.get_model('orgs', 'AccessLevelInstance')
 
-    import_records = ImportRecord.objects.all()
-    for import_record in import_records:
-        org = import_record.super_organization
+    if ImportRecord.objects.filter(super_organization=None).exists():
+        raise ValueError("Some ImportRecords have no super_organization, and are orphaned. This shouldn't have happened and these ImportRecords cannot be migrated. Please add a super_organization or delete the orphaned ImportRecords and try again.")
 
-        if org is None:
-            raise ValueError(f"ImportRecord with pk {import_record.pk} has no super_organization, and is thus orphaned. This shouldn't have happened and this ImportRecord cannot be migrated. Please add a super_organization or delete the ImportRecord and try again.")
-        else:
-            ali = AccessLevelInstance.objects.get(organization=org, depth=1)
+    root_alis = {ali.organization_id: ali for ali in AccessLevelInstance.objects.filter(depth=1)}
 
-        import_record.access_level_instance = ali
-        import_record.save()
+    for import_record in ImportRecord.objects.all().iterator():
+        import_record.access_level_instance = root_alis[import_record.super_organization_id]
+        import_record.save(update_fields=['access_level_instance'])
 
 
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('orgs', '0029_auto_20230413_1250'),
+        ('orgs', '0030_accountability_hierarchy'),
         ('data_importer', '0018_importfile_multiple_cycle_upload'),
     ]
 

diff --git a/seed/data_importer/tasks.py b/seed/data_importer/tasks.py
@@ -1452,6 +1452,9 @@ def _save_raw_data_create_tasks(file_pk, progress_key):
             _log.debug(f'Error reading XLSX file: {str(e)}')
             return progress_data.finish_with_error('Failed to parse XLSX file. Please review your import file - all headers should be present and non-numeric.')
 
+    if any('{' in header or '}' in header for header in parser.headers):
+        return progress_data.finish_with_error('Failed to import. Please review your import file - headers cannot contain braces: { }')
+
     import_file.has_generated_headers = False
     if hasattr(parser, 'has_generated_headers'):
         import_file.has_generated_headers = parser.has_generated_headers

diff --git a/...rgs/migrations/0030_auto_20240105_1257.py → ...rgs/migrations/0029_auto_20240105_1257.py b/...rgs/migrations/0030_auto_20240105_1257.py → ...rgs/migrations/0029_auto_20240105_1257.py
@@ -6,7 +6,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('orgs', '0029_auto_20230413_1250'),
+        ('orgs', '0028_organization_audit_template_report_type'),
     ]
 
     operations = [

diff --git a/...rgs/migrations/0029_auto_20230413_1250.py → ...grations/0030_accountability_hierarchy.py b/...rgs/migrations/0029_auto_20230413_1250.py → ...grations/0030_accountability_hierarchy.py
@@ -1,46 +1,45 @@
-# Generated by Django 3.2.18 on 2023-04-13 19:50
+# Generated by Django 3.2.23 on 2024-03-08 06:05
 
 import django.db.models.deletion
 from django.db import migrations, models, transaction
 
 
 @transaction.atomic
-def give_org_access_level_root(apps, schema_editor):
+def create_root_access_levels(apps, schema_editor):
     Organization = apps.get_model('orgs', 'Organization')
     AccessLevelInstance = apps.get_model('orgs', 'AccessLevelInstance')
 
-    orgs = Organization.objects.all()
-    for i, org in enumerate(orgs):
+    for i, org in enumerate(Organization.objects.all()):
         org.access_level_names = [org.name]
         org.save()
 
         AccessLevelInstance.objects.create(
             tree_id=i,
             organization=org,
-            name="root",
-            path={org.access_level_names[0]: "root"},
+            name='root',
+            path={org.access_level_names[0]: 'root'},
             depth=1,
             lft=1,
             rgt=2,
-        ).save()
+        )
 
 
 @transaction.atomic
 def assign_users_to_root_access_level(apps, schema_editor):
     OrganizationUser = apps.get_model('orgs', 'OrganizationUser')
     AccessLevelInstance = apps.get_model('orgs', 'AccessLevelInstance')
 
-    users = OrganizationUser.objects.all()
-    for user in users:
-        root = AccessLevelInstance.objects.get(organization=user.organization, depth=1)
-        user.access_level_instance = root
-        user.save()
+    root_alis = {ali.organization_id: ali for ali in AccessLevelInstance.objects.filter(depth=1)}
+
+    for user in OrganizationUser.objects.all():
+        user.access_level_instance = root_alis[user.organization_id]
+        user.save(update_fields=['access_level_instance'])
 
 
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('orgs', '0028_organization_audit_template_report_type'),
+        ('orgs', '0029_auto_20240105_1257'),
     ]
 
     operations = [
@@ -58,23 +57,23 @@ class Migration(migrations.Migration):
                 ('tree_id', models.PositiveIntegerField(db_index=True)),
                 ('depth', models.PositiveIntegerField(db_index=True)),
                 ('name', models.CharField(max_length=100)),
+                ('path', models.JSONField()),
                 ('organization', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='orgs.organization')),
-                ('path', models.JSONField(null=False))
             ],
             options={
                 'abstract': False,
             },
         ),
-        migrations.RunPython(give_org_access_level_root),
         migrations.AddField(
             model_name='organizationuser',
             name='access_level_instance',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='orgs.accesslevelinstance', related_name="users"),
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='users', to='orgs.accesslevelinstance'),
         ),
+        migrations.RunPython(create_root_access_levels),
         migrations.RunPython(assign_users_to_root_access_level, reverse_code=migrations.RunPython.noop),
         migrations.AlterField(
             model_name='organizationuser',
             name='access_level_instance',
-            field=models.ForeignKey(null=False, on_delete=django.db.models.deletion.CASCADE, to='orgs.accesslevelinstance', related_name="users"),
+            field=models.ForeignKey(null=False, on_delete=django.db.models.deletion.CASCADE, related_name='users', to='orgs.accesslevelinstance'),
         )
     ]
diff --git a/...migrations/0213_add_filtergroup_labels.py → ...migrations/0212_add_filtergroup_labels.py b/...migrations/0213_add_filtergroup_labels.py → ...migrations/0212_add_filtergroup_labels.py
@@ -6,7 +6,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('seed', '0212_auto_add_props_to_tree_squashed_0213_analysis_access_level_instance'),
+        ('seed', '0211_auto_20240109_1348'),
     ]
 
     operations = [

diff --git a/seed/migrations/0212_auto_add_props_to_tree_squashed_0213_analysis_access_level_instance.py b/seed/migrations/0212_auto_add_props_to_tree_squashed_0213_analysis_access_level_instance.py
diff --git a/...igrations/0214_move_filtergroup_labels.py → ...igrations/0213_move_filtergroup_labels.py b/...igrations/0214_move_filtergroup_labels.py → ...igrations/0213_move_filtergroup_labels.py
@@ -35,7 +35,7 @@ def move_filter_groups_back(apps, schema_editor):
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('seed', '0213_add_filtergroup_labels'),
+        ('seed', '0212_add_filtergroup_labels'),
     ]
 
     operations = [

diff --git a/...rations/0215_delete_filtergroup_labels.py → ...rations/0214_delete_filtergroup_labels.py b/...rations/0215_delete_filtergroup_labels.py → ...rations/0214_delete_filtergroup_labels.py
@@ -6,7 +6,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('seed', '0214_move_filtergroup_labels'),
+        ('seed', '0213_move_filtergroup_labels'),
     ]
 
     operations = [