Create record spreadsheet that can import/export batches as an Excel …

…file

dear_petition/petition/api/viewsets.py

@@ -3,6 +3,7 @@
 import logging
 import tablib
 import tempfile
+import os
 
 from django.db import transaction
 from django.db.models import Count
@@ -11,13 +12,13 @@
 from django.contrib.auth.models import update_last_login
 from django.http import FileResponse, HttpResponse
 from django_filters.rest_framework import DjangoFilterBackend
-from rest_framework import filters, generics, parsers, permissions, status, viewsets, views
+from rest_framework import filters, generics, parsers, permissions, status, viewsets
 from rest_framework.decorators import action
 from rest_framework.response import Response
 from rest_framework_simplejwt import exceptions
 from rest_framework_simplejwt import views as simplejwt_views
 from rest_framework_simplejwt.serializers import TokenRefreshSerializer
-
+from openpyxl import load_workbook
 
 from dear_petition.petition import constants
 from dear_petition.petition import models as pm
@@ -363,6 +364,25 @@ def generate_summary(self, request, pk):
             resp["Content-Disposition"] = 'attachment; filename="Records Summary.docx"'
             return resp
 
+    @action(
+        detail=True,
+        methods=[
+            "post",
+        ],
+    )
+    def generate_spreadsheet(self, request, pk):
+        batch = self.get_object()
+        resource = resources.RecordSummaryResource()
+        dataset = resource.export(batch)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            filepath = tmpdir + f"/{batch.label}.xlsx"
+            dataset.save(filepath)
+            resp = FileResponse(open(filepath, "rb"))
+            resp[
+                "Content-Type"
+            ] = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+            return resp
 
     @action(
         detail=False,
@@ -407,6 +427,26 @@ def assign_client_to_batch(self, request, pk):
         batch.save()
         batch.adjust_for_new_client_dob()
         return Response({"batch_id": batch.pk})
+
+    @action(
+        detail=False,
+        methods=[
+            "post",
+        ],
+    )
+    def import_spreadsheet(self, request):
+        files = request.data.getlist("files")
+        user = request.user
+        resource = resources.RecordSummaryResource()
+        for file in files:
+            label = os.path.basename(file.name)
+            batch = pm.Batch.objects.create(label=label, user=user)
+            batch.files.create(file=file) 
+            file.seek(0)
+            workbook = load_workbook(filename=file)
+            resource.import_data(workbook, batch)
+
+        return Response({"batch_id": batch.pk})
 
 
 class MyInboxView(generics.ListAPIView):

dear_petition/petition/constants.py

@@ -44,6 +44,13 @@
 ARRAIGNED = "ARRAIGNED"
 GUILTY = "GUILTY"
 
+ACTIONS = Choices(
+    (CHARGED, CHARGED),
+    (CONVICTED, CONVICTED),
+    (ARRAIGNED, ARRAIGNED),
+    (GUILTY, GUILTY),
+)
+
 COMMENT_MAX_LENGTH = 8192
 
 NEW_COMMENT_EMAIL_SUBJECT = "New comment available for batch #{batch}"

dear_petition/petition/etl/load.py

@@ -74,6 +74,7 @@ def create_petitions_from_records(batch, form_type):
     logger.info(f"Searching for {form_type} records (batch {batch})")
     record_set = petition_offense_records(batch, form_type)
     petition_types = identify_distinct_petitions(record_set)
+
     for petition_type in petition_types:
         petition = batch.petitions.create(
             form_type=form_type,

dear_petition/petition/etl/tests/test_transform.py

@@ -69,7 +69,7 @@ def test_combine_batches(batch, batch_file, fake_pdf):
             "Date of Birth/Estimated Age": "1990-01-01",
             "Name": "DOE,JON,BOJACK",
             "Race": "WHITE",
-            "Sex": "MALE",
+            "Sex": "M",
         },
         "District Court Offense Information": [
             {

dear_petition/petition/etl/transform.py

@@ -1,8 +1,10 @@
 import os
 from typing import List
 from django.db import transaction
+import tablib
 
 from dear_petition.petition import models as pm
+from dear_petition.petition import resources as pr
 
 from .load import create_batch_petitions, create_documents, assign_agencies_to_documents
 
@@ -25,6 +27,11 @@ def recalculate_petitions(petition_id, offense_record_ids):
 
 def combine_batches(batch_ids: List[int], label: str, user_id: int):
 
+    client_resource = pr.ClientResource()
+    record_resource = pr.RecordResource()
+    offense_resource = pr.OffenseResource()
+    offense_record_resource = pr.OffenseRecordResource()
+
     with transaction.atomic():
         new_batch = pm.Batch.objects.create(label=label, user_id=user_id)
         batches = pm.Batch.objects.filter(id__in=batch_ids)
@@ -34,6 +41,9 @@ def combine_batches(batch_ids: List[int], label: str, user_id: int):
         for batch in batches:
             for record in batch.records.iterator():
 
+                if record.file_no in saved_file_nos:
+                    continue # Duplicate record of one already imported
+
                 if record.batch_file:
                     file = record.batch_file.file.file
                     file_name = os.path.basename(file.name)
@@ -47,10 +57,34 @@ def combine_batches(batch_ids: List[int], label: str, user_id: int):
                 else:
                     new_batch_file=None
 
-                new_record = new_batch.records.create(batch=new_batch, batch_file=new_batch_file, data=record.data)
-                # Pass file numbers of CIPRS records that have already been saved in this batch of CIPRS records.
-                # If this CIPRS record is in the list, it will not be saved again.
-                new_record.refresh_record_from_data(saved_file_nos)
+                export_record_dataset = record_resource.export(queryset=pm.CIPRSRecord.objects.filter(id=record.id))
+                import_record_dataset = tablib.Dataset()
+                import_record_dataset.headers = [col.column_name for col in record_resource.get_import_fields()]
+                import_record_dataset.append([new_batch.id] + [value if value else None for value in export_record_dataset[0]])
+                record_resource.import_data(import_record_dataset)
+                record_id = record_resource.saved_instance_ids[-1]
+
+                if new_batch_file:
+                    new_record = pm.CIPRSRecord.objects.get(id=record_id)
+                    new_record.batch_file = new_batch_file
+                    new_record.save()
+
+                for offense in record.offenses.all():
+                    export_offense_dataset = offense_resource.export(queryset=pm.Offense.objects.filter(id=offense.id))
+                    import_offense_dataset = tablib.Dataset()
+                    import_offense_dataset.headers = [col.column_name for col in offense_resource.get_import_fields()]
+                    import_offense_dataset.append([record_id] + [value if value else None for value in export_offense_dataset[0]])
+                    offense_resource.import_data(import_offense_dataset)
+                    offense_id = offense_resource.saved_instance_ids[-1]
+
+                    export_offense_record_dataset = offense_record_resource.export(queryset=pm.OffenseRecord.objects.filter(offense_id=offense.id))
+                    import_offense_record_dataset = tablib.Dataset()
+                    import_offense_record_dataset.headers = [col.column_name for col in offense_record_resource.get_import_fields()]
+                    for row in export_offense_record_dataset.dict:
+                        import_offense_record_dataset.append([offense_id] + [value if value else None for value in row.values()])
+
+                    offense_record_resource.import_data(import_offense_record_dataset)
+
                 saved_file_nos.append(record.file_no)
 
         create_batch_petitions(new_batch)