Merge pull request #351 from NGEET/issue-348-refactor-pubworkflow

Refactor Publication Workflow

Closes  #348

CHANGES.md

@@ -1,5 +1,20 @@
 # NGEE Tropics Archive Releases
 
+## v2.4.0
+Refactors the publication workflow
+
+ - Issue #348 - Refactor the publication workflow
+    + Adds approval date to Dataset model
+    + Removes unapprove and unsubmit actions and permissions
+    + Changes edit permission to allow editing at any status
+      by the managed_by user or NGEE Tropics Admin
+    + Updates Edit Datasets to list all dataset editable by the logged
+      in users
+    + Adds historical dataset functionality for tracking Dataset changes and
+      allows revert to previous versions
+    + Removes duplicated checking on Dataset metadata form
+
+
 ## v2.3.2
 Minor change to user registrations notification
 

archive_api/admin.py

@@ -9,8 +9,36 @@
 from django.shortcuts import redirect, render
 from django.urls import path
 from django.utils.datetime_safe import datetime
+from django.utils.safestring import mark_safe
 
 from archive_api.models import DataSet, DataSetDownloadLog, MeasurementVariable, Person, Plot, Site
+from simple_history.admin import SimpleHistoryAdmin
+
+
+@admin.register(DataSet)
+class DataSetHistoryAdmin(SimpleHistoryAdmin):
+    list_display = ["data_set_id", "version", "status", "name"]
+    history_list_display = ["list_changes"]
+    search_fields = ['name', 'status', "ngt_id", "version"]
+
+    def list_changes(self, obj):
+        """
+        Lists the changes between revisions in the Admin UI
+
+        See: https://django-simple-history.readthedocs.io/en/latest/history_diffing.html
+
+
+        :param obj:
+        :return:
+        """
+        diff = []
+        if obj.prev_record:
+            delta = obj.diff_against(obj.prev_record)
+            for change in delta.changes:
+                diff.append("<b>- {}:</b> changed from `{}` to `{}`".format(change.field, change.old, change.new))
+
+        # Mark safe (https://docs.djangoproject.com/en/4.0/ref/utils/#django.utils.safestring.mark_safe)
+        return mark_safe("\n<br>".join(diff))
 
 
 class CsvImportForm(forms.Form):
@@ -155,91 +183,6 @@ class SiteAdmin(ModelAdmin):
     list_display = ('site_id', 'name', 'description',)
 
 
-@admin.register(DataSet)
-class DraftDataSetAdmin(ModelAdmin):
-    actions = ("mark_as_deleted",)
-    exclude = ("archive", "cdiac_import", "doi", "status", "description", "start_date", "end_date", "qaqc_status",
-               "ngee_tropics_resources", "qaqc_method_description", "submission_date", "publication_date",
-               "status_comment", "funding_organizations", "acknowledgement", "reference", "doe_funding_contract_numbers",
-               "additional_reference_information", "originating_institution", "additional_access_information",
-               "sites", "plots", "variables", "contact", "cdiac_submission_contact", "access_level")
-    list_display = ('ngt_id', 'version', 'name', 'managed_by', 'created_date', 'modified_by', 'modified_date',)
-    readonly_fields = ('ngt_id', 'version', 'name', 'created_date', 'modified_by', 'modified_date',)
-
-    def __init__(self, *args, **kwargs):
-        """
-        Override the parent method in order to remove display links
-        that navigate to show info page.
-        :param args:
-        :param kwargs:
-        """
-        super(self.__class__, self).__init__(*args, **kwargs)
-
-        # This will affect the Title of the page on in the Admin site
-        self.model._meta.verbose_name = 'Draft data set'
-        self.model._meta.verbose_name_plural = 'Draft data sets'
-
-    def save_model(self, request, obj, form, change):
-        """Override save to set modified by"""
-        obj.modified_by = request.user
-        super().save_model(request, obj, form, change)
-
-    def get_actions(self, request):
-        """Overrides parent. Removed the delete selected action"""
-        actions = super(self.__class__, self).get_actions(request)
-        if 'delete_selected' in actions:
-            del actions['delete_selected']
-        return actions
-
-    def get_queryset(self, request):
-        """
-        Returns a QuerySet of all DRAFT DataSets
-        """
-        qs = super(DraftDataSetAdmin, self).get_queryset(request)
-        return qs.filter(status=DataSet.STATUS_DRAFT)
-
-    def has_add_permission(self, request):
-        """
-        Disallow add through the admin interface. These records
-        should only be created in the main site
-
-        param request:
-        :return: False
-        """
-        return False
-
-    def mark_as_deleted(self, request, queryset):
-        """
-        Mark the DRAFT Datasets as deleted. Datasets marked
-        as deleted will not show up in the Archive Service
-
-        :param request: The current http request
-        :param queryset: the selected objects to mark deleted
-        :return: None
-        """
-
-        # Check that the user has delete permission for the actual model
-        if not self.has_delete_permission(request):
-            raise PermissionDenied
-
-        n = queryset.count()
-        if n:
-            for obj in queryset:
-                # We just want to mark deleted NOT physically delte
-                obj.status = DataSet.STATUS_DELETED
-                obj.save()
-
-            self.message_user(request,
-                              "Successfully marked %(count)d %(items)s as DELETED." % {
-                                  "count": n,
-                                  "items": model_ngettext(self.opts, n)
-                              }, messages.SUCCESS)
-        # Return None to display the change list page again.
-        return None
-
-    mark_as_deleted.short_description = "Mark draft data sets as DELETED"
-
-
 @admin.register(DataSetDownloadLog)
 class DataSetDownloadLogAdmin(ModelAdmin):
     """

archive_api/apps.py

@@ -17,36 +17,39 @@ def load_groups(sender, **kwargs):
     change_contact = Permission.objects.get(codename='change_person')
     change_measurementvariable = Permission.objects.get(codename='change_measurementvariable')
     can_approve_submitted = Permission.objects.get(codename='approve_submitted_dataset')
-    can_unsubmit_submitted = Permission.objects.get(codename='unsubmit_submitted_dataset')
-    can_unapprove_submitted = Permission.objects.get(codename='unapprove_approved_dataset')
-    can_edit_own_draft = Permission.objects.get(codename='edit_own_draft_dataset')
-    can_edit_all_draft = Permission.objects.get(codename='edit_all_draft_dataset')
-    can_edit_all_submitted = Permission.objects.get(codename='edit_all_submitted_dataset')
+    can_edit_own = Permission.objects.get(codename='edit_own_dataset')
+    can_edit_all = Permission.objects.get(codename='edit_all_dataset')
     can_view_all_datasets = Permission.objects.get(codename='view_all_datasets')
     can_view_ngeet_approved_datasets = Permission.objects.get(codename='view_ngeet_approved_datasets')
     can_upload_large_file = Permission.objects.get(codename='upload_large_file_dataset')
 
     admin = Group.objects.filter(name='NGT Administrator')
     if len(admin) == 0:
         admin = Group.objects.create(name='NGT Administrator')
-        for perm in [add_measurementvariable, change_measurementvariable, change_dataset, add_dataset, add_site,
-                     add_plot, add_contact, change_site, change_plot, change_contact, can_approve_submitted,
-                     can_unsubmit_submitted, can_unapprove_submitted,
-                     can_edit_own_draft, can_edit_all_draft, can_edit_all_submitted, can_view_all_datasets,
-                     can_upload_large_file]:
-            admin.permissions.add(perm)
-        admin.save()
         print("{} group created".format(admin.name))
+    else:
+        admin = admin[0]
+
+    admin.permissions.clear()
+    admin.permissions.add(add_measurementvariable, change_measurementvariable, change_dataset, add_dataset, add_site,
+                 add_plot, add_contact, change_site, change_plot, change_contact, can_approve_submitted,
+                 can_edit_own, can_edit_all, can_view_all_datasets,
+                 can_upload_large_file)
+    admin.save()
+    print("{} group permissions updated".format(admin.name))
 
     for id, name in archive_api.models.PERSON_ROLE_CHOICES:
-        ngt_user = Group.objects.filter(name='NGT {}'.format(name))
-        if len(ngt_user) == 0:
-            ngt_user = Group.objects.create(name='NGT {}'.format(name))
-            for perm in [change_dataset, add_dataset, add_contact, can_edit_own_draft,
-                         can_view_ngeet_approved_datasets]:
-                ngt_user.permissions.add(perm)
-            ngt_user.save()
-            print("{} group created".format(ngt_user.name))
+        ngt_group = Group.objects.filter(name='NGT {}'.format(name))
+        if len(ngt_group) == 0:
+            ngt_group = Group.objects.create(name='NGT {}'.format(name))
+            print("{} group created".format(ngt_group))
+        else:
+            ngt_group = ngt_group[0]
+        ngt_group.permissions.clear()
+        ngt_group.permissions.add(change_dataset, add_dataset, add_contact, can_edit_own,
+                         can_view_ngeet_approved_datasets)
+        ngt_group.save()
+        print("{} group permissions updated".format(ngt_group))
 
 
 class ArchiveApiConfig(AppConfig):

archive_api/management/commands/importcdiac.py

@@ -60,6 +60,7 @@ def handle(self, *args, **options):
                             with open(archive_file, 'rb') as f:
                                 dataset.archive.save(filename,
                                                      File(f))
+                            dataset._change_reason = f'Uploaded {archive_file}'
                             dataset.save()
                             archived = True
 
@@ -71,6 +72,7 @@ def handle(self, *args, **options):
                         with open(archive_file, 'rb') as f:
                             dataset.archive.save(filename,
                                                  File(f))
+                        dataset._change_reason = f'Zipped and uploaded {archive_file}'
                         dataset.save()
 
     def extract_metadata(self, options, ngt_id,xml_file):
@@ -91,7 +93,8 @@ def extract_metadata(self, options, ngt_id,xml_file):
             return dataset
         except DataSet.DoesNotExist:
 
-            dataset = DataSet(ngt_id=int(ngt_id), managed_by=create_by, modified_by=create_by,version="1.0")
+            dataset = DataSet(ngt_id=int(ngt_id), managed_by=create_by, modified_by=create_by, version="1.0")
+            dataset._change_reason = f'New Dataset'
             dataset.save()
 
         # Origin are the authors in order of precedence

archive_api/migrations/0013_publication_workflow_refactor.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.1.2 on 2022-02-28 17:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('archive_api', '0012_dataset_managed_by_editable'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='dataset',
+            options={'ordering': ('-modified_date',), 'permissions': (('approve_submitted_dataset', "Can approve a 'submitted' dataset"), ('edit_own_dataset', 'Can edit own dataset'), ('edit_all_dataset', 'Can edit any  dataset'), ('view_all_datasets', 'Can view all datasets'), ('view_ngeet_approved_datasets', 'Can view all approved NGEE Tropics datasets'), ('upload_large_file_dataset', 'Can upload a large file to a dataset'))},
+        ),
+        migrations.AddField(
+            model_name='dataset',
+            name='approval_date',
+            field=models.DateField(blank=True, null=True),
+        ),
+    ]

archive_api/migrations/0014_status_change_datetime.py

@@ -0,0 +1,30 @@
+# Generated by Django 3.1.2 on 2022-02-28 19:45
+
+from django.db import migrations, models
+
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('archive_api', '0013_publication_workflow_refactor'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='dataset',
+            name='approval_date',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='publication_date',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='dataset',
+            name='submission_date',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+
+    ]

archive_api/migrations/0015_managed_modified_date.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.2 on 2022-02-28 20:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('archive_api', '0014_status_change_datetime'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='dataset',
+            name='modified_date',
+            field=models.DateTimeField(blank=True),
+        ),
+    ]

archive_api/migrations/0016_update_approval_publication_dates.py

@@ -0,0 +1,28 @@
+from django.db import migrations, models
+
+
+def update_approval_publication_dates(apps, schema_editor):
+    """
+    Set the approval date and publication date to the modifed_date
+
+    :param apps:
+    :param schema_editor:
+    :return:
+    """
+    DataSet = apps.get_model('archive_api', 'DataSet')
+    for row in DataSet.objects.all():
+        if row.status == 2:
+            row.publication_date = row.modified_date
+            row.approval_date = row.modified_date
+            row.save(update_fields=['publication_date', 'approval_date'])
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('archive_api', '0015_managed_modified_date'),
+    ]
+
+    operations = [
+        migrations.RunPython(update_approval_publication_dates,
+                             reverse_code=migrations.RunPython.noop),
+    ]