Merge pull request #36 from RockefellerArchiveCenter/issue-35

Moves files before sending SNS messsage

.gitignore

@@ -5,6 +5,11 @@
 .coverage
 .tox
 
+### App
+/destination
+/storage
+
+
 ### Django ###
 *.log
 *.pot

digitized_image_qc/settings.py

@@ -134,6 +134,7 @@
 X_FRAME_OPTIONS = "SAMEORIGIN"
 
 BASE_STORAGE_DIR = BASE_DIR / getenv('STORAGE_PATH')
+BASE_DESTINATION_DIR = BASE_DIR / getenv('DESTINATION_PATH')
 
 MEDIA_ROOT = BASE_STORAGE_DIR
 MEDIA_URL = '/media/'

docker-compose.yml

@@ -22,6 +22,7 @@ services:
     environment:
       - ALLOWED_HOSTS=localhost
       - APPLICATION_PORT=${APPLICATION_PORT:-80}
+      - DESTINATION_PATH=DESTINATION
       - SQL_ENGINE=django.db.backends.postgresql # Django database engine
       - SQL_DB_NAME=postgres # Database to connect to
       - SQL_DB_USER=postgres # Name of database user

entrypoint.prod.sh

@@ -9,6 +9,8 @@ declare -p | grep -Ev 'BASHOPTS|BASH_VERSINFO|EUID|PPID|SHELLOPTS|UID' > /contai
 python ./manage.py migrate
 # collect static assets
 python ./manage.py collectstatic --no-input
+# remove approved
+python ./manage.py remove_approved
 # discover packages
 python ./manage.py discover_packages
 # fetch rights statements

package_review/management/commands/remove_approved.py

@@ -0,0 +1,34 @@
+import logging
+from os import getenv
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from package_review.models import Package
+
+logging.basicConfig(
+    level=int(getenv('LOGGING_LEVEL', logging.INFO)),
+    format='%(filename)s::%(funcName)s::%(lineno)s %(message)s')
+
+
+class Command(BaseCommand):
+    """
+    This command is designed to clean up erroneously created packages,
+    and should not need to be run regularly. It should only be run
+    once when the service is first started.
+    """
+
+    help = "Removes packages which have already been approved."
+
+    def handle(self, *args, **options):
+        if not settings.BASE_STORAGE_DIR.is_dir():
+            self.stdout.write(self.style.ERROR(f'Root directory {str(settings.BASE_STORAGE_DIR)} for files waiting to be QCed does not exist.'))
+            exit()
+        deleted_list = []
+        for package in Package.objects.filter(process_status=Package.PENDING):
+            if not (settings.BASE_STORAGE_DIR / package.refid).exists():
+                package.delete()
+                deleted_list.append(package.refid)
+
+        message = f'Packages deleted: {", ".join(deleted_list)}' if len(deleted_list) else 'No approved packages to delete.'
+        self.stdout.write(self.style.SUCCESS(message))

package_review/tests.py

@@ -14,7 +14,7 @@
 from .clients import ArchivesSpaceClient, AWSClient
 from .helpers import get_config
 from .management.commands import (check_qc_status, discover_packages,
-                                  fetch_rights_statements,
+                                  fetch_rights_statements, remove_approved,
                                   send_startup_message)
 from .models import Package, RightsStatement
 
@@ -263,6 +263,18 @@ def test_handle(self, mock_rights):
         self.assertEqual(RightsStatement.objects.all().count(), len(rights_statements))
 
 
+class RemoveApprovedCommandTests(TestCase):
+
+    def test_handle(self):
+        """Asserts command deletes only packages with missing binaries"""
+        create_packages()
+        copy_binaries()
+        shutil.rmtree(Path(settings.BASE_STORAGE_DIR, "9ba10e5461d401517b0e1a53d514ec87"))
+        remove_approved.Command().handle()
+        self.assertEqual(Package.objects.all().count(), 1)
+        self.assertEqual(Package.objects.all().first().refid, "f7d3dd6dc9c4732fa17dbd88fbe652b6")
+
+
 class ViewMixinTests(TestCase):
 
     def setUp(self):
@@ -304,7 +316,8 @@ def test_approve_view(self, mock_deliver, mock_init):
         for package in Package.objects.all():
             self.assertEqual(package.process_status, Package.APPROVED)
             self.assertEqual(package.rights_ids, rights_list)
-        self.assertEqual(len(list(Path(settings.BASE_STORAGE_DIR).iterdir())), Package.objects.all().count())
+        self.assertEqual(len(list(Path(settings.BASE_DESTINATION_DIR).iterdir())), Package.objects.all().count())
+        self.assertEqual(len(list(Path(settings.BASE_STORAGE_DIR).iterdir())), 0)
         self.assertEqual(response.status_code, 302)
         self.assertEqual(response.url, reverse('package-list'))
 

package_review/views.py

@@ -1,6 +1,6 @@
 from os import getenv
 from pathlib import Path
-from shutil import rmtree
+from shutil import copytree, rmtree
 
 from django.conf import settings
 from django.shortcuts import redirect
@@ -74,6 +74,7 @@ def post(self, request, *args, **kwargs):
         rights_ids = request.GET['rights_ids']
         aws_client = AWSClient('sns', settings.AWS['role_arn'])
         for package in queryset:
+            self.move_files(package)
             aws_client.deliver_message(
                 settings.AWS['sns_topic'],
                 package,
@@ -85,6 +86,13 @@ def post(self, request, *args, **kwargs):
             package.save()
         return redirect('package-list')
 
+    def move_files(self, package):
+        """Moves files to packaging directory."""
+        bag_dir = Path(settings.BASE_STORAGE_DIR, package.refid)
+        new_path = Path(settings.BASE_DESTINATION_DIR, package.refid)
+        copytree(bag_dir, new_path)
+        rmtree(bag_dir)
+
 
 class PackageRejectView(PackageActionView):
     """Rejects a list of packages."""