work-in-progress: convert heic to jpeg on Asset creation

mediathread/assetmgr/models.py

@@ -14,7 +14,9 @@
 from tagging.models import Tag
 
 from mediathread.assetmgr.custom_storage import private_storage
-from mediathread.assetmgr.utils import get_signed_s3_url
+from mediathread.assetmgr.utils import (
+    get_signed_s3_url, get_s3_private_bucket_name
+)
 
 
 METADATA_ORIGINAL_OWNER = 'Original Owner'
@@ -397,10 +399,7 @@ def is_panopto(self):
         return self.label == 'mp4_panopto'
 
     def signed_url(self):
-        s3_private_bucket = getattr(
-            settings,
-            'S3_PRIVATE_STORAGE_BUCKET_NAME',
-            'mediathread-private-uploads')
+        s3_private_bucket = get_s3_private_bucket_name()
         if s3_private_bucket in self.url:
             return get_signed_s3_url(
                 self.url, s3_private_bucket,
@@ -413,10 +412,7 @@ def signed_url(self):
         return self.url
 
     def url_processed(self, request):
-        s3_private_bucket = getattr(
-            settings,
-            'S3_PRIVATE_STORAGE_BUCKET_NAME',
-            'mediathread-private-uploads')
+        s3_private_bucket = get_s3_private_bucket_name()
         if s3_private_bucket in self.url:
             return get_signed_s3_url(
                 self.url, s3_private_bucket,

mediathread/assetmgr/urls.py

@@ -78,7 +78,7 @@
     path('pdfjs/<int:pk>/', PDFViewerDetailView.as_view(), {},
          name='pdfjs'),
 
-    path('sign_s3/', S3SignView.as_view()),
+    path('sign_s3/', S3SignView.as_view(), name='s3sign'),
 
     # Load this CSS asset as a TemplateView so we can use the static
     # template tag.

mediathread/assetmgr/utils.py

@@ -1,15 +1,80 @@
-from urllib.parse import urlparse
 import boto3
-from s3sign.utils import create_presigned_url, s3_config
+import requests
+from django.conf import settings
+from os.path import basename, splitext, join
+from PIL import Image
+from pi_heif import register_heif_opener
+from s3sign.utils import create_presigned_url, s3_config, upload_file
+from urllib.parse import urlparse
+
+
+s3_sign_view_settings = {
+    'private': True,
+    'root': 'private/',
+    'acl': None,
+    'expiration_time': 3600 * 8,  # 8 hours
+    'max_file_size': 50000000,  # 50mb
+}
 
 
-def get_signed_s3_url(url, bucket, aws_key, aws_secret):
-    s3_client = boto3.client(
+def get_s3_private_bucket_name() -> str:
+    return getattr(
+        settings,
+        'S3_PRIVATE_STORAGE_BUCKET_NAME',
+        'mediathread-private-uploads')
+
+
+def get_s3_client(aws_key, aws_secret):
+    return boto3.client(
         's3', config=s3_config,
         aws_access_key_id=aws_key,
         aws_secret_access_key=aws_secret)
 
+
+def get_signed_s3_url(url: str, bucket: str, aws_key: str, aws_secret: str):
+    s3_client = get_s3_client(aws_key, aws_secret)
+
     url = urlparse(url)
     object_name = url.path.lstrip('/')
     object_name = object_name.replace(bucket + '/', '')
     return create_presigned_url(s3_client, bucket, object_name, 3600)
+
+
+def convert_heic_to_jpg(
+        url: str, request: object, bucket: str,
+        aws_key: str, aws_secret: str
+) -> str:
+    """
+    Given an heic image url, convert it to a JPEG. This comprises a
+    few steps:
+    * Download the file
+    * Do the conversion
+    * Upload jpeg to S3
+    * Return new url
+    """
+    response = requests.get(url, stream=True)
+
+    # Sort out the new filename
+    parsed_url = urlparse(url)
+    filename = splitext(basename(parsed_url.path))[0]
+    filename = filename + '.jpg'
+
+    # Open the file and convert it
+    register_heif_opener()
+    im = Image.open(response.raw)
+    rgb_im = im.convert('RGB')
+    tmp_jpeg = join('/tmp/', filename)
+    rgb_im.save(tmp_jpeg)
+
+    # upload to S3, return source url
+    s3_client = get_s3_client(aws_key, aws_secret)
+    mime_type = 'image/jpeg'
+
+    data = upload_file(
+        s3_client, bucket, mime_type, object_name,
+        s3_sign_view_settings.get('max_file_size'),
+        s3_sign_view_settings.get('acl'),
+        s3_sign_view_settings.get('expiration_time'),
+        s3_sign_view_settings.get('private')
+    )
+    print(data)

mediathread/assetmgr/views.py

@@ -37,6 +37,10 @@
     Asset, Source, ExternalCollection,
     SuggestedExternalCollection
 )
+from mediathread.assetmgr.utils import (
+    s3_sign_view_settings,
+    get_signed_s3_url, get_s3_private_bucket_name, convert_heic_to_jpg
+)
 from mediathread.djangosherd.api import DiscussionIndexResource
 from mediathread.djangosherd.models import SherdNote, DiscussionIndex
 from mediathread.djangosherd.views import create_annotation, edit_annotation, \
@@ -375,6 +379,19 @@ def post(self, request, *args, **kwargs):
             width = request.POST.get('width')
             height = request.POST.get('height')
 
+        if url.endswith('.heic') or url.endswith('.heif'):
+            # Convert to JPG
+            s3_private_bucket = get_s3_private_bucket_name()
+            signed_url = get_signed_s3_url(
+                url, s3_private_bucket,
+                settings.AWS_ACCESS_KEY,
+                settings.AWS_SECRET_KEY)
+
+            # Pass in the signed url because we need to download it.
+            url = convert_heic_to_jpg(
+                signed_url, request, s3_private_bucket,
+                settings.AWS_ACCESS_KEY, settings.AWS_SECRET_KEY)
+
         # If the form passed in a valid label, use it.
         lbl = request.POST.get('label')
         if lbl and lbl in Asset.primary_labels:
@@ -1395,14 +1412,11 @@ def dispatch(self, request, *args, **kwargs):
 
 
 class S3SignView(SignS3View):
-    private = True
-    root = 'private/'
-    acl = None
-    expiration_time = 3600 * 8  # 8 hours
-    max_file_size = 50000000  # 50mb
+    private = s3_sign_view_settings.get('private', True)
+    root = s3_sign_view_settings.get('root', 'private/')
+    acl = s3_sign_view_settings.get('acl', None)
+    expiration_time = s3_sign_view_settings.get('expiration_time', 3600 * 8)
+    max_file_size = s3_sign_view_settings.get('max_file_size', 50000000)
 
     def get_bucket(self):
-        return getattr(
-            settings,
-            'S3_PRIVATE_STORAGE_BUCKET_NAME',
-            'mediathread-private-uploads')
+        return get_s3_private_bucket_name()

requirements.txt

@@ -143,6 +143,8 @@ click-plugins>=1.1.1
 wcwidth==0.2.5
 django-celery-results<2.6.0
 
+Pillow==11.1.0
+pi-heif==0.21.0
 
 # memcached
 pylibmc==1.6.3;sys_platform == "linux" and python_version<"3.9"