work-in-progress: convert heic to jpeg on Asset creation

* Consolidated S3 logic and settings in a few places to be more DRY.

mediathread/assetmgr/models.py

@@ -14,7 +14,9 @@
 from tagging.models import Tag
 
 from mediathread.assetmgr.custom_storage import private_storage
-from mediathread.assetmgr.utils import get_signed_s3_url
+from mediathread.assetmgr.utils import (
+    get_signed_s3_url, get_s3_private_bucket_name
+)
 
 
 METADATA_ORIGINAL_OWNER = 'Original Owner'
@@ -397,10 +399,7 @@ def is_panopto(self):
         return self.label == 'mp4_panopto'
 
     def signed_url(self):
-        s3_private_bucket = getattr(
-            settings,
-            'S3_PRIVATE_STORAGE_BUCKET_NAME',
-            'mediathread-private-uploads')
+        s3_private_bucket = get_s3_private_bucket_name()
         if s3_private_bucket in self.url:
             return get_signed_s3_url(
                 self.url, s3_private_bucket,
@@ -413,10 +412,7 @@ def signed_url(self):
         return self.url
 
     def url_processed(self, request):
-        s3_private_bucket = getattr(
-            settings,
-            'S3_PRIVATE_STORAGE_BUCKET_NAME',
-            'mediathread-private-uploads')
+        s3_private_bucket = get_s3_private_bucket_name()
         if s3_private_bucket in self.url:
             return get_signed_s3_url(
                 self.url, s3_private_bucket,

mediathread/assetmgr/utils.py

@@ -1,15 +1,88 @@
-from urllib.parse import urlparse
 import boto3
-from s3sign.utils import create_presigned_url, s3_config
+import requests
+from django.conf import settings
+from django.utils import timezone
+from os.path import basename, splitext, join
+from PIL import Image
+from pi_heif import register_heif_opener
+from s3sign.utils import (
+    DEFAULT_AWS_REGION,
+    create_presigned_url, s3_config, get_object_name, upload_file
+)
+from urllib.parse import urlparse
+
+
+s3_sign_view_settings = {
+    'private': True,
+    'root': 'private/',
+    'acl': None,
+    'expiration_time': 3600 * 8,  # 8 hours
+    'max_file_size': 50000000,  # 50mb
+}
 
 
-def get_signed_s3_url(url, bucket, aws_key, aws_secret):
-    s3_client = boto3.client(
+def get_s3_private_bucket_name() -> str:
+    return getattr(
+        settings,
+        'S3_PRIVATE_STORAGE_BUCKET_NAME',
+        'mediathread-private-uploads')
+
+
+def get_s3_client(aws_key, aws_secret):
+    aws_region_name = DEFAULT_AWS_REGION
+    if hasattr(settings, 'AWS_S3_REGION_NAME'):
+        aws_region_name = settings.AWS_S3_REGION_NAME
+
+    return boto3.client(
         's3', config=s3_config,
+        region_name=self.aws_region_name,
         aws_access_key_id=aws_key,
         aws_secret_access_key=aws_secret)
 
+
+def get_signed_s3_url(url: str, bucket: str, aws_key: str, aws_secret: str):
+    s3_client = get_s3_client(aws_key, aws_secret)
+
     url = urlparse(url)
     object_name = url.path.lstrip('/')
     object_name = object_name.replace(bucket + '/', '')
     return create_presigned_url(s3_client, bucket, object_name, 3600)
+
+
+def convert_heic_to_jpg(
+        url: str, request: object, bucket: str,
+        aws_key: str, aws_secret: str
+) -> str:
+    """
+    Given an heic image url, convert it to a JPEG. This comprises a
+    few steps:
+    * Download the file
+    * Do the conversion
+    * Upload jpeg to S3
+    * Return new url
+    """
+    print('convert_heic_to_jpg', url, bucket)
+    response = requests.get(url, stream=True)
+
+    # Sort out the new filename
+    parsed_url = urlparse(url)
+    filename = splitext(basename(parsed_url.path))[0]
+    filename = filename + '.jpg'
+    print(filename)
+
+    # Open the file and convert it
+    register_heif_opener()
+    im = Image.open(response.raw)
+    rgb_im = im.convert('RGB')
+    tmp_jpeg = join('/tmp/', filename)
+    rgb_im.save(tmp_jpeg)
+    print(tmp_jpeg)
+
+    # upload to S3, return source url
+    s3_client = get_s3_client(aws_key, aws_secret)
+    object_name = get_object_name(timezone.now(), '.jpg')
+    print(object_name)
+    uploaded = upload_file(s3_client, filename, bucket, object_name)
+    print(uploaded)
+
+    return object_name

mediathread/assetmgr/views.py

@@ -37,6 +37,10 @@
     Asset, Source, ExternalCollection,
     SuggestedExternalCollection
 )
+from mediathread.assetmgr.utils import (
+    s3_sign_view_settings,
+    get_signed_s3_url, get_s3_private_bucket_name, convert_heic_to_jpg
+)
 from mediathread.djangosherd.api import DiscussionIndexResource
 from mediathread.djangosherd.models import SherdNote, DiscussionIndex
 from mediathread.djangosherd.views import create_annotation, edit_annotation, \
@@ -375,6 +379,20 @@ def post(self, request, *args, **kwargs):
             width = request.POST.get('width')
             height = request.POST.get('height')
 
+        print('url', url)
+        if url.endswith('.heic') or url.endswith('.heif'):
+            # Convert to JPG
+            s3_private_bucket = get_s3_private_bucket_name()
+            signed_url = get_signed_s3_url(
+                url, s3_private_bucket,
+                settings.AWS_ACCESS_KEY,
+                settings.AWS_SECRET_KEY)
+
+            # Pass in the signed url because we need to download it.
+            url = convert_heic_to_jpg(
+                signed_url, request, s3_private_bucket,
+                settings.AWS_ACCESS_KEY, settings.AWS_SECRET_KEY)
+
         # If the form passed in a valid label, use it.
         lbl = request.POST.get('label')
         if lbl and lbl in Asset.primary_labels:
@@ -1395,14 +1413,11 @@ def dispatch(self, request, *args, **kwargs):
 
 
 class S3SignView(SignS3View):
-    private = True
-    root = 'private/'
-    acl = None
-    expiration_time = 3600 * 8  # 8 hours
-    max_file_size = 50000000  # 50mb
+    private = s3_sign_view_settings.get('private', True)
+    root = s3_sign_view_settings.get('root', 'private/')
+    acl = s3_sign_view_settings.get('acl', None)
+    expiration_time = s3_sign_view_settings.get('expiration_time', 3600 * 8)
+    max_file_size = s3_sign_view_settings.get('max_file_size', 50000000)
 
     def get_bucket(self):
-        return getattr(
-            settings,
-            'S3_PRIVATE_STORAGE_BUCKET_NAME',
-            'mediathread-private-uploads')
+        return get_s3_private_bucket_name()

mediathread/templates/main/collection_add.html

@@ -76,9 +76,10 @@ <h6 class="card-title mt-2">IMAGE / PDF</h6>
                                 <p class="card-text">
                                     Mediathread supports image files
                                     that end in .bmp, .gif, .jpg,
-                                    .jpeg, .png, .svg, .webp, .heic,
-                                    and also PDF files.  Submitted
-                                    files must be less than 50MB.
+                                    .jpeg, .png, .svg, .webp,
+                                    .heic/.heif, .avif, and also PDF
+                                    files.  Submitted files must be
+                                    less than 50MB.
                                 </p>
                                <div class="card-text">
                                    {% csrf_token %}
@@ -113,7 +114,7 @@ <h6 class="card-title mt-2">IMAGE / PDF</h6>
                                                        class="custom-file-input"
                                                        id="mediaUploadFilename"
                                                        aria-describedby="inputGroupFileAddon02"
-                                                       accept="image/bmp,image/gif,image/jpeg,image/png,image/svg,image/svg+xml,image/webp,image/heic,application/pdf" />
+                                                       accept="image/bmp,image/gif,image/jpeg,image/png,image/svg,image/svg+xml,image/webp,image/heic,image/heif,image/avif,application/pdf" />
                                                    <input type="hidden" name="url" id="uploaded-url" />
                                                    <label class="custom-file-label" for="mediaUploadFilename">
                                                        Choose file

requirements.txt

@@ -143,6 +143,8 @@ click-plugins>=1.1.1
 wcwidth==0.2.5
 django-celery-results<2.6.0
 
+Pillow==11.1.0
+pi-heif==0.21.0
 
 # memcached
 pylibmc==1.6.3;sys_platform == "linux" and python_version<"3.9"