diff --git a/mediathread/assetmgr/models.py b/mediathread/assetmgr/models.py index 7945d7cf0..c0789d4b3 100755 --- a/mediathread/assetmgr/models.py +++ b/mediathread/assetmgr/models.py @@ -14,7 +14,9 @@ from tagging.models import Tag from mediathread.assetmgr.custom_storage import private_storage -from mediathread.assetmgr.utils import get_signed_s3_url +from mediathread.assetmgr.utils import ( + get_signed_s3_url, get_s3_private_bucket_name +) METADATA_ORIGINAL_OWNER = 'Original Owner' @@ -397,10 +399,7 @@ def is_panopto(self): return self.label == 'mp4_panopto' def signed_url(self): - s3_private_bucket = getattr( - settings, - 'S3_PRIVATE_STORAGE_BUCKET_NAME', - 'mediathread-private-uploads') + s3_private_bucket = get_s3_private_bucket_name() if s3_private_bucket in self.url: return get_signed_s3_url( self.url, s3_private_bucket, @@ -413,10 +412,7 @@ def signed_url(self): return self.url def url_processed(self, request): - s3_private_bucket = getattr( - settings, - 'S3_PRIVATE_STORAGE_BUCKET_NAME', - 'mediathread-private-uploads') + s3_private_bucket = get_s3_private_bucket_name() if s3_private_bucket in self.url: return get_signed_s3_url( self.url, s3_private_bucket, diff --git a/mediathread/assetmgr/utils.py b/mediathread/assetmgr/utils.py index 00224581f..10a13cf2c 100644 --- a/mediathread/assetmgr/utils.py +++ b/mediathread/assetmgr/utils.py @@ -1,15 +1,99 @@ -from urllib.parse import urlparse import boto3 -from s3sign.utils import create_presigned_url, s3_config +import requests +import os +import tempfile +from django.conf import settings +from django.utils import timezone +from os.path import basename, splitext +from PIL import Image +from pi_heif import register_heif_opener +from s3sign.utils import ( + DEFAULT_AWS_REGION, + create_presigned_url, s3_config, get_object_name, upload_file +) +from urllib.parse import urlparse + + +s3_sign_view_settings = { + 'private': True, + 'root': 'private/', + 'acl': None, + 'expiration_time': 3600 * 8, # 8 hours + 'max_file_size': 50000000, # 50mb +} + +def get_s3_private_bucket_name() -> str: + return getattr( + settings, + 'S3_PRIVATE_STORAGE_BUCKET_NAME', + 'mediathread-private-uploads') -def get_signed_s3_url(url, bucket, aws_key, aws_secret): - s3_client = boto3.client( + +def get_s3_client(aws_key, aws_secret): + aws_region_name = DEFAULT_AWS_REGION + if hasattr(settings, 'AWS_S3_REGION_NAME'): + aws_region_name = settings.AWS_S3_REGION_NAME + + return boto3.client( 's3', config=s3_config, + region_name=aws_region_name, aws_access_key_id=aws_key, aws_secret_access_key=aws_secret) + +def get_signed_s3_url(url: str, bucket: str, aws_key: str, aws_secret: str): + s3_client = get_s3_client(aws_key, aws_secret) + url = urlparse(url) object_name = url.path.lstrip('/') object_name = object_name.replace(bucket + '/', '') return create_presigned_url(s3_client, bucket, object_name, 3600) + + +def convert_heic_to_jpg( + url: str, request: object, bucket: str, + aws_key: str, aws_secret: str +) -> str: + """ + Given an heic image url, convert it to a JPEG. This comprises a + few steps: + * Download the file + * Do the conversion + * Upload jpeg to S3 + * Return new url + """ + response = requests.get(url, stream=True) + + # Sort out the new filename + parsed_url = urlparse(url) + filename = splitext(basename(parsed_url.path))[0] + filename = filename + '.jpg' + + # Open the file and convert it + register_heif_opener() + im = Image.open(response.raw) + rgb_im = im.convert('RGB') + tmp_jpeg, tmp_jpeg_path = tempfile.mkstemp(suffix='.jpg') + rgb_im.save(tmp_jpeg_path) + width, height = rgb_im.size + + # upload to S3, return source url + s3_client = get_s3_client(aws_key, aws_secret) + object_name = get_object_name(timezone.now(), '.jpg', + s3_sign_view_settings.get('root')) + uploaded = upload_file(s3_client, tmp_jpeg_path, bucket, object_name) + + os.remove(tmp_jpeg_path) + + data = { + 'url': url, + 'width': width, + 'height': height, + } + + if uploaded: + url = 'https://{}.s3.amazonaws.com/{}'.format(bucket, object_name) + data['url'] = url + + return data diff --git a/mediathread/assetmgr/views.py b/mediathread/assetmgr/views.py index 9636b5bb7..acd476f8c 100644 --- a/mediathread/assetmgr/views.py +++ b/mediathread/assetmgr/views.py @@ -37,6 +37,10 @@ Asset, Source, ExternalCollection, SuggestedExternalCollection ) +from mediathread.assetmgr.utils import ( + s3_sign_view_settings, + get_signed_s3_url, get_s3_private_bucket_name, convert_heic_to_jpg +) from mediathread.djangosherd.api import DiscussionIndexResource from mediathread.djangosherd.models import SherdNote, DiscussionIndex from mediathread.djangosherd.views import create_annotation, edit_annotation, \ @@ -375,6 +379,22 @@ def post(self, request, *args, **kwargs): width = request.POST.get('width') height = request.POST.get('height') + if url.endswith('.heic') or url.endswith('.heif'): + # Convert to JPG + s3_private_bucket = get_s3_private_bucket_name() + signed_url = get_signed_s3_url( + url, s3_private_bucket, + settings.AWS_ACCESS_KEY, + settings.AWS_SECRET_KEY) + + # Pass in the signed url because we need to download it. + jpg_data = convert_heic_to_jpg( + signed_url, request, s3_private_bucket, + settings.AWS_ACCESS_KEY, settings.AWS_SECRET_KEY) + url = jpg_data.get('url') + width = jpg_data.get('width') + height = jpg_data.get('height') + # If the form passed in a valid label, use it. lbl = request.POST.get('label') if lbl and lbl in Asset.primary_labels: @@ -1395,14 +1415,11 @@ def dispatch(self, request, *args, **kwargs): class S3SignView(SignS3View): - private = True - root = 'private/' - acl = None - expiration_time = 3600 * 8 # 8 hours - max_file_size = 50000000 # 50mb + private = s3_sign_view_settings.get('private', True) + root = s3_sign_view_settings.get('root', 'private/') + acl = s3_sign_view_settings.get('acl', None) + expiration_time = s3_sign_view_settings.get('expiration_time', 3600 * 8) + max_file_size = s3_sign_view_settings.get('max_file_size', 50000000) def get_bucket(self): - return getattr( - settings, - 'S3_PRIVATE_STORAGE_BUCKET_NAME', - 'mediathread-private-uploads') + return get_s3_private_bucket_name() diff --git a/mediathread/templates/main/collection_add.html b/mediathread/templates/main/collection_add.html index cfb086db4..d72529469 100644 --- a/mediathread/templates/main/collection_add.html +++ b/mediathread/templates/main/collection_add.html @@ -76,9 +76,10 @@
IMAGE / PDF

Mediathread supports image files that end in .bmp, .gif, .jpg, - .jpeg, .png, .svg, .webp, .heic, - and also PDF files. Submitted - files must be less than 50MB. + .jpeg, .png, .svg, .webp, + .heic/.heif, .avif, and also PDF + files. Submitted files must be + less than 50MB.

{% csrf_token %} @@ -113,7 +114,7 @@
IMAGE / PDF
class="custom-file-input" id="mediaUploadFilename" aria-describedby="inputGroupFileAddon02" - accept="image/bmp,image/gif,image/jpeg,image/png,image/svg,image/svg+xml,image/webp,image/heic,application/pdf" /> + accept="image/bmp,image/gif,image/jpeg,image/png,image/svg,image/svg+xml,image/webp,image/heic,image/heif,image/avif,application/pdf" />