Merge pull request #60 from ImperialCollegeLondon/release/0.4.0

Release v0.4.0

.github/workflows/python-test.yml

@@ -5,9 +5,9 @@ name: Python test
 
 on:
   push:
-    branches: [ master ]
+    branches: [ main ]
   pull_request:
-    branches: [ master ]
+    branches: [ main ]
   release:
     types: [created, edited, published]
 

README.md

@@ -6,8 +6,13 @@
 
 django-drf-filepond supports remote storage of uploads via [*django-storages*](https://django-storages.readthedocs.io/)
 
+:new: Support for large file uploads (> ~2GB) now available from version 0.4.0 (see [#57](https://github.com/ImperialCollegeLondon/django-drf-filepond/issues/57)).<br/>
 :new: Support for filepond chunked uploads now available from version 0.3.0.
 
+
+:warning: **Release v0.4.0 includes a database update** in relation to the large upload support. If you don't wish to apply DB migrations within your application at this stage, please continue to use django-drf-filepond [v0.3.1](https://pypi.org/project/django-drf-filepond/0.3.1/). :warning:
+
+
 Further documentation and a tutorial are available at [https://django-drf-filepond.readthedocs.io](https://django-drf-filepond.readthedocs.io).
 
 ### Installation

django_drf_filepond/uploaders.py

@@ -202,7 +202,7 @@ def _handle_new_chunk_upload(self, request, upload_id, file_id):
                             status=status.HTTP_500_INTERNAL_SERVER_ERROR)
         if os.path.exists(base_loc):
             try:
-                os.makedirs(chunk_dir, exist_ok=False)
+                os.makedirs(chunk_dir)
             except OSError as e:
                 LOG.debug('Unable to create chunk storage dir: %s' %
                           (str(e)))

tests/test_process_view.py

@@ -1,3 +1,4 @@
+from django_drf_filepond.models import TemporaryUploadChunked
 from io import BytesIO
 import logging
 import os
@@ -46,6 +47,9 @@
 # test_new_chunked_upload_request: Check that a new chunked upload request
 #    results in handle_upload being called on the chunked uploader class.
 #
+# test_chunked_upload_large_file: Test that a new chunked upload request for
+#    a file greater than ~2GB in size is correctly handled.
+#
 # UPDATE: June 2021:
 # test_process_data_BASE_DIR_pathlib: Tests the upload process when BASE_DIR
 #    is set as a pathlib.Path object as it is by default in more recent Django
@@ -58,6 +62,8 @@ def setUp(self):
         data.write(os.urandom(16384))
         self.test_data = data
 
+        self.rf = RequestFactory()
+
     def test_process_data(self):
         self._process_data()
 
@@ -93,8 +99,7 @@ def test_UPLOAD_TMP_not_set(self):
         # Set up and run request
         (encoded_form, content_type) = self._get_encoded_form('testfile.dat')
 
-        rf = RequestFactory()
-        req = rf.post(reverse('process'),
+        req = self.rf.post(reverse('process'),
                       data=encoded_form, content_type=content_type)
         pv = views.ProcessView.as_view()
         response = pv(req)
@@ -109,8 +114,7 @@ def test_process_invalid_storage_location(self):
         views.storage = FileSystemStorage(location='/django_test')
         (encoded_form, content_type) = self._get_encoded_form('testfile.dat')
 
-        rf = RequestFactory()
-        req = rf.post(reverse('process'),
+        req = self.rf.post(reverse('process'),
                       data=encoded_form, content_type=content_type)
         pv = views.ProcessView.as_view()
         response = pv(req)
@@ -127,8 +131,7 @@ def test_process_invalid_data(self):
         upload_form = {'somekey':
                        SimpleUploadedFile('test.txt', self.test_data.read())}
         enc_form = encode_multipart('abc', upload_form)
-        rf = RequestFactory()
-        req = rf.post(reverse('process'), data=enc_form,
+        req = self.rf.post(reverse('process'), data=enc_form,
                       content_type='multipart/form-data; boundary=abc')
         pv = views.ProcessView.as_view()
         response = pv(req)
@@ -143,8 +146,7 @@ def test_upload_non_file_data(self):
         cf = ContentFile(self.test_data.read(), name='test.txt')
         upload_form = {'filepond': cf}
         enc_form = encode_multipart('abc', upload_form)
-        rf = RequestFactory()
-        req = rf.post(reverse('process'), data=enc_form,
+        req = self.rf.post(reverse('process'), data=enc_form,
                       content_type='multipart/form-data; boundary=abc')
         req.FILES['filepond'] = cf
         pv = views.ProcessView.as_view()
@@ -170,8 +172,7 @@ def test_process_data_invalid_different_field_name(self):
         boundary = str(uuid.uuid4()).replace('-', '')
         enc_form = encode_multipart(boundary, upload_form)
 
-        rf = RequestFactory()
-        req = rf.post(reverse('process'), data=enc_form,
+        req = self.rf.post(reverse('process'), data=enc_form,
                       content_type='multipart/form-data; boundary=%s'
                       % boundary)
         pv = views.ProcessView.as_view()
@@ -188,8 +189,7 @@ def test_store_upload_with_storage_outside_BASE_DIR_without_enable(self):
         views.storage = FileSystemStorage(location='/tmp/uploads')
         (encoded_form, content_type) = self._get_encoded_form('testfile.dat')
 
-        rf = RequestFactory()
-        req = rf.post(reverse('process'),
+        req = self.rf.post(reverse('process'),
                       data=encoded_form, content_type=content_type)
         pv = views.ProcessView.as_view()
         response = pv(req)
@@ -213,8 +213,7 @@ def test_store_upload_with_storage_outside_BASE_DIR_with_enable(self):
 
         (encoded_form, content_type) = self._get_encoded_form('testfile.dat')
 
-        rf = RequestFactory()
-        req = rf.post(reverse('process'),
+        req = self.rf.post(reverse('process'),
                       data=encoded_form, content_type=content_type)
         pv = views.ProcessView.as_view()
         response = pv(req)
@@ -232,8 +231,7 @@ def test_relative_UPLOAD_TMP_outside_base_dir_not_allowed(self):
         # Set up and run request
         (encoded_form, content_type) = self._get_encoded_form('testfile.dat')
 
-        rf = RequestFactory()
-        req = rf.post(reverse('process'),
+        req = self.rf.post(reverse('process'),
                       data=encoded_form, content_type=content_type)
         pv = views.ProcessView.as_view()
         response = pv(req)
@@ -264,15 +262,38 @@ def test_new_chunked_upload_request(self, mock_chunked_ul):
                                                 content_type='text/plain')
         (encoded_form, content_type) = self._get_encoded_form('testfile.dat',
                                                               file_spec='{}')
-        rf = RequestFactory(HTTP_UPLOAD_LENGTH=1048576)
-        req = rf.post(reverse('process'),
-                      data=encoded_form, content_type=content_type)
+        req = self.rf.post(reverse('process'),
+                           data=encoded_form, content_type=content_type,
+                           HTTP_UPLOAD_LENGTH=1048576)
         pv = views.ProcessView.as_view()
         pv(req)
         django_drf_filepond.views._get_file_id = original_gfid
 
         mock_chunked_ul.assert_called_once_with(ANY, upload_id, file_id)
 
+    def test_chunked_upload_large_file(self):
+        # Mock _get_file_id to return the specified file and upload IDs.
+        upload_id = ensure_text('ababababababababababab')
+        file_id = ensure_text('xyxyxyxyxyxyxyxyxyxyxy')
+        mock_gfid = MagicMock(spec='django_drf_filepond.utils._get_file_id')
+        mock_gfid.side_effect = [upload_id, file_id, upload_id, file_id]
+        original_gfid = django_drf_filepond.views._get_file_id
+        django_drf_filepond.views._get_file_id = mock_gfid
+
+        (encoded_form, content_type) = self._get_encoded_form('largefile.dat',
+                                                              file_spec='{}')
+        # Create a post request for a 3GB file...
+        req = self.rf.post(reverse('process'),
+                           data=encoded_form, content_type=content_type,
+                           HTTP_UPLOAD_LENGTH=3221225472)
+        # Run the request and check for the TemporaryUploadChunked object
+        pv = views.ProcessView.as_view()
+        response = pv(req)
+        django_drf_filepond.views._get_file_id = original_gfid
+        self.assertEqual(response.status_code, 200)
+        tuc = TemporaryUploadChunked.objects.get(upload_id=upload_id)
+        self.assertEqual(tuc.total_size, 3221225472)
+
     def _process_data(self, upload_field_name=None):
         tmp_upload_dir = drf_filepond_settings.UPLOAD_TMP
         self.uploaddir_exists_pre_test = os.path.exists(tmp_upload_dir)