diff --git a/deployment/docker-compose.test.yml b/deployment/docker-compose.test.yml index 8e8e223..5e66988 100644 --- a/deployment/docker-compose.test.yml +++ b/deployment/docker-compose.test.yml @@ -3,18 +3,34 @@ volumes: static-data: media-data: + minio-data: # Exactly the same as production but for dev env, we expose the port and uses # different port for the web. version: '3.4' services: + minio: + image: quay.io/minio/minio:RELEASE.2024-03-30T09-41-56Z.fips + command: minio server /data --console-address ":9001" + ports: + - "9010:9000" + - "9011:9001" + environment: + - MINIO_ROOT_USER=minio_user + - MINIO_ROOT_PASSWORD=minio_password + - MINIO_HTTP_TRACE + volumes: + - minio-data:/data + restart: always + dev: image: ${APP_IMAGE}:dev container_name: "dev" links: - db - redis + - minio volumes: - static-data:/home/web/static - media-data:/home/web/media @@ -41,6 +57,13 @@ services: - ADMIN_EMAIL=admin@example.com - SENTRY_DSN= - SENTRY_ENVIRONMENT=staging + + # Minio + - MINIO_AWS_ACCESS_KEY_ID=minio_user + - MINIO_AWS_SECRET_ACCESS_KEY=minio_password + - MINIO_AWS_ENDPOINT_URL=http://minio:9000/ + - MINIO_AWS_BUCKET_NAME=tomorrownow + - MINIO_AWS_DIR_PREFIX=dev/media entrypoint: [ ] ports: # for django test server diff --git a/django_project/core/settings/test.py b/django_project/core/settings/test.py index 18799c7..8a78a62 100644 --- a/django_project/core/settings/test.py +++ b/django_project/core/settings/test.py @@ -16,18 +16,4 @@ 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', } } - -STORAGES = { - "default": { - "BACKEND": "django.core.files.storage.FileSystemStorage", - "OPTIONS": { - "location": "/home/web/media/default_test", - }, - }, - "staticfiles": { - "BACKEND": ( - "django.contrib.staticfiles.storage.ManifestStaticFilesStorage" - ), - } -} EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' diff --git a/django_project/core/tests/test_s3.py b/django_project/core/tests/test_s3.py new file mode 100644 index 0000000..5504e9d --- /dev/null +++ b/django_project/core/tests/test_s3.py @@ -0,0 +1,37 @@ +# coding=utf-8 +""" +Tomorrow Now GAP. + +.. note:: Unit test for S3 utils. +""" +import os + +from django.core.files.base import ContentFile +from django.core.files.storage import default_storage +from django.test import TestCase + +from core.utils.s3 import zip_folder_in_s3, remove_s3_folder + + +class TestS3Utilities(TestCase): + """Test S3 utilities.""" + + def test_zip_folder_in_s3(self): + """Test zip folder in S3.""" + folder = 'test_folder' + remove_s3_folder(default_storage, folder) + default_storage.save( + os.path.join(folder, 'test'), ContentFile(b"new content") + ) + default_storage.save( + os.path.join(folder, 'test_2'), ContentFile(b"new content") + ) + zip_folder_in_s3( + default_storage, folder, 'test_folder.zip' + ) + self.assertTrue( + default_storage.exists('test_folder.zip') + ) + self.assertFalse( + default_storage.exists(folder) + ) diff --git a/django_project/core/utils/s3.py b/django_project/core/utils/s3.py index d0c2aa3..0b27c09 100644 --- a/django_project/core/utils/s3.py +++ b/django_project/core/utils/s3.py @@ -18,6 +18,9 @@ def zip_folder_in_s3( """Zip folder contents into a zip file on S3.""" zip_buffer = io.BytesIO() + if s3_storage.exists(zip_file_name): + s3_storage.delete(zip_file_name) + # Create buffer zip file with zipfile.ZipFile(zip_buffer, 'w') as zip_file: # Get file list @@ -37,6 +40,7 @@ def zip_folder_in_s3( # Save it to S3 zip_buffer.seek(0) s3_storage.save(zip_file_name, ContentFile(zip_buffer.read())) + remove_s3_folder(s3_storage, folder_path) def remove_s3_folder(s3_storage: S3Boto3Storage, folder_path: str): diff --git a/django_project/gap/ingestor/tio_shortterm.py b/django_project/gap/ingestor/tio_shortterm.py index c0a668b..a44a92f 100644 --- a/django_project/gap/ingestor/tio_shortterm.py +++ b/django_project/gap/ingestor/tio_shortterm.py @@ -18,7 +18,7 @@ from django.core.files.storage import default_storage from django.utils import timezone -from core.utils.s3 import zip_folder_in_s3, remove_s3_folder +from core.utils.s3 import zip_folder_in_s3 from gap.ingestor.base import BaseIngestor from gap.models import ( CastType, CollectorSession, DataSourceFile, DatasetStore, Grid @@ -106,7 +106,6 @@ def _run(self): zip_folder_in_s3( s3_storage, folder_path=folder, zip_file_name=zip_file ) - remove_s3_folder(s3_storage, folder) def run(self): """Run Tio Short Term Ingestor.""" diff --git a/django_project/spw/tests/test_crop_insight_generator.py b/django_project/spw/tests/test_crop_insight_generator.py index 5e02d09..4abd6c7 100644 --- a/django_project/spw/tests/test_crop_insight_generator.py +++ b/django_project/spw/tests/test_crop_insight_generator.py @@ -294,7 +294,7 @@ def create_timeline_data( self.request.refresh_from_db() # Check the if of farm group in the path - self.assertTrue(f'{self.farm_group.id}/' in self.request.file.path) + self.assertTrue(f'{self.farm_group.id}/' in self.request.file.name) with self.request.file.open(mode='r') as csv_file: csv_reader = csv.reader(csv_file)