Add minio to tests

deployment/docker-compose.test.yml

@@ -3,18 +3,34 @@
 volumes:
   static-data:
   media-data:
+  minio-data:
 
 # Exactly the same as production but for dev env, we expose the port and uses
 # different port for the web.
 version: '3.4'
 services:
 
+  minio:
+    image: quay.io/minio/minio:RELEASE.2024-03-30T09-41-56Z.fips
+    command: minio server /data --console-address ":9001"
+    ports:
+      - "9010:9000"
+      - "9011:9001"
+    environment:
+      - MINIO_ROOT_USER=minio_user
+      - MINIO_ROOT_PASSWORD=minio_password
+      - MINIO_HTTP_TRACE
+    volumes:
+      - minio-data:/data
+    restart: always
+
   dev:
     image: ${APP_IMAGE}:dev
     container_name: "dev"
     links:
       - db
       - redis
+      - minio
     volumes:
       - static-data:/home/web/static
       - media-data:/home/web/media
@@ -41,6 +57,13 @@ services:
       - [email protected]
       - SENTRY_DSN=
       - SENTRY_ENVIRONMENT=staging
+
+      # Minio
+      - MINIO_AWS_ACCESS_KEY_ID=minio_user
+      - MINIO_AWS_SECRET_ACCESS_KEY=minio_password
+      - MINIO_AWS_ENDPOINT_URL=http://minio:9000/
+      - MINIO_AWS_BUCKET_NAME=tomorrownow
+      - MINIO_AWS_DIR_PREFIX=dev/media
     entrypoint: [ ]
     ports:
       # for django test server

django_project/core/settings/test.py

@@ -16,18 +16,4 @@
         'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
     }
 }
-
-STORAGES = {
-    "default": {
-        "BACKEND": "django.core.files.storage.FileSystemStorage",
-        "OPTIONS": {
-            "location": "/home/web/media/default_test",
-        },
-    },
-    "staticfiles": {
-        "BACKEND": (
-            "django.contrib.staticfiles.storage.ManifestStaticFilesStorage"
-        ),
-    }
-}
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'

django_project/core/tests/test_s3.py

@@ -0,0 +1,37 @@
+# coding=utf-8
+"""
+Tomorrow Now GAP.
+
+.. note:: Unit test for S3 utils.
+"""
+import os
+
+from django.core.files.base import ContentFile
+from django.core.files.storage import default_storage
+from django.test import TestCase
+
+from core.utils.s3 import zip_folder_in_s3, remove_s3_folder
+
+
+class TestS3Utilities(TestCase):
+    """Test S3 utilities."""
+
+    def test_zip_folder_in_s3(self):
+        """Test zip folder in S3."""
+        folder = 'test_folder'
+        remove_s3_folder(default_storage, folder)
+        default_storage.save(
+            os.path.join(folder, 'test'), ContentFile(b"new content")
+        )
+        default_storage.save(
+            os.path.join(folder, 'test_2'), ContentFile(b"new content")
+        )
+        zip_folder_in_s3(
+            default_storage, folder, 'test_folder.zip'
+        )
+        self.assertTrue(
+            default_storage.exists('test_folder.zip')
+        )
+        self.assertFalse(
+            default_storage.exists(folder)
+        )

django_project/core/utils/s3.py

@@ -18,6 +18,9 @@ def zip_folder_in_s3(
     """Zip folder contents into a zip file on S3."""
     zip_buffer = io.BytesIO()
 
+    if s3_storage.exists(zip_file_name):
+        s3_storage.delete(zip_file_name)
+
     # Create buffer zip file
     with zipfile.ZipFile(zip_buffer, 'w') as zip_file:
         # Get file list
@@ -37,6 +40,7 @@ def zip_folder_in_s3(
     # Save it to S3
     zip_buffer.seek(0)
     s3_storage.save(zip_file_name, ContentFile(zip_buffer.read()))
+    remove_s3_folder(s3_storage, folder_path)
 
 
 def remove_s3_folder(s3_storage: S3Boto3Storage, folder_path: str):

django_project/gap/ingestor/tio_shortterm.py

@@ -18,7 +18,7 @@
 from django.core.files.storage import default_storage
 from django.utils import timezone
 
-from core.utils.s3 import zip_folder_in_s3, remove_s3_folder
+from core.utils.s3 import zip_folder_in_s3
 from gap.ingestor.base import BaseIngestor
 from gap.models import (
     CastType, CollectorSession, DataSourceFile, DatasetStore, Grid
@@ -106,7 +106,6 @@ def _run(self):
         zip_folder_in_s3(
             s3_storage, folder_path=folder, zip_file_name=zip_file
         )
-        remove_s3_folder(s3_storage, folder)
 
     def run(self):
         """Run Tio Short Term Ingestor."""

django_project/spw/tests/test_crop_insight_generator.py

@@ -294,7 +294,7 @@ def create_timeline_data(
         self.request.refresh_from_db()
 
         # Check the if of farm group in the path
-        self.assertTrue(f'{self.farm_group.id}/' in self.request.file.path)
+        self.assertTrue(f'{self.farm_group.id}/' in self.request.file.name)
 
         with self.request.file.open(mode='r') as csv_file:
             csv_reader = csv.reader(csv_file)