DjangoCSV - v0.6.4

- Changed .csv file generation logic (now created in buffer than uploaded to AWS S3 media folder, and deleted from buffer)
- Returned database in production

DjangoCSV/settings.py

@@ -73,24 +73,24 @@
 # Database
 # https://docs.djangoproject.com/en/3.1/ref/settings/#databases
 
-"""if DEBUG:
+if DEBUG:
     DATABASES = {
         'default': {
             'ENGINE': 'django.db.backends.sqlite3',
             'NAME': BASE_DIR / 'db.sqlite3',
         }
     }
-else:"""
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.postgresql',
-        'NAME': env("DATABASE_NAME"),
-        'HOST': env("DATABASE_HOST"),
-        'PORT': 5432,
-        'USER': env("DATABASE_USER"),
-        'PASSWORD': env("DATABASE_PASSWORD"),
+else:
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.postgresql',
+            'NAME': env("DATABASE_NAME"),
+            'HOST': env("DATABASE_HOST"),
+            'PORT': 5432,
+            'USER': env("DATABASE_USER"),
+            'PASSWORD': env("DATABASE_PASSWORD"),
+        }
     }
-}
 
 # Password validation
 # https://docs.djangoproject.com/en/3.1/ref/settings/#auth-password-validators
@@ -134,7 +134,7 @@
 ]
 
 MEDIA_URL = '/media/'
-MEDIA_ROOT = BASE_DIR / "resources/media"
+#MEDIA_ROOT = BASE_DIR / "resources/media"
 
 # Configuring celery broker in our case it is Redis
 
@@ -159,3 +159,6 @@
 STATICFILES_LOCATION = 'static'
 STATICFILES_STORAGE = 'custom_storages.StaticStorage'
 
+MEDIAFILES_LOCATION = 'media'
+DEFAULT_FILE_STORAGE = 'custom_storages.MediaStorage'
+

custom_storages.py

@@ -4,3 +4,6 @@
 
 class StaticStorage(S3Boto3Storage):
     location = settings.STATICFILES_LOCATION
+
+class MediaStorage(S3Boto3Storage):
+    location = settings.MEDIAFILES_LOCATION

extensions/data_generator.py

@@ -1,11 +1,12 @@
 from main.models import SchemasColumn, DataSets
-from pathlib import Path
+from django.core.files import File
+from os import remove
 
 from .small_generators import *
 import csv
 
 
-def generate_file(dataset_id, user_dir):
+def generate_file(dataset_id):
     """Generating .csv file from schema to user directory"""
     dataset = DataSets.objects.get(id=dataset_id)
     column_separator = dataset.schema.column_separator
@@ -21,20 +22,18 @@ def generate_file(dataset_id, user_dir):
     for i in range(rows_amount):
         row_list.append(generate_row_from_pattern(pattern))
 
-    Path(user_dir).mkdir(parents=True, exist_ok=True)
-    filename = f"{user_dir}/CSV_{dataset.date_created.strftime('%Y-%m-%d %H:%M:%S')}.csv"
+    filename = f"CSV_{dataset.date_created.strftime('%Y-%m-%d %H:%M:%S')}.csv"
 
     with open(filename, "w", newline='') as file:
         writer = csv.writer(file, quoting=csv.QUOTE_NONNUMERIC, delimiter=column_separator, quotechar=string_character)
         writer.writerows(row_list)
 
-    filename = Path(filename)
-
-    dataset.file.name = f"{filename.parent / filename.name}"
+    dataset.file.save(filename, File(open(filename)))
     dataset.ready = True
-
     dataset.save()
 
+    remove(filename)
+
 def generate_column_pattern(column_set):
     """
     Generating pattern for future parsing and row generation

main/tasks.py

@@ -4,7 +4,7 @@
 
 
 @shared_task
-def generate_file_async(dataset_form, user_dir):
+def generate_file_async(dataset_form):
     """Celery function wrapper for simple function"""
-    generate_file(dataset_form, user_dir)
+    generate_file(dataset_form)
 

main/views.py

@@ -115,11 +115,8 @@ def dataset(request, id):
                 dataset_form.schema = schema
                 dataset_form.save()
 
-                # Getting user directory location
-                user_dir = settings.MEDIA_ROOT / f"user_{request.user.id}"
-
                 # Sending file generation to Celery | Redis
-                generate_file_async.apply_async((dataset_form.id, str(user_dir)))
+                generate_file_async.apply_async((dataset_form.id,))
 
                 messages.add_message(request, messages.SUCCESS, "All clear, now please wait till Python generate file "
                                                                 "and AJAX update status")