From cd0cec7ae2b2e4875962fb1cd7bd1c0182ced7db Mon Sep 17 00:00:00 2001 From: akochari Date: Wed, 11 Dec 2024 13:40:36 +0100 Subject: [PATCH 01/10] additional env variables: --- ptp/Dockerfile | 6 ++++-- ptp/ptp/settings.py | 44 ++++++++++++++++++++++++++++++-------------- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/ptp/Dockerfile b/ptp/Dockerfile index ccdbad6..79cf584 100644 --- a/ptp/Dockerfile +++ b/ptp/Dockerfile @@ -30,7 +30,7 @@ FROM djangobase AS djangoapp COPY . /app/ WORKDIR /app -RUN mkdir /app/inference/models/ +RUN mkdir -p /app/ext_storage/{models,media,database} # Expose the port the Django app runs on EXPOSE 8000 @@ -48,8 +48,10 @@ USER $USER ENV CELERY_BROKER_URL="redis://localhost:6379/0" ENV REDIS_URL="redis://localhost:6379/0" -ENV MODEL_DIR="/app/inference/models/models" +ENV MODEL_DIR="/app/ext_storage/models" # Models will be downloaded and placed in /app/inference/models/models through SciLifeLab Serve interface +ENV DATABASE_DIR="/app/ext_storage/database" +ENV MEDIA_DIR="/app/ext_storage/media" # Start supervisord CMD ["sh", "-c", "/app/start-script.sh"] diff --git a/ptp/ptp/settings.py b/ptp/ptp/settings.py index 3024208..8cb82ac 100644 --- a/ptp/ptp/settings.py +++ b/ptp/ptp/settings.py @@ -8,11 +8,15 @@ SECRET_KEY = 'ptp_for_president_CC44aAFG' # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True + +#DEBUG = True +if os.environ.get('DEBUG', False) == 'True': + DEBUG = True + ALLOWED_HOSTS = ['*', 'localhost'] -CSRF_TRUSTED_ORIGINS=['https://ptp-inference.serve.scilifelab.se'] +CSRF_TRUSTED_ORIGINS=['https://ptp2-inference.serve.scilifelab.se'] @@ -61,11 +65,12 @@ WSGI_APPLICATION = 'ptp.wsgi.application' +DATABASE_DIR = os.environ.get('DATABASE_DIR', BASE_DIR) # Database DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': BASE_DIR / 'db.sqlite3', + 'NAME': DATABASE_DIR / 'db.sqlite3', } } @@ -103,18 +108,28 @@ # Media files (Uploaded files, results) MEDIA_URL = '/media/' -MEDIA_ROOT = os.path.join(BASE_DIR, 'media') + +MEDIA_DIR = os.environ.get('MEDIA_DIR', BASE_DIR) +MEDIA_ROOT = os.path.join(MEDIA_DIR, 'media') # Email settings (for sending job completion notifications) -if DEBUG: - EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' # Use console backend for testing -else: +if os.environ.get('EMAIL_HOST', False): EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' - EMAIL_HOST = 'smtp.your-email-provider.com' - EMAIL_PORT = 587 - EMAIL_USE_TLS = True - EMAIL_HOST_USER = 'your-email@example.com' - EMAIL_HOST_PASSWORD = 'your-email-password' + EMAIL_HOST = os.environ.get('EMAIL_HOST', 'smtp.example.com') + EMAIL_PORT = os.environ.get('EMAIL_PORT',587) + EMAIL_USE_TLS = os.environ.get('EMAIL_USE_TLS',True) + EMAIL_HOST_USER = os.environ.get('EMAIL_HOST_USER','your-email@example.com') + + # Not superfond of this.. + EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD','your-email-password') + # Perhaps? + if os.environ.get("EMAIL_PASSWORD_FILE", False): + filename = os.environ.get("EMAIL_PASSWORD_FILE", False) + with open(filename) as f: + EMAIL_HOST_PASSWORD = f.read().strip() + +else: + EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' # Use console backend for testing # Celery configuration CELERY_BROKER_URL = "redis://localhost:6379/0" @@ -133,11 +148,12 @@ # In production, it's better to use a cloud storage solution like AWS S3 # Set the SITE_URL for generating download links in emails -SITE_URL = 'https://yourdomain.com' + +SITE_URL = os.environ.get('EMAIL_DOMAIN','https://ptp2-inference.serve.scilifelab.se') # Security settings SECURE_BROWSER_XSS_FILTER = True X_FRAME_OPTIONS = 'DENY' SECURE_SSL_REDIRECT = not DEBUG CSRF_COOKIE_SECURE = not DEBUG -SESSION_COOKIE_SECURE = not DEBUG +SESSION_COOKIE_SECURE = not DEBUG \ No newline at end of file From 7b7dada63acedeae292d2d50a4a238e19d8be4cd Mon Sep 17 00:00:00 2001 From: akochari Date: Wed, 11 Dec 2024 15:07:58 +0100 Subject: [PATCH 02/10] database dir as path --- ptp/ptp/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ptp/ptp/settings.py b/ptp/ptp/settings.py index 8cb82ac..2879304 100644 --- a/ptp/ptp/settings.py +++ b/ptp/ptp/settings.py @@ -65,7 +65,7 @@ WSGI_APPLICATION = 'ptp.wsgi.application' -DATABASE_DIR = os.environ.get('DATABASE_DIR', BASE_DIR) +DATABASE_DIR = Path(os.environ.get('DATABASE_DIR', BASE_DIR)) # Database DATABASES = { 'default': { From 57a591d84c5807bff906945fe20931a473404ad9 Mon Sep 17 00:00:00 2001 From: akochari Date: Wed, 11 Dec 2024 15:17:36 +0100 Subject: [PATCH 03/10] define default debug --- ptp/ptp/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ptp/ptp/settings.py b/ptp/ptp/settings.py index 2879304..429ad74 100644 --- a/ptp/ptp/settings.py +++ b/ptp/ptp/settings.py @@ -9,7 +9,7 @@ # SECURITY WARNING: don't run with debug turned on in production! -#DEBUG = True +DEBUG = False if os.environ.get('DEBUG', False) == 'True': DEBUG = True From 432bc020bbce6f507f423322383534898ab216ea Mon Sep 17 00:00:00 2001 From: akochari Date: Wed, 11 Dec 2024 15:25:20 +0100 Subject: [PATCH 04/10] correct creation of folders --- ptp/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ptp/Dockerfile b/ptp/Dockerfile index 79cf584..853d940 100644 --- a/ptp/Dockerfile +++ b/ptp/Dockerfile @@ -30,8 +30,7 @@ FROM djangobase AS djangoapp COPY . /app/ WORKDIR /app -RUN mkdir -p /app/ext_storage/{models,media,database} - +RUN mkdir -p /app/ext_storage/models /app/ext_storage/media /app/ext_storage/database # Expose the port the Django app runs on EXPOSE 8000 From b2c70a11ff92d05342443e069c0612ef0d66b1f6 Mon Sep 17 00:00:00 2001 From: akochari Date: Wed, 11 Dec 2024 16:40:06 +0100 Subject: [PATCH 05/10] disable secure session cookie --- ptp/ptp/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ptp/ptp/settings.py b/ptp/ptp/settings.py index 429ad74..2d4cb25 100644 --- a/ptp/ptp/settings.py +++ b/ptp/ptp/settings.py @@ -156,4 +156,4 @@ X_FRAME_OPTIONS = 'DENY' SECURE_SSL_REDIRECT = not DEBUG CSRF_COOKIE_SECURE = not DEBUG -SESSION_COOKIE_SECURE = not DEBUG \ No newline at end of file +#SESSION_COOKIE_SECURE = not DEBUG From 3f6a86967c93dade144dd4e7b9facaca7034f5de Mon Sep 17 00:00:00 2001 From: akochari Date: Wed, 11 Dec 2024 16:56:22 +0100 Subject: [PATCH 06/10] try disabling ssl redirect --- ptp/ptp/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ptp/ptp/settings.py b/ptp/ptp/settings.py index 2d4cb25..b9b2dc9 100644 --- a/ptp/ptp/settings.py +++ b/ptp/ptp/settings.py @@ -154,6 +154,6 @@ # Security settings SECURE_BROWSER_XSS_FILTER = True X_FRAME_OPTIONS = 'DENY' -SECURE_SSL_REDIRECT = not DEBUG +#SECURE_SSL_REDIRECT = not DEBUG CSRF_COOKIE_SECURE = not DEBUG -#SESSION_COOKIE_SECURE = not DEBUG +SESSION_COOKIE_SECURE = not DEBUG From 80a1299db06a7c7b630920450623bc129823924f Mon Sep 17 00:00:00 2001 From: akochari Date: Wed, 11 Dec 2024 17:12:36 +0100 Subject: [PATCH 07/10] serve static in debug false --- ptp/inference/urls.py | 7 ++++++- ptp/ptp/settings.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ptp/inference/urls.py b/ptp/inference/urls.py index e0337df..e51d26c 100644 --- a/ptp/inference/urls.py +++ b/ptp/inference/urls.py @@ -1,5 +1,7 @@ +from django.conf import settings from django.urls import path from . import views +from django.conf.urls.static import static app_name = 'inference' @@ -8,4 +10,7 @@ path('', views.submit_job, name='submit_job'), path('status//', views.job_status, name='job_status'), path('stat//', views.job_status_json, name='job_status_json'), -] \ No newline at end of file +] + +if settings.DEBUG or settings.SERVE_STATIC: + urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) \ No newline at end of file diff --git a/ptp/ptp/settings.py b/ptp/ptp/settings.py index b9b2dc9..d5de58e 100644 --- a/ptp/ptp/settings.py +++ b/ptp/ptp/settings.py @@ -104,7 +104,7 @@ BASE_DIR / "staticsource", #'/var/www/static', ] - +SERVE_STATIC = True # Media files (Uploaded files, results) MEDIA_URL = '/media/' From f938cacc2814144cc4df9b24052b2d8951e15f36 Mon Sep 17 00:00:00 2001 From: akochari Date: Thu, 12 Dec 2024 17:08:44 +0100 Subject: [PATCH 08/10] restructuring to set better default env variables --- ptp/Dockerfile | 10 ++-------- ptp/inference/tasks.py | 4 +++- ptp/inference/urls.py | 7 +------ ptp/ptp/settings.py | 39 ++++++++++++++++----------------------- ptp/ptp/urls.py | 9 +++++++-- 5 files changed, 29 insertions(+), 40 deletions(-) diff --git a/ptp/Dockerfile b/ptp/Dockerfile index 853d940..d1d0e26 100644 --- a/ptp/Dockerfile +++ b/ptp/Dockerfile @@ -30,7 +30,8 @@ FROM djangobase AS djangoapp COPY . /app/ WORKDIR /app -RUN mkdir -p /app/ext_storage/models /app/ext_storage/media /app/ext_storage/database +# Models also need to be added. They be downloaded and placed in /app/ext_storage/models/ through the SciLifeLab Serve interface + # Expose the port the Django app runs on EXPOSE 8000 @@ -45,12 +46,5 @@ RUN chmod +x /app/start-django.sh # Make sure the container is running as non-root USER $USER -ENV CELERY_BROKER_URL="redis://localhost:6379/0" -ENV REDIS_URL="redis://localhost:6379/0" -ENV MODEL_DIR="/app/ext_storage/models" -# Models will be downloaded and placed in /app/inference/models/models through SciLifeLab Serve interface -ENV DATABASE_DIR="/app/ext_storage/database" -ENV MEDIA_DIR="/app/ext_storage/media" - # Start supervisord CMD ["sh", "-c", "/app/start-script.sh"] diff --git a/ptp/inference/tasks.py b/ptp/inference/tasks.py index ccfada4..8ae99ca 100644 --- a/ptp/inference/tasks.py +++ b/ptp/inference/tasks.py @@ -6,8 +6,10 @@ from django.conf import settings import pandas as pd from datetime import timezone +from pathlib import Path -model_dir = os.environ.get("MODEL_DIR", "/app/inference/models/models/") +model_dir = os.environ.get("MODEL_DIR", "/app/ext_storage/models") +Path(model_dir).mkdir(parents=True, exist_ok=True) max_models = os.environ.get("MAX_MODELS", False) @shared_task diff --git a/ptp/inference/urls.py b/ptp/inference/urls.py index e51d26c..029936c 100644 --- a/ptp/inference/urls.py +++ b/ptp/inference/urls.py @@ -1,8 +1,6 @@ from django.conf import settings from django.urls import path from . import views -from django.conf.urls.static import static - app_name = 'inference' @@ -10,7 +8,4 @@ path('', views.submit_job, name='submit_job'), path('status//', views.job_status, name='job_status'), path('stat//', views.job_status_json, name='job_status_json'), -] - -if settings.DEBUG or settings.SERVE_STATIC: - urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) \ No newline at end of file +] \ No newline at end of file diff --git a/ptp/ptp/settings.py b/ptp/ptp/settings.py index d5de58e..25d0147 100644 --- a/ptp/ptp/settings.py +++ b/ptp/ptp/settings.py @@ -13,14 +13,10 @@ if os.environ.get('DEBUG', False) == 'True': DEBUG = True - ALLOWED_HOSTS = ['*', 'localhost'] CSRF_TRUSTED_ORIGINS=['https://ptp2-inference.serve.scilifelab.se'] - - - # Application definition INSTALLED_APPS = [ 'django.contrib.admin', @@ -65,7 +61,8 @@ WSGI_APPLICATION = 'ptp.wsgi.application' -DATABASE_DIR = Path(os.environ.get('DATABASE_DIR', BASE_DIR)) +DATABASE_DIR = Path(os.environ.get('DATABASE_DIR', '/app/ext_storage/database')) +DATABASE_DIR.mkdir(parents=True, exist_ok=True) # Database DATABASES = { 'default': { @@ -109,27 +106,23 @@ # Media files (Uploaded files, results) MEDIA_URL = '/media/' -MEDIA_DIR = os.environ.get('MEDIA_DIR', BASE_DIR) +MEDIA_DIR = os.environ.get('MEDIA_DIR', '/app/ext_storage/media') MEDIA_ROOT = os.path.join(MEDIA_DIR, 'media') # Email settings (for sending job completion notifications) -if os.environ.get('EMAIL_HOST', False): - EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' - EMAIL_HOST = os.environ.get('EMAIL_HOST', 'smtp.example.com') - EMAIL_PORT = os.environ.get('EMAIL_PORT',587) - EMAIL_USE_TLS = os.environ.get('EMAIL_USE_TLS',True) - EMAIL_HOST_USER = os.environ.get('EMAIL_HOST_USER','your-email@example.com') - - # Not superfond of this.. - EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD','your-email-password') - # Perhaps? - if os.environ.get("EMAIL_PASSWORD_FILE", False): - filename = os.environ.get("EMAIL_PASSWORD_FILE", False) - with open(filename) as f: - EMAIL_HOST_PASSWORD = f.read().strip() - -else: - EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' # Use console backend for testing +EMAIL_BACKEND = ( + "django.core.mail.backends.smtp.EmailBackend" if not DEBUG else "django.core.mail.backends.console.EmailBackend" +) +EMAIL_HOST = os.environ.get('EMAIL_HOST', "smtp.gmail.com") +EMAIL_PORT = os.environ.get('EMAIL_PORT', 465) +EMAIL_USE_SSL = os.environ.get('EMAIL_USE_SSL', True) +EMAIL_USE_TLS = os.environ.get('EMAIL_USE_TLS', False) +EMAIL_HOST_USER = os.environ.get('EMAIL_HOST_USER','your-email@example.com') +EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD','your-email-password') +if os.environ.get("EMAIL_PASSWORD_FILE", False): + filename = os.environ.get("EMAIL_PASSWORD_FILE", False) + with open(filename) as f: + EMAIL_HOST_PASSWORD = f.read().strip() # Celery configuration CELERY_BROKER_URL = "redis://localhost:6379/0" diff --git a/ptp/ptp/urls.py b/ptp/ptp/urls.py index e765a2b..ab652aa 100644 --- a/ptp/ptp/urls.py +++ b/ptp/ptp/urls.py @@ -15,13 +15,18 @@ 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ from django.contrib import admin -from django.urls import path, include +from django.urls import path, include, re_path from django.conf import settings from django.conf.urls.static import static +from django.views.static import serve urlpatterns = [ path('admin/', admin.site.urls), path('', include('inference.urls')), -] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) +] +urlpatterns += [ + re_path(r'^static/(?P.*)$', serve, {'document_root': settings.STATIC_ROOT}), + re_path(r'^media/(?P.*)$', serve, {'document_root': settings.MEDIA_ROOT}), +] \ No newline at end of file From d3767e36cc6178b422da0b668d02f193adbb930d Mon Sep 17 00:00:00 2001 From: akochari Date: Thu, 12 Dec 2024 17:18:39 +0100 Subject: [PATCH 09/10] clean up --- ptp/Dockerfile | 1 + ptp/ptp/settings.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ptp/Dockerfile b/ptp/Dockerfile index d1d0e26..6ab3ab5 100644 --- a/ptp/Dockerfile +++ b/ptp/Dockerfile @@ -30,6 +30,7 @@ FROM djangobase AS djangoapp COPY . /app/ WORKDIR /app +RUN mkdir -p /app/ext_storage/ # Models also need to be added. They be downloaded and placed in /app/ext_storage/models/ through the SciLifeLab Serve interface # Expose the port the Django app runs on diff --git a/ptp/ptp/settings.py b/ptp/ptp/settings.py index 25d0147..dfd6caa 100644 --- a/ptp/ptp/settings.py +++ b/ptp/ptp/settings.py @@ -101,7 +101,6 @@ BASE_DIR / "staticsource", #'/var/www/static', ] -SERVE_STATIC = True # Media files (Uploaded files, results) MEDIA_URL = '/media/' @@ -142,7 +141,7 @@ # Set the SITE_URL for generating download links in emails -SITE_URL = os.environ.get('EMAIL_DOMAIN','https://ptp2-inference.serve.scilifelab.se') +SITE_URL = os.environ.get('SITE_URL','https://ptp2-inference.serve.scilifelab.se') # Security settings SECURE_BROWSER_XSS_FILTER = True From 84858771d2d09e277b776186c62fe816d5b43a09 Mon Sep 17 00:00:00 2001 From: akochari Date: Thu, 12 Dec 2024 17:27:50 +0100 Subject: [PATCH 10/10] random secret --- README.md | 20 +++++++++++--------- ptp/ptp/settings.py | 3 ++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index bdc0496..76e4794 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ To use this project, follow these steps: docker-compose up --build ``` -### Scilifelab Deployment +### Scilifelab Serve Deployment 1. Build container ```sh @@ -54,17 +54,19 @@ To use this project, follow these steps: ``` 2. Push container to registry - 3. Deploy + - By default the folder /app/ext_storage/ should be mounted to outside storage to avoid ram bloat. It will contain database, media, and models. + - Place the downloaded models (from https://huggingface.co/pharmbio/ptp) to /app/ext_storage/models/ 4. Configuration - - Set environment variables - Use `DOWNLOAD=true` to download models prior to startup. - - preferebly mount `/app/inference/models` directory to outside storage to avoid ram bloat. - - - Use `MAX_MODELS` to limit the number of active models (for debug purposes) .otherwise it iterates all models present (currently 800+). - -- If desired use `MODEL_DIR` to change path where model are found (default /app/inference/models/models) + - Currently static files are served by Django itself even with DEBUG=False + - All defaults of environmental variables make sense for SciLifeLab Serve. Here are some environmental variables that can be changed if needed: + - `EMAIL_HOST`, `EMAIL_PORT`, `EMAIL_HOST_USER` etc for email settings. + - `MAX_MODELS` to limit the number of active models (for debug purposes) .otherwise it iterates all models present (currently 800+). + - `MEDIA_DIR` for media directory + - `MODEL_DIR` for models directory + - `DATABASE_DIR` for database directory + - `SITE_URL` for generating download links in emails ## References diff --git a/ptp/ptp/settings.py b/ptp/ptp/settings.py index dfd6caa..bd6ce57 100644 --- a/ptp/ptp/settings.py +++ b/ptp/ptp/settings.py @@ -5,7 +5,8 @@ BASE_DIR = Path(__file__).resolve().parent.parent # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'ptp_for_president_CC44aAFG' +import random +SECRET_KEY = os.environ.get('SECRET_KEY', ''.join([random.SystemRandom().choice('abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') for i in range(50)])) # SECURITY WARNING: don't run with debug turned on in production!