diff --git a/README.md b/README.md index bdc0496..76e4794 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ To use this project, follow these steps: docker-compose up --build ``` -### Scilifelab Deployment +### Scilifelab Serve Deployment 1. Build container ```sh @@ -54,17 +54,19 @@ To use this project, follow these steps: ``` 2. Push container to registry - 3. Deploy + - By default the folder /app/ext_storage/ should be mounted to outside storage to avoid ram bloat. It will contain database, media, and models. + - Place the downloaded models (from https://huggingface.co/pharmbio/ptp) to /app/ext_storage/models/ 4. Configuration - - Set environment variables - Use `DOWNLOAD=true` to download models prior to startup. - - preferebly mount `/app/inference/models` directory to outside storage to avoid ram bloat. - - - Use `MAX_MODELS` to limit the number of active models (for debug purposes) .otherwise it iterates all models present (currently 800+). - -- If desired use `MODEL_DIR` to change path where model are found (default /app/inference/models/models) + - Currently static files are served by Django itself even with DEBUG=False + - All defaults of environmental variables make sense for SciLifeLab Serve. Here are some environmental variables that can be changed if needed: + - `EMAIL_HOST`, `EMAIL_PORT`, `EMAIL_HOST_USER` etc for email settings. + - `MAX_MODELS` to limit the number of active models (for debug purposes) .otherwise it iterates all models present (currently 800+). + - `MEDIA_DIR` for media directory + - `MODEL_DIR` for models directory + - `DATABASE_DIR` for database directory + - `SITE_URL` for generating download links in emails ## References diff --git a/ptp/Dockerfile b/ptp/Dockerfile index d4fb681..f43534f 100644 --- a/ptp/Dockerfile +++ b/ptp/Dockerfile @@ -32,7 +32,8 @@ FROM djangobase AS djangoapp COPY . /app/ WORKDIR /app -RUN mkdir /app/inference/models/ +RUN mkdir -p /app/ext_storage/ +# Models also need to be added. They be downloaded and placed in /app/ext_storage/models/ through the SciLifeLab Serve interface # Expose the port the Django app runs on EXPOSE 8000 @@ -50,10 +51,5 @@ RUN chmod +x /app/start-django.sh # Make sure the container is running as non-root USER $USER -ENV CELERY_BROKER_URL="redis://localhost:6379/0" -ENV REDIS_URL="redis://localhost:6379/0" -ENV MODEL_DIR="/app/inference/models/models" -# Models will be downloaded and placed in /app/inference/models/models through SciLifeLab Serve interface - # Start supervisord CMD ["sh", "-c", "/app/start-script.sh"] diff --git a/ptp/inference/tasks.py b/ptp/inference/tasks.py index fda2eb8..2fb8b78 100644 --- a/ptp/inference/tasks.py +++ b/ptp/inference/tasks.py @@ -6,8 +6,10 @@ from django.conf import settings import pandas as pd from datetime import timezone +from pathlib import Path -model_dir = os.environ.get("MODEL_DIR", "/app/inference/models/models/") +model_dir = os.environ.get("MODEL_DIR", "/app/ext_storage/models") +Path(model_dir).mkdir(parents=True, exist_ok=True) max_models = os.environ.get("MAX_MODELS", False) @shared_task diff --git a/ptp/inference/urls.py b/ptp/inference/urls.py index e0337df..029936c 100644 --- a/ptp/inference/urls.py +++ b/ptp/inference/urls.py @@ -1,7 +1,7 @@ +from django.conf import settings from django.urls import path from . import views - app_name = 'inference' urlpatterns = [ diff --git a/ptp/ptp/settings.py b/ptp/ptp/settings.py index c03e3ef..7f1f833 100644 --- a/ptp/ptp/settings.py +++ b/ptp/ptp/settings.py @@ -10,20 +10,19 @@ # SECURITY WARNING: don't run with debug turned on in production! -#DEBUG = True +DEBUG = False if os.environ.get('DEBUG', False) == 'True': DEBUG = True -else: - DEBUG = False ALLOWED_HOSTS = ['*', 'localhost'] -CSRF_TRUSTED_ORIGINS=['http://localhost:8000'] -if os.environ.get('CSRF_TRUSTED_ORIGINS', False): - CSRF_TRUSTED_ORIGINS = os.environ.get('CSRF_TRUSTED_ORIGINS').split(',') +CSRF_TRUSTED_ORIGINS=['https://ptp2-inference.serve.scilifelab.se'] +if os.environ.get('CSRF_TRUSTED_ORIGINS', False): + CSRF_TRUSTED_ORIGINS = os.environ.get('CSRF_TRUSTED_ORIGINS').split(',') + # Application definition INSTALLED_APPS = [ 'django.contrib.admin', @@ -68,7 +67,8 @@ WSGI_APPLICATION = 'ptp.wsgi.application' -DATABASE_DIR = os.environ.get('DATABASE_DIR', BASE_DIR) +DATABASE_DIR = Path(os.environ.get('DATABASE_DIR', '/app/ext_storage/database')) +DATABASE_DIR.mkdir(parents=True, exist_ok=True) # Database DATABASES = { 'default': { @@ -108,31 +108,26 @@ #'/var/www/static', ] - # Media files (Uploaded files, results) MEDIA_URL = '/media/' -MEDIA_DIR = os.environ.get('MEDIA_DIR', BASE_DIR) +MEDIA_DIR = os.environ.get('MEDIA_DIR', '/app/ext_storage/media') MEDIA_ROOT = os.path.join(MEDIA_DIR, 'media') # Email settings (for sending job completion notifications) -if os.environ.get('EMAIL_HOST', False): - EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' - EMAIL_HOST = os.environ.get('EMAIL_HOST', 'smtp.example.com') - EMAIL_PORT = os.environ.get('EMAIL_PORT',587) - EMAIL_USE_TLS = os.environ.get('EMAIL_USE_TLS',True) - EMAIL_HOST_USER = os.environ.get('EMAIL_HOST_USER','your-email@example.com') - - # Not superfond of this.. - EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD','your-email-password') - # Perhaps? - if os.environ.get("EMAIL_PASSWORD_FILE", False): - filename = os.environ.get("EMAIL_PASSWORD_FILE", False) - with open(filename) as f: - EMAIL_HOST_PASSWORD = f.read().strip() - -else: - EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' # Use console backend for testing +EMAIL_BACKEND = ( + "django.core.mail.backends.smtp.EmailBackend" if not DEBUG else "django.core.mail.backends.console.EmailBackend" +) +EMAIL_HOST = os.environ.get('EMAIL_HOST', "smtp.gmail.com") +EMAIL_PORT = os.environ.get('EMAIL_PORT', 465) +EMAIL_USE_SSL = os.environ.get('EMAIL_USE_SSL', True) +EMAIL_USE_TLS = os.environ.get('EMAIL_USE_TLS', False) +EMAIL_HOST_USER = os.environ.get('EMAIL_HOST_USER','your-email@example.com') +EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD','your-email-password') +if os.environ.get("EMAIL_PASSWORD_FILE", False): + filename = os.environ.get("EMAIL_PASSWORD_FILE", False) + with open(filename) as f: + EMAIL_HOST_PASSWORD = f.read().strip() # Celery configuration CELERY_BROKER_URL = "redis://localhost:6379/0" @@ -152,7 +147,8 @@ # Set the SITE_URL for generating download links in emails -SITE_URL = os.environ.get('EMAIL_DOMAIN','https://yourdomain.com') +SITE_URL = os.environ.get('SITE_URL','https://ptp2-inference.serve.scilifelab.se') + STAGE_ENV = os.environ.get('STAGE', False) @@ -160,6 +156,7 @@ # Security settings SECURE_BROWSER_XSS_FILTER = True X_FRAME_OPTIONS = 'DENY' - SECURE_SSL_REDIRECT = not DEBUG + #SECURE_SSL_REDIRECT = not DEBUG CSRF_COOKIE_SECURE = not DEBUG SESSION_COOKIE_SECURE = not DEBUG + diff --git a/ptp/ptp/urls.py b/ptp/ptp/urls.py index e765a2b..ab652aa 100644 --- a/ptp/ptp/urls.py +++ b/ptp/ptp/urls.py @@ -15,13 +15,18 @@ 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ from django.contrib import admin -from django.urls import path, include +from django.urls import path, include, re_path from django.conf import settings from django.conf.urls.static import static +from django.views.static import serve urlpatterns = [ path('admin/', admin.site.urls), path('', include('inference.urls')), -] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) +] +urlpatterns += [ + re_path(r'^static/(?P.*)$', serve, {'document_root': settings.STATIC_ROOT}), + re_path(r'^media/(?P.*)$', serve, {'document_root': settings.MEDIA_ROOT}), +] \ No newline at end of file