Merge branch 'dev' into ms-add_host

.env.example

@@ -23,6 +23,10 @@ CONTACT_EMAIL='PhysioNet Contact <[email protected]>'
 SERVER_EMAIL='PhysioNet System <[email protected]>'
 ERROR_EMAIL='[email protected]'
 
+# Contact address for project editors. This address may be viewable by authors.
+# Optionally, add "PROJECT-SLUG" to include the project slug.
+PROJECT_EDITOR_EMAIL='[email protected]'
+
 # Admins
 ADMINS_NAME=PhysioNet Technical
 ADMINS_MAIL=[email protected]
@@ -60,6 +64,18 @@ AWS_VALUE=secret
 AWS_VALUE2=secret
 AWS_CLOUD_FORMATION=url
 
+# AWS credentials (Access Key and Secret Key): Configure AWS credentials in the AWS CLI profile using the 'aws configure' command.
+AWS_PROFILE=
+# AWS account ID
+AWS_ACCOUNT_ID=
+# Path to the file containing credentials for AWS
+# (https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#shared-credentials-file)
+AWS_SHARED_CREDENTIALS_FILE=
+# The default bucket name to store projects with an 'OPEN' access policy.
+S3_OPEN_ACCESS_BUCKET=
+# The default bucket name to store logs and metrics related to project usage.
+S3_SERVER_ACCESS_LOG_BUCKET=
+
 # Datacite
 # Used to assign the DOIs
 # Changing the password can be done at the settings tab in DataCite website
@@ -158,6 +174,10 @@ GRADIENT_85 = 'rgba(42, 47, 52, 0.85)'
 # maximum number of emails that can be associated to a user model
 MAX_EMAILS_PER_USER = 10
 
+# maximum number of active projects that can be created by a submitting author at any time.
+# if MAX_SUBMITTABLE_PROJECTS is reached, the user must wait for a project to be archived or published before starting another.
+MAX_SUBMITTABLE_PROJECTS = 10
+
 # Max training report size in bytes
 MAX_TRAINING_REPORT_UPLOAD_SIZE = 1048576
 ENABLE_LIGHTWAVE=True

README.md

@@ -41,6 +41,12 @@ The code should dynamically reload in development, however, if there are any iss
 
 Docker-compose uses volumes to persist the database contents and data directories (media and static files). To clean up the created containers, networks and volumes stop `docker-compose up` and run `docker-compose down -v`. Do not run `docker-compose down -v` if you want to retain current database contents.
 
+## Background tasks
+
+Background tasks are managed by [Django Q2](https://django-q2.readthedocs.io/en/master/), "a native Django task queue, scheduler and worker application using Python multiprocessing".
+
+If you would like to run background tasks on your development server, you will need to start the task manager with `python manage.py qcluster`
+
 ## Using a debugger with Docker
 
 To access a debug prompt raised using `breakpoint()`:

demo-files/aws_credentials

@@ -0,0 +1,3 @@
+[default]
+aws_access_key_id = AKIAZZZZZZZZZZZZZZZZ
+aws_secret_access_key = xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

deploy/common/etc/sudoers.d/physionet

@@ -16,3 +16,7 @@ pn ALL=(root:root) NOPASSWD: \
 # Restart django-background-tasks.service (if it is running)
 pn ALL=(root:root) NOPASSWD: \
     /bin/systemctl try-restart django-background-tasks.service
+
+# Restart django-q2-tasks.service (if it is running)
+pn ALL=(root:root) NOPASSWD: \
+    /bin/systemctl try-restart django-q2-tasks.service

deploy/post-receive

@@ -112,6 +112,7 @@ else
 fi
 
 # Restart the django-background-tasks server
+# Background tasks will soon be migrated to Django Q2
 (
     cd $working_dir/physionet-django
     if cmp -s OLD-TARGETS LATE-TARGETS; then
@@ -130,6 +131,25 @@ fi
     fi
 )
 
+# Restart the Django Q2 server
+(
+    cd $working_dir/physionet-django
+    if cmp -s OLD-TARGETS LATE-TARGETS; then
+        # assume that if there are no migrations, no need to restart
+        # background tasks (and don't even bother showing a message)
+        :
+    elif [ -n "$no_bgtasks" ]; then
+        echo "- SKIPPING restarting django-q2-tasks due to"
+        echo "  --push-option=no-bgtasks"
+    elif [ -n "$no_install" ]; then
+        echo "- SKIPPING restarting django-q2-tasks due to"
+        echo "  --push-option=no-install"
+    else
+        echo "* Restarting django-q2-tasks..."
+        sudo systemctl try-restart django-q2-tasks.service
+    fi
+)
+
 # Run late migrations (as well as early migrations, if they were
 # skipped before)
 (

deploy/production/etc/rsyslog.d/django_q.conf

@@ -0,0 +1,3 @@
+# Filter to get all background tasks and send them to a custom location
+:programname, isequal, "django-q2-tasks" -/data/log/background_tasks/django_q_tasks.log
+& stop

deploy/production/etc/systemd/system/django-q2-tasks.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Command that runs Django Q2 tasks
+After=emperor.uwsgi.service
+
+[Service]
+Environment=DJANGO_SETTINGS_MODULE=physionet.settings.production
+ExecStart=/physionet/python-env/physionet/bin/python /physionet/physionet-build/physionet-django/manage.py qcluster
+StandardError=syslog
+SyslogIdentifier=django-q2-tasks
+Restart=always
+KillSignal=SIGINT
+Type=simple
+NotifyAccess=all
+User=www-data
+Group=www-data
+
+[Install]
+WantedBy=multi-user.target

deploy/staging/etc/rsyslog.d/django_q.conf

@@ -0,0 +1,3 @@
+# Filter to get all background tasks and send them to a custom location
+:programname, isequal, "django-q2-tasks" -/data/log/background_tasks/django_q_tasks.log
+& stop

deploy/staging/etc/systemd/system/django-q2-tasks.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Command that runs Django Q2 tasks
+After=emperor.uwsgi.service
+
+[Service]
+Environment=DJANGO_SETTINGS_MODULE=physionet.settings.staging
+ExecStart=/physionet/python-env/physionet/bin/python /physionet/physionet-build/physionet-django/manage.py qcluster
+StandardError=syslog
+SyslogIdentifier=django-q2-tasks
+Restart=always
+KillSignal=SIGINT
+Type=simple
+NotifyAccess=all
+User=www-data
+Group=www-data
+
+[Install]
+WantedBy=multi-user.target

deploy/test-server/install-pn-test-server

@@ -234,9 +234,14 @@ systemctl daemon-reload
 systemctl enable emperor.uwsgi
 systemctl restart emperor.uwsgi
 systemctl restart nginx
+
+# django-background-tasks will be replaced with Django Q2 soon
 systemctl enable django-background-tasks
 systemctl restart django-background-tasks
 
+systemctl enable django-q2-tasks
+systemctl restart django-q2-tasks
+
 su pn -c '
    cd /physionet/physionet-build.git
    echo

physionet-django/.coveragerc

@@ -0,0 +1,5 @@
+[run]
+plugins = django_coverage_plugin
+
+[django_coverage_plugin]
+template_extensions = html, txt, json, xml

physionet-django/console/forms.py

@@ -223,9 +223,6 @@ def save(self):
             # Reject
             if edit_log.decision == 0:
                 project.reject()
-                # Have to reload this object which is changed by the reject
-                # function
-                edit_log = EditLog.objects.get(id=edit_log.id)
             # Resubmit with revisions
             elif edit_log.decision == 1:
                 project.submission_status = SubmissionStatus.NEEDS_RESUBMISSION
@@ -635,7 +632,7 @@ class NewsForm(forms.ModelForm):
 
     class Meta:
         model = News
-        fields = ('title', 'content', 'url', 'project', 'front_page_banner')
+        fields = ('slug', 'title', 'content', 'url', 'project', 'link_all_versions', 'front_page_banner')
 
 
 class FeaturedForm(forms.Form):