upgrade to django-sodar-core v1.0.2 (#1922)

.github/ISSUE_TEMPLATE/release_cleanup.md

@@ -19,13 +19,14 @@ TBA
 
 - [ ] Review code style and cleanup
 - [ ] Review and update docs entries
-- [ ] Update `SODAR_API_DEFAULT_VERSION` and `SODAR_API_ALLOWED_VERSIONS`
+- [ ] Ensure REST API versions are up to date and documented
 - [ ] Run `npx update-browserslist-db@latest` for Vue app
 - [ ] Update Vue app version with `npm version`
 - [ ] Update version in CHANGELOG and SODAR Release Notes doc
 - [ ] Update version in docs conf.py
 - [ ] Ensure both SODAR and SODAR Core API versioning is correct in API docs
 - [ ] Ensure docs can be built without errors
+- [ ] Ensure `generateschema` runs without errors or warnings (until in CI)
 
 ## Notes
 

.github/workflows/build.yml

@@ -7,12 +7,12 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - '3.8'
           - '3.9'
           - '3.10'
+          - '3.11'
     services:
       postgres:
-        image: postgres:11
+        image: postgres:16
         env:
           POSTGRES_DB: sodar
           POSTGRES_USER: postgres
@@ -24,6 +24,17 @@ jobs:
           --health-retries 10
         ports:
           - 5432:5432
+      # TODO: Remove temporary iRODS postgres server once on iRODS 4.3
+      postgres-irods:
+        image: postgres:11
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
       redis:
         image: redis
         options: >-
@@ -39,7 +50,7 @@ jobs:
         env:
           IRODS_AUTHENTICATION_SCHEME: native
           IRODS_DEFAULT_HASH_SCHEME: MD5
-          IRODS_ICAT_DBSERVER: postgres
+          IRODS_ICAT_DBSERVER: postgres-irods  # TODO: Use main postgres on 4.3
           IRODS_ICAT_DBUSER: postgres
           IRODS_ICAT_DBPASS: postgres
           IRODS_ZONE_NAME: sodarZone
@@ -64,7 +75,7 @@ jobs:
         uses: actions/checkout@v3
       - name: Install project Python dependencies
         run: |
-          pip install wheel==0.40.0
+          pip install wheel==0.42.0
           pip install -r requirements/local.txt
           pip install -r requirements/test.txt
       - name: Setup Node.js
@@ -86,14 +97,15 @@ jobs:
           coverage report
       - name: Run Vue app tests
         run: make test_samplesheets_vue
-        if: ${{ matrix.python-version == '3.8' }}
+        if: ${{ matrix.python-version == '3.11' }}
       - name: Check Python linting
         run: flake8 .
       - name: Check Python formatting
         run: make black arg=--check
+        if: ${{ matrix.python-version == '3.11' }}
       - name: Report coverage with Coveralls
         uses: coverallsapp/github-action@master
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           path-to-lcov: './coverage.lcov'
-        if: ${{ matrix.python-version == '3.8' }}
+        if: ${{ matrix.python-version == '3.11' }}

.readthedocs.yaml

@@ -9,7 +9,7 @@ version: 2
 build:
   os: ubuntu-20.04
   tools:
-    python: '3.8'
+    python: '3.11'
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:

Makefile

@@ -5,6 +5,7 @@ define USAGE=
 @echo -e "Usage:"
 @echo -e "\tmake black [arg=--<arg>]                    -- format python with black"
 @echo -e "\tmake serve [arg=sync]                       -- start server"
+@echo -e "\tmake flake                                  -- run flake8"
 @echo -e "\tmake celery                                 -- start celery & celerybeat"
 @echo -e "\tmake demo                                   -- start demo server"
 @echo -e "\tmake samplesheets_vue                       -- start samplesheet vue.js app"
@@ -39,6 +40,11 @@ endif
 	$(MANAGE) runserver 0.0.0.0:8000 --settings=config.settings.local
 
 
+.PHONY: flake
+flake:
+	flake8 .
+
+
 .PHONY: celery
 celery:
 	celery -A config worker -l info --beat

config/settings/base.py

@@ -8,6 +8,7 @@
 https://docs.djangoproject.com/en/3.2/ref/settings/
 """
 
+import itertools
 import os
 import re
 
@@ -61,6 +62,7 @@
     'markupfield',  # For markdown
     'rest_framework',  # For API views
     'knox',  # For token auth
+    'social_django',  # For OIDC authentication
     'docs',  # For the online user documentation/manual
     'dal',  # For user search combo box
     'dal_select2',
@@ -304,7 +306,7 @@
 AUTOSLUG_SLUGIFY_FUNCTION = 'slugify.slugify'
 
 # Location of root django.contrib.admin URL, use {% url 'admin:index' %}
-ADMIN_URL = r'^admin/'
+ADMIN_URL = 'admin/'
 
 
 # Celery
@@ -326,6 +328,8 @@
 CELERYD_TASK_TIME_LIMIT = 5 * 60
 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-soft-time-limit
 CELERYD_TASK_SOFT_TIME_LIMIT = 60
+# https://docs.celeryq.dev/en/latest/userguide/configuration.html#broker-connection-retry-on-startup
+CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = False
 CELERY_IMPORTS = [
     'landingzones.tasks_celery',
     'samplesheets.tasks_celery',
@@ -355,7 +359,6 @@
 LDAP_ALT_DOMAINS = env.list('LDAP_ALT_DOMAINS', None, default=[])
 
 if ENABLE_LDAP:
-    import itertools
     import ldap
     from django_auth_ldap.config import LDAPSearch
 
@@ -384,8 +387,9 @@
     AUTH_LDAP_USER_FILTER = env.str(
         'AUTH_LDAP_USER_FILTER', '(sAMAccountName=%(user)s)'
     )
+    AUTH_LDAP_USER_SEARCH_BASE = env.str('AUTH_LDAP_USER_SEARCH_BASE', None)
     AUTH_LDAP_USER_SEARCH = LDAPSearch(
-        env.str('AUTH_LDAP_USER_SEARCH_BASE', None),
+        AUTH_LDAP_USER_SEARCH_BASE,
         ldap.SCOPE_SUBTREE,
         AUTH_LDAP_USER_FILTER,
     )
@@ -417,8 +421,11 @@
         AUTH_LDAP2_USER_FILTER = env.str(
             'AUTH_LDAP2_USER_FILTER', '(sAMAccountName=%(user)s)'
         )
+        AUTH_LDAP2_USER_SEARCH_BASE = env.str(
+            'AUTH_LDAP2_USER_SEARCH_BASE', None
+        )
         AUTH_LDAP2_USER_SEARCH = LDAPSearch(
-            env.str('AUTH_LDAP2_USER_SEARCH_BASE', None),
+            AUTH_LDAP2_USER_SEARCH_BASE,
             ldap.SCOPE_SUBTREE,
             AUTH_LDAP2_USER_FILTER,
         )
@@ -435,79 +442,40 @@
         )
 
 
-# SAML configuration
+# OpenID Connect (OIDC) configuration
 # ------------------------------------------------------------------------------
 
-ENABLE_SAML = env.bool('ENABLE_SAML', False)
-SAML2_AUTH = {
-    # Required setting
-    # Pysaml2 Saml client settings
-    # See: https://pysaml2.readthedocs.io/en/latest/howto/config.html
-    'SAML_CLIENT_SETTINGS': {
-        # Optional entity ID string to be passed in the 'Issuer' element of
-        # authn request, if required by the IDP.
-        'entityid': env.str('SAML_CLIENT_ENTITY_ID', 'SODAR'),
-        'entitybaseurl': env.str(
-            'SAML_CLIENT_ENTITY_URL', 'https://localhost:8000'
-        ),
-        # The auto(dynamic) metadata configuration URL of SAML2
-        'metadata': {
-            'local': [
-                env.str('SAML_CLIENT_METADATA_FILE', 'metadata.xml'),
-            ],
-        },
-        'service': {
-            'sp': {
-                'idp': env.str(
-                    'SAML_CLIENT_IPD',
-                    'https://sso.hpc.bihealth.org/auth/realms/cubi',
-                ),
-                # Keycloak expects client signature
-                'authn_requests_signed': 'true',
-                # Enforce POST binding which is required by keycloak
-                'binding': 'urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST',
-            },
-        },
-        'key_file': env.str('SAML_CLIENT_KEY_FILE', 'key.pem'),
-        'cert_file': env.str('SAML_CLIENT_CERT_FILE', 'cert.pem'),
-        'xmlsec_binary': env.str('SAML_CLIENT_XMLSEC1', '/usr/bin/xmlsec1'),
-        'encryption_keypairs': [
-            {
-                'key_file': env.str('SAML_CLIENT_KEY_FILE', 'key.pem'),
-                'cert_file': env.str('SAML_CLIENT_CERT_FILE', 'cert.pem'),
-            }
-        ],
-    },
-    # Custom target redirect URL after the user get logged in.
-    # Defaults to /admin if not set. This setting will be overwritten if you
-    # have parameter ?next= specificed in the login URL.
-    'DEFAULT_NEXT_URL': '/',
-    # # Optional settings below
-    # 'NEW_USER_PROFILE': {
-    #     'USER_GROUPS': [],  # The default group name when a new user logs in
-    #     'ACTIVE_STATUS': True,  # The default active status for new users
-    #     'STAFF_STATUS': True,  # The staff status for new users
-    #     'SUPERUSER_STATUS': False,  # The superuser status for new users
-    # },
-    'ATTRIBUTES_MAP': env.dict(
-        'SAML_ATTRIBUTES_MAP',
-        default={
-            # Change values to corresponding SAML2 userprofile attributes.
-            'email': 'Email',
-            'username': 'UserName',
-            'first_name': 'FirstName',
-            'last_name': 'LastName',
-        },
-    ),
-    # 'TRIGGER': {
-    #     'FIND_USER': 'path.to.your.find.user.hook.method',
-    #     'NEW_USER': 'path.to.your.new.user.hook.method',
-    #     'CREATE_USER': 'path.to.your.create.user.hook.method',
-    #     'BEFORE_LOGIN': 'path.to.your.login.hook.method',
-    # },
-    # Custom URL to validate incoming SAML requests against
-    # 'ASSERTION_URL': 'https://your.url.here',
-}
+ENABLE_OIDC = env.bool('ENABLE_OIDC', False)
+
+if ENABLE_OIDC:
+    AUTHENTICATION_BACKENDS = tuple(
+        itertools.chain(
+            ('social_core.backends.open_id_connect.OpenIdConnectAuth',),
+            AUTHENTICATION_BACKENDS,
+        )
+    )
+    TEMPLATES[0]['OPTIONS']['context_processors'] += [
+        'social_django.context_processors.backends',
+        'social_django.context_processors.login_redirect',
+    ]
+    SOCIAL_AUTH_JSONFIELD_ENABLED = True
+    SOCIAL_AUTH_JSONFIELD_CUSTOM = 'django.db.models.JSONField'
+    SOCIAL_AUTH_USER_MODEL = AUTH_USER_MODEL
+    SOCIAL_AUTH_ADMIN_USER_SEARCH_FIELDS = [
+        'username',
+        'name',
+        'first_name',
+        'last_name',
+        'email',
+    ]
+    SOCIAL_AUTH_OIDC_OIDC_ENDPOINT = env.str(
+        'SOCIAL_AUTH_OIDC_OIDC_ENDPOINT', None
+    )
+    SOCIAL_AUTH_OIDC_KEY = env.str('SOCIAL_AUTH_OIDC_KEY', 'CHANGEME')
+    SOCIAL_AUTH_OIDC_SECRET = env.str('SOCIAL_AUTH_OIDC_SECRET', 'CHANGEME')
+    SOCIAL_AUTH_OIDC_USERNAME_KEY = env.str(
+        'SOCIAL_AUTH_OIDC_USERNAME_KEY', 'username'
+    )
 
 
 # Logging

config/urls.py

@@ -1,5 +1,5 @@
 from django.conf import settings
-from django.conf.urls import include, url
+from django.conf.urls import include
 from django.conf.urls.static import static
 from django.contrib import admin
 from django.contrib.auth import views as auth_views
@@ -26,7 +26,7 @@ def handler500(request, *args, **argv):
 urlpatterns = [
     path(route='', view=HomeView.as_view(), name='home'),
     # Django Admin, use {% url 'admin:index' %}
-    url(settings.ADMIN_URL, admin.site.urls),
+    path(settings.ADMIN_URL, admin.site.urls),
     # Login and logout
     path(
         route='login/',
@@ -40,6 +40,8 @@ def handler500(request, *args, **argv):
     path('api/auth/', include('knox.urls')),
     # Iconify SVG icons
     path('icons/', include('dj_iconify.urls')),
+    # Social auth for OIDC support
+    path('social/', include('social_django.urls')),
     # General site apps
     path('alerts/adm/', include('adminalerts.urls')),
     path('alerts/app/', include('appalerts.urls')),

docs_manual/source/admin_commands.rst

@@ -19,6 +19,9 @@ These commands originate in SODAR Core. More information can be found in the
     Add remote site for remote project synchronization.
 ``batchupdateroles``
     Batch update project roles and send invites.
+``checkusers``
+    Check the status of LDAP users. Returns info on users whose accounts are
+    disabled or removed on an LDAP server.
 ``cleanappsettings``
     Clean up unused application settings.
 ``deletecache``

docs_manual/source/admin_install.rst

@@ -17,26 +17,22 @@ environment. All these components are required for running the complete feature
 set of SODAR. However, it is also possible to run some of these outside of the
 Docker Compose network if e.g. you already have a separate iRODS server running.
 
-- Essential SODAR Components
+Essential SODAR Components
     - ``sodar-web``: The SODAR web server for main program logic and UIs.
     - ``sodar-celeryd-default``: Celery daemon for background jobs.
     - ``sodar-celerybeat``: Celery service for periodic tasks.
-- Database Servers
-    - ``postgres``: PostgreSQL server for SODAR and iRODS databases.
+Database Servers
+    - ``postgres``: PostgreSQL server for SODAR and iRODS databases. Minimum
+      supported version is v12, recommended version is v16.
     - ``redis``: In-memory database for Celery jobs and caching.
-- iRODS Servers
-    - ``irods``: An iRODS iCAT server for file storage.
+iRODS Servers
+    - ``irods``: An iRODS iCAT server for file storage. The minimum supported
+      version is v4.3.
     - ``davrods``: iRODS WebDAV server for web access and IGV/UCSC integration.
-- Networking
+Networking
     - ``traefik``: Reverse proxy for TLS/SSL routing.
     - ``sssd``: System Security Service Daemon for LDAP/AD authentication.
 
-.. note::
-
-    Currently the sodar-docker-compose environment only supports iRODS v4.2.
-    Support for v4.3 is being worked on. iRODS v4.3 will be the default
-    supported version from SODAR v1.0 onwards.
-
 
 Quickstart Guide
 ================
@@ -60,15 +56,15 @@ Prerequisites
 Ensure your system matches the following operating system and software
 requirements.
 
-- Hardware
+Hardware
     - ~10 GB of disk space for the Docker images
-- Operating System
+Operating System
     - A modern Linux distribution that is
       `supported by Docker <https://docs.docker.com/engine/install/#server>`_.
     - Outgoing HTTPS connections to the internet are allowed to download data
       and Docker images.
     - Server ports 80 and 443 are open and free on the host.
-- Software
+Software
     - `Docker <https://docs.docker.com/get-docker/>`_
     - `Docker Compose <https://docs.docker.com/compose/install/>`_
     - `OpenSSL <https://www.openssl.org/>`_
@@ -279,7 +275,7 @@ production is generally recommended only for experienced SODAR admins.
 
 .. note::
 
-    SODAR v1.0 will be upgraded to use iRODS 4.3 and Postgres v16. This version
+    SODAR v1.0 has been upgraded to use iRODS 4.3 and Postgres v16. This version
     may require special steps for upgrading an existing environment. Make sure
     to refer to the sodar-docker-compose README for instructions.
 
@@ -348,7 +344,7 @@ Production Prerequisites
 In addition to the :ref:`general prerequisites <admin_install_prerequisites>`,
 we recommend the following for a production deployment of SODAR:
 
-- Recommended Hardware
+Recommended Hardware
     - Memory: 64 GB of RAM
     - CPU: 16 cores
     - Disk: 600+ GB of free and **fast** disk space