diff --git a/.gitignore b/.gitignore index ac5a7677..0dce4a37 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ htmlcov .DS_Store .pytest_cache/ export/ +config_dev.env diff --git a/.travis.yml b/.travis.yml index 035f0893..a20b9892 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,14 +15,21 @@ addons: - postgresql-10 - postgresql-client-10 +env: + - ELASTICSEARCH_HOST: localhost:9200 +before_install: + - curl https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.3.2-amd64.deb -o elasticsearch.deb + - sudo dpkg -i --force-confnew elasticsearch.deb + - sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch + - sudo service elasticsearch restart before_script: - - psql template1 -c 'create extension hstore;' + - sleep 10 # Needed for ElasticSearch install: - pip install --upgrade pip - pip install coveralls -r requirements.txt -r dev-requirements.txt script: - flake8 - - pytest -ra -vvv + - pytest -ra -vvv --cov=. before_cache: - rm -f $HOME/.cache/pip/log/debug.log @@ -30,4 +37,5 @@ notifications: slack: secure: gm6+6ekPIw0WcNuul94MoAzsAJ1/rlP0++UdnB01uf5boXIJiQjKZn+BVyhpMX1CN3KQFyf2tXEsMj2AY8IEF0YzJJXc9+ae70p+5OQXInLUcea7SZJ/7q7Tuw2AJoXUxbDmoto83N828waEIdWjCKW5qCxM248+FG9wKNipkjswv/obASOBzlhGQ67kzRaUpCsCHYlpbgxhlg1lPZs32vKL9YOtjCjyCerxnE+SIomANE+djpd8eGFUz90SEcfR5ypGHeuIic4xsX6VhGHyzevfgEix5aq3QDiNSbH8GWClyMeiU82ov3dEsRvMheRH9vRYl6xzdKuAgWRfW61biApki8sPn2w7W6AMtD0MaHMNi3QsQVW4z0saKNICtvi7ZQXtw8DvajEYLn1GuyF6hBUh9LEoqFXN52HlBVjZ/0XnCPyCdyQy03u3pOxz/OQ1/9gOH2SQMkQUqMYbiHeELXg4KHnPpvfgNBmFAA8WybbAzLXAqF2/67nMiNOi1a0oHOMfHlEAFnxITzYcU4eQxfapHHzXdfObZXwcna7T6RXdCVEX5LmOUN35oJkKe4y4T3ngumD8thbVX07HDYrbAaeE85JoT5ok5xe7WHimf2XTuOUNogNALnsop19EROrHcRB6d88BluVoS5S3e6uS7SliUCjH2HqMkDMR0T5zRy0= -after_success: coveralls +after_success: + - pip install codecov && codecov diff --git a/README.md b/README.md index 919f8ed7..71e5042b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ## Prerequisites - Python 3.6 -- PostgreSQL 10.6 +- PostgreSQL 10.x ## Installation @@ -19,7 +19,9 @@ pip install -r requirements.txt ``` -- Create `local_settings.py` file in the project root and use it to override settings as needed. +- Copy `config_dev.env.example` to `config_dev.env` and edit according + to your needs. DATABASE_URL is the one setting that you will need to + uncomment and possibly edit. - Create a database @@ -48,29 +50,19 @@ python manage.py createsuperuser ### Docker compose setup -- Build and start the containers. The `--build` flag is needed only for first time setup **or** if something in the Dockerfile changes. +- Copy `config_dev.env.example` to `config_dev.env` and edit according + to your needs. The file is copiously commented. ``` -docker-compose up --build +cp config_dev.env.example config_dev.env ``` -- Create `local_settings.py` to checkout root +- Build and start the containers. Docker-compose will automatically build + the container if it is missing. By default the container initializes + database and starts Django dev server. -```python -DEBUG = True - -SECRET_KEY = 'xxx' - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql', - 'NAME': 'helerm', - 'USER': 'root', - 'PASSWORD': 'root', - 'PORT': 5432, - 'HOST': 'helerm_postgres96', - } -} +``` +docker-compose up ``` - Access the application container shell @@ -79,12 +71,6 @@ DATABASES = { docker-compose exec django bash ``` -- Enable necessary postgresql extensions for the database - -``` -echo 'CREATE EXTENSION hstore; \q' | ./manage.py dbshell -``` - - Run `migrate`, `compilemessages`, and `createsuperuser` as usual. Detailed info in manual setup steps. ## Development @@ -153,7 +139,7 @@ python manage.py export_data The export uses [pyxb](http://pyxb.sourceforge.net/) library and needs Python bindings to be generated from XSD schema files. -The repo contains two sets of JHS XML Schema files located in `data` directory. In addition to original ones, there are also HKI customized versions, which are in use at least for now. +The repo contains two sets of JHS XML Schema files located in `data` directory. In addition to original ones, there are also HKI customized versions, which are in use at least for now. Generated bindings are included in `metarecord/binding/` so the export should work out of the box. diff --git a/config_dev.env.example b/config_dev.env.example new file mode 100644 index 00000000..22a23ead --- /dev/null +++ b/config_dev.env.example @@ -0,0 +1,162 @@ +# HelERM environment configuration +# This file defines a set of (environment) variables that configure most +# of the functionality of helerm. In order for helerm to read +# this file, rename it to `config_dev.env`. As the name implies, this +# file is supposed to be used only in development. For production use +# we recommend setting the environment variables using the facilities +# of your runtime environment. +# HelERM reads this file by itself. However, it can also be +# used in conjunction with the included docker-compose.yml. Then +# you don't need to inject the file into the container. Instead +# Docker defines environment variables that helerm will read. +# Following are the settings and their explanations, with example values +# that might be useful for development: + +# HelERM generates a JHS191-compliant XML export for consumption by +# other systems. This sets the human readable description fields for +# that document. +# HELERM_JHS191_EXPORT_DESCRIPTION="Export from development environment" + +# Whether to run Django in debug mode +# Django setting: DEBUG https://docs.djangoproject.com/en/3.0/ref/settings/#debug +DEBUG=True + +# Helsinki user management library (django-helusers) brings this in as +# a generic setting for indicating the "mode" of the site. It can be +# either 'dev', 'test' or 'production'. Only effect in HelERM is +# changing header color in Django admin +# Does not correspond to a standard Django setting +HEL_SITE_TYPE=dev + +# Level of Django logging. All events above the given level will be logged. +# Django setting: DJANGO_LOG_LEVEL https://docs.djangoproject.com/en/3.0/topics/logging/#examples +# DJANGO_LOG_LEVEL=INFO + +# Configures database for HelERM using URL style. Format is: +# postgres://USER:PASSWORD@HOST:PORT/NAME +# Unused components may be left out, only Postgres is supported +# The example below configures HelERM to use local PostgreSQL database +# called "helerm", connecting same as username as Django is running as. +# Django setting: DATABASES (but not directly) https://docs.djangoproject.com/en/3.0/ref/settings/#databases +# DATABASE_URL=postgres:///helerm + +# HelERM will use JWT tokens for authentication. This settings Specifies +# the value that must be present in the "aud"-key of the token presented +# by a client when making an authenticated request. HelERM uses this +# key for verifying that the token was meant for accessing this particular +# HelERM instance (the tokens are signed, see below). +# Does not correspond to standard Django setting +#TOKEN_AUTH_ACCEPTED_AUDIENCE=string-identifying-this-helerm-instance + +# This key will be used by HelERM to verify the JWT token is from trusted +# Identity Provider (OpenID terminology). The provider must have signed +# the JWT TOKEN using this shared secret +# Does not correspond to standard Django setting +#TOKEN_AUTH_SHARED_SECRET=abcdefghacbdefgabcdefghacbdefgabcdefghacbdefgabcdefghacbdefgabcdefghacbdefg + +# Secret used for various functions within Django. This setting is +# mandatory for Django, but HelERM will generate an ephemeral key, +# if it is not defined here. Currently HelERM does not use any +# functionality that needs this. +# Django setting: SECRET_KEY https://docs.djangoproject.com/en/3.0/ref/settings/#secret-key +#SECRET_KEY=gaehgoes89yrtp9384ygpe9r8ahgaerui8ghpae98rgh + +# List of Host-values, that HelERM will accept in requests. +# This setting is a Django protection measure against HTTP Host-header attacks +# https://docs.djangoproject.com/en/3.0/topics/security/#host-headers-virtual-hosting +# Specified as a comma separated list of allowed values. Note that this does +# NOT matter if you are running with DEBUG +# Django setting: ALLOWED_HOSTS https://docs.djangoproject.com/en/3.0/ref/settings/#allowed-hosts +#ALLOWED_HOSTS=api.hel.ninja,helerm-api.hel.ninja + +# List of tuples (or just e-mail addresses) specifying Administrators of this +# HelERM instance. Django uses this only when logging is configured to +# send exceptions to admins. HelERM does not do this. Still you may want +# to set this for documentation +# Django setting: ADMINS https://docs.djangoproject.com/en/3.0/ref/settings/#admins +ADMINS=admin@this.helerm.instance,another-admin@this.helerm.instance + +# Whether cookies set by Django are only allowed to be sent over +# secure (HTTPS) connection. This sets the "Secure" tag on the Set-Cookie +# header. For development, you will likely to want to set this to False. +# On server you will most certainly want to set this to True. +# Typical failure mode is when admin login returns back to empty form. +# Django setting: CSRF_COOKIE_SECURE and SESSION_COOKIE_SECURE +# https://docs.djangoproject.com/en/3.1/ref/settings/#session-cookie-secure +# COOKIE_SECURE=False + +# Cookie prefix is added to the every cookie set by HelERM. These are +# mostly used when accessing the internal Django admin site. This applies +# to django session cookie and csrf cookie +# Django setting: prepended to CSRF_COOKIE_NAME and SESSION_COOKIE_NAME +COOKIE_PREFIX=helerm + +# This sets the path for both session cookie and CSRF cookie. Useful if +# helerm is served from a sub-path (ie. there is a prefix the path, +# like https://host.name/helerm). Note that this must include the +# leading '/'. +# Django setting: sets both CSRF_COOKIE_PATH and SESSION_COOKIE_PATH +#PATH_PREFIX=/helerm + +# Django INTERNAL_IPS setting allows some debugging aids for the addresses +# specified here +# DJango setting: INTERNAL_IPS https://docs.djangoproject.com/en/3.0/ref/settings/#internal-ips +INTERNAL_IPS=127.0.0.1 + +# Specifies a header that is trusted to indicate that the request was using +# https while traversing over the Internet at large. This is used when +# a proxy terminates the TLS connection and forwards the request over +# a secure network. Specified using a tuple. +# Django setting: SECURE_PROXY_SSL_HEADER https://docs.djangoproject.com/en/3.0/ref/settings/#secure-proxy-ssl-header +#SECURE_PROXY_SSL_HEADER=('HTTP_X_FORWARDED_PROTO', 'https') + +# Media root is the place in file system where Django and, by extension +# HelERM stores "uploaded" files. This means any and all files +# that are inputted through API and generated by JHS191 exporter +# Django setting: MEDIA_ROOT https://docs.djangoproject.com/en/3.0/ref/settings/#media-root +#MEDIA_ROOT=/home/helerm/media + +# Static root is the place where HelERM will install any static +# files that need to be served to clients. For HelERM this is mostly +# JS and CSS for the API exploration interface + admin +# Django setting: STATIC_ROOT +#STATIC_ROOT=/home/helerm/static + +# Media URL is address (URL) where users can access files in MEDIA_ROOT +# through http. Ie. where your uploaded files are publicly accessible. +# In the simple case this is a relative URL to same server as API +# Django setting: MEDIA_URL https://docs.djangoproject.com/en/3.0/ref/settings/#media-url +MEDIA_URL=/media/ + +# Static URL is address (URL) where users can access files in STATIC_ROOT +# through http. Same factors apply as to MEDIA_URL +# Django setting: STATIC_URL https://docs.djangoproject.com/en/3.0/ref/settings/#static-url +STATIC_URL=/static/ + +# Specifies that Django is to use `X-Forwarded-Host` as it would normally +# use the `Host`-header. This is necessary when `Host`-header is used for +# routing the requests in a network of reverse proxies. `X-Forwarded-Host` +# is then used to carry the Host-header value supplied by the origin client. +# This affects how ALLOWED_HOSTS behaves, as well. +# Django setting: https://docs.djangoproject.com/en/3.0/ref/settings/#use-x-forwarded-host +# TRUST_X_FORWARDED_HOST=False + +# Sentry is an error tracking sentry (sentry.io) that can be self hosted +# or purchased as PaaS. SENTRY_DSN setting specifies the URL where reports +# for this HelERM instance should be sent. You can find this in +# your Sentry interface (or through its API) +#SENTRY_DSN=http://your.sentry.here/fsdafads/13 + +# Sentry environment is an optional tag that can be included in sentry +# reports. It is used to separate deployments within Sentry UI +SENTRY_ENVIRONMENT=local-development-unconfigured + +# Host for ElasticSearch connection. +#ELASTICSEARCH_HOST=helerm_elasticsearch:9200 + +# Helsinki user management library (django-helusers) related OIDC settings. +SOCIAL_AUTH_TUNNISTAMO_KEY=https://i/am/clientid/in/url/style +SOCIAL_AUTH_TUNNISTAMO_SECRET=iamyoursecret +SOCIAL_AUTH_TUNNISTAMO_OIDC_ENDPOINT=https://api.hel.fi/sso/openid +OIDC_API_TOKEN_AUTH_AUDIENCE=https://api.hel.fi/auth/projects +OIDC_API_TOKEN_AUTH_ISSUER=https://api.hel.fi/sso diff --git a/dev-requirements.in b/dev-requirements.in index ef41fef0..a9152791 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -3,4 +3,5 @@ freezegun isort pip-tools pytest +pytest-cov pytest-django diff --git a/dev-requirements.txt b/dev-requirements.txt index d6043c40..4aaceefc 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,28 +4,27 @@ # # pip-compile dev-requirements.in # -attrs==20.2.0 # via pytest +attrs==20.3.0 # via pytest click==7.1.2 # via pip-tools -flake8==3.8.3 # via -r dev-requirements.in +coverage==5.3.1 # via pytest-cov +flake8==3.8.4 # via -r dev-requirements.in freezegun==1.0.0 # via -r dev-requirements.in -importlib-metadata==1.7.0 # via flake8, pluggy, pytest -iniconfig==1.0.1 # via pytest -isort==5.5.3 # via -r dev-requirements.in +iniconfig==1.1.1 # via pytest +isort==5.7.0 # via -r dev-requirements.in mccabe==0.6.1 # via flake8 -more-itertools==8.5.0 # via pytest -packaging==20.4 # via pytest -pip-tools==5.3.1 # via -r dev-requirements.in +packaging==20.8 # via pytest +pip-tools==5.5.0 # via -r dev-requirements.in pluggy==0.13.1 # via pytest -py==1.9.0 # via pytest +py==1.10.0 # via pytest pycodestyle==2.6.0 # via flake8 pyflakes==2.2.0 # via flake8 pyparsing==2.4.7 # via packaging -pytest-django==3.10.0 # via -r dev-requirements.in -pytest==6.0.2 # via -r dev-requirements.in, pytest-django +pytest-cov==2.10.1 # via -r dev-requirements.in +pytest-django==4.1.0 # via -r dev-requirements.in +pytest==6.2.1 # via -r dev-requirements.in, pytest-cov, pytest-django python-dateutil==2.8.1 # via freezegun -six==1.15.0 # via packaging, pip-tools, python-dateutil -toml==0.10.1 # via pytest -zipp==3.1.0 # via importlib-metadata +six==1.15.0 # via python-dateutil +toml==0.10.2 # via pytest # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/docker-compose.yml b/docker-compose.yml index 76384a69..da501115 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,9 +12,27 @@ services: - helerm_postgres-data-volume:/var/lib/postgresql/data container_name: helerm_postgres + elasticsearch: + image: elasticsearch:7.3.2 + build: + context: . + dockerfile: ./docker/elasticsearch/Dockerfile + ports: + - "9200:9200" + environment: + - discovery.type=single-node + - xpack.security.enabled=false + - cluster.routing.allocation.disk.watermark.low=97% + - cluster.routing.allocation.disk.watermark.high=98% + - cluster.routing.allocation.disk.watermark.flood_stage=99% + - "ES_JAVA_OPTS=-Xms512m -Xmx512m -Djava.security.policy=file:/usr/share/elasticsearch/plugins/elasticsearch-analysis-voikko/plugin-security.policy" + container_name: helerm_elasticsearch + django: build: . command: bash -c 'tail -f /dev/null' + environment: + ELASTICSEARCH_HOST: helerm_elasticsearch volumes: - .:/code ports: diff --git a/docker/elasticsearch/Dockerfile b/docker/elasticsearch/Dockerfile new file mode 100644 index 00000000..babbf2db --- /dev/null +++ b/docker/elasticsearch/Dockerfile @@ -0,0 +1,14 @@ +FROM elasticsearch:7.3.2 + +# Installation of the voikko library and plugin for ElasticSearch +# https://github.com/EvidentSolutions/elasticsearch-analysis-voikko +RUN yum install -y \ + unzip \ + libvoikko + +RUN curl -o /usr/lib64/voikko/morpho.zip https://www.puimula.org/htp/testing/voikko-snapshot/dict-morpho.zip +RUN unzip -o /usr/lib64/voikko/morpho.zip -d /usr/lib64/voikko/ + +RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch https://github.com/EvidentSolutions/elasticsearch-analysis-voikko/releases/download/v0.6.0/elasticsearch-analysis-voikko-0.6.0.zip \ +&& echo '-Djava.security.policy=file:/usr/share/elasticsearch/plugins/elasticsearch-analysis-voikko/plugin-security.policy' >> /usr/share/elasticsearch/config/jvm.options \ +|| echo "Plugin already exists." diff --git a/helerm/settings.py b/helerm/settings.py index 693cb9df..5167f1fe 100644 --- a/helerm/settings.py +++ b/helerm/settings.py @@ -1,29 +1,104 @@ """ Django settings for helerm project. +""" -Generated by 'django-admin startproject' using Django 1.9.5. +import environ +import logging +import os +import sentry_sdk +import subprocess -For more information on this file, see -https://docs.djangoproject.com/en/1.9/topics/settings/ -For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.9/ref/settings/ -""" +from django.utils.translation import gettext_lazy as _ +from sentry_sdk.integrations.django import DjangoIntegration -import os -import sys -from django.utils.translation import ugettext_lazy as _ +CONFIG_FILE_NAME = "config_dev.env" + +# This will get default settings, as Django has not yet initialized +# logging when importing this file +logger = logging.getLogger(__name__) + + +def get_git_revision_hash() -> str: + """ + Retrieve the git hash for the underlying git repository or die trying + + We need a way to retrieve git revision hash for sentry reports + I assume that if we have a git repository available we will + have git-the-comamand as well + """ + try: + # We are not interested in gits complaints + git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], stderr=subprocess.DEVNULL, encoding='utf8') + # ie. "git" was not found + # should we return a more generic meta hash here? + # like "undefined"? + except FileNotFoundError: + git_hash = "git_not_available" + except subprocess.CalledProcessError: + # Ditto + git_hash = "no_repository" + return git_hash.rstrip() + + +root = environ.Path(__file__) - 2 # two levels back in hierarchy +env = environ.Env( + DEBUG=(bool, False), + DJANGO_LOG_LEVEL=(str, 'INFO'), + CONN_MAX_AGE=(int, 0), + DATABASE_URL=(str, 'postgres:///helerm'), + HEL_SITE_TYPE=(str, 'dev'), + TOKEN_AUTH_ACCEPTED_AUDIENCE=(str, ''), + TOKEN_AUTH_SHARED_SECRET=(str, ''), + SECRET_KEY=(str, ''), + ALLOWED_HOSTS=(list, []), + ADMINS=(list, []), + SECURE_PROXY_SSL_HEADER=(tuple, None), + MEDIA_ROOT=(environ.Path(), root('media')), + STATIC_ROOT=(environ.Path(), root('static')), + MEDIA_URL=(str, '/media/'), + STATIC_URL=(str, '/static/'), + TRUST_X_FORWARDED_HOST=(bool, False), + SENTRY_DSN=(str, ''), + SENTRY_ENVIRONMENT=(str, 'development'), + COOKIE_SECURE=(bool, True), + COOKIE_PREFIX=(str, 'helerm'), + PATH_PREFIX=(str, '/'), + INTERNAL_IPS=(list, []), + HELERM_JHS191_EXPORT_DESCRIPTION=(str, 'exported from undefined environment'), + ELASTICSEARCH_HOST=(str, "helerm_elasticsearch:9200"), + SOCIAL_AUTH_TUNNISTAMO_KEY=(str, ''), + SOCIAL_AUTH_TUNNISTAMO_SECRET=(str, ''), + SOCIAL_AUTH_TUNNISTAMO_OIDC_ENDPOINT=(str, 'https://api.hel.fi/sso/openid'), + OIDC_API_TOKEN_AUTH_AUDIENCE=(str, ''), + OIDC_API_TOKEN_AUTH_ISSUER=(str, 'https://api.hel.fi/sso'), +) # Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +BASE_DIR = root() +# Helsinki specific setting specifying whether the site +# is in ('dev','test' or 'production'). Only sets the background +# color in admin for HelERM +SITE_TYPE = env('HEL_SITE_TYPE') + +# Django environ has a nasty habit of complanining at level +# WARN about env file not being preset. Here we pre-empt it. +env_file_path = os.path.join(BASE_DIR, CONFIG_FILE_NAME) +if os.path.exists(env_file_path): + # Logging configuration is not available at this point + print(f'Reading config from {env_file_path}') + environ.Env.read_env(env_file_path) # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False +DEBUG = env('DEBUG') -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = env('ALLOWED_HOSTS') +ADMINS = env('ADMINS') +INTERNAL_IPS = env('INTERNAL_IPS', + default=(['127.0.0.1'] if DEBUG else [])) # Application definition @@ -39,17 +114,27 @@ 'django.contrib.sites', 'rest_framework', 'corsheaders', - 'allauth', - 'allauth.account', - 'allauth.socialaccount', - 'helusers.providers.helsinki', + 'social_django', 'adminsortable2', 'django_filters', + 'django_elasticsearch_dsl', + 'django_elasticsearch_dsl_drf', + 'django_admin_json_editor', 'metarecord', + 'search_indices', 'users', ] +if env('SENTRY_DSN'): + sentry_sdk.init( + dsn=env("SENTRY_DSN"), + environment=env("SENTRY_ENVIRONMENT"), + release=get_git_revision_hash(), + integrations=[DjangoIntegration()], + ) + + MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', @@ -82,32 +167,40 @@ WSGI_APPLICATION = 'helerm.wsgi.application' - -# Database -# https://docs.djangoproject.com/en/1.9/ref/settings/#databases - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql', - 'NAME': 'helerm' - } + 'default': env.db() } +# Persistent connections +# https://docs.djangoproject.com/en/3.1/ref/settings/#conn-max-age +DATABASES['default']['CONN_MAX_AGE'] = env('CONN_MAX_AGE') + LOGGING = { 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'timestamped_named': { + 'format': '%(asctime)s %(name)s %(levelname)s: %(message)s', + }, + }, 'handlers': { 'console': { 'class': 'logging.StreamHandler', - 'stream': sys.stdout, - } + 'formatter': 'timestamped_named', + }, + # Just for reference, not used + 'blackhole': { + 'class': 'logging.NullHandler', + }, }, - 'root': { - 'handlers': ['console'], - 'level': 'INFO' + 'loggers': { + 'django': { + 'handlers': ['console'], + 'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'), + }, } } - # Password validation # https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators @@ -150,46 +243,56 @@ ) -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.9/howto/static-files/ +STATIC_URL = env('STATIC_URL') +MEDIA_URL = env('MEDIA_URL') +STATIC_ROOT = env('STATIC_ROOT') +MEDIA_ROOT = env('MEDIA_ROOT') -STATIC_URL = '/static/' +# Whether to trust X-Forwarded-Host headers for all purposes +# where Django would need to make use of its own hostname +# fe. generating absolute URLs pointing to itself +# Most often used in reverse proxy setups +USE_X_FORWARDED_HOST = env('TRUST_X_FORWARDED_HOST') +# HelERM is a very public API CORS_ALLOW_CREDENTIALS = True CORS_ORIGIN_ALLOW_ALL = True +CSRF_COOKIE_NAME = '{}-csrftoken'.format(env('COOKIE_PREFIX')) +CSRF_COOKIE_PATH = env('PATH_PREFIX') +CSRF_COOKIE_SECURE = env('COOKIE_SECURE') +SESSION_COOKIE_NAME = '{}-sessionid'.format(env('COOKIE_PREFIX')) +SESSION_COOKIE_PATH = env('PATH_PREFIX') +SESSION_COOKIE_SECURE = env('COOKIE_SECURE') + AUTH_USER_MODEL = 'users.User' AUTHENTICATION_BACKENDS = ( 'django.contrib.auth.backends.ModelBackend', - 'allauth.account.auth_backends.AuthenticationBackend', + 'helusers.tunnistamo_oidc.TunnistamoOIDCAuth', ) -SOCIALACCOUNT_PROVIDERS = { - 'helsinki': { - 'VERIFIED_EMAIL': True - } -} -SOCIALACCOUNT_ADAPTER = 'helusers.adapter.SocialAccountAdapter' LOGIN_REDIRECT_URL = '/' +LOGOUT_REDIRECT_URL = '/' ACCOUNT_LOGOUT_ON_GET = True SITE_ID = 1 JWT_AUTH = { 'JWT_PAYLOAD_GET_USER_ID_HANDLER': 'helusers.jwt.get_user_id_from_payload_handler', - # JWT_AUDIENCE and JWT_SECRET_KEY must be set in local_settings.py + 'JWT_AUDIENCE': env('TOKEN_AUTH_ACCEPTED_AUDIENCE'), + 'JWT_SECRET_KEY': env('TOKEN_AUTH_SHARED_SECRET'), } # Used for descriptive comment in the headers of JHS191 XML export -XML_EXPORT_DESCRIPTION = 'exported from undefined environment' +XML_EXPORT_DESCRIPTION = env('HELERM_JHS191_EXPORT_DESCRIPTION') REST_FRAMEWORK = { 'DEFAULT_PERMISSION_CLASSES': ( 'rest_framework.permissions.IsAuthenticatedOrReadOnly', ), 'DEFAULT_AUTHENTICATION_CLASSES': ( - 'helusers.jwt.JWTAuthentication', + 'helusers.oidc.ApiTokenAuthentication', ), 'DEFAULT_PAGINATION_CLASS': 'metarecord.pagination.MetaRecordPagination', 'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',), @@ -214,32 +317,58 @@ ), } +SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer' + +SOCIAL_AUTH_TUNNISTAMO_KEY = env('SOCIAL_AUTH_TUNNISTAMO_KEY') +SOCIAL_AUTH_TUNNISTAMO_SECRET = env('SOCIAL_AUTH_TUNNISTAMO_SECRET') +SOCIAL_AUTH_TUNNISTAMO_OIDC_ENDPOINT = env('SOCIAL_AUTH_TUNNISTAMO_OIDC_ENDPOINT') + +OIDC_API_TOKEN_AUTH = { + # Audience that must be present in the token for the request to be + # accepted. Value must be agreed between your SSO service and your + # application instance. Essentially this allows your application to + # know that the token in meant to be used with it. + 'AUDIENCE': env('OIDC_API_TOKEN_AUTH_AUDIENCE'), + # Who we trust to sign the tokens. The library will request the + # public signature keys from standard locations below this URL + 'ISSUER': env('OIDC_API_TOKEN_AUTH_ISSUER'), +} + -# local_settings.py can be used to override environment-specific settings -# like database and email that differ between development and production. +# Elasticsearch configuration +ELASTICSEARCH_DSL = { + 'default': { + 'hosts': env("ELASTICSEARCH_HOST") + }, +} + +# Name of the Elasticsearch index +ELASTICSEARCH_INDEX_NAMES = { + 'search_indices.documents.action': 'action', + 'search_indices.documents.function': 'function', + 'search_indices.documents.classification': 'classification', + 'search_indices.documents.phase': 'phase', + 'search_indices.documents.record': 'record', +} + + +# Django SECRET_KEY setting, used for password reset links and such +SECRET_KEY = env('SECRET_KEY') +# If a secret key was not supplied elsewhere, generate a random one and log +# a warning (note that logging is not configured yet). This means that any +# functionality expecting SECRET_KEY to stay same will break upon restart. +# Should not be a problem for development. +if not SECRET_KEY: + logger.warning("SECRET_KEY was not defined in configuration. Generating an ephemeral key.") + import random + system_random = random.SystemRandom() + SECRET_KEY = ''.join([system_random.choice('abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') + for i in range(64)]) + +# local_settings.py is useful for overriding settings not available +# through environment and when developing new stuff local_settings_path = os.path.join(BASE_DIR, "local_settings.py") if os.path.exists(local_settings_path): with open(local_settings_path) as fp: code = compile(fp.read(), local_settings_path, 'exec') exec(code, globals(), locals()) - -# If a secret key was not supplied from elsewhere, generate a random one -# and store it into a file called .django_secret. -if 'SECRET_KEY' not in locals(): - secret_file = os.path.join(BASE_DIR, '.django_secret') - try: - with open(secret_file) as f: - SECRET_KEY = f.read().strip() - except IOError: - import random - system_random = random.SystemRandom() - try: - SECRET_KEY = ''.join( - [system_random.choice('abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') for i in range(64)] - ) - secret = open(secret_file, 'w') - os.chmod(secret_file, 0o0600) - secret.write(SECRET_KEY) - secret.close() - except IOError: - Exception('Please create a %s file with random characters to generate your secret key!' % secret_file) diff --git a/helerm/urls.py b/helerm/urls.py index 185668b8..188febd9 100644 --- a/helerm/urls.py +++ b/helerm/urls.py @@ -4,22 +4,38 @@ from metarecord.views import ( AttributeViewSet, BulkUpdateViewSet, ClassificationViewSet, ExportView, FunctionViewSet, JHSExportViewSet, - TemplateViewSet + RecordViewSet, TemplateViewSet +) +from search_indices.views import ( + ActionSearchDocumentViewSet, + AllSearchDocumentViewSet, + ClassificationSearchDocumentViewSet, + FunctionSearchDocumentViewSet, + PhaseSearchDocumentViewSet, + RecordSearchDocumentViewSet, ) from users.views import UserViewSet router = DefaultRouter() router.register(r'function', FunctionViewSet) +router.register(r'record', RecordViewSet) router.register(r'attribute', AttributeViewSet) router.register(r'template', TemplateViewSet, basename='template') router.register(r'user', UserViewSet) router.register(r'classification', ClassificationViewSet) router.register(r'export/jhs191', JHSExportViewSet, basename='jhs191_export') router.register(r'bulk-update', BulkUpdateViewSet) +router.register(r"action-search", ActionSearchDocumentViewSet, basename='action_search') +router.register(r"classification-search", ClassificationSearchDocumentViewSet, basename='classification_search') +router.register(r"function-search", FunctionSearchDocumentViewSet, basename='function_search') +router.register(r"phase-search", PhaseSearchDocumentViewSet, basename='phase_search') +router.register(r"record-search", RecordSearchDocumentViewSet, basename='record_search') +router.register(r"all-search", AllSearchDocumentViewSet, basename='all_search') urlpatterns = [ path('v1/', include(router.urls)), path('admin/', admin.site.urls), - path('accounts/', include('allauth.urls')), + path('pysocial/', include('social_django.urls', namespace='social')), + path('helauth/', include('helusers.urls')), path('export/', ExportView.as_view(), name='export') ] diff --git a/locale/fi/LC_MESSAGES/django.po b/locale/fi/LC_MESSAGES/django.po index 15a9d824..f8667d0a 100644 --- a/locale/fi/LC_MESSAGES/django.po +++ b/locale/fi/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2018-01-22 08:42+0200\n" +"POT-Creation-Date: 2020-09-29 13:12+0300\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -26,29 +26,49 @@ msgstr "suomi" msgid "English" msgstr "englanti" -#: metarecord/admin.py:32 +#: metarecord/admin/action.py:21 metarecord/admin/phase.py:21 +#: metarecord/admin/record.py:21 metarecord/models/classification.py:66 +msgid "code" +msgstr "tunnus" + +#: metarecord/admin/action.py:25 metarecord/admin/phase.py:25 +#: metarecord/admin/record.py:25 metarecord/models/function.py:99 +#: metarecord/models/function.py:193 metarecord/models/phase.py:9 +msgid "function" +msgstr "käsittelyprosessi" + +#: metarecord/admin/action.py:29 metarecord/admin/record.py:29 +#: metarecord/models/action.py:9 metarecord/models/phase.py:25 +msgid "phase" +msgstr "käsittelyvaihe" + +#: metarecord/admin/action.py:33 metarecord/admin/function.py:44 +#: metarecord/admin/phase.py:29 metarecord/admin/record.py:37 +#: metarecord/models/attribute.py:13 metarecord/models/attribute.py:25 +#: metarecord/models/attribute.py:74 metarecord/models/function.py:49 +msgid "name" +msgstr "nimi" + +#: metarecord/admin/function.py:13 msgid "State history" msgstr "Tilahistoria" -#: metarecord/admin.py:52 metarecord/models/function.py:104 +#: metarecord/admin/function.py:22 metarecord/models/bulk_update.py:33 +#: metarecord/models/classification.py:53 metarecord/models/function.py:197 +#: metarecord/models/structural_element.py:19 +msgid "modified by" +msgstr "muokkaaja" + +#: metarecord/admin/function.py:39 metarecord/models/function.py:113 msgid "template" msgstr "malli" -#: metarecord/admin.py:53 +#: metarecord/admin/function.py:40 msgid "classification code" msgstr "luokitustunnus" -#: metarecord/admin.py:57 metarecord/admin.py:82 metarecord/admin.py:94 -#: metarecord/admin.py:106 metarecord/models/attribute.py:13 -#: metarecord/models/attribute.py:25 metarecord/models/function.py:42 -msgid "name" -msgstr "nimi" - -#: metarecord/models/action.py:9 metarecord/models/phase.py:25 -msgid "phase" -msgstr "käsittelyvaihe" - -#: metarecord/models/action.py:25 metarecord/models/record.py:9 +#: metarecord/admin/record.py:33 metarecord/models/action.py:25 +#: metarecord/models/record.py:9 msgid "action" msgstr "toimenpide" @@ -72,7 +92,7 @@ msgstr "tunniste" msgid "group" msgstr "ryhmä" -#: metarecord/models/attribute.py:31 +#: metarecord/models/attribute.py:31 metarecord/models/attribute.py:75 msgid "help text" msgstr "ohjeteksti" @@ -80,11 +100,11 @@ msgstr "ohjeteksti" msgid "attribute" msgstr "attribuutti" -#: metarecord/models/attribute.py:35 metarecord/models/structural_element.py:23 +#: metarecord/models/attribute.py:35 metarecord/models/structural_element.py:25 msgid "attributes" msgstr "attribuutit" -#: metarecord/models/attribute.py:61 metarecord/views/base.py:101 +#: metarecord/models/attribute.py:61 metarecord/views/base.py:113 msgid "Invalid attribute." msgstr "Epäkelpo attribuutti." @@ -92,11 +112,11 @@ msgstr "Epäkelpo attribuutti." msgid "value" msgstr "arvo" -#: metarecord/models/attribute.py:76 +#: metarecord/models/attribute.py:78 msgid "attribute value" msgstr "attribuutin arvo" -#: metarecord/models/attribute.py:77 +#: metarecord/models/attribute.py:79 msgid "attribute values" msgstr "attribuuttien arvot" @@ -108,135 +128,188 @@ msgstr "lisäysaika" msgid "time of modification" msgstr "muokkausaika" -#: metarecord/models/classification.py:13 metarecord/models/record.py:11 +#: metarecord/models/bulk_update.py:21 metarecord/models/classification.py:68 +msgid "description" +msgstr "kuvaus" + +#: metarecord/models/bulk_update.py:24 metarecord/models/classification.py:44 +#: metarecord/models/structural_element.py:16 +msgid "created by" +msgstr "lisääjä" + +#: metarecord/models/bulk_update.py:42 +msgid "approved by" +msgstr "hyväksyjä" + +#: metarecord/models/bulk_update.py:49 metarecord/models/classification.py:60 +#: metarecord/models/structural_element.py:22 +msgid "created by (text)" +msgstr "lisääjä (teksti)" + +#: metarecord/models/bulk_update.py:50 metarecord/models/classification.py:61 +#: metarecord/models/function.py:199 metarecord/models/structural_element.py:23 +msgid "modified by (text)" +msgstr "muokkaaja (teksti)" + +#: metarecord/models/bulk_update.py:51 +msgid "approved by (text)" +msgstr "hyväksyjä (teksti)" + +#: metarecord/models/bulk_update.py:53 +msgid "is approved" +msgstr "Hyväksytty" + +#: metarecord/models/bulk_update.py:54 +msgid "changes" +msgstr "muutokset" + +#: metarecord/models/bulk_update.py:56 metarecord/models/classification.py:39 +#: metarecord/models/function.py:53 metarecord/models/function.py:201 +msgid "state" +msgstr "tila" + +#: metarecord/models/bulk_update.py:59 +msgid "The state that is assigned to functions after applying the updates" +msgstr "Tila johon käsittelyprosessit asetetaan muutosten jälkeen" + +#: metarecord/models/bulk_update.py:63 metarecord/models/function.py:62 +msgid "bulk update" +msgstr "massamuutos" + +#: metarecord/models/bulk_update.py:64 +msgid "bulk updates" +msgstr "massamuutokset" + +#: metarecord/models/bulk_update.py:66 +msgid "Can approve bulk update" +msgstr "Saa hyväksyä massamuutoksia" + +#: metarecord/models/bulk_update.py:82 +msgid "No permission to approve." +msgstr "Ei lupaa hyväksyä." + +#: metarecord/models/classification.py:26 metarecord/models/function.py:38 +msgid "Draft" +msgstr "Luonnos" + +#: metarecord/models/classification.py:27 metarecord/models/function.py:39 +msgid "Sent for review" +msgstr "Lähetetty hyväksyttäväksi" + +#: metarecord/models/classification.py:28 metarecord/models/function.py:40 +msgid "Waiting for approval" +msgstr "Odottaa hyväksyntää" + +#: metarecord/models/classification.py:29 metarecord/models/function.py:41 +msgid "Approved" +msgstr "Hyväksytty" + +#: metarecord/models/classification.py:40 metarecord/models/function.py:54 +#: metarecord/models/function.py:203 +msgid "valid from" +msgstr "voimassaoloaika alkaa" + +#: metarecord/models/classification.py:41 metarecord/models/function.py:55 +#: metarecord/models/function.py:204 +msgid "valid to" +msgstr "voimassaoloaika päättyy" + +#: metarecord/models/classification.py:64 metarecord/models/record.py:11 msgid "parent" msgstr "vanhempi" -#: metarecord/models/classification.py:15 -msgid "code" -msgstr "tunnus" - -#: metarecord/models/classification.py:16 +#: metarecord/models/classification.py:67 msgid "title" msgstr "nimeke" -#: metarecord/models/classification.py:17 -msgid "description" -msgstr "kuvaus" - -#: metarecord/models/classification.py:18 +#: metarecord/models/classification.py:69 msgid "description internal" msgstr "sisäinen kuvaus" -#: metarecord/models/classification.py:19 +#: metarecord/models/classification.py:70 msgid "related classification" msgstr "liittyvä tehtäväluokka" -#: metarecord/models/classification.py:20 +#: metarecord/models/classification.py:71 msgid "additional information" msgstr "lisätiedot" -#: metarecord/models/classification.py:21 +#: metarecord/models/classification.py:72 msgid "function allowed" msgstr "käsittelyprosessi sallittu" -#: metarecord/models/classification.py:24 metarecord/models/function.py:50 +#: metarecord/models/classification.py:77 metarecord/models/function.py:57 msgid "classification" msgstr "tehtäväluokka" -#: metarecord/models/classification.py:25 +#: metarecord/models/classification.py:78 msgid "classifications" msgstr "tehtäväluokat" -#: metarecord/models/function.py:30 -msgid "Draft" -msgstr "Luonnos" +#: metarecord/models/classification.py:81 +msgid "Can edit classification" +msgstr "Saa muokata tehtäväluokkia" -#: metarecord/models/function.py:31 -msgid "Sent for review" -msgstr "Lähetetty hyväksyttäväksi" +#: metarecord/models/classification.py:82 +msgid "Can review classification" +msgstr "Saa tarkastaa tehtäväluokkia" -#: metarecord/models/function.py:32 -msgid "Waiting for approval" -msgstr "Odottaa hyväksyntää" +#: metarecord/models/classification.py:83 +msgid "Can approve classification" +msgstr "Saa hyväksyä tehtäväluokkia" -#: metarecord/models/function.py:33 -msgid "Approved" -msgstr "Hyväksytty" +#: metarecord/models/classification.py:84 +msgid "Can view classification modified by" +msgstr "Saa nähdä tehtäväluokan muokkaajan" -#: metarecord/models/function.py:34 -msgid "Deleted" -msgstr "" - -#: metarecord/models/function.py:43 +#: metarecord/models/function.py:50 msgid "error count" msgstr "virheiden määrä" -#: metarecord/models/function.py:44 +#: metarecord/models/function.py:51 msgid "is template" msgstr "malli" -#: metarecord/models/function.py:46 metarecord/models/function.py:173 -msgid "state" -msgstr "tila" - -#: metarecord/models/function.py:47 metarecord/models/function.py:175 -msgid "valid from" -msgstr "voimassaoloaika alkaa" - -#: metarecord/models/function.py:48 metarecord/models/function.py:176 -msgid "valid to" -msgstr "voimassaoloaika päättyy" - -#: metarecord/models/function.py:90 metarecord/models/function.py:166 -#: metarecord/models/phase.py:9 -msgid "function" -msgstr "käsittelyprosessi" - -#: metarecord/models/function.py:91 +#: metarecord/models/function.py:100 #: metarecord/templates/admin/metarecord/function/import_tos.html:55 msgid "functions" msgstr "käsittelyprosessit" -#: metarecord/models/function.py:94 +#: metarecord/models/function.py:103 msgid "Can edit" msgstr "Saa muokata" -#: metarecord/models/function.py:95 +#: metarecord/models/function.py:104 msgid "Can review" msgstr "Saa tarkastaa" -#: metarecord/models/function.py:96 +#: metarecord/models/function.py:105 msgid "Can approve" msgstr "Saa hyväksyä" -#: metarecord/models/function.py:97 +#: metarecord/models/function.py:106 msgid "Can view modified by" msgstr "Saa nähdä muokkaajan" -#: metarecord/models/function.py:168 +#: metarecord/models/function.py:195 msgid "modified at" msgstr "muokkausaika" -#: metarecord/models/function.py:170 metarecord/models/structural_element.py:19 -msgid "modified by" -msgstr "muokkaaja" - #: metarecord/models/phase.py:26 msgid "phases" msgstr "käsittelyvaiheet" -#: metarecord/models/record.py:51 +#: metarecord/models/record.py:45 msgid "record" msgstr "asiakirja" -#: metarecord/models/record.py:52 +#: metarecord/models/record.py:46 msgid "records" msgstr "asiakirjat" -#: metarecord/models/structural_element.py:16 -msgid "created by" -msgstr "lisääjä" +#: metarecord/templates/admin/metarecord/classification/change_form.html:9 +msgid "Create new version" +msgstr "Luo uusi versio" #: metarecord/templates/admin/metarecord/function/change_list.html:9 #: metarecord/templates/admin/metarecord/function/import_tos.html:25 @@ -259,79 +332,104 @@ msgstr "Etusivu" msgid "TOS import" msgstr "TOS-tuonti" -#: metarecord/views/admin.py:37 +#: metarecord/tests/test_api.py:2040 metarecord/views/classification.py:163 +#: metarecord/views/function.py:239 +msgid "Invalid state change." +msgstr "Virheellinen tilan vaihto." + +#: metarecord/views/admin.py:40 #, python-format msgid "File \"%s\" was imported successfully!" msgstr "Tiedosto \"%s\" tuotu onnistuneesti!" -#: metarecord/views/admin.py:39 +#: metarecord/views/admin.py:42 #, python-format msgid "Error importing file \"%s\"" msgstr "Tiedoston \"%s\" tuonti epäonnistui" -#: metarecord/views/base.py:97 metarecord/views/base.py:129 -#: metarecord/views/base.py:130 +#: metarecord/views/base.py:109 metarecord/views/base.py:141 +#: metarecord/views/base.py:142 msgid "This attribute is required." msgstr "Tämä attribuutti on pakollinen." -#: metarecord/views/base.py:105 +#: metarecord/views/base.py:117 msgid "This attribute isn't allowed." msgstr "Tämä attribuutti ei ole sallittu." -#: metarecord/views/base.py:109 +#: metarecord/views/base.py:121 msgid "This attribute does not allow multiple values." msgstr "Tälle attribuutille ei voi antaa kuin yhden arvon." -#: metarecord/views/base.py:119 +#: metarecord/views/base.py:131 msgid "Value must be a string." msgstr "Arvo täytyy olla merkkijono." -#: metarecord/views/base.py:125 +#: metarecord/views/base.py:137 msgid "Invalid value." msgstr "Epäkelpo arvo." -#: metarecord/views/function.py:115 -msgid "\"valid_from\" cannot be after \"valid_to\"." -msgstr "\"valid_from\" ei voi olla myöhemmin kuin \"valid_to\"." +#: metarecord/views/bulk_update.py:66 +msgid "No permission to create bulk update" +msgstr "Ei lupaa luoda massamuutosta" -#: metarecord/views/function.py:120 -#, python-format -msgid "Classification %s already has a function." -msgstr "Luokalla %s on jo käsittelyprosessi." +#: metarecord/views/bulk_update.py:77 +msgid "No permission to update bulk update" +msgstr "Ei lupaa päivittää massamuutosta" -#: metarecord/views/function.py:124 -#, python-format -msgid "Classification %s does not allow function creation." -msgstr "Luokalle %s ei voi lisätä käsittelyprosessia." - -#: metarecord/views/function.py:134 -msgid "No permission to create." -msgstr "Ei lupaa luoda." +#: metarecord/views/bulk_update.py:103 +msgid "No permission to delete bulk update" +msgstr "Ei lupaa poistaa massamuutosta" -#: metarecord/views/function.py:162 +#: metarecord/views/classification.py:143 metarecord/views/function.py:170 msgid "\"state\", \"valid_from\" or \"valid_to\" required." msgstr "" "Joku kentistä \"state\", \"valid_from\" tai \"valid_to\" on pakollinen." -#: metarecord/views/function.py:192 -msgid "No permission to edit." -msgstr "Ei lupaa muokata." +#: metarecord/views/classification.py:175 metarecord/views/function.py:251 +msgid "No permission for the state change." +msgstr "Ei lupaa tilan vaihtoon." + +#: metarecord/views/classification.py:182 metarecord/views/function.py:141 +msgid "No permission to create." +msgstr "Ei lupaa luoda." -#: metarecord/views/function.py:196 +#: metarecord/views/classification.py:196 +msgid "No permission to update." +msgstr "Ei lupaa päivittää." + +#: metarecord/views/classification.py:215 metarecord/views/function.py:213 msgid "" "Cannot edit while in state \"sent_for_review\" or \"waiting_for_approval\"" msgstr "" "Muokkaus ei ole mahdollista kun tila on \"sent_for_review\" tai " "\"waiting_for_approval\"" -#: metarecord/views/function.py:220 -msgid "Invalid state change." -msgstr "Virheellinen tilan vaihto." +#: metarecord/views/function.py:122 +msgid "\"valid_from\" cannot be after \"valid_to\"." +msgstr "\"valid_from\" ei voi olla myöhemmin kuin \"valid_to\"." -#: metarecord/views/function.py:232 -msgid "No permission for the state change." -msgstr "Ei oikeutta tilan vaihtoon." +#: metarecord/views/function.py:127 +#, python-format +msgid "Classification %s already has a function." +msgstr "Luokalla %s on jo käsittelyprosessi." + +#: metarecord/views/function.py:131 +#, python-format +msgid "Classification %s does not allow function creation." +msgstr "Luokalle %s ei voi lisätä käsittelyprosessia." + +#: metarecord/views/function.py:185 +msgid "Changing classification is not allowed. Only version can be changed." +msgstr "Tehtäväluokan muuttaminen ei ole sallittu. Vain version muuttaminen on sallittua." + +#: metarecord/views/function.py:209 +msgid "No permission to edit." +msgstr "Ei lupaa muokata." + +#: metarecord/views/function.py:317 +msgid "Invalid UUID" +msgstr "Epäkelpo UUID." -#: metarecord/views/function.py:298 +#: metarecord/views/function.py:349 msgid "No permission to delete or state is not \"draft\"." msgstr "Ei oikeutta poistaa tai tila ei ole luonnos." diff --git a/metarecord/admin/__init__.py b/metarecord/admin/__init__.py index aa60afd6..dda373a0 100644 --- a/metarecord/admin/__init__.py +++ b/metarecord/admin/__init__.py @@ -1,5 +1,6 @@ from .action import ActionAdmin # noqa from .attribute import AttributeAdmin, AttributeGroupAdmin # noqa +from .attribute_validation import AttributeValidationRuleAdmin # noqa from .bulk_update import BulkUpdateAdmin # noqa from .classification import ClassificationAdmin # noqa from .function import FunctionAdmin # noqa diff --git a/metarecord/admin/_common.py b/metarecord/admin/_common.py deleted file mode 100644 index ebf5c151..00000000 --- a/metarecord/admin/_common.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.contrib import admin - -from metarecord.forms import UTF8HStoreField - - -class StructuralElementAdmin(admin.ModelAdmin): - def get_form(self, request, obj=None, **kwargs): - kwargs['field_classes'] = {'attributes': UTF8HStoreField} - return super().get_form(request, obj, **kwargs) diff --git a/metarecord/admin/action.py b/metarecord/admin/action.py index e023fdce..064a305b 100644 --- a/metarecord/admin/action.py +++ b/metarecord/admin/action.py @@ -1,12 +1,11 @@ from django.contrib import admin -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ -from metarecord.admin._common import StructuralElementAdmin from metarecord.models.action import Action @admin.register(Action) -class ActionAdmin(StructuralElementAdmin): +class ActionAdmin(admin.ModelAdmin): list_display = ('get_classification_code', 'get_function_name', 'get_phase_name', 'get_name') list_filter = ('phase__function__classification__code',) search_fields = ('attributes',) diff --git a/metarecord/admin/attribute_validation.py b/metarecord/admin/attribute_validation.py new file mode 100644 index 00000000..026d7aac --- /dev/null +++ b/metarecord/admin/attribute_validation.py @@ -0,0 +1,200 @@ +from django.contrib import admin +from django.utils.translation import gettext_lazy as _ +from django_admin_json_editor import JSONEditorWidget + +from metarecord.models.attribute import Attribute, AttributeValue +from metarecord.models.attribute_validation import AttributeValidationRule + + +def dynamic_schema(widget): + """ + Dynamic schema created with: https://github.com/abogushov/django-admin-json-editor. + Example dynamic schemas can be found here: https://github.com/json-editor/json-editor. + """ + attribute_identifiers = list( + Attribute.objects.order_by("index", "identifier").values_list( + "identifier", flat=True + ) + ) + attribute_names = list( + Attribute.objects.order_by("index", "identifier").values_list("name", flat=True) + ) + attribute_values = list( + AttributeValue.objects.order_by("value").values_list("value", flat=True) + ) + + return { + "type": "object", + "title": str(_("Attribute validation rules")), + "properties": { + "allowed": { + "title": str(_("Allowed")), + "type": "array", + "items": { + "title": str(_("Allowed")), + "type": "string", + "enum": attribute_identifiers, + "options": { + "enum_titles": attribute_names, + }, + }, + }, + "required": { + "title": str(_("Required")), + "type": "array", + "items": { + "title": str(_("Required")), + "type": "string", + "enum": attribute_identifiers, + "options": { + "enum_titles": attribute_names, + }, + }, + }, + "conditionally_required": { + "title": str(_("Conditionally Required")), + "type": "array", + "items": { + "title": str(_("Conditionally Required")), + "type": "object", + "properties": { + "attribute": { + "title": str(_("Attribute")), + "type": "string", + "enum": attribute_identifiers, + "options": { + "enum_titles": attribute_names, + }, + }, + "conditions": { + "title": str(_("Required Conditions")), + "type": "array", + "items": { + "title": str(_("Required Condition")), + "type": "object", + "properties": { + "attribute": { + "title": str(_("Attribute")), + "type": "string", + "enum": attribute_identifiers, + "options": { + "enum_titles": attribute_names, + }, + }, + "value": { + "title": str(_("Value")), + "type": "string", + "enum": attribute_values, + }, + }, + "required": ["attribute", "value"], + }, + }, + }, + "required": ["attribute", "conditions"], + }, + }, + "conditionally_disallowed": { + "title": str(_("Conditionally Disallowed")), + "type": "array", + "items": { + "title": str(_("Conditionally Disallowed")), + "type": "object", + "properties": { + "attribute": { + "title": str(_("Attribute")), + "type": "string", + "enum": attribute_identifiers, + "options": { + "enum_titles": attribute_names, + }, + }, + "conditions": { + "title": str(_("Disallowed Conditions")), + "type": "array", + "items": { + "title": str(_("Disallowed Condition")), + "type": "object", + "properties": { + "attribute": { + "title": str(_("Attribute")), + "type": "string", + "enum": attribute_identifiers, + "options": { + "enum_titles": attribute_names, + }, + }, + "value": { + "title": str(_("Value")), + "type": "string", + "enum": attribute_values, + }, + }, + "required": ["attribute", "value"], + }, + }, + }, + "required": ["attribute", "conditions"], + }, + }, + "multivalued": { + "title": str(_("Multivalued")), + "type": "array", + "items": { + "title": str(_("Multivalued")), + "type": "string", + "enum": attribute_identifiers, + "options": { + "enum_titles": attribute_names, + }, + }, + }, + "all_or_none": { + "title": str(_("All or None")), + "type": "array", + "items": { + "title": str(_("Attribute set")), + "type": "array", + "items": { + "title": str(_("Attribute")), + "type": "string", + "enum": attribute_identifiers, + "options": { + "enum_titles": attribute_names, + }, + }, + }, + }, + "allow_values_outside_choices": { + "title": str(_("Values Outside Choices")), + "type": "array", + "items": { + "title": str(_("Values Outside Choices")), + "type": "string", + "enum": attribute_identifiers, + "options": { + "enum_titles": attribute_names, + }, + }, + }, + }, + "required": [ + "allowed", + "required", + "conditionally_required", + "conditionally_disallowed", + "multivalued", + "all_or_none", + "allow_values_outside_choices", + ], + } + + +@admin.register(AttributeValidationRule) +class AttributeValidationRuleAdmin(admin.ModelAdmin): + def get_form(self, request, obj=None, **kwargs): + widget = JSONEditorWidget(dynamic_schema, collapsed=False) + form = super().get_form( + request, obj, widgets={"validation_json": widget}, **kwargs + ) + return form diff --git a/metarecord/admin/classification.py b/metarecord/admin/classification.py index 1f6f4d6e..bae2eb0b 100644 --- a/metarecord/admin/classification.py +++ b/metarecord/admin/classification.py @@ -1,10 +1,48 @@ from django.contrib import admin +from django.http import HttpResponseForbidden, HttpResponseNotAllowed, HttpResponseRedirect +from django.shortcuts import get_object_or_404 +from django.urls import path, reverse from metarecord.models.classification import Classification @admin.register(Classification) class ClassificationAdmin(admin.ModelAdmin): - list_display = ('code', 'title', 'function_allowed') + change_form_template = "admin/metarecord/classification/change_form.html" + list_display = ('code', 'version', 'state', 'title', 'function_allowed') search_fields = ('code', 'title') - ordering = ('code',) + ordering = ('code', 'version') + list_filter = ('state', 'code') + + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path( + '/new-version/', + self.admin_site.admin_view(self.create_new_version_view), + name='metarecord_classification_create-new-version' + ), + ] + return custom_urls + urls + + def create_new_version_view(self, request, object_id): + """ + Create new classification draft version with the information + carried over from the latest version + """ + if request.method != 'POST': + return HttpResponseNotAllowed(permitted_methods='POST') + + if not request.user.has_perm(Classification.CAN_EDIT): + return HttpResponseForbidden() + + classification = get_object_or_404(Classification, pk=object_id) + + latest_version = Classification.objects.latest_version().get(uuid=classification.uuid) + latest_version.pk = None # New version will be created if pk is none + latest_version.state = Classification.DRAFT + latest_version.save() + + return HttpResponseRedirect( + reverse('admin:metarecord_classification_change', kwargs={"object_id": latest_version.pk}) + ) diff --git a/metarecord/admin/function.py b/metarecord/admin/function.py index 5245dc6e..3d2b070a 100644 --- a/metarecord/admin/function.py +++ b/metarecord/admin/function.py @@ -1,9 +1,8 @@ from django.contrib import admin from django.db import transaction from django.urls import path -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ -from metarecord.admin._common import StructuralElementAdmin from metarecord.models.function import Function, MetadataVersion from metarecord.views.admin import tos_import_view @@ -26,7 +25,7 @@ def has_add_permission(self, request, obj=None): @admin.register(Function) -class FunctionAdmin(StructuralElementAdmin): +class FunctionAdmin(admin.ModelAdmin): list_display = ('get_classification_code', 'get_name', 'state', 'version') list_filter = ('state', 'classification__code') search_fields = ('classification__code', 'classification__title') diff --git a/metarecord/admin/phase.py b/metarecord/admin/phase.py index 78590233..9127df47 100644 --- a/metarecord/admin/phase.py +++ b/metarecord/admin/phase.py @@ -1,12 +1,11 @@ from django.contrib import admin -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ -from metarecord.admin._common import StructuralElementAdmin from metarecord.models.phase import Phase @admin.register(Phase) -class PhaseAdmin(StructuralElementAdmin): +class PhaseAdmin(admin.ModelAdmin): list_display = ('get_classification_code', 'get_function_name', 'get_name') list_filter = ('function__classification__code',) search_fields = ('attributes',) diff --git a/metarecord/admin/record.py b/metarecord/admin/record.py index 59034efb..a237be0a 100644 --- a/metarecord/admin/record.py +++ b/metarecord/admin/record.py @@ -1,12 +1,11 @@ from django.contrib import admin -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ -from metarecord.admin._common import StructuralElementAdmin from metarecord.models.record import Record @admin.register(Record) -class RecordAdmin(StructuralElementAdmin): +class RecordAdmin(admin.ModelAdmin): list_display = ('get_classification_code', 'get_function_name', 'get_phase_name', 'get_action_name', 'get_name') list_filter = ('action__phase__function__classification__code',) search_fields = ('attributes',) diff --git a/metarecord/exporter/jhs.py b/metarecord/exporter/jhs.py index 08afc4da..2a2d7a89 100644 --- a/metarecord/exporter/jhs.py +++ b/metarecord/exporter/jhs.py @@ -134,7 +134,7 @@ def _handle_classification(self, classification): try: function = Function.objects.prefetch_related( 'phases', 'phases__actions', 'phases__actions__records' - ).filter(classification=classification).latest_approved().get() + ).filter(classification__uuid=classification.uuid).latest_approved().get() except Function.DoesNotExist: return jhs.Luokka( diff --git a/metarecord/forms.py b/metarecord/forms.py deleted file mode 100644 index b7b0c0d9..00000000 --- a/metarecord/forms.py +++ /dev/null @@ -1,13 +0,0 @@ -import json - -from django.contrib.postgres.forms import HStoreField - - -class UTF8HStoreField(HStoreField): - """ - Disable non ASCII character escaping in HStoreField - """ - def prepare_value(self, value): - if isinstance(value, dict): - return json.dumps(value, ensure_ascii=False) - return value diff --git a/metarecord/importer/tos.py b/metarecord/importer/tos.py index 1ae89c68..bc402def 100644 --- a/metarecord/importer/tos.py +++ b/metarecord/importer/tos.py @@ -176,28 +176,31 @@ def _get_classification_code(self, sheet): return str(classification_codes[index]) + def _get_classification(self, code): + queryset = Classification.objects.filter(code=code) + + if not queryset.exists(): + raise TOSImporterException( + 'Classification %s does not exist' % code + ) + + classification = queryset.latest_approved().first() + if not classification: + classification = queryset.latest_version().first() + return classification + def _import_function(self, sheet): classification_code = self._get_classification_code(sheet) if not classification_code: return - try: - classification = Classification.objects.get(code=classification_code) - except Classification.DoesNotExist: - raise TOSImporterException( - 'Classification %s does not exist' % classification_code - ) + classification = self._get_classification(classification_code) Function.objects.filter(classification=classification).delete() if not classification.function_allowed: print('Skipping, classification %s does not allow function creation.' % classification_code) return - if Function.objects.latest_version().filter(classification=classification).exists(): - raise TOSImporterException( - 'Classification %s already has a function.' % classification_code - ) - return Function.objects.create(classification=classification) def _clean_attributes(self, original_attribute_data, row_num=None): diff --git a/metarecord/management/commands/import_attributes.py b/metarecord/management/commands/import_attributes.py index 7e1d0ba6..7e28cc4e 100644 --- a/metarecord/management/commands/import_attributes.py +++ b/metarecord/management/commands/import_attributes.py @@ -16,7 +16,8 @@ def add_arguments(self, parser): def handle(self, *args, **options): filename = options['filename'] try: - tos_importer = TOSImporter(filename) + tos_importer = TOSImporter(options) + tos_importer.open(filename) except Exception as e: print("Cannot open file '%s': %s" % (filename, e)) return diff --git a/metarecord/migrations/0001_squashed_0048.py b/metarecord/migrations/0001_squashed_0048.py new file mode 100644 index 00000000..0bfff727 --- /dev/null +++ b/metarecord/migrations/0001_squashed_0048.py @@ -0,0 +1,1757 @@ +# Generated by Django 2.2.16 on 2021-01-15 06:25 + +import uuid + +import django.utils.timezone +from django.conf import settings +from django.db import migrations, models +from django.db.models import OuterRef, Subquery, Value as V +from django.db.models.functions import Concat, Trim + + +def add_indexes(apps, schema_editor): + Attribute = apps.get_model("metarecord", "Attribute") + for attribute in Attribute.objects.all(): + attribute.index = ( + max(Attribute.objects.values_list("index", flat=True) or [0]) + 1 + ) + attribute.save() + + +def add_initial_metarecord_versions(apps, schema_editor): + Function = apps.get_model("metarecord", "Function") + MetadataVersion = apps.get_model("metarecord", "MetadataVersion") + + for function in Function.objects.filter(is_template=False, metadata_versions=None): + MetadataVersion.objects.create( + function=function, + modified_at=function.modified_at, + modified_by=None, + state=function.state, + ) + + +def add_attribute_value_indexes(apps, schema_editor): + AttributeValue = apps.get_model("metarecord", "AttributeValue") + + for attribute_value in AttributeValue.objects.filter(index=0).order_by( + "created_at" + ): + new_index = ( + max( + AttributeValue.objects.filter( + attribute=attribute_value.attribute + ).values_list("index", flat=True) + or [0] + ) + + 1 + ) + AttributeValue.objects.filter(id=attribute_value.id).update(index=new_index) + + +def populate_classification(apps, schema_editor): + Classification = apps.get_model("metarecord", "Classification") + Function = apps.get_model("metarecord", "Function") + + for function in Function.objects.filter( + is_template=False, function_id__isnull=False + ).order_by("function_id"): + parent = ( + Classification.objects.get(code=function.parent.function_id) + if function.parent + else None + ) + classification, created = Classification.objects.update_or_create( + code=function.function_id, + defaults={ + "title": function.name, + "parent": parent, + }, + ) + function.classification = classification + function.name = "" + function.save(update_fields=("classification", "name")) + + +def populate_function_allowed(apps, schema_editor): + Classification = apps.get_model("metarecord", "Classification") + + for classification in Classification.objects.all(): + classification.function_allowed = not classification.children.exists() + classification.save(update_fields=("function_allowed",)) + + +def update_function_modified_by(apps, schema_editor): + Function = apps.get_model("metarecord", "Function") + + for function in Function.objects.exclude(metadata_versions=None): + metadata_version = function.metadata_versions.latest("id") + Function.objects.filter(id=function.id).update( + modified_by=metadata_version.modified_by + ) + + +def delete_soft_deleted_functions(apps, schema_editor): + Function = apps.get_model("metarecord", "Function") + Function.objects.filter(state="deleted").delete() + + +def populate_user_name_field(apps, field_name, model_class): + """ + Populate related user's full name to equivalent char field prefixed with + single underscore. The char field is expected to be in the same table as + the foreign key. + """ + User = apps.get_model("users", "User") + char_field_name = "_%s" % field_name + + full_name = ( + User.objects.filter(pk=OuterRef(field_name)) + .annotate(full_name=Trim(Concat("first_name", V(" "), "last_name"))) + .values_list("full_name")[:1] + ) + + model_class.objects.exclude(**{field_name: None}).update( + **{char_field_name: Subquery(full_name)} + ) + + +def populate_function_user_name_fields(apps, schema_editor): + Function = apps.get_model("metarecord", "Function") + populate_user_name_field(apps, "created_by", Function) + populate_user_name_field(apps, "modified_by", Function) + + +def populate_phase_user_name_fields(apps, schema_editor): + Phase = apps.get_model("metarecord", "Phase") + populate_user_name_field(apps, "created_by", Phase) + populate_user_name_field(apps, "modified_by", Phase) + + +def populate_action_user_name_fields(apps, schema_editor): + Action = apps.get_model("metarecord", "Action") + populate_user_name_field(apps, "created_by", Action) + populate_user_name_field(apps, "modified_by", Action) + + +def populate_record_user_name_fields(apps, schema_editor): + Record = apps.get_model("metarecord", "Record") + populate_user_name_field(apps, "created_by", Record) + populate_user_name_field(apps, "modified_by", Record) + + +def populate_bulk_update_user_name_fields(apps, schema_editor): + BulkUpdate = apps.get_model("metarecord", "BulkUpdate") + populate_user_name_field(apps, "approved_by", BulkUpdate) + populate_user_name_field(apps, "created_by", BulkUpdate) + populate_user_name_field(apps, "modified_by", BulkUpdate) + + +def populate_metadata_version_user_name_fields(apps, schema_editor): + MetadataVersion = apps.get_model("metarecord", "MetadataVersion") + populate_user_name_field(apps, "modified_by", MetadataVersion) + + +def set_existing_classifications_as_approved(apps, schema_editor): + Classification = apps.get_model("metarecord", "Classification") + Classification.objects.all().update(state="approved") + + +class Migration(migrations.Migration): + + replaces = [ + ("metarecord", "0001_initial"), + ("metarecord", "0002_function_id_unique_and_db_index"), + ("metarecord", "0003_auto_20160417_1157"), + ("metarecord", "0004_add_missing_attributes"), + ("metarecord", "0005_freetext_attributes"), + ("metarecord", "0006_new_data_model"), + ("metarecord", "0007_attribute_is_free_text"), + ("metarecord", "0008_longer_attribute_values"), + ("metarecord", "0009_add_record_attachment"), + ("metarecord", "0010_rename_order_to_index"), + ("metarecord", "0011_remove_record_attachment"), + ("metarecord", "0012_hstore_attributes"), + ("metarecord", "0013_add_function_is_template"), + ("metarecord", "0014_int_id_structural_elements"), + ("metarecord", "0015_versionable_functions"), + ("metarecord", "0016_add_attribute_index"), + ("metarecord", "0017_remove_record_type"), + ("metarecord", "0018_function_uuid_version_unique"), + ("metarecord", "0019_add_metarecord_version"), + ("metarecord", "0020_add_attribute_group"), + ("metarecord", "0021_add_validation_dates"), + ("metarecord", "0022_remove_name"), + ("metarecord", "0023_add_attribute_value_index"), + ("metarecord", "0024_attibute_value_index_unique"), + ("metarecord", "0025_add_attribute_help_text"), + ("metarecord", "0026_use_native_hstore"), + ("metarecord", "0027_add_classification"), + ("metarecord", "0028_remove_function_parent"), + ("metarecord", "0029_add_db_index_for_uuids"), + ("metarecord", "0030_add_can_view_modified_by_permission"), + ("metarecord", "0031_add_deleted_state"), + ("metarecord", "0032_add_additional_information_and_related_classification"), + ("metarecord", "0033_change_attributes_fields_to_jsonb"), + ("metarecord", "0034_add_function_classification_related_name"), + ("metarecord", "0035_add_on_deletes"), + ("metarecord", "0036_add_classification_function_allowed"), + ("metarecord", "0037_update_function_modified_by_to_latest_metadata"), + ("metarecord", "0038_disable_unique_attribute_value_index"), + ("metarecord", "0039_remove_deleted_state"), + ("metarecord", "0040_bulk_update"), + ("metarecord", "0041_excessmeta"), + ("metarecord", "0042_bulkupdate_approved_by"), + ("metarecord", "0043_add_name_and_help_text_to_attribute_value"), + ("metarecord", "0044_user_relation_texts"), + ("metarecord", "0045_populate_persistent_user_name_fields"), + ("metarecord", "0046_classification_versioning"), + ("metarecord", "0047_approve_existing_classifications"), + ("metarecord", "0048_replace_pg_jsonfield_with_django_jsonfield"), + ] + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="Action", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created_at", + models.DateTimeField( + default=django.utils.timezone.now, + editable=False, + verbose_name="time of creation", + ), + ), + ( + "modified_at", + models.DateTimeField( + default=django.utils.timezone.now, + editable=False, + verbose_name="time of modification", + ), + ), + ("uuid", models.UUIDField(default=uuid.uuid4, editable=False)), + ( + "index", + models.PositiveSmallIntegerField( + db_index=True, editable=False, null=True + ), + ), + ( + "attributes", + models.JSONField( + blank=True, default=dict, verbose_name="attributes" + ), + ), + ("name", models.CharField(max_length=256, verbose_name="name")), + ( + "created_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="action_created", + to=settings.AUTH_USER_MODEL, + verbose_name="created by", + ), + ), + ( + "modified_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="action_modified", + to=settings.AUTH_USER_MODEL, + verbose_name="modified by", + ), + ), + ], + options={ + "ordering": ("phase", "index"), + "verbose_name": "action", + "verbose_name_plural": "actions", + }, + ), + migrations.CreateModel( + name="Attribute", + fields=[ + ( + "created_at", + models.DateTimeField( + default=django.utils.timezone.now, + editable=False, + verbose_name="time of creation", + ), + ), + ( + "modified_at", + models.DateTimeField( + default=django.utils.timezone.now, + editable=False, + verbose_name="time of modification", + ), + ), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ( + "identifier", + models.CharField( + db_index=True, + max_length=64, + unique=True, + verbose_name="identifier", + ), + ), + ("name", models.CharField(max_length=256, verbose_name="name")), + ("index", models.PositiveSmallIntegerField(default=0)), + ], + options={ + "verbose_name": "attribute", + "verbose_name_plural": "attributes", + "ordering": ("index",), + }, + ), + migrations.CreateModel( + name="Function", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created_at", + models.DateTimeField( + default=django.utils.timezone.now, + editable=False, + verbose_name="time of creation", + ), + ), + ( + "modified_at", + models.DateTimeField( + default=django.utils.timezone.now, + editable=False, + verbose_name="time of modification", + ), + ), + ("uuid", models.UUIDField(default=uuid.uuid4, editable=False)), + ( + "index", + models.PositiveSmallIntegerField( + db_index=True, editable=False, null=True + ), + ), + ( + "attributes", + models.JSONField( + blank=True, default=dict, verbose_name="attributes" + ), + ), + ( + "function_id", + models.CharField( + db_index=True, + max_length=16, + null=True, + verbose_name="function ID", + ), + ), + ("name", models.CharField(max_length=256, verbose_name="name")), + ("error_count", models.PositiveIntegerField(default=0)), + ( + "is_template", + models.BooleanField(default=False, verbose_name="is template"), + ), + ( + "created_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="function_created", + to=settings.AUTH_USER_MODEL, + verbose_name="created by", + ), + ), + ( + "modified_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="function_modified", + to=settings.AUTH_USER_MODEL, + verbose_name="modified by", + ), + ), + ( + "parent", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="children", + to="metarecord.Function", + verbose_name="parent", + ), + ), + ( + "state", + models.CharField( + choices=[ + ("draft", "Draft"), + ("sent_for_review", "Sent for review"), + ("waiting_for_approval", "Waiting for approval"), + ("approved", "Approved"), + ], + default="draft", + max_length=20, + ), + ), + ( + "version", + models.PositiveIntegerField( + blank=True, db_index=True, default=1, null=True + ), + ), + ], + options={ + "verbose_name": "function", + "verbose_name_plural": "functions", + "unique_together": {("uuid", "version")}, + }, + ), + migrations.CreateModel( + name="Phase", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created_at", + models.DateTimeField( + default=django.utils.timezone.now, + editable=False, + verbose_name="time of creation", + ), + ), + ( + "modified_at", + models.DateTimeField( + default=django.utils.timezone.now, + editable=False, + verbose_name="time of modification", + ), + ), + ("uuid", models.UUIDField(default=uuid.uuid4, editable=False)), + ( + "index", + models.PositiveSmallIntegerField( + db_index=True, editable=False, null=True + ), + ), + ( + "attributes", + models.JSONField( + blank=True, default=dict, verbose_name="attributes" + ), + ), + ("name", models.CharField(max_length=256, verbose_name="name")), + ( + "created_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="phase_created", + to=settings.AUTH_USER_MODEL, + verbose_name="created by", + ), + ), + ( + "function", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="phases", + to="metarecord.Function", + verbose_name="function", + ), + ), + ( + "modified_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="phase_modified", + to=settings.AUTH_USER_MODEL, + verbose_name="modified by", + ), + ), + ], + options={ + "ordering": ("function", "index"), + "verbose_name": "phase", + "verbose_name_plural": "phases", + }, + ), + migrations.CreateModel( + name="RecordType", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ( + "created_at", + models.DateTimeField( + default=django.utils.timezone.now, + editable=False, + verbose_name="time of creation", + ), + ), + ( + "modified_at", + models.DateTimeField( + default=django.utils.timezone.now, + editable=False, + verbose_name="time of modification", + ), + ), + ("value", models.CharField(max_length=256, verbose_name="name")), + ], + options={ + "verbose_name": "record type", + "verbose_name_plural": "record types", + }, + ), + migrations.CreateModel( + name="Record", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created_at", + models.DateTimeField( + default=django.utils.timezone.now, + editable=False, + verbose_name="time of creation", + ), + ), + ( + "modified_at", + models.DateTimeField( + default=django.utils.timezone.now, + editable=False, + verbose_name="time of modification", + ), + ), + ("uuid", models.UUIDField(default=uuid.uuid4, editable=False)), + ( + "index", + models.PositiveSmallIntegerField( + db_index=True, editable=False, null=True + ), + ), + ( + "attributes", + models.JSONField( + blank=True, default=dict, verbose_name="attributes" + ), + ), + ( + "name", + models.CharField(max_length=256, verbose_name="type specifier"), + ), + ( + "action", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="records", + to="metarecord.Action", + verbose_name="action", + ), + ), + ( + "created_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="record_created", + to=settings.AUTH_USER_MODEL, + verbose_name="created by", + ), + ), + ( + "modified_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="record_modified", + to=settings.AUTH_USER_MODEL, + verbose_name="modified by", + ), + ), + ( + "parent", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="children", + to="metarecord.Record", + verbose_name="parent", + ), + ), + ( + "type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="records", + to="metarecord.RecordType", + verbose_name="type", + ), + ), + ], + options={ + "ordering": ("action", "index"), + "verbose_name": "record", + "verbose_name_plural": "records", + }, + ), + migrations.AddField( + model_name="action", + name="phase", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="actions", + to="metarecord.Phase", + verbose_name="phase", + ), + ), + migrations.CreateModel( + name="AttributeValue", + fields=[ + ( + "created_at", + models.DateTimeField( + default=django.utils.timezone.now, + editable=False, + verbose_name="time of creation", + ), + ), + ( + "modified_at", + models.DateTimeField( + default=django.utils.timezone.now, + editable=False, + verbose_name="time of modification", + ), + ), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("value", models.CharField(max_length=1024, verbose_name="value")), + ( + "attribute", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="values", + to="metarecord.Attribute", + verbose_name="attribute", + ), + ), + ], + options={ + "verbose_name": "attribute value", + "verbose_name_plural": "attribute values", + "unique_together": {("attribute", "value")}, + }, + ), + migrations.RunPython( + code=add_indexes, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.AlterField( + model_name="attribute", + name="index", + field=models.PositiveSmallIntegerField(db_index=True), + ), + migrations.RemoveField( + model_name="record", + name="type", + ), + migrations.DeleteModel( + name="RecordType", + ), + migrations.AlterModelOptions( + name="function", + options={ + "permissions": ( + ("can_edit", "Can edit"), + ("can_review", "Can review"), + ("can_approve", "Can approve"), + ), + "verbose_name": "function", + "verbose_name_plural": "functions", + }, + ), + migrations.AlterField( + model_name="action", + name="created_at", + field=models.DateTimeField( + auto_now_add=True, verbose_name="time of creation" + ), + ), + migrations.AlterField( + model_name="action", + name="modified_at", + field=models.DateTimeField( + auto_now=True, verbose_name="time of modification" + ), + ), + migrations.AlterField( + model_name="attribute", + name="created_at", + field=models.DateTimeField( + auto_now_add=True, verbose_name="time of creation" + ), + ), + migrations.AlterField( + model_name="attribute", + name="modified_at", + field=models.DateTimeField( + auto_now=True, verbose_name="time of modification" + ), + ), + migrations.AlterField( + model_name="attributevalue", + name="created_at", + field=models.DateTimeField( + auto_now_add=True, verbose_name="time of creation" + ), + ), + migrations.AlterField( + model_name="attributevalue", + name="modified_at", + field=models.DateTimeField( + auto_now=True, verbose_name="time of modification" + ), + ), + migrations.AlterField( + model_name="function", + name="created_at", + field=models.DateTimeField( + auto_now_add=True, verbose_name="time of creation" + ), + ), + migrations.AlterField( + model_name="function", + name="error_count", + field=models.PositiveIntegerField(default=0, verbose_name="error count"), + ), + migrations.AlterField( + model_name="function", + name="modified_at", + field=models.DateTimeField( + auto_now=True, verbose_name="time of modification" + ), + ), + migrations.AlterField( + model_name="function", + name="state", + field=models.CharField( + choices=[ + ("draft", "Draft"), + ("sent_for_review", "Sent for review"), + ("waiting_for_approval", "Waiting for approval"), + ("approved", "Approved"), + ], + default="draft", + max_length=20, + verbose_name="state", + ), + ), + migrations.AlterField( + model_name="phase", + name="created_at", + field=models.DateTimeField( + auto_now_add=True, verbose_name="time of creation" + ), + ), + migrations.AlterField( + model_name="phase", + name="modified_at", + field=models.DateTimeField( + auto_now=True, verbose_name="time of modification" + ), + ), + migrations.AlterField( + model_name="record", + name="created_at", + field=models.DateTimeField( + auto_now_add=True, verbose_name="time of creation" + ), + ), + migrations.AlterField( + model_name="record", + name="modified_at", + field=models.DateTimeField( + auto_now=True, verbose_name="time of modification" + ), + ), + migrations.CreateModel( + name="MetadataVersion", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("modified_at", models.DateTimeField(verbose_name="modified at")), + ( + "state", + models.CharField( + choices=[ + ("draft", "Draft"), + ("sent_for_review", "Sent for review"), + ("waiting_for_approval", "Waiting for approval"), + ("approved", "Approved"), + ], + default="draft", + max_length=20, + verbose_name="state", + ), + ), + ( + "function", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="metadata_versions", + to="metarecord.Function", + verbose_name="function", + ), + ), + ( + "modified_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + verbose_name="modified by", + ), + ), + ], + options={ + "ordering": ("id",), + }, + ), + migrations.RunPython( + code=add_initial_metarecord_versions, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.CreateModel( + name="AttributeGroup", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=64, verbose_name="name")), + ], + options={ + "verbose_name": "attribute group", + "verbose_name_plural": "attribute groups", + }, + ), + migrations.AddField( + model_name="attribute", + name="group", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="attributes", + to="metarecord.AttributeGroup", + verbose_name="group", + ), + ), + migrations.AddField( + model_name="function", + name="valid_to", + field=models.DateField(blank=True, null=True, verbose_name="valid to"), + ), + migrations.AddField( + model_name="function", + name="valid_from", + field=models.DateField(blank=True, null=True, verbose_name="valid from"), + ), + migrations.AddField( + model_name="metadataversion", + name="valid_to", + field=models.DateField(blank=True, null=True, verbose_name="valid to"), + ), + migrations.AddField( + model_name="metadataversion", + name="valid_from", + field=models.DateField(blank=True, null=True, verbose_name="valid from"), + ), + migrations.RemoveField( + model_name="action", + name="name", + ), + migrations.RemoveField( + model_name="phase", + name="name", + ), + migrations.RemoveField( + model_name="record", + name="name", + ), + migrations.AlterModelOptions( + name="attributevalue", + options={ + "ordering": ("index",), + "verbose_name": "attribute value", + "verbose_name_plural": "attribute values", + }, + ), + migrations.AddField( + model_name="attributevalue", + name="index", + field=models.PositiveSmallIntegerField(db_index=True, default=0), + preserve_default=False, + ), + migrations.RunPython( + code=add_attribute_value_indexes, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.AlterUniqueTogether( + name="attributevalue", + unique_together={("attribute", "index"), ("attribute", "value")}, + ), + migrations.AddField( + model_name="attribute", + name="help_text", + field=models.TextField(blank=True, verbose_name="help text"), + ), + migrations.CreateModel( + name="Classification", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "uuid", + models.UUIDField(db_index=True, default=uuid.uuid4, editable=False), + ), + ( + "code", + models.CharField(db_index=True, max_length=16, verbose_name="code"), + ), + ("title", models.CharField(max_length=256, verbose_name="title")), + ( + "description", + models.TextField(blank=True, verbose_name="description"), + ), + ( + "description_internal", + models.TextField(blank=True, verbose_name="description internal"), + ), + ( + "parent", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="children", + to="metarecord.Classification", + verbose_name="parent", + ), + ), + ( + "created_at", + models.DateTimeField( + auto_now_add=True, verbose_name="time of creation" + ), + ), + ( + "modified_at", + models.DateTimeField( + auto_now=True, verbose_name="time of modification" + ), + ), + ], + options={ + "verbose_name_plural": "classifications", + "verbose_name": "classification", + }, + ), + migrations.AddField( + model_name="function", + name="classification", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="metarecord.Classification", + verbose_name="classification", + ), + ), + migrations.RunPython( + code=populate_classification, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.RemoveField( + model_name="function", + name="parent", + ), + migrations.RemoveField( + model_name="function", + name="function_id", + ), + migrations.AlterField( + model_name="function", + name="name", + field=models.CharField(blank=True, max_length=256, verbose_name="name"), + ), + migrations.AlterField( + model_name="classification", + name="parent", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="children", + to="metarecord.Classification", + verbose_name="parent", + ), + ), + migrations.AlterField( + model_name="action", + name="uuid", + field=models.UUIDField(db_index=True, default=uuid.uuid4, editable=False), + ), + migrations.AlterField( + model_name="function", + name="uuid", + field=models.UUIDField(db_index=True, default=uuid.uuid4, editable=False), + ), + migrations.AlterField( + model_name="phase", + name="uuid", + field=models.UUIDField(db_index=True, default=uuid.uuid4, editable=False), + ), + migrations.AlterField( + model_name="record", + name="uuid", + field=models.UUIDField(db_index=True, default=uuid.uuid4, editable=False), + ), + migrations.AlterModelOptions( + name="function", + options={ + "permissions": ( + ("can_edit", "Can edit"), + ("can_review", "Can review"), + ("can_approve", "Can approve"), + ("can_view_modified_by", "Can view modified by"), + ), + "verbose_name": "function", + "verbose_name_plural": "functions", + }, + ), + migrations.AlterField( + model_name="function", + name="state", + field=models.CharField( + choices=[ + ("draft", "Draft"), + ("sent_for_review", "Sent for review"), + ("waiting_for_approval", "Waiting for approval"), + ("approved", "Approved"), + ("deleted", "Deleted"), + ], + default="draft", + max_length=20, + verbose_name="state", + ), + ), + migrations.AlterField( + model_name="metadataversion", + name="state", + field=models.CharField( + choices=[ + ("draft", "Draft"), + ("sent_for_review", "Sent for review"), + ("waiting_for_approval", "Waiting for approval"), + ("approved", "Approved"), + ("deleted", "Deleted"), + ], + default="draft", + max_length=20, + verbose_name="state", + ), + ), + migrations.AddField( + model_name="classification", + name="additional_information", + field=models.TextField(blank=True, verbose_name="additional information"), + ), + migrations.AddField( + model_name="classification", + name="related_classification", + field=models.TextField(blank=True, verbose_name="related classification"), + ), + migrations.AlterField( + model_name="function", + name="classification", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="functions", + to="metarecord.Classification", + verbose_name="classification", + ), + ), + migrations.AlterField( + model_name="action", + name="created_by", + field=models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="action_created", + to=settings.AUTH_USER_MODEL, + verbose_name="created by", + ), + ), + migrations.AlterField( + model_name="action", + name="modified_by", + field=models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="action_modified", + to=settings.AUTH_USER_MODEL, + verbose_name="modified by", + ), + ), + migrations.AlterField( + model_name="attribute", + name="group", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="attributes", + to="metarecord.AttributeGroup", + verbose_name="group", + ), + ), + migrations.AlterField( + model_name="classification", + name="parent", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="children", + to="metarecord.Classification", + verbose_name="parent", + ), + ), + migrations.AlterField( + model_name="function", + name="classification", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="functions", + to="metarecord.Classification", + verbose_name="classification", + ), + ), + migrations.AlterField( + model_name="function", + name="created_by", + field=models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="function_created", + to=settings.AUTH_USER_MODEL, + verbose_name="created by", + ), + ), + migrations.AlterField( + model_name="function", + name="modified_by", + field=models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="function_modified", + to=settings.AUTH_USER_MODEL, + verbose_name="modified by", + ), + ), + migrations.AlterField( + model_name="metadataversion", + name="modified_by", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + verbose_name="modified by", + ), + ), + migrations.AlterField( + model_name="phase", + name="created_by", + field=models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="phase_created", + to=settings.AUTH_USER_MODEL, + verbose_name="created by", + ), + ), + migrations.AlterField( + model_name="phase", + name="modified_by", + field=models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="phase_modified", + to=settings.AUTH_USER_MODEL, + verbose_name="modified by", + ), + ), + migrations.AlterField( + model_name="record", + name="created_by", + field=models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="record_created", + to=settings.AUTH_USER_MODEL, + verbose_name="created by", + ), + ), + migrations.AlterField( + model_name="record", + name="modified_by", + field=models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="record_modified", + to=settings.AUTH_USER_MODEL, + verbose_name="modified by", + ), + ), + migrations.AddField( + model_name="classification", + name="function_allowed", + field=models.BooleanField(default=False, verbose_name="function allowed"), + ), + migrations.RunPython( + code=populate_function_allowed, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.RunPython( + code=update_function_modified_by, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.AlterUniqueTogether( + name="attributevalue", + unique_together={("attribute", "value")}, + ), + migrations.RunPython( + code=delete_soft_deleted_functions, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.AlterField( + model_name="function", + name="state", + field=models.CharField( + choices=[ + ("draft", "Draft"), + ("sent_for_review", "Sent for review"), + ("waiting_for_approval", "Waiting for approval"), + ("approved", "Approved"), + ], + default="draft", + max_length=20, + verbose_name="state", + ), + ), + migrations.AlterField( + model_name="metadataversion", + name="state", + field=models.CharField( + choices=[ + ("draft", "Draft"), + ("sent_for_review", "Sent for review"), + ("waiting_for_approval", "Waiting for approval"), + ("approved", "Approved"), + ], + default="draft", + max_length=20, + verbose_name="state", + ), + ), + migrations.CreateModel( + name="BulkUpdate", + fields=[ + ( + "created_at", + models.DateTimeField( + auto_now_add=True, verbose_name="time of creation" + ), + ), + ( + "modified_at", + models.DateTimeField( + auto_now=True, verbose_name="time of modification" + ), + ), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ( + "description", + models.CharField( + blank=True, max_length=512, verbose_name="description" + ), + ), + ( + "is_approved", + models.BooleanField(default=False, verbose_name="is approved"), + ), + ( + "changes", + models.JSONField(blank=True, default=dict, verbose_name="changes"), + ), + ( + "state", + models.CharField( + choices=[ + ("draft", "Draft"), + ("sent_for_review", "Sent for review"), + ("waiting_for_approval", "Waiting for approval"), + ("approved", "Approved"), + ], + help_text="The state that is assigned to functions after applying the updates", + max_length=20, + verbose_name="state", + ), + ), + ( + "created_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="bulkupdate_created", + to=settings.AUTH_USER_MODEL, + verbose_name="created by", + ), + ), + ( + "modified_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="bulkupdate_modified", + to=settings.AUTH_USER_MODEL, + verbose_name="modified by", + ), + ), + ( + "approved_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="bulkupdate_approved", + to=settings.AUTH_USER_MODEL, + verbose_name="approved by", + ), + ), + ( + "_approved_by", + models.CharField( + blank=True, + editable=False, + max_length=200, + verbose_name="approved by (text)", + ), + ), + ( + "_created_by", + models.CharField( + blank=True, + editable=False, + max_length=200, + verbose_name="created by (text)", + ), + ), + ( + "_modified_by", + models.CharField( + blank=True, + editable=False, + max_length=200, + verbose_name="modified by (text)", + ), + ), + ], + options={ + "verbose_name": "bulk update", + "verbose_name_plural": "bulk updates", + "permissions": (("approve_bulkupdate", "Can approve bulk update"),), + }, + ), + migrations.AddField( + model_name="function", + name="bulk_update", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="functions", + to="metarecord.BulkUpdate", + verbose_name="bulk update", + ), + ), + migrations.AddField( + model_name="attributevalue", + name="help_text", + field=models.TextField(blank=True, verbose_name="help text"), + ), + migrations.AddField( + model_name="attributevalue", + name="name", + field=models.CharField(blank=True, max_length=256, verbose_name="name"), + ), + migrations.AddField( + model_name="action", + name="_created_by", + field=models.CharField( + blank=True, + editable=False, + max_length=200, + verbose_name="created by (text)", + ), + ), + migrations.AddField( + model_name="action", + name="_modified_by", + field=models.CharField( + blank=True, + editable=False, + max_length=200, + verbose_name="modified by (text)", + ), + ), + migrations.AddField( + model_name="function", + name="_created_by", + field=models.CharField( + blank=True, + editable=False, + max_length=200, + verbose_name="created by (text)", + ), + ), + migrations.AddField( + model_name="function", + name="_modified_by", + field=models.CharField( + blank=True, + editable=False, + max_length=200, + verbose_name="modified by (text)", + ), + ), + migrations.AddField( + model_name="metadataversion", + name="_modified_by", + field=models.CharField( + blank=True, + editable=False, + max_length=200, + verbose_name="modified by (text)", + ), + ), + migrations.AddField( + model_name="phase", + name="_created_by", + field=models.CharField( + blank=True, + editable=False, + max_length=200, + verbose_name="created by (text)", + ), + ), + migrations.AddField( + model_name="phase", + name="_modified_by", + field=models.CharField( + blank=True, + editable=False, + max_length=200, + verbose_name="modified by (text)", + ), + ), + migrations.AddField( + model_name="record", + name="_created_by", + field=models.CharField( + blank=True, + editable=False, + max_length=200, + verbose_name="created by (text)", + ), + ), + migrations.AddField( + model_name="record", + name="_modified_by", + field=models.CharField( + blank=True, + editable=False, + max_length=200, + verbose_name="modified by (text)", + ), + ), + migrations.RunPython( + code=populate_function_user_name_fields, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.RunPython( + code=populate_phase_user_name_fields, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.RunPython( + code=populate_action_user_name_fields, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.RunPython( + code=populate_record_user_name_fields, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.RunPython( + code=populate_bulk_update_user_name_fields, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.RunPython( + code=populate_metadata_version_user_name_fields, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.AlterModelOptions( + name="classification", + options={ + "permissions": ( + ("can_edit_classification", "Can edit classification"), + ("can_review_classification", "Can review classification"), + ("can_approve_classification", "Can approve classification"), + ( + "can_view_classification_modified_by", + "Can view classification modified by", + ), + ), + "verbose_name": "classification", + "verbose_name_plural": "classifications", + }, + ), + migrations.AddField( + model_name="classification", + name="_created_by", + field=models.CharField( + blank=True, + editable=False, + max_length=200, + verbose_name="created by (text)", + ), + ), + migrations.AddField( + model_name="classification", + name="_modified_by", + field=models.CharField( + blank=True, + editable=False, + max_length=200, + verbose_name="modified by (text)", + ), + ), + migrations.AddField( + model_name="classification", + name="created_by", + field=models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="classification_created", + to=settings.AUTH_USER_MODEL, + verbose_name="created by", + ), + ), + migrations.AddField( + model_name="classification", + name="modified_by", + field=models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="classification_modified", + to=settings.AUTH_USER_MODEL, + verbose_name="modified by", + ), + ), + migrations.AddField( + model_name="classification", + name="state", + field=models.CharField( + choices=[ + ("draft", "Draft"), + ("sent_for_review", "Sent for review"), + ("waiting_for_approval", "Waiting for approval"), + ("approved", "Approved"), + ], + default="draft", + max_length=20, + verbose_name="state", + ), + ), + migrations.AddField( + model_name="classification", + name="valid_from", + field=models.DateField(blank=True, null=True, verbose_name="valid from"), + ), + migrations.AddField( + model_name="classification", + name="valid_to", + field=models.DateField(blank=True, null=True, verbose_name="valid to"), + ), + migrations.AddField( + model_name="classification", + name="version", + field=models.PositiveIntegerField( + blank=True, db_index=True, default=1, null=True + ), + ), + migrations.AlterUniqueTogether( + name="classification", + unique_together={("uuid", "version")}, + ), + migrations.RunPython( + code=set_existing_classifications_as_approved, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + ] diff --git a/metarecord/migrations/0002_attributevalidationrule.py b/metarecord/migrations/0002_attributevalidationrule.py new file mode 100644 index 00000000..abb25156 --- /dev/null +++ b/metarecord/migrations/0002_attributevalidationrule.py @@ -0,0 +1,65 @@ +# Generated by Django 3.1.5 on 2021-02-15 09:20 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ("contenttypes", "0002_remove_content_type_name"), + ("metarecord", "0001_squashed_0048"), + ] + + operations = [ + migrations.CreateModel( + name="AttributeValidationRule", + fields=[ + ( + "created_at", + models.DateTimeField( + auto_now_add=True, verbose_name="time of creation" + ), + ), + ( + "modified_at", + models.DateTimeField( + auto_now=True, verbose_name="time of modification" + ), + ), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("validation_json", models.JSONField()), + ( + "content_type", + models.OneToOneField( + help_text="Relation to the chosen type: Action (Toimenpide) / Function (Käsittelyprosessi) / Phase (Käsittelyvaihe) / Record (Asiakirja). Relation to 'Attribute validation rule' is applied for each of the types.", + limit_choices_to=models.Q( + ("model", "action"), + ("model", "function"), + ("model", "phase"), + ("model", "record"), + ("model", "attributevalidationrule"), + _connector="OR", + ), + on_delete=django.db.models.deletion.CASCADE, + to="contenttypes.contenttype", + verbose_name="content type", + ), + ), + ], + options={ + "verbose_name": "attribute validation rule", + "verbose_name_plural": "attribute validation rules", + "ordering": ("content_type",), + }, + ), + ] diff --git a/metarecord/migrations/0046_classification_versioning.py b/metarecord/migrations/0046_classification_versioning.py new file mode 100644 index 00000000..836461fa --- /dev/null +++ b/metarecord/migrations/0046_classification_versioning.py @@ -0,0 +1,99 @@ +# Generated by Django 2.2.16 on 2020-09-23 11:46 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('metarecord', '0045_populate_persistent_user_name_fields'), + ] + + operations = [ + migrations.AlterModelOptions( + name='classification', + options={ + 'permissions': ( + ('can_edit_classification', 'Can edit classification'), + ('can_review_classification', 'Can review classification'), + ('can_approve_classification', 'Can approve classification'), + ('can_view_classification_modified_by', 'Can view classification modified by') + ), + 'verbose_name': 'classification', + 'verbose_name_plural': 'classifications', + }, + ), + migrations.AddField( + model_name='classification', + name='_created_by', + field=models.CharField(blank=True, editable=False, max_length=200, verbose_name='created by (text)'), + ), + migrations.AddField( + model_name='classification', + name='_modified_by', + field=models.CharField(blank=True, editable=False, max_length=200, verbose_name='modified by (text)'), + ), + migrations.AddField( + model_name='classification', + name='created_by', + field=models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='classification_created', + to=settings.AUTH_USER_MODEL, + verbose_name='created by' + ), + ), + migrations.AddField( + model_name='classification', + name='modified_by', + field=models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='classification_modified', + to=settings.AUTH_USER_MODEL, + verbose_name='modified by' + ), + ), + migrations.AddField( + model_name='classification', + name='state', + field=models.CharField( + choices=[ + ('draft', 'Draft'), + ('sent_for_review', 'Sent for review'), + ('waiting_for_approval', 'Waiting for approval'), + ('approved', 'Approved') + ], + default='draft', + max_length=20, + verbose_name='state' + ), + ), + migrations.AddField( + model_name='classification', + name='valid_from', + field=models.DateField(blank=True, null=True, verbose_name='valid from'), + ), + migrations.AddField( + model_name='classification', + name='valid_to', + field=models.DateField(blank=True, null=True, verbose_name='valid to'), + ), + migrations.AddField( + model_name='classification', + name='version', + field=models.PositiveIntegerField(blank=True, db_index=True, default=1, null=True), + ), + migrations.AlterUniqueTogether( + name='classification', + unique_together={('uuid', 'version')}, + ), + ] diff --git a/metarecord/migrations/0047_approve_existing_classifications.py b/metarecord/migrations/0047_approve_existing_classifications.py new file mode 100644 index 00000000..cc955621 --- /dev/null +++ b/metarecord/migrations/0047_approve_existing_classifications.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.16 on 2020-10-02 11:27 + +from django.db import migrations + + +def set_existing_classifications_as_approved(apps, schema_editor): + Classification = apps.get_model('metarecord', 'Classification') + Classification.objects.all().update(state='approved') + + +class Migration(migrations.Migration): + + dependencies = [ + ('metarecord', '0046_classification_versioning'), + ] + + operations = [ + migrations.RunPython(set_existing_classifications_as_approved, migrations.RunPython.noop) + ] diff --git a/metarecord/migrations/0048_replace_pg_jsonfield_with_django_jsonfield.py b/metarecord/migrations/0048_replace_pg_jsonfield_with_django_jsonfield.py new file mode 100644 index 00000000..ebc5ee92 --- /dev/null +++ b/metarecord/migrations/0048_replace_pg_jsonfield_with_django_jsonfield.py @@ -0,0 +1,38 @@ +# Generated by Django 3.1.5 on 2021-01-11 09:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("metarecord", "0047_approve_existing_classifications"), + ] + + operations = [ + migrations.AlterField( + model_name="action", + name="attributes", + field=models.JSONField(blank=True, default=dict, verbose_name="attributes"), + ), + migrations.AlterField( + model_name="bulkupdate", + name="changes", + field=models.JSONField(blank=True, default=dict, verbose_name="changes"), + ), + migrations.AlterField( + model_name="function", + name="attributes", + field=models.JSONField(blank=True, default=dict, verbose_name="attributes"), + ), + migrations.AlterField( + model_name="phase", + name="attributes", + field=models.JSONField(blank=True, default=dict, verbose_name="attributes"), + ), + migrations.AlterField( + model_name="record", + name="attributes", + field=models.JSONField(blank=True, default=dict, verbose_name="attributes"), + ), + ] diff --git a/metarecord/models/__init__.py b/metarecord/models/__init__.py index 2edb8fba..4f753d44 100644 --- a/metarecord/models/__init__.py +++ b/metarecord/models/__init__.py @@ -1,5 +1,6 @@ from .action import Action # noqa from .attribute import Attribute, AttributeGroup, AttributeValue # noqa +from .attribute_validation import AttributeValidationRule # noqa from .classification import Classification # noqa from .function import Function, MetadataVersion # noqa from .phase import Phase # noqa diff --git a/metarecord/models/action.py b/metarecord/models/action.py index 04106619..aa690c96 100644 --- a/metarecord/models/action.py +++ b/metarecord/models/action.py @@ -1,5 +1,5 @@ from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from .phase import Phase from .structural_element import StructuralElement diff --git a/metarecord/models/attribute.py b/metarecord/models/attribute.py index 6e252455..cb45e0bf 100644 --- a/metarecord/models/attribute.py +++ b/metarecord/models/attribute.py @@ -1,7 +1,7 @@ import logging from django.db import models, transaction -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from .base import TimeStampedModel, UUIDPrimaryKeyModel from .predefined_attributes import PREDEFINED_ATTRIBUTES diff --git a/metarecord/models/attribute_validation.py b/metarecord/models/attribute_validation.py new file mode 100644 index 00000000..9318b8b3 --- /dev/null +++ b/metarecord/models/attribute_validation.py @@ -0,0 +1,33 @@ +from django.contrib.contenttypes.models import ContentType +from django.db import models +from django.db.models import Q +from django.utils.translation import gettext_lazy as _ + +from metarecord.models.base import TimeStampedModel, UUIDPrimaryKeyModel + + +class AttributeValidationRule(TimeStampedModel, UUIDPrimaryKeyModel): + content_type = models.OneToOneField( + ContentType, + verbose_name=_("content type"), + on_delete=models.CASCADE, + limit_choices_to=Q(model="action") + | Q(model="function") + | Q(model="phase") + | Q(model="record") + | Q(model="attributevalidationrule"), + help_text=_( + "Relation to the chosen type: Action (Toimenpide) / Function (Käsittelyprosessi) / " + "Phase (Käsittelyvaihe) / Record (Asiakirja). " + "Relation to 'Attribute validation rule' is applied for each of the types." + ), + ) + validation_json = models.JSONField() + + class Meta: + verbose_name = _("attribute validation rule") + verbose_name_plural = _("attribute validation rules") + ordering = ("content_type",) + + def __str__(self): + return f"{self.content_type}" diff --git a/metarecord/models/base.py b/metarecord/models/base.py index ae839545..eceb67ad 100644 --- a/metarecord/models/base.py +++ b/metarecord/models/base.py @@ -1,7 +1,7 @@ import uuid from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class TimeStampedModel(models.Model): diff --git a/metarecord/models/bulk_update.py b/metarecord/models/bulk_update.py index ee16cee2..81d8e5f3 100644 --- a/metarecord/models/bulk_update.py +++ b/metarecord/models/bulk_update.py @@ -1,10 +1,9 @@ from copy import deepcopy from django.conf import settings -from django.contrib.postgres.fields import JSONField from django.core.exceptions import PermissionDenied from django.db import models, transaction -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from ..utils import create_new_function_version, update_nested_dictionary from .base import TimeStampedModel, UUIDPrimaryKeyModel @@ -51,7 +50,7 @@ class BulkUpdate(TimeStampedModel, UUIDPrimaryKeyModel): _approved_by = models.CharField(verbose_name=_('approved by (text)'), max_length=200, blank=True, editable=False) is_approved = models.BooleanField(verbose_name=_('is approved'), default=False) - changes = JSONField(verbose_name=_('changes'), blank=True, default=dict) + changes = models.JSONField(verbose_name=_('changes'), blank=True, default=dict) state = models.CharField( verbose_name=_('state'), max_length=20, diff --git a/metarecord/models/classification.py b/metarecord/models/classification.py index e0a1ccda..57991dfd 100644 --- a/metarecord/models/classification.py +++ b/metarecord/models/classification.py @@ -1,14 +1,79 @@ import uuid from collections import Iterable -from django.db import models, transaction -from django.utils.translation import ugettext_lazy as _ +from django.contrib.auth import get_user_model +from django.db import connection, models, transaction +from django.utils.translation import gettext_lazy as _ from .base import TimeStampedModel +class ClassificationQuerySet(models.QuerySet): + def latest_version(self): + return self.order_by('code', '-version').distinct('code') + + def latest_approved(self): + return self.filter(state=Classification.APPROVED).latest_version() + + def filter_for_user(self, user): + if not user.is_authenticated: + return self.filter(state=Classification.APPROVED) + return self + + def previous_versions(self, classification): + return self.filter( + version__lt=classification.version, + uuid=classification.uuid + ) + + def non_approved(self): + return self.exclude(state=Classification.APPROVED) + + class Classification(TimeStampedModel): + DRAFT = 'draft' + SENT_FOR_REVIEW = 'sent_for_review' + WAITING_FOR_APPROVAL = 'waiting_for_approval' + APPROVED = 'approved' + + STATE_CHOICES = ( + (DRAFT, _('Draft')), + (SENT_FOR_REVIEW, _('Sent for review')), + (WAITING_FOR_APPROVAL, _('Waiting for approval')), + (APPROVED, _('Approved')), + ) + + CAN_EDIT = 'metarecord.can_edit_classification' + CAN_REVIEW = 'metarecord.can_review_classification' + CAN_APPROVE = 'metarecord.can_approve_classification' + CAN_VIEW_MODIFIED_BY = 'metarecord.can_view_classification_modified_by' + uuid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True) + version = models.PositiveIntegerField(db_index=True, default=1, null=True, blank=True) + state = models.CharField(verbose_name=_('state'), max_length=20, choices=STATE_CHOICES, default=DRAFT) + valid_from = models.DateField(verbose_name=_('valid from'), null=True, blank=True) + valid_to = models.DateField(verbose_name=_('valid to'), null=True, blank=True) + created_by = models.ForeignKey( + get_user_model(), + verbose_name=_('created by'), + null=True, + blank=True, + related_name='%(class)s_created', + editable=False, + on_delete=models.SET_NULL + ) + modified_by = models.ForeignKey( + get_user_model(), + verbose_name=_('modified by'), + null=True, + blank=True, + related_name='%(class)s_modified', + editable=False, + on_delete=models.SET_NULL + ) + _created_by = models.CharField(verbose_name=_('created by (text)'), max_length=200, blank=True, editable=False) + _modified_by = models.CharField(verbose_name=_('modified by (text)'), max_length=200, blank=True, editable=False) + parent = models.ForeignKey( 'self', verbose_name=_('parent'), related_name='children', blank=True, null=True, on_delete=models.SET_NULL ) @@ -20,13 +85,58 @@ class Classification(TimeStampedModel): additional_information = models.TextField(verbose_name=_('additional information'), blank=True) function_allowed = models.BooleanField(verbose_name=_('function allowed'), default=False) + objects = ClassificationQuerySet.as_manager() + class Meta: verbose_name = _('classification') verbose_name_plural = _('classifications') + unique_together = (('uuid', 'version'),) + permissions = ( + ('can_edit_classification', _('Can edit classification')), + ('can_review_classification', _('Can review classification')), + ('can_approve_classification', _('Can approve classification')), + ('can_view_classification_modified_by', _('Can view classification modified by')), + ) def __str__(self): return self.code + @transaction.atomic + def save(self, *args, **kwargs): + if not self.id: + with connection.cursor() as cursor: + cursor.execute('LOCK TABLE %s' % self._meta.db_table) + + try: + latest = Classification.objects.latest_version().get(code=self.code) + self.version = latest.version + 1 + self.uuid = latest.uuid + except Classification.DoesNotExist: + self.version = 1 + + # Only update `_created_by` and `_modified_by` value if the relations + # are set set. Text values should persist even if related user is deleted. + if self.created_by: + self._created_by = self.created_by.get_full_name() + + if self.modified_by: + self._modified_by = self.modified_by.get_full_name() + + super().save(*args, **kwargs) + + if self.state == Classification.APPROVED: + # Delete old non-approved versions leading to current version if newly saved version is approved. + self.delete_old_non_approved_versions() + + def delete_old_non_approved_versions(self): + if self.state != Classification.APPROVED: + raise Exception('Function must be approved before old non-approved versions can be deleted.') + + Classification.objects.previous_versions(self).non_approved().delete() + + def get_modified_by_display(self): + return self._modified_by or None + @transaction.atomic def update_function_allowed(classifications): diff --git a/metarecord/models/function.py b/metarecord/models/function.py index a90c0959..2c7b2e63 100644 --- a/metarecord/models/function.py +++ b/metarecord/models/function.py @@ -1,6 +1,6 @@ from django.conf import settings from django.db import connection, models, transaction -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from .classification import Classification from .structural_element import StructuralElement @@ -21,7 +21,7 @@ def filter_for_user(self, user): def previous_versions(self, function): return self.filter( version__lt=function.version, - classification=function.classification + classification__uuid=function.classification.uuid ) def non_approved(self): @@ -142,13 +142,15 @@ def save(self, *args, **kwargs): if not self.classification: raise Exception('Classification is required.') - if not self.id: + if self.state == Function.APPROVED and self.classification.state != Classification.APPROVED: + raise Exception('Approved function must have approved classification') + if not self.id: # lock Function table to prevent possible race condition when adding the new latest version number with connection.cursor() as cursor: cursor.execute('LOCK TABLE %s' % self._meta.db_table) try: - latest = Function.objects.latest_version().get(classification=self.classification) + latest = Function.objects.latest_version().get(classification__uuid=self.classification.uuid) self.version = latest.version + 1 self.uuid = latest.uuid except Function.DoesNotExist: diff --git a/metarecord/models/phase.py b/metarecord/models/phase.py index 2d73143d..a0a0476f 100644 --- a/metarecord/models/phase.py +++ b/metarecord/models/phase.py @@ -1,5 +1,5 @@ from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from .function import Function from .structural_element import StructuralElement diff --git a/metarecord/models/record.py b/metarecord/models/record.py index 1961c107..2c1ae4b8 100644 --- a/metarecord/models/record.py +++ b/metarecord/models/record.py @@ -1,5 +1,5 @@ from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from .action import Action from .structural_element import StructuralElement diff --git a/metarecord/models/structural_element.py b/metarecord/models/structural_element.py index a2c2e285..3c2287f0 100644 --- a/metarecord/models/structural_element.py +++ b/metarecord/models/structural_element.py @@ -3,11 +3,12 @@ from copy import deepcopy from django.conf import settings -from django.contrib.postgres.fields import JSONField +from django.contrib.contenttypes.models import ContentType from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from .attribute import Attribute +from .attribute_validation import AttributeValidationRule from .base import TimeStampedModel @@ -22,7 +23,7 @@ class StructuralElement(TimeStampedModel): _created_by = models.CharField(verbose_name=_('created by (text)'), max_length=200, blank=True, editable=False) _modified_by = models.CharField(verbose_name=_('modified by (text)'), max_length=200, blank=True, editable=False) index = models.PositiveSmallIntegerField(null=True, editable=False, db_index=True) - attributes = JSONField(verbose_name=_('attributes'), blank=True, default=dict) + attributes = models.JSONField(verbose_name=_('attributes'), blank=True, default=dict) _attribute_validations = { 'allowed': None, @@ -38,41 +39,90 @@ class Meta: abstract = True ordering = ('index',) + @classmethod + def get_attribute_validations(cls): + """ + Returns the attribute validation rules with the priority: + 1. Type specific attribute validation rules, combined with the general attribute validation rules. + 2. General attribute validation rules if the type specific do not exist. + 3. Hard coded `_attribute_validations` if neither type specific or general attribute validation rules exist. + """ + content_type = ContentType.objects.get_for_model(cls) + general_content_type = ContentType.objects.get_for_model( + AttributeValidationRule + ) + + attr_validation_rule = AttributeValidationRule.objects.filter( + content_type=content_type + ).first() + general_attr_validation_rule = AttributeValidationRule.objects.filter( + content_type=general_content_type + ).first() + + if attr_validation_rule and general_attr_validation_rule: + attr_validations = attr_validation_rule.validation_json + general_attr_validations = general_attr_validation_rule.validation_json + + # Combine type specific attribute validation rules with the general validation rules. + for attr_key in StructuralElement._attribute_validations.keys(): + attr_validations[attr_key] += general_attr_validations[attr_key] + + elif attr_validation_rule: + attr_validations = attr_validation_rule.validation_json + elif general_attr_validation_rule: + attr_validations = general_attr_validation_rule.validation_json + else: + return cls._attribute_validations + + refactored_validations = deepcopy(attr_validations) + + cond_type = "conditionally_required" + refactor_conditional_validation( + attr_validations, refactored_validations, cond_type + ) + + cond_type = "conditionally_disallowed" + refactor_conditional_validation( + attr_validations, refactored_validations, cond_type + ) + + return refactored_validations + @classmethod def get_attribute_json_schema(cls): - return get_attribute_json_schema(**cls._attribute_validations) + return get_attribute_json_schema(**cls.get_attribute_validations()) @classmethod def get_required_attributes(cls): - return set(cls._attribute_validations.get('required') or []) + return set(cls.get_attribute_validations().get('required') or []) @classmethod def get_multivalued_attributes(cls): - return set(cls._attribute_validations.get('multivalued') or []) + return set(cls.get_attribute_validations().get('multivalued') or []) @classmethod def get_conditionally_required_attributes(cls): - return deepcopy(cls._attribute_validations.get('conditionally_required')) or {} + return deepcopy(cls.get_attribute_validations().get('conditionally_required')) or {} @classmethod def get_conditionally_disallowed_attributes(cls): - return deepcopy(cls._attribute_validations.get('conditionally_disallowed')) or {} + return deepcopy(cls.get_attribute_validations().get('conditionally_disallowed')) or {} @classmethod def get_all_or_none_attributes(cls): return [ set(validation) - for validation in cls._attribute_validations.get('all_or_none') or [] + for validation in cls.get_attribute_validations().get('all_or_none') or [] if validation ] @classmethod def get_allow_values_outside_choices_attributes(cls): - return set(cls._attribute_validations.get('allow_values_outside_choices') or []) + return set(cls.get_attribute_validations().get('allow_values_outside_choices') or []) @classmethod def is_attribute_allowed(cls, attribute_identifier): - allowed = cls._attribute_validations.get('allowed') + allowed = cls.get_attribute_validations().get('allowed') if allowed is None: # None means the validation isn't enabled return True @@ -231,3 +281,53 @@ def get_attribute_json_schema(**kwargs): schema['extra_validations'] = extra_validations return schema + + +def refactor_conditional_validation( + attr_validations, refactored_validations, cond_type +): + """ + Refactor the conditional (conditionally_required, conditionally_disallowed) validation rules from the JSON schema + of the Django admin to the schema supported by the rest of the application. This needs to be done because the + django-admin-json-editor schema does not support the existing validation schema. + + Example: + + 'conditionally_required': [ + { + 'attribute': 'SecurityPeriod', + 'conditions': [ + { + 'attribute': 'PublicityClass', + 'value': 'Salassa pidettävä', + }, + { + 'attribute': 'PublicityClass', + 'value': 'Osittain salassa pidettävä', + }, + ] + }, + ] + + -> + + 'conditionally_required': { + 'SecurityPeriod': {'PublicityClass': ['Salassa pidettävä', 'Osittain salassa pidettävä']}, + }, + """ + refactored_validations[cond_type] = {} + for conditional_validation_obj in attr_validations.get(cond_type, []): + attribute = conditional_validation_obj["attribute"] + + if attribute not in refactored_validations[cond_type]: + refactored_validations[cond_type][attribute] = {} + + conditions = conditional_validation_obj["conditions"] + for condition_obj in conditions: + cond_attribute = condition_obj["attribute"] + cond_value = condition_obj["value"] + + if cond_attribute not in refactored_validations[cond_type][attribute]: + refactored_validations[cond_type][attribute][cond_attribute] = [] + + refactored_validations[cond_type][attribute][cond_attribute].append(cond_value) diff --git a/metarecord/pagination.py b/metarecord/pagination.py index 9fe54049..64216cc6 100644 --- a/metarecord/pagination.py +++ b/metarecord/pagination.py @@ -1,3 +1,4 @@ +from django_elasticsearch_dsl_drf.pagination import PageNumberPagination as ESPageNumberPagination from rest_framework.pagination import PageNumberPagination @@ -5,3 +6,7 @@ class MetaRecordPagination(PageNumberPagination): page_size = 20 page_size_query_param = 'page_size' max_page_size = 10000 + + +class ESRecordPagination(ESPageNumberPagination, MetaRecordPagination): + pass diff --git a/metarecord/templates/admin/metarecord/classification/change_form.html b/metarecord/templates/admin/metarecord/classification/change_form.html new file mode 100644 index 00000000..5c8aaf65 --- /dev/null +++ b/metarecord/templates/admin/metarecord/classification/change_form.html @@ -0,0 +1,12 @@ +{% extends "admin/change_form.html" %} +{% load i18n %} + +{% block object-tools %} + {{ block.super }} +
+
+ {% csrf_token %} + +
+
+{% endblock %} diff --git a/metarecord/tests/conftest.py b/metarecord/tests/conftest.py index 336d64d9..bff1e8a5 100644 --- a/metarecord/tests/conftest.py +++ b/metarecord/tests/conftest.py @@ -1,3 +1,4 @@ +import os import uuid import pytest @@ -16,12 +17,22 @@ def parent_classification(): @pytest.fixture def classification(): - return Classification.objects.create(title='test classification', code='00 00', function_allowed=True) + return Classification.objects.create( + title='test classification', + code='00 00', + state=Classification.APPROVED, + function_allowed=True, + ) @pytest.fixture def classification_2(): - return Classification.objects.create(title='test classification 2', code='00 01', function_allowed=True) + return Classification.objects.create( + title='test classification 2', + code='00 01', + state=Classification.APPROVED, + function_allowed=True + ) @pytest.fixture @@ -176,3 +187,11 @@ def attribute_group(choice_attribute): choice_attribute.group = group choice_attribute.save(update_fields=('group',)) return group + + +@pytest.fixture +def tos_importer_excel_file_path(): + excel_filename = "00_00_01_02.xlsx" + current_file = os.path.realpath(__file__) + current_directory = os.path.dirname(current_file) + return os.path.join(current_directory, "test_data", excel_filename) diff --git a/metarecord/tests/test_0045_migration.py b/metarecord/tests/test_0045_migration.py index 575b733a..3c6cfd73 100644 --- a/metarecord/tests/test_0045_migration.py +++ b/metarecord/tests/test_0045_migration.py @@ -19,8 +19,11 @@ def test_function_migration(function, user, user_2): assert function._created_by == '' assert function._modified_by == '' - call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) - call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + try: + call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) + call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + except KeyError: + pytest.skip() function.refresh_from_db() assert function._created_by == 'John Rambo' @@ -34,8 +37,11 @@ def test_phase_migration(phase, user, user_2): assert phase._created_by == '' assert phase._modified_by == '' - call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) - call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + try: + call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) + call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + except KeyError: + pytest.skip() phase.refresh_from_db() assert phase._created_by == 'John Rambo' @@ -49,8 +55,11 @@ def test_action_migration(action, user, user_2): assert action._created_by == '' assert action._modified_by == '' - call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) - call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + try: + call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) + call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + except KeyError: + pytest.skip() action.refresh_from_db() assert action._created_by == 'John Rambo' @@ -64,8 +73,11 @@ def test_record_migration(record, user, user_2): assert record._created_by == '' assert record._modified_by == '' - call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) - call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + try: + call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) + call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + except KeyError: + pytest.skip() record.refresh_from_db() assert record._created_by == 'John Rambo' @@ -84,8 +96,11 @@ def test_bulk_update_migration(bulk_update, user, user_2, super_user): assert bulk_update._created_by == '' assert bulk_update._modified_by == '' - call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) - call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + try: + call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) + call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + except KeyError: + pytest.skip() bulk_update.refresh_from_db() assert bulk_update._approved_by == 'Kurt Sloane' @@ -102,8 +117,11 @@ def test_metadata_version_migration(function, user): metadata_version = function.metadata_versions.first() assert metadata_version._modified_by == '' - call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) - call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + try: + call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) + call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + except KeyError: + pytest.skip() metadata_version.refresh_from_db() assert metadata_version._modified_by == 'John Rambo' @@ -114,8 +132,11 @@ def test_migration_without_user(function): assert function.created_by == None assert function._created_by == '' - call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) - call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + try: + call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) + call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + except KeyError: + pytest.skip() function.refresh_from_db() assert function._created_by == '' @@ -130,8 +151,11 @@ def test_migration_without_first_name(function, user, user_2): assert function._created_by == '' assert function.created_by.get_full_name() == 'Rambo' - call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) - call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + try: + call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) + call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + except KeyError: + pytest.skip() function.refresh_from_db() assert function._created_by == 'Rambo' @@ -146,8 +170,11 @@ def test_migration_without_last_name(function, user, user_2): assert function._created_by == '' assert function.created_by.get_full_name() == 'John' - call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) - call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + try: + call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) + call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + except KeyError: + pytest.skip() function.refresh_from_db() assert function._created_by == 'John' @@ -163,8 +190,11 @@ def test_migration_wouthout_names(function, user, user_2): assert function._created_by == '' assert function.created_by.get_full_name() == '' - call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) - call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + try: + call_command('migrate', app_label='metarecord', migration_name='0044', skip_checks=True, fake=True) + call_command('migrate', app_label='metarecord', migration_name='0045', skip_checks=True) + except KeyError: + pytest.skip() function.refresh_from_db() assert function._created_by == '' diff --git a/metarecord/tests/test_api.py b/metarecord/tests/test_api.py index 7ea543af..1c27a7df 100644 --- a/metarecord/tests/test_api.py +++ b/metarecord/tests/test_api.py @@ -6,12 +6,14 @@ import freezegun import pytest import pytz +from django.utils.translation import gettext_lazy as _ from rest_framework.reverse import reverse -from metarecord.models import Action, Attribute, Classification, Function, Phase, Record +from metarecord.models import Action, Classification, Function, Phase, Record from metarecord.models.bulk_update import BulkUpdate from metarecord.tests.utils import ( - assert_response_functions, check_attribute_errors, get_bulk_update_function_key, set_permissions + assert_response_functions, check_attribute_errors, FunctionTestDetailSerializer, get_bulk_update_function_key, + set_permissions ) from metarecord.views.classification import include_related @@ -42,10 +44,17 @@ def get_bulk_update_approve_url(bulk_update): return reverse('bulkupdate-approve', kwargs={'pk': bulk_update.pk}) +def get_record_detail_url(record): + return reverse('record-detail', kwargs={'uuid': record.uuid}) + + @pytest.fixture def post_function_data(classification, free_text_attribute, choice_attribute): return { - 'classification': str(classification.uuid), + 'classification': { + 'id': classification.uuid.hex, + 'version': classification.version, + }, 'attributes': { free_text_attribute.identifier: 'new function attribute value', }, @@ -70,11 +79,15 @@ def post_function_data(classification, free_text_attribute, choice_attribute): @pytest.fixture -def put_function_data(function, free_text_attribute, choice_attribute): +def put_function_data(function, free_text_attribute, choice_attribute, bulk_update): return { 'name': 'new function version', 'function_id': function.classification.code, 'parent': 'xyz', + 'classification': { + 'id': function.classification.uuid.hex, + 'version': function.classification.version, + }, 'attributes': { free_text_attribute.identifier: 'new function version attribute value', }, @@ -92,7 +105,8 @@ def put_function_data(function, free_text_attribute, choice_attribute): } ] } - ] + ], + 'bulk_update': bulk_update.id, } @@ -247,10 +261,37 @@ def test_function_post(post_function_data, user_api_client): assert response.data['function_id'] == response.data['classification_code'] == new_function.classification.code +@pytest.mark.django_db +def test_function_post_with_multiple_available_classification_versions( + post_function_data, classification, user_api_client +): + """ + Test that creating new function will set classification relation to the correct version + specified in request data + """ + set_permissions(user_api_client, Function.CAN_EDIT) + classification_v2 = Classification.objects.create( + uuid=classification.uuid, + title='test classification v2', + code=classification.code, + function_allowed=classification.function_allowed, + ) + + post_function_data['classification']['version'] = classification_v2.version + + response = user_api_client.post(FUNCTION_LIST_URL, data=post_function_data) + + assert response.status_code == 201 + new_function = Function.objects.last() + assert new_function.classification == classification_v2 + + @pytest.mark.django_db def test_function_post_empty_function(user_api_client, classification): set_permissions(user_api_client, Function.CAN_EDIT) - response = user_api_client.post(FUNCTION_LIST_URL, data={'classification': str(classification.uuid)}) + response = user_api_client.post(FUNCTION_LIST_URL, data={ + 'classification': {'id': classification.uuid.hex, 'version': classification.version }, + }) assert response.status_code == 201 new_function = Function.objects.last() @@ -275,7 +316,7 @@ def test_cannot_post_more_than_one_function_for_classification(post_function_dat response = user_api_client.post(FUNCTION_LIST_URL, data=post_function_data) assert response.status_code == 400 - assert 'Classification %s already has a function.' % post_function_data['classification'] in str(response.data) + assert 'Classification %s already has a function.' % post_function_data['classification']['id'] in str(response.data) @pytest.mark.django_db @@ -300,6 +341,35 @@ def test_function_put(put_function_data, user_api_client, function, phase, actio assert obj.modified_at == modified_ats[index] +@pytest.mark.django_db +def test_function_put_with_new_classification_version(put_function_data, user_api_client, classification, function): + set_permissions(user_api_client, Function.CAN_EDIT) + classification_v2 = Classification.objects.create( + uuid=classification.uuid, + title='test classification v2', + code=classification.code, + function_allowed=classification.function_allowed, + ) + # Sanity checks before writing anything + assert function.classification.title == classification.title + assert function.classification.version == classification.version + assert classification_v2.version == 2 + put_function_data['classification'] = { + 'id': classification_v2.uuid.hex, + 'version': classification_v2.version, + } + + response = user_api_client.put(get_function_detail_url(function), data=put_function_data) + assert response.status_code == 200 + + new_function = Function.objects.latest_version().get(uuid=function.uuid) + _check_function_object_matches_data(new_function, put_function_data) + assert Function.objects.count() == 2 + assert new_function.uuid == function.uuid + assert new_function.classification.title == classification_v2.title + assert new_function.classification.version == classification_v2.version + + @pytest.mark.django_db def test_function_post_invalid_attributes(post_function_data, user_api_client): post_function_data['attributes'] = {'InvalidFunctionAttribute': 'value'} @@ -327,12 +397,19 @@ def test_function_put_invalid_attributes(put_function_data, user_api_client, fun def test_function_put_not_able_to_change_classification(put_function_data, user_api_client, function, classification, classification_2): set_permissions(user_api_client, Function.CAN_EDIT) - put_function_data['classification'] = str(classification_2.uuid) + put_function_data['classification'] = { + 'id': classification_2.uuid.hex, + 'version': classification_2.version, + } response = user_api_client.put(get_function_detail_url(function), data=put_function_data) - assert response.status_code == 200 - new_function = Function.objects.last() - assert new_function.classification == classification + + assert response.status_code == 400 + latest_version = Function.objects.filter(uuid=function.uuid).latest_version().first() + assert latest_version.version == function.version + assert response.json() == { + 'non_field_errors': ['Changing classification is not allowed. Only version can be changed.'] + } @pytest.mark.django_db @@ -1389,6 +1466,34 @@ def test_function_version_history_field(user_api_client, classification, user_2) assert last_version['modified_by'] == 'Rocky Balboa' +@pytest.mark.django_db +def test_classification_version_history(user_api_client, classification): + + classification.pk = None + classification.state = Classification.SENT_FOR_REVIEW + classification.valid_from = datetime.datetime.now() + classification.valid_to = datetime.datetime.now() + classification.save() + + response = user_api_client.get(get_classification_detail_url(classification)) + assert response.status_code == 200 + version_history = response.data['version_history'] + + assert len(version_history) == 2 + + first_version = version_history[0] + assert first_version.get('modified_at') + assert first_version['state'] == Classification.APPROVED + assert first_version['version'] == 1 + assert 'modified_by' not in version_history[0] + + last_version = version_history[1] + assert last_version['state'] == Classification.SENT_FOR_REVIEW + assert type(last_version['valid_to']) == datetime.date + assert type(last_version['valid_from']) == datetime.date + assert last_version['version'] == 2 + + @pytest.mark.django_db def test_classification_function_state_field(user_api_client, classification, classification_2): Function.objects.create(classification=classification, state=Function.DRAFT) @@ -1764,16 +1869,79 @@ def test_function_delete_on_approve(user_api_client, classification): @pytest.mark.django_db def test_function_post_when_not_allowed(post_function_data, user_api_client): set_permissions(user_api_client, Function.CAN_EDIT) - parent_classification = Classification.objects.get(uuid=post_function_data['classification']) + parent_classification = Classification.objects.get( + uuid=post_function_data['classification']['id'], + version=post_function_data['classification']['version'], + ) parent_classification.function_allowed = False parent_classification.save(update_fields=('function_allowed',)) response = user_api_client.post(FUNCTION_LIST_URL, data=post_function_data) assert response.status_code == 400 - expected_error = 'Classification %s does not allow function creation.' % parent_classification.uuid + expected_error = 'Classification %s does not allow function creation.' % parent_classification.uuid.hex assert expected_error in response.data['non_field_errors'] +@pytest.mark.django_db +def test_function_post_new_when_existing_function(rf, function, phase, action, record, user_api_client, super_user_api_client): + set_permissions(user_api_client, Function.CAN_EDIT) + classification = function.classification + classification.pk = None + classification.save() + + function.refresh_from_db() + + function_key = get_bulk_update_function_key(function) + post_data = { + 'description': 'Bulk update description', + 'state': Function.APPROVED, + 'changes': { + function_key: { + 'attributes': {'TypeSpecifier': 'bulk updated test thing'}, + } + }, + } + + with freezegun.freeze_time('2020-01-01 12:00'): + response = super_user_api_client.post(BULK_UPDATE_LIST_URL, post_data) + + function.bulk_update = BulkUpdate.objects.first() + function.save() + + new_classification = Classification.objects.latest_version().get(code=function.classification.code) + post_data = {'classification': { + 'id': new_classification.uuid.hex, + 'version': new_classification.version, + }} + + response = user_api_client.post(FUNCTION_LIST_URL, data=post_data) + + class view(): + action = "create" + + dummy_request = rf.get(get_function_detail_url(function)) + dummy_request.user = user_api_client.user + dummy_context = {"view": view(), "request": dummy_request} + function_data = FunctionTestDetailSerializer(function, context=dummy_context).data + response_data_json = response.json() + for attr in ['modified_at', 'created_at', 'actions', 'id']: + response_data_json['phases'][0].pop(attr) + function_data['phases'][0].pop(attr) + assert response_data_json['phases'][0] == function_data['phases'][0] + response_data_json.pop('phases') + function_data.pop('phases') + + new_phase = Function.objects.filter(classification=new_classification).latest_version().first().phases.first() + assert (new_phase.attributes == phase.attributes) + assert new_phase.actions.first().attributes == phase.actions.first().attributes + assert new_phase.actions.first().records.first().attributes == phase.actions.first().records.first().attributes + assert response_data_json["version"] != function_data["version"] + assert response_data_json["classification"]["version"] != function_data["classification"]["version"] + for attr in ["created_at", "modified_at", "version", "classification", "bulk_update"]: + function_data.pop(attr) + response_data_json.pop(attr) + assert response_data_json == function_data + @pytest.mark.parametrize('authenticated', (False, True)) @pytest.mark.django_db def test_classification_fields_visibility(api_client, user_api_client, classification, authenticated): @@ -1790,6 +1958,225 @@ def test_classification_fields_visibility(api_client, user_api_client, classific assert 'additional_information' not in response.data +@pytest.mark.parametrize('has_permission', (False, True)) +@pytest.mark.django_db +def test_classification_create_requires_permission(user_api_client, user_2_api_client, has_permission): + set_permissions(user_api_client, Classification.CAN_EDIT) + client = user_api_client if has_permission else user_2_api_client + data = { + 'code': '05', + 'title': 'test classification created through the API' + } + + response = client.post(CLASSIFICATION_LIST_URL, data=data) + + if has_permission: + assert response.status_code == 201 + assert Classification.objects.count() == 1 + classification = Classification.objects.first() + assert classification.code == data['code'] + assert classification.title == data['title'] + assert classification.version == 1 + assert classification.created_by == client.user + assert classification.modified_by == client.user + else: + assert response.status_code == 403 + assert Classification.objects.count() == 0 + + +@pytest.mark.parametrize('has_permission', (False, True)) +@pytest.mark.django_db +def test_classification_update_requires_permission(user_api_client, user_2_api_client, classification, has_permission): + set_permissions(user_api_client, Classification.CAN_EDIT) + client = user_api_client if has_permission else user_2_api_client + classification.state = Classification.DRAFT + classification.save(update_fields=['state']) + data = { + 'state': Classification.SENT_FOR_REVIEW, + } + + response = client.patch(get_classification_detail_url(classification), data=data) + + if has_permission: + assert response.status_code == 200 + assert Classification.objects.count() == 1 + classification.refresh_from_db() + assert classification.state == Classification.SENT_FOR_REVIEW + assert classification.modified_by == client.user + else: + assert response.status_code == 403 + assert Classification.objects.count() == 1 + classification.refresh_from_db() + assert classification.state == Classification.DRAFT + + +@pytest.mark.django_db +def test_classification_patch_does_not_create_new_version(user_api_client, classification): + set_permissions(user_api_client, Classification.CAN_EDIT) + classification.state = Classification.DRAFT + classification.save(update_fields=['state']) + data = { + 'state': Classification.SENT_FOR_REVIEW, + } + + assert classification.version == 1 + response = user_api_client.patch(get_classification_detail_url(classification), data=data) + + assert response.status_code == 200 + assert Classification.objects.count() == 1 + classification.refresh_from_db() + assert classification.state == Classification.SENT_FOR_REVIEW + assert classification.version == 1 + + +@pytest.mark.django_db +def test_classification_put_creates_new_version(user_api_client, classification): + set_permissions(user_api_client, Classification.CAN_EDIT) + data = { + 'code': classification.code, + 'title': 'Updated classification title', + 'description': 'Updated classification description', + } + + response = user_api_client.put(get_classification_detail_url(classification), data=data) + + assert response.status_code == 200 + assert response.json()['version'] == 2 + assert Classification.objects.count() == 2 + new_version = Classification.objects.latest_version().get(code=classification.code) + assert new_version.code == classification.code + assert new_version.uuid == classification.uuid + assert new_version.version == 2 + assert new_version.title == data['title'] + assert new_version.description == data['description'] + assert new_version.state == Classification.DRAFT + + +@pytest.mark.parametrize('parent_state', (Classification.APPROVED, Classification.DRAFT)) +@pytest.mark.django_db +def test_classification_put_parent_version_change(user_api_client, parent_classification, classification, parent_state): + set_permissions(user_api_client, Classification.CAN_EDIT) + classification.parent = parent_classification + classification.save(update_fields=['parent']) + parent_classification_v2 = Classification.objects.create( + uuid=parent_classification.uuid, + title='Updated title', + code=parent_classification.code, + function_allowed=parent_classification.function_allowed, + state=parent_state + ) + assert parent_classification_v2.version == 2 + assert classification.parent == parent_classification + + data = { + 'code': classification.code, + 'title': 'Updated classification title', + 'description': 'Updated classification description', + 'parent': { + 'id': parent_classification_v2.uuid.hex, + 'version': parent_classification_v2.version, + } + } + response = user_api_client.put(get_classification_detail_url(classification), data=data) + response_data = response.json() + + assert response.status_code == 200 + new_version = Classification.objects.latest_version().get(uuid=classification.uuid) + assert new_version.version == 2 + assert new_version.parent == parent_classification_v2 + assert response_data['parent']['id'] == parent_classification_v2.uuid.hex + assert response_data['parent']['version'] == parent_classification_v2.version + + +@pytest.mark.parametrize( + 'old_state,new_state,is_ok', + ( + (Classification.DRAFT, Classification.SENT_FOR_REVIEW, True), + (Classification.DRAFT, Classification.WAITING_FOR_APPROVAL, False), + (Classification.DRAFT, Classification.APPROVED, False), + (Classification.SENT_FOR_REVIEW, Classification.DRAFT, True), + (Classification.SENT_FOR_REVIEW, Classification.WAITING_FOR_APPROVAL, True), + (Classification.SENT_FOR_REVIEW, Classification.APPROVED, False), + (Classification.WAITING_FOR_APPROVAL, Classification.DRAFT, True), + (Classification.WAITING_FOR_APPROVAL, Classification.SENT_FOR_REVIEW, False), + (Classification.WAITING_FOR_APPROVAL, Classification.APPROVED, True), + (Classification.APPROVED, Classification.DRAFT, True), + (Classification.APPROVED, Classification.SENT_FOR_REVIEW, False), + (Classification.APPROVED, Classification.WAITING_FOR_APPROVAL, False), + ), +) +@pytest.mark.django_db +def test_classification_state_change(user_api_client, classification, old_state, new_state, is_ok): + set_permissions( + user_api_client, + [ + Classification.CAN_EDIT, + Classification.CAN_REVIEW, + Classification.CAN_APPROVE, + ] + ) + classification.state = old_state + classification.save(update_fields=['state']) + data = {'state': new_state, 'name': 'this should be ignored'} + + response = user_api_client.patch(get_classification_detail_url(classification), data=data) + + if is_ok: + assert response.status_code == 200 + assert response.data['version'] == 1 + assert response.data['state'] == new_state + + classification.refresh_from_db() + assert classification.version == 1 + assert classification.state == new_state + else: + assert response.status_code == 400 + assert response.data['state'] == [_('Invalid state change.')] + + +@pytest.mark.django_db +def test_classification_modified_by(classification, user_api_client, user): + set_permissions(user_api_client, Classification.CAN_VIEW_MODIFIED_BY) + + response = user_api_client.get(get_classification_detail_url(classification)) + assert response.status_code == 200 + assert response.data['modified_by'] is None + + classification.modified_by = user + classification.save() + + response = user_api_client.get(get_classification_detail_url(classification)) + assert response.status_code == 200 + assert response.data['modified_by'] == '%s %s' % (user.first_name, user.last_name) + + +@pytest.mark.django_db +def test_function_anonymous_cannot_view_modified_by(classification, api_client, user): + classification.state = Function.APPROVED + classification.modified_by = user + classification.save() + + response = api_client.get(get_classification_detail_url(classification)) + assert response.status_code == 200 + assert 'modified_by' not in response.data + + +@pytest.mark.django_db +def test_classification_version_filter(user_api_client, classification): + classification_v2 = Classification.objects.create( + uuid=classification.uuid, + title='test classification v2', + code=classification.code, + function_allowed=classification.function_allowed, + ) + assert classification_v2.version == 2 + + response = user_api_client.get(get_classification_detail_url(classification), {'version': 2}) + + data = response.json() + assert data['title'] == 'test classification v2' + + @pytest.mark.django_db def test_version_history_modified_by(user_2_api_client, super_user_api_client, function, classification, user): set_permissions(user_2_api_client, Function.CAN_EDIT) @@ -1975,3 +2362,62 @@ def test_bulk_update_modified_by_display(bulk_update, user_api_client, permissio assert 'modified_by' not in response_data.keys() else: assert response_data['modified_by'] == 'John Rambo' + + +@pytest.mark.django_db +def test_record_api_put(record, super_user_api_client): + data = { + 'attributes': {'TypeSpecifier': 'updated record'}, + 'index': 123, + } + + response = super_user_api_client.put(get_record_detail_url(record), data=data) + + record.refresh_from_db() + assert response.status_code == 405 + assert record.index == 1 + assert record.attributes == {'TypeSpecifier': 'test record'} + + +@pytest.mark.django_db +def test_record_api_patch(record, super_user_api_client): + data = { + 'attributes': {'TypeSpecifier': 'updated record'}, + } + + response = super_user_api_client.patch(get_record_detail_url(record), data=data) + + record.refresh_from_db() + assert response.status_code == 405 + assert record.index == 1 + assert record.attributes == {'TypeSpecifier': 'test record'} + + +@pytest.mark.django_db +def test_record_api_delete(record, super_user_api_client): + response = super_user_api_client.delete(get_record_detail_url(record)) + + assert response.status_code == 405 + assert Record.objects.filter(pk=record.pk).exists() + + +@pytest.mark.parametrize('permission', (None, 'view_modified_by', 'superuser')) +@pytest.mark.django_db +def test_record_modified_by_display(record, user_api_client, permission): + if permission == 'view_modified_by': + set_permissions(user_api_client, Function.CAN_VIEW_MODIFIED_BY) + elif permission == 'superuser': + user_api_client.user.is_superuser = True + user_api_client.user.save(update_fields=['is_superuser']) + + record.created_by = user_api_client.user + record.modified_by = user_api_client.user + record.save() + + response = user_api_client.get(get_record_detail_url(record)) + response_data = response.json() + + if not permission: + assert 'modified_by' not in response_data.keys() + else: + assert response_data['modified_by'] == 'John Rambo' diff --git a/metarecord/tests/test_attribute_validations.py b/metarecord/tests/test_attribute_validations.py index 8b379eeb..b59ef348 100644 --- a/metarecord/tests/test_attribute_validations.py +++ b/metarecord/tests/test_attribute_validations.py @@ -1,6 +1,56 @@ import pytest +from copy import deepcopy +from django.contrib.contenttypes.models import ContentType from metarecord.models import Action, Function, Phase, Record +from metarecord.models.attribute import create_predefined_attributes +from metarecord.models.attribute_validation import AttributeValidationRule +from metarecord.models.structural_element import refactor_conditional_validation + +VALIDATION_JSON = { + 'allowed': [ + 'PersonalData', 'PublicityClass', 'RetentionPeriod', 'RetentionPeriodStart', 'SecurityPeriod', + 'InformationSystem', 'Subject' + ], + 'required': [ + 'PersonalData', 'PublicityClass', 'RetentionPeriod', + ], + 'conditionally_required': [ + { + 'attribute': 'SecurityPeriod', + 'conditions': [ + { + 'attribute': 'PublicityClass', + 'value': 'Salassa pidettävä', + }, + { + 'attribute': 'PublicityClass', + 'value': 'Osittain salassa pidettävä', + }, + ] + }, + ], + 'conditionally_disallowed': [ + { + 'attribute': 'RetentionPeriodStart', + 'conditions': [ + { + 'attribute': 'RetentionPeriod', + 'value': '-1', + } + ] + }, + ], + 'multivalued': [ + 'InformationSystem', 'Subject' + ], + 'all_or_none': [ + ['InformationSystem', 'Subject'], + ], + 'allow_values_outside_choices': [ + 'InformationSystem', 'Subject' + ], +} @pytest.mark.parametrize('model', [Action, Function, Phase, Record]) @@ -11,3 +61,158 @@ def test_allowed_contains_required_keys(model): assert set(model._attribute_validations['allowed']).issuperset(required_keys), \ 'allowedKeys does not contain all the required keys in the model "{}" attribute validations'.format( model.__name__) + + +def test_refactor_conditional_validation(): + attr_validations = { + "conditionally_required": [ + { + "attribute": "SecurityPeriod", + "conditions": [ + { + "attribute": "PublicityClass", + "value": "Salassa pidettävä", + }, + { + "attribute": "PublicityClass", + "value": "Osittain salassa pidettävä", + }, + ], + }, + ] + } + expected_output = { + 'conditionally_required': { + 'SecurityPeriod': { + 'PublicityClass': [ + 'Salassa pidettävä', 'Osittain salassa pidettävä' + ] + }, + }, + } + + refactored_validations = deepcopy(attr_validations) + refactor_conditional_validation(attr_validations, refactored_validations, "conditionally_required") + + assert refactored_validations == expected_output + + +@pytest.mark.django_db +def test_structural_element_json_schema_with_defined_validation(function): + # Function is a subclass of StructuralElement + create_predefined_attributes() + + content_type = ContentType.objects.get_for_model(function) + AttributeValidationRule.objects.create( + content_type=content_type, + validation_json=VALIDATION_JSON, + ) + + allowed_attributes = VALIDATION_JSON["allowed"] + json_schema = function.get_attribute_json_schema() + + assert all([prop in allowed_attributes for prop in json_schema["properties"]]) + assert len(json_schema["properties"]) == len(allowed_attributes) + + +@pytest.mark.django_db +def test_structural_element_required_attributes_with_defined_validation(function): + # Function is a subclass of StructuralElement + content_type = ContentType.objects.get_for_model(function) + AttributeValidationRule.objects.create( + content_type=content_type, + validation_json=VALIDATION_JSON, + ) + + required_attributes = VALIDATION_JSON["required"] + function_required_attributes = function.get_required_attributes() + + assert all([prop in required_attributes for prop in function_required_attributes]) + + +@pytest.mark.django_db +def test_structural_element_multivalued_attributes_with_defined_validation(function): + # Function is a subclass of StructuralElement + content_type = ContentType.objects.get_for_model(function) + AttributeValidationRule.objects.create( + content_type=content_type, + validation_json=VALIDATION_JSON, + ) + + multivalued_attributes = VALIDATION_JSON["multivalued"] + function_multivalued_attributes = function.get_multivalued_attributes() + + assert all([prop in multivalued_attributes for prop in function_multivalued_attributes]) + + +@pytest.mark.django_db +def test_structural_element_conditionally_required_attributes_with_defined_validation(function): + # Function is a subclass of StructuralElement + content_type = ContentType.objects.get_for_model(function) + AttributeValidationRule.objects.create( + content_type=content_type, + validation_json=VALIDATION_JSON, + ) + + conditionally_required_attributes = VALIDATION_JSON["conditionally_required"] + function_conditionally_required_attributes = function.get_conditionally_required_attributes() + + assert len(function_conditionally_required_attributes) == len(conditionally_required_attributes) + + +@pytest.mark.django_db +def test_structural_element_conditionally_disallowed_attributes_with_defined_validation(function): + # Function is a subclass of StructuralElement + content_type = ContentType.objects.get_for_model(function) + AttributeValidationRule.objects.create( + content_type=content_type, + validation_json=VALIDATION_JSON, + ) + + conditionally_disallowed_attributes = VALIDATION_JSON["conditionally_disallowed"] + function_conditionally_disallowed_attributes = function.get_conditionally_required_attributes() + + assert len(function_conditionally_disallowed_attributes) == len(conditionally_disallowed_attributes) + + +@pytest.mark.django_db +def test_structural_element_all_or_none_attributes_with_defined_validation(function): + # Function is a subclass of StructuralElement + content_type = ContentType.objects.get_for_model(function) + AttributeValidationRule.objects.create( + content_type=content_type, + validation_json=VALIDATION_JSON, + ) + + all_or_none = VALIDATION_JSON["all_or_none"][0] + function_all_or_none = function.get_all_or_none_attributes()[0] + + assert all([prop in all_or_none for prop in function_all_or_none]) + + +@pytest.mark.django_db +def test_structural_element_allow_values_outside_choices_attributes_with_defined_validation(function): + # Function is a subclass of StructuralElement + content_type = ContentType.objects.get_for_model(function) + AttributeValidationRule.objects.create( + content_type=content_type, + validation_json=VALIDATION_JSON, + ) + + allow_values_outside_choices = VALIDATION_JSON["allow_values_outside_choices"] + function_allow_values_outside_choices = function.get_allow_values_outside_choices_attributes() + + assert all([prop in allow_values_outside_choices for prop in function_allow_values_outside_choices]) + + +@pytest.mark.django_db +def test_structural_element_is_attribute_allowed_with_defined_validation(function): + # Function is a subclass of StructuralElement + content_type = ContentType.objects.get_for_model(function) + AttributeValidationRule.objects.create( + content_type=content_type, + validation_json=VALIDATION_JSON, + ) + + allowed = VALIDATION_JSON["allowed"] + assert function.is_attribute_allowed(allowed[0]) diff --git a/metarecord/tests/test_classification_model.py b/metarecord/tests/test_classification_model.py new file mode 100644 index 00000000..109305e2 --- /dev/null +++ b/metarecord/tests/test_classification_model.py @@ -0,0 +1,42 @@ +import pytest + +from metarecord.models import Classification + + +@pytest.mark.django_db +def test__classification__delete_old_unapproved_versions_on_save(classification): + v1 = classification + v1.state = Classification.APPROVED + v1.save() + assert v1.version == 1 + + v1.pk = None + v1.state = Classification.DRAFT + v1.save() + v2 = Classification.objects.latest_version().get(uuid=classification.uuid) + assert v2.version == 2 + + v2.pk = None + v2.state = Classification.SENT_FOR_REVIEW + v2.save() + v3 = Classification.objects.latest_version().get(uuid=classification.uuid) + assert v3.version == 3 + + v3.pk = None + v3.state = Classification.WAITING_FOR_APPROVAL + v3.save() + v4 = Classification.objects.latest_version().get(uuid=classification.uuid) + assert v4.version == 4 + + # Check that nothing has been deleted by mistake so far + assert Classification.objects.filter(uuid=v1.uuid).count() == 4 + + # v5 is approved and saving it should delete all older unapproved versions + v4.pk = None + v4.state = Classification.APPROVED + v4.save() + v5 = Classification.objects.latest_version().get(uuid=classification.uuid) + assert v5.version == 5 + + assert not Classification.objects.filter(uuid=v5.uuid).exclude(state=Classification.APPROVED).exists() + assert Classification.objects.count() == 2 diff --git a/metarecord/tests/test_data/00_00_01_02.xlsx b/metarecord/tests/test_data/00_00_01_02.xlsx new file mode 100644 index 00000000..622a6f08 Binary files /dev/null and b/metarecord/tests/test_data/00_00_01_02.xlsx differ diff --git a/metarecord/tests/test_function_model.py b/metarecord/tests/test_function_model.py new file mode 100644 index 00000000..fdeb6af3 --- /dev/null +++ b/metarecord/tests/test_function_model.py @@ -0,0 +1,44 @@ +import pytest + +from metarecord.models import Classification, Function + + +@pytest.mark.django_db +def test_function_get_name(function): + assert function.get_name() == function.classification.title + + function.is_template = True + function.save(update_fields=("is_template",)) + + assert function.get_name() == function.name + + +@pytest.mark.django_db +def test_function_draft_delete(function): + function.state = Function.APPROVED + can_delete = function.can_user_delete(None) + assert can_delete is False + + +@pytest.mark.django_db +def test_function_save_without_classification(function): + function.classification = None + with pytest.raises(Exception) as excinfo: + function.save() + assert str(excinfo.value) == "Classification is required." + + +@pytest.mark.django_db +def test_approved_function_with_unapproved_classification(function): + function.state = Function.APPROVED + function.classification.state = Classification.DRAFT + with pytest.raises(Exception) as excinfo: + function.save() + assert str(excinfo.value) == "Approved function must have approved classification" + + +@pytest.mark.django_db +def test_function_unapproved_delete_old_versions(function): + with pytest.raises(Exception) as excinfo: + function.delete_old_non_approved_versions() + assert str(excinfo.value) == "Function must be approved before old non-approved versions can be deleted." diff --git a/metarecord/tests/test_importer.py b/metarecord/tests/test_importer.py new file mode 100644 index 00000000..b47119e8 --- /dev/null +++ b/metarecord/tests/test_importer.py @@ -0,0 +1,24 @@ +import pytest + +from metarecord.importer.tos import TOSImporter, TOSImporterException + + +@pytest.mark.django_db +def test_tos_importer_with_no_classification(tos_importer_excel_file_path): + tos_importer = TOSImporter() + tos_importer.open(tos_importer_excel_file_path) + with pytest.raises(TOSImporterException) as excinfo: + tos_importer.import_data() + assert str(excinfo.value) == "Classification 00 00 01 02 does not exist" + + +@pytest.mark.django_db +def test_tos_importer_with_classification_and_function(function, tos_importer_excel_file_path): + function.classification.code = "00 00 01 02" + function.classification.save(update_fields=["code"]) + + tos_importer = TOSImporter() + tos_importer.open(tos_importer_excel_file_path) + + tos_importer.import_data() + tos_importer.import_attributes() diff --git a/metarecord/tests/test_jhs_exporter.py b/metarecord/tests/test_jhs_exporter.py index 47d46311..cae0a2fb 100644 --- a/metarecord/tests/test_jhs_exporter.py +++ b/metarecord/tests/test_jhs_exporter.py @@ -3,9 +3,11 @@ import freezegun import pytest +from rest_framework.test import APIClient from metarecord.exporter.jhs import JHSExporter from metarecord.models import Function +from metarecord.views.export import JHSExportViewSet @pytest.mark.django_db @@ -65,3 +67,24 @@ def test_exporter_xml_generation_is_successful(function, phase, action, record): action_id=action.uuid, rec_id=record.uuid ) + + +@pytest.mark.django_db +def test_export_view_file_creation(function, phase, action, record): + client = APIClient() + response = client.get("/export/") + assert 'Content-Disposition' in response + + +@pytest.mark.django_db +def test_jhs_export_view_file_creation(function, phase, action, record): + jhs_export_view = JHSExportViewSet() + + open_mock = mock.mock_open() + open_mock.side_effect = [FileNotFoundError, mock.DEFAULT] + with mock.patch("metarecord.views.export.open", open_mock, create=True): + response = jhs_export_view.list(None) + + open_mock.assert_called() + open_mock.return_value.write.assert_called_once() + assert 'Content-Disposition' in response diff --git a/metarecord/tests/test_structural_element.py b/metarecord/tests/test_structural_element.py index 8f360f1b..ed0d7dda 100644 --- a/metarecord/tests/test_structural_element.py +++ b/metarecord/tests/test_structural_element.py @@ -1,5 +1,7 @@ import pytest +from metarecord.models.attribute import create_predefined_attributes + @pytest.mark.django_db def test_structural_element_persistent_user_name_fields(user, function): @@ -18,3 +20,13 @@ def test_structural_element_persistent_user_name_fields(user, function): assert function._created_by == 'John Rambo' assert not function.modified_by assert function._modified_by == 'John Rambo' + + +@pytest.mark.django_db +def test_structural_element_json_schema_without_defined_validation(function): + # Function is a subclass of StructuralElement + create_predefined_attributes() + + allowed_attributes = function.get_attribute_validations()["allowed"] + json_schema = function.get_attribute_json_schema() + assert all([prop in allowed_attributes for prop in json_schema["properties"]]) diff --git a/metarecord/tests/utils.py b/metarecord/tests/utils.py index 8fab1cc2..75c9f719 100644 --- a/metarecord/tests/utils.py +++ b/metarecord/tests/utils.py @@ -1,5 +1,10 @@ from django.contrib.auth import get_user_model from django.contrib.auth.models import Permission +from rest_framework import serializers + +from metarecord.models import Function +from metarecord.views.base import ClassificationRelationSerializer, HexRelatedField, StructuralElementSerializer +from metarecord.views.function import FunctionListSerializer, PhaseSerializer def set_permissions(api_client, permissions): @@ -51,3 +56,42 @@ def assert_response_functions(response, objects): def get_bulk_update_function_key(function): return '{uuid}__{version}'.format(uuid=function.uuid.hex, version=function.version) + + +class FunctionTestDetailSerializer(StructuralElementSerializer): + version = serializers.IntegerField(read_only=True) + modified_by = serializers.SerializerMethodField() + state = serializers.CharField(read_only=True) + + # TODO these three are here to maintain backwards compatibility, + # should be removed as soon as the UI doesn't need these anymore + function_id = serializers.ReadOnlyField(source='get_classification_code') + # there is also Function.name field which should be hidden for other than templates when this is removed + name = serializers.ReadOnlyField(source='get_name') + parent = serializers.SerializerMethodField() + classification_code = serializers.ReadOnlyField(source='get_classification_code') + classification_title = serializers.ReadOnlyField(source='get_name') + + classification = ClassificationRelationSerializer() + + class Meta(StructuralElementSerializer.Meta): + model = Function + exclude = StructuralElementSerializer.Meta.exclude + ('index', 'is_template') + + + def get_parent(self, obj): + if obj.classification and obj.classification.parent: + parent_functions = ( + Function.objects + .filter(classification__uuid=obj.classification.parent.uuid) + ) + if parent_functions.exists(): + return parent_functions[0].uuid.hex + return None + + def get_fields(self): + fields = super().get_fields() + + fields['phases'] = PhaseSerializer(many=True, required=False) + + return fields \ No newline at end of file diff --git a/metarecord/views/__init__.py b/metarecord/views/__init__.py index 47b6b1cb..2ec4781c 100644 --- a/metarecord/views/__init__.py +++ b/metarecord/views/__init__.py @@ -3,4 +3,5 @@ from .classification import ClassificationViewSet # noqa from .export import ExportView, JHSExportViewSet # noqa from .function import FunctionViewSet # noqa +from .record import RecordViewSet # noqa from .template import TemplateViewSet # noqa diff --git a/metarecord/views/base.py b/metarecord/views/base.py index 7e0156ff..ec59a93e 100644 --- a/metarecord/views/base.py +++ b/metarecord/views/base.py @@ -1,16 +1,28 @@ import collections from collections import defaultdict -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import exceptions, serializers -from metarecord.models import Attribute, StructuralElement +from metarecord.models import Attribute, Classification, StructuralElement class BaseModelSerializer(serializers.ModelSerializer): id = serializers.UUIDField(read_only=True, format='hex') +class ClassificationRelationSerializer(serializers.ModelSerializer): + id = serializers.UUIDField(source='uuid', format='hex', read_only=True) + version = serializers.IntegerField(min_value=1, read_only=True) + + class Meta: + model = Classification + fields = ('id', 'version') + + def to_internal_value(self, data): + return Classification.objects.get(uuid=data['id'], version=data['version']) + + class StructuralElementSerializer(serializers.ModelSerializer): id = serializers.UUIDField(source='uuid', format='hex', read_only=True) # TODO: This DictFields child should allow only strings or array of strings. @@ -18,7 +30,7 @@ class StructuralElementSerializer(serializers.ModelSerializer): class Meta: ordering = ('index',) - exclude = ('uuid', 'created_by') + exclude = ('uuid', 'created_by', '_created_by', '_modified_by') def get_fields(self): fields = super().get_fields() diff --git a/metarecord/views/bulk_update.py b/metarecord/views/bulk_update.py index da0db86a..3a70a474 100644 --- a/metarecord/views/bulk_update.py +++ b/metarecord/views/bulk_update.py @@ -1,5 +1,5 @@ from django.core.exceptions import ObjectDoesNotExist, PermissionDenied, ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers, status, viewsets from rest_framework.decorators import action from rest_framework.response import Response diff --git a/metarecord/views/classification.py b/metarecord/views/classification.py index 8566d7a6..d567aec6 100644 --- a/metarecord/views/classification.py +++ b/metarecord/views/classification.py @@ -1,11 +1,14 @@ -from django.db.models import Prefetch +import django_filters +from django.core import exceptions +from django.db import transaction +from django.db.models import Prefetch, Q +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers, viewsets from metarecord.models import Classification, Function +from metarecord.views.base import ClassificationRelationSerializer from metarecord.views.function import PhaseSerializer -from .base import HexRelatedField - def include_related(request): """ @@ -18,27 +21,56 @@ def include_related(request): class ClassificationSerializer(serializers.ModelSerializer): id = serializers.UUIDField(source='uuid', format='hex', read_only=True) - parent = HexRelatedField(read_only=True) + parent = ClassificationRelationSerializer(required=False) + modified_by = serializers.SerializerMethodField() + version_history = serializers.SerializerMethodField() class Meta: model = Classification - fields = ('id', 'created_at', 'modified_at', 'code', 'title', 'parent', 'description', 'description_internal', - 'related_classification', 'additional_information', 'function_allowed') + fields = ( + 'id', + 'created_at', + 'modified_at', + 'modified_by', + 'version', + 'state', + 'valid_from', + 'valid_to', + 'code', + 'title', + 'parent', + 'description', + 'description_internal', + 'related_classification', + 'additional_information', + 'function_allowed', + 'version_history', + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) request = self.context['request'] - self.include_related = include_related(request) + self.include_related = False + + if request.method == 'GET': + self.include_related = include_related(request) def get_fields(self): + user = self.context['request'].user fields = super().get_fields() if self.include_related: fields['phases'] = serializers.SerializerMethodField(method_name='_get_phases') + if not user.has_perm(Classification.CAN_VIEW_MODIFIED_BY): + del fields['modified_by'] + return fields + def get_modified_by(self, obj): + return obj.get_modified_by_display() + def _get_function(self, obj): functions = obj.prefetched_functions num_of_functions = len(functions) @@ -81,38 +113,191 @@ def _get_phases(self, obj): return serializer.data + def get_version_history(self, obj): + request = self.context['request'] + classifications = Classification.objects.filter_for_user(request.user).filter(uuid=obj.uuid).order_by('version') + ret = [] + + for classification in classifications: + version_data = { + attr: getattr(classification, attr) for attr in ( + 'state', + 'version', + 'modified_at', + 'valid_from', + 'valid_to' + ) + } + + if not request or request.user.has_perm(Classification.CAN_VIEW_MODIFIED_BY): + version_data['modified_by'] = classification.get_modified_by_display() + + ret.append(version_data) + + return ret + def to_representation(self, obj): data = super().to_representation(obj) - data = self._append_function_fields_to_repr(obj, data) - request = self.context['request'] - if request and not request.user.is_authenticated: - data.pop('description_internal', None) - data.pop('additional_information', None) + + if request and request.method == 'GET': + data = self._append_function_fields_to_repr(obj, data) + + if not request.user.is_authenticated: + data.pop('description_internal', None) + data.pop('additional_information', None) return data + def _create_new_version(self, validated_data): + user = self.context['request'].user + user_data = {'created_by': user, 'modified_by': user} + validated_data.update(user_data) + return Classification.objects.create(**validated_data) + + def validate(self, data): + request = self.context['request'] + if request.method in ['PUT', 'PATCH']: + self.validate_update(data) + return super().validate(data) + + def validate_update(self, data): + if self.partial: + if not any(field in data for field in ('state', 'valid_from', 'valid_to')): + raise exceptions.ValidationError(_('"state", "valid_from" or "valid_to" required.')) + + new_state = data.get('state') + if new_state: + self.validate_state_change(self.instance.state, new_state) + + def validate_state_change(self, old_state, new_state): + user = self.context['request'].user + + if old_state == new_state: + return + + valid_changes = { + Classification.DRAFT: {Classification.SENT_FOR_REVIEW}, + Classification.SENT_FOR_REVIEW: {Classification.WAITING_FOR_APPROVAL, Classification.DRAFT}, + Classification.WAITING_FOR_APPROVAL: {Classification.APPROVED, Classification.DRAFT}, + Classification.APPROVED: {Classification.DRAFT}, + } + + if new_state not in valid_changes[old_state]: + raise exceptions.ValidationError({'state': [_('Invalid state change.')]}) + + state_change_required_permissions = { + Classification.SENT_FOR_REVIEW: Classification.CAN_EDIT, + Classification.WAITING_FOR_APPROVAL: Classification.CAN_REVIEW, + Classification.APPROVED: Classification.CAN_APPROVE, + } + + relevant_state = new_state if new_state != Classification.DRAFT else old_state + required_permission = state_change_required_permissions[relevant_state] + + if not user.has_perm(required_permission): + raise exceptions.PermissionDenied(_('No permission for the state change.')) -class ClassificationViewSet(viewsets.ReadOnlyModelViewSet): + @transaction.atomic + def create(self, validated_data): + user = self.context['request'].user + + if not user.has_perm(Classification.CAN_EDIT): + raise exceptions.PermissionDenied(_('No permission to create.')) + + validated_data.update({ + 'created_by': user, + 'modified_by': user, + }) + + return super().create(validated_data) + + @transaction.atomic + def update(self, instance, validated_data): + user = self.context['request'].user + + if not user.has_perm(Classification.CAN_EDIT): + raise exceptions.PermissionDenied(_('No permission to update.')) + + if self.partial: + allowed_fields = {'state', 'valid_from', 'valid_to'} + data = { + field: validated_data[field] + for field in allowed_fields + if field in validated_data + } + if not data: + return instance + data['modified_by'] = user + + # Update only state, valid_from and valid_to fields + # and do an actual update instead of a new version + return super().update(instance, data) + + if instance.state in (Classification.SENT_FOR_REVIEW, Classification.WAITING_FOR_APPROVAL): + raise exceptions.ValidationError( + _('Cannot edit while in state "sent_for_review" or "waiting_for_approval"') + ) + + return self._create_new_version(validated_data) + + +class ClassificationFilterSet(django_filters.FilterSet): + valid_at = django_filters.DateFilter(method='filter_valid_at') + + class Meta: + model = Classification + fields = ('valid_at', 'version') + + def filter_valid_at(self, queryset, name, value): + # Classification is considered invalid if neither date is set + queryset = queryset.exclude(valid_from__isnull=True, valid_to__isnull=True) + + # Null value means there's no bound to that direction + queryset = queryset.filter( + (Q(valid_from__isnull=True) | Q(valid_from__lte=value)) & + (Q(valid_to__isnull=True) | Q(valid_to__gte=value)) + ) + return queryset + + +class ClassificationViewSet(viewsets.ModelViewSet): queryset = Classification.objects.order_by('code').select_related('parent').prefetch_related('children') serializer_class = ClassificationSerializer + filter_backends = (django_filters.rest_framework.DjangoFilterBackend,) + filterset_class = ClassificationFilterSet lookup_field = 'uuid' + def apply_queryset_filters(self, queryset): + queryset = queryset.filter_for_user(self.request.user) + + if 'version' in self.request.query_params: + return queryset + + state = self.request.query_params.get('state') + if state == 'approved': + return queryset.latest_approved() + + return queryset.latest_version() + def get_queryset(self): user = self.request.user - queryset = Function.objects.filter_for_user(user).latest_version() + function_qs = Function.objects.filter_for_user(user).latest_version() if include_related(self.request): - queryset = queryset.prefetch_related( + function_qs = function_qs.prefetch_related( 'phases', 'phases__actions', 'phases__actions__records' ) - return super().get_queryset().prefetch_related( + queryset = super().get_queryset() + queryset = self.apply_queryset_filters(queryset) + + return queryset.prefetch_related( Prefetch( 'functions', - queryset=queryset, + queryset=function_qs, to_attr='prefetched_functions' ) ) diff --git a/metarecord/views/function.py b/metarecord/views/function.py index 56f81b68..aa0d8e7f 100644 --- a/metarecord/views/function.py +++ b/metarecord/views/function.py @@ -1,16 +1,18 @@ +import uuid + import django_filters from django.db import transaction from django.db.models import Q from django.http import Http404 -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import exceptions, serializers, status, viewsets from rest_framework.response import Response -from metarecord.models import Action, Function, Phase, Record +from metarecord.models import Action, Classification, Function, Phase, Record +from metarecord.models.bulk_update import BulkUpdate from ..utils import validate_uuid4 -from .base import DetailSerializerMixin, HexRelatedField, StructuralElementSerializer -from .classification import Classification +from .base import ClassificationRelationSerializer, DetailSerializerMixin, HexRelatedField, StructuralElementSerializer class RecordSerializer(StructuralElementSerializer): @@ -51,6 +53,8 @@ class FunctionListSerializer(StructuralElementSerializer): classification_code = serializers.ReadOnlyField(source='get_classification_code') classification_title = serializers.ReadOnlyField(source='get_name') + bulk_update = serializers.SerializerMethodField() + # TODO these three are here to maintain backwards compatibility, # should be removed as soon as the UI doesn't need these anymore function_id = serializers.ReadOnlyField(source='get_classification_code') @@ -58,7 +62,7 @@ class FunctionListSerializer(StructuralElementSerializer): name = serializers.ReadOnlyField(source='get_name') parent = serializers.SerializerMethodField() - classification = HexRelatedField(queryset=Classification.objects.all()) + classification = ClassificationRelationSerializer() class Meta(StructuralElementSerializer.Meta): model = Function @@ -74,27 +78,62 @@ def get_fields(self): return fields - def _create_new_version(self, function_data): + def _create_new_version(self, function_data, copy_from_previous=False, phases=None): + + if not phases: + phases = [] + user = self.context['request'].user user_data = {'created_by': user, 'modified_by': user} phase_data = function_data.pop('phases', []) + if function_data.get("bulk_update"): + function_data["bulk_update"] = BulkUpdate.objects.get(id=function_data["bulk_update"]) + function_data.update(user_data) + function = Function.objects.create(**function_data) - for index, phase_datum in enumerate(phase_data, 1): - action_data = phase_datum.pop('actions', []) - phase_datum.update(user_data) - phase = Phase.objects.create(function=function, index=index, **phase_datum) + if copy_from_previous: + for phase in phases: + actions = [] + for action in phase.actions.all(): + records = [] + for record in action.records.all(): + record.pk = None + record.uuid = uuid.uuid4() + record.save() + records.append(record) + + action.pk = None + action.uuid = uuid.uuid4() + action.save() + for record in records: + action.records.add(record) + actions.append(action) + + phase.pk = None + phase.uuid = uuid.uuid4() + phase.save() + for action in actions: + phase.actions.add(action) + function.phases.add(phase) - for index, action_datum in enumerate(action_data, 1): - record_data = action_datum.pop('records', []) - action_datum.update(user_data) - action = Action.objects.create(phase=phase, index=index, **action_datum) + else: + for index, phase_datum in enumerate(phase_data, 1): + action_data = phase_datum.pop('actions', []) + phase_datum.update(user_data) - for index, record_datum in enumerate(record_data, 1): - record_datum.update(user_data) - Record.objects.create(action=action, index=index, **record_datum) + phase = Phase.objects.create(function=function, index=index, **phase_datum) + + for index, action_datum in enumerate(action_data, 1): + record_data = action_datum.pop('records', []) + action_datum.update(user_data) + action = Action.objects.create(phase=phase, index=index, **action_datum) + + for index, record_datum in enumerate(record_data, 1): + record_datum.update(user_data) + Record.objects.create(action=action, index=index, **record_datum) return function @@ -103,11 +142,17 @@ def get_modified_by(self, obj): def get_parent(self, obj): if obj.classification and obj.classification.parent: - parent_functions = Function.objects.filter(classification=obj.classification.parent) + parent_functions = ( + Function.objects + .filter(classification__uuid=obj.classification.parent.uuid) + ) if parent_functions.exists(): return parent_functions[0].uuid.hex return None + def get_bulk_update(self, obj): + return obj.bulk_update.id if obj.bulk_update else None + def validate(self, data): new_valid_from = data.get('valid_from') new_valid_to = data.get('valid_to') @@ -117,11 +162,11 @@ def validate(self, data): if not self.instance: if Function.objects.filter(classification=data['classification']).exists(): raise exceptions.ValidationError( - _('Classification %s already has a function.') % data['classification'].uuid + _('Classification %s already has a function.') % data['classification'].uuid.hex ) if not data['classification'].function_allowed: raise exceptions.ValidationError( - _('Classification %s does not allow function creation.') % data['classification'].uuid + _('Classification %s does not allow function creation.') % data['classification'].uuid.hex ) return data @@ -133,10 +178,50 @@ def create(self, validated_data): if not user.has_perm(Function.CAN_EDIT): raise exceptions.PermissionDenied(_('No permission to create.')) - validated_data['modified_by'] = user - new_function = self._create_new_version(validated_data) - new_function.create_metadata_version() + previous_version_function = None + if list(validated_data.keys()) == ['classification']: + previous_classifications = Classification.objects.filter( + code=validated_data["classification"].code + ).order_by('-version') + + previous_classification = previous_classifications[1] if len(previous_classifications) > 1 else None + + previous_version_function = Function.objects.filter( + classification=previous_classification + ).latest_version().first() + + classification = validated_data["classification"] + previous_function_data = FunctionDetailSerializer(previous_version_function, context=self.context).data + + if previous_version_function: + previous_function_data['classification'] = classification + previous_function_data['modified_by'] = user + previous_function_data['error_count'] = 0 + extra_fields = [ + 'classification_code', + 'classification_title', + 'function_id', + 'parent', + 'version_history', + 'id' + ] + for field in extra_fields: + previous_function_data.pop(field) + + if previous_version_function.phases: + phases = previous_version_function.phases.all() + + new_function = self._create_new_version( + self.validate(previous_function_data), + copy_from_previous=True, + phases=phases + ) + + else: + validated_data['modified_by'] = user + new_function = self._create_new_version(validated_data) + new_function.create_metadata_version() return new_function @@ -150,7 +235,7 @@ def get_fields(self): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['classification'].read_only = True + self.fields['classification'].required = False if self.partial: self.fields['state'].read_only = False @@ -170,6 +255,14 @@ def validate(self, data): errors = self.get_attribute_validation_errors(self.instance) if errors: raise exceptions.ValidationError(errors) + else: + classification = data['classification'] + + if classification.uuid != self.instance.classification.uuid: + raise exceptions.ValidationError( + _('Changing classification is not allowed. Only version can be changed.') + ) + return data @transaction.atomic @@ -198,7 +291,9 @@ def update(self, instance, validated_data): _('Cannot edit while in state "sent_for_review" or "waiting_for_approval"') ) - validated_data['classification'] = instance.classification + if not validated_data.get('classification'): + validated_data['classification'] = instance.classification + validated_data['modified_by'] = user new_function = self._create_new_version(validated_data) new_function.create_metadata_version() @@ -239,8 +334,15 @@ def get_version_history(self, obj): ret = [] for function in functions: - version_data = {attr: getattr(function, attr) for attr in ('state', 'version', 'modified_at')} - + version_data = { + attr: getattr(function, attr) for attr in ( + 'state', + 'version', + 'modified_at', + 'valid_from', + 'valid_to' + ) + } if not request or function.can_view_modified_by(request.user): version_data['modified_by'] = function.get_modified_by_display() diff --git a/metarecord/views/record.py b/metarecord/views/record.py new file mode 100644 index 00000000..bf7529bd --- /dev/null +++ b/metarecord/views/record.py @@ -0,0 +1,21 @@ +from rest_framework import serializers, viewsets +from rest_framework.mixins import RetrieveModelMixin + +from metarecord.models import Record +from metarecord.views.base import StructuralElementSerializer + + +class RecordSerializer(StructuralElementSerializer): + modified_by = serializers.SerializerMethodField() + + class Meta(StructuralElementSerializer.Meta): + model = Record + + def get_modified_by(self, obj): + return obj._modified_by or None + + +class RecordViewSet(RetrieveModelMixin, viewsets.GenericViewSet): + serializer_class = RecordSerializer + queryset = Record.objects.all() + lookup_field = 'uuid' diff --git a/openapi.yaml b/openapi.yaml index 266e6fad..f309a30b 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -141,6 +141,12 @@ paths: To update a function ie. to create a new version of it, PUT method must be used. All new versions start from state "draft", and this cannot be changed with PUT. parameters: + - name: function_id + in: path + required: true + description: ID of the function to put + schema: + type: string - $ref: '#/components/parameters/includeApproved' responses: '200': @@ -316,7 +322,7 @@ paths: $ref: '#/components/schemas/Template' '404': $ref: '#/components/responses/NotFound' - /export/: + /export/jhs191/: get: tags: - export @@ -332,7 +338,7 @@ paths: content: application/xml: schema: - type: string + $ref: '#/components/schemas/tos:Tos' /bulk-update/: get: tags: @@ -429,13 +435,20 @@ paths: tags: - bulk_update description: Approve bulk update and apply its changes + parameters: + - name: bulk_update_id + in: path + required: true + description: ID of the bulk update to update + schema: + type: string responses: '200': description: The bulk update was approved succesfully '400': - $ref: '#/component/responses/BadRequest' + $ref: '#/components/responses/BadRequest' '401': - $ref: '#/component/responses/NotAuthenticated' + $ref: '#/components/responses/NotAuthenticated' '403': $ref: '#/components/responses/Forbidden' '404': @@ -448,15 +461,40 @@ components: scheme: bearer bearerFormat: JWT schemas: + RelatedClassification: + type: object + example: + id: 150c41a055804f7d97f1ee47be2d109d + version: 1 + properties: + id: + description: Unique identifier + type: string + version: + description: Classification version number + type: integer + readOnly: true Classification: type: object example: id: 150c41a055804f7d97f1ee47be2d109d created_at: '2017-08-28T09:25:22.097Z' modified_at: '2017-08-28T09:25:22.190Z' + modified_by: 'Matti Meikäläinen' + version: 2 + state: 'draft' + valid_from: '2017-08-28T09:25:22.097Z' + valid_to: '2018-08-28T09:25:22.097Z' + version_history: + state: 'draft' + version: 1 + modified_at: '2017-08-28T20:46:34.796669Z' + valid_from: '2017-08-28T20:46:34.796644Z' + valid_to: '2017-08-28T20:46:34.796644Z' + modified_by: Matti Meikäläinen code: 00 00 title: Hallintoasioiden ohjaus - parent: 1d75967c89334bb5b61681e1a5118e6b + parent: {} description: Hallintoasioiden ohjaus on tarkkaa puuhaa description_internal: Hallintoasioiden ohjaus on tarkkaa puuhaa related_classification: Katso myös tehtäväluokka 00 @@ -478,6 +516,32 @@ components: description: Last modification time type: string format: dateTime + modified_by: + description: First and last name of User + type: string + nullable: true + version: + description: Classification version number + type: integer + readOnly: true + state: + description: Classification state + type: string + enum: + - draft + - sent_for_review + - waiting_for_approval + - approved + valid_from: + type: string + format: date + description: The classification is valid starting from this date + valid_to: + type: string + format: date + description: The classification is valid until this date + version_history: + $ref: '#/components/schemas/VersionHistory' code: description: Classification code type: string @@ -485,8 +549,7 @@ components: description: Title type: string parent: - description: ID of the parent classification of this classification - type: string + $ref: '#/components/schemas/RelatedClassification' description: description: Description type: string @@ -529,13 +592,20 @@ components: version: 4 function_id: '00' name: Hallintoasiat - parent: null + parent: {} classification: 112d25ac6dab4e59b9827f94ac2c06fe created_at: '2017-08-25T10:10:00.346962Z' modified_at: '2017-08-25T10:10:00.347001Z' state: draft valid_from: '2017-06-25' valid_to: '2017-07-25' + version_history: + state: 'draft' + version: 1 + modified_at: '2017-08-28T20:46:34.796669Z' + valid_from: '2017-08-28T20:46:34.796644Z' + valid_to: '2017-08-28T20:46:34.796644Z' + modified_by: Matti Meikäläinen modified_by: 'Matti Meikäläinen' classification_code: 01 00 00 classification_title: Työnantaja- ja henkilöstöpolitiikka @@ -564,8 +634,7 @@ components: type: string deprecated: true classification: - description: ID of the function's classification - type: string + $ref: '#/components/schemas/RelatedClassification' created_at: description: Creation time type: string @@ -592,6 +661,8 @@ components: type: string format: date description: The function is valid until this date + version_history: + $ref: '#/components/schemas/VersionHistory' modified_by: description: First and last name of User type: string @@ -907,6 +978,44 @@ components: type: string previous: type: string + VersionHistory: + type: object + example: + state: 'draft' + version: 1 + modified_at: '2017-08-28T20:46:34.796669Z' + valid_from: '2017-08-28T20:46:34.796644Z' + valid_to: '2017-08-28T20:46:34.796644Z' + modified_by: Matti Meikäläinen + properties: + state: + description: Object state + type: string + enum: + - draft + - sent_for_review + - waiting_for_approval + - approved + version: + description: Object version number + type: integer + readOnly: true + modified_at: + description: Last modification time + type: string + format: dateTime + valid_from: + description: Valid from date + type: string + format: dateTime + valid_to: + description: Valid to date + type: string + format: dateTime + modified_by: + description: First and last name of User + type: string + nullable: true BulkUpdate: type: object description: Bulk update that contains changes to Functions and any related Phases, Actions, and Records @@ -998,6 +1107,89 @@ components: } } } + tos:Tos: + type: object + description: Export of the data in JHS191 XML format. + example: + xmlns:tos: "http://skeemat.jhs-suositukset.fi/tos/2015/01/15" + tos:TosTiedot: + tos:id: "cee34db0-de3e-47fb-b937-b74dd87ea759" + tos:Nimeke: + tos:NimekeKielella: + tos:kieliKoodi: "fi" + tos:NimekeTeksti: Helsingin kaupungin Tiedonohjaussuunnitelma + tos:YhteyshenkiloNimi: Tiedonhallinta + tos:TosVersio: 1 + tos:TilaKoodi: 3 + tos:OrganisaatioNimi: Helsingin kaupunki + tos:LisatiedotTeksti: JHS 191 XML 2021-01-11 12:15EET exported from undefined environment + tos:Luokka: + tos:Luokitustunnus: '00' + tos:Nimeke: + tos:NimekeKielella: + tos:kieliKoodi: "fi" + tos:NimekeTeksti: Helsingin kaupungin Tiedonohjaussuunnitelma + properties: + xmlns:tos: + type: string + xml: + attribute: true + tos:TosTiedot: + $ref: '#/components/schemas/tos:TosTiedot' + tos:Luokka: + $ref: '#/components/schemas/tos:Luokka' + tos:Luokka: + type: object + example: + tos:Luokitustunnus: '00' + properties: + tos:Luokitustunnus: + type: string + tos:Nimeke: + $ref: '#/components/schemas/tos:Nimeke' + tos:TosTiedot: + type: object + example: + tos:id: "cee34db0-de3e-47fb-b937-b74dd87ea759" + tos:YhteyshenkiloNimi: Tiedonhallinta + tos:TosVersio: 1 + tos:TilaKoodi: 3 + tos:OrganisaatioNimi: Helsingin kaupunki + tos:LisatiedotTeksti: JHS 191 XML 2021-01-11 12:15EET exported from undefined environment + properties: + tos:id: + type: string + xml: + attribute: true + tos:Nimeke: + $ref: '#/components/schemas/tos:Nimeke' + tos:YhteyshenkiloNimi: + type: string + tos:TosVersio: + type: integer + tos:TilaKoodi: + type: integer + tos:OrganisaatioNimi: + type: string + tos:LisatiedotTeksti: + type: string + tos:Nimeke: + type: object + properties: + tos:NimekeKielella: + $ref: '#/components/schemas/tos:NimekeKielella' + tos:NimekeKielella: + type: object + example: + tos:kieliKoodi: "fi" + tos:NimekeTeksti: Helsingin kaupungin Tiedonohjaussuunnitelma + properties: + tos:kieliKoodi: + type: string + xml: + attribute: true + tos:NimekeTeksti: + type: string responses: BadRequest: description: Bad request, details about the error in the request body diff --git a/requirements.in b/requirements.in index e65e65bb..1d9649a8 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,11 @@ -Django~=2.2.16 +Django~=3.1 +django-elasticsearch-dsl-drf +django-environ django-helusers +# 0.10.0 is needed because 1.0.0 switches to a more strict +# JWT validation library. Our "Tunnistamo"-implementation +# generates invalid amr claims in tokens. +drf-oidc-auth==0.10.0 pytz psycopg2 openpyxl @@ -7,7 +13,9 @@ pyxb djangorestframework django-cors-headers django-filter -django-allauth +django-admin-json-editor djangorestframework-jwt django-admin-sortable2 djangorestframework-xml +sentry-sdk +social-auth-app-django diff --git a/requirements.txt b/requirements.txt index 38635508..93073110 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,40 +2,151 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile requirements.in +# pip-compile # -certifi==2020.6.20 # via requests -chardet==3.0.4 # via requests -defusedxml==0.6.0 # via djangorestframework-xml, python3-openid -django-admin-sortable2==0.7.7 # via -r requirements.in -django-allauth==0.42.0 # via -r requirements.in -django-cors-headers==3.5.0 # via -r requirements.in -django-filter==2.3.0 # via -r requirements.in -django-helusers==0.5.6 # via -r requirements.in -django==2.2.16 # via -r requirements.in, django-admin-sortable2, django-allauth, django-cors-headers, django-filter, django-helusers, djangorestframework, drf-oidc-auth -djangorestframework-jwt==1.11.0 # via -r requirements.in -djangorestframework-xml==2.0.0 # via -r requirements.in -djangorestframework==3.11.1 # via -r requirements.in, drf-oidc-auth -drf-oidc-auth==0.10.0 # via django-helusers -ecdsa==0.14.1 # via python-jose -et-xmlfile==1.0.1 # via openpyxl -future==0.18.2 # via pyjwkest -idna==2.10 # via requests -jdcal==1.4.1 # via openpyxl -oauthlib==3.1.0 # via requests-oauthlib -openpyxl==3.0.5 # via -r requirements.in -psycopg2==2.8.6 # via -r requirements.in -pyasn1==0.4.8 # via python-jose, rsa -pycryptodomex==3.9.8 # via pyjwkest -pyjwkest==1.4.2 # via drf-oidc-auth -pyjwt==1.7.1 # via djangorestframework-jwt -python-jose==3.2.0 # via django-helusers -python3-openid==3.2.0 # via django-allauth -pytz==2020.1 # via -r requirements.in, django -pyxb==1.2.6 # via -r requirements.in -requests-oauthlib==1.3.0 # via django-allauth -requests==2.24.0 # via django-allauth, django-helusers, pyjwkest, requests-oauthlib -rsa==4.6 # via python-jose -six==1.15.0 # via ecdsa, pyjwkest, python-jose -sqlparse==0.3.1 # via django -urllib3==1.25.10 # via requests +asgiref==3.3.1 + # via django +certifi==2020.12.5 + # via + # elasticsearch + # requests + # sentry-sdk +cffi==1.14.4 + # via cryptography +chardet==4.0.0 + # via requests +cryptography==3.4.6 + # via social-auth-core +defusedxml==0.6.0 + # via + # djangorestframework-xml + # python3-openid + # social-auth-core +django-admin-json-editor==0.2.3 + # via -r requirements.in +django-admin-sortable2==0.7.7 + # via -r requirements.in +django-cors-headers==3.6.0 + # via -r requirements.in +django-elasticsearch-dsl-drf==0.20.9 + # via -r requirements.in +django-elasticsearch-dsl==7.1.4 + # via django-elasticsearch-dsl-drf +django-environ==0.4.5 + # via -r requirements.in +django-filter==2.4.0 + # via -r requirements.in +django-helusers==0.5.6 + # via -r requirements.in +django-nine==0.2.4 + # via django-elasticsearch-dsl-drf +django==3.1.8 + # via + # -r requirements.in + # django-admin-json-editor + # django-admin-sortable2 + # django-cors-headers + # django-filter + # django-helusers + # django-nine + # djangorestframework + # drf-oidc-auth +djangorestframework-jwt==1.11.0 + # via -r requirements.in +djangorestframework-xml==2.0.0 + # via -r requirements.in +djangorestframework==3.12.2 + # via + # -r requirements.in + # django-elasticsearch-dsl-drf + # drf-oidc-auth +drf-oidc-auth==0.10.0 + # via + # -r requirements.in + # django-helusers +ecdsa==0.14.1 + # via python-jose +elasticsearch-dsl==7.3.0 + # via + # django-elasticsearch-dsl + # django-elasticsearch-dsl-drf +elasticsearch==7.10.1 + # via + # django-elasticsearch-dsl-drf + # elasticsearch-dsl +et-xmlfile==1.0.1 + # via openpyxl +future==0.18.2 + # via pyjwkest +idna==2.10 + # via requests +jdcal==1.4.1 + # via openpyxl +oauthlib==3.1.0 + # via + # requests-oauthlib + # social-auth-core +openpyxl==3.0.5 + # via -r requirements.in +psycopg2==2.8.6 + # via -r requirements.in +pyasn1==0.4.8 + # via + # python-jose + # rsa +pycparser==2.20 + # via cffi +pycryptodomex==3.10.1 + # via pyjwkest +pyjwkest==1.4.2 + # via drf-oidc-auth +pyjwt==1.7.1 + # via + # djangorestframework-jwt + # social-auth-core +python-dateutil==2.8.1 + # via elasticsearch-dsl +python-jose==3.2.0 + # via django-helusers +python3-openid==3.2.0 + # via social-auth-core +pytz==2020.5 + # via + # -r requirements.in + # django +pyxb==1.2.6 + # via -r requirements.in +requests-oauthlib==1.3.0 + # via social-auth-core +requests==2.25.1 + # via + # django-helusers + # pyjwkest + # requests-oauthlib + # social-auth-core +rsa==4.7 + # via python-jose +sentry-sdk==0.19.5 + # via -r requirements.in +six==1.15.0 + # via + # django-elasticsearch-dsl + # django-elasticsearch-dsl-drf + # ecdsa + # elasticsearch-dsl + # pyjwkest + # python-dateutil + # python-jose + # social-auth-app-django + # social-auth-core +social-auth-app-django==4.0.0 + # via -r requirements.in +social-auth-core==4.0.2 + # via social-auth-app-django +sqlparse==0.4.1 + # via django +urllib3==1.26.4 + # via + # elasticsearch + # requests + # sentry-sdk diff --git a/search_indices/__init__.py b/search_indices/__init__.py new file mode 100644 index 00000000..b875c450 --- /dev/null +++ b/search_indices/__init__.py @@ -0,0 +1,56 @@ +from elasticsearch_dsl import analyzer, token_filter + +from search_indices.utils import create_elasticsearch_connection + + +def get_finnish_stop_filter() -> token_filter: + return token_filter("finnish_stop", type="stop", stopwords="_finnish_") + + +def get_finnish_stem_filter() -> token_filter: + return token_filter("finnish_stemmer", type="stemmer", language="finnish") + + +def get_edge_ngram_filter() -> token_filter: + return token_filter( + "custom_edge_ngram_filter", + type="edge_ngram", + min_gram=3, + max_gram=15, + ) + + +def get_voikko_filter() -> token_filter: + return token_filter( + "voikko", + type="voikko", + ) + + +def get_finnish_analyzer() -> analyzer: + es_connection = create_elasticsearch_connection() + if "voikko" in es_connection.cat.plugins(): + return analyzer( + "finnish_analyzer", + tokenizer="finnish", + filter=[ + "lowercase", + "asciifolding", + "unique", + get_edge_ngram_filter(), + get_voikko_filter(), + ], + ) + else: + return analyzer( + "finnish_analyzer", + tokenizer="standard", + filter=[ + "lowercase", + "asciifolding", + "unique", + get_edge_ngram_filter(), + get_finnish_stop_filter(), + get_finnish_stem_filter(), + ], + ) diff --git a/search_indices/backends/__init__.py b/search_indices/backends/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/search_indices/backends/faceted_attribute_backend.py b/search_indices/backends/faceted_attribute_backend.py new file mode 100644 index 00000000..187554fc --- /dev/null +++ b/search_indices/backends/faceted_attribute_backend.py @@ -0,0 +1,63 @@ +from typing import List, Type + +from django_elasticsearch_dsl import Document +from django_elasticsearch_dsl_drf.filter_backends.mixins import FilterBackendMixin +from elasticsearch_dsl.search import Search +from rest_framework.filters import BaseFilterBackend +from rest_framework.request import Request +from rest_framework.viewsets import ReadOnlyModelViewSet + +from search_indices.documents import ActionDocument, FunctionDocument, PhaseDocument, RecordDocument + +DOCUMENT_TYPES = [ + ActionDocument, + FunctionDocument, + PhaseDocument, + RecordDocument, +] + + +class FacetedAttributeBackend(BaseFilterBackend, FilterBackendMixin): + """Adds faceted search for attributes.""" + + faceted_search_param = "facet_attribute" + + @staticmethod + def get_attributes(documents: List[Type[Document]] = None) -> list: + if not documents: + documents = DOCUMENT_TYPES + attrs = [] + for document in documents: + model = document.Django.model + attributes = getattr(model, "_attribute_validations", None) + if attributes: + attrs += map( + lambda x: f"{str(model._meta.verbose_name)}_{x}", + attributes["allowed"], + ) + return attrs + + def filter_queryset( + self, request: Request, queryset: Search, view: Type[ReadOnlyModelViewSet] + ) -> Search: + """Filter the queryset. + :param request: Django REST framework request. + :param queryset: Base queryset. + :param view: View. + :type request: rest_framework.request.Request + :type queryset: elasticsearch_dsl.search.Search + :type view: rest_framework.viewsets.ReadOnlyModelViewSet + :return: Updated queryset. + :rtype: elasticsearch_dsl.search.Search + """ + attribute_validations = view.attributes + if attribute_validations: + for attribute in attribute_validations: + attribute_replaced = attribute.replace(".", "+") + # Example ("_attribute_Subject.Scheme", "terms", "attributes.Subject+Scheme.keyword") + queryset.aggs.bucket( + f"_attribute_{attribute}", + "terms", + field=f"attributes.{attribute_replaced}.keyword", + ) + return queryset diff --git a/search_indices/documents/__init__.py b/search_indices/documents/__init__.py new file mode 100644 index 00000000..f2b10ec9 --- /dev/null +++ b/search_indices/documents/__init__.py @@ -0,0 +1,5 @@ +from .action import ActionDocument # noqa +from .classification import ClassificationDocument # noqa +from .function import FunctionDocument # noqa +from .phase import PhaseDocument # noqa +from .record import RecordDocument # noqa diff --git a/search_indices/documents/action.py b/search_indices/documents/action.py new file mode 100644 index 00000000..4facf84c --- /dev/null +++ b/search_indices/documents/action.py @@ -0,0 +1,23 @@ +from django.conf import settings +from django_elasticsearch_dsl import Index + +from metarecord.models import Action +from search_indices import get_finnish_analyzer +from search_indices.documents.base import BaseDocument + +# Name of the Elasticsearch index +INDEX = Index(settings.ELASTICSEARCH_INDEX_NAMES[__name__]) + +finnish_analyzer = get_finnish_analyzer() + +INDEX.analyzer(finnish_analyzer) + +INDEX.settings( + max_result_window=500000, +) + + +@INDEX.document +class ActionDocument(BaseDocument): + class Django: + model = Action diff --git a/search_indices/documents/base.py b/search_indices/documents/base.py new file mode 100644 index 00000000..d7d23d20 --- /dev/null +++ b/search_indices/documents/base.py @@ -0,0 +1,106 @@ +from typing import Optional + +from django_elasticsearch_dsl import Document, fields + +from metarecord.models import Action, Classification, Function, Phase, StructuralElement +from search_indices import get_finnish_analyzer +from search_indices.documents.utils import prepare_attributes + +finnish_analyzer = get_finnish_analyzer() + + +class BaseDocument(Document): + id = fields.KeywordField(attr="uuid.hex") + + title = fields.TextField( + analyzer=finnish_analyzer, + fields={ + "keyword": fields.KeywordField(), + }, + ) + + description = fields.TextField( + analyzer=finnish_analyzer, + fields={ + "keyword": fields.KeywordField(), + }, + ) + + description_internal = fields.TextField( + analyzer=finnish_analyzer, + fields={ + "keyword": fields.KeywordField(), + }, + ) + + additional_information = fields.TextField( + analyzer=finnish_analyzer, + fields={ + "keyword": fields.KeywordField(), + }, + ) + + state = fields.KeywordField() + + code = fields.KeywordField() + + type = fields.IntegerField() + + parent = fields.ObjectField( + properties={ + "id": fields.KeywordField(attr="uuid.hex"), + "title": fields.KeywordField(), + "version": fields.IntegerField(), + } + ) + + attributes = fields.ObjectField(dynamic=True) + + def prepare_title(self, obj: StructuralElement) -> Optional[str]: + title = getattr(obj, "title", None) + if not title: + title = getattr(obj, "name", None) + return title + + def prepare_description(self, obj: StructuralElement) -> Optional[str]: + return getattr(obj, "description", None) + + def prepare_description_internal(self, obj: StructuralElement) -> Optional[str]: + return getattr(obj, "description_internal", None) + + def prepare_additional_information(self, obj: StructuralElement) -> Optional[str]: + return getattr(obj, "additional_information", None) + + def prepare_state(self, obj: StructuralElement) -> Optional[str]: + return getattr(obj, "state", None) + + def prepare_code(self, obj: StructuralElement) -> Optional[str]: + return getattr(obj, "code", None) + + def prepare_type(self, obj: StructuralElement) -> int: + obj_type = type(obj) + if obj_type == Classification: + return 1 + elif obj_type == Function: + return 2 + elif obj_type == Phase: + return 3 + elif obj_type == Action: + return 4 + return 5 + + def prepare_parent(self, obj: StructuralElement) -> Optional[dict]: + parent = getattr(obj, "parent", None) + if parent: + return { + "id": parent.uuid.hex, + "title": self.prepare_title(parent), + "version": getattr(parent, "version", None), + } + return None + + def prepare_attributes(self, obj: Function) -> Optional[dict]: + if hasattr(obj, "attributes"): + model_name = str(self.Django.model._meta.verbose_name) + return prepare_attributes(obj, prefix=f"{model_name}_") + return None diff --git a/search_indices/documents/classification.py b/search_indices/documents/classification.py new file mode 100644 index 00000000..5002d376 --- /dev/null +++ b/search_indices/documents/classification.py @@ -0,0 +1,27 @@ +from django.conf import settings +from django.db.models import QuerySet +from django_elasticsearch_dsl import Index + +from metarecord.models import Classification +from search_indices import get_finnish_analyzer +from search_indices.documents.base import BaseDocument + +# Name of the Elasticsearch index +INDEX = Index(settings.ELASTICSEARCH_INDEX_NAMES[__name__]) + +finnish_analyzer = get_finnish_analyzer() + +INDEX.analyzer(finnish_analyzer) + +INDEX.settings( + max_result_window=500000, +) + + +@INDEX.document +class ClassificationDocument(BaseDocument): + class Django: + model = Classification + + def get_queryset(self) -> QuerySet: + return Classification.objects.latest_version() diff --git a/search_indices/documents/function.py b/search_indices/documents/function.py new file mode 100644 index 00000000..c11db81a --- /dev/null +++ b/search_indices/documents/function.py @@ -0,0 +1,27 @@ +from django.conf import settings +from django.db.models import QuerySet +from django_elasticsearch_dsl import Index + +from metarecord.models import Function +from search_indices import get_finnish_analyzer +from search_indices.documents.base import BaseDocument + +# Name of the Elasticsearch index +INDEX = Index(settings.ELASTICSEARCH_INDEX_NAMES[__name__]) + +finnish_analyzer = get_finnish_analyzer() + +INDEX.analyzer(finnish_analyzer) + +INDEX.settings( + max_result_window=500000, +) + + +@INDEX.document +class FunctionDocument(BaseDocument): + class Django: + model = Function + + def get_queryset(self) -> QuerySet: + return Function.objects.latest_version() diff --git a/search_indices/documents/phase.py b/search_indices/documents/phase.py new file mode 100644 index 00000000..0d846c90 --- /dev/null +++ b/search_indices/documents/phase.py @@ -0,0 +1,23 @@ +from django.conf import settings +from django_elasticsearch_dsl import Index + +from metarecord.models import Phase +from search_indices import get_finnish_analyzer +from search_indices.documents.base import BaseDocument + +# Name of the Elasticsearch index +INDEX = Index(settings.ELASTICSEARCH_INDEX_NAMES[__name__]) + +finnish_analyzer = get_finnish_analyzer() + +INDEX.analyzer(finnish_analyzer) + +INDEX.settings( + max_result_window=500000, +) + + +@INDEX.document +class PhaseDocument(BaseDocument): + class Django: + model = Phase diff --git a/search_indices/documents/record.py b/search_indices/documents/record.py new file mode 100644 index 00000000..a470939b --- /dev/null +++ b/search_indices/documents/record.py @@ -0,0 +1,23 @@ +from django.conf import settings +from django_elasticsearch_dsl import Index + +from metarecord.models import Record +from search_indices import get_finnish_analyzer +from search_indices.documents.base import BaseDocument + +# Name of the Elasticsearch index +INDEX = Index(settings.ELASTICSEARCH_INDEX_NAMES[__name__]) + +finnish_analyzer = get_finnish_analyzer() + +INDEX.analyzer(finnish_analyzer) + +INDEX.settings( + max_result_window=500000, +) + + +@INDEX.document +class RecordDocument(BaseDocument): + class Django: + model = Record diff --git a/search_indices/documents/utils.py b/search_indices/documents/utils.py new file mode 100644 index 00000000..6b53d565 --- /dev/null +++ b/search_indices/documents/utils.py @@ -0,0 +1,16 @@ +from metarecord.models.structural_element import StructuralElement + + +def prepare_attributes(obj: StructuralElement, prefix: str = "") -> dict: + """ + ElasticSearch cannot handle attribute names with dots, for example "Subject.Scheme". + This is because it tries to interpret Scheme as an attribute for Subject. + Fix this by replacing "." with "+" when indexing the Documents. + """ + attributes = {} + if obj: + attrs = obj.attributes + for key, value in attrs.items(): + key = key.replace(".", "+") + attributes[prefix + key] = str(value) + return attributes diff --git a/search_indices/serializers/__init__.py b/search_indices/serializers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/search_indices/serializers/action.py b/search_indices/serializers/action.py new file mode 100644 index 00000000..965e4db7 --- /dev/null +++ b/search_indices/serializers/action.py @@ -0,0 +1,7 @@ +from search_indices.documents import ActionDocument +from search_indices.serializers.base import BaseSearchSerializer + + +class ActionSearchSerializer(BaseSearchSerializer): + class Meta: + document = ActionDocument diff --git a/search_indices/serializers/base.py b/search_indices/serializers/base.py new file mode 100644 index 00000000..5c98efa6 --- /dev/null +++ b/search_indices/serializers/base.py @@ -0,0 +1,34 @@ +from typing import Optional + +from django_elasticsearch_dsl_drf.serializers import DocumentSerializer +from elasticsearch_dsl.response.hit import Hit +from rest_framework import serializers + +from search_indices.serializers.utils import get_attributes + + +class BaseSearchSerializer(DocumentSerializer): + attributes = serializers.SerializerMethodField() + + score = serializers.SerializerMethodField() + + class Meta: + fields = ( + "id", + "title", + "description", + "description_internal", + "additional_information", + "state", + "code", + "type", + "parent", + "attributes", + "score", + ) + + def get_score(self, obj: Hit) -> int: + return obj.meta.score + + def get_attributes(self, obj: Hit) -> Optional[dict]: + return get_attributes(obj, "attributes") diff --git a/search_indices/serializers/classification.py b/search_indices/serializers/classification.py new file mode 100644 index 00000000..1569fb15 --- /dev/null +++ b/search_indices/serializers/classification.py @@ -0,0 +1,7 @@ +from search_indices.documents import ClassificationDocument +from search_indices.serializers.base import BaseSearchSerializer + + +class ClassificationSearchSerializer(BaseSearchSerializer): + class Meta: + document = ClassificationDocument diff --git a/search_indices/serializers/function.py b/search_indices/serializers/function.py new file mode 100644 index 00000000..6238154b --- /dev/null +++ b/search_indices/serializers/function.py @@ -0,0 +1,7 @@ +from search_indices.documents import FunctionDocument +from search_indices.serializers.base import BaseSearchSerializer + + +class FunctionSearchSerializer(BaseSearchSerializer): + class Meta: + document = FunctionDocument diff --git a/search_indices/serializers/phase.py b/search_indices/serializers/phase.py new file mode 100644 index 00000000..7b8f7f4f --- /dev/null +++ b/search_indices/serializers/phase.py @@ -0,0 +1,7 @@ +from search_indices.documents import PhaseDocument +from search_indices.serializers.base import BaseSearchSerializer + + +class PhaseSearchSerializer(BaseSearchSerializer): + class Meta: + document = PhaseDocument diff --git a/search_indices/serializers/record.py b/search_indices/serializers/record.py new file mode 100644 index 00000000..efd3c127 --- /dev/null +++ b/search_indices/serializers/record.py @@ -0,0 +1,7 @@ +from search_indices.documents import RecordDocument +from search_indices.serializers.base import BaseSearchSerializer + + +class RecordSearchSerializer(BaseSearchSerializer): + class Meta: + document = RecordDocument diff --git a/search_indices/serializers/utils.py b/search_indices/serializers/utils.py new file mode 100644 index 00000000..eb17d972 --- /dev/null +++ b/search_indices/serializers/utils.py @@ -0,0 +1,19 @@ +from typing import Optional + +from elasticsearch_dsl.response.hit import Hit + + +def get_attributes(obj: Hit, attribute_field_name: str) -> Optional[dict]: + """ + Fetch attributes from index and revert the attribute names that + have "." replaced with "+". + """ + attrs = getattr(obj, attribute_field_name) + if attrs: + attributes = {} + attrs = attrs.to_dict() + for key, value in attrs.items(): + key = key.replace("+", ".") + attributes[key] = value + return attributes + return None diff --git a/search_indices/tests/__init__.py b/search_indices/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/search_indices/tests/conftest.py b/search_indices/tests/conftest.py new file mode 100644 index 00000000..0954ef65 --- /dev/null +++ b/search_indices/tests/conftest.py @@ -0,0 +1,77 @@ +from elasticsearch.helpers.test import get_test_client +from elasticsearch_dsl.connections import add_connection +from pytest import fixture + +from metarecord.models import Action, Classification, Function, Phase, Record +from metarecord.tests.conftest import user, user_api_client # noqa +from search_indices.documents import ( + ActionDocument, ClassificationDocument, FunctionDocument, PhaseDocument, RecordDocument +) + + +@fixture(scope="session", autouse=True) +def create_indices(): + """ + Initialize all indices with the custom analyzers. + """ + ActionDocument._index.create(ignore=[400, 404]) + ClassificationDocument._index.create(ignore=[400, 404]) + FunctionDocument._index.create(ignore=[400, 404]) + PhaseDocument._index.create(ignore=[400, 404]) + RecordDocument._index.create(ignore=[400, 404]) + + +@fixture(scope="session") +def es_connection(): + es_connection = get_test_client() + add_connection("default", es_connection) + yield es_connection + + +@fixture +def action(phase): + return Action.objects.create( + attributes={"AdditionalInformation": "testisana"}, phase=phase, index=1 + ) + + +@fixture +def classification(): + return Classification.objects.create( + title="testisana", + code="00 00", + state=Classification.APPROVED, + function_allowed=True, + ) + + +@fixture +def classification2(): + return Classification.objects.create( + title="testisana ja toinen testisana", + code="00 00", + state=Classification.APPROVED, + function_allowed=True, + ) + + +@fixture +def function(classification): + return Function.objects.create( + attributes={"AdditionalInformation": "testisana"}, + classification=classification, + ) + + +@fixture +def phase(function): + return Phase.objects.create( + attributes={"AdditionalInformation": "testisana"}, function=function, index=1 + ) + + +@fixture +def record(action): + return Record.objects.create( + attributes={"AdditionalInformation": "testisana"}, action=action, index=1 + ) diff --git a/search_indices/tests/test_elastic_api.py b/search_indices/tests/test_elastic_api.py new file mode 100644 index 00000000..6b17f845 --- /dev/null +++ b/search_indices/tests/test_elastic_api.py @@ -0,0 +1,116 @@ +import pytest +from rest_framework.reverse import reverse + +ACTION_LIST_URL = reverse("action_search-list") +ALL_LIST_URL = reverse("all_search-list") +CLASSIFICATION_LIST_URL = reverse("classification_search-list") +FUNCTION_LIST_URL = reverse("function_search-list") +PHASE_LIST_URL = reverse("phase_search-list") +RECORD_LIST_URL = reverse("record_search-list") + + +@pytest.mark.django_db +def test_classification_search_exact(user_api_client, classification): + url = ALL_LIST_URL + "?search=testisana" + response = user_api_client.get(url) + assert response.status_code == 200 + + results = response.data["results"] if "results" in response.data else response.data + uuids = list(result["id"] for result in results) + assert classification.uuid.hex in uuids + + +@pytest.mark.django_db +def test_classification_search_fuzzy1(user_api_client, es_connection, classification): + if "voikko" not in str(es_connection.indices.get_settings()): + pytest.skip() + url = ALL_LIST_URL + "?search=testi" + response = user_api_client.get(url) + assert response.status_code == 200 + + results = response.data["results"] if "results" in response.data else response.data + uuids = list(result["id"] for result in results) + assert classification.uuid.hex in uuids + + +@pytest.mark.django_db +def test_classification_search_fuzzy2(user_api_client, es_connection, classification): + if "voikko" not in str(es_connection.indices.get_settings()): + pytest.skip() + url = ALL_LIST_URL + "?search=testisanojat" + response = user_api_client.get(url) + assert response.status_code == 200 + + results = response.data["results"] if "results" in response.data else response.data + uuids = list(result["id"] for result in results) + assert classification.uuid.hex in uuids + + +@pytest.mark.django_db +def test_classification_search_query_string(user_api_client, classification2): + url = ALL_LIST_URL + '?search_simple_query_string="testisana ja toinen testisana"' + response = user_api_client.get(url) + assert response.status_code == 200 + + results = response.data["results"] if "results" in response.data else response.data + uuids = list(result["id"] for result in results) + assert classification2.uuid.hex in uuids + + +@pytest.mark.django_db +def test_action_filter_attribute_exact(user_api_client, action): + url = ACTION_LIST_URL + "?action_AdditionalInformation=testisana" + response = user_api_client.get(url) + assert response.status_code == 200 + + results = response.data["results"] if "results" in response.data else response.data + uuids = list(result["id"] for result in results) + assert action.uuid.hex in uuids + + +@pytest.mark.django_db +def test_classification_filter_title_exact( + user_api_client, es_connection, classification +): + if "voikko" not in str(es_connection.indices.get_settings()): + pytest.skip() + url = CLASSIFICATION_LIST_URL + f"?title=testisana" + response = user_api_client.get(url) + assert response.status_code == 200 + + results = response.data["results"] if "results" in response.data else response.data + uuids = list(result["id"] for result in results) + assert classification.uuid.hex in uuids + + +@pytest.mark.django_db +def test_function_filter_attribute_exact(user_api_client, function): + url = FUNCTION_LIST_URL + "?function_AdditionalInformation=testisana" + response = user_api_client.get(url) + assert response.status_code == 200 + + results = response.data["results"] if "results" in response.data else response.data + uuids = list(result["id"] for result in results) + assert function.uuid.hex in uuids + + +@pytest.mark.django_db +def test_phase_filter_attribute_exact(user_api_client, phase): + url = PHASE_LIST_URL + "?phase_AdditionalInformation=testisana" + response = user_api_client.get(url) + assert response.status_code == 200 + + results = response.data["results"] if "results" in response.data else response.data + uuids = list(result["id"] for result in results) + assert phase.uuid.hex in uuids + + +@pytest.mark.django_db +def test_record_filter_attribute_exact(user_api_client, record): + url = RECORD_LIST_URL + "?record_AdditionalInformation=testisana" + response = user_api_client.get(url) + assert response.status_code == 200 + + results = response.data["results"] if "results" in response.data else response.data + uuids = list(result["id"] for result in results) + assert record.uuid.hex in uuids diff --git a/search_indices/utils.py b/search_indices/utils.py new file mode 100644 index 00000000..8958832a --- /dev/null +++ b/search_indices/utils.py @@ -0,0 +1,7 @@ +from django.conf import settings +from elasticsearch_dsl import connections + + +def create_elasticsearch_connection(): + es_host = settings.ELASTICSEARCH_DSL["default"]["hosts"] + return connections.create_connection(hosts=[es_host]) diff --git a/search_indices/views/__init__.py b/search_indices/views/__init__.py new file mode 100644 index 00000000..1e51ef54 --- /dev/null +++ b/search_indices/views/__init__.py @@ -0,0 +1,6 @@ +from .action import ActionSearchDocumentViewSet # noqa +from .all import AllSearchDocumentViewSet # noqa +from .classification import ClassificationSearchDocumentViewSet # noqa +from .function import FunctionSearchDocumentViewSet # noqa +from .phase import PhaseSearchDocumentViewSet # noqa +from .record import RecordSearchDocumentViewSet # noqa diff --git a/search_indices/views/action.py b/search_indices/views/action.py new file mode 100644 index 00000000..293516c6 --- /dev/null +++ b/search_indices/views/action.py @@ -0,0 +1,16 @@ +from search_indices.backends.faceted_attribute_backend import FacetedAttributeBackend +from search_indices.documents.action import ActionDocument +from search_indices.serializers.action import ActionSearchSerializer +from search_indices.views.base import BaseSearchDocumentViewSet +from search_indices.views.utils import populate_filter_fields_with_attributes + + +class ActionSearchDocumentViewSet(BaseSearchDocumentViewSet): + document = ActionDocument + serializer_class = ActionSearchSerializer + + filter_fields = {} + attributes = FacetedAttributeBackend.get_attributes([ActionDocument]) + populate_filter_fields_with_attributes(filter_fields, attributes) + + search_fields = tuple(f"attributes.{attribute}" for attribute in attributes) diff --git a/search_indices/views/all.py b/search_indices/views/all.py new file mode 100644 index 00000000..63b4e07d --- /dev/null +++ b/search_indices/views/all.py @@ -0,0 +1,23 @@ +from django.conf import settings +from elasticsearch_dsl import Search + +from search_indices.documents.action import ActionDocument +from search_indices.serializers.action import ActionSearchSerializer +from search_indices.views.base import BaseSearchDocumentViewSet + + +class AllSearchDocumentViewSet(BaseSearchDocumentViewSet): + document = ActionDocument # This needs to be filled with a valid Document + serializer_class = ( + ActionSearchSerializer # This needs to be filled with a valid Serializer + ) + + def __init__(self, *args, **kwargs): + super(AllSearchDocumentViewSet, self).__init__(*args, **kwargs) + + self.search = Search( + using=self.client, + index=list(settings.ELASTICSEARCH_INDEX_NAMES.values()), + doc_type=self.document._doc_type.name, + ).sort(*self.ordering) + self.search.params(preserve_order=False) diff --git a/search_indices/views/base.py b/search_indices/views/base.py new file mode 100644 index 00000000..c1b7e1d4 --- /dev/null +++ b/search_indices/views/base.py @@ -0,0 +1,67 @@ +from django_elasticsearch_dsl_drf.filter_backends import ( + CompoundSearchFilterBackend, FacetedSearchFilterBackend, FilteringFilterBackend, + SimpleQueryStringSearchFilterBackend +) +from django_elasticsearch_dsl_drf.viewsets import BaseDocumentViewSet + +from metarecord.pagination import ESRecordPagination +from search_indices.backends.faceted_attribute_backend import FacetedAttributeBackend +from search_indices.views.utils import populate_filter_fields_with_attributes + + +class BaseSearchDocumentViewSet(BaseDocumentViewSet): + pagination_class = ESRecordPagination + lookup_field = "id" + filter_backends = [ + CompoundSearchFilterBackend, + FacetedAttributeBackend, + FacetedSearchFilterBackend, + FilteringFilterBackend, + SimpleQueryStringSearchFilterBackend, + ] + + faceted_search_fields = { + "title": { + "field": "title.keyword", + "enabled": True, + }, + "description": { + "field": "description.keyword", + "enabled": True, + }, + "related_classification": { + "field": "related_classification.keyword", + "enabled": True, + }, + "internal_description": { + "field": "internal_description.keyword", + "enabled": True, + }, + "additional_information": { + "field": "additional_information.keyword", + "enabled": True, + }, + "type": { + "field": "type", + "enabled": True, + }, + } + + filter_fields = {} + attributes = FacetedAttributeBackend.get_attributes() + populate_filter_fields_with_attributes(filter_fields, attributes) + + search_fields = ( + "title", + "description", + "related_classification", + "internal_description", + "additional_information", + ) + + search_fields += tuple(f"attributes.{attribute}" for attribute in attributes) + + ordering = ( + "type", + "_score", + ) diff --git a/search_indices/views/classification.py b/search_indices/views/classification.py new file mode 100644 index 00000000..f139a691 --- /dev/null +++ b/search_indices/views/classification.py @@ -0,0 +1,24 @@ +from search_indices.documents.classification import ClassificationDocument +from search_indices.serializers.classification import ClassificationSearchSerializer +from search_indices.views.base import BaseSearchDocumentViewSet + + +class ClassificationSearchDocumentViewSet(BaseSearchDocumentViewSet): + document = ClassificationDocument + serializer_class = ClassificationSearchSerializer + + filter_fields = { + "title": "title", + "description": "description", + "related_classification": "related_classification", + "internal_description": "internal_description", + "additional_information": "additional_information", + } + + search_fields = ( + "title", + "description", + "related_classification", + "internal_description", + "additional_information", + ) diff --git a/search_indices/views/function.py b/search_indices/views/function.py new file mode 100644 index 00000000..2d8a1046 --- /dev/null +++ b/search_indices/views/function.py @@ -0,0 +1,16 @@ +from search_indices.backends.faceted_attribute_backend import FacetedAttributeBackend +from search_indices.documents.function import FunctionDocument +from search_indices.serializers.function import FunctionSearchSerializer +from search_indices.views.base import BaseSearchDocumentViewSet +from search_indices.views.utils import populate_filter_fields_with_attributes + + +class FunctionSearchDocumentViewSet(BaseSearchDocumentViewSet): + document = FunctionDocument + serializer_class = FunctionSearchSerializer + + filter_fields = {} + attributes = FacetedAttributeBackend.get_attributes([FunctionDocument]) + populate_filter_fields_with_attributes(filter_fields, attributes) + + search_fields = tuple(f"attributes.{attribute}" for attribute in attributes) diff --git a/search_indices/views/phase.py b/search_indices/views/phase.py new file mode 100644 index 00000000..9b41f826 --- /dev/null +++ b/search_indices/views/phase.py @@ -0,0 +1,16 @@ +from search_indices.backends.faceted_attribute_backend import FacetedAttributeBackend +from search_indices.documents.phase import PhaseDocument +from search_indices.serializers.phase import PhaseSearchSerializer +from search_indices.views.base import BaseSearchDocumentViewSet +from search_indices.views.utils import populate_filter_fields_with_attributes + + +class PhaseSearchDocumentViewSet(BaseSearchDocumentViewSet): + document = PhaseDocument + serializer_class = PhaseSearchSerializer + + filter_fields = {} + attributes = FacetedAttributeBackend.get_attributes([PhaseDocument]) + populate_filter_fields_with_attributes(filter_fields, attributes) + + search_fields = tuple(f"attributes.phase_{attribute}" for attribute in attributes) diff --git a/search_indices/views/record.py b/search_indices/views/record.py new file mode 100644 index 00000000..c07fdeb5 --- /dev/null +++ b/search_indices/views/record.py @@ -0,0 +1,16 @@ +from search_indices.backends.faceted_attribute_backend import FacetedAttributeBackend +from search_indices.documents.record import RecordDocument +from search_indices.serializers.record import RecordSearchSerializer +from search_indices.views.base import BaseSearchDocumentViewSet +from search_indices.views.utils import populate_filter_fields_with_attributes + + +class RecordSearchDocumentViewSet(BaseSearchDocumentViewSet): + document = RecordDocument + serializer_class = RecordSearchSerializer + + filter_fields = {} + attributes = FacetedAttributeBackend.get_attributes([RecordDocument]) + populate_filter_fields_with_attributes(filter_fields, attributes) + + search_fields = tuple(f"attributes.{attribute}" for attribute in attributes) diff --git a/search_indices/views/utils.py b/search_indices/views/utils.py new file mode 100644 index 00000000..b59e2738 --- /dev/null +++ b/search_indices/views/utils.py @@ -0,0 +1,12 @@ +def populate_filter_fields_with_attributes( + filter_fields: dict, attributes: list +) -> None: + """ + Fetch the filter fields from the indexed attributes. + """ + if not attributes: + return + for attribute in attributes: + attribute_replaced = attribute.replace(".", "+") + # Example {"Subject.Scheme": "attributes.Subject+Scheme"} + filter_fields[attribute] = f"attributes.{attribute_replaced}" diff --git a/users/migrations/0004_longer_first_name.py b/users/migrations/0004_longer_first_name.py new file mode 100644 index 00000000..0a242b20 --- /dev/null +++ b/users/migrations/0004_longer_first_name.py @@ -0,0 +1,20 @@ +# Generated by Django 3.1.5 on 2021-01-11 09:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0003_longer_last_name"), + ] + + operations = [ + migrations.AlterField( + model_name="user", + name="first_name", + field=models.CharField( + blank=True, max_length=150, verbose_name="first name" + ), + ), + ] diff --git a/users/migrations/0005_transfer_to_pysocial.py b/users/migrations/0005_transfer_to_pysocial.py new file mode 100644 index 00000000..213bf190 --- /dev/null +++ b/users/migrations/0005_transfer_to_pysocial.py @@ -0,0 +1,42 @@ +from django.db import connection, migrations +from django.db.utils import IntegrityError + + +def transfer_users_from_allauth_to_pysocial(apps, schema_editor): + """ + Transfer user UIDs from Allauth to Python Social Auth. + Has to be done in raw SQL since Django does not recognize allauth after removing it from the project. + """ + all_tables = connection.introspection.table_names() + if "socialaccount_socialaccount" in all_tables: + try: + with connection.cursor() as cursor: + cursor.execute( + "INSERT INTO social_auth_usersocialauth (user_id, provider, uid, extra_data, created, modified)" + "SELECT user_id, 'tunnistamo', uid, extra_data, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP FROM socialaccount_socialaccount;" + ) + except IntegrityError: + connection._rollback() + + +def remove_sessions(apps, schema_editor): + """ + This needs to be done to get rid of the old sessions because of the + added setting SESSION_SERIALIZER. + """ + Session = apps.get_model("sessions", "Session") + Session.objects.all().delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0004_longer_first_name"), + ] + + operations = [ + migrations.RunPython( + transfer_users_from_allauth_to_pysocial, migrations.RunPython.noop, + ), + migrations.RunPython(remove_sessions, migrations.RunPython.noop,), + ]