Merge pull request #319 from City-of-Helsinki/develop

release 0.4.12

.gitignore

@@ -14,3 +14,4 @@ htmlcov
 .DS_Store
 .pytest_cache/
 export/
+config_dev.env

.travis.yml

@@ -15,19 +15,27 @@ addons:
     - postgresql-10
     - postgresql-client-10
 
+env:
+  - ELASTICSEARCH_HOST: localhost:9200
+before_install:
+  - curl https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.3.2-amd64.deb -o elasticsearch.deb
+  - sudo dpkg -i --force-confnew elasticsearch.deb
+  - sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch
+  - sudo service elasticsearch restart
 before_script:
-  - psql template1 -c 'create extension hstore;'
+  - sleep 10 # Needed for ElasticSearch
 install:
 - pip install --upgrade pip
 - pip install coveralls -r requirements.txt -r dev-requirements.txt
 script:
   - flake8
-  - pytest -ra -vvv
+  - pytest -ra -vvv --cov=.
 before_cache:
 - rm -f $HOME/.cache/pip/log/debug.log
 
 notifications:
   slack:
     secure: gm6+6ekPIw0WcNuul94MoAzsAJ1/rlP0++UdnB01uf5boXIJiQjKZn+BVyhpMX1CN3KQFyf2tXEsMj2AY8IEF0YzJJXc9+ae70p+5OQXInLUcea7SZJ/7q7Tuw2AJoXUxbDmoto83N828waEIdWjCKW5qCxM248+FG9wKNipkjswv/obASOBzlhGQ67kzRaUpCsCHYlpbgxhlg1lPZs32vKL9YOtjCjyCerxnE+SIomANE+djpd8eGFUz90SEcfR5ypGHeuIic4xsX6VhGHyzevfgEix5aq3QDiNSbH8GWClyMeiU82ov3dEsRvMheRH9vRYl6xzdKuAgWRfW61biApki8sPn2w7W6AMtD0MaHMNi3QsQVW4z0saKNICtvi7ZQXtw8DvajEYLn1GuyF6hBUh9LEoqFXN52HlBVjZ/0XnCPyCdyQy03u3pOxz/OQ1/9gOH2SQMkQUqMYbiHeELXg4KHnPpvfgNBmFAA8WybbAzLXAqF2/67nMiNOi1a0oHOMfHlEAFnxITzYcU4eQxfapHHzXdfObZXwcna7T6RXdCVEX5LmOUN35oJkKe4y4T3ngumD8thbVX07HDYrbAaeE85JoT5ok5xe7WHimf2XTuOUNogNALnsop19EROrHcRB6d88BluVoS5S3e6uS7SliUCjH2HqMkDMR0T5zRy0=
 
-after_success: coveralls
+after_success:
+  - pip install codecov && codecov

README.md

@@ -5,7 +5,7 @@
 ## Prerequisites
 
 - Python 3.6
-- PostgreSQL 10.6
+- PostgreSQL 10.x
 
 ## Installation
 
@@ -19,7 +19,9 @@
 pip install -r requirements.txt
 ```
 
-- Create `local_settings.py` file in the project root and use it to override settings as needed.
+- Copy `config_dev.env.example` to `config_dev.env` and edit according
+  to your needs. DATABASE_URL is the one setting that you will need to
+  uncomment and possibly edit.
 
 - Create a database
 
@@ -48,29 +50,19 @@ python manage.py createsuperuser
 
 ### Docker compose setup
 
-- Build and start the containers. The `--build` flag is needed only for first time setup **or** if something in the Dockerfile changes.
+- Copy `config_dev.env.example` to `config_dev.env` and edit according
+  to your needs. The file is copiously commented.
 
 ```
-docker-compose up --build
+cp config_dev.env.example config_dev.env
 ```
 
-- Create `local_settings.py` to checkout root
+- Build and start the containers. Docker-compose will automatically build
+  the container if it is missing. By default the container initializes
+  database and starts Django dev server.
 
-```python
-DEBUG = True
-
-SECRET_KEY = 'xxx'
-
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.postgresql',
-        'NAME': 'helerm',
-        'USER': 'root',
-        'PASSWORD': 'root',
-        'PORT': 5432,
-        'HOST': 'helerm_postgres96',
-    }
-}
+```
+docker-compose up
 ```
 
 - Access the application container shell
@@ -79,12 +71,6 @@ DATABASES = {
 docker-compose exec django bash
 ```
 
-- Enable necessary postgresql extensions for the database
-
-```
-echo 'CREATE EXTENSION hstore; \q' | ./manage.py dbshell
-```
-
 - Run `migrate`, `compilemessages`, and `createsuperuser` as usual. Detailed info in manual setup steps.
 
 ## Development
@@ -153,7 +139,7 @@ python manage.py export_data <xml file>
 
 The export uses [pyxb](http://pyxb.sourceforge.net/) library and needs Python bindings to be generated from XSD schema files.
 
-The repo contains two sets of JHS XML Schema files located in `data` directory. In addition to original ones, there are also HKI customized versions, which are in use at least for now. 
+The repo contains two sets of JHS XML Schema files located in `data` directory. In addition to original ones, there are also HKI customized versions, which are in use at least for now.
 
 Generated bindings are included in `metarecord/binding/` so the export should work out of the box.
 

config_dev.env.example

@@ -0,0 +1,162 @@
+# HelERM environment configuration
+# This file defines a set of (environment) variables that configure most
+# of the functionality of helerm. In order for helerm to read
+# this file, rename it to `config_dev.env`. As the name implies, this
+# file is supposed to be used only in development. For production use
+# we recommend setting the environment variables using the facilities
+# of your runtime environment.
+# HelERM reads this file by itself. However, it can also be
+# used in conjunction with the included docker-compose.yml. Then
+# you don't need to inject the file into the container. Instead
+# Docker defines environment variables that helerm will read.
+# Following are the settings and their explanations, with example values
+# that might be useful for development:
+
+# HelERM generates a JHS191-compliant XML export for consumption by
+# other systems. This sets the human readable description fields for
+# that document.
+# HELERM_JHS191_EXPORT_DESCRIPTION="Export from development environment"
+
+# Whether to run Django in debug mode
+# Django setting: DEBUG https://docs.djangoproject.com/en/3.0/ref/settings/#debug
+DEBUG=True
+
+# Helsinki user management library (django-helusers) brings this in as
+# a generic setting for indicating the "mode" of the site. It can be
+# either 'dev', 'test' or 'production'. Only effect in HelERM is
+# changing header color in Django admin
+# Does not correspond to a standard Django setting
+HEL_SITE_TYPE=dev
+
+# Level of Django logging. All events above the given level will be logged.
+# Django setting: DJANGO_LOG_LEVEL https://docs.djangoproject.com/en/3.0/topics/logging/#examples
+# DJANGO_LOG_LEVEL=INFO
+
+# Configures database for HelERM using URL style. Format is:
+# postgres://USER:PASSWORD@HOST:PORT/NAME
+# Unused components may be left out, only Postgres is supported
+# The example below configures HelERM to use local PostgreSQL database
+# called "helerm", connecting same as username as Django is running as.
+# Django setting: DATABASES (but not directly) https://docs.djangoproject.com/en/3.0/ref/settings/#databases
+# DATABASE_URL=postgres:///helerm
+
+# HelERM will use JWT tokens for authentication. This settings Specifies
+# the value that must be present in the "aud"-key of the token presented
+# by a client when making an authenticated request. HelERM uses this
+# key for verifying that the token was meant for accessing this particular
+# HelERM instance (the tokens are signed, see below).
+# Does not correspond to standard Django setting
+#TOKEN_AUTH_ACCEPTED_AUDIENCE=string-identifying-this-helerm-instance
+
+# This key will be used by HelERM to verify the JWT token is from trusted
+# Identity Provider (OpenID terminology). The provider must have signed
+# the JWT TOKEN using this shared secret
+# Does not correspond to standard Django setting
+#TOKEN_AUTH_SHARED_SECRET=abcdefghacbdefgabcdefghacbdefgabcdefghacbdefgabcdefghacbdefgabcdefghacbdefg
+
+# Secret used for various functions within Django. This setting is
+# mandatory for Django, but HelERM will generate an ephemeral key,
+# if it is not defined here. Currently HelERM does not use any
+# functionality that needs this.
+# Django setting: SECRET_KEY https://docs.djangoproject.com/en/3.0/ref/settings/#secret-key
+#SECRET_KEY=gaehgoes89yrtp9384ygpe9r8ahgaerui8ghpae98rgh
+
+# List of Host-values, that HelERM will accept in requests.
+# This setting is a Django protection measure against HTTP Host-header attacks
+# https://docs.djangoproject.com/en/3.0/topics/security/#host-headers-virtual-hosting
+# Specified as a comma separated list of allowed values. Note that this does
+# NOT matter if you are running with DEBUG
+# Django setting: ALLOWED_HOSTS https://docs.djangoproject.com/en/3.0/ref/settings/#allowed-hosts
+#ALLOWED_HOSTS=api.hel.ninja,helerm-api.hel.ninja
+
+# List of tuples (or just e-mail addresses) specifying Administrators of this
+# HelERM instance. Django uses this only when logging is configured to
+# send exceptions to admins. HelERM does not do this. Still you may want
+# to set this for documentation
+# Django setting: ADMINS https://docs.djangoproject.com/en/3.0/ref/settings/#admins
+[email protected],[email protected]
+
+# Whether cookies set by Django are only allowed to be sent over
+# secure (HTTPS) connection. This sets the "Secure" tag on the Set-Cookie
+# header. For development, you will likely to want to set this to False.
+# On server you will most certainly want to set this to True.
+# Typical failure mode is when admin login returns back to empty form.
+# Django setting: CSRF_COOKIE_SECURE and SESSION_COOKIE_SECURE
+# https://docs.djangoproject.com/en/3.1/ref/settings/#session-cookie-secure
+# COOKIE_SECURE=False
+
+# Cookie prefix is added to the every cookie set by HelERM. These are
+# mostly used when accessing the internal Django admin site. This applies
+# to django session cookie and csrf cookie
+# Django setting: prepended to CSRF_COOKIE_NAME and SESSION_COOKIE_NAME
+COOKIE_PREFIX=helerm
+
+# This sets the path for both session cookie and CSRF cookie. Useful if
+# helerm is served from a sub-path (ie. there is a prefix the path,
+# like https://host.name/helerm). Note that this must include the
+# leading '/'.
+# Django setting: sets both CSRF_COOKIE_PATH and SESSION_COOKIE_PATH
+#PATH_PREFIX=/helerm
+
+# Django INTERNAL_IPS setting allows some debugging aids for the addresses
+# specified here
+# DJango setting: INTERNAL_IPS https://docs.djangoproject.com/en/3.0/ref/settings/#internal-ips
+INTERNAL_IPS=127.0.0.1
+
+# Specifies a header that is trusted to indicate that the request was using
+# https while traversing over the Internet at large. This is used when
+# a proxy terminates the TLS connection and forwards the request over
+# a secure network. Specified using a tuple.
+# Django setting: SECURE_PROXY_SSL_HEADER https://docs.djangoproject.com/en/3.0/ref/settings/#secure-proxy-ssl-header
+#SECURE_PROXY_SSL_HEADER=('HTTP_X_FORWARDED_PROTO', 'https')
+
+# Media root is the place in file system where Django and, by extension
+# HelERM stores "uploaded" files. This means any and all files
+# that are inputted through API and generated by JHS191 exporter
+# Django setting: MEDIA_ROOT https://docs.djangoproject.com/en/3.0/ref/settings/#media-root
+#MEDIA_ROOT=/home/helerm/media
+
+# Static root is the place where HelERM will install any static
+# files that need to be served to clients. For HelERM this is mostly
+# JS and CSS for the API exploration interface + admin
+# Django setting: STATIC_ROOT
+#STATIC_ROOT=/home/helerm/static
+
+# Media URL is address (URL) where users can access files in MEDIA_ROOT
+# through http. Ie. where your uploaded files are publicly accessible.
+# In the simple case this is a relative URL to same server as API
+# Django setting: MEDIA_URL https://docs.djangoproject.com/en/3.0/ref/settings/#media-url
+MEDIA_URL=/media/
+
+# Static URL is address (URL) where users can access files in STATIC_ROOT
+# through http. Same factors apply as to MEDIA_URL
+# Django setting: STATIC_URL https://docs.djangoproject.com/en/3.0/ref/settings/#static-url
+STATIC_URL=/static/
+
+# Specifies that Django is to use `X-Forwarded-Host` as it would normally
+# use the `Host`-header. This is necessary when `Host`-header is used for
+# routing the requests in a network of reverse proxies. `X-Forwarded-Host`
+# is then used to carry the Host-header value supplied by the origin client.
+# This affects how ALLOWED_HOSTS behaves, as well.
+# Django setting: https://docs.djangoproject.com/en/3.0/ref/settings/#use-x-forwarded-host
+# TRUST_X_FORWARDED_HOST=False
+
+# Sentry is an error tracking sentry (sentry.io) that can be self hosted
+# or purchased as PaaS. SENTRY_DSN setting specifies the URL where reports
+# for this HelERM instance should be sent. You can find this in
+# your Sentry interface (or through its API)
+#SENTRY_DSN=http://your.sentry.here/fsdafads/13
+
+# Sentry environment is an optional tag that can be included in sentry
+# reports. It is used to separate deployments within Sentry UI
+SENTRY_ENVIRONMENT=local-development-unconfigured
+
+# Host for ElasticSearch connection.
+#ELASTICSEARCH_HOST=helerm_elasticsearch:9200
+
+# Helsinki user management library (django-helusers) related OIDC settings.
+SOCIAL_AUTH_TUNNISTAMO_KEY=https://i/am/clientid/in/url/style
+SOCIAL_AUTH_TUNNISTAMO_SECRET=iamyoursecret
+SOCIAL_AUTH_TUNNISTAMO_OIDC_ENDPOINT=https://api.hel.fi/sso/openid
+OIDC_API_TOKEN_AUTH_AUDIENCE=https://api.hel.fi/auth/projects
+OIDC_API_TOKEN_AUTH_ISSUER=https://api.hel.fi/sso

dev-requirements.in

@@ -3,4 +3,5 @@ freezegun
 isort
 pip-tools
 pytest
+pytest-cov
 pytest-django

dev-requirements.txt

@@ -4,28 +4,27 @@
 #
 #    pip-compile dev-requirements.in
 #
-attrs==20.2.0             # via pytest
+attrs==20.3.0             # via pytest
 click==7.1.2              # via pip-tools
-flake8==3.8.3             # via -r dev-requirements.in
+coverage==5.3.1           # via pytest-cov
+flake8==3.8.4             # via -r dev-requirements.in
 freezegun==1.0.0          # via -r dev-requirements.in
-importlib-metadata==1.7.0  # via flake8, pluggy, pytest
-iniconfig==1.0.1          # via pytest
-isort==5.5.3              # via -r dev-requirements.in
+iniconfig==1.1.1          # via pytest
+isort==5.7.0              # via -r dev-requirements.in
 mccabe==0.6.1             # via flake8
-more-itertools==8.5.0     # via pytest
-packaging==20.4           # via pytest
-pip-tools==5.3.1          # via -r dev-requirements.in
+packaging==20.8           # via pytest
+pip-tools==5.5.0          # via -r dev-requirements.in
 pluggy==0.13.1            # via pytest
-py==1.9.0                 # via pytest
+py==1.10.0                # via pytest
 pycodestyle==2.6.0        # via flake8
 pyflakes==2.2.0           # via flake8
 pyparsing==2.4.7          # via packaging
-pytest-django==3.10.0     # via -r dev-requirements.in
-pytest==6.0.2             # via -r dev-requirements.in, pytest-django
+pytest-cov==2.10.1        # via -r dev-requirements.in
+pytest-django==4.1.0      # via -r dev-requirements.in
+pytest==6.2.1             # via -r dev-requirements.in, pytest-cov, pytest-django
 python-dateutil==2.8.1    # via freezegun
-six==1.15.0               # via packaging, pip-tools, python-dateutil
-toml==0.10.1              # via pytest
-zipp==3.1.0               # via importlib-metadata
+six==1.15.0               # via python-dateutil
+toml==0.10.2              # via pytest
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

docker-compose.yml

@@ -12,9 +12,27 @@ services:
       - helerm_postgres-data-volume:/var/lib/postgresql/data
     container_name: helerm_postgres
 
+  elasticsearch:
+    image: elasticsearch:7.3.2
+    build:
+      context: .
+      dockerfile: ./docker/elasticsearch/Dockerfile
+    ports:
+      - "9200:9200"
+    environment:
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+      - cluster.routing.allocation.disk.watermark.low=97%
+      - cluster.routing.allocation.disk.watermark.high=98%
+      - cluster.routing.allocation.disk.watermark.flood_stage=99%
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m -Djava.security.policy=file:/usr/share/elasticsearch/plugins/elasticsearch-analysis-voikko/plugin-security.policy"
+    container_name: helerm_elasticsearch
+
   django:
     build: .
     command: bash -c 'tail -f /dev/null'
+    environment:
+      ELASTICSEARCH_HOST: helerm_elasticsearch
     volumes:
       - .:/code
     ports:

docker/elasticsearch/Dockerfile

@@ -0,0 +1,14 @@
+FROM elasticsearch:7.3.2
+
+# Installation of the voikko library and plugin for ElasticSearch
+# https://github.com/EvidentSolutions/elasticsearch-analysis-voikko
+RUN yum install -y \
+    unzip \
+    libvoikko
+
+RUN curl -o /usr/lib64/voikko/morpho.zip https://www.puimula.org/htp/testing/voikko-snapshot/dict-morpho.zip
+RUN unzip -o /usr/lib64/voikko/morpho.zip -d /usr/lib64/voikko/
+
+RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch https://github.com/EvidentSolutions/elasticsearch-analysis-voikko/releases/download/v0.6.0/elasticsearch-analysis-voikko-0.6.0.zip \
+&& echo '-Djava.security.policy=file:/usr/share/elasticsearch/plugins/elasticsearch-analysis-voikko/plugin-security.policy' >> /usr/share/elasticsearch/config/jvm.options \
+|| echo "Plugin already exists."