From 97dfed6bbca4678681575efced274b074750d124 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Mon, 16 Sep 2024 12:36:53 +0200 Subject: [PATCH 01/25] Update CKAN deployment to CKAN 2.10 - Improve start_ckan setup scripts (DEV/PRO) to debian-based images. - Improve ennvars files. - Improve Solr schema to add language. --- .env.example | 17 ++++++++--- README.md | 1 + ckan/Dockerfile | 12 +++++--- ckan/Dockerfile.dev | 11 +++++--- .../01_setup_ckanext_config.sh | 8 +++++- .../patches/ckan/00_plugin_slug_preview.patch | 2 +- ckan/setup/start_ckan.sh.override | 7 +++-- ckan/setup/start_ckan_development.sh.override | 28 +++++++++++++++++-- samples/.env.es.example | 16 +++++++++-- samples/.env.localhost | 2 +- samples/ckan.ini.example | 2 +- solr/Dockerfile.spatial | 2 +- 12 files changed, 83 insertions(+), 25 deletions(-) diff --git a/.env.example b/.env.example index 11dd700d..9de28a5b 100644 --- a/.env.example +++ b/.env.example @@ -95,6 +95,10 @@ CKAN_SITE_URL=${PROXY_SERVER_URL} CKAN__ROOT_PATH=${PROXY_CKAN_LOCATION}/{{LANG}} CKAN__FAVICON=${PROXY_CKAN_LOCATION}/base/images/ckan.ico CKAN__SITE_LOGO=/images/default/ckan-logo.png +# CKAN site info +CKAN__SITE_INTRO_TEXT="" +CKAN__SITE_DESCRIPTION="" +CKAN__SITE_ABOUT="" # Custom licenses that are aligned with DCAT-AP CKAN___LICENSES_GROUP_URL=https://raw.githubusercontent.com/mjanez/ckanext-schemingdcat/main/ckanext/schemingdcat/public/static/licenses.json CKAN___BEAKER__SESSION__SECRET=CHANGE_ME @@ -122,6 +126,11 @@ CKAN__SEARCH__SOLR_ALLOWED_QUERY_PARSERS=field # CORS Settings. If True, all origins will be allowed (the response header Access-Control-Allow-Origin is set to ‘*’). If False, only origins from the ckan.cors.origin_whitelist setting will be allowed. CKAN__CORS__ORIGIN_ALLOW_ALL=False CKAN__CORS__ORIGIN_WHITELIST="" +# Enables or disable collaborators in individual datasets (https://docs.ckan.org/en/2.9/maintaining/authorization.html#dataset-collaborators) +CKAN__AUTH__ALLOW_DATASET_COLLABORATORS=False +CKAN__AUTH__ALLOW_ADMIN_COLLABORATORS=False +# Default number of facets shown in search results. Default 10. +SEARCH__FACETS__DEFAULT=4 ## Resource Proxy settings ### Preview size limit, default: 1MB @@ -130,7 +139,7 @@ CKAN__RESOURCE_PROXY__MAX_FILE_SIZE=50048576 CKAN__RESOURCE_PROXY__CHUNK_SIZE=4096 ## Default timeout for fetching proxied items CKAN__RESOURCE_PROXY__TIMEOUT=10 -CKAN__VIEWS__DEFAULT_VIEWS="image_view webpage_view text_view recline_view wmts_view geojson_view geo_view shp_view pdf_view" +CKAN__VIEWS__DEFAULT_VIEWS="image_view video_view audio_view webpage_view text_view datatables_view wmts_view geojson_view geo_view shp_view pdf_view" # Localization CKAN__LOCALE_DEFAULT="en" @@ -138,7 +147,7 @@ CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru CKAN__LOCALES_OFFERED="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" # Extensions -CKAN__PLUGINS="stats image_view text_view recline_view webpage_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" +CKAN__PLUGINS="stats image_view video_view audio_view webpage_view text_view datatables_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" # ckanext-harvest CKAN__HARVEST__MQ__TYPE=redis @@ -176,12 +185,12 @@ CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8 ## CSW Endpoint for spatial metadata CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI=${PYCSW_URL} ## Scheming: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_2.yaml ckanext.schemingdcat:schemas/resources/dcat_3_document.yaml" +CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_full.yaml ckanext.schemingdcat:schemas/resources/dcat_document.yaml" CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_group.json" CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_org.json" CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/default_presets.json ckanext.fluent:presets.json" ## Facets: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_eu dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" +CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_eu language dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" CKANEXT__SCHEMINGDCAT_ORGANIZATION_CUSTOM_FACETS=True CKANEXT__SCHEMINGDCAT_GROUP_CUSTOM_FACETS=True CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_ICON="theme" diff --git a/README.md b/README.md index ac853f21..579cf174 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ Available components: | 2.9.x | custom spatial image | `alpine:3.15` | `ghcr.io/mjanez/ckan-spatial:ckan-2.9.8`, `ghcr.io/mjanez/ckan-spatial:ckan-2.9.8`, `ghcr.io/mjanez/ckan-docker:ckan-2.9.9`, `ghcr.io/mjanez/ckan-docker:ckan-2.9.10`, `ghcr.io/mjanez/ckan-docker:ckan-2.9.11`, `ghcr.io/mjanez/ckan-docker:2.9.12` | Stable official versions of CKAN `2.9.8`, `2.9.10` and `2.9.11`, including a security backport: `2.9.12`. As of `2.9.12`, repo images are aligned with the [`ckan-docker-spatial`](https://github.com/mjanez/ckan-docker-spatial#pre-configured-ckan-docker-images) and [`ckan-docker-base](https://github.com/ckan/ckan-docker-base#pre-configured-ckan-docker-images)` tags. | | 2.10.x | custom spatial image | `python:3.10-slim-bookworm` | `ghcr.io/mjanez/ckan-docker:2.10.5` | From `2.10` images only [Debian-based official Python images](https://hub.docker.com/_/python) rather than Alpine-based images will be provided. | | 2.11.x | custom spatial image | `python:3.10-slim-bookworm` | `ghcr.io/mjanez/ckan-docker:2.11.0` | CKAN's latest official version. Only [Debian-based official Python images](https://hub.docker.com/_/python). | +| master | custom spatial image | `python:3.10-slim-bookworm` | `ghcr.io/mjanez/ckan-docker:master` | Latest image available. Not for use in production. | The non-CKAN images are as follows: * PostgreSQL: [Custom image](/postgresql/Dockerfile) based on official PostgreSQL image. Database files are stored in a named volume. diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 9b96915d..88109c36 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/mjanez/ckan-spatial-base:2.9.12 +FROM ghcr.io/mjanez/ckan-spatial-base:2.10.5-py3.10 LABEL maintainer="mnl.janez@gmail.com" # Set up environment variables @@ -25,8 +25,9 @@ COPY req_fixes req_fixes ## Scheming DCAT - v3.2.2 (mjanez/GeoDCAT-AP/NTI-RISP extended version) ## RUN echo ${TZ} > /etc/timezone && \ if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime; fi && \ - # Remove apk cache - rm -rf /var/cache/apk/* && \ + # Install patch utility + apt-get update && \ + apt-get install -y patch && \ # Install CKAN extensions echo "ckan/ckanext-xloader" && \ pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-xloader.git@1.0.1#egg=ckanext-xloader && \ @@ -55,7 +56,10 @@ RUN echo ${TZ} > /etc/timezone && \ pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-fluent.git@v1.0.1#egg=ckanext-fluent && \ echo "mjanez/ckanext-schemingdcat" && \ pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-schemingdcat.git@v3.2.2#egg=ckanext_schemingdcat && \ - pip3 install --no-cache-dir -r ${APP_DIR}/src/ckanext-schemingdcat/requirements.txt + pip3 install --no-cache-dir -r ${APP_DIR}/src/ckanext-schemingdcat/requirements.txt && \ + # Remove system cache + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Used to configure the container environment by setting environment variables, creating users, running initialization scripts, .etc COPY docker-entrypoint.d/* /docker-entrypoint.d/ diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index 228902ef..86fbca08 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM ghcr.io/mjanez/ckan-spatial-dev:2.9.12 +FROM ghcr.io/mjanez/ckan-spatial-dev:2.10.5-py3.10 LABEL maintainer="mnl.janez@gmail.com" # Set up environment variables @@ -10,9 +10,12 @@ ENV APP_DIR=/srv/app \ WORKDIR ${APP_DIR} RUN echo ${TZ} > /etc/timezone && \ if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime; fi && \ - apk --no-cache add sudo && \ - # Remove apk cache - rm -rf /var/cache/apk/* + # Install patch utility + apt-get update && \ + apt-get install -y patch && \ + # Remove system cache + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Install any extensions needed by your CKAN instance # - Make sure to add the plugins to CKAN__PLUGINS in the .env file diff --git a/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh b/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh index 8ba4079e..d786aa95 100644 --- a/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh +++ b/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh @@ -4,6 +4,12 @@ echo "[docker-entrypoint.01_setup_ckanext_config] Clear index" ckan -c $CKAN_INI search-index clear +# ckan: Update core settings +echo "[docker-entrypoint.01_setup_ckanext_config] Loading CKAN core settings in the CKAN config file" +ckan config-tool $CKAN_INI \ + "search.facets.default = $SEARCH__FACETS__DEFAULT" + +# ckanext-schemingdcat: Update settings echo "[docker-entrypoint.01_setup_ckanext_config] Loading ckanext-scheming and ckanext-schemingdcat settings into ckan.ini" ckan config-tool $CKAN_INI \ "scheming.dataset_schemas=$CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA" \ @@ -18,7 +24,7 @@ ckan config-tool $CKAN_INI \ "schemingdcat.default_package_item_show_spatial=$CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_SHOW_SPATIAL" \ "schemingdcat.show_metadata_templates_toolbar=$CKANEXT__SCHEMINGDCAT_SHOW_METADATA_TEMPLATES_TOOLBAR" \ "schemingdcat.metadata_templates_search_identifier=$CKANEXT__METADATA_TEMPLATES_SEARCH_IDENTIFIER" \ - "schemingdcat.endpoints_yaml=$CKANEXT__SCHEMINGDCAT_ENDPOINTS_YAML" \ + "schemingdcat.endpoints_yaml=$CKANEXT__SCHEMINGDCAT_ENDPOINTS_YAML" # ckanext-dcat: Add settings to the CKAN config file echo "[docker-entrypoint.01_setup_ckanext_config] Loading ckanext-dcat settings in the CKAN config file" diff --git a/ckan/patches/ckan/00_plugin_slug_preview.patch b/ckan/patches/ckan/00_plugin_slug_preview.patch index 3590303e..b1be4173 100644 --- a/ckan/patches/ckan/00_plugin_slug_preview.patch +++ b/ckan/patches/ckan/00_plugin_slug_preview.patch @@ -1,5 +1,5 @@ diff --git a/ckan/public/base/javascript/plugins/jquery.slug-preview.js b/ckan/public/base/javascript/plugins/jquery.slug-preview.js -index 4a4f78fe25..8c03e816d2 100644 +index 4a4f78fe2..8c03e816d 100644 --- a/ckan/public/base/javascript/plugins/jquery.slug-preview.js +++ b/ckan/public/base/javascript/plugins/jquery.slug-preview.js @@ -33,6 +33,11 @@ diff --git a/ckan/setup/start_ckan.sh.override b/ckan/setup/start_ckan.sh.override index 7d525853..1ffcbf06 100644 --- a/ckan/setup/start_ckan.sh.override +++ b/ckan/setup/start_ckan.sh.override @@ -1,11 +1,12 @@ -#!/bin/sh +#!/bin/bash # Set up the Secret key used by Beaker and Flask # This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var -if grep -E "beaker.session.secret ?= ?$" ckan.ini +if grep -qE "beaker.session.secret ?= ?$" ckan.ini then echo "Setting beaker.session.secret in ini file" ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" @@ -46,7 +47,7 @@ UWSGI_OPTS="--plugins http,python \ --wsgi-file /srv/app/wsgi.py \ --module wsgi:application \ --uid 92 --gid 92 \ - --http 0.0.0.0:5000 \ + --http [::]:5000 \ --master --enable-threads \ --lazy-apps \ -p 2 -L -b 32768 --vacuum \ diff --git a/ckan/setup/start_ckan_development.sh.override b/ckan/setup/start_ckan_development.sh.override index 4669dac6..5a5a9f4c 100644 --- a/ckan/setup/start_ckan_development.sh.override +++ b/ckan/setup/start_ckan_development.sh.override @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Install any local extensions in the src_extensions volume echo "Looking for local extensions to install..." @@ -31,6 +31,13 @@ do echo "Found setup.py file in $i" cd $APP_DIR fi + if [ -f $i/pyproject.toml ]; + then + cd $i + pip install -e . + echo "Found pyproject.toml file in $i" + cd $APP_DIR + fi # Point `use` in test.ini to location of `test-core.ini` if [ -f $i/test.ini ]; @@ -51,6 +58,7 @@ if grep -E "beaker.session.secret ?= ?$" ckan.ini then echo "Setting beaker.session.secret in ini file" ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" @@ -90,6 +98,18 @@ then done fi +CKAN_RUN="ckan -c $CKAN_INI run -H 0.0.0.0" +CKAN_OPTIONS="" +if [ "$USE_DEBUGPY_FOR_DEV" = true ] ; then + pip install debugpy + CKAN_RUN="/usr/bin/python -m debugpy --listen 0.0.0.0:5678 $CKAN_RUN" + CKAN_OPTIONS="$CKAN_OPTIONS --disable-reloader" +fi + +if [ "$USE_HTTPS_FOR_DEV" = true ] ; then + CKAN_OPTIONS="$CKAN_OPTIONS -C unsafe.cert -K unsafe.key" +fi + # Create Harvester logs directory and change its ownership mkdir -p $CKAN_LOGS_PATH/harvester chown -R ckan:ckan $CKAN_LOGS_PATH/harvester @@ -102,7 +122,11 @@ chown -R ckan:ckan $CKAN_LOGS_PATH/xloader #supervisord --configuration /etc/supervisord.conf & # Start the development server as the ckan user with automatic reload -su ckan -c "/usr/bin/ckan -c $CKAN_INI run -H 0.0.0.0" +while true; do + su ckan -c "$CKAN_RUN $CKAN_OPTIONS" + echo Exit with status $?. Restarting. + sleep 2 +done # Workers # To start the Harvester worker diff --git a/samples/.env.es.example b/samples/.env.es.example index 55554036..e19c32c8 100644 --- a/samples/.env.es.example +++ b/samples/.env.es.example @@ -95,6 +95,10 @@ CKAN_SITE_URL=${PROXY_SERVER_URL} CKAN__ROOT_PATH=${PROXY_CKAN_LOCATION}/{{LANG}} CKAN__FAVICON=${PROXY_CKAN_LOCATION}/base/images/ckan.ico CKAN__SITE_LOGO=/images/default/ckan-logo.png +# CKAN site info +CKAN__SITE_INTRO_TEXT="" +CKAN__SITE_DESCRIPTION="" +CKAN__SITE_ABOUT="" # Custom licenses that are aligned with DCAT-AP CKAN___LICENSES_GROUP_URL=https://raw.githubusercontent.com/mjanez/ckanext-schemingdcat/main/ckanext/schemingdcat/public/static/licenses.json CKAN___BEAKER__SESSION__SECRET=CHANGE_ME @@ -122,6 +126,11 @@ CKAN__SEARCH__SOLR_ALLOWED_QUERY_PARSERS=field # CORS Settings. If True, all origins will be allowed (the response header Access-Control-Allow-Origin is set to ‘*’). If False, only origins from the ckan.cors.origin_whitelist setting will be allowed. CKAN__CORS__ORIGIN_ALLOW_ALL=False CKAN__CORS__ORIGIN_WHITELIST="" +# Enables or disable collaborators in individual datasets (https://docs.ckan.org/en/2.9/maintaining/authorization.html#dataset-collaborators) +CKAN__AUTH__ALLOW_DATASET_COLLABORATORS=False +CKAN__AUTH__ALLOW_ADMIN_COLLABORATORS=False +# Default number of facets shown in search results. Default 10. +SEARCH__FACETS__DEFAULT=4 ## Resource Proxy settings ### Preview size limit, default: 1MB @@ -135,9 +144,10 @@ CKAN__VIEWS__DEFAULT_VIEWS="image_view webpage_view text_view recline_view wmts_ # Localization CKAN__LOCALE_DEFAULT="es" CKAN__LOCALE_ORDER="es en pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" +CKAN__LOCALES_OFFERED="es en pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" # Extensions -CKAN__PLUGINS="stats image_view text_view recline_view webpage_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" +CKAN__PLUGINS="stats image_view video_view audio_view webpage_view text_view datatables_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" # ckanext-harvest CKAN__HARVEST__MQ__TYPE=redis @@ -175,12 +185,12 @@ CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8 ## CSW Endpoint for spatial metadata CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI=${PYCSW_URL} ## Scheming: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/es_geodcat_ap_2.yaml ckanext.schemingdcat:schemas/resources/dcat_3_document.yaml" +CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/es_geodcat_ap_full.yaml ckanext.schemingdcat:schemas/resources/dcat_document.yaml" CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/es_geodcat_ap_group.json" CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/es_geodcat_ap_org.json" CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/default_presets.json ckanext.fluent:presets.json" ## Facets: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_es dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" +CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_es language dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" CKANEXT__SCHEMINGDCAT_ORGANIZATION_CUSTOM_FACETS=True CKANEXT__SCHEMINGDCAT_GROUP_CUSTOM_FACETS=True CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_ICON="theme" diff --git a/samples/.env.localhost b/samples/.env.localhost index 11dd700d..5d622ed3 100644 --- a/samples/.env.localhost +++ b/samples/.env.localhost @@ -176,7 +176,7 @@ CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8 ## CSW Endpoint for spatial metadata CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI=${PYCSW_URL} ## Scheming: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_2.yaml ckanext.schemingdcat:schemas/resources/dcat_3_document.yaml" +CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/es_geodcat_ap_full.yaml ckanext.schemingdcat:schemas/resources/dcat_document.yaml" CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_group.json" CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_org.json" CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/default_presets.json ckanext.fluent:presets.json" diff --git a/samples/ckan.ini.example b/samples/ckan.ini.example index 8b7ceb97..6529b931 100644 --- a/samples/ckan.ini.example +++ b/samples/ckan.ini.example @@ -116,7 +116,7 @@ ckan.site_id = default # Add ``datapusher`` to enable DataPusher # Add ``resource_proxy`` to enable resorce proxying and get around the # same origin policy -ckan.plugins = stats text_view image_view recline_view resourcedictionary datastore datapusher harvest ckan_harvester spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface scheming_datasets scheming_groups scheming_organizations pdf_view pages +ckan.plugins = stats text_view image_view recline_view resourcedictionary datastore datapusher harvest ckan_harvester spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations pdf_view pages ## ckanext-scheming diff --git a/solr/Dockerfile.spatial b/solr/Dockerfile.spatial index cb63e1ce..f74d6adf 100644 --- a/solr/Dockerfile.spatial +++ b/solr/Dockerfile.spatial @@ -59,6 +59,7 @@ ENV SOLR_BBOX_FIELDS ' \ \ \ + \ \ \ ' \ @@ -67,7 +68,6 @@ ENV SOLR_BBOX_FIELDS '/a $SOLR_BBOX_FIELDS" $SOLR_SCHEMA_FILE - RUN chmod 644 $SOLR_SCHEMA_FILE USER solr From 71366560b32b5e6e6de7a2e26ec1b93afde6cd46 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Mon, 16 Sep 2024 12:40:11 +0200 Subject: [PATCH 02/25] Add docs about ckanet-schemingdcat harvesters. --- doc/harvesters/README.md | 29 +++ doc/harvesters/ckan_template_config.json | 272 +++++++++++++++++++++++ 2 files changed, 301 insertions(+) create mode 100644 doc/harvesters/README.md create mode 100644 doc/harvesters/ckan_template_config.json diff --git a/doc/harvesters/README.md b/doc/harvesters/README.md new file mode 100644 index 00000000..c1e5fb7b --- /dev/null +++ b/doc/harvesters/README.md @@ -0,0 +1,29 @@ +# Harvesters. Configuration examples +>[!TIP] +> For more information about scheming cat harvesters, see [docs](https://github.com/mjanez/ckanext-schemingdcat?tab=readme-ov-file#harvesters). + +This subdirectory contains examples of JSON configuration files for harvesters. These configurations include various settings such as API version, tags, groups, and field mappings. These JSON files are used to configure the harvesting process from different sources. + +For example, a configuration might include settings like: +- `api_version` +- `clean_tags` +- `default_groups` +- `default_tags` +- `default_extras` +- `override_extras` +- `organizations_filter_include` +- `groups_filter_include` +- `allow_private_datasets` +- `user` +- `api_key` +- `read_only` +- `remote_groups` +- `remote_orgs` +- `default_group_dicts` +- `schema` +- `allow_harvest_datasets` +- `field_mapping_schema_version` +- `dataset_field_mapping` +- `distribution_field_mapping` + +These settings help in defining how the data should be harvested, processed, and stored. \ No newline at end of file diff --git a/doc/harvesters/ckan_template_config.json b/doc/harvesters/ckan_template_config.json new file mode 100644 index 00000000..942dfb3c --- /dev/null +++ b/doc/harvesters/ckan_template_config.json @@ -0,0 +1,272 @@ +{ + "api_version":2, + "clean_tags":false, + "default_groups":[], + "default_tags":[], + "default_extras":{ + "encoding":"UTF-8", + "harvest_description":"Harvest from Sample Portal", + "harvest_url":"{harvest_source_url}/dataset/{dataset_id}" + }, + "override_extras":false, + "organizations_filter_include":[ + + ], + "groups_filter_include":[], + "allow_private_datasets": true, + "user":"admin", + "api_key":"eyJ0easdasdasdasdasd", + "read_only":true, + "remote_groups":"only_local", + "remote_orgs":"only_local", + "default_group_dicts":[ + + ], + "schema":"es_geodcat_ap_full", + "allow_harvest_datasets":false, + "field_mapping_schema_version":2, + "dataset_field_mapping":{ + "title_translated":{ + "field_name":"title_translated" + }, + "schemingdcat_xls_metadata_template":{ + "field_name":"schemingdcat_xls_metadata_template" + }, + "dataset_scope":{ + "field_name":"dataset_scope" + }, + "graphic_overview":{ + "field_name":"graphic_overview" + }, + "identifier":{ + "field_name":"identifier" + }, + "name":{ + "field_name":"name" + }, + "dcat_type":{ + "field_name":"dcat_type" + }, + "theme_es":{ + "field_name":"theme_es" + }, + "hvd_category":{ + "field_name":"hvd_category" + }, + "language":{ + "field_name":"language" + }, + "created":{ + "field_name":"created" + }, + "modified":{ + "field_name":"modified" + }, + "notes_translated":{ + "field_name":"notes_translated" + }, + "topic":{ + "field_name":"topic" + }, + "tag_string":{ + "field_name":"tag_string" + }, + "tag_uri":{ + "field_name":"tag_uri" + }, + "contact_name":{ + "field_name":"contact_name" + }, + "contact_email":{ + "field_name":"contact_email" + }, + "contact_url":{ + "field_name":"contact_url" + }, + "contact_uri":{ + "field_name":"contact_uri" + }, + "publisher_name":{ + "field_name":"publisher_name" + }, + "publisher_email":{ + "field_name":"publisher_email" + }, + "publisher_url":{ + "field_name":"publisher_url" + }, + "publisher_uri":{ + "field_name":"publisher_uri" + }, + "publisher_type":{ + "field_name":"publisher_type" + }, + "maintainer":{ + "field_name":"maintainer" + }, + "maintainer_email":{ + "field_name":"maintainer_email" + }, + "maintainer_url":{ + "field_name":"maintainer_url" + }, + "maintainer_uri":{ + "field_name":"maintainer_uri" + }, + "author":{ + "field_name":"author" + }, + "author_email":{ + "field_name":"author_email" + }, + "author_url":{ + "field_name":"author_url" + }, + "author_uri":{ + "field_name":"author_uri" + }, + "alternate_identifier":{ + "field_name":"alternate_identifier" + }, + "provenance":{ + "field_name":"provenance" + }, + "conforms_to":{ + "field_name":"conforms_to" + }, + "metadata_profile":{ + "field_name":"metadata_profile" + }, + "temporal_start":{ + "field_name":"temporal_start" + }, + "temporal_end":{ + "field_name":"temporal_end" + }, + "frequency":{ + "field_name":"frequency" + }, + "source":{ + "field_name":"source" + }, + "lineage_source":{ + "field_name":"lineage_source" + }, + "lineage_process_steps":{ + "field_name":"lineage_process_steps" + }, + "reference":{ + "field_name":"reference" + }, + "purpose":{ + "field_name":"purpose" + }, + "encoding":{ + "field_name":"encoding" + }, + "temporal_resolution":{ + "field_name":"temporal_resolution" + }, + "theme":{ + "field_name":"theme" + }, + "inspire_id":{ + "field_name":"inspire_id" + }, + "reference_system":{ + "field_name":"reference_system" + }, + "spatial":{ + "field_name":"spatial" + }, + "spatial_uri":{ + "field_name":"spatial_uri" + }, + "representation_type":{ + "field_name":"representation_type" + }, + "spatial_resolution_in_meters":{ + "field_name":"spatial_resolution_in_meters" + }, + "license_id":{ + "field_name":"license_id" + }, + "access_rights":{ + "field_name":"access_rights" + }, + "version":{ + "field_name":"version" + }, + "version_notes":{ + "field_name":"version_notes" + }, + "valid":{ + "field_name":"valid" + }, + "groups":{ + "field_name":"groups" + }, + "private":{ + "field_name":"private" + } + }, + "distribution_field_mapping":{ + "dataset_id":{ + "field_name":"id" + }, + "url":{ + "field_name":"url" + }, + "created":{ + "field_name":"created" + }, + "name":{ + "field_name":"name" + }, + "description":{ + "field_name":"description" + }, + "encoding":{ + "field_name":"encoding" + }, + "availability":{ + "field_name":"availability" + }, + "issued":{ + "field_name":"issued" + }, + "modified":{ + "field_name":"modified" + }, + "language":{ + "field_name":"language" + }, + "rights":{ + "field_name":"rights" + }, + "format":{ + "field_name":"format" + }, + "mimetype":{ + "field_name":"mimetype" + }, + "status":{ + "field_name":"status" + }, + "size":{ + "field_name":"size" + }, + "resource_relation":{ + "field_name":"resource_relation" + }, + "license":{ + "field_name":"license" + }, + "conforms_to":{ + "field_name":"conforms_to" + }, + "reference_system":{ + "field_name":"reference_system" + } + } + } \ No newline at end of file From ec0b2fa8dff17fff0f9bf6551107004e3ee9768c Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:20:04 +0200 Subject: [PATCH 03/25] Update plugins and remove who.ini - Update ckanext-xloader to fixed version. - Update ckanext-geoview to latest. - Update ckanext-schemingdcat to latest alpha --- ckan/Dockerfile | 15 ++++++--------- ckan/Dockerfile.dev | 3 --- ckan/Dockerfile.ghcr | 3 --- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 88109c36..a8b57d77 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -12,9 +12,9 @@ WORKDIR ${APP_DIR} COPY req_fixes req_fixes # CKAN configuration & extensions -## XLoader - 1.0.1 ## +## XLoader - 1.0.1-stable (mjanez/Forked fixed version) ## ## Harvest - v1.5.6 (Worker with supervisor) ## -## Geoview - v0.1.0 ## +## Geoview - v0.2.0 ## ## Spatial - v2.1.1 ## ## DCAT - v1.8.0 (Latest stable version of ckanext-dcat with minor fixes) ## ## Scheming - release-3.0.0 ## @@ -22,7 +22,7 @@ COPY req_fixes req_fixes ## Pages - v0.5.2 ## ## PDFView - 0.0.8 ## ## Fluent - v1.0.1 (mjanez/Forked stable version) ## -## Scheming DCAT - v3.2.2 (mjanez/GeoDCAT-AP/NTI-RISP extended version) ## +## Scheming DCAT - v4.0.0-alpha (mjanez/GeoDCAT-AP/NTI-RISP extended version) ## RUN echo ${TZ} > /etc/timezone && \ if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime; fi && \ # Install patch utility @@ -30,14 +30,14 @@ RUN echo ${TZ} > /etc/timezone && \ apt-get install -y patch && \ # Install CKAN extensions echo "ckan/ckanext-xloader" && \ - pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-xloader.git@1.0.1#egg=ckanext-xloader && \ + pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-xloader.git@1.0.1-stable#egg=ckanext-xloader && \ pip3 install --no-cache-dir -r ${APP_DIR}/src/ckanext-xloader/requirements.txt && \ pip3 install --no-cache-dir -U requests[security] && \ echo "ckan/ckanext-harvest" && \ pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-harvest.git@v1.5.6#egg=ckanext-harvest && \ pip3 install --no-cache-dir -r ${APP_DIR}/src/ckanext-harvest/requirements.txt && \ echo "ckan/ckanext-geoview" && \ - pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-geoview.git@v0.1.0#egg=ckanext-geoview && \ + pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-geoview.git@v0.2.0#egg=ckanext-geoview && \ echo "ckan/ckanext-spatial" && \ pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-spatial.git@v2.1.1#egg=ckanext-spatial && \ pip3 install --no-cache-dir -r ${APP_DIR}/req_fixes/ckanext-spatial/requirements.txt && \ @@ -55,7 +55,7 @@ RUN echo ${TZ} > /etc/timezone && \ echo "mjanez/ckanext-fluent" && \ pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-fluent.git@v1.0.1#egg=ckanext-fluent && \ echo "mjanez/ckanext-schemingdcat" && \ - pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-schemingdcat.git@v3.2.2#egg=ckanext_schemingdcat && \ + pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-schemingdcat.git@v4.0.0-alpha#egg=ckanext_schemingdcat && \ pip3 install --no-cache-dir -r ${APP_DIR}/src/ckanext-schemingdcat/requirements.txt && \ # Remove system cache apt-get clean && \ @@ -64,9 +64,6 @@ RUN echo ${TZ} > /etc/timezone && \ # Used to configure the container environment by setting environment variables, creating users, running initialization scripts, .etc COPY docker-entrypoint.d/* /docker-entrypoint.d/ -# Update who.ini with PROXY_CKAN_LOCATION -COPY setup/who.ini ./ - # Apply any patches needed to CKAN core COPY patches patches diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index 86fbca08..25cbcb40 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -57,9 +57,6 @@ RUN echo ${TZ} > /etc/timezone && \ # Used to configure the container environment by setting environment variables, creating users, running initialization scripts, .etc COPY docker-entrypoint.d/* /docker-entrypoint.d/ -# Update who.ini with PROXY_CKAN_LOCATION -COPY setup/who.ini ./ - # Override start_ckan.sh with DEV sh COPY setup/start_ckan_development.sh.override ./start_ckan_development.sh RUN chmod +x ./start_ckan_development.sh diff --git a/ckan/Dockerfile.ghcr b/ckan/Dockerfile.ghcr index 4e1e5a05..f1d1276b 100644 --- a/ckan/Dockerfile.ghcr +++ b/ckan/Dockerfile.ghcr @@ -17,9 +17,6 @@ RUN echo ${TZ} > /etc/timezone && \ # Used to configure the container environment by setting environment variables, creating users, running initialization scripts, .etc COPY docker-entrypoint.d/* docker-entrypoint.d/ -# Update who.ini with PROXY_CKAN_LOCATION -COPY setup/who.ini ./ - # Apply any patches needed to CKAN core COPY patches patches From e082565a85a2f2a86da4ee30d79645e36a3be038 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:20:15 +0200 Subject: [PATCH 04/25] Refactor prerun.py.override to improve database connection checks and ownership of storage directory --- ckan/setup/prerun.py.override | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/ckan/setup/prerun.py.override b/ckan/setup/prerun.py.override index 3d686969..956b35bf 100644 --- a/ckan/setup/prerun.py.override +++ b/ckan/setup/prerun.py.override @@ -33,6 +33,7 @@ def check_main_db_connection(retry=None): conn_str = os.environ.get("CKAN_SQLALCHEMY_URL") if not conn_str: print("[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db") + return return check_db_connection(conn_str, retry) @@ -41,6 +42,7 @@ def check_datastore_db_connection(retry=None): conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL") if not conn_str: print("[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db") + return return check_db_connection(conn_str, retry) @@ -83,29 +85,29 @@ def check_solr_connection(retry=None): time.sleep(10) check_solr_connection(retry=retry - 1) else: - import re - conn_info = connection.read() - schema_name = json.loads(conn_info) - if 'ckan' in schema_name['name']: - print('[prerun] Succesfully connected to solr and CKAN schema loaded') - else: + import re + conn_info = connection.read() + schema_name = json.loads(conn_info) + if 'ckan' in schema_name['name']: + print('[prerun] Succesfully connected to solr and CKAN schema loaded') + else: print('[prerun] Succesfully connected to solr, but CKAN schema not found') def init_db(): - db_command = ["ckan", "-c", ckan_ini, "db", "init"] + db_command = ["ckan", "-c", ckan_ini, "db", "upgrade"] print("[prerun] Initializing or upgrading db - start") try: subprocess.check_output(db_command, stderr=subprocess.STDOUT) print("[prerun] Initializing or upgrading db - end") except subprocess.CalledProcessError as e: - if "OperationalError" in e.output: - print(e.output) + if "OperationalError" in str(e.output): print("[prerun] Database not ready, waiting a bit before exit...") time.sleep(5) sys.exit(1) else: + print(str(e)) print(e.output) raise e @@ -194,6 +196,13 @@ def create_sysadmin(): subprocess.call(command) print("[prerun] Made user {0} a sysadmin".format(name)) + # cleanup permissions + # We're running as root before pivoting to uwsgi and dropping privs + data_dir = "%s/storage" % os.environ['CKAN_STORAGE_PATH'] + + command = ["chown", "-R", "ckan:ckan", data_dir] + subprocess.call(command) + print("[prerun] Ensured storage directory is owned by ckan") if __name__ == "__main__": @@ -208,5 +217,4 @@ if __name__ == "__main__": check_datastore_db_connection() init_datastore_db() check_solr_connection() - create_sysadmin() - \ No newline at end of file + create_sysadmin() \ No newline at end of file From cb278d10b3e08550b9c098c256768c7a6ee72a03 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:20:37 +0200 Subject: [PATCH 05/25] Refactor start_ckan_development.sh.override to update licenses group URL in CKAN configuration --- ckan/setup/start_ckan_development.sh.override | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/ckan/setup/start_ckan_development.sh.override b/ckan/setup/start_ckan_development.sh.override index 5a5a9f4c..724e7322 100644 --- a/ckan/setup/start_ckan_development.sh.override +++ b/ckan/setup/start_ckan_development.sh.override @@ -54,7 +54,7 @@ ckan config-tool $CKAN_INI -s DEFAULT "debug = true" # Set up the Secret key used by Beaker and Flask # This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var -if grep -E "beaker.session.secret ?= ?$" ckan.ini +if grep -qE "beaker.session.secret ?= ?$" ckan.ini then echo "Setting beaker.session.secret in ini file" ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" @@ -62,11 +62,6 @@ then JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" - - echo "Update licenses in CKAN" - if [ -n "$CKAN___LICENSES_GROUP_URL" ]; then - ckan config-tool $CKAN_INI "licenses_group_url=${CKAN___LICENSES_GROUP_URL}" - fi fi # Update the plugins setting in the ini file with the values defined in the env var @@ -82,6 +77,25 @@ ckan config-tool $SRC_DIR/ckan/test-core.ini \ "solr_url = $TEST_CKAN_SOLR_URL" \ "ckan.redis.url = $TEST_CKAN_REDIS_URL" +# Update local (file://) licenses for Development mode +if [ -n "$LICENSES_GROUP_URL" ]; then + expanded_app_dir=$(eval echo "$APP_DIR") + expanded_src_dir=$(eval echo "$SRC_DIR") + expanded_src_extensions_dir=$(eval echo "$SRC_EXTENSIONS_DIR") + + if [[ "$LICENSES_GROUP_URL" == *"$expanded_src_dir"* ]]; then + LICENSES_GROUP_URL=${LICENSES_GROUP_URL//$expanded_src_dir/$expanded_src_extensions_dir} + echo "Updating licenses group URL to: $LICENSES_GROUP_URL" + ckan config-tool $CKAN_INI "licenses_group_url=${LICENSES_GROUP_URL}" + elif [[ "$LICENSES_GROUP_URL" == "file://$expanded_app_dir/src/"* ]]; then + LICENSES_GROUP_URL=${LICENSES_GROUP_URL//file:\/\/$expanded_app_dir\/src\//file:\/\/$expanded_app_dir\/src_extensions\/} + echo "Updating licenses group URL to: $LICENSES_GROUP_URL" + ckan config-tool $CKAN_INI "licenses_group_url=${LICENSES_GROUP_URL}" + else + echo "LICENSES_GROUP_URL does not contain $expanded_src_dir or file://$expanded_app_dir/src/. No changes made." + fi +fi + # Run the prerun script to init CKAN and create the default admin user python3 prerun.py From c126c2434efcf483a1e4be2d36692560cdf5795b Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:21:11 +0200 Subject: [PATCH 06/25] Update licenses group URL in CKAN configuration and add a workaround for Github Codespaces --- ckan/setup/start_ckan.sh.override | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ckan/setup/start_ckan.sh.override b/ckan/setup/start_ckan.sh.override index 1ffcbf06..e30913d5 100644 --- a/ckan/setup/start_ckan.sh.override +++ b/ckan/setup/start_ckan.sh.override @@ -10,11 +10,18 @@ then JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" +fi + +# Github Codespaces workaround. Check if WTF_CSRF_ENABLED is explicitly set to False and update CKAN_INI +if grep -qE "^WTF_CSRF_ENABLED ?= ?False$" $CKAN_INI; then + echo "WTF_CSRF_ENABLED is explicitly set to False. Updating CKAN_INI." + ckan config-tool $CKAN_INI "WTF_CSRF_ENABLED=False" +fi +# Update local (file://) licenses +if [ -n "$CKAN___LICENSES_GROUP_URL" ]; then + ckan config-tool $CKAN_INI "licenses_group_url=${CKAN___LICENSES_GROUP_URL}" echo "Update licenses in CKAN" - if [ -n "$CKAN___LICENSES_GROUP_URL" ]; then - ckan config-tool $CKAN_INI "licenses_group_url=${CKAN___LICENSES_GROUP_URL}" - fi fi # Run the prerun script to init CKAN and create the default admin user From 5d2aaf1865c7bd2db7278cd201f3ac58a7448a78 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:29:28 +0200 Subject: [PATCH 07/25] Update ennvars - Improve proxy_server hosts - Include HTTP/HTTPS protocol in config. - Update CKAN_VERSION - Improve licenses.json from ckanext-schemingdcat. - Add datastore_sql_search option. - Remove unnecesary CKANEXT__XLOADER__API_TOKEN var. --- .env.example | 21 ++-- samples/.env.codespaces | 216 ++++++++++++++++++++++++++++++++++++++++ samples/.env.es.example | 23 +++-- samples/.env.localhost | 38 ++++--- 4 files changed, 269 insertions(+), 29 deletions(-) create mode 100644 samples/.env.codespaces diff --git a/.env.example b/.env.example index 9de28a5b..51d1b79c 100644 --- a/.env.example +++ b/.env.example @@ -7,11 +7,13 @@ APP_DIR=/srv/app CKAN_PORT_HOST=5000 # Common proxy server for Apache or NGINX. # Change all the PROXY_SERVER_PORT_HOST to PROXY_SERVER_HTTPS_PORT_HOST if you can use HTTPS instead of HTTP (Only for NGINX and not development compose) -PROXY_SERVER_PORT_HOST=81 +PROXY_SERVER_BASE_PORT_HOST=81 PROXY_SERVER_HTTPS_PORT_HOST=8443 -NGINX_PORT_HOST=${PROXY_SERVER_PORT_HOST} +# Server port for the proxy server. Better using the SSL port for production. +PROXY_SERVER_PORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} +NGINX_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} NGINX_SSLPORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} -APACHE_PORT_HOST=${PROXY_SERVER_PORT_HOST} +APACHE_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} PYCSW_PORT_HOST=8000 # Solr @@ -41,7 +43,9 @@ APACHE_LOG_DIR=/var/log/apache #NGINX/APACHE ## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI and CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. If you dont need PROXY_SERVER_PORT_HOST (e.g. is 80), edit the PROXY_SERVER_URL and remove :${PROXY_SERVER_PORT_HOST} PROXY_SERVER_NAME=localhost -PROXY_SERVER_URL=http://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} +# URL protocol. Change to https if you use SSL +PROXY_SERVER_PROTOCOL=http +PROXY_SERVER_URL=${PROXY_SERVER_PROTOCOL}://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} PROXY_CKAN_LOCATION=/catalog PROXY_PYCSW_LOCATION=/csw @@ -88,7 +92,7 @@ CKAN_DEV_COMPOSE_SERVICE=ckan-dev # CKAN core ## If use docker-compose.ghcr.yml only "*.*.*" versions available in: https://github.com/mjanez/ckan-docker/pkgs/container/ckan-docker -CKAN_VERSION=2.9.11 +CKAN_VERSION=2.10.5 CKAN_SITE_ID=default # CKAN_SITE_URL = http:/ or https:/ + PROXY_SERVER_NAME. Optionally the APACHE_HOST_PORT if different from 80 CKAN_SITE_URL=${PROXY_SERVER_URL} @@ -99,8 +103,8 @@ CKAN__SITE_LOGO=/images/default/ckan-logo.png CKAN__SITE_INTRO_TEXT="" CKAN__SITE_DESCRIPTION="" CKAN__SITE_ABOUT="" -# Custom licenses that are aligned with DCAT-AP -CKAN___LICENSES_GROUP_URL=https://raw.githubusercontent.com/mjanez/ckanext-schemingdcat/main/ckanext/schemingdcat/public/static/licenses.json +# Custom licenses that are aligned with DCAT-AP (Also works with DEV Mode) +LICENSES_GROUP_URL=file://${APP_DIR}/src/ckanext-schemingdcat/ckanext/schemingdcat/public/static/licenses.json CKAN___BEAKER__SESSION__SECRET=CHANGE_ME # See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME @@ -131,6 +135,8 @@ CKAN__AUTH__ALLOW_DATASET_COLLABORATORS=False CKAN__AUTH__ALLOW_ADMIN_COLLABORATORS=False # Default number of facets shown in search results. Default 10. SEARCH__FACETS__DEFAULT=4 +# Enable or disable the DataStore SQL search backend. Default is False. +CKAN__DATASTORE__SQLSEARCH__ENABLED=True ## Resource Proxy settings ### Preview size limit, default: 1MB @@ -158,7 +164,6 @@ CKAN__HARVEST__MQ__REDIS_DB=${REDIS_CKAN_DATABASE} CKAN__HARVEST__LOG_TIMEFRAME=40 # ckanext-xloader -CKANEXT__XLOADER__API_TOKEN=api_token CKANEXT__XLOADER__JOBS__DB_URI=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB} # ckanext-dcat diff --git a/samples/.env.codespaces b/samples/.env.codespaces new file mode 100644 index 00000000..4ec58c1f --- /dev/null +++ b/samples/.env.codespaces @@ -0,0 +1,216 @@ +# Base +APP_DIR=/srv/app +# Sets Docker Compose project name to avoid resource conflicts between projects. Defaults to the folder name "ckan-docker" if not set. +#COMPOSE_PROJECT_NAME=ckan-docker-mytheme + +# Github Codespaces avoid errors with the CSRF token. +WTF_CSRF_ENABLED=False + +# Host Ports +CKAN_PORT_HOST=5000 +# Common proxy server for Apache or NGINX. +# Change all the PROXY_SERVER_PORT_HOST to PROXY_SERVER_HTTPS_PORT_HOST if you can use HTTPS instead of HTTP (Only for NGINX and not development compose) +PROXY_SERVER_BASE_PORT_HOST=81 +PROXY_SERVER_HTTPS_PORT_HOST=8443 +# Server port for the proxy server. Better using the SSL port for production. +PROXY_SERVER_PORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} +NGINX_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} +NGINX_SSLPORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} +APACHE_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} +PYCSW_PORT_HOST=8000 + +# Solr +SOLR_IMAGE_VERSION=2.9-solr9-spatial +SOLR_PORT=8983 +SOLR_CKAN_DATABASE=ckan +CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/${SOLR_CKAN_DATABASE} +TEST_CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/${SOLR_CKAN_DATABASE} + +# Redis +REDIS_VERSION=7-alpine +REDIS_PORT=6379 +REDIS_CKAN_DATABASE=1 +CKAN_REDIS_URL=redis://redis:${REDIS_PORT}/${REDIS_CKAN_DATABASE} +TEST_CKAN_REDIS_URL=redis://redis:${REDIS_PORT}/${REDIS_CKAN_DATABASE} + +# NGINX +NGINX_PORT=80 +NGINX_SSLPORT=443 +NGINX_LOG_DIR=/var/log/nginx + +# Apache HTTP Server +APACHE_VERSION=2.4-alpine +APACHE_PORT=80 +APACHE_LOG_DIR=/var/log/apache + +#NGINX/APACHE +## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI and CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. If you dont need PROXY_SERVER_PORT_HOST (e.g. is 81), edit the PROXY_SERVER_URL and remove :${PROXY_SERVER_PORT_HOST} +PROXY_SERVER_NAME=localhost +# URL protocol. Change to https if you use SSL +PROXY_SERVER_PROTOCOL=https +PROXY_SERVER_URL=${PROXY_SERVER_PROTOCOL}://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} +PROXY_CKAN_LOCATION=/catalog +PROXY_PYCSW_LOCATION=/csw + +# pycsw +PYCSW_PORT=8000 +CKAN_URL=${PROXY_SERVER_URL}${PROXY_CKAN_LOCATION} +PYCSW_URL=${PROXY_SERVER_URL}${PROXY_PYCSW_LOCATION} +# SCHEMAS: ckan2pycsw/model/dataset.py - Dataset type +PYCSW_CKAN_SCHEMA=iso19139_geodcatap +PYCSW_OUPUT_SCHEMA=iso19139_inspire +# ckan-pycsw schedule +## ckan2pycsw days between each scheduler job +PYCSW_CRON_DAYS_INTERVAL=2 +# ckan2pycsw hour of start of the scheduler job (0-23) +PYCSW_CRON_HOUR_START=4 +## Timezone +TZ=UTC + +# CKAN databases +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=postgres +POSTGRES_HOST=db +CKAN_DB_USER=ckandbuser +CKAN_DB_PASSWORD=ckandbpassword +CKAN_DB=ckandb +DATASTORE_READONLY_USER=datastore_ro +DATASTORE_READONLY_PASSWORD=datastore +DATASTORE_DB=datastore +CKAN_SQLALCHEMY_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB} +CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_DB} +CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_DB} + +# Test database connections +CKAN_TEST_DB=ckan_test +DATASTORE_TEST_DB=datastore_test +TEST_CKAN_SQLALCHEMY_URL=postgres://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_TEST_DB} +TEST_CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_TEST_DB} +TEST_CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_TEST_DB} + +# Dev settings +USE_HTTPS_FOR_DEV=false +CKAN_DEV_COMPOSE_SERVICE=ckan-dev + +# CKAN core +## If use docker-compose.ghcr.yml only "*.*.*" versions available in: https://github.com/mjanez/ckan-docker/pkgs/container/ckan-docker +CKAN_VERSION=2.10.5 +CKAN_SITE_ID=default +# CKAN_SITE_URL = http:/ or https:/ + PROXY_SERVER_NAME. Optionally the APACHE_HOST_PORT if different from 80 +CKAN_SITE_URL=${PROXY_SERVER_URL} +CKAN__ROOT_PATH=${PROXY_CKAN_LOCATION}/{{LANG}} +CKAN__FAVICON=${PROXY_CKAN_LOCATION}/base/images/ckan.ico +CKAN__SITE_LOGO=/images/default/ckan-logo.png +# CKAN site info +CKAN__SITE_INTRO_TEXT="" +CKAN__SITE_DESCRIPTION="" +CKAN__SITE_ABOUT="" +# Custom licenses that are aligned with DCAT-AP (Also works with DEV Mode) +LICENSES_GROUP_URL=file://${APP_DIR}/src/ckanext-schemingdcat/ckanext/schemingdcat/public/static/licenses.json +CKAN___BEAKER__SESSION__SECRET=CHANGE_ME +# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings +CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME +CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME +CKAN_SYSADMIN_NAME=ckan_admin +CKAN_SYSADMIN_PASSWORD=test1234 +CKAN_SYSADMIN_EMAIL=your_email@example.com +CKAN_STORAGE_PATH=/var/lib/ckan +CKAN_LOGS_PATH=/var/log +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@${PROXY_SERVER_NAME} +## Customize which text formats the text_view plugin will show +CKAN__PREVIEW__JSON_FORMATS="json jsonld" +# html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json +CKAN__PREVIEW__XML_FORMATS="xml rdf rdf+xml owl+xml atom rss turtle ttl n3 n-triples" +CKAN__PREVIEW__TEXT_FORMATS="text plain text/plain text/turtle csv tsv rss txt json" +CKAN__PREVIEW__LOADABLE="html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json arcgis_rest" +# ckanext-spatial: Allow Solr local params: https://github.com/ckan/ckanext-spatial/issues/328 +CKAN__SEARCH__SOLR_ALLOWED_QUERY_PARSERS=field +# CORS Settings. If True, all origins will be allowed (the response header Access-Control-Allow-Origin is set to ‘*’). If False, only origins from the ckan.cors.origin_whitelist setting will be allowed. +CKAN__CORS__ORIGIN_ALLOW_ALL=False +CKAN__CORS__ORIGIN_WHITELIST="" +# Enables or disable collaborators in individual datasets (https://docs.ckan.org/en/2.9/maintaining/authorization.html#dataset-collaborators) +CKAN__AUTH__ALLOW_DATASET_COLLABORATORS=False +CKAN__AUTH__ALLOW_ADMIN_COLLABORATORS=False +# Default number of facets shown in search results. Default 10. +SEARCH__FACETS__DEFAULT=4 +# Enable or disable the DataStore SQL search backend. Default is False. +CKAN__DATASTORE__SQLSEARCH__ENABLED=True + +## Resource Proxy settings +### Preview size limit, default: 1MB +CKAN__RESOURCE_PROXY__MAX_FILE_SIZE=50048576 +## Size of chunks to read/write__ +CKAN__RESOURCE_PROXY__CHUNK_SIZE=4096 +## Default timeout for fetching proxied items +CKAN__RESOURCE_PROXY__TIMEOUT=10 +CKAN__VIEWS__DEFAULT_VIEWS="image_view video_view audio_view webpage_view text_view datatables_view wmts_view geojson_view geo_view shp_view pdf_view" + +# Localization +CKAN__LOCALE_DEFAULT="en" +CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" +CKAN__LOCALES_OFFERED="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" + +# Extensions +CKAN__PLUGINS="stats image_view video_view audio_view webpage_view text_view datatables_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" + +# ckanext-harvest +CKAN__HARVEST__MQ__TYPE=redis +CKAN__HARVEST__MQ__HOSTNAME=redis +CKAN__HARVEST__MQ__PORT=${REDIS_PORT} +CKAN__HARVEST__MQ__REDIS_DB=${REDIS_CKAN_DATABASE} +# Clean-up mechanism for the harvest log table. The default is 30 days. +CKAN__HARVEST__LOG_TIMEFRAME=40 + +# ckanext-xloader +CKANEXT__XLOADER__JOBS__DB_URI=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB} + +# ckanext-dcat +CKANEXT__DCAT__BASE_URI=${CKAN_URL} +# Default profile(s). Instead of this envvar, it's possible to specify all the profile(s) availables to be used for serialization using the profiles parameter: http://localhost:5000/catalog.xml?profiles=eu_dcat_ap_2,es_dcat +CKANEXT__DCAT__RDF_PROFILES='eu_dcat_ap_2 eu_dcat_ap_scheming' +# The custom endpoint **must** start with a forward slash (`/`) and contain the `{_format}` placeholder. The endpoint is added to the CKAN_SITE_URL and CKAN__ROOT_PATH, example: http://localhost:5000/catalog/catalog.rdf +CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT='/catalog.{_format}' + +# ckanext-spatial (Solr Backend - solr9-spatial: https://docs.ckan.org/projects/ckanext-spatial/en/latest/spatial-search.html#choosing-a-backend-for-the-spatial-search) +CKANEXT__SPATIAL__SEARCH_BACKEND=solr-spatial-field +CKAN__SPATIAL__SRID=3857 +CKANEXT__SPATIAL__COMMON_MAP__TYPE=custom +CKANEXT__SPATIAL__COMMON_MAP__CUSTOM__URL=http://a.tile.openstreetmap.org/{z}/{x}/{y}.png +CKANEXT__SPATIAL__COMMON_MAP__ATTRIBUTION='Map tiles by OpenStreetMap (CC BY SA)' + +# ckanext-geoview +CKANEXT__GEOVIEW__GEOJSON__MAX_FILE_SIZE=100000000 +CKANEXT__GEOVIEW__OL_VIEWER__FORMATS="wms wfs geojson gml kml" +CKANEXT__GEOVIEW__SHP_VIEWER__SRID=3857 +CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8 + +# ckanext-schemingdcat +## CSW Endpoint for spatial metadata +CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI=${PYCSW_URL} +## Scheming: setup_scheming.sh +CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_full.yaml ckanext.schemingdcat:schemas/resources/dcat_document.yaml" +CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_group.json" +CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_org.json" +CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/default_presets.json ckanext.fluent:presets.json" +## Facets: setup_scheming.sh +CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_eu language dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" +CKANEXT__SCHEMINGDCAT_ORGANIZATION_CUSTOM_FACETS=True +CKANEXT__SCHEMINGDCAT_GROUP_CUSTOM_FACETS=True +CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_ICON="theme" +CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_SHOW_SPATIAL=True +CKANEXT__SCHEMINGDCAT_SHOW_METADATA_TEMPLATES_TOOLBAR=False +CKANEXT__METADATA_TEMPLATES_SEARCH_IDENTIFIER="schemingdcat_xls-template" +CKANEXT__SCHEMINGDCAT_ENDPOINTS_YAML="endpoints.yaml" + +# ckanext-pages +CKANEXT__PAGES__ALOW_HTML=False +CKANEXT__PAGES__ORGANIZATION=True +CKANEXT__PAGES__GROUP=True +CKANEXT__PAGES__ABOUT_MENU=False +CKANEXT__PAGES__GROUP_MENU=True +CKANEXT__PAGES__ORGANIZATION_MENU=True \ No newline at end of file diff --git a/samples/.env.es.example b/samples/.env.es.example index e19c32c8..be78eddd 100644 --- a/samples/.env.es.example +++ b/samples/.env.es.example @@ -7,11 +7,13 @@ APP_DIR=/srv/app CKAN_PORT_HOST=5000 # Common proxy server for Apache or NGINX. # Change all the PROXY_SERVER_PORT_HOST to PROXY_SERVER_HTTPS_PORT_HOST if you can use HTTPS instead of HTTP (Only for NGINX and not development compose) -PROXY_SERVER_PORT_HOST=81 +PROXY_SERVER_BASE_PORT_HOST=81 PROXY_SERVER_HTTPS_PORT_HOST=8443 -NGINX_PORT_HOST=${PROXY_SERVER_PORT_HOST} +# Server port for the proxy server. Better using the SSL port for production. +PROXY_SERVER_PORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} +NGINX_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} NGINX_SSLPORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} -APACHE_PORT_HOST=${PROXY_SERVER_PORT_HOST} +APACHE_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} PYCSW_PORT_HOST=8000 # Solr @@ -41,7 +43,9 @@ APACHE_LOG_DIR=/var/log/apache #NGINX/APACHE ## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI and CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. If you dont need PROXY_SERVER_PORT_HOST (e.g. is 80), edit the PROXY_SERVER_URL and remove :${PROXY_SERVER_PORT_HOST} PROXY_SERVER_NAME=localhost -PROXY_SERVER_URL=http://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} +# URL protocol. Change to https if you use SSL +PROXY_SERVER_PROTOCOL=http +PROXY_SERVER_URL=${PROXY_SERVER_PROTOCOL}://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} PROXY_CKAN_LOCATION=/catalog PROXY_PYCSW_LOCATION=/csw @@ -88,7 +92,7 @@ CKAN_DEV_COMPOSE_SERVICE=ckan-dev # CKAN core ## If use docker-compose.ghcr.yml only "*.*.*" versions available in: https://github.com/mjanez/ckan-docker/pkgs/container/ckan-docker -CKAN_VERSION=2.9.11 +CKAN_VERSION=2.10.5 CKAN_SITE_ID=default # CKAN_SITE_URL = http:/ or https:/ + PROXY_SERVER_NAME. Optionally the APACHE_HOST_PORT if different from 80 CKAN_SITE_URL=${PROXY_SERVER_URL} @@ -99,8 +103,8 @@ CKAN__SITE_LOGO=/images/default/ckan-logo.png CKAN__SITE_INTRO_TEXT="" CKAN__SITE_DESCRIPTION="" CKAN__SITE_ABOUT="" -# Custom licenses that are aligned with DCAT-AP -CKAN___LICENSES_GROUP_URL=https://raw.githubusercontent.com/mjanez/ckanext-schemingdcat/main/ckanext/schemingdcat/public/static/licenses.json +# Custom licenses that are aligned with DCAT-AP (Also works with DEV Mode) +LICENSES_GROUP_URL=file://${APP_DIR}/src/ckanext-schemingdcat/ckanext/schemingdcat/public/static/licenses.json CKAN___BEAKER__SESSION__SECRET=CHANGE_ME # See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME @@ -131,6 +135,8 @@ CKAN__AUTH__ALLOW_DATASET_COLLABORATORS=False CKAN__AUTH__ALLOW_ADMIN_COLLABORATORS=False # Default number of facets shown in search results. Default 10. SEARCH__FACETS__DEFAULT=4 +# Enable or disable the DataStore SQL search backend. Default is False. +CKAN__DATASTORE__SQLSEARCH__ENABLED=True ## Resource Proxy settings ### Preview size limit, default: 1MB @@ -139,7 +145,7 @@ CKAN__RESOURCE_PROXY__MAX_FILE_SIZE=50048576 CKAN__RESOURCE_PROXY__CHUNK_SIZE=4096 ## Default timeout for fetching proxied items CKAN__RESOURCE_PROXY__TIMEOUT=10 -CKAN__VIEWS__DEFAULT_VIEWS="image_view webpage_view text_view recline_view wmts_view geojson_view geo_view shp_view pdf_view" +CKAN__VIEWS__DEFAULT_VIEWS="image_view video_view audio_view webpage_view text_view datatables_view wmts_view geojson_view geo_view shp_view pdf_view" # Localization CKAN__LOCALE_DEFAULT="es" @@ -158,7 +164,6 @@ CKAN__HARVEST__MQ__REDIS_DB=${REDIS_CKAN_DATABASE} CKAN__HARVEST__LOG_TIMEFRAME=40 # ckanext-xloader -CKANEXT__XLOADER__API_TOKEN=api_token CKANEXT__XLOADER__JOBS__DB_URI=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB} # ckanext-dcat diff --git a/samples/.env.localhost b/samples/.env.localhost index 5d622ed3..51d1b79c 100644 --- a/samples/.env.localhost +++ b/samples/.env.localhost @@ -7,11 +7,13 @@ APP_DIR=/srv/app CKAN_PORT_HOST=5000 # Common proxy server for Apache or NGINX. # Change all the PROXY_SERVER_PORT_HOST to PROXY_SERVER_HTTPS_PORT_HOST if you can use HTTPS instead of HTTP (Only for NGINX and not development compose) -PROXY_SERVER_PORT_HOST=81 +PROXY_SERVER_BASE_PORT_HOST=81 PROXY_SERVER_HTTPS_PORT_HOST=8443 -NGINX_PORT_HOST=${PROXY_SERVER_PORT_HOST} +# Server port for the proxy server. Better using the SSL port for production. +PROXY_SERVER_PORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} +NGINX_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} NGINX_SSLPORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} -APACHE_PORT_HOST=${PROXY_SERVER_PORT_HOST} +APACHE_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} PYCSW_PORT_HOST=8000 # Solr @@ -41,7 +43,9 @@ APACHE_LOG_DIR=/var/log/apache #NGINX/APACHE ## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI and CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. If you dont need PROXY_SERVER_PORT_HOST (e.g. is 80), edit the PROXY_SERVER_URL and remove :${PROXY_SERVER_PORT_HOST} PROXY_SERVER_NAME=localhost -PROXY_SERVER_URL=http://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} +# URL protocol. Change to https if you use SSL +PROXY_SERVER_PROTOCOL=http +PROXY_SERVER_URL=${PROXY_SERVER_PROTOCOL}://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} PROXY_CKAN_LOCATION=/catalog PROXY_PYCSW_LOCATION=/csw @@ -88,15 +92,19 @@ CKAN_DEV_COMPOSE_SERVICE=ckan-dev # CKAN core ## If use docker-compose.ghcr.yml only "*.*.*" versions available in: https://github.com/mjanez/ckan-docker/pkgs/container/ckan-docker -CKAN_VERSION=2.9.11 +CKAN_VERSION=2.10.5 CKAN_SITE_ID=default # CKAN_SITE_URL = http:/ or https:/ + PROXY_SERVER_NAME. Optionally the APACHE_HOST_PORT if different from 80 CKAN_SITE_URL=${PROXY_SERVER_URL} CKAN__ROOT_PATH=${PROXY_CKAN_LOCATION}/{{LANG}} CKAN__FAVICON=${PROXY_CKAN_LOCATION}/base/images/ckan.ico CKAN__SITE_LOGO=/images/default/ckan-logo.png -# Custom licenses that are aligned with DCAT-AP -CKAN___LICENSES_GROUP_URL=https://raw.githubusercontent.com/mjanez/ckanext-schemingdcat/main/ckanext/schemingdcat/public/static/licenses.json +# CKAN site info +CKAN__SITE_INTRO_TEXT="" +CKAN__SITE_DESCRIPTION="" +CKAN__SITE_ABOUT="" +# Custom licenses that are aligned with DCAT-AP (Also works with DEV Mode) +LICENSES_GROUP_URL=file://${APP_DIR}/src/ckanext-schemingdcat/ckanext/schemingdcat/public/static/licenses.json CKAN___BEAKER__SESSION__SECRET=CHANGE_ME # See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME @@ -122,6 +130,13 @@ CKAN__SEARCH__SOLR_ALLOWED_QUERY_PARSERS=field # CORS Settings. If True, all origins will be allowed (the response header Access-Control-Allow-Origin is set to ‘*’). If False, only origins from the ckan.cors.origin_whitelist setting will be allowed. CKAN__CORS__ORIGIN_ALLOW_ALL=False CKAN__CORS__ORIGIN_WHITELIST="" +# Enables or disable collaborators in individual datasets (https://docs.ckan.org/en/2.9/maintaining/authorization.html#dataset-collaborators) +CKAN__AUTH__ALLOW_DATASET_COLLABORATORS=False +CKAN__AUTH__ALLOW_ADMIN_COLLABORATORS=False +# Default number of facets shown in search results. Default 10. +SEARCH__FACETS__DEFAULT=4 +# Enable or disable the DataStore SQL search backend. Default is False. +CKAN__DATASTORE__SQLSEARCH__ENABLED=True ## Resource Proxy settings ### Preview size limit, default: 1MB @@ -130,7 +145,7 @@ CKAN__RESOURCE_PROXY__MAX_FILE_SIZE=50048576 CKAN__RESOURCE_PROXY__CHUNK_SIZE=4096 ## Default timeout for fetching proxied items CKAN__RESOURCE_PROXY__TIMEOUT=10 -CKAN__VIEWS__DEFAULT_VIEWS="image_view webpage_view text_view recline_view wmts_view geojson_view geo_view shp_view pdf_view" +CKAN__VIEWS__DEFAULT_VIEWS="image_view video_view audio_view webpage_view text_view datatables_view wmts_view geojson_view geo_view shp_view pdf_view" # Localization CKAN__LOCALE_DEFAULT="en" @@ -138,7 +153,7 @@ CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru CKAN__LOCALES_OFFERED="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" # Extensions -CKAN__PLUGINS="stats image_view text_view recline_view webpage_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" +CKAN__PLUGINS="stats image_view video_view audio_view webpage_view text_view datatables_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" # ckanext-harvest CKAN__HARVEST__MQ__TYPE=redis @@ -149,7 +164,6 @@ CKAN__HARVEST__MQ__REDIS_DB=${REDIS_CKAN_DATABASE} CKAN__HARVEST__LOG_TIMEFRAME=40 # ckanext-xloader -CKANEXT__XLOADER__API_TOKEN=api_token CKANEXT__XLOADER__JOBS__DB_URI=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB} # ckanext-dcat @@ -176,12 +190,12 @@ CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8 ## CSW Endpoint for spatial metadata CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI=${PYCSW_URL} ## Scheming: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/es_geodcat_ap_full.yaml ckanext.schemingdcat:schemas/resources/dcat_document.yaml" +CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_full.yaml ckanext.schemingdcat:schemas/resources/dcat_document.yaml" CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_group.json" CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_org.json" CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/default_presets.json ckanext.fluent:presets.json" ## Facets: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_eu dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" +CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_eu language dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" CKANEXT__SCHEMINGDCAT_ORGANIZATION_CUSTOM_FACETS=True CKANEXT__SCHEMINGDCAT_GROUP_CUSTOM_FACETS=True CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_ICON="theme" From 10e3db2ba5bf2678578b0eadc3c37de17fa58190 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:30:11 +0200 Subject: [PATCH 08/25] Update .gitignore to include .env.codespaces file --- .gitignore | 1 + doc/info_envfile.md | 1 - src/README.md | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 94e41881..d4f38ae4 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ index.html.* # environment allowlist !.env.example +!.env.codespaces !.env.es.example !.env.localhost samples/api/* diff --git a/doc/info_envfile.md b/doc/info_envfile.md index 9d379b1d..2fb1867c 100644 --- a/doc/info_envfile.md +++ b/doc/info_envfile.md @@ -156,7 +156,6 @@ This is a CKAN extension that allows users to harvest dataset metadata from mult ### ckanext-xloader This is a CKAN extension that allows users to load data into CKAN's data store for faster and more efficient processing. -- `CKANEXT__XLOADER__API_TOKEN`: The API token for the XLoader extension. - `CKANEXT__XLOADER__JOBS__DB_URI`: The database URI for the XLoader jobs. ### ckanext-dcat diff --git a/src/README.md b/src/README.md index 9979d741..c6e73072 100644 --- a/src/README.md +++ b/src/README.md @@ -18,9 +18,9 @@ fi # Define repositories and tags declare -A repos=( - ["ckan/ckanext-xloader"]="1.0.1" + ["mjanez/ckanext-xloader"]="1.0.1-stable" ["ckan/ckanext-harvest"]="v1.5.6" - ["ckan/ckanext-geoview"]="v0.1.0" + ["ckan/ckanext-geoview"]="v0.2.0" ["ckan/ckanext-spatial"]="v2.1.1" ["mjanez/ckanext-dcat"]="v1.8.0" ["ckan/ckanext-scheming"]="release-3.0.0" From 7a819c3a30142d6ad342de9306dafdab53376016 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:03:52 +0200 Subject: [PATCH 09/25] Update .gitignore to include docker-compose.local-*.yml file --- .env.example | 2 +- .gitignore | 3 +++ samples/.env.codespaces | 2 +- samples/.env.es.example | 2 +- samples/.env.localhost | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 51d1b79c..ae6207c3 100644 --- a/.env.example +++ b/.env.example @@ -9,7 +9,7 @@ CKAN_PORT_HOST=5000 # Change all the PROXY_SERVER_PORT_HOST to PROXY_SERVER_HTTPS_PORT_HOST if you can use HTTPS instead of HTTP (Only for NGINX and not development compose) PROXY_SERVER_BASE_PORT_HOST=81 PROXY_SERVER_HTTPS_PORT_HOST=8443 -# Server port for the proxy server. Better using the SSL port for production. +# Server port for the proxy server. Better using the SSL port for production, in DEV or if using balancer: PROXY_SERVER_PORT_HOST=PROXY_SERVER_BASE_PORT_HOST PROXY_SERVER_PORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} NGINX_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} NGINX_SSLPORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} diff --git a/.gitignore b/.gitignore index d4f38ae4..609b3048 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ index.html.* !.env.es.example !.env.localhost samples/api/* + +# docker compose files +docker-compose.local-*.yml diff --git a/samples/.env.codespaces b/samples/.env.codespaces index 4ec58c1f..390cb336 100644 --- a/samples/.env.codespaces +++ b/samples/.env.codespaces @@ -12,7 +12,7 @@ CKAN_PORT_HOST=5000 # Change all the PROXY_SERVER_PORT_HOST to PROXY_SERVER_HTTPS_PORT_HOST if you can use HTTPS instead of HTTP (Only for NGINX and not development compose) PROXY_SERVER_BASE_PORT_HOST=81 PROXY_SERVER_HTTPS_PORT_HOST=8443 -# Server port for the proxy server. Better using the SSL port for production. +# Server port for the proxy server. Better using the SSL port for production, in DEV or if using balancer: PROXY_SERVER_PORT_HOST=PROXY_SERVER_BASE_PORT_HOST PROXY_SERVER_PORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} NGINX_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} NGINX_SSLPORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} diff --git a/samples/.env.es.example b/samples/.env.es.example index be78eddd..103e734f 100644 --- a/samples/.env.es.example +++ b/samples/.env.es.example @@ -9,7 +9,7 @@ CKAN_PORT_HOST=5000 # Change all the PROXY_SERVER_PORT_HOST to PROXY_SERVER_HTTPS_PORT_HOST if you can use HTTPS instead of HTTP (Only for NGINX and not development compose) PROXY_SERVER_BASE_PORT_HOST=81 PROXY_SERVER_HTTPS_PORT_HOST=8443 -# Server port for the proxy server. Better using the SSL port for production. +# Server port for the proxy server. Better using the SSL port for production, in DEV or if using balancer: PROXY_SERVER_PORT_HOST=PROXY_SERVER_BASE_PORT_HOST PROXY_SERVER_PORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} NGINX_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} NGINX_SSLPORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} diff --git a/samples/.env.localhost b/samples/.env.localhost index 51d1b79c..ae6207c3 100644 --- a/samples/.env.localhost +++ b/samples/.env.localhost @@ -9,7 +9,7 @@ CKAN_PORT_HOST=5000 # Change all the PROXY_SERVER_PORT_HOST to PROXY_SERVER_HTTPS_PORT_HOST if you can use HTTPS instead of HTTP (Only for NGINX and not development compose) PROXY_SERVER_BASE_PORT_HOST=81 PROXY_SERVER_HTTPS_PORT_HOST=8443 -# Server port for the proxy server. Better using the SSL port for production. +# Server port for the proxy server. Better using the SSL port for production, in DEV or if using balancer: PROXY_SERVER_PORT_HOST=PROXY_SERVER_BASE_PORT_HOST PROXY_SERVER_PORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} NGINX_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} NGINX_SSLPORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} From d31279779b432edf69cfc0e2a354895bb4c55ff2 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:00:41 +0000 Subject: [PATCH 10/25] Fix start_ckan.sh.override for clearer CSRF check and update PROXY_SERVER_URL in .env.codespaces --- ckan/setup/start_ckan.sh.override | 4 ++-- samples/.env.codespaces | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/setup/start_ckan.sh.override b/ckan/setup/start_ckan.sh.override index e30913d5..71a08431 100644 --- a/ckan/setup/start_ckan.sh.override +++ b/ckan/setup/start_ckan.sh.override @@ -13,8 +13,8 @@ then fi # Github Codespaces workaround. Check if WTF_CSRF_ENABLED is explicitly set to False and update CKAN_INI -if grep -qE "^WTF_CSRF_ENABLED ?= ?False$" $CKAN_INI; then - echo "WTF_CSRF_ENABLED is explicitly set to False. Updating CKAN_INI." +if [ "$WTF_CSRF_ENABLED" = "False" ]; then + echo "WTF_CSRF_ENABLED is set to False. Updating CKAN_INI." ckan config-tool $CKAN_INI "WTF_CSRF_ENABLED=False" fi diff --git a/samples/.env.codespaces b/samples/.env.codespaces index 4ec58c1f..2e42ef3a 100644 --- a/samples/.env.codespaces +++ b/samples/.env.codespaces @@ -48,7 +48,7 @@ APACHE_LOG_DIR=/var/log/apache PROXY_SERVER_NAME=localhost # URL protocol. Change to https if you use SSL PROXY_SERVER_PROTOCOL=https -PROXY_SERVER_URL=${PROXY_SERVER_PROTOCOL}://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} +PROXY_SERVER_URL=${PROXY_SERVER_PROTOCOL}://${PROXY_SERVER_NAME} PROXY_CKAN_LOCATION=/catalog PROXY_PYCSW_LOCATION=/csw From 6adab9d26294b8f115c4f4a9e471dde4aefdcad2 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:01:08 +0000 Subject: [PATCH 11/25] Remove unnecesary who.ini config --- ckan/docker-entrypoint.d/00_update_who.sh | 15 ----------- ckan/setup/who.ini | 32 ----------------------- 2 files changed, 47 deletions(-) delete mode 100644 ckan/docker-entrypoint.d/00_update_who.sh delete mode 100644 ckan/setup/who.ini diff --git a/ckan/docker-entrypoint.d/00_update_who.sh b/ckan/docker-entrypoint.d/00_update_who.sh deleted file mode 100644 index b214ffc6..00000000 --- a/ckan/docker-entrypoint.d/00_update_who.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -# Update who.ini when exists PROXY_CKAN_LOCATION -echo "[docker-entrypoint.00_update_who] Update who.ini" -if [ -n "$PROXY_CKAN_LOCATION" ] && [ "$PROXY_CKAN_LOCATION" != "/" ]; then - sed -i "s|\${WHO_LOCATION}|$PROXY_CKAN_LOCATION|g" "${APP_DIR}/who.ini"; -else - # Check if the value is exactly "/" - if [ "$PROXY_CKAN_LOCATION" = "/" ]; then - sed -i "s|\${WHO_LOCATION}||g" "${APP_DIR}/who.ini"; - else - # Handle the case when $PROXY_CKAN_LOCATION is empty or not set - sed -i "s|\${WHO_LOCATION}|/|g" "${APP_DIR}/who.ini"; - fi -fi diff --git a/ckan/setup/who.ini b/ckan/setup/who.ini deleted file mode 100644 index 34a753ff..00000000 --- a/ckan/setup/who.ini +++ /dev/null @@ -1,32 +0,0 @@ -[plugin:auth_tkt] -use=ckan.lib.repoze_plugins.auth_tkt:make_plugin -# If no secret key is defined here, beaker.session.secret will be used -#secret = somesecret - -[plugin:friendlyform] -use=ckan.lib.repoze_plugins.friendly_form:FriendlyFormPlugin -login_form_url=${WHO_LOCATION}/user/login -login_handler_path=/login_generic -logout_handler_path=/user/logout -rememberer_name=auth_tkt -post_login_url=${WHO_LOCATION}/user/logged_in -post_logout_url=${WHO_LOCATION}/user/logged_out -charset=utf-8 - -[general] -request_classifier=repoze.who.classifiers:default_request_classifier -challenge_decider=repoze.who.classifiers:default_challenge_decider - -[identifiers] -plugins = - friendlyform;browser - auth_tkt - -[authenticators] -plugins = - auth_tkt - ckan.lib.authenticator:UsernamePasswordAuthenticator - -[challengers] -plugins = - friendlyform;browser \ No newline at end of file From 916d40542ed7cd7cdd59da115066fd1661ca26db Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:08:32 +0200 Subject: [PATCH 12/25] Update CSW catalog information with envvars - Add activity plugin - Add icons and improve ckan_site/description info. --- .env.example | 41 ++++++++++++++++++---- ckan-pycsw/Dockerfile | 21 ++++++++++++ ckan-pycsw/pycsw/conf/pycsw.conf.template | 42 +++++++++++------------ 3 files changed, 77 insertions(+), 27 deletions(-) diff --git a/.env.example b/.env.example index ae6207c3..6f6798b4 100644 --- a/.env.example +++ b/.env.example @@ -98,11 +98,12 @@ CKAN_SITE_ID=default CKAN_SITE_URL=${PROXY_SERVER_URL} CKAN__ROOT_PATH=${PROXY_CKAN_LOCATION}/{{LANG}} CKAN__FAVICON=${PROXY_CKAN_LOCATION}/base/images/ckan.ico -CKAN__SITE_LOGO=/images/default/ckan-logo.png +CKAN__SITE_LOGO=${PROXY_CKAN_LOCATION}/images/default/ckan-logo.png # CKAN site info -CKAN__SITE_INTRO_TEXT="" -CKAN__SITE_DESCRIPTION="" -CKAN__SITE_ABOUT="" +CKAN__SITE_TITLE="CKAN Open Data" +CKAN__SITE_DESCRIPTION="ckan-docker *spatial Open Data portal" +CKAN__SITE_INTRO_TEXT="[![ckan-docker-spatial](https://img.shields.io/badge/ckan_docker_spatial-CKAN_${CKAN_VERSION}-brightgreen?style=social&logo=github#center)](https://github.com/mjanez/ckan-docker)\n[`ckan-docker *spatial`](https://github.com/mjanez/ckan-docker) is a ready-to-use development environment for [CKAN](https://github.com/ckan/ckan), the world’s leading open-source data portal platform." +CKAN__SITE_ABOUT=${CKAN__SITE_INTRO_TEXT} # Custom licenses that are aligned with DCAT-AP (Also works with DEV Mode) LICENSES_GROUP_URL=file://${APP_DIR}/src/ckanext-schemingdcat/ckanext/schemingdcat/public/static/licenses.json CKAN___BEAKER__SESSION__SECRET=CHANGE_ME @@ -153,7 +154,7 @@ CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru CKAN__LOCALES_OFFERED="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" # Extensions -CKAN__PLUGINS="stats image_view video_view audio_view webpage_view text_view datatables_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" +CKAN__PLUGINS="activity stats image_view video_view audio_view webpage_view text_view datatables_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" # ckanext-harvest CKAN__HARVEST__MQ__TYPE=redis @@ -203,6 +204,9 @@ CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_SHOW_SPATIAL=True CKANEXT__SCHEMINGDCAT_SHOW_METADATA_TEMPLATES_TOOLBAR=False CKANEXT__METADATA_TEMPLATES_SEARCH_IDENTIFIER="schemingdcat_xls-template" CKANEXT__SCHEMINGDCAT_ENDPOINTS_YAML="endpoints.yaml" +CKANEXT__SCHEMINGDCAT__SOCIAL_GITHUB="https://github.com/mjanez/ckanext-schemingdcat" +CKANEXT__SCHEMINGDCAT__SOCIAL_X="https://x.com/ckanproject" +CKANEXT__SCHEMINGDCAT__SOCIAL_LINKEDIN="https://www.linkedin.com/company/ckanproject" # ckanext-pages CKANEXT__PAGES__ALOW_HTML=False @@ -210,4 +214,29 @@ CKANEXT__PAGES__ORGANIZATION=True CKANEXT__PAGES__GROUP=True CKANEXT__PAGES__ABOUT_MENU=False CKANEXT__PAGES__GROUP_MENU=True -CKANEXT__PAGES__ORGANIZATION_MENU=True \ No newline at end of file +CKANEXT__PAGES__ORGANIZATION_MENU=True + +# ckan-pycsw +CSW_IDENTIFICATION_TITLE="Sample Geospatial Catalogue" +CSW_IDENTIFICATION_ABSTRACT="OGC CSW server powered by pycsw" +CSW_PROVIDER_NAME="ckan-docker development team" +CSW_PROVIDER_URL="https://github.com/mjanez/ckan-docker" +CSW_CONTACT_NAME="ckan-docker development team" +CSW_CONTACT_POSITION="Site Administrator" +CSW_CONTACT_ADDRESS="ckan-docker development team" +CSW_CONTACT_CITY="Madrid" +CSW_CONTACT_STATE_OR_PROVINCE="Madrid" +CSW_CONTACT_POSTAL_CODE="28001" +CSW_CONTACT_COUNTRY="Spain" +CSW_CONTACT_EMAIL=${CKAN_SYSADMIN_EMAIL} +CSW_CONTACT_URL=${CKAN_URL} +## Date of the last update of the metadata. ISO 8601 format): YYYY-MM-DD +CSW_INSPIRE_DATE="2024-01-01" +## Keywords separated by commas based on the GEMET Thesaurus keywordValue: https://inspire.ec.europa.eu/schemas/common/1.0/enums/enum_eng.xsd +CSW_INSPIRE_GEMET_KEYWORDS="Utility and governmental services" +# The level of INSPIRE conformance for spatial data sets and services (conformant, notConformant, notEvaluated) +CSW_INSPIRE_CONFORMITY="notEvaluated" +CSW_INSPIRE_CONTACT_NAME=${CSW_CONTACT_NAME} +CSW_INSPIRE_CONTACT_EMAIL=${CKAN_SYSADMIN_EMAIL} +# Temporal extent of the service (in ISO 8601 format). Either a single date (i.e. YYYY-MM-DD), or an extent (i.e. YYYY-MM-DD/YYYY-MM-DD) +CSW_INSPIRE_TEMP_EXTENT="2024-01-01" diff --git a/ckan-pycsw/Dockerfile b/ckan-pycsw/Dockerfile index 4fa1f0e9..4052ed4d 100644 --- a/ckan-pycsw/Dockerfile +++ b/ckan-pycsw/Dockerfile @@ -15,6 +15,27 @@ ENV TIMEOUT=300 ENV PYCSW_CRON_DAYS_INTERVAL=2 ENV PYCSW_CRON_HOUR_START=4 +# PYCSW Catalog configuration +ENV CSW_IDENTIFICATION_TITLE="Sample Geospatial Catalogue" \ + CSW_IDENTIFICATION_ABSTRACT="OGC CSW server powered by pycsw" \ + CSW_PROVIDER_NAME="ckan-docker development team" \ + CSW_PROVIDER_URL="https://github.com/mjanez/ckan-docker" \ + CSW_CONTACT_NAME="ckan-docker development team" \ + CSW_CONTACT_POSITION="Site Administrator" \ + CSW_CONTACT_ADDRESS="ckan-docker development team" \ + CSW_CONTACT_CITY="Madrid" \ + CSW_CONTACT_STATE_OR_PROVINCE="Madrid" \ + CSW_CONTACT_POSTAL_CODE="28001" \ + CSW_CONTACT_COUNTRY="Spain" \ + CSW_CONTACT_EMAIL=${CKAN_SYSADMIN_EMAIL} \ + CSW_CONTACT_URL=${CKAN_URL} \ + CSW_INSPIRE_DATE="2024-01-01" \ + CSW_INSPIRE_GEMET_KEYWORDS="Utility and governmental services" \ + CSW_INSPIRE_CONFORMITY="notEvaluated" \ + CSW_INSPIRE_CONTACT_NAME=${CSW_CONTACT_NAME} \ + CSW_INSPIRE_CONTACT_EMAIL=${CKAN_SYSADMIN_EMAIL} \ + CSW_INSPIRE_TEMP_EXTENT="2024-01-01/2024-12-31" + WORKDIR ${APP_DIR} # Update files if needed diff --git a/ckan-pycsw/pycsw/conf/pycsw.conf.template b/ckan-pycsw/pycsw/conf/pycsw.conf.template index 8fa945b2..e15908d9 100644 --- a/ckan-pycsw/pycsw/conf/pycsw.conf.template +++ b/ckan-pycsw/pycsw/conf/pycsw.conf.template @@ -23,27 +23,27 @@ allowed_ips=127.0.0.1 #csw_harvest_pagesize=10 [metadata:main] -identification_title=Sample Geospatial Catalogue -identification_abstract=OGC CSW server powered by pycsw +identification_title=${CSW_IDENTIFICATION_TITLE} +identification_abstract=${CSW_IDENTIFICATION_ABSTRACT} identification_keywords=catalogue,discovery,metadata,spatial identification_keywords_type=theme identification_fees=None identification_accessconstraints=None -provider_name=Sample -provider_url=https://example.org/ -#contact_name=admin -#contact_position=Position Title -#contact_address=Mailing Address -#contact_city=Madrid -#contact_stateorprovince=Administrative Area -#contact_postalcode=Zip or Postal Code -contact_country=Spain +provider_name=${CSW_PROVIDER_NAME} +provider_url=${CSW_PROVIDER_URL} +contact_name=${CSW_CONTACT_NAME} +contact_position=${CSW_CONTACT_POSITION} +contact_address=${CSW_CONTACT_ADDRESS} +contact_city=${CSW_CONTACT_CITY} +contact_stateorprovince=${CSW_CONTACT_STATE_OR_PROVINCE} +contact_postalcode=${CSW_CONTACT_POSTAL_CODE} +contact_country=${CSW_CONTACT_COUNTRY} #contact_phone=+xx-xxx-xxx-xxxx #contact_fax=+xx-xxx-xxx-xxxx -contact_email=admin@example.org -#contact_url=https://example.org/members/admin -#contact_hours=Hours of Service -#contact_instructions=During hours of service. Off on weekends. +contact_email=${CSW_CONTACT_EMAIL} +contact_url=${CSW_CONTACT_URL} +contact_hours=Hours of Service +contact_instructions=During hours of service. Off on weekends. contact_role=pointOfContact [repository] @@ -61,9 +61,9 @@ table=records enabled=true languages_supported=eng,spa default_language=eng -date=YYYY-MM-DD -gemet_keywords=Utility and governmental services -conformity_service=notEvaluated -contact_name=Organization Name -contact_email=Email Address -temp_extent=YYYY-MM-DD/YYYY-MM-DD \ No newline at end of file +date=${CSW_INSPIRE_DATE} +gemet_keywords=${CSW_INSPIRE_GEMET_KEYWORDS} +conformity_service=${CSW_INSPIRE_CONFORMITY} +contact_name=${CSW_INSPIRE_CONTACT_NAME} +contact_email=${CSW_INSPIRE_CONTACT_EMAIL} +temp_extent=${CSW_INSPIRE_TEMP_EXTENT} \ No newline at end of file From ae97c9120b5bab908a1e8985d112639d23789618 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:08:47 +0200 Subject: [PATCH 13/25] Remove unnecessary files related to updating who.ini --- ckan/docker-entrypoint.d/00_update_who.sh | 15 ----------- ckan/setup/who.ini | 32 ----------------------- 2 files changed, 47 deletions(-) delete mode 100644 ckan/docker-entrypoint.d/00_update_who.sh delete mode 100644 ckan/setup/who.ini diff --git a/ckan/docker-entrypoint.d/00_update_who.sh b/ckan/docker-entrypoint.d/00_update_who.sh deleted file mode 100644 index b214ffc6..00000000 --- a/ckan/docker-entrypoint.d/00_update_who.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -# Update who.ini when exists PROXY_CKAN_LOCATION -echo "[docker-entrypoint.00_update_who] Update who.ini" -if [ -n "$PROXY_CKAN_LOCATION" ] && [ "$PROXY_CKAN_LOCATION" != "/" ]; then - sed -i "s|\${WHO_LOCATION}|$PROXY_CKAN_LOCATION|g" "${APP_DIR}/who.ini"; -else - # Check if the value is exactly "/" - if [ "$PROXY_CKAN_LOCATION" = "/" ]; then - sed -i "s|\${WHO_LOCATION}||g" "${APP_DIR}/who.ini"; - else - # Handle the case when $PROXY_CKAN_LOCATION is empty or not set - sed -i "s|\${WHO_LOCATION}|/|g" "${APP_DIR}/who.ini"; - fi -fi diff --git a/ckan/setup/who.ini b/ckan/setup/who.ini deleted file mode 100644 index 34a753ff..00000000 --- a/ckan/setup/who.ini +++ /dev/null @@ -1,32 +0,0 @@ -[plugin:auth_tkt] -use=ckan.lib.repoze_plugins.auth_tkt:make_plugin -# If no secret key is defined here, beaker.session.secret will be used -#secret = somesecret - -[plugin:friendlyform] -use=ckan.lib.repoze_plugins.friendly_form:FriendlyFormPlugin -login_form_url=${WHO_LOCATION}/user/login -login_handler_path=/login_generic -logout_handler_path=/user/logout -rememberer_name=auth_tkt -post_login_url=${WHO_LOCATION}/user/logged_in -post_logout_url=${WHO_LOCATION}/user/logged_out -charset=utf-8 - -[general] -request_classifier=repoze.who.classifiers:default_request_classifier -challenge_decider=repoze.who.classifiers:default_challenge_decider - -[identifiers] -plugins = - friendlyform;browser - auth_tkt - -[authenticators] -plugins = - auth_tkt - ckan.lib.authenticator:UsernamePasswordAuthenticator - -[challengers] -plugins = - friendlyform;browser \ No newline at end of file From a7c2031f2a451b54c8449008defe9d6cb79a646c Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:08:55 +0200 Subject: [PATCH 14/25] Update CKAN core settings and ckanext-schemingdcat settings --- ckan/docker-entrypoint.d/01_setup_ckanext_config.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh b/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh index d786aa95..d4ead00e 100644 --- a/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh +++ b/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh @@ -7,7 +7,8 @@ ckan -c $CKAN_INI search-index clear # ckan: Update core settings echo "[docker-entrypoint.01_setup_ckanext_config] Loading CKAN core settings in the CKAN config file" ckan config-tool $CKAN_INI \ - "search.facets.default = $SEARCH__FACETS__DEFAULT" + "search.facets.default = $SEARCH__FACETS__DEFAULT" \ + "ckan.datastore.sqlsearch.enabled = $CKAN__DATASTORE__SQLSEARCH__ENABLED" # ckanext-schemingdcat: Update settings echo "[docker-entrypoint.01_setup_ckanext_config] Loading ckanext-scheming and ckanext-schemingdcat settings into ckan.ini" From a646b491251c3554230f488f4b2ac6b4bd9ae4f8 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:12:15 +0200 Subject: [PATCH 15/25] Add patch to fix ckanext-pages - org/group templates using c.group_dict instead of newer group_dict. https://github.com/mjanez/ckanext-pages/pull/2 --- .../ckanext-pages/01_org_group_templates | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 ckan/patches/ckanext-pages/01_org_group_templates diff --git a/ckan/patches/ckanext-pages/01_org_group_templates b/ckan/patches/ckanext-pages/01_org_group_templates new file mode 100644 index 00000000..94b74a69 --- /dev/null +++ b/ckan/patches/ckanext-pages/01_org_group_templates @@ -0,0 +1,30 @@ +diff --git a/ckanext/pages/theme/templates_group/group/read_base.html b/ckanext/pages/theme/templates_group/group/read_base.html +index bbdae8f..6bf00fa 100644 +--- a/ckanext/pages/theme/templates_group/group/read_base.html ++++ b/ckanext/pages/theme/templates_group/group/read_base.html +@@ -3,8 +3,8 @@ + {% block content_primary_nav %} + {{ super() }} + {% if h.ckan_version().split('.')[1] | int >= 9 %} +- {{ h.build_nav_icon('pages.group_pages_index', _('Pages'), id=c.group_dict.name, icon='file') }} ++ {{ h.build_nav_icon('pages.group_pages_index', _('Pages'), id=group_dict.name, icon='file') }} + {% else %} +- {{ h.build_nav_icon('group_pages_index', _('Pages'), id=c.group_dict.name, icon='file') }} ++ {{ h.build_nav_icon('group_pages_index', _('Pages'), id=group_dict.name, icon='file') }} + {% endif %} + {% endblock %} +diff --git a/ckanext/pages/theme/templates_organization/organization/read_base.html b/ckanext/pages/theme/templates_organization/organization/read_base.html +index e5f3b47..eb2f39b 100644 +--- a/ckanext/pages/theme/templates_organization/organization/read_base.html ++++ b/ckanext/pages/theme/templates_organization/organization/read_base.html +@@ -3,8 +3,8 @@ + {% block content_primary_nav %} + {{ super() }} + {% if h.ckan_version().split('.')[1] | int >= 9 %} +- {{ h.build_nav_icon('pages.organization_pages_index', _('Pages'), id=c.group_dict.name, icon='file') }} ++ {{ h.build_nav_icon('pages.organization_pages_index', _('Pages'), id=group_dict.name, icon='file') }} + {% else %} +- {{ h.build_nav_icon('organization_pages_index', _('Pages'), id=c.group_dict.name, icon='file') }} ++ {{ h.build_nav_icon('organization_pages_index', _('Pages'), id=group_dict.name, icon='file') }} + {% endif %} + {% endblock %} From ad6977a0109fb057d50e3806f830a47995056ddb Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Fri, 20 Sep 2024 14:40:54 +0200 Subject: [PATCH 16/25] Update Scheming DCAT version to v4.0.0 --- ckan/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/Dockerfile b/ckan/Dockerfile index a8b57d77..ecc2ba95 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -22,7 +22,7 @@ COPY req_fixes req_fixes ## Pages - v0.5.2 ## ## PDFView - 0.0.8 ## ## Fluent - v1.0.1 (mjanez/Forked stable version) ## -## Scheming DCAT - v4.0.0-alpha (mjanez/GeoDCAT-AP/NTI-RISP extended version) ## +## Scheming DCAT - v4.0.0 (mjanez/GeoDCAT-AP/NTI-RISP extended version) ## RUN echo ${TZ} > /etc/timezone && \ if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime; fi && \ # Install patch utility @@ -55,7 +55,7 @@ RUN echo ${TZ} > /etc/timezone && \ echo "mjanez/ckanext-fluent" && \ pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-fluent.git@v1.0.1#egg=ckanext-fluent && \ echo "mjanez/ckanext-schemingdcat" && \ - pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-schemingdcat.git@v4.0.0-alpha#egg=ckanext_schemingdcat && \ + pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-schemingdcat.git@v4.0.0#egg=ckanext_schemingdcat && \ pip3 install --no-cache-dir -r ${APP_DIR}/src/ckanext-schemingdcat/requirements.txt && \ # Remove system cache apt-get clean && \ From 05edcca600a64c8a0fb269c1d1dd034c8808fbec Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Fri, 20 Sep 2024 14:41:15 +0200 Subject: [PATCH 17/25] Fix CKAN xloader API token creation - Using CKAN_SYSADMIN_NAME envvar --- ckan/docker-entrypoint.d/02_setup_xloader.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/docker-entrypoint.d/02_setup_xloader.sh b/ckan/docker-entrypoint.d/02_setup_xloader.sh index b360431d..0caca95e 100644 --- a/ckan/docker-entrypoint.d/02_setup_xloader.sh +++ b/ckan/docker-entrypoint.d/02_setup_xloader.sh @@ -4,7 +4,7 @@ TOKEN_NAME="xloader" # Get the list of tokens and extract the IDs for tokens with the specified name -TOKEN_IDS=$(ckan -c $CKAN_INI user token list ckan_admin | grep "$TOKEN_NAME" | awk -F'[][]' '{print $2}' | tr -d '[]') +TOKEN_IDS=$(ckan -c $CKAN_INI user token list $CKAN_SYSADMIN_NAME | grep "$TOKEN_NAME" | awk -F'[][]' '{print $2}' | tr -d '[]') # Revoke each previous token of xloader for TOKEN_ID in $TOKEN_IDS @@ -27,4 +27,4 @@ ckan config-tool $CKAN_INI \ # Create ckanext-xloader API_TOKEN echo "[docker-entrypoint.01_setup_xloader] Set up ckanext.xloader.api_token in the CKAN config file" -ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $CKAN_INI user token add ckan_admin xloader | tail -n 1 | tr -d '\t')" \ No newline at end of file +ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $CKAN_INI user token add $CKAN_SYSADMIN_NAME xloader | tail -n 1 | tr -d '\t')" \ No newline at end of file From 09b87877c3b4f9fff6170a0c01e55d5e300ab146 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Sun, 22 Sep 2024 02:34:58 +0200 Subject: [PATCH 18/25] Update CKAN core and ckanext-schemingdcat settings --- .../01_setup_ckanext_config.sh | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh b/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh index d4ead00e..9d986e90 100644 --- a/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh +++ b/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh @@ -7,8 +7,8 @@ ckan -c $CKAN_INI search-index clear # ckan: Update core settings echo "[docker-entrypoint.01_setup_ckanext_config] Loading CKAN core settings in the CKAN config file" ckan config-tool $CKAN_INI \ - "search.facets.default = $SEARCH__FACETS__DEFAULT" \ - "ckan.datastore.sqlsearch.enabled = $CKAN__DATASTORE__SQLSEARCH__ENABLED" + "search.facets.default=$SEARCH__FACETS__DEFAULT" \ + "ckan.datastore.sqlsearch.enabled=$CKAN__DATASTORE__SQLSEARCH__ENABLED" # ckanext-schemingdcat: Update settings echo "[docker-entrypoint.01_setup_ckanext_config] Loading ckanext-scheming and ckanext-schemingdcat settings into ckan.ini" @@ -25,41 +25,44 @@ ckan config-tool $CKAN_INI \ "schemingdcat.default_package_item_show_spatial=$CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_SHOW_SPATIAL" \ "schemingdcat.show_metadata_templates_toolbar=$CKANEXT__SCHEMINGDCAT_SHOW_METADATA_TEMPLATES_TOOLBAR" \ "schemingdcat.metadata_templates_search_identifier=$CKANEXT__METADATA_TEMPLATES_SEARCH_IDENTIFIER" \ - "schemingdcat.endpoints_yaml=$CKANEXT__SCHEMINGDCAT_ENDPOINTS_YAML" + "schemingdcat.endpoints_yaml=$CKANEXT__SCHEMINGDCAT_ENDPOINTS_YAML" \ + "schemingdcat.social_github=$CKANEXT__SCHEMINGDCAT__SOCIAL_GITHUB" \ + "schemingdcat.social_x=$CKANEXT__SCHEMINGDCAT__SOCIAL_X" \ + "schemingdcat.social_linkedin=$CKANEXT__SCHEMINGDCAT__SOCIAL_LINKEDIN" # ckanext-dcat: Add settings to the CKAN config file echo "[docker-entrypoint.01_setup_ckanext_config] Loading ckanext-dcat settings in the CKAN config file" ckan config-tool $CKAN_INI \ - "ckanext.dcat.base_uri = $CKANEXT__DCAT__BASE_URI" \ - "ckanext.dcat.catalog_endpoint = $CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT" \ - "ckanext.dcat.rdf.profiles = $CKANEXT__DCAT__RDF_PROFILES" + "ckanext.dcat.base_uri=$CKANEXT__DCAT__BASE_URI" \ + "ckanext.dcat.catalog_endpoint=$CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT" \ + "ckanext.dcat.rdf.profiles=$CKANEXT__DCAT__RDF_PROFILES" # ckan previews: Add CKAN Resource views to the CKAN config file echo "[docker-entrypoint.01_setup_ckanext_config] Loading resource views in the CKAN config file" ckan config-tool $CKAN_INI \ - "ckan.views.default_views = $CKAN__VIEWS__DEFAULT_VIEWS" \ - "ckan.preview.json_formats = $CKAN__PREVIEW__JSON_FORMATS" \ - "ckan.preview.xml_formats = $CKAN__PREVIEW__XML_FORMATS" \ - "ckan.preview.text_formats = $CKAN__PREVIEW__TEXT_FORMATS" \ - "ckan.preview.loadable = $CKAN__PREVIEW__LOADABLE" + "ckan.views.default_views=$CKAN__VIEWS__DEFAULT_VIEWS" \ + "ckan.preview.json_formats=$CKAN__PREVIEW__JSON_FORMATS" \ + "ckan.preview.xml_formats=$CKAN__PREVIEW__XML_FORMATS" \ + "ckan.preview.text_formats=$CKAN__PREVIEW__TEXT_FORMATS" \ + "ckan.preview.loadable=$CKAN__PREVIEW__LOADABLE" # ckanext-geoview: Add geoviews CKAN config file echo "[docker-entrypoint.01_setup_ckanext_config] Loading geoviews in the CKAN config file" ckan config-tool $CKAN_INI \ - "ckanext.geoview.ol_viewer.formats = $CKANEXT__GEOVIEW__OL_VIEWER__FORMATS" \ - "ckanext.geoview.shp_viewer.srid = $CKANEXT__GEOVIEW__SHP_VIEWER__SRID" \ - "ckanext.geoview.shp_viewer.encoding = $CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING" \ - "ckanext.geoview.geojson.max_file_size = $CKANEXT__GEOVIEW__GEOJSON__MAX_FILE_SIZE" + "ckanext.geoview.ol_viewer.formats=$CKANEXT__GEOVIEW__OL_VIEWER__FORMATS" \ + "ckanext.geoview.shp_viewer.srid=$CKANEXT__GEOVIEW__SHP_VIEWER__SRID" \ + "ckanext.geoview.shp_viewer.encoding=$CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING" \ + "ckanext.geoview.geojson.max_file_size=$CKANEXT__GEOVIEW__GEOJSON__MAX_FILE_SIZE" # ckanext-pages: Add pages CKAN config file echo "[docker-entrypoint.01_setup_ckanext_config] Loading pages config in the CKAN config file" ckan config-tool $CKAN_INI \ - "ckan.pages.allow_html = $CKANEXT__PAGES__ALOW_HTML" \ - "ckanext.pages.organization = $CKANEXT__PAGES__ORGANIZATION" \ - "ckanext.pages.group = $CKANEXT__PAGES__GROUP" \ - "ckanext.pages.about_menu = $CKANEXT__PAGES__ABOUT_MENU" \ - "ckanext.pages.group_menu = $CKANEXT__PAGES__GROUP_MENU" \ - "ckanext.pages.organization_menu = $CKANEXT__PAGES__ORGANIZATION_MENU" + "ckan.pages.allow_html=$CKANEXT__PAGES__ALOW_HTML" \ + "ckanext.pages.organization=$CKANEXT__PAGES__ORGANIZATION" \ + "ckanext.pages.group=$CKANEXT__PAGES__GROUP" \ + "ckanext.pages.about_menu=$CKANEXT__PAGES__ABOUT_MENU" \ + "ckanext.pages.group_menu=$CKANEXT__PAGES__GROUP_MENU" \ + "ckanext.pages.organization_menu=$CKANEXT__PAGES__ORGANIZATION_MENU" # Rebuild index echo "[docker-entrypoint.01_setup_ckanext_config] Rebuild index" From c4c58f6ef1b0e87e1d49d3b3f518e84735e5a47f Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Mon, 23 Sep 2024 08:10:25 +0200 Subject: [PATCH 19/25] Update CKAN preview and DCAT default profiles --- .env.example | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 9b5ca893..2b49c9df 100644 --- a/.env.example +++ b/.env.example @@ -124,7 +124,7 @@ CKAN_SMTP_MAIL_FROM=ckan@${PROXY_SERVER_NAME} CKAN__PREVIEW__JSON_FORMATS="json jsonld" # html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json CKAN__PREVIEW__XML_FORMATS="xml rdf rdf+xml owl+xml atom rss turtle ttl n3 n-triples" -CKAN__PREVIEW__TEXT_FORMATS="text plain text/plain text/turtle csv tsv rss txt json" +CKAN__PREVIEW__TEXT_FORMATS="text plain text/plain text/turtle rss txt" CKAN__PREVIEW__LOADABLE="html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json arcgis_rest" # ckanext-spatial: Allow Solr local params: https://github.com/ckan/ckanext-spatial/issues/328 CKAN__SEARCH__SOLR_ALLOWED_QUERY_PARSERS=field @@ -146,7 +146,7 @@ CKAN__RESOURCE_PROXY__MAX_FILE_SIZE=50048576 CKAN__RESOURCE_PROXY__CHUNK_SIZE=4096 ## Default timeout for fetching proxied items CKAN__RESOURCE_PROXY__TIMEOUT=10 -CKAN__VIEWS__DEFAULT_VIEWS="image_view video_view audio_view webpage_view text_view datatables_view wmts_view geojson_view geo_view shp_view pdf_view" +CKAN__VIEWS__DEFAULT_VIEWS="image_view video_view audio_view webpage_view datatables_view text_view wmts_view geojson_view geo_view shp_view pdf_view" # Localization CKAN__LOCALE_DEFAULT="en" @@ -169,8 +169,8 @@ CKANEXT__XLOADER__JOBS__DB_URI=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@ # ckanext-dcat CKANEXT__DCAT__BASE_URI=${CKAN_URL} -# Default profile(s). Instead of this envvar, it's possible to specify all the profile(s) availables to be used for serialization using the profiles parameter: http://localhost:5000/catalog.xml?profiles=euro_dcat_ap,spain_dcat -CKANEXT__DCAT__RDF_PROFILES='euro_dcat_ap_2' +# Default profile(s). Instead of this envvar, it's possible to specify all the profile(s) availables to be used for serialization using the profiles parameter: http://localhost:5000/catalog.xml?profiles=eu_dcat_ap_2,es_dcat +CKANEXT__DCAT__RDF_PROFILES='eu_dcat_ap_2 eu_dcat_ap_scheming' # The custom endpoint **must** start with a forward slash (`/`) and contain the `{_format}` placeholder. The endpoint is added to the CKAN_SITE_URL and CKAN__ROOT_PATH, example: http://localhost:5000/catalog/catalog.rdf CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT='/catalog.{_format}' From 3fdab45448d5372bd5e1e2eb9bb4f34a5e2b3e83 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Mon, 23 Sep 2024 18:54:38 +0000 Subject: [PATCH 20/25] Remove --plugins options from uWSGI configuration - The --plugins options were removed from the uWSGI configuration because the plugins are already included when uWSGI is installed via pip. Specifying the plugins explicitly is unnecessary and can lead to errors if the plugins are not found in the expected locations. By removing the --plugins options, we rely on uWSGI's dynamic plugin loading mechanism, which ensures that the required plugins (e.g., python, http) are loaded correctly without the need for manual specification. - Add performance options. --- ckan/setup/start_ckan.sh.override | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ckan/setup/start_ckan.sh.override b/ckan/setup/start_ckan.sh.override index 71a08431..bf50b4d5 100644 --- a/ckan/setup/start_ckan.sh.override +++ b/ckan/setup/start_ckan.sh.override @@ -48,8 +48,11 @@ chown -R ckan:ckan $CKAN_LOGS_PATH/harvester mkdir -p $CKAN_LOGS_PATH/xloader chown -R ckan:ckan $CKAN_LOGS_PATH/xloader -# Set the common uwsgi options -UWSGI_OPTS="--plugins http,python \ +# Set the common uwsgi options. +## Add thunder-lock to prevent multiple workers from running the same job and buffer-size to prevent large headers +UWSGI_OPTS="--thunder-lock \ + --buffer-size=65535 \ + --ignore-sigpipe \ --socket /tmp/uwsgi.sock \ --wsgi-file /srv/app/wsgi.py \ --module wsgi:application \ From 4578814d50a18459e0ca4e6a3aa32eaa401078cf Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Mon, 23 Sep 2024 19:12:27 +0000 Subject: [PATCH 21/25] Replace crond with debian-based cron in CKAN startup script for background execution and improve SMTP options - Prevent duplicate entries in crontab for harvester background processes. --- .env.example | 2 ++ .../01_setup_ckanext_config.sh | 14 ++++++++++ ckan/setup/start_ckan.sh.override | 27 ++++++++++++++++--- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 2b49c9df..4e5c49d8 100644 --- a/.env.example +++ b/.env.example @@ -115,6 +115,8 @@ CKAN_SYSADMIN_PASSWORD=test1234 CKAN_SYSADMIN_EMAIL=your_email@example.com CKAN_STORAGE_PATH=/var/lib/ckan CKAN_LOGS_PATH=/var/log +# SMTP settings +CKAN__SMTP_ENABLED=False CKAN_SMTP_SERVER=smtp.corporateict.domain:25 CKAN_SMTP_STARTTLS=True CKAN_SMTP_USER=user diff --git a/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh b/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh index 9d986e90..ec958146 100644 --- a/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh +++ b/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh @@ -10,6 +10,20 @@ ckan config-tool $CKAN_INI \ "search.facets.default=$SEARCH__FACETS__DEFAULT" \ "ckan.datastore.sqlsearch.enabled=$CKAN__DATASTORE__SQLSEARCH__ENABLED" +# Add SMTP settings if CKAN__SMTP_ENABLED is True +if [ "$CKAN__SMTP_ENABLED" = "True" ]; then + echo "[docker-entrypoint.01_setup_ckanext_config] Adding SMTP settings to the CKAN config file" + ckan config-tool $CKAN_INI \ + "smtp.server=$CKAN_SMTP_SERVER" \ + "smtp.starttls=$CKAN_SMTP_STARTTLS" \ + "smtp.user=$CKAN_SMTP_USER" \ + "smtp.password=$CKAN_SMTP_PASSWORD" \ + "smtp.mail_from=$CKAN_SMTP_MAIL_FROM" \ + "smtp.reply_to=" \ + "email_to=" \ + "error_email_from=" +fi + # ckanext-schemingdcat: Update settings echo "[docker-entrypoint.01_setup_ckanext_config] Loading ckanext-scheming and ckanext-schemingdcat settings into ckan.ini" ckan config-tool $CKAN_INI \ diff --git a/ckan/setup/start_ckan.sh.override b/ckan/setup/start_ckan.sh.override index bf50b4d5..91cfeb4a 100644 --- a/ckan/setup/start_ckan.sh.override +++ b/ckan/setup/start_ckan.sh.override @@ -24,6 +24,12 @@ if [ -n "$CKAN___LICENSES_GROUP_URL" ]; then echo "Update licenses in CKAN" fi +# Comment out smtp.* settings if CKAN__SMTP_ENABLED is not True +if [ "$CKAN__SMTP_ENABLED" != "True" ]; then + echo "Commenting out smtp.* settings in ini file" + sed -i 's/^\(smtp\..*\)$/#\1/' $CKAN_INI +fi + # Run the prerun script to init CKAN and create the default admin user python3 prerun.py @@ -70,14 +76,27 @@ then supervisord --configuration /etc/supervisord.conf & # Workers - ## Add harvester background procces to crontab + ## Add the harvester background processes to crontab if they do not already exist echo "[prerun.workers] Add harvester background procceses to crontab" - crontab -l | { cat; echo "*/15 * * * * /usr/bin/supervisorctl start ckan_harvester_run"; } | crontab - + if ! crontab -l | grep -q "/usr/bin/supervisorctl start ckan_harvester_run"; then + { crontab -l; echo "*/15 * * * * /usr/bin/supervisorctl start ckan_harvester_run"; } | crontab - + fi ## Clean-up mechanism for the harvest log table. 'ckan.harvest.log_timeframe'. The default time frame is 30 days - crontab -l | { cat; echo "0 5 */30 * * /usr/bin/supervisorctl start ckan_harvester_clean_log"; } | crontab - + if ! crontab -l | grep -q "/usr/bin/supervisorctl start ckan_harvester_clean_log"; then + { crontab -l; echo "0 5 */30 * * /usr/bin/supervisorctl start ckan_harvester_clean_log"; } | crontab - + fi + + ## Add SMTP notification process to crontab if CKAN__SMTP_ENABLED is True + if [ "$CKAN__SMTP_ENABLED" = "True" ]; then + echo "[prerun.workers] Add SMTP notification process to crontab" + if ! crontab -l | grep -q "ckan -c $CKAN_INI notify send_emails"; then + { crontab -l; echo "@hourly echo '{}' | ckan -c $CKAN_INI notify send_emails > /dev/null"; } | crontab - + fi + fi + ## Execute cron in the background echo "[prerun.workers] Execute cron in the background" - crond -b -l 8 + cron -f & # Start uwsgi uwsgi $UWSGI_OPTS From f415ab6f9700c07aa9d9da7bc8c3953fefef2688 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Mon, 23 Sep 2024 19:12:33 +0000 Subject: [PATCH 22/25] Update xloader log file paths to use /var/log/xloader for consistency --- ckan/setup/workers/xloader.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/setup/workers/xloader.conf b/ckan/setup/workers/xloader.conf index c7749ddb..7d12cef1 100644 --- a/ckan/setup/workers/xloader.conf +++ b/ckan/setup/workers/xloader.conf @@ -2,9 +2,9 @@ command=ckan jobs worker default user=ckan numprocs=1 -stdout_logfile=/var/log/harvester/ckan_xloader.log +stdout_logfile=/var/log/xloader/ckan_xloader.log stdout_logfile_maxbytes=100MB -stderr_logfile=/var/log/harvester/ckan_xloader.log +stderr_logfile=/var/log/xloader/ckan_xloader.log stderr_logfile_maxbytes=100MB autostart=true autorestart=true From c0185a51eb6d9a2e127ccd914d58eb6a7099b8e6 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Mon, 23 Sep 2024 19:23:42 +0000 Subject: [PATCH 23/25] Update Dockerfile to install cron for scheduled tasks and upgrade CKAN ghcr image to 2.10.5 --- ckan/Dockerfile | 4 +++- ckan/Dockerfile.ghcr | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ckan/Dockerfile b/ckan/Dockerfile index ecc2ba95..d0eb048d 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -27,7 +27,9 @@ RUN echo ${TZ} > /etc/timezone && \ if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime; fi && \ # Install patch utility apt-get update && \ - apt-get install -y patch && \ + apt-get install -y patch \ + # Install cron for scheduled tasks + cron && \ # Install CKAN extensions echo "ckan/ckanext-xloader" && \ pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-xloader.git@1.0.1-stable#egg=ckanext-xloader && \ diff --git a/ckan/Dockerfile.ghcr b/ckan/Dockerfile.ghcr index 27da3492..5c37c934 100644 --- a/ckan/Dockerfile.ghcr +++ b/ckan/Dockerfile.ghcr @@ -1,4 +1,4 @@ -FROM ghcr.io/mjanez/ckan-docker:ckan-2.9.11 +FROM ghcr.io/mjanez/ckan-docker:ckan-2.10.5 LABEL maintainer="mnl.janez@gmail.com" # Set up environment variables From ee9bfa1dd53183a1863502144af04dd3fdccb00d Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Mon, 23 Sep 2024 19:42:15 +0000 Subject: [PATCH 24/25] Optimize uWSGI configuration for improved performance and resource management --- ckan/setup/start_ckan.sh.override | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/ckan/setup/start_ckan.sh.override b/ckan/setup/start_ckan.sh.override index 91cfeb4a..b025cc1a 100644 --- a/ckan/setup/start_ckan.sh.override +++ b/ckan/setup/start_ckan.sh.override @@ -56,6 +56,7 @@ chown -R ckan:ckan $CKAN_LOGS_PATH/xloader # Set the common uwsgi options. ## Add thunder-lock to prevent multiple workers from running the same job and buffer-size to prevent large headers +## To increase performance, you can adjust the number of processes (-p) (More info: https://uwsgi-docs.readthedocs.io/en/latest/Options.html & https://www.bloomberg.com/company/stories/configuring-uwsgi-production-deployment/) UWSGI_OPTS="--thunder-lock \ --buffer-size=65535 \ --ignore-sigpipe \ @@ -66,8 +67,18 @@ UWSGI_OPTS="--thunder-lock \ --http [::]:5000 \ --master --enable-threads \ --lazy-apps \ - -p 2 -L -b 32768 --vacuum \ - --harakiri $UWSGI_HARAKIRI" + --vacuum \ + --harakiri $UWSGI_HARAKIRI \ + --max-requests 500 \ + --max-worker-lifetime 3600 \ + --reload-on-rss 1024 \ + --processes 3 \ + --buffer-size 32768 \ + --disable-logging \ + --log-4xx \ + --log-5xx \ + --auto-procname \ + --procname-prefix-spaced 'ckan '" if [ $? -eq 0 ] then From 6a75fd056ec6c327fff3d48e59d68609c6581e23 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Mon, 23 Sep 2024 22:06:02 +0200 Subject: [PATCH 25/25] Refactor uWSGI options in start_ckan.sh.override to using lower options - 2 processes - 32768 buffer-size --- ckan/setup/start_ckan.sh.override | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ckan/setup/start_ckan.sh.override b/ckan/setup/start_ckan.sh.override index b025cc1a..ec3e3ace 100644 --- a/ckan/setup/start_ckan.sh.override +++ b/ckan/setup/start_ckan.sh.override @@ -58,27 +58,27 @@ chown -R ckan:ckan $CKAN_LOGS_PATH/xloader ## Add thunder-lock to prevent multiple workers from running the same job and buffer-size to prevent large headers ## To increase performance, you can adjust the number of processes (-p) (More info: https://uwsgi-docs.readthedocs.io/en/latest/Options.html & https://www.bloomberg.com/company/stories/configuring-uwsgi-production-deployment/) UWSGI_OPTS="--thunder-lock \ - --buffer-size=65535 \ --ignore-sigpipe \ --socket /tmp/uwsgi.sock \ --wsgi-file /srv/app/wsgi.py \ --module wsgi:application \ --uid 92 --gid 92 \ --http [::]:5000 \ - --master --enable-threads \ + --master \ + --enable-threads \ --lazy-apps \ --vacuum \ --harakiri $UWSGI_HARAKIRI \ --max-requests 500 \ --max-worker-lifetime 3600 \ --reload-on-rss 1024 \ - --processes 3 \ + --processes 2 \ --buffer-size 32768 \ --disable-logging \ --log-4xx \ --log-5xx \ --auto-procname \ - --procname-prefix-spaced 'ckan '" + --procname-prefix 'ckan-'" if [ $? -eq 0 ] then