diff --git a/.env.example b/.env.example index 2d3dfeb4..4e5c49d8 100644 --- a/.env.example +++ b/.env.example @@ -7,11 +7,13 @@ APP_DIR=/srv/app CKAN_PORT_HOST=5000 # Common proxy server for Apache or NGINX. # Change all the PROXY_SERVER_PORT_HOST to PROXY_SERVER_HTTPS_PORT_HOST if you can use HTTPS instead of HTTP (Only for NGINX and not development compose) -PROXY_SERVER_PORT_HOST=81 +PROXY_SERVER_BASE_PORT_HOST=81 PROXY_SERVER_HTTPS_PORT_HOST=8443 -NGINX_PORT_HOST=${PROXY_SERVER_PORT_HOST} +# Server port for the proxy server. Better using the SSL port for production, in DEV or if using balancer: PROXY_SERVER_PORT_HOST=PROXY_SERVER_BASE_PORT_HOST +PROXY_SERVER_PORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} +NGINX_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} NGINX_SSLPORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} -APACHE_PORT_HOST=${PROXY_SERVER_PORT_HOST} +APACHE_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} PYCSW_PORT_HOST=8000 # Solr @@ -41,7 +43,9 @@ APACHE_LOG_DIR=/var/log/apache #NGINX/APACHE ## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI and CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. If you dont need PROXY_SERVER_PORT_HOST (e.g. is 80), edit the PROXY_SERVER_URL and remove :${PROXY_SERVER_PORT_HOST} PROXY_SERVER_NAME=localhost -PROXY_SERVER_URL=http://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} +# URL protocol. Change to https if you use SSL +PROXY_SERVER_PROTOCOL=http +PROXY_SERVER_URL=${PROXY_SERVER_PROTOCOL}://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} PROXY_CKAN_LOCATION=/catalog PROXY_PYCSW_LOCATION=/csw @@ -88,15 +92,20 @@ CKAN_DEV_COMPOSE_SERVICE=ckan-dev # CKAN core ## If use docker-compose.ghcr.yml only "*.*.*" versions available in: https://github.com/mjanez/ckan-docker/pkgs/container/ckan-docker -CKAN_VERSION=2.9.11 +CKAN_VERSION=2.10.5 CKAN_SITE_ID=default # CKAN_SITE_URL = http:/ or https:/ + PROXY_SERVER_NAME. Optionally the APACHE_HOST_PORT if different from 80 CKAN_SITE_URL=${PROXY_SERVER_URL} CKAN__ROOT_PATH=${PROXY_CKAN_LOCATION}/{{LANG}} CKAN__FAVICON=${PROXY_CKAN_LOCATION}/base/images/ckan.ico -CKAN__SITE_LOGO=/images/default/ckan-logo.png -# Custom licenses that are aligned with DCAT-AP -CKAN___LICENSES_GROUP_URL=https://raw.githubusercontent.com/mjanez/ckanext-schemingdcat/main/ckanext/schemingdcat/public/static/licenses.json +CKAN__SITE_LOGO=${PROXY_CKAN_LOCATION}/images/default/ckan-logo.png +# CKAN site info +CKAN__SITE_TITLE="CKAN Open Data" +CKAN__SITE_DESCRIPTION="ckan-docker *spatial Open Data portal" +CKAN__SITE_INTRO_TEXT="[![ckan-docker-spatial](https://img.shields.io/badge/ckan_docker_spatial-CKAN_${CKAN_VERSION}-brightgreen?style=social&logo=github#center)](https://github.com/mjanez/ckan-docker)\n[`ckan-docker *spatial`](https://github.com/mjanez/ckan-docker) is a ready-to-use development environment for [CKAN](https://github.com/ckan/ckan), the world’s leading open-source data portal platform." +CKAN__SITE_ABOUT=${CKAN__SITE_INTRO_TEXT} +# Custom licenses that are aligned with DCAT-AP (Also works with DEV Mode) +LICENSES_GROUP_URL=file://${APP_DIR}/src/ckanext-schemingdcat/ckanext/schemingdcat/public/static/licenses.json CKAN___BEAKER__SESSION__SECRET=CHANGE_ME # See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME @@ -106,6 +115,8 @@ CKAN_SYSADMIN_PASSWORD=test1234 CKAN_SYSADMIN_EMAIL=your_email@example.com CKAN_STORAGE_PATH=/var/lib/ckan CKAN_LOGS_PATH=/var/log +# SMTP settings +CKAN__SMTP_ENABLED=False CKAN_SMTP_SERVER=smtp.corporateict.domain:25 CKAN_SMTP_STARTTLS=True CKAN_SMTP_USER=user @@ -115,7 +126,7 @@ CKAN_SMTP_MAIL_FROM=ckan@${PROXY_SERVER_NAME} CKAN__PREVIEW__JSON_FORMATS="json jsonld" # html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json CKAN__PREVIEW__XML_FORMATS="xml rdf rdf+xml owl+xml atom rss turtle ttl n3 n-triples" -CKAN__PREVIEW__TEXT_FORMATS="text plain text/plain text/turtle csv tsv rss txt json" +CKAN__PREVIEW__TEXT_FORMATS="text plain text/plain text/turtle rss txt" CKAN__PREVIEW__LOADABLE="html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json arcgis_rest" # ckanext-spatial: Allow Solr local params: https://github.com/ckan/ckanext-spatial/issues/328 CKAN__SEARCH__SOLR_ALLOWED_QUERY_PARSERS=field @@ -125,6 +136,10 @@ CKAN__CORS__ORIGIN_WHITELIST="" # Enables or disable collaborators in individual datasets (https://docs.ckan.org/en/2.9/maintaining/authorization.html#dataset-collaborators) CKAN__AUTH__ALLOW_DATASET_COLLABORATORS=False CKAN__AUTH__ALLOW_ADMIN_COLLABORATORS=False +# Default number of facets shown in search results. Default 10. +SEARCH__FACETS__DEFAULT=4 +# Enable or disable the DataStore SQL search backend. Default is False. +CKAN__DATASTORE__SQLSEARCH__ENABLED=True ## Resource Proxy settings ### Preview size limit, default: 1MB @@ -133,7 +148,7 @@ CKAN__RESOURCE_PROXY__MAX_FILE_SIZE=50048576 CKAN__RESOURCE_PROXY__CHUNK_SIZE=4096 ## Default timeout for fetching proxied items CKAN__RESOURCE_PROXY__TIMEOUT=10 -CKAN__VIEWS__DEFAULT_VIEWS="image_view webpage_view text_view recline_view wmts_view geojson_view geo_view shp_view pdf_view" +CKAN__VIEWS__DEFAULT_VIEWS="image_view video_view audio_view webpage_view datatables_view text_view wmts_view geojson_view geo_view shp_view pdf_view" # Localization CKAN__LOCALE_DEFAULT="en" @@ -141,7 +156,7 @@ CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru CKAN__LOCALES_OFFERED="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" # Extensions -CKAN__PLUGINS="stats image_view text_view recline_view webpage_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" +CKAN__PLUGINS="activity stats image_view video_view audio_view webpage_view text_view datatables_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" # ckanext-harvest CKAN__HARVEST__MQ__TYPE=redis @@ -152,13 +167,12 @@ CKAN__HARVEST__MQ__REDIS_DB=${REDIS_CKAN_DATABASE} CKAN__HARVEST__LOG_TIMEFRAME=40 # ckanext-xloader -CKANEXT__XLOADER__API_TOKEN=api_token CKANEXT__XLOADER__JOBS__DB_URI=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB} # ckanext-dcat CKANEXT__DCAT__BASE_URI=${CKAN_URL} -# Default profile(s). Instead of this envvar, it's possible to specify all the profile(s) availables to be used for serialization using the profiles parameter: http://localhost:5000/catalog.xml?profiles=euro_dcat_ap,spain_dcat -CKANEXT__DCAT__RDF_PROFILES='euro_dcat_ap_2' +# Default profile(s). Instead of this envvar, it's possible to specify all the profile(s) availables to be used for serialization using the profiles parameter: http://localhost:5000/catalog.xml?profiles=eu_dcat_ap_2,es_dcat +CKANEXT__DCAT__RDF_PROFILES='eu_dcat_ap_2 eu_dcat_ap_scheming' # The custom endpoint **must** start with a forward slash (`/`) and contain the `{_format}` placeholder. The endpoint is added to the CKAN_SITE_URL and CKAN__ROOT_PATH, example: http://localhost:5000/catalog/catalog.rdf CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT='/catalog.{_format}' @@ -179,12 +193,12 @@ CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8 ## CSW Endpoint for spatial metadata CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI=${PYCSW_URL} ## Scheming: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcatap_eu/geodcatap_eu_dataset.yaml" -CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap_eu/geodcatap_eu_group.json" -CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap_eu/geodcatap_eu_org.json" -CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/default_presets.json ckanext.fluent:presets.json ckanext.iepnb:schemas/presets.json" +CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_full.yaml" +CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_group.json" +CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_org.json" +CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/default_presets.json ckanext.fluent:presets.json" ## Facets: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_eu dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" +CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_eu language dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" CKANEXT__SCHEMINGDCAT_ORGANIZATION_CUSTOM_FACETS=True CKANEXT__SCHEMINGDCAT_GROUP_CUSTOM_FACETS=True CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_ICON="theme" @@ -192,6 +206,9 @@ CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_SHOW_SPATIAL=True CKANEXT__SCHEMINGDCAT_SHOW_METADATA_TEMPLATES_TOOLBAR=False CKANEXT__METADATA_TEMPLATES_SEARCH_IDENTIFIER="schemingdcat_xls-template" CKANEXT__SCHEMINGDCAT_ENDPOINTS_YAML="endpoints.yaml" +CKANEXT__SCHEMINGDCAT__SOCIAL_GITHUB="https://github.com/mjanez/ckanext-schemingdcat" +CKANEXT__SCHEMINGDCAT__SOCIAL_X="https://x.com/ckanproject" +CKANEXT__SCHEMINGDCAT__SOCIAL_LINKEDIN="https://www.linkedin.com/company/ckanproject" # ckanext-pages CKANEXT__PAGES__ALOW_HTML=False @@ -199,4 +216,29 @@ CKANEXT__PAGES__ORGANIZATION=True CKANEXT__PAGES__GROUP=True CKANEXT__PAGES__ABOUT_MENU=False CKANEXT__PAGES__GROUP_MENU=True -CKANEXT__PAGES__ORGANIZATION_MENU=True \ No newline at end of file +CKANEXT__PAGES__ORGANIZATION_MENU=True + +# ckan-pycsw +CSW_IDENTIFICATION_TITLE="Sample Geospatial Catalogue" +CSW_IDENTIFICATION_ABSTRACT="OGC CSW server powered by pycsw" +CSW_PROVIDER_NAME="ckan-docker development team" +CSW_PROVIDER_URL="https://github.com/mjanez/ckan-docker" +CSW_CONTACT_NAME="ckan-docker development team" +CSW_CONTACT_POSITION="Site Administrator" +CSW_CONTACT_ADDRESS="ckan-docker development team" +CSW_CONTACT_CITY="Madrid" +CSW_CONTACT_STATE_OR_PROVINCE="Madrid" +CSW_CONTACT_POSTAL_CODE="28001" +CSW_CONTACT_COUNTRY="Spain" +CSW_CONTACT_EMAIL=${CKAN_SYSADMIN_EMAIL} +CSW_CONTACT_URL=${CKAN_URL} +## Date of the last update of the metadata. ISO 8601 format): YYYY-MM-DD +CSW_INSPIRE_DATE="2024-01-01" +## Keywords separated by commas based on the GEMET Thesaurus keywordValue: https://inspire.ec.europa.eu/schemas/common/1.0/enums/enum_eng.xsd +CSW_INSPIRE_GEMET_KEYWORDS="Utility and governmental services" +# The level of INSPIRE conformance for spatial data sets and services (conformant, notConformant, notEvaluated) +CSW_INSPIRE_CONFORMITY="notEvaluated" +CSW_INSPIRE_CONTACT_NAME=${CSW_CONTACT_NAME} +CSW_INSPIRE_CONTACT_EMAIL=${CKAN_SYSADMIN_EMAIL} +# Temporal extent of the service (in ISO 8601 format). Either a single date (i.e. YYYY-MM-DD), or an extent (i.e. YYYY-MM-DD/YYYY-MM-DD) +CSW_INSPIRE_TEMP_EXTENT="2024-01-01" diff --git a/.gitignore b/.gitignore index 94e41881..609b3048 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,10 @@ index.html.* # environment allowlist !.env.example +!.env.codespaces !.env.es.example !.env.localhost samples/api/* + +# docker compose files +docker-compose.local-*.yml diff --git a/README.md b/README.md index d864e173..137e32a7 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ Available components: | 2.9.x | custom spatial image | `alpine:3.15` | `ghcr.io/mjanez/ckan-spatial:ckan-2.9.8`, `ghcr.io/mjanez/ckan-spatial:ckan-2.9.8`, `ghcr.io/mjanez/ckan-docker:ckan-2.9.9`, `ghcr.io/mjanez/ckan-docker:ckan-2.9.10`, `ghcr.io/mjanez/ckan-docker:ckan-2.9.11` | Stable official versions of CKAN `2.9.8`, `2.9.10` and `2.9.11` | | 2.10.x | custom spatial image | `python:3.10-slim-bookworm` | `ghcr.io/mjanez/ckan-docker:2.10.5` | From `2.10` images only [Debian-based official Python images](https://hub.docker.com/_/python) rather than Alpine-based images will be provided. | | 2.11.x | custom spatial image | `python:3.10-slim-bookworm` | `ghcr.io/mjanez/ckan-docker:2.11.0` | CKAN's latest official version. Only [Debian-based official Python images](https://hub.docker.com/_/python). | +| master | custom spatial image | `python:3.10-slim-bookworm` | `ghcr.io/mjanez/ckan-docker:master` | Latest image available. Not for use in production. | The non-CKAN images are as follows: * PostgreSQL: [Custom image](/postgresql/Dockerfile) based on official PostgreSQL image. Database files are stored in a named volume. diff --git a/ckan-pycsw/Dockerfile b/ckan-pycsw/Dockerfile index 4fa1f0e9..4052ed4d 100644 --- a/ckan-pycsw/Dockerfile +++ b/ckan-pycsw/Dockerfile @@ -15,6 +15,27 @@ ENV TIMEOUT=300 ENV PYCSW_CRON_DAYS_INTERVAL=2 ENV PYCSW_CRON_HOUR_START=4 +# PYCSW Catalog configuration +ENV CSW_IDENTIFICATION_TITLE="Sample Geospatial Catalogue" \ + CSW_IDENTIFICATION_ABSTRACT="OGC CSW server powered by pycsw" \ + CSW_PROVIDER_NAME="ckan-docker development team" \ + CSW_PROVIDER_URL="https://github.com/mjanez/ckan-docker" \ + CSW_CONTACT_NAME="ckan-docker development team" \ + CSW_CONTACT_POSITION="Site Administrator" \ + CSW_CONTACT_ADDRESS="ckan-docker development team" \ + CSW_CONTACT_CITY="Madrid" \ + CSW_CONTACT_STATE_OR_PROVINCE="Madrid" \ + CSW_CONTACT_POSTAL_CODE="28001" \ + CSW_CONTACT_COUNTRY="Spain" \ + CSW_CONTACT_EMAIL=${CKAN_SYSADMIN_EMAIL} \ + CSW_CONTACT_URL=${CKAN_URL} \ + CSW_INSPIRE_DATE="2024-01-01" \ + CSW_INSPIRE_GEMET_KEYWORDS="Utility and governmental services" \ + CSW_INSPIRE_CONFORMITY="notEvaluated" \ + CSW_INSPIRE_CONTACT_NAME=${CSW_CONTACT_NAME} \ + CSW_INSPIRE_CONTACT_EMAIL=${CKAN_SYSADMIN_EMAIL} \ + CSW_INSPIRE_TEMP_EXTENT="2024-01-01/2024-12-31" + WORKDIR ${APP_DIR} # Update files if needed diff --git a/ckan-pycsw/pycsw/conf/pycsw.conf.template b/ckan-pycsw/pycsw/conf/pycsw.conf.template index 8fa945b2..e15908d9 100644 --- a/ckan-pycsw/pycsw/conf/pycsw.conf.template +++ b/ckan-pycsw/pycsw/conf/pycsw.conf.template @@ -23,27 +23,27 @@ allowed_ips=127.0.0.1 #csw_harvest_pagesize=10 [metadata:main] -identification_title=Sample Geospatial Catalogue -identification_abstract=OGC CSW server powered by pycsw +identification_title=${CSW_IDENTIFICATION_TITLE} +identification_abstract=${CSW_IDENTIFICATION_ABSTRACT} identification_keywords=catalogue,discovery,metadata,spatial identification_keywords_type=theme identification_fees=None identification_accessconstraints=None -provider_name=Sample -provider_url=https://example.org/ -#contact_name=admin -#contact_position=Position Title -#contact_address=Mailing Address -#contact_city=Madrid -#contact_stateorprovince=Administrative Area -#contact_postalcode=Zip or Postal Code -contact_country=Spain +provider_name=${CSW_PROVIDER_NAME} +provider_url=${CSW_PROVIDER_URL} +contact_name=${CSW_CONTACT_NAME} +contact_position=${CSW_CONTACT_POSITION} +contact_address=${CSW_CONTACT_ADDRESS} +contact_city=${CSW_CONTACT_CITY} +contact_stateorprovince=${CSW_CONTACT_STATE_OR_PROVINCE} +contact_postalcode=${CSW_CONTACT_POSTAL_CODE} +contact_country=${CSW_CONTACT_COUNTRY} #contact_phone=+xx-xxx-xxx-xxxx #contact_fax=+xx-xxx-xxx-xxxx -contact_email=admin@example.org -#contact_url=https://example.org/members/admin -#contact_hours=Hours of Service -#contact_instructions=During hours of service. Off on weekends. +contact_email=${CSW_CONTACT_EMAIL} +contact_url=${CSW_CONTACT_URL} +contact_hours=Hours of Service +contact_instructions=During hours of service. Off on weekends. contact_role=pointOfContact [repository] @@ -61,9 +61,9 @@ table=records enabled=true languages_supported=eng,spa default_language=eng -date=YYYY-MM-DD -gemet_keywords=Utility and governmental services -conformity_service=notEvaluated -contact_name=Organization Name -contact_email=Email Address -temp_extent=YYYY-MM-DD/YYYY-MM-DD \ No newline at end of file +date=${CSW_INSPIRE_DATE} +gemet_keywords=${CSW_INSPIRE_GEMET_KEYWORDS} +conformity_service=${CSW_INSPIRE_CONFORMITY} +contact_name=${CSW_INSPIRE_CONTACT_NAME} +contact_email=${CSW_INSPIRE_CONTACT_EMAIL} +temp_extent=${CSW_INSPIRE_TEMP_EXTENT} \ No newline at end of file diff --git a/ckan/Dockerfile b/ckan/Dockerfile index eec6ed80..d0eb048d 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/mjanez/ckan-base-spatial:ckan-2.9.11 +FROM ghcr.io/mjanez/ckan-spatial-base:2.10.5-py3.10 LABEL maintainer="mnl.janez@gmail.com" # Set up environment variables @@ -12,9 +12,9 @@ WORKDIR ${APP_DIR} COPY req_fixes req_fixes # CKAN configuration & extensions -## XLoader - 1.0.1 ## +## XLoader - 1.0.1-stable (mjanez/Forked fixed version) ## ## Harvest - v1.5.6 (Worker with supervisor) ## -## Geoview - v0.1.0 ## +## Geoview - v0.2.0 ## ## Spatial - v2.1.1 ## ## DCAT - v1.8.0 (Latest stable version of ckanext-dcat with minor fixes) ## ## Scheming - release-3.0.0 ## @@ -22,21 +22,24 @@ COPY req_fixes req_fixes ## Pages - v0.5.2 ## ## PDFView - 0.0.8 ## ## Fluent - v1.0.1 (mjanez/Forked stable version) ## -## Scheming DCAT - v3.2.2 (mjanez/GeoDCAT-AP/NTI-RISP extended version) ## +## Scheming DCAT - v4.0.0 (mjanez/GeoDCAT-AP/NTI-RISP extended version) ## RUN echo ${TZ} > /etc/timezone && \ if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime; fi && \ - # Remove apk cache - rm -rf /var/cache/apk/* && \ + # Install patch utility + apt-get update && \ + apt-get install -y patch \ + # Install cron for scheduled tasks + cron && \ # Install CKAN extensions echo "ckan/ckanext-xloader" && \ - pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-xloader.git@1.0.1#egg=ckanext-xloader && \ + pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-xloader.git@1.0.1-stable#egg=ckanext-xloader && \ pip3 install --no-cache-dir -r ${APP_DIR}/src/ckanext-xloader/requirements.txt && \ pip3 install --no-cache-dir -U requests[security] && \ echo "ckan/ckanext-harvest" && \ pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-harvest.git@v1.5.6#egg=ckanext-harvest && \ pip3 install --no-cache-dir -r ${APP_DIR}/src/ckanext-harvest/requirements.txt && \ echo "ckan/ckanext-geoview" && \ - pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-geoview.git@v0.1.0#egg=ckanext-geoview && \ + pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-geoview.git@v0.2.0#egg=ckanext-geoview && \ echo "ckan/ckanext-spatial" && \ pip3 install --no-cache-dir -e git+https://github.com/ckan/ckanext-spatial.git@v2.1.1#egg=ckanext-spatial && \ pip3 install --no-cache-dir -r ${APP_DIR}/req_fixes/ckanext-spatial/requirements.txt && \ @@ -54,15 +57,15 @@ RUN echo ${TZ} > /etc/timezone && \ echo "mjanez/ckanext-fluent" && \ pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-fluent.git@v1.0.1#egg=ckanext-fluent && \ echo "mjanez/ckanext-schemingdcat" && \ - pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-schemingdcat.git@v3.2.2#egg=ckanext_schemingdcat && \ - pip3 install --no-cache-dir -r ${APP_DIR}/src/ckanext-schemingdcat/requirements.txt + pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-schemingdcat.git@v4.0.0#egg=ckanext_schemingdcat && \ + pip3 install --no-cache-dir -r ${APP_DIR}/src/ckanext-schemingdcat/requirements.txt && \ + # Remove system cache + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Used to configure the container environment by setting environment variables, creating users, running initialization scripts, .etc COPY docker-entrypoint.d/* /docker-entrypoint.d/ -# Update who.ini with PROXY_CKAN_LOCATION -COPY setup/who.ini ./ - # Apply any patches needed to CKAN core COPY patches patches diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index 21f3a8a4..25cbcb40 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM ghcr.io/mjanez/ckan-base-spatial:ckan-2.9.11-dev +FROM ghcr.io/mjanez/ckan-spatial-dev:2.10.5-py3.10 LABEL maintainer="mnl.janez@gmail.com" # Set up environment variables @@ -10,9 +10,12 @@ ENV APP_DIR=/srv/app \ WORKDIR ${APP_DIR} RUN echo ${TZ} > /etc/timezone && \ if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime; fi && \ - apk --no-cache add sudo && \ - # Remove apk cache - rm -rf /var/cache/apk/* + # Install patch utility + apt-get update && \ + apt-get install -y patch && \ + # Remove system cache + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Install any extensions needed by your CKAN instance # - Make sure to add the plugins to CKAN__PLUGINS in the .env file @@ -54,9 +57,6 @@ RUN echo ${TZ} > /etc/timezone && \ # Used to configure the container environment by setting environment variables, creating users, running initialization scripts, .etc COPY docker-entrypoint.d/* /docker-entrypoint.d/ -# Update who.ini with PROXY_CKAN_LOCATION -COPY setup/who.ini ./ - # Override start_ckan.sh with DEV sh COPY setup/start_ckan_development.sh.override ./start_ckan_development.sh RUN chmod +x ./start_ckan_development.sh diff --git a/ckan/Dockerfile.ghcr b/ckan/Dockerfile.ghcr index 03dc78a3..5c37c934 100644 --- a/ckan/Dockerfile.ghcr +++ b/ckan/Dockerfile.ghcr @@ -1,4 +1,4 @@ -FROM ghcr.io/mjanez/ckan-docker:ckan-2.9.11 +FROM ghcr.io/mjanez/ckan-docker:ckan-2.10.5 LABEL maintainer="mnl.janez@gmail.com" # Set up environment variables @@ -17,9 +17,6 @@ RUN echo ${TZ} > /etc/timezone && \ # Used to configure the container environment by setting environment variables, creating users, running initialization scripts, .etc COPY docker-entrypoint.d/* docker-entrypoint.d/ -# Update who.ini with PROXY_CKAN_LOCATION -COPY setup/who.ini ./ - # Apply any patches needed to CKAN core COPY patches patches diff --git a/ckan/docker-entrypoint.d/00_update_who.sh b/ckan/docker-entrypoint.d/00_update_who.sh deleted file mode 100644 index b214ffc6..00000000 --- a/ckan/docker-entrypoint.d/00_update_who.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -# Update who.ini when exists PROXY_CKAN_LOCATION -echo "[docker-entrypoint.00_update_who] Update who.ini" -if [ -n "$PROXY_CKAN_LOCATION" ] && [ "$PROXY_CKAN_LOCATION" != "/" ]; then - sed -i "s|\${WHO_LOCATION}|$PROXY_CKAN_LOCATION|g" "${APP_DIR}/who.ini"; -else - # Check if the value is exactly "/" - if [ "$PROXY_CKAN_LOCATION" = "/" ]; then - sed -i "s|\${WHO_LOCATION}||g" "${APP_DIR}/who.ini"; - else - # Handle the case when $PROXY_CKAN_LOCATION is empty or not set - sed -i "s|\${WHO_LOCATION}|/|g" "${APP_DIR}/who.ini"; - fi -fi diff --git a/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh b/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh index 8ba4079e..ec958146 100644 --- a/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh +++ b/ckan/docker-entrypoint.d/01_setup_ckanext_config.sh @@ -4,6 +4,27 @@ echo "[docker-entrypoint.01_setup_ckanext_config] Clear index" ckan -c $CKAN_INI search-index clear +# ckan: Update core settings +echo "[docker-entrypoint.01_setup_ckanext_config] Loading CKAN core settings in the CKAN config file" +ckan config-tool $CKAN_INI \ + "search.facets.default=$SEARCH__FACETS__DEFAULT" \ + "ckan.datastore.sqlsearch.enabled=$CKAN__DATASTORE__SQLSEARCH__ENABLED" + +# Add SMTP settings if CKAN__SMTP_ENABLED is True +if [ "$CKAN__SMTP_ENABLED" = "True" ]; then + echo "[docker-entrypoint.01_setup_ckanext_config] Adding SMTP settings to the CKAN config file" + ckan config-tool $CKAN_INI \ + "smtp.server=$CKAN_SMTP_SERVER" \ + "smtp.starttls=$CKAN_SMTP_STARTTLS" \ + "smtp.user=$CKAN_SMTP_USER" \ + "smtp.password=$CKAN_SMTP_PASSWORD" \ + "smtp.mail_from=$CKAN_SMTP_MAIL_FROM" \ + "smtp.reply_to=" \ + "email_to=" \ + "error_email_from=" +fi + +# ckanext-schemingdcat: Update settings echo "[docker-entrypoint.01_setup_ckanext_config] Loading ckanext-scheming and ckanext-schemingdcat settings into ckan.ini" ckan config-tool $CKAN_INI \ "scheming.dataset_schemas=$CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA" \ @@ -19,40 +40,43 @@ ckan config-tool $CKAN_INI \ "schemingdcat.show_metadata_templates_toolbar=$CKANEXT__SCHEMINGDCAT_SHOW_METADATA_TEMPLATES_TOOLBAR" \ "schemingdcat.metadata_templates_search_identifier=$CKANEXT__METADATA_TEMPLATES_SEARCH_IDENTIFIER" \ "schemingdcat.endpoints_yaml=$CKANEXT__SCHEMINGDCAT_ENDPOINTS_YAML" \ + "schemingdcat.social_github=$CKANEXT__SCHEMINGDCAT__SOCIAL_GITHUB" \ + "schemingdcat.social_x=$CKANEXT__SCHEMINGDCAT__SOCIAL_X" \ + "schemingdcat.social_linkedin=$CKANEXT__SCHEMINGDCAT__SOCIAL_LINKEDIN" # ckanext-dcat: Add settings to the CKAN config file echo "[docker-entrypoint.01_setup_ckanext_config] Loading ckanext-dcat settings in the CKAN config file" ckan config-tool $CKAN_INI \ - "ckanext.dcat.base_uri = $CKANEXT__DCAT__BASE_URI" \ - "ckanext.dcat.catalog_endpoint = $CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT" \ - "ckanext.dcat.rdf.profiles = $CKANEXT__DCAT__RDF_PROFILES" + "ckanext.dcat.base_uri=$CKANEXT__DCAT__BASE_URI" \ + "ckanext.dcat.catalog_endpoint=$CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT" \ + "ckanext.dcat.rdf.profiles=$CKANEXT__DCAT__RDF_PROFILES" # ckan previews: Add CKAN Resource views to the CKAN config file echo "[docker-entrypoint.01_setup_ckanext_config] Loading resource views in the CKAN config file" ckan config-tool $CKAN_INI \ - "ckan.views.default_views = $CKAN__VIEWS__DEFAULT_VIEWS" \ - "ckan.preview.json_formats = $CKAN__PREVIEW__JSON_FORMATS" \ - "ckan.preview.xml_formats = $CKAN__PREVIEW__XML_FORMATS" \ - "ckan.preview.text_formats = $CKAN__PREVIEW__TEXT_FORMATS" \ - "ckan.preview.loadable = $CKAN__PREVIEW__LOADABLE" + "ckan.views.default_views=$CKAN__VIEWS__DEFAULT_VIEWS" \ + "ckan.preview.json_formats=$CKAN__PREVIEW__JSON_FORMATS" \ + "ckan.preview.xml_formats=$CKAN__PREVIEW__XML_FORMATS" \ + "ckan.preview.text_formats=$CKAN__PREVIEW__TEXT_FORMATS" \ + "ckan.preview.loadable=$CKAN__PREVIEW__LOADABLE" # ckanext-geoview: Add geoviews CKAN config file echo "[docker-entrypoint.01_setup_ckanext_config] Loading geoviews in the CKAN config file" ckan config-tool $CKAN_INI \ - "ckanext.geoview.ol_viewer.formats = $CKANEXT__GEOVIEW__OL_VIEWER__FORMATS" \ - "ckanext.geoview.shp_viewer.srid = $CKANEXT__GEOVIEW__SHP_VIEWER__SRID" \ - "ckanext.geoview.shp_viewer.encoding = $CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING" \ - "ckanext.geoview.geojson.max_file_size = $CKANEXT__GEOVIEW__GEOJSON__MAX_FILE_SIZE" + "ckanext.geoview.ol_viewer.formats=$CKANEXT__GEOVIEW__OL_VIEWER__FORMATS" \ + "ckanext.geoview.shp_viewer.srid=$CKANEXT__GEOVIEW__SHP_VIEWER__SRID" \ + "ckanext.geoview.shp_viewer.encoding=$CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING" \ + "ckanext.geoview.geojson.max_file_size=$CKANEXT__GEOVIEW__GEOJSON__MAX_FILE_SIZE" # ckanext-pages: Add pages CKAN config file echo "[docker-entrypoint.01_setup_ckanext_config] Loading pages config in the CKAN config file" ckan config-tool $CKAN_INI \ - "ckan.pages.allow_html = $CKANEXT__PAGES__ALOW_HTML" \ - "ckanext.pages.organization = $CKANEXT__PAGES__ORGANIZATION" \ - "ckanext.pages.group = $CKANEXT__PAGES__GROUP" \ - "ckanext.pages.about_menu = $CKANEXT__PAGES__ABOUT_MENU" \ - "ckanext.pages.group_menu = $CKANEXT__PAGES__GROUP_MENU" \ - "ckanext.pages.organization_menu = $CKANEXT__PAGES__ORGANIZATION_MENU" + "ckan.pages.allow_html=$CKANEXT__PAGES__ALOW_HTML" \ + "ckanext.pages.organization=$CKANEXT__PAGES__ORGANIZATION" \ + "ckanext.pages.group=$CKANEXT__PAGES__GROUP" \ + "ckanext.pages.about_menu=$CKANEXT__PAGES__ABOUT_MENU" \ + "ckanext.pages.group_menu=$CKANEXT__PAGES__GROUP_MENU" \ + "ckanext.pages.organization_menu=$CKANEXT__PAGES__ORGANIZATION_MENU" # Rebuild index echo "[docker-entrypoint.01_setup_ckanext_config] Rebuild index" diff --git a/ckan/docker-entrypoint.d/02_setup_xloader.sh b/ckan/docker-entrypoint.d/02_setup_xloader.sh index b360431d..0caca95e 100644 --- a/ckan/docker-entrypoint.d/02_setup_xloader.sh +++ b/ckan/docker-entrypoint.d/02_setup_xloader.sh @@ -4,7 +4,7 @@ TOKEN_NAME="xloader" # Get the list of tokens and extract the IDs for tokens with the specified name -TOKEN_IDS=$(ckan -c $CKAN_INI user token list ckan_admin | grep "$TOKEN_NAME" | awk -F'[][]' '{print $2}' | tr -d '[]') +TOKEN_IDS=$(ckan -c $CKAN_INI user token list $CKAN_SYSADMIN_NAME | grep "$TOKEN_NAME" | awk -F'[][]' '{print $2}' | tr -d '[]') # Revoke each previous token of xloader for TOKEN_ID in $TOKEN_IDS @@ -27,4 +27,4 @@ ckan config-tool $CKAN_INI \ # Create ckanext-xloader API_TOKEN echo "[docker-entrypoint.01_setup_xloader] Set up ckanext.xloader.api_token in the CKAN config file" -ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $CKAN_INI user token add ckan_admin xloader | tail -n 1 | tr -d '\t')" \ No newline at end of file +ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $CKAN_INI user token add $CKAN_SYSADMIN_NAME xloader | tail -n 1 | tr -d '\t')" \ No newline at end of file diff --git a/ckan/patches/ckan/00_plugin_slug_preview.patch b/ckan/patches/ckan/00_plugin_slug_preview.patch index 3590303e..b1be4173 100644 --- a/ckan/patches/ckan/00_plugin_slug_preview.patch +++ b/ckan/patches/ckan/00_plugin_slug_preview.patch @@ -1,5 +1,5 @@ diff --git a/ckan/public/base/javascript/plugins/jquery.slug-preview.js b/ckan/public/base/javascript/plugins/jquery.slug-preview.js -index 4a4f78fe25..8c03e816d2 100644 +index 4a4f78fe2..8c03e816d 100644 --- a/ckan/public/base/javascript/plugins/jquery.slug-preview.js +++ b/ckan/public/base/javascript/plugins/jquery.slug-preview.js @@ -33,6 +33,11 @@ diff --git a/ckan/patches/ckanext-pages/01_org_group_templates b/ckan/patches/ckanext-pages/01_org_group_templates new file mode 100644 index 00000000..94b74a69 --- /dev/null +++ b/ckan/patches/ckanext-pages/01_org_group_templates @@ -0,0 +1,30 @@ +diff --git a/ckanext/pages/theme/templates_group/group/read_base.html b/ckanext/pages/theme/templates_group/group/read_base.html +index bbdae8f..6bf00fa 100644 +--- a/ckanext/pages/theme/templates_group/group/read_base.html ++++ b/ckanext/pages/theme/templates_group/group/read_base.html +@@ -3,8 +3,8 @@ + {% block content_primary_nav %} + {{ super() }} + {% if h.ckan_version().split('.')[1] | int >= 9 %} +- {{ h.build_nav_icon('pages.group_pages_index', _('Pages'), id=c.group_dict.name, icon='file') }} ++ {{ h.build_nav_icon('pages.group_pages_index', _('Pages'), id=group_dict.name, icon='file') }} + {% else %} +- {{ h.build_nav_icon('group_pages_index', _('Pages'), id=c.group_dict.name, icon='file') }} ++ {{ h.build_nav_icon('group_pages_index', _('Pages'), id=group_dict.name, icon='file') }} + {% endif %} + {% endblock %} +diff --git a/ckanext/pages/theme/templates_organization/organization/read_base.html b/ckanext/pages/theme/templates_organization/organization/read_base.html +index e5f3b47..eb2f39b 100644 +--- a/ckanext/pages/theme/templates_organization/organization/read_base.html ++++ b/ckanext/pages/theme/templates_organization/organization/read_base.html +@@ -3,8 +3,8 @@ + {% block content_primary_nav %} + {{ super() }} + {% if h.ckan_version().split('.')[1] | int >= 9 %} +- {{ h.build_nav_icon('pages.organization_pages_index', _('Pages'), id=c.group_dict.name, icon='file') }} ++ {{ h.build_nav_icon('pages.organization_pages_index', _('Pages'), id=group_dict.name, icon='file') }} + {% else %} +- {{ h.build_nav_icon('organization_pages_index', _('Pages'), id=c.group_dict.name, icon='file') }} ++ {{ h.build_nav_icon('organization_pages_index', _('Pages'), id=group_dict.name, icon='file') }} + {% endif %} + {% endblock %} diff --git a/ckan/setup/prerun.py.override b/ckan/setup/prerun.py.override index 3d686969..956b35bf 100644 --- a/ckan/setup/prerun.py.override +++ b/ckan/setup/prerun.py.override @@ -33,6 +33,7 @@ def check_main_db_connection(retry=None): conn_str = os.environ.get("CKAN_SQLALCHEMY_URL") if not conn_str: print("[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db") + return return check_db_connection(conn_str, retry) @@ -41,6 +42,7 @@ def check_datastore_db_connection(retry=None): conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL") if not conn_str: print("[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db") + return return check_db_connection(conn_str, retry) @@ -83,29 +85,29 @@ def check_solr_connection(retry=None): time.sleep(10) check_solr_connection(retry=retry - 1) else: - import re - conn_info = connection.read() - schema_name = json.loads(conn_info) - if 'ckan' in schema_name['name']: - print('[prerun] Succesfully connected to solr and CKAN schema loaded') - else: + import re + conn_info = connection.read() + schema_name = json.loads(conn_info) + if 'ckan' in schema_name['name']: + print('[prerun] Succesfully connected to solr and CKAN schema loaded') + else: print('[prerun] Succesfully connected to solr, but CKAN schema not found') def init_db(): - db_command = ["ckan", "-c", ckan_ini, "db", "init"] + db_command = ["ckan", "-c", ckan_ini, "db", "upgrade"] print("[prerun] Initializing or upgrading db - start") try: subprocess.check_output(db_command, stderr=subprocess.STDOUT) print("[prerun] Initializing or upgrading db - end") except subprocess.CalledProcessError as e: - if "OperationalError" in e.output: - print(e.output) + if "OperationalError" in str(e.output): print("[prerun] Database not ready, waiting a bit before exit...") time.sleep(5) sys.exit(1) else: + print(str(e)) print(e.output) raise e @@ -194,6 +196,13 @@ def create_sysadmin(): subprocess.call(command) print("[prerun] Made user {0} a sysadmin".format(name)) + # cleanup permissions + # We're running as root before pivoting to uwsgi and dropping privs + data_dir = "%s/storage" % os.environ['CKAN_STORAGE_PATH'] + + command = ["chown", "-R", "ckan:ckan", data_dir] + subprocess.call(command) + print("[prerun] Ensured storage directory is owned by ckan") if __name__ == "__main__": @@ -208,5 +217,4 @@ if __name__ == "__main__": check_datastore_db_connection() init_datastore_db() check_solr_connection() - create_sysadmin() - \ No newline at end of file + create_sysadmin() \ No newline at end of file diff --git a/ckan/setup/start_ckan.sh.override b/ckan/setup/start_ckan.sh.override index 7d525853..ec3e3ace 100644 --- a/ckan/setup/start_ckan.sh.override +++ b/ckan/setup/start_ckan.sh.override @@ -1,19 +1,33 @@ -#!/bin/sh +#!/bin/bash # Set up the Secret key used by Beaker and Flask # This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var -if grep -E "beaker.session.secret ?= ?$" ckan.ini +if grep -qE "beaker.session.secret ?= ?$" ckan.ini then echo "Setting beaker.session.secret in ini file" ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" +fi + +# Github Codespaces workaround. Check if WTF_CSRF_ENABLED is explicitly set to False and update CKAN_INI +if [ "$WTF_CSRF_ENABLED" = "False" ]; then + echo "WTF_CSRF_ENABLED is set to False. Updating CKAN_INI." + ckan config-tool $CKAN_INI "WTF_CSRF_ENABLED=False" +fi +# Update local (file://) licenses +if [ -n "$CKAN___LICENSES_GROUP_URL" ]; then + ckan config-tool $CKAN_INI "licenses_group_url=${CKAN___LICENSES_GROUP_URL}" echo "Update licenses in CKAN" - if [ -n "$CKAN___LICENSES_GROUP_URL" ]; then - ckan config-tool $CKAN_INI "licenses_group_url=${CKAN___LICENSES_GROUP_URL}" - fi +fi + +# Comment out smtp.* settings if CKAN__SMTP_ENABLED is not True +if [ "$CKAN__SMTP_ENABLED" != "True" ]; then + echo "Commenting out smtp.* settings in ini file" + sed -i 's/^\(smtp\..*\)$/#\1/' $CKAN_INI fi # Run the prerun script to init CKAN and create the default admin user @@ -40,17 +54,31 @@ chown -R ckan:ckan $CKAN_LOGS_PATH/harvester mkdir -p $CKAN_LOGS_PATH/xloader chown -R ckan:ckan $CKAN_LOGS_PATH/xloader -# Set the common uwsgi options -UWSGI_OPTS="--plugins http,python \ +# Set the common uwsgi options. +## Add thunder-lock to prevent multiple workers from running the same job and buffer-size to prevent large headers +## To increase performance, you can adjust the number of processes (-p) (More info: https://uwsgi-docs.readthedocs.io/en/latest/Options.html & https://www.bloomberg.com/company/stories/configuring-uwsgi-production-deployment/) +UWSGI_OPTS="--thunder-lock \ + --ignore-sigpipe \ --socket /tmp/uwsgi.sock \ --wsgi-file /srv/app/wsgi.py \ --module wsgi:application \ --uid 92 --gid 92 \ - --http 0.0.0.0:5000 \ - --master --enable-threads \ + --http [::]:5000 \ + --master \ + --enable-threads \ --lazy-apps \ - -p 2 -L -b 32768 --vacuum \ - --harakiri $UWSGI_HARAKIRI" + --vacuum \ + --harakiri $UWSGI_HARAKIRI \ + --max-requests 500 \ + --max-worker-lifetime 3600 \ + --reload-on-rss 1024 \ + --processes 2 \ + --buffer-size 32768 \ + --disable-logging \ + --log-4xx \ + --log-5xx \ + --auto-procname \ + --procname-prefix 'ckan-'" if [ $? -eq 0 ] then @@ -59,14 +87,27 @@ then supervisord --configuration /etc/supervisord.conf & # Workers - ## Add harvester background procces to crontab + ## Add the harvester background processes to crontab if they do not already exist echo "[prerun.workers] Add harvester background procceses to crontab" - crontab -l | { cat; echo "*/15 * * * * /usr/bin/supervisorctl start ckan_harvester_run"; } | crontab - + if ! crontab -l | grep -q "/usr/bin/supervisorctl start ckan_harvester_run"; then + { crontab -l; echo "*/15 * * * * /usr/bin/supervisorctl start ckan_harvester_run"; } | crontab - + fi ## Clean-up mechanism for the harvest log table. 'ckan.harvest.log_timeframe'. The default time frame is 30 days - crontab -l | { cat; echo "0 5 */30 * * /usr/bin/supervisorctl start ckan_harvester_clean_log"; } | crontab - + if ! crontab -l | grep -q "/usr/bin/supervisorctl start ckan_harvester_clean_log"; then + { crontab -l; echo "0 5 */30 * * /usr/bin/supervisorctl start ckan_harvester_clean_log"; } | crontab - + fi + + ## Add SMTP notification process to crontab if CKAN__SMTP_ENABLED is True + if [ "$CKAN__SMTP_ENABLED" = "True" ]; then + echo "[prerun.workers] Add SMTP notification process to crontab" + if ! crontab -l | grep -q "ckan -c $CKAN_INI notify send_emails"; then + { crontab -l; echo "@hourly echo '{}' | ckan -c $CKAN_INI notify send_emails > /dev/null"; } | crontab - + fi + fi + ## Execute cron in the background echo "[prerun.workers] Execute cron in the background" - crond -b -l 8 + cron -f & # Start uwsgi uwsgi $UWSGI_OPTS diff --git a/ckan/setup/start_ckan_development.sh.override b/ckan/setup/start_ckan_development.sh.override index 4669dac6..724e7322 100644 --- a/ckan/setup/start_ckan_development.sh.override +++ b/ckan/setup/start_ckan_development.sh.override @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Install any local extensions in the src_extensions volume echo "Looking for local extensions to install..." @@ -31,6 +31,13 @@ do echo "Found setup.py file in $i" cd $APP_DIR fi + if [ -f $i/pyproject.toml ]; + then + cd $i + pip install -e . + echo "Found pyproject.toml file in $i" + cd $APP_DIR + fi # Point `use` in test.ini to location of `test-core.ini` if [ -f $i/test.ini ]; @@ -47,18 +54,14 @@ ckan config-tool $CKAN_INI -s DEFAULT "debug = true" # Set up the Secret key used by Beaker and Flask # This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var -if grep -E "beaker.session.secret ?= ?$" ckan.ini +if grep -qE "beaker.session.secret ?= ?$" ckan.ini then echo "Setting beaker.session.secret in ini file" ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" - - echo "Update licenses in CKAN" - if [ -n "$CKAN___LICENSES_GROUP_URL" ]; then - ckan config-tool $CKAN_INI "licenses_group_url=${CKAN___LICENSES_GROUP_URL}" - fi fi # Update the plugins setting in the ini file with the values defined in the env var @@ -74,6 +77,25 @@ ckan config-tool $SRC_DIR/ckan/test-core.ini \ "solr_url = $TEST_CKAN_SOLR_URL" \ "ckan.redis.url = $TEST_CKAN_REDIS_URL" +# Update local (file://) licenses for Development mode +if [ -n "$LICENSES_GROUP_URL" ]; then + expanded_app_dir=$(eval echo "$APP_DIR") + expanded_src_dir=$(eval echo "$SRC_DIR") + expanded_src_extensions_dir=$(eval echo "$SRC_EXTENSIONS_DIR") + + if [[ "$LICENSES_GROUP_URL" == *"$expanded_src_dir"* ]]; then + LICENSES_GROUP_URL=${LICENSES_GROUP_URL//$expanded_src_dir/$expanded_src_extensions_dir} + echo "Updating licenses group URL to: $LICENSES_GROUP_URL" + ckan config-tool $CKAN_INI "licenses_group_url=${LICENSES_GROUP_URL}" + elif [[ "$LICENSES_GROUP_URL" == "file://$expanded_app_dir/src/"* ]]; then + LICENSES_GROUP_URL=${LICENSES_GROUP_URL//file:\/\/$expanded_app_dir\/src\//file:\/\/$expanded_app_dir\/src_extensions\/} + echo "Updating licenses group URL to: $LICENSES_GROUP_URL" + ckan config-tool $CKAN_INI "licenses_group_url=${LICENSES_GROUP_URL}" + else + echo "LICENSES_GROUP_URL does not contain $expanded_src_dir or file://$expanded_app_dir/src/. No changes made." + fi +fi + # Run the prerun script to init CKAN and create the default admin user python3 prerun.py @@ -90,6 +112,18 @@ then done fi +CKAN_RUN="ckan -c $CKAN_INI run -H 0.0.0.0" +CKAN_OPTIONS="" +if [ "$USE_DEBUGPY_FOR_DEV" = true ] ; then + pip install debugpy + CKAN_RUN="/usr/bin/python -m debugpy --listen 0.0.0.0:5678 $CKAN_RUN" + CKAN_OPTIONS="$CKAN_OPTIONS --disable-reloader" +fi + +if [ "$USE_HTTPS_FOR_DEV" = true ] ; then + CKAN_OPTIONS="$CKAN_OPTIONS -C unsafe.cert -K unsafe.key" +fi + # Create Harvester logs directory and change its ownership mkdir -p $CKAN_LOGS_PATH/harvester chown -R ckan:ckan $CKAN_LOGS_PATH/harvester @@ -102,7 +136,11 @@ chown -R ckan:ckan $CKAN_LOGS_PATH/xloader #supervisord --configuration /etc/supervisord.conf & # Start the development server as the ckan user with automatic reload -su ckan -c "/usr/bin/ckan -c $CKAN_INI run -H 0.0.0.0" +while true; do + su ckan -c "$CKAN_RUN $CKAN_OPTIONS" + echo Exit with status $?. Restarting. + sleep 2 +done # Workers # To start the Harvester worker diff --git a/ckan/setup/who.ini b/ckan/setup/who.ini deleted file mode 100644 index 34a753ff..00000000 --- a/ckan/setup/who.ini +++ /dev/null @@ -1,32 +0,0 @@ -[plugin:auth_tkt] -use=ckan.lib.repoze_plugins.auth_tkt:make_plugin -# If no secret key is defined here, beaker.session.secret will be used -#secret = somesecret - -[plugin:friendlyform] -use=ckan.lib.repoze_plugins.friendly_form:FriendlyFormPlugin -login_form_url=${WHO_LOCATION}/user/login -login_handler_path=/login_generic -logout_handler_path=/user/logout -rememberer_name=auth_tkt -post_login_url=${WHO_LOCATION}/user/logged_in -post_logout_url=${WHO_LOCATION}/user/logged_out -charset=utf-8 - -[general] -request_classifier=repoze.who.classifiers:default_request_classifier -challenge_decider=repoze.who.classifiers:default_challenge_decider - -[identifiers] -plugins = - friendlyform;browser - auth_tkt - -[authenticators] -plugins = - auth_tkt - ckan.lib.authenticator:UsernamePasswordAuthenticator - -[challengers] -plugins = - friendlyform;browser \ No newline at end of file diff --git a/ckan/setup/workers/xloader.conf b/ckan/setup/workers/xloader.conf index c7749ddb..7d12cef1 100644 --- a/ckan/setup/workers/xloader.conf +++ b/ckan/setup/workers/xloader.conf @@ -2,9 +2,9 @@ command=ckan jobs worker default user=ckan numprocs=1 -stdout_logfile=/var/log/harvester/ckan_xloader.log +stdout_logfile=/var/log/xloader/ckan_xloader.log stdout_logfile_maxbytes=100MB -stderr_logfile=/var/log/harvester/ckan_xloader.log +stderr_logfile=/var/log/xloader/ckan_xloader.log stderr_logfile_maxbytes=100MB autostart=true autorestart=true diff --git a/doc/harvesters/README.md b/doc/harvesters/README.md new file mode 100644 index 00000000..c1e5fb7b --- /dev/null +++ b/doc/harvesters/README.md @@ -0,0 +1,29 @@ +# Harvesters. Configuration examples +>[!TIP] +> For more information about scheming cat harvesters, see [docs](https://github.com/mjanez/ckanext-schemingdcat?tab=readme-ov-file#harvesters). + +This subdirectory contains examples of JSON configuration files for harvesters. These configurations include various settings such as API version, tags, groups, and field mappings. These JSON files are used to configure the harvesting process from different sources. + +For example, a configuration might include settings like: +- `api_version` +- `clean_tags` +- `default_groups` +- `default_tags` +- `default_extras` +- `override_extras` +- `organizations_filter_include` +- `groups_filter_include` +- `allow_private_datasets` +- `user` +- `api_key` +- `read_only` +- `remote_groups` +- `remote_orgs` +- `default_group_dicts` +- `schema` +- `allow_harvest_datasets` +- `field_mapping_schema_version` +- `dataset_field_mapping` +- `distribution_field_mapping` + +These settings help in defining how the data should be harvested, processed, and stored. \ No newline at end of file diff --git a/doc/harvesters/ckan_template_config.json b/doc/harvesters/ckan_template_config.json new file mode 100644 index 00000000..942dfb3c --- /dev/null +++ b/doc/harvesters/ckan_template_config.json @@ -0,0 +1,272 @@ +{ + "api_version":2, + "clean_tags":false, + "default_groups":[], + "default_tags":[], + "default_extras":{ + "encoding":"UTF-8", + "harvest_description":"Harvest from Sample Portal", + "harvest_url":"{harvest_source_url}/dataset/{dataset_id}" + }, + "override_extras":false, + "organizations_filter_include":[ + + ], + "groups_filter_include":[], + "allow_private_datasets": true, + "user":"admin", + "api_key":"eyJ0easdasdasdasdasd", + "read_only":true, + "remote_groups":"only_local", + "remote_orgs":"only_local", + "default_group_dicts":[ + + ], + "schema":"es_geodcat_ap_full", + "allow_harvest_datasets":false, + "field_mapping_schema_version":2, + "dataset_field_mapping":{ + "title_translated":{ + "field_name":"title_translated" + }, + "schemingdcat_xls_metadata_template":{ + "field_name":"schemingdcat_xls_metadata_template" + }, + "dataset_scope":{ + "field_name":"dataset_scope" + }, + "graphic_overview":{ + "field_name":"graphic_overview" + }, + "identifier":{ + "field_name":"identifier" + }, + "name":{ + "field_name":"name" + }, + "dcat_type":{ + "field_name":"dcat_type" + }, + "theme_es":{ + "field_name":"theme_es" + }, + "hvd_category":{ + "field_name":"hvd_category" + }, + "language":{ + "field_name":"language" + }, + "created":{ + "field_name":"created" + }, + "modified":{ + "field_name":"modified" + }, + "notes_translated":{ + "field_name":"notes_translated" + }, + "topic":{ + "field_name":"topic" + }, + "tag_string":{ + "field_name":"tag_string" + }, + "tag_uri":{ + "field_name":"tag_uri" + }, + "contact_name":{ + "field_name":"contact_name" + }, + "contact_email":{ + "field_name":"contact_email" + }, + "contact_url":{ + "field_name":"contact_url" + }, + "contact_uri":{ + "field_name":"contact_uri" + }, + "publisher_name":{ + "field_name":"publisher_name" + }, + "publisher_email":{ + "field_name":"publisher_email" + }, + "publisher_url":{ + "field_name":"publisher_url" + }, + "publisher_uri":{ + "field_name":"publisher_uri" + }, + "publisher_type":{ + "field_name":"publisher_type" + }, + "maintainer":{ + "field_name":"maintainer" + }, + "maintainer_email":{ + "field_name":"maintainer_email" + }, + "maintainer_url":{ + "field_name":"maintainer_url" + }, + "maintainer_uri":{ + "field_name":"maintainer_uri" + }, + "author":{ + "field_name":"author" + }, + "author_email":{ + "field_name":"author_email" + }, + "author_url":{ + "field_name":"author_url" + }, + "author_uri":{ + "field_name":"author_uri" + }, + "alternate_identifier":{ + "field_name":"alternate_identifier" + }, + "provenance":{ + "field_name":"provenance" + }, + "conforms_to":{ + "field_name":"conforms_to" + }, + "metadata_profile":{ + "field_name":"metadata_profile" + }, + "temporal_start":{ + "field_name":"temporal_start" + }, + "temporal_end":{ + "field_name":"temporal_end" + }, + "frequency":{ + "field_name":"frequency" + }, + "source":{ + "field_name":"source" + }, + "lineage_source":{ + "field_name":"lineage_source" + }, + "lineage_process_steps":{ + "field_name":"lineage_process_steps" + }, + "reference":{ + "field_name":"reference" + }, + "purpose":{ + "field_name":"purpose" + }, + "encoding":{ + "field_name":"encoding" + }, + "temporal_resolution":{ + "field_name":"temporal_resolution" + }, + "theme":{ + "field_name":"theme" + }, + "inspire_id":{ + "field_name":"inspire_id" + }, + "reference_system":{ + "field_name":"reference_system" + }, + "spatial":{ + "field_name":"spatial" + }, + "spatial_uri":{ + "field_name":"spatial_uri" + }, + "representation_type":{ + "field_name":"representation_type" + }, + "spatial_resolution_in_meters":{ + "field_name":"spatial_resolution_in_meters" + }, + "license_id":{ + "field_name":"license_id" + }, + "access_rights":{ + "field_name":"access_rights" + }, + "version":{ + "field_name":"version" + }, + "version_notes":{ + "field_name":"version_notes" + }, + "valid":{ + "field_name":"valid" + }, + "groups":{ + "field_name":"groups" + }, + "private":{ + "field_name":"private" + } + }, + "distribution_field_mapping":{ + "dataset_id":{ + "field_name":"id" + }, + "url":{ + "field_name":"url" + }, + "created":{ + "field_name":"created" + }, + "name":{ + "field_name":"name" + }, + "description":{ + "field_name":"description" + }, + "encoding":{ + "field_name":"encoding" + }, + "availability":{ + "field_name":"availability" + }, + "issued":{ + "field_name":"issued" + }, + "modified":{ + "field_name":"modified" + }, + "language":{ + "field_name":"language" + }, + "rights":{ + "field_name":"rights" + }, + "format":{ + "field_name":"format" + }, + "mimetype":{ + "field_name":"mimetype" + }, + "status":{ + "field_name":"status" + }, + "size":{ + "field_name":"size" + }, + "resource_relation":{ + "field_name":"resource_relation" + }, + "license":{ + "field_name":"license" + }, + "conforms_to":{ + "field_name":"conforms_to" + }, + "reference_system":{ + "field_name":"reference_system" + } + } + } \ No newline at end of file diff --git a/doc/info_envfile.md b/doc/info_envfile.md index 9d379b1d..2fb1867c 100644 --- a/doc/info_envfile.md +++ b/doc/info_envfile.md @@ -156,7 +156,6 @@ This is a CKAN extension that allows users to harvest dataset metadata from mult ### ckanext-xloader This is a CKAN extension that allows users to load data into CKAN's data store for faster and more efficient processing. -- `CKANEXT__XLOADER__API_TOKEN`: The API token for the XLoader extension. - `CKANEXT__XLOADER__JOBS__DB_URI`: The database URI for the XLoader jobs. ### ckanext-dcat diff --git a/samples/.env.codespaces b/samples/.env.codespaces new file mode 100644 index 00000000..116a3877 --- /dev/null +++ b/samples/.env.codespaces @@ -0,0 +1,216 @@ +# Base +APP_DIR=/srv/app +# Sets Docker Compose project name to avoid resource conflicts between projects. Defaults to the folder name "ckan-docker" if not set. +#COMPOSE_PROJECT_NAME=ckan-docker-mytheme + +# Github Codespaces avoid errors with the CSRF token. +WTF_CSRF_ENABLED=False + +# Host Ports +CKAN_PORT_HOST=5000 +# Common proxy server for Apache or NGINX. +# Change all the PROXY_SERVER_PORT_HOST to PROXY_SERVER_HTTPS_PORT_HOST if you can use HTTPS instead of HTTP (Only for NGINX and not development compose) +PROXY_SERVER_BASE_PORT_HOST=81 +PROXY_SERVER_HTTPS_PORT_HOST=8443 +# Server port for the proxy server. Better using the SSL port for production, in DEV or if using balancer: PROXY_SERVER_PORT_HOST=PROXY_SERVER_BASE_PORT_HOST +PROXY_SERVER_PORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} +NGINX_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} +NGINX_SSLPORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} +APACHE_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} +PYCSW_PORT_HOST=8000 + +# Solr +SOLR_IMAGE_VERSION=2.9-solr9-spatial +SOLR_PORT=8983 +SOLR_CKAN_DATABASE=ckan +CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/${SOLR_CKAN_DATABASE} +TEST_CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/${SOLR_CKAN_DATABASE} + +# Redis +REDIS_VERSION=7-alpine +REDIS_PORT=6379 +REDIS_CKAN_DATABASE=1 +CKAN_REDIS_URL=redis://redis:${REDIS_PORT}/${REDIS_CKAN_DATABASE} +TEST_CKAN_REDIS_URL=redis://redis:${REDIS_PORT}/${REDIS_CKAN_DATABASE} + +# NGINX +NGINX_PORT=80 +NGINX_SSLPORT=443 +NGINX_LOG_DIR=/var/log/nginx + +# Apache HTTP Server +APACHE_VERSION=2.4-alpine +APACHE_PORT=80 +APACHE_LOG_DIR=/var/log/apache + +#NGINX/APACHE +## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI and CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. If you dont need PROXY_SERVER_PORT_HOST (e.g. is 81), edit the PROXY_SERVER_URL and remove :${PROXY_SERVER_PORT_HOST} +PROXY_SERVER_NAME=localhost +# URL protocol. Change to https if you use SSL +PROXY_SERVER_PROTOCOL=https +PROXY_SERVER_URL=${PROXY_SERVER_PROTOCOL}://${PROXY_SERVER_NAME} +PROXY_CKAN_LOCATION=/catalog +PROXY_PYCSW_LOCATION=/csw + +# pycsw +PYCSW_PORT=8000 +CKAN_URL=${PROXY_SERVER_URL}${PROXY_CKAN_LOCATION} +PYCSW_URL=${PROXY_SERVER_URL}${PROXY_PYCSW_LOCATION} +# SCHEMAS: ckan2pycsw/model/dataset.py - Dataset type +PYCSW_CKAN_SCHEMA=iso19139_geodcatap +PYCSW_OUPUT_SCHEMA=iso19139_inspire +# ckan-pycsw schedule +## ckan2pycsw days between each scheduler job +PYCSW_CRON_DAYS_INTERVAL=2 +# ckan2pycsw hour of start of the scheduler job (0-23) +PYCSW_CRON_HOUR_START=4 +## Timezone +TZ=UTC + +# CKAN databases +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=postgres +POSTGRES_HOST=db +CKAN_DB_USER=ckandbuser +CKAN_DB_PASSWORD=ckandbpassword +CKAN_DB=ckandb +DATASTORE_READONLY_USER=datastore_ro +DATASTORE_READONLY_PASSWORD=datastore +DATASTORE_DB=datastore +CKAN_SQLALCHEMY_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB} +CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_DB} +CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_DB} + +# Test database connections +CKAN_TEST_DB=ckan_test +DATASTORE_TEST_DB=datastore_test +TEST_CKAN_SQLALCHEMY_URL=postgres://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_TEST_DB} +TEST_CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_TEST_DB} +TEST_CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_TEST_DB} + +# Dev settings +USE_HTTPS_FOR_DEV=false +CKAN_DEV_COMPOSE_SERVICE=ckan-dev + +# CKAN core +## If use docker-compose.ghcr.yml only "*.*.*" versions available in: https://github.com/mjanez/ckan-docker/pkgs/container/ckan-docker +CKAN_VERSION=2.10.5 +CKAN_SITE_ID=default +# CKAN_SITE_URL = http:/ or https:/ + PROXY_SERVER_NAME. Optionally the APACHE_HOST_PORT if different from 80 +CKAN_SITE_URL=${PROXY_SERVER_URL} +CKAN__ROOT_PATH=${PROXY_CKAN_LOCATION}/{{LANG}} +CKAN__FAVICON=${PROXY_CKAN_LOCATION}/base/images/ckan.ico +CKAN__SITE_LOGO=/images/default/ckan-logo.png +# CKAN site info +CKAN__SITE_INTRO_TEXT="" +CKAN__SITE_DESCRIPTION="" +CKAN__SITE_ABOUT="" +# Custom licenses that are aligned with DCAT-AP (Also works with DEV Mode) +LICENSES_GROUP_URL=file://${APP_DIR}/src/ckanext-schemingdcat/ckanext/schemingdcat/public/static/licenses.json +CKAN___BEAKER__SESSION__SECRET=CHANGE_ME +# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings +CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME +CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME +CKAN_SYSADMIN_NAME=ckan_admin +CKAN_SYSADMIN_PASSWORD=test1234 +CKAN_SYSADMIN_EMAIL=your_email@example.com +CKAN_STORAGE_PATH=/var/lib/ckan +CKAN_LOGS_PATH=/var/log +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@${PROXY_SERVER_NAME} +## Customize which text formats the text_view plugin will show +CKAN__PREVIEW__JSON_FORMATS="json jsonld" +# html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json +CKAN__PREVIEW__XML_FORMATS="xml rdf rdf+xml owl+xml atom rss turtle ttl n3 n-triples" +CKAN__PREVIEW__TEXT_FORMATS="text plain text/plain text/turtle csv tsv rss txt json" +CKAN__PREVIEW__LOADABLE="html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json arcgis_rest" +# ckanext-spatial: Allow Solr local params: https://github.com/ckan/ckanext-spatial/issues/328 +CKAN__SEARCH__SOLR_ALLOWED_QUERY_PARSERS=field +# CORS Settings. If True, all origins will be allowed (the response header Access-Control-Allow-Origin is set to ‘*’). If False, only origins from the ckan.cors.origin_whitelist setting will be allowed. +CKAN__CORS__ORIGIN_ALLOW_ALL=False +CKAN__CORS__ORIGIN_WHITELIST="" +# Enables or disable collaborators in individual datasets (https://docs.ckan.org/en/2.9/maintaining/authorization.html#dataset-collaborators) +CKAN__AUTH__ALLOW_DATASET_COLLABORATORS=False +CKAN__AUTH__ALLOW_ADMIN_COLLABORATORS=False +# Default number of facets shown in search results. Default 10. +SEARCH__FACETS__DEFAULT=4 +# Enable or disable the DataStore SQL search backend. Default is False. +CKAN__DATASTORE__SQLSEARCH__ENABLED=True + +## Resource Proxy settings +### Preview size limit, default: 1MB +CKAN__RESOURCE_PROXY__MAX_FILE_SIZE=50048576 +## Size of chunks to read/write__ +CKAN__RESOURCE_PROXY__CHUNK_SIZE=4096 +## Default timeout for fetching proxied items +CKAN__RESOURCE_PROXY__TIMEOUT=10 +CKAN__VIEWS__DEFAULT_VIEWS="image_view video_view audio_view webpage_view text_view datatables_view wmts_view geojson_view geo_view shp_view pdf_view" + +# Localization +CKAN__LOCALE_DEFAULT="en" +CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" +CKAN__LOCALES_OFFERED="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" + +# Extensions +CKAN__PLUGINS="stats image_view video_view audio_view webpage_view text_view datatables_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" + +# ckanext-harvest +CKAN__HARVEST__MQ__TYPE=redis +CKAN__HARVEST__MQ__HOSTNAME=redis +CKAN__HARVEST__MQ__PORT=${REDIS_PORT} +CKAN__HARVEST__MQ__REDIS_DB=${REDIS_CKAN_DATABASE} +# Clean-up mechanism for the harvest log table. The default is 30 days. +CKAN__HARVEST__LOG_TIMEFRAME=40 + +# ckanext-xloader +CKANEXT__XLOADER__JOBS__DB_URI=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB} + +# ckanext-dcat +CKANEXT__DCAT__BASE_URI=${CKAN_URL} +# Default profile(s). Instead of this envvar, it's possible to specify all the profile(s) availables to be used for serialization using the profiles parameter: http://localhost:5000/catalog.xml?profiles=eu_dcat_ap_2,es_dcat +CKANEXT__DCAT__RDF_PROFILES='eu_dcat_ap_2 eu_dcat_ap_scheming' +# The custom endpoint **must** start with a forward slash (`/`) and contain the `{_format}` placeholder. The endpoint is added to the CKAN_SITE_URL and CKAN__ROOT_PATH, example: http://localhost:5000/catalog/catalog.rdf +CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT='/catalog.{_format}' + +# ckanext-spatial (Solr Backend - solr9-spatial: https://docs.ckan.org/projects/ckanext-spatial/en/latest/spatial-search.html#choosing-a-backend-for-the-spatial-search) +CKANEXT__SPATIAL__SEARCH_BACKEND=solr-spatial-field +CKAN__SPATIAL__SRID=3857 +CKANEXT__SPATIAL__COMMON_MAP__TYPE=custom +CKANEXT__SPATIAL__COMMON_MAP__CUSTOM__URL=http://a.tile.openstreetmap.org/{z}/{x}/{y}.png +CKANEXT__SPATIAL__COMMON_MAP__ATTRIBUTION='Map tiles by OpenStreetMap (CC BY SA)' + +# ckanext-geoview +CKANEXT__GEOVIEW__GEOJSON__MAX_FILE_SIZE=100000000 +CKANEXT__GEOVIEW__OL_VIEWER__FORMATS="wms wfs geojson gml kml" +CKANEXT__GEOVIEW__SHP_VIEWER__SRID=3857 +CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8 + +# ckanext-schemingdcat +## CSW Endpoint for spatial metadata +CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI=${PYCSW_URL} +## Scheming: setup_scheming.sh +CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_full.yaml ckanext.schemingdcat:schemas/resources/dcat_document.yaml" +CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_group.json" +CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_org.json" +CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/default_presets.json ckanext.fluent:presets.json" +## Facets: setup_scheming.sh +CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_eu language dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" +CKANEXT__SCHEMINGDCAT_ORGANIZATION_CUSTOM_FACETS=True +CKANEXT__SCHEMINGDCAT_GROUP_CUSTOM_FACETS=True +CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_ICON="theme" +CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_SHOW_SPATIAL=True +CKANEXT__SCHEMINGDCAT_SHOW_METADATA_TEMPLATES_TOOLBAR=False +CKANEXT__METADATA_TEMPLATES_SEARCH_IDENTIFIER="schemingdcat_xls-template" +CKANEXT__SCHEMINGDCAT_ENDPOINTS_YAML="endpoints.yaml" + +# ckanext-pages +CKANEXT__PAGES__ALOW_HTML=False +CKANEXT__PAGES__ORGANIZATION=True +CKANEXT__PAGES__GROUP=True +CKANEXT__PAGES__ABOUT_MENU=False +CKANEXT__PAGES__GROUP_MENU=True +CKANEXT__PAGES__ORGANIZATION_MENU=True \ No newline at end of file diff --git a/samples/.env.es.example b/samples/.env.es.example index 50a20ddc..c0d7da11 100644 --- a/samples/.env.es.example +++ b/samples/.env.es.example @@ -7,11 +7,13 @@ APP_DIR=/srv/app CKAN_PORT_HOST=5000 # Common proxy server for Apache or NGINX. # Change all the PROXY_SERVER_PORT_HOST to PROXY_SERVER_HTTPS_PORT_HOST if you can use HTTPS instead of HTTP (Only for NGINX and not development compose) -PROXY_SERVER_PORT_HOST=81 +PROXY_SERVER_BASE_PORT_HOST=81 PROXY_SERVER_HTTPS_PORT_HOST=8443 -NGINX_PORT_HOST=${PROXY_SERVER_PORT_HOST} +# Server port for the proxy server. Better using the SSL port for production, in DEV or if using balancer: PROXY_SERVER_PORT_HOST=PROXY_SERVER_BASE_PORT_HOST +PROXY_SERVER_PORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} +NGINX_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} NGINX_SSLPORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} -APACHE_PORT_HOST=${PROXY_SERVER_PORT_HOST} +APACHE_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} PYCSW_PORT_HOST=8000 # Solr @@ -41,7 +43,9 @@ APACHE_LOG_DIR=/var/log/apache #NGINX/APACHE ## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI and CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. If you dont need PROXY_SERVER_PORT_HOST (e.g. is 80), edit the PROXY_SERVER_URL and remove :${PROXY_SERVER_PORT_HOST} PROXY_SERVER_NAME=localhost -PROXY_SERVER_URL=http://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} +# URL protocol. Change to https if you use SSL +PROXY_SERVER_PROTOCOL=http +PROXY_SERVER_URL=${PROXY_SERVER_PROTOCOL}://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} PROXY_CKAN_LOCATION=/catalog PROXY_PYCSW_LOCATION=/csw @@ -88,15 +92,19 @@ CKAN_DEV_COMPOSE_SERVICE=ckan-dev # CKAN core ## If use docker-compose.ghcr.yml only "*.*.*" versions available in: https://github.com/mjanez/ckan-docker/pkgs/container/ckan-docker -CKAN_VERSION=2.9.11 +CKAN_VERSION=2.10.5 CKAN_SITE_ID=default # CKAN_SITE_URL = http:/ or https:/ + PROXY_SERVER_NAME. Optionally the APACHE_HOST_PORT if different from 80 CKAN_SITE_URL=${PROXY_SERVER_URL} CKAN__ROOT_PATH=${PROXY_CKAN_LOCATION}/{{LANG}} CKAN__FAVICON=${PROXY_CKAN_LOCATION}/base/images/ckan.ico CKAN__SITE_LOGO=/images/default/ckan-logo.png -# Custom licenses that are aligned with DCAT-AP -CKAN___LICENSES_GROUP_URL=https://raw.githubusercontent.com/mjanez/ckanext-schemingdcat/main/ckanext/schemingdcat/public/static/licenses.json +# CKAN site info +CKAN__SITE_INTRO_TEXT="" +CKAN__SITE_DESCRIPTION="" +CKAN__SITE_ABOUT="" +# Custom licenses that are aligned with DCAT-AP (Also works with DEV Mode) +LICENSES_GROUP_URL=file://${APP_DIR}/src/ckanext-schemingdcat/ckanext/schemingdcat/public/static/licenses.json CKAN___BEAKER__SESSION__SECRET=CHANGE_ME # See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME @@ -125,6 +133,10 @@ CKAN__CORS__ORIGIN_WHITELIST="" # Enables or disable collaborators in individual datasets (https://docs.ckan.org/en/2.9/maintaining/authorization.html#dataset-collaborators) CKAN__AUTH__ALLOW_DATASET_COLLABORATORS=False CKAN__AUTH__ALLOW_ADMIN_COLLABORATORS=False +# Default number of facets shown in search results. Default 10. +SEARCH__FACETS__DEFAULT=4 +# Enable or disable the DataStore SQL search backend. Default is False. +CKAN__DATASTORE__SQLSEARCH__ENABLED=True ## Resource Proxy settings ### Preview size limit, default: 1MB @@ -133,14 +145,15 @@ CKAN__RESOURCE_PROXY__MAX_FILE_SIZE=50048576 CKAN__RESOURCE_PROXY__CHUNK_SIZE=4096 ## Default timeout for fetching proxied items CKAN__RESOURCE_PROXY__TIMEOUT=10 -CKAN__VIEWS__DEFAULT_VIEWS="image_view webpage_view text_view recline_view wmts_view geojson_view geo_view shp_view pdf_view" +CKAN__VIEWS__DEFAULT_VIEWS="image_view video_view audio_view webpage_view text_view datatables_view wmts_view geojson_view geo_view shp_view pdf_view" # Localization CKAN__LOCALE_DEFAULT="es" CKAN__LOCALE_ORDER="es en pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" +CKAN__LOCALES_OFFERED="es en pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" # Extensions -CKAN__PLUGINS="stats image_view text_view recline_view webpage_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" +CKAN__PLUGINS="stats image_view video_view audio_view webpage_view text_view datatables_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" # ckanext-harvest CKAN__HARVEST__MQ__TYPE=redis @@ -151,7 +164,6 @@ CKAN__HARVEST__MQ__REDIS_DB=${REDIS_CKAN_DATABASE} CKAN__HARVEST__LOG_TIMEFRAME=40 # ckanext-xloader -CKANEXT__XLOADER__API_TOKEN=api_token CKANEXT__XLOADER__JOBS__DB_URI=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB} # ckanext-dcat @@ -178,12 +190,12 @@ CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8 ## CSW Endpoint for spatial metadata CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI=${PYCSW_URL} ## Scheming: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcatap_es/geodcatap_es_dataset.yaml" -CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap_es/geodcatap_es_group.json" -CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap_es/geodcatap_es_org.json" -CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/default_presets.json ckanext.fluent:presets.json ckanext.iepnb:schemas/presets.json" +CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/es_geodcat_ap_full.yaml" +CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/es_geodcat_ap_group.json" +CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/es_geodcat_ap_org.json" +CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/default_presets.json ckanext.fluent:presets.json" ## Facets: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_es dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" +CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_es language dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" CKANEXT__SCHEMINGDCAT_ORGANIZATION_CUSTOM_FACETS=True CKANEXT__SCHEMINGDCAT_GROUP_CUSTOM_FACETS=True CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_ICON="theme" diff --git a/samples/.env.localhost b/samples/.env.localhost index 2d3dfeb4..8120d9f2 100644 --- a/samples/.env.localhost +++ b/samples/.env.localhost @@ -7,11 +7,13 @@ APP_DIR=/srv/app CKAN_PORT_HOST=5000 # Common proxy server for Apache or NGINX. # Change all the PROXY_SERVER_PORT_HOST to PROXY_SERVER_HTTPS_PORT_HOST if you can use HTTPS instead of HTTP (Only for NGINX and not development compose) -PROXY_SERVER_PORT_HOST=81 +PROXY_SERVER_BASE_PORT_HOST=81 PROXY_SERVER_HTTPS_PORT_HOST=8443 -NGINX_PORT_HOST=${PROXY_SERVER_PORT_HOST} +# Server port for the proxy server. Better using the SSL port for production, in DEV or if using balancer: PROXY_SERVER_PORT_HOST=PROXY_SERVER_BASE_PORT_HOST +PROXY_SERVER_PORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} +NGINX_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} NGINX_SSLPORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST} -APACHE_PORT_HOST=${PROXY_SERVER_PORT_HOST} +APACHE_PORT_HOST=${PROXY_SERVER_BASE_PORT_HOST} PYCSW_PORT_HOST=8000 # Solr @@ -41,7 +43,9 @@ APACHE_LOG_DIR=/var/log/apache #NGINX/APACHE ## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI and CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. If you dont need PROXY_SERVER_PORT_HOST (e.g. is 80), edit the PROXY_SERVER_URL and remove :${PROXY_SERVER_PORT_HOST} PROXY_SERVER_NAME=localhost -PROXY_SERVER_URL=http://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} +# URL protocol. Change to https if you use SSL +PROXY_SERVER_PROTOCOL=http +PROXY_SERVER_URL=${PROXY_SERVER_PROTOCOL}://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST} PROXY_CKAN_LOCATION=/catalog PROXY_PYCSW_LOCATION=/csw @@ -88,15 +92,19 @@ CKAN_DEV_COMPOSE_SERVICE=ckan-dev # CKAN core ## If use docker-compose.ghcr.yml only "*.*.*" versions available in: https://github.com/mjanez/ckan-docker/pkgs/container/ckan-docker -CKAN_VERSION=2.9.11 +CKAN_VERSION=2.10.5 CKAN_SITE_ID=default # CKAN_SITE_URL = http:/ or https:/ + PROXY_SERVER_NAME. Optionally the APACHE_HOST_PORT if different from 80 CKAN_SITE_URL=${PROXY_SERVER_URL} CKAN__ROOT_PATH=${PROXY_CKAN_LOCATION}/{{LANG}} CKAN__FAVICON=${PROXY_CKAN_LOCATION}/base/images/ckan.ico CKAN__SITE_LOGO=/images/default/ckan-logo.png -# Custom licenses that are aligned with DCAT-AP -CKAN___LICENSES_GROUP_URL=https://raw.githubusercontent.com/mjanez/ckanext-schemingdcat/main/ckanext/schemingdcat/public/static/licenses.json +# CKAN site info +CKAN__SITE_INTRO_TEXT="" +CKAN__SITE_DESCRIPTION="" +CKAN__SITE_ABOUT="" +# Custom licenses that are aligned with DCAT-AP (Also works with DEV Mode) +LICENSES_GROUP_URL=file://${APP_DIR}/src/ckanext-schemingdcat/ckanext/schemingdcat/public/static/licenses.json CKAN___BEAKER__SESSION__SECRET=CHANGE_ME # See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME @@ -125,6 +133,10 @@ CKAN__CORS__ORIGIN_WHITELIST="" # Enables or disable collaborators in individual datasets (https://docs.ckan.org/en/2.9/maintaining/authorization.html#dataset-collaborators) CKAN__AUTH__ALLOW_DATASET_COLLABORATORS=False CKAN__AUTH__ALLOW_ADMIN_COLLABORATORS=False +# Default number of facets shown in search results. Default 10. +SEARCH__FACETS__DEFAULT=4 +# Enable or disable the DataStore SQL search backend. Default is False. +CKAN__DATASTORE__SQLSEARCH__ENABLED=True ## Resource Proxy settings ### Preview size limit, default: 1MB @@ -133,7 +145,7 @@ CKAN__RESOURCE_PROXY__MAX_FILE_SIZE=50048576 CKAN__RESOURCE_PROXY__CHUNK_SIZE=4096 ## Default timeout for fetching proxied items CKAN__RESOURCE_PROXY__TIMEOUT=10 -CKAN__VIEWS__DEFAULT_VIEWS="image_view webpage_view text_view recline_view wmts_view geojson_view geo_view shp_view pdf_view" +CKAN__VIEWS__DEFAULT_VIEWS="image_view video_view audio_view webpage_view text_view datatables_view wmts_view geojson_view geo_view shp_view pdf_view" # Localization CKAN__LOCALE_DEFAULT="en" @@ -141,7 +153,7 @@ CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru CKAN__LOCALES_OFFERED="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" # Extensions -CKAN__PLUGINS="stats image_view text_view recline_view webpage_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" +CKAN__PLUGINS="stats image_view video_view audio_view webpage_view text_view datatables_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars" # ckanext-harvest CKAN__HARVEST__MQ__TYPE=redis @@ -152,7 +164,6 @@ CKAN__HARVEST__MQ__REDIS_DB=${REDIS_CKAN_DATABASE} CKAN__HARVEST__LOG_TIMEFRAME=40 # ckanext-xloader -CKANEXT__XLOADER__API_TOKEN=api_token CKANEXT__XLOADER__JOBS__DB_URI=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB} # ckanext-dcat @@ -179,12 +190,12 @@ CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8 ## CSW Endpoint for spatial metadata CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI=${PYCSW_URL} ## Scheming: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcatap_eu/geodcatap_eu_dataset.yaml" -CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap_eu/geodcatap_eu_group.json" -CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap_eu/geodcatap_eu_org.json" -CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/default_presets.json ckanext.fluent:presets.json ckanext.iepnb:schemas/presets.json" +CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_full.yaml" +CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_group.json" +CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_org.json" +CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/default_presets.json ckanext.fluent:presets.json" ## Facets: setup_scheming.sh -CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_eu dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" +CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_eu language dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to" CKANEXT__SCHEMINGDCAT_ORGANIZATION_CUSTOM_FACETS=True CKANEXT__SCHEMINGDCAT_GROUP_CUSTOM_FACETS=True CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_ICON="theme" diff --git a/samples/ckan.ini.example b/samples/ckan.ini.example index 8b7ceb97..6529b931 100644 --- a/samples/ckan.ini.example +++ b/samples/ckan.ini.example @@ -116,7 +116,7 @@ ckan.site_id = default # Add ``datapusher`` to enable DataPusher # Add ``resource_proxy`` to enable resorce proxying and get around the # same origin policy -ckan.plugins = stats text_view image_view recline_view resourcedictionary datastore datapusher harvest ckan_harvester spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface scheming_datasets scheming_groups scheming_organizations pdf_view pages +ckan.plugins = stats text_view image_view recline_view resourcedictionary datastore datapusher harvest ckan_harvester spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations pdf_view pages ## ckanext-scheming diff --git a/solr/Dockerfile.spatial b/solr/Dockerfile.spatial index cb63e1ce..f74d6adf 100644 --- a/solr/Dockerfile.spatial +++ b/solr/Dockerfile.spatial @@ -59,6 +59,7 @@ ENV SOLR_BBOX_FIELDS ' \ \ \ + \ \ \ ' \ @@ -67,7 +68,6 @@ ENV SOLR_BBOX_FIELDS '/a $SOLR_BBOX_FIELDS" $SOLR_SCHEMA_FILE - RUN chmod 644 $SOLR_SCHEMA_FILE USER solr diff --git a/src/README.md b/src/README.md index 9979d741..c6e73072 100644 --- a/src/README.md +++ b/src/README.md @@ -18,9 +18,9 @@ fi # Define repositories and tags declare -A repos=( - ["ckan/ckanext-xloader"]="1.0.1" + ["mjanez/ckanext-xloader"]="1.0.1-stable" ["ckan/ckanext-harvest"]="v1.5.6" - ["ckan/ckanext-geoview"]="v0.1.0" + ["ckan/ckanext-geoview"]="v0.2.0" ["ckan/ckanext-spatial"]="v2.1.1" ["mjanez/ckanext-dcat"]="v1.8.0" ["ckan/ckanext-scheming"]="release-3.0.0"