New Jupyter env, Raven and notebook autodeploy fix (#332)

## Overview

- Jupyter env: new version with latest Xclim and RavenPy

See Ouranosinc/PAVICS-e2e-workflow-tests#115 for
more details.

- Raven WPS: new version to match with new RavenPy inside the Jupyter
env

See Ouranosinc/raven@v0.14.2...v0.18.1 for
more details.

- Notebook autodeploy: unable to read the `env.local`

When `env.local` is a symlink we need to volume-mount the destination of
the symlink so it resolves inside the notebook autodeploy container.

This will allow notebook autodeploy config variable to be set in
`env.local`.

Also had someone changed the value of `JUPYTERHUB_USER_DATA_DIR` in
`env.local`, it would not have worked without this fix.

  This is a non breaking change.

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.26.2
+current_version = 1.26.3
 commit = True
 tag = False
 tag_name = {new_version}
@@ -30,11 +30,11 @@ search = {current_version}
 replace = {new_version}
 
 [bumpversion:file:RELEASE.txt]
-search = {current_version} 2023-05-25T03:11:29Z
+search = {current_version} 2023-06-01T20:36:33Z
 replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ}
 
 [bumpversion:part:releaseTime]
-values = 2023-05-25T03:11:29Z
+values = 2023-06-01T20:36:33Z
 
 [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template]
 search = 'version': '{current_version}'

CHANGES.md

@@ -17,6 +17,34 @@
 
 [//]: # (list changes here, using '-' for each new entry, remove this when items are added)
 
+[1.26.3](https://github.com/bird-house/birdhouse-deploy/tree/1.26.3) (2023-06-01)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+- Jupyter env: new version with latest Xclim and RavenPy
+
+  See https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/pull/115 for more
+  details.
+
+- Raven WPS: new version to match with new RavenPy inside the Jupyter env
+
+  See https://github.com/Ouranosinc/raven/compare/v0.14.2...v0.18.1 for more
+  details.
+
+## Fixes
+- Notebook autodeploy: unable to read the `env.local`
+
+  When `env.local` is a symlink we need to volume-mount the destination of the
+  symlink so it resolves inside the notebook autodeploy container.
+
+  This will allow notebook autodeploy config variable to be set in `env.local`.
+
+  Also had someone changed the value of `JUPYTERHUB_USER_DATA_DIR` in `env.local`,
+  it would not have worked without this fix.
+
+  This is a non-breaking fix.
+
+
 [1.26.2](https://github.com/bird-house/birdhouse-deploy/tree/1.26.2) (2023-05-25)
 ------------------------------------------------------------------------------------------------------------------
 

Makefile

@@ -1,7 +1,7 @@
 # Generic variables
 override SHELL       := bash
 override APP_NAME    := birdhouse-deploy
-override APP_VERSION := 1.26.2
+override APP_VERSION := 1.26.3
 
 # utility to remove comments after value of an option variable
 override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g")

README.rst

@@ -14,13 +14,13 @@ for a full-fledged production platform.
     * - releases
       - | |latest-version| |commits-since|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.2.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.26.3.svg
     :alt: Commits since latest release
-    :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.2...master
+    :target: https://github.com/bird-house/birdhouse-deploy/compare/1.26.3...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.2-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-1.26.3-blue.svg?style=flat
     :alt: Latest Tag
-    :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.2
+    :target: https://github.com/bird-house/birdhouse-deploy/tree/1.26.3
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
     :alt: ReadTheDocs Build Status (latest version)

RELEASE.txt

@@ -1 +1 @@
-1.26.2 2023-05-25T03:11:29Z
+1.26.3 2023-06-01T20:36:33Z

birdhouse/components/scheduler/config.yml.template

@@ -23,7 +23,7 @@
     --volume /tmp/notebookdeploy:/tmp/notebookdeploy:rw
     --env COMPOSE_DIR=${COMPOSE_DIR}
     --env TMP_BASE_DIR=/tmp/notebookdeploy
-    --env JUPYTERHUB_USER_DATA_DIR=${JUPYTERHUB_USER_DATA_DIR}
+    --env JUPYTERHUB_USER_DATA_DIR=${JUPYTERHUB_USER_DATA_DIR}${AUTODEPLOY_NOTEBOOK_EXTRA_DOCKER_ARGS}
   image: 'docker:19.03.6-git'
 
 - name: autodeploy
@@ -40,7 +40,7 @@
     --env COMPOSE_DIR=${COMPOSE_DIR}
     --env AUTODEPLOY_DEPLOY_KEY_ROOT_DIR=${AUTODEPLOY_DEPLOY_KEY_ROOT_DIR}
     --env JUPYTERHUB_USER_DATA_DIR=${JUPYTERHUB_USER_DATA_DIR}
-    --env AUTODEPLOY_SILENT=true
+    --env AUTODEPLOY_SILENT=true${AUTODEPLOY_PLATFORM_EXTRA_DOCKER_ARGS}
   image: 'pavics/docker-compose-git:docker-18.09.7-compose-1.25.1'
 
 ${AUTODEPLOY_EXTRA_SCHEDULER_JOBS}

birdhouse/components/scheduler/default.env

@@ -17,11 +17,23 @@ export AUTODEPLOY_PLATFORM_FREQUENCY="7 5 * * *"
 # Hourly
 export AUTODEPLOY_NOTEBOOK_FREQUENCY="@hourly"
 
+if [ -L "$BIRDHOUSE_LOCAL_ENV" ]; then
+    # Add volume-mount if env.local is a symlink and not a real file,
+    # so the symlink can resolve.
+    # This would likely be on a production deployment, not in a Vagrant test VM.
+    if [ -z "$(echo "$AUTODEPLOY_NOTEBOOK_EXTRA_DOCKER_ARGS" | grep "volume ${BIRDHOUSE_LOCAL_ENV_REAL_DIR}:")" ]; then
+        export AUTODEPLOY_NOTEBOOK_EXTRA_DOCKER_ARGS="$AUTODEPLOY_NOTEBOOK_EXTRA_DOCKER_ARGS
+    --volume ${BIRDHOUSE_LOCAL_ENV_REAL_DIR}:${BIRDHOUSE_LOCAL_ENV_REAL_DIR}:ro"
+    fi
+fi
+
 # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here
 OPTIONAL_VARS="
   $OPTIONAL_VARS
   \$AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES
   \$AUTODEPLOY_PLATFORM_FREQUENCY
   \$AUTODEPLOY_NOTEBOOK_FREQUENCY
   \$AUTODEPLOY_EXTRA_SCHEDULER_JOBS
+  \$AUTODEPLOY_NOTEBOOK_EXTRA_DOCKER_ARGS
+  \$AUTODEPLOY_PLATFORM_EXTRA_DOCKER_ARGS
 "

birdhouse/config/canarie-api/docker_configuration.py.template

@@ -109,8 +109,8 @@ SERVICES = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '1.26.2',
-            'releaseTime': '2023-05-25T03:11:29Z',
+            'version': '1.26.3',
+            'releaseTime': '2023-06-01T20:36:33Z',
             'institution': 'Ouranos',
             'researchSubject': 'Climatology',
             'supportEmail': '${SUPPORT_EMAIL}',
@@ -142,8 +142,8 @@ PLATFORMS = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '1.26.2',
-            'releaseTime': '2023-05-25T03:11:29Z',
+            'version': '1.26.3',
+            'releaseTime': '2023-06-01T20:36:33Z',
             'institution': 'Ouranos',
             'researchSubject': 'Climatology',
             'supportEmail': '${SUPPORT_EMAIL}',

birdhouse/config/jupyterhub/default.env

@@ -8,7 +8,7 @@ export JUPYTERHUB_DOCKER=pavics/jupyterhub
 export JUPYTERHUB_VERSION=1.4.0-20210506
 
 # Jupyter single-user server images, can be overriden in env.local to have a space separated list of multiple images
-export DOCKER_NOTEBOOK_IMAGES="pavics/workflow-tests:221130-update230403"
+export DOCKER_NOTEBOOK_IMAGES="pavics/workflow-tests:230530-1"
 
 # Name of the image displayed on the JupyterHub image selection page
 # Can be overriden in env.local to have a space separated list of multiple images, the name order must correspond

birdhouse/config/raven/default.env

@@ -7,7 +7,7 @@
 # Ex: RAVEN_GEO_URL="https://${PAVICS_FQDN}/geoserver/"
 export RAVEN_GEO_URL="https://pavics.ouranos.ca/geoserver/"
 
-export RAVEN_VERSION=0.14.2
+export RAVEN_VERSION="0.18.1"
 
 # This tells the postgres service to create a database on startup
 export POSTGRES_DATABASES_TO_CREATE="

birdhouse/deployment/trigger-deploy-notebook

@@ -29,8 +29,6 @@ START_TIME="`date -Isecond`"
 echo "==========
 notebookdeploy START_TIME=$START_TIME"
 
-set -x
-
 # running script manually (not with cron) source env.local file.
 if [ -z "$COMPOSE_DIR" ]; then
     COMPOSE_DIR="$(dirname -- "$(dirname -- "$(realpath "$0")")")"
@@ -43,6 +41,8 @@ if [ -f "$COMPOSE_DIR/read-configs.include.sh" ]; then
     read_configs
 fi
 
+set -x
+
 NOTEBOOK_DIR_MNT="/notebook_dir"
 TUTORIAL_NOTEBOOKS_DIR="tutorial-notebooks"
 if [ -z "$WORKFLOW_TESTS_BRANCH" ]; then

birdhouse/read-configs.include.sh

@@ -54,6 +54,20 @@ discover_compose_dir() {
 }
 
 
+discover_env_local() {
+    if [ -z "$BIRDHOUSE_LOCAL_ENV" ]; then
+        BIRDHOUSE_LOCAL_ENV="$COMPOSE_DIR/env.local"
+    fi
+
+    # env.local can be a symlink to the private config repo where the real
+    # env.local file is source controlled.
+    # Docker volume-mount will need the real dir of the file for symlink to
+    # resolve inside the container.
+    BIRDHOUSE_LOCAL_ENV_REAL_PATH="$(realpath "$BIRDHOUSE_LOCAL_ENV")"
+    BIRDHOUSE_LOCAL_ENV_REAL_DIR="$(dirname "$BIRDHOUSE_LOCAL_ENV_REAL_PATH")"
+}
+
+
 read_default_env() {
     if [ -e "$COMPOSE_DIR/default.env" ]; then
         # Ensure DELAYED_EVAL is properly initialized before being appended to.
@@ -69,7 +83,7 @@ read_default_env() {
 read_env_local() {
     # we don't use usual .env filename, because docker-compose uses it
 
-    echo "Using local environment file at: ${BIRDHOUSE_LOCAL_ENV:="$COMPOSE_DIR/env.local"}"
+    echo "Using local environment file at: ${BIRDHOUSE_LOCAL_ENV}"
 
     if [ -e "$BIRDHOUSE_LOCAL_ENV" ]; then
         saved_shell_options="$(set +o)"
@@ -228,10 +242,11 @@ create_compose_conf_list() {
 # process_delayed_eval() at the appropriate moment.
 read_configs() {
     discover_compose_dir
+    discover_env_local
     read_default_env
-    read_env_local  # for EXTRA_CONF_DIRS and DEFAULT_CONF_DIRS
+    read_env_local  # for EXTRA_CONF_DIRS and DEFAULT_CONF_DIRS, need discover_env_local
     read_components_default_env  # uses EXTRA_CONF_DIRS and DEFAULT_CONF_DIRS, sets ALL_CONF_DIRS
-    read_env_local  # again to override components default.env
+    read_env_local  # again to override components default.env, need discover_env_local
     process_delayed_eval
 }
 
@@ -241,7 +256,8 @@ read_configs() {
 # read_configs() to be safe.
 read_basic_configs_only() {
     discover_compose_dir
+    discover_env_local
     read_default_env
-    read_env_local
+    read_env_local  # need discover_env_local
     process_delayed_eval
 }

docs/source/conf.py

@@ -69,9 +69,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.26.2'
+version = '1.26.3'
 # The full version, including alpha/beta/rc tags.
-release = '1.26.2'
+release = '1.26.3'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.