Add PAVICS homepage notebooks to Jenkins testsuite and new Jupyter env build

Jenkinsfile

@@ -10,7 +10,7 @@ pipeline {
     // https://jenkins.io/doc/book/pipeline/syntax/
     agent {
         docker {
-            image "pavics/workflow-tests:210527.1-update20210618"
+            image "pavics/workflow-tests:210527.1-update20210705"
             label 'linux && docker'
         }
     }
@@ -50,6 +50,12 @@ Note: This test suite might require manual clean-up on failure (if critical erro
                description: 'FINCH_REPO branch to test against.', trim: true)
         string(name: 'FINCH_REPO', defaultValue: 'bird-house/finch',
                description: 'https://github.com/bird-house/finch repo or fork to test against.', trim: true)
+        booleanParam(name: 'TEST_PAVICS_LANDING_REPO', defaultValue: true,
+                     description: 'Check the box to test pavics-landing repo.')
+        string(name: 'PAVICS_LANDING_BRANCH', defaultValue: 'master',
+               description: 'PAVICS_LANDING_REPO branch to test against.', trim: true)
+        string(name: 'PAVICS_LANDING_REPO', defaultValue: 'Ouranosinc/PAVICS-landing',
+               description: 'https://github.com/Ouranosinc/PAVICS-landing repo or fork to test against.', trim: true)
         booleanParam(name: 'TEST_RAVEN_REPO', defaultValue: false,
                      description: 'Check the box to test raven repo.')
         string(name: 'RAVEN_BRANCH', defaultValue: 'master',
@@ -99,7 +105,7 @@ Note this is another run, will double the time and no guaranty to have same erro
 
     post {
         always {
-            archiveArtifacts(artifacts: 'notebooks/*.ipynb, pavics-sdi-*/docs/source/notebooks/*.ipynb, finch-*/docs/source/notebooks/*.ipynb, raven-*/docs/source/notebooks/*.ipynb, esgf-compute-api-*/examples/*.ipynb, buildout/*.output.ipynb, buildout/env-dump/',
+            archiveArtifacts(artifacts: 'notebooks/*.ipynb, pavics-sdi-*/docs/source/notebooks/*.ipynb, finch-*/docs/source/notebooks/*.ipynb, raven-*/docs/source/notebooks/*.ipynb, esgf-compute-api-*/examples/*.ipynb, PAVICS-landing-*/content/notebooks/climate_indicators/*.ipynb, buildout/*.output.ipynb, buildout/env-dump/',
                              fingerprint: true)
         }
 	unsuccessful {  // Run if the current builds status is "Aborted", "Failure" or "Unstable"

binder/Dockerfile

@@ -1,4 +1,4 @@
-FROM pavics/workflow-tests:210527.1-update20210618
+FROM pavics/workflow-tests:210527.1-update20210705
 
 USER root
 
@@ -10,7 +10,7 @@ COPY binder/reorg-notebooks /notebooks
 
 WORKDIR /notebooks
 
-RUN ./downloadrepos; ./reorg-notebooks; rm -rfv downloadrepos default_build_params reorg-notebooks; chown -R jenkins:jenkins .
+RUN ./downloadrepos; DEPLOY_PAVICS_LANDING_NB=1 ./reorg-notebooks; rm -rfv downloadrepos default_build_params reorg-notebooks; chown -R jenkins:jenkins .
 
 RUN pip install https://github.com/Ouranosinc/xclim/archive/master.tar.gz
 

binder/reorg-notebooks

@@ -8,6 +8,15 @@
 mv -v esgf-compute-api-$ESGF_COMPUTE_API_BRANCH/examples ./esgf-compute-api-examples-$ESGF_COMPUTE_API_BRANCH
 rm -rfv esgf-compute-api-$ESGF_COMPUTE_API_BRANCH
 
+if [ -n "$DEPLOY_PAVICS_LANDING_NB" ]; then
+    # Pavics landing notebooks already deployed to Jupyter env via
+    # https://github.com/bird-house/birdhouse-deploy-ouranos/blob/5dd7608e0639feec4b70cdee6325fad137bc5993/scheduler-jobs/deploy_pavics_landing_notebooks.yml#L16-L17.
+    # This switch is meant for Binder only.
+    mv -v PAVICS-landing-$PAVICS_LANDING_BRANCH/content/notebooks/climate_indicators ./pavics-homepage
+    rm -v ./pavics-homepage/*.html
+fi
+rm -rfv PAVICS-landing-$PAVICS_LANDING_BRANCH
+
 mv -v pavics-sdi-$PAVICS_SDI_BRANCH/docs/source/notebooks/*.ipynb ./
 rm -rfv pavics-sdi-$PAVICS_SDI_BRANCH
 

default_build_params

@@ -42,6 +42,27 @@ else
     echo "FINCH_REPO has been set to '$FINCH_REPO'"
 fi
 
+if [ -z "$TEST_PAVICS_LANDING_REPO" ]; then
+    TEST_PAVICS_LANDING_REPO=true
+    echo "TEST_PAVICS_LANDING_REPO not set, default to '$TEST_PAVICS_LANDING_REPO'"
+else
+    echo "TEST_PAVICS_LANDING_REPO has been set to '$TEST_PAVICS_LANDING_REPO'"
+fi
+
+if [ -z "$PAVICS_LANDING_BRANCH" ]; then
+    PAVICS_LANDING_BRANCH=master
+    echo "PAVICS_LANDING_BRANCH not set, default to '$PAVICS_LANDING_BRANCH'"
+else
+    echo "PAVICS_LANDING_BRANCH has been set to '$PAVICS_LANDING_BRANCH'"
+fi
+
+if [ -z "$PAVICS_LANDING_REPO" ]; then
+    PAVICS_LANDING_REPO='Ouranosinc/PAVICS-landing'
+    echo "PAVICS_LANDING_REPO not set, default to '$PAVICS_LANDING_REPO'"
+else
+    echo "PAVICS_LANDING_REPO has been set to '$PAVICS_LANDING_REPO'"
+fi
+
 if [ -z "$TEST_RAVEN_REPO" ]; then
     TEST_RAVEN_REPO=false
     echo "TEST_RAVEN_REPO not set, default to '$TEST_RAVEN_REPO'"

docker/Dockerfile

@@ -4,7 +4,7 @@ RUN conda update conda
 
 # to checkout other notebooks and to run pip install
 RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y git mercurial gcc fonts-humor-sans && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y git mercurial gcc unzip fonts-humor-sans && \
     apt-get clean
 
 COPY environment.yml /environment.yml
@@ -60,6 +60,15 @@ ADD https://raw.githubusercontent.com/jupyter/docker-stacks/master/base-notebook
 RUN chmod a+rx /usr/local/bin/start.sh /usr/local/bin/start-singleuser.sh /usr/local/bin/start-notebook.sh /usr/local/bin/fix-permissions; \
     chmod a+r /etc/jupyter/jupyter_notebook_config.py
 
+# For Pavics-landing notebooks to re-create Jupyter env layout:
+# /notebook_dir/writable-workspace, /notebook_dir/pavics-homepage.
+#
+# Path to the /notebook_dir/pavics-homepage/tutorial_data/*.geojson files are
+# hardcoded so users can copy the nb to writable-workspace/ dir and still be able
+# to run them seemlessly from the Jupyter env (without having to also copy
+# those *.geojson files with the notebooks).
+RUN mkdir /notebook_dir && chown jenkins /notebook_dir
+
 # problem running start-notebook.sh when being root
 # the jupyter/base-notebook image also do not default to root user so we do the same here
 USER jenkins

docker/Dockerfile.testing

@@ -1,6 +1,6 @@
 # For testing quickly without having to do a full rebuild.
 
-FROM pavics/workflow-tests:210527.1-update20210615
+FROM pavics/workflow-tests:210527.1-update20210618
 
 USER root
 
@@ -11,8 +11,11 @@ USER root
 #    && pip uninstall -y ravenpy \
 #    && conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy ravenpy aiohttp
 
-RUN umask 0000 \
-    && conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy intake-xarray intake-geopandas intake-thredds
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y unzip && \
+    apt-get clean && \
+    mkdir /notebook_dir && \
+    chown jenkins /notebook_dir
 
 #RUN umask 0000 \
 #    && pip install https://github.com/CSHS-CWRA/RavenPy/archive/refs/heads/master.zip

downloadrepos

@@ -27,6 +27,7 @@ downloadgithubrepos() {
 if [ -z "$1" ]; then
     downloadgithubrepos $PAVICS_SDI_REPO $PAVICS_SDI_BRANCH
     downloadgithubrepos $FINCH_REPO $FINCH_BRANCH
+    downloadgithubrepos $PAVICS_LANDING_REPO $PAVICS_LANDING_BRANCH
     downloadgithubrepos $RAVEN_REPO $RAVEN_BRANCH
     downloadgithubrepos $ESGF_COMPUTE_API_REPO $ESGF_COMPUTE_API_BRANCH
 else

launchcontainer

@@ -1,7 +1,7 @@
 #!/bin/sh -x
 
 if [ -z "$DOCKER_IMAGE" ]; then
-    DOCKER_IMAGE="pavics/workflow-tests:210527.1-update20210618"
+    DOCKER_IMAGE="pavics/workflow-tests:210527.1-update20210705"
 fi
 
 if [ -z "$CONTAINER_NAME" ]; then

launchnotebook

@@ -7,7 +7,7 @@ if [ -z "$PORT" ]; then
 fi
 
 if [ -z "$DOCKER_IMAGE" ]; then
-    DOCKER_IMAGE="pavics/workflow-tests:210527.1-update20210618"
+    DOCKER_IMAGE="pavics/workflow-tests:210527.1-update20210705"
 fi
 
 if [ -z "$CONTAINER_NAME" ]; then

runtest

@@ -16,6 +16,9 @@ fi
 
 if [ ! -z "$PAVICS_HOST" ]; then
     echo "Will run notebooks against $PAVICS_HOST"
+    # Any lines marked with TEST_USE_PROD_DATA will replace pavics to boreas to
+    # use data from prod and avoid having to replicate data to test servers.
+    sed -i "/TEST_USE_PROD_DATA/s/pavics.ouranos.ca/boreas.ouranos.ca/g" $NOTEBOOKS
     # .ncml links will always comes from the production server, not the server
     # under test since we do not perform regex replace for .ncml links.
     sed -i "/\.ncml/!s/pavics.ouranos.ca/$PAVICS_HOST/g" $NOTEBOOKS

testall

@@ -17,6 +17,8 @@ PAVICS_SDI_BRANCH="`echo "$PAVICS_SDI_BRANCH" | sed "s@/@-@g"`"
 PAVICS_SDI_REPO_NAME="`echo "$PAVICS_SDI_REPO" | sed "s@^.*/@@g"`"
 FINCH_BRANCH="`echo "$FINCH_BRANCH" | sed "s@/@-@g"`"
 FINCH_REPO_NAME="`echo "$FINCH_REPO" | sed "s@^.*/@@g"`"
+PAVICS_LANDING_BRANCH="`echo "$PAVICS_LANDING_BRANCH" | sed "s@/@-@g"`"
+PAVICS_LANDING_REPO_NAME="`echo "$PAVICS_LANDING_REPO" | sed "s@^.*/@@g"`"
 RAVEN_BRANCH="`echo "$RAVEN_BRANCH" | sed "s@/@-@g"`"
 RAVEN_REPO_NAME="`echo "$RAVEN_REPO" | sed "s@^.*/@@g"`"
 ESGF_COMPUTE_API_BRANCH="`echo "$ESGF_COMPUTE_API_BRANCH" | sed "s@/@-@g"`"
@@ -25,6 +27,7 @@ ESGF_COMPUTE_API_REPO_NAME="`echo "$ESGF_COMPUTE_API_REPO" | sed "s@^.*/@@g"`"
 # branches that have allowed characters such as '+' other than alphanum, '-' and '_' are converted to '-' in archives
 PAVICS_SDI_DIR=`echo "${PAVICS_SDI_REPO_NAME}-${PAVICS_SDI_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
 FINCH_DIR=`echo "${FINCH_REPO_NAME}-${FINCH_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
+PAVICS_LANDING_DIR=`echo "${PAVICS_LANDING_REPO_NAME}-${PAVICS_LANDING_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
 RAVEN_DIR=`echo "${RAVEN_REPO_NAME}-${RAVEN_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
 ESGF_COMPUTE_API_DIR=`echo "${ESGF_COMPUTE_API_REPO_NAME}-${ESGF_COMPUTE_API_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"`
 
@@ -49,6 +52,7 @@ rm -v $ESGF_COMPUTE_API_REPO_NAME-$ESGF_COMPUTE_API_BRANCH/tox.ini
 TEST_MAGPIE_AUTH="`echo "$TEST_MAGPIE_AUTH" | tr '[:upper:]' '[:lower:]'`"
 TEST_PAVICS_SDI_REPO="`echo "$TEST_PAVICS_SDI_REPO" | tr '[:upper:]' '[:lower:]'`"
 TEST_FINCH_REPO="`echo "$TEST_FINCH_REPO" | tr '[:upper:]' '[:lower:]'`"
+TEST_PAVICS_LANDING_REPO="`echo "$TEST_PAVICS_LANDING_REPO" | tr '[:upper:]' '[:lower:]'`"
 TEST_RAVEN_REPO="`echo "$TEST_RAVEN_REPO" | tr '[:upper:]' '[:lower:]'`"
 TEST_ESGF_COMPUTE_API_REPO="`echo "$TEST_ESGF_COMPUTE_API_REPO" | tr '[:upper:]' '[:lower:]'`"
 
@@ -62,6 +66,21 @@ fi
 if [ x"$TEST_FINCH_REPO" = xtrue ]; then
     NOTEBOOKS_TO_TEST="$NOTEBOOKS_TO_TEST ${FINCH_DIR}/docs/source/notebooks/*.ipynb"
 fi
+
+if [ x"$TEST_PAVICS_LANDING_REPO" = xtrue ]; then
+    NOTEBOOKS_TO_TEST="$NOTEBOOKS_TO_TEST ${PAVICS_LANDING_DIR}/content/notebooks/climate_indicators/*.ipynb"
+
+    SETUP_SCRIPT="${PAVICS_LANDING_DIR}/content/notebooks/climate_indicators/setup_dirlayout.sh"
+    if [ -x "$SETUP_SCRIPT" ]; then
+
+        # Delegate extra files layout setup to source repo so this setup_dirlayout.sh
+        # becomes a stable interface between this repo and the source repo.
+        # Any future layout changes will be seemless for this repo.
+
+        $SETUP_SCRIPT
+    fi
+fi
+
 if [ x"$TEST_RAVEN_REPO" = xtrue ]; then
     NOTEBOOKS_TO_TEST="$NOTEBOOKS_TO_TEST ${RAVEN_DIR}/docs/source/notebooks/*.ipynb"
 fi