diff --git a/Jenkinsfile b/Jenkinsfile index e1f74211..232252d5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ pipeline { // https://jenkins.io/doc/book/pipeline/syntax/ agent { docker { - image "pavics/workflow-tests:210527.1-update20210618" + image "pavics/workflow-tests:210527.1-update20210705" label 'linux && docker' } } @@ -50,6 +50,12 @@ Note: This test suite might require manual clean-up on failure (if critical erro description: 'FINCH_REPO branch to test against.', trim: true) string(name: 'FINCH_REPO', defaultValue: 'bird-house/finch', description: 'https://github.com/bird-house/finch repo or fork to test against.', trim: true) + booleanParam(name: 'TEST_PAVICS_LANDING_REPO', defaultValue: true, + description: 'Check the box to test pavics-landing repo.') + string(name: 'PAVICS_LANDING_BRANCH', defaultValue: 'master', + description: 'PAVICS_LANDING_REPO branch to test against.', trim: true) + string(name: 'PAVICS_LANDING_REPO', defaultValue: 'Ouranosinc/PAVICS-landing', + description: 'https://github.com/Ouranosinc/PAVICS-landing repo or fork to test against.', trim: true) booleanParam(name: 'TEST_RAVEN_REPO', defaultValue: false, description: 'Check the box to test raven repo.') string(name: 'RAVEN_BRANCH', defaultValue: 'master', @@ -99,7 +105,7 @@ Note this is another run, will double the time and no guaranty to have same erro post { always { - archiveArtifacts(artifacts: 'notebooks/*.ipynb, pavics-sdi-*/docs/source/notebooks/*.ipynb, finch-*/docs/source/notebooks/*.ipynb, raven-*/docs/source/notebooks/*.ipynb, esgf-compute-api-*/examples/*.ipynb, buildout/*.output.ipynb, buildout/env-dump/', + archiveArtifacts(artifacts: 'notebooks/*.ipynb, pavics-sdi-*/docs/source/notebooks/*.ipynb, finch-*/docs/source/notebooks/*.ipynb, raven-*/docs/source/notebooks/*.ipynb, esgf-compute-api-*/examples/*.ipynb, PAVICS-landing-*/content/notebooks/climate_indicators/*.ipynb, buildout/*.output.ipynb, buildout/env-dump/', fingerprint: true) } unsuccessful { // Run if the current builds status is "Aborted", "Failure" or "Unstable" diff --git a/binder/Dockerfile b/binder/Dockerfile index 4545ab24..dde3928d 100644 --- a/binder/Dockerfile +++ b/binder/Dockerfile @@ -1,4 +1,4 @@ -FROM pavics/workflow-tests:210527.1-update20210618 +FROM pavics/workflow-tests:210527.1-update20210705 USER root @@ -10,7 +10,7 @@ COPY binder/reorg-notebooks /notebooks WORKDIR /notebooks -RUN ./downloadrepos; ./reorg-notebooks; rm -rfv downloadrepos default_build_params reorg-notebooks; chown -R jenkins:jenkins . +RUN ./downloadrepos; DEPLOY_PAVICS_LANDING_NB=1 ./reorg-notebooks; rm -rfv downloadrepos default_build_params reorg-notebooks; chown -R jenkins:jenkins . RUN pip install https://github.com/Ouranosinc/xclim/archive/master.tar.gz diff --git a/binder/reorg-notebooks b/binder/reorg-notebooks index 7e71e1c0..8ec894ea 100755 --- a/binder/reorg-notebooks +++ b/binder/reorg-notebooks @@ -8,6 +8,15 @@ mv -v esgf-compute-api-$ESGF_COMPUTE_API_BRANCH/examples ./esgf-compute-api-examples-$ESGF_COMPUTE_API_BRANCH rm -rfv esgf-compute-api-$ESGF_COMPUTE_API_BRANCH +if [ -n "$DEPLOY_PAVICS_LANDING_NB" ]; then + # Pavics landing notebooks already deployed to Jupyter env via + # https://github.com/bird-house/birdhouse-deploy-ouranos/blob/5dd7608e0639feec4b70cdee6325fad137bc5993/scheduler-jobs/deploy_pavics_landing_notebooks.yml#L16-L17. + # This switch is meant for Binder only. + mv -v PAVICS-landing-$PAVICS_LANDING_BRANCH/content/notebooks/climate_indicators ./pavics-homepage + rm -v ./pavics-homepage/*.html +fi +rm -rfv PAVICS-landing-$PAVICS_LANDING_BRANCH + mv -v pavics-sdi-$PAVICS_SDI_BRANCH/docs/source/notebooks/*.ipynb ./ rm -rfv pavics-sdi-$PAVICS_SDI_BRANCH diff --git a/default_build_params b/default_build_params index 060a5c3c..afc82342 100644 --- a/default_build_params +++ b/default_build_params @@ -42,6 +42,27 @@ else echo "FINCH_REPO has been set to '$FINCH_REPO'" fi +if [ -z "$TEST_PAVICS_LANDING_REPO" ]; then + TEST_PAVICS_LANDING_REPO=true + echo "TEST_PAVICS_LANDING_REPO not set, default to '$TEST_PAVICS_LANDING_REPO'" +else + echo "TEST_PAVICS_LANDING_REPO has been set to '$TEST_PAVICS_LANDING_REPO'" +fi + +if [ -z "$PAVICS_LANDING_BRANCH" ]; then + PAVICS_LANDING_BRANCH=master + echo "PAVICS_LANDING_BRANCH not set, default to '$PAVICS_LANDING_BRANCH'" +else + echo "PAVICS_LANDING_BRANCH has been set to '$PAVICS_LANDING_BRANCH'" +fi + +if [ -z "$PAVICS_LANDING_REPO" ]; then + PAVICS_LANDING_REPO='Ouranosinc/PAVICS-landing' + echo "PAVICS_LANDING_REPO not set, default to '$PAVICS_LANDING_REPO'" +else + echo "PAVICS_LANDING_REPO has been set to '$PAVICS_LANDING_REPO'" +fi + if [ -z "$TEST_RAVEN_REPO" ]; then TEST_RAVEN_REPO=false echo "TEST_RAVEN_REPO not set, default to '$TEST_RAVEN_REPO'" diff --git a/docker/Dockerfile b/docker/Dockerfile index 713aa9c4..700e743b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,7 +4,7 @@ RUN conda update conda # to checkout other notebooks and to run pip install RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y git mercurial gcc fonts-humor-sans && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y git mercurial gcc unzip fonts-humor-sans && \ apt-get clean COPY environment.yml /environment.yml @@ -60,6 +60,15 @@ ADD https://raw.githubusercontent.com/jupyter/docker-stacks/master/base-notebook RUN chmod a+rx /usr/local/bin/start.sh /usr/local/bin/start-singleuser.sh /usr/local/bin/start-notebook.sh /usr/local/bin/fix-permissions; \ chmod a+r /etc/jupyter/jupyter_notebook_config.py +# For Pavics-landing notebooks to re-create Jupyter env layout: +# /notebook_dir/writable-workspace, /notebook_dir/pavics-homepage. +# +# Path to the /notebook_dir/pavics-homepage/tutorial_data/*.geojson files are +# hardcoded so users can copy the nb to writable-workspace/ dir and still be able +# to run them seemlessly from the Jupyter env (without having to also copy +# those *.geojson files with the notebooks). +RUN mkdir /notebook_dir && chown jenkins /notebook_dir + # problem running start-notebook.sh when being root # the jupyter/base-notebook image also do not default to root user so we do the same here USER jenkins diff --git a/docker/Dockerfile.testing b/docker/Dockerfile.testing index 99b42c63..e55de7f2 100644 --- a/docker/Dockerfile.testing +++ b/docker/Dockerfile.testing @@ -1,6 +1,6 @@ # For testing quickly without having to do a full rebuild. -FROM pavics/workflow-tests:210527.1-update20210615 +FROM pavics/workflow-tests:210527.1-update20210618 USER root @@ -11,8 +11,11 @@ USER root # && pip uninstall -y ravenpy \ # && conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy ravenpy aiohttp -RUN umask 0000 \ - && conda install -c conda-forge -c cdat -c bokeh -c plotly -c defaults -n birdy intake-xarray intake-geopandas intake-thredds +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y unzip && \ + apt-get clean && \ + mkdir /notebook_dir && \ + chown jenkins /notebook_dir #RUN umask 0000 \ # && pip install https://github.com/CSHS-CWRA/RavenPy/archive/refs/heads/master.zip diff --git a/downloadrepos b/downloadrepos index d04b1d94..8742a869 100755 --- a/downloadrepos +++ b/downloadrepos @@ -27,6 +27,7 @@ downloadgithubrepos() { if [ -z "$1" ]; then downloadgithubrepos $PAVICS_SDI_REPO $PAVICS_SDI_BRANCH downloadgithubrepos $FINCH_REPO $FINCH_BRANCH + downloadgithubrepos $PAVICS_LANDING_REPO $PAVICS_LANDING_BRANCH downloadgithubrepos $RAVEN_REPO $RAVEN_BRANCH downloadgithubrepos $ESGF_COMPUTE_API_REPO $ESGF_COMPUTE_API_BRANCH else diff --git a/launchcontainer b/launchcontainer index a3cfbde9..3ac5e6ca 100755 --- a/launchcontainer +++ b/launchcontainer @@ -1,7 +1,7 @@ #!/bin/sh -x if [ -z "$DOCKER_IMAGE" ]; then - DOCKER_IMAGE="pavics/workflow-tests:210527.1-update20210618" + DOCKER_IMAGE="pavics/workflow-tests:210527.1-update20210705" fi if [ -z "$CONTAINER_NAME" ]; then diff --git a/launchnotebook b/launchnotebook index f0ea7eeb..4157aee1 100755 --- a/launchnotebook +++ b/launchnotebook @@ -7,7 +7,7 @@ if [ -z "$PORT" ]; then fi if [ -z "$DOCKER_IMAGE" ]; then - DOCKER_IMAGE="pavics/workflow-tests:210527.1-update20210618" + DOCKER_IMAGE="pavics/workflow-tests:210527.1-update20210705" fi if [ -z "$CONTAINER_NAME" ]; then diff --git a/runtest b/runtest index 3d3534c0..433d5d88 100755 --- a/runtest +++ b/runtest @@ -16,6 +16,9 @@ fi if [ ! -z "$PAVICS_HOST" ]; then echo "Will run notebooks against $PAVICS_HOST" + # Any lines marked with TEST_USE_PROD_DATA will replace pavics to boreas to + # use data from prod and avoid having to replicate data to test servers. + sed -i "/TEST_USE_PROD_DATA/s/pavics.ouranos.ca/boreas.ouranos.ca/g" $NOTEBOOKS # .ncml links will always comes from the production server, not the server # under test since we do not perform regex replace for .ncml links. sed -i "/\.ncml/!s/pavics.ouranos.ca/$PAVICS_HOST/g" $NOTEBOOKS diff --git a/testall b/testall index 93990009..29821cc4 100755 --- a/testall +++ b/testall @@ -17,6 +17,8 @@ PAVICS_SDI_BRANCH="`echo "$PAVICS_SDI_BRANCH" | sed "s@/@-@g"`" PAVICS_SDI_REPO_NAME="`echo "$PAVICS_SDI_REPO" | sed "s@^.*/@@g"`" FINCH_BRANCH="`echo "$FINCH_BRANCH" | sed "s@/@-@g"`" FINCH_REPO_NAME="`echo "$FINCH_REPO" | sed "s@^.*/@@g"`" +PAVICS_LANDING_BRANCH="`echo "$PAVICS_LANDING_BRANCH" | sed "s@/@-@g"`" +PAVICS_LANDING_REPO_NAME="`echo "$PAVICS_LANDING_REPO" | sed "s@^.*/@@g"`" RAVEN_BRANCH="`echo "$RAVEN_BRANCH" | sed "s@/@-@g"`" RAVEN_REPO_NAME="`echo "$RAVEN_REPO" | sed "s@^.*/@@g"`" ESGF_COMPUTE_API_BRANCH="`echo "$ESGF_COMPUTE_API_BRANCH" | sed "s@/@-@g"`" @@ -25,6 +27,7 @@ ESGF_COMPUTE_API_REPO_NAME="`echo "$ESGF_COMPUTE_API_REPO" | sed "s@^.*/@@g"`" # branches that have allowed characters such as '+' other than alphanum, '-' and '_' are converted to '-' in archives PAVICS_SDI_DIR=`echo "${PAVICS_SDI_REPO_NAME}-${PAVICS_SDI_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"` FINCH_DIR=`echo "${FINCH_REPO_NAME}-${FINCH_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"` +PAVICS_LANDING_DIR=`echo "${PAVICS_LANDING_REPO_NAME}-${PAVICS_LANDING_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"` RAVEN_DIR=`echo "${RAVEN_REPO_NAME}-${RAVEN_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"` ESGF_COMPUTE_API_DIR=`echo "${ESGF_COMPUTE_API_REPO_NAME}-${ESGF_COMPUTE_API_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"` @@ -49,6 +52,7 @@ rm -v $ESGF_COMPUTE_API_REPO_NAME-$ESGF_COMPUTE_API_BRANCH/tox.ini TEST_MAGPIE_AUTH="`echo "$TEST_MAGPIE_AUTH" | tr '[:upper:]' '[:lower:]'`" TEST_PAVICS_SDI_REPO="`echo "$TEST_PAVICS_SDI_REPO" | tr '[:upper:]' '[:lower:]'`" TEST_FINCH_REPO="`echo "$TEST_FINCH_REPO" | tr '[:upper:]' '[:lower:]'`" +TEST_PAVICS_LANDING_REPO="`echo "$TEST_PAVICS_LANDING_REPO" | tr '[:upper:]' '[:lower:]'`" TEST_RAVEN_REPO="`echo "$TEST_RAVEN_REPO" | tr '[:upper:]' '[:lower:]'`" TEST_ESGF_COMPUTE_API_REPO="`echo "$TEST_ESGF_COMPUTE_API_REPO" | tr '[:upper:]' '[:lower:]'`" @@ -62,6 +66,21 @@ fi if [ x"$TEST_FINCH_REPO" = xtrue ]; then NOTEBOOKS_TO_TEST="$NOTEBOOKS_TO_TEST ${FINCH_DIR}/docs/source/notebooks/*.ipynb" fi + +if [ x"$TEST_PAVICS_LANDING_REPO" = xtrue ]; then + NOTEBOOKS_TO_TEST="$NOTEBOOKS_TO_TEST ${PAVICS_LANDING_DIR}/content/notebooks/climate_indicators/*.ipynb" + + SETUP_SCRIPT="${PAVICS_LANDING_DIR}/content/notebooks/climate_indicators/setup_dirlayout.sh" + if [ -x "$SETUP_SCRIPT" ]; then + + # Delegate extra files layout setup to source repo so this setup_dirlayout.sh + # becomes a stable interface between this repo and the source repo. + # Any future layout changes will be seemless for this repo. + + $SETUP_SCRIPT + fi +fi + if [ x"$TEST_RAVEN_REPO" = xtrue ]; then NOTEBOOKS_TO_TEST="$NOTEBOOKS_TO_TEST ${RAVEN_DIR}/docs/source/notebooks/*.ipynb" fi