From 7f2e02335255b05cc96e81b380c418d6f65b8758 Mon Sep 17 00:00:00 2001 From: ana espinoza Date: Mon, 13 Jan 2025 11:04:08 -0700 Subject: [PATCH] Init ulv25s --- jupyter-images/ulv25s/.condarc | 2 + jupyter-images/ulv25s/Acknowledgements.ipynb | 43 ++++++ jupyter-images/ulv25s/Dockerfile | 42 ++++++ .../ulv25s/additional_kernels.ipynb | 52 +++++++ jupyter-images/ulv25s/build.sh | 44 ++++++ jupyter-images/ulv25s/default_kernel.py | 67 +++++++++ jupyter-images/ulv25s/environment.yml | 25 ++++ jupyter-images/ulv25s/overrides.json | 7 + jupyter-images/ulv25s/secrets.yaml | 112 +++++++++++++++ jupyter-images/ulv25s/update_material.ipynb | 136 ++++++++++++++++++ 10 files changed, 530 insertions(+) create mode 100644 jupyter-images/ulv25s/.condarc create mode 100644 jupyter-images/ulv25s/Acknowledgements.ipynb create mode 100644 jupyter-images/ulv25s/Dockerfile create mode 100644 jupyter-images/ulv25s/additional_kernels.ipynb create mode 100755 jupyter-images/ulv25s/build.sh create mode 100755 jupyter-images/ulv25s/default_kernel.py create mode 100644 jupyter-images/ulv25s/environment.yml create mode 100644 jupyter-images/ulv25s/overrides.json create mode 100644 jupyter-images/ulv25s/secrets.yaml create mode 100644 jupyter-images/ulv25s/update_material.ipynb diff --git a/jupyter-images/ulv25s/.condarc b/jupyter-images/ulv25s/.condarc new file mode 100644 index 00000000..7c536d94 --- /dev/null +++ b/jupyter-images/ulv25s/.condarc @@ -0,0 +1,2 @@ +envs_dirs: + - /home/jovyan/additional-envs diff --git a/jupyter-images/ulv25s/Acknowledgements.ipynb b/jupyter-images/ulv25s/Acknowledgements.ipynb new file mode 100644 index 00000000..fae591fc --- /dev/null +++ b/jupyter-images/ulv25s/Acknowledgements.ipynb @@ -0,0 +1,43 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c86cd54f-b73c-4781-b6eb-89c79d3d3b22", + "metadata": {}, + "source": [ + "## Acknowledgements\n", + "\n", + "Launching this JupyterHub server is the result of a collaboration between several research and academic institutions and their staff. For Jetstream2 and JupyterHub expertise, we thank Andrea Zonca (San Diego Supercomputing Center), Jeremy Fischer, Mike Lowe (Indiana University), the NSF Jetstream2 (`doi:10.1145/3437359.3465565`) team.\n", + "\n", + "This work employs the NSF Jetstream2 Cloud at Indiana University through allocation EES220002 from the Advanced Cyberinfrastructure Coordination Ecosystem: Services & Support (ACCESS) program, which is supported by National Science Foundation grants #2138259, #2138286, #2138307, #2137603, and #2138296.\n", + "\n", + "Unidata is one of the University Corporation for Atmospheric Research (UCAR)'s Community Programs (UCP), and is funded primarily by the National Science Foundation (AGS-2403649).\n", + "\n", + "## To Acknowledge This JupyterHub and the Unidata Science Gateway\n", + "\n", + "If you have benefited from the Unidata Science Gateway, please cite `doi:10.5065/688s-2w73`. Additional citation information can be found in this [Citation File Format file](https://raw.githubusercontent.com/Unidata/science-gateway/master/CITATION.cff).\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/jupyter-images/ulv25s/Dockerfile b/jupyter-images/ulv25s/Dockerfile new file mode 100644 index 00000000..70043656 --- /dev/null +++ b/jupyter-images/ulv25s/Dockerfile @@ -0,0 +1,42 @@ +# Heavily borrowed from docker-stacks/minimal-notebook/ +# https://github.com/jupyter/docker-stacks/blob/main/minimal-notebook/Dockerfile + +ARG BASE_CONTAINER=jupyter/minimal-notebook +FROM $BASE_CONTAINER + +ENV DEFAULT_ENV_NAME=ulv25s EDITOR=nano VISUAL=nano + +LABEL maintainer="Unidata " + +USER root + +RUN apt-get update && \ + apt-get install -y --no-install-recommends vim nano curl zip unzip && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +USER $NB_UID + +ADD environment.yml /tmp + +RUN mamba install --quiet --yes \ + 'conda-forge::nb_conda_kernels' \ + 'conda-forge::jupyterlab-git' \ + 'conda-forge::ipywidgets' && \ + mamba env update --name $DEFAULT_ENV_NAME -f /tmp/environment.yml && \ + pip install --no-cache-dir nbgitpuller && \ + mamba clean --all -f -y && \ + jupyter lab clean -y && \ + npm cache clean --force && \ + rm -rf /home/$NB_USER/.cache/yarn && \ + rm -rf /home/$NB_USER/.node-gyp && \ + fix-permissions $CONDA_DIR && \ + fix-permissions /home/$NB_USER + +COPY additional_kernels.ipynb update_material.ipynb Acknowledgements.ipynb \ + default_kernel.py .condarc / + +ARG JUPYTER_SETTINGS_DIR=/opt/conda/share/jupyter/lab/settings/ +COPY overrides.json $JUPYTER_SETTINGS_DIR + +USER $NB_UID diff --git a/jupyter-images/ulv25s/additional_kernels.ipynb b/jupyter-images/ulv25s/additional_kernels.ipynb new file mode 100644 index 00000000..e59cdd0f --- /dev/null +++ b/jupyter-images/ulv25s/additional_kernels.ipynb @@ -0,0 +1,52 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a9d9cf3f-590d-40ef-8421-a9789a03bb07", + "metadata": {}, + "source": [ + "### Creating Additional Kernels\n", + "\n", + "You can also create additional kernels and have them be available via the kernel menu. Your kernel must contain the `nb_conda_kernels` and `ipykernel` packages for this to work. For example, if you wish to have a kernel with the `seaborn` package, you can create the following `environment.yml` from the terminal with the `pico` editor:\n", + "\n", + "```yaml\n", + " name: myenv\n", + " channels:\n", + " - conda-forge\n", + " dependencies:\n", + " - python=3\n", + " - seaborn\n", + " - nb_conda_kernels\n", + " - ipykernel\n", + "```\n", + "\n", + "followed by\n", + "\n", + "`mamba env update --name myenv -f environment.yml`\n", + "\n", + "at this point `myenv` will be available via the `Kernel → Change kernel...` menu." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/jupyter-images/ulv25s/build.sh b/jupyter-images/ulv25s/build.sh new file mode 100755 index 00000000..a3abfb74 --- /dev/null +++ b/jupyter-images/ulv25s/build.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Exit immediately if a command exits with a non-zero status +set -e + +usage() { + echo "Usage: $0 [--push]" + echo " Name of the Docker image to build" + echo " --push Optionally push the Docker image to DockerHub" + exit 1 +} + +if [ -z "$1" ]; then + echo "Error: No image name provided." + usage +fi + +IMAGE_NAME=$1 +PUSH_IMAGE=false + +if [ "$2" == "--push" ]; then + PUSH_IMAGE=true +fi + +DATE_TAG=$(date "+%Y%b%d_%H%M%S") +RANDOM_HEX=$(openssl rand -hex 2) +TAG="${DATE_TAG}_${RANDOM_HEX}" +FULL_TAG="unidata/$IMAGE_NAME:$TAG" + +# Build the Docker image +echo "Building Docker image with tag: $FULL_TAG" +docker build --no-cache --pull --tag "$FULL_TAG" . + +echo "Docker image built successfully: $FULL_TAG" + +if $PUSH_IMAGE; then + echo "Pushing Docker image to DockerHub: $FULL_TAG" + docker push "$FULL_TAG" + echo "Docker image pushed successfully: $FULL_TAG" +else + echo "Skipping Docker image push. Use '--push' to push the image." +fi + +exit 0 diff --git a/jupyter-images/ulv25s/default_kernel.py b/jupyter-images/ulv25s/default_kernel.py new file mode 100755 index 00000000..80a432f3 --- /dev/null +++ b/jupyter-images/ulv25s/default_kernel.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +import argparse +import glob +import json +import os +import re + + +def update_kernelspec_in_notebooks(directory, new_name): + """ + Updates the kernelspec in all Jupyter Notebook files within the specified + directory and its subdirectories, while preserving the original file + formatting. + + Args: + directory (str): The path to the directory containing .ipynb files. + new_name (str): The new name to set in the kernelspec. + """ + for file_path in glob.glob(f'{directory}/**/*.ipynb', recursive=True): + try: + with open(file_path, 'r', encoding='utf-8') as file: + file_contents = file.read() + notebook = json.loads(file_contents) + + if 'kernelspec' not in notebook.get('metadata', {}): + print(f"No kernelspec found in {file_path}. Skipping file.") + continue + + kernelspec = notebook['metadata']['kernelspec'] + kernelspec['display_name'] = f"Python [conda env:{new_name}]" + kernelspec['name'] = f"conda-env-{new_name}-py" + + # Convert the updated kernelspec dictionary to a JSON-formatted + # string with indentation + updated_kernelspec = json.dumps(kernelspec, indent=4) + + # Replace the existing kernelspec section in the original file + # contents with the updated JSON string. The regular expression + # looks for the "kernelspec" key and replaces its entire value + # (including nested structures), preserving the overall structure + # and formatting of the file. + updated_contents = re.sub( + r'"kernelspec": *\{.*?\}', + f'"kernelspec": {updated_kernelspec}', + file_contents, flags=re.DOTALL + ) + + with open(file_path, 'w', encoding='utf-8') as file: + file.write(updated_contents) + + except Exception as e: + print(f"Error processing file {file_path}: {e}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Update the kernel name in " + "Jupyter Notebook files in directory " + "tree.") + parser.add_argument("new_kernel_name", help="New kernel name to set.") + parser.add_argument("directory_path", nargs='?', default=os.getcwd(), + help="Directory containing .ipynb files (default: " + "current directory).") + + args = parser.parse_args() + + update_kernelspec_in_notebooks(args.directory_path, args.new_kernel_name) diff --git a/jupyter-images/ulv25s/environment.yml b/jupyter-images/ulv25s/environment.yml new file mode 100644 index 00000000..e93d7a9b --- /dev/null +++ b/jupyter-images/ulv25s/environment.yml @@ -0,0 +1,25 @@ +name: ulv25s +channels: + - conda-forge +dependencies: + # Required by JupyterLab + - 'python<3.13' + - nb_conda_kernels + - ipykernel + # User requested packages + - numpy + - matplotlib + - cartopy + - metpy + - siphon + - pandas + - pip + - xarray + - ipywidgets + - python-awips + - scikit-learn + - seaborn + - pip: + # It is recommended to install a package using pip as a last resort, i.e. + # when it is not found in the conda repos + - palmerpenguins diff --git a/jupyter-images/ulv25s/overrides.json b/jupyter-images/ulv25s/overrides.json new file mode 100644 index 00000000..c60d6b61 --- /dev/null +++ b/jupyter-images/ulv25s/overrides.json @@ -0,0 +1,7 @@ +{ + "@jupyterlab/docmanager-extension:plugin": { + "defaultViewers": { + "markdown": "Markdown Preview" + } + } +} diff --git a/jupyter-images/ulv25s/secrets.yaml b/jupyter-images/ulv25s/secrets.yaml new file mode 100644 index 00000000..74acdf2d --- /dev/null +++ b/jupyter-images/ulv25s/secrets.yaml @@ -0,0 +1,112 @@ +hub: + cookieSecret: "xxx" + config: + Authenticator: + admin_users: + - admins + #If you have a large list of users, consider using allowed_users.yaml + allowed_users: + - users + # necessary for jhub admins to add user via admin page `/hub/admin` + allow_existing_users: true + GitHubOAuthenticator: + client_id: "xxx" + client_secret: "xxx" + oauth_callback_url: "https://ulv25s-1.ees220002.projects.jetstream-cloud.org:443/oauth_callback" + JupyterHub: + authenticator_class: github + extraConfig: + 01-no-labels: | + from kubespawner import KubeSpawner + class CustomSpawner(KubeSpawner): + def _build_common_labels(self, extra_labels): + labels = super()._build_common_labels(extra_labels) + # Until https://github.com/jupyterhub/kubespawner/issues/498 + # is fixed + del labels['hub.jupyter.org/username'] + return labels + c.JupyterHub.spawner_class = CustomSpawner + + +proxy: + secretToken: "xxx" + +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: "nginx" + cert-manager.io/cluster-issuer: "letsencrypt" + #For manually issuing certificates: see vms/jupyter/readme.md + #cert-manager.io/issuer: "incommon" + nginx.ingress.kubernetes.io/proxy-body-size: 500m + hosts: + - "ulv25s-1.ees220002.projects.jetstream-cloud.org" + tls: + - hosts: + - "ulv25s-1.ees220002.projects.jetstream-cloud.org" + secretName: certmanager-tls-jupyterhub + +#For having a dedicated core node: see vms/jupyter/readme.md +#scheduling: +# corePods: +# tolerations: +# - key: hub.jupyter.org/dedicated +# operator: Equal +# value: core +# effect: NoSchedule +# - key: hub.jupyter.org_dedicated +# operator: Equal +# value: core +# effect: NoSchedule +# nodeAffinity: +# matchNodePurpose: require + +singleuser: + extraEnv: + NBGITPULLER_DEPTH: "0" + storage: + capacity: 10Gi + startTimeout: 600 + memory: + guarantee: 4G + limit: 4G + cpu: + guarantee: 1 + limit: 2 + defaultUrl: "/lab" + image: + name: "unidata/ulv25s" + tag: "xxx" + lifecycleHooks: + postStart: + exec: + command: + - "bash" + - "-c" + - > + dir="/home/jovyan/.ssh"; [ -d $dir ] && { chmod 700 $dir && chmod -f 600 $dir/* && chmod -f 644 $dir/*.pub; } || true; + cp -t /home/jovyan /Acknowledgements.ipynb /additional_kernels.ipynb; + python /default_kernel.py $DEFAULT_ENV_NAME /home/jovyan; + [[ -f $HOME/.bashrc ]] || cp /etc/skel/.bashrc $HOME/; + [[ -f $HOME/.profile ]] || cp /etc/skel/.profile $HOME/; + [[ -f $HOME/.bash_logout ]] || cp /etc/skel/.bash_logout $HOME/; + [[ -f $HOME/.condarc ]] || cp /.condarc $HOME/; + [ -d "/share" ] && [ ! -L ~/share ] && ln -s /share ~/share || true; + #gitpuller ; + #Multiple profiles: see vms/jupyter/readme.md + #profileList: + #- display_name: "High Power (default)" + # description: "12 GB of memory; up to 4 vCPUs" + # kubespawner_override: + # mem_guarantee: 12G + # mem_limit: 12G + # cpu_guarantee: 2 + # cpu_limit: 4 + # default: true + #- display_name: "Low Power" + # description: "6 GB of memory; up to 2 vCPUS" + # kubespawner_override: + # mem_guarantee: 6G + # mem_limit: 6G + # cpu_guarantee: 1 + # cpu_limit: 2 diff --git a/jupyter-images/ulv25s/update_material.ipynb b/jupyter-images/ulv25s/update_material.ipynb new file mode 100644 index 00000000..ae49f3aa --- /dev/null +++ b/jupyter-images/ulv25s/update_material.ipynb @@ -0,0 +1,136 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "
\n", + "\n", + "
\n", + "\"Unidata\n", + "
\n", + "\n", + "

Update Notebook Material

\n", + "

by Unidata

\n", + "\n", + "
\n", + "
\n", + "\n", + "---\n", + "\n", + "This notebook can be used to update material whenever updates are posted.\n", + "\n", + "---\n", + "\n", + "## Running the Update\n", + "\n", + "When you run the following cell, any changes from the workshop GitHub repository will be applied to the material under the directory defined by `gitdir` in your workspace.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "repourl = \"https://github.com/unidata/ucar-intern-pdws/\"\n", + "branch = \"main\"\n", + "gitdir = \"ucar-intern-pdws\"\n", + "\n", + "!gitpuller {repourl} {branch} {gitdir}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## What to expect?\n", + "\n", + "### No Updates Available\n", + "\n", + "If there are no updates to the material, the output from the cell will look something like the following:\n", + "\n", + "~~~bash\n", + "$ git fetch\n", + "\n", + "$ git reset --mixed\n", + "\n", + "$ git -c user.email=nbgitpuller@nbgitpuller.link -c user.name=nbgitpuller merge -Xours origin/main\n", + "\n", + "Already up to date.\n", + "~~~\n", + "\n", + "### Updates are Available\n", + "If there are updates to the material, the output will show the updated files being pulled, for example:\n", + "\n", + "~~~bash\n", + "$ git fetch\n", + "\n", + "From https://github.com/Unidata/users-workshop-2023\n", + "\n", + " 193f5d9..392761e main -> origin/main\n", + "\n", + "$ git reset --mixed\n", + "\n", + "$ git -c user.email=nbgitpuller@nbgitpuller.link -c user.name=nbgitpuller merge -Xours origin/main\n", + "\n", + "Updating 193f5d9..392761e\n", + "\n", + "Fast-forward\n", + "\n", + " {monday => 1_monday}/martin_tabular/TabularSummer2023.ipynb | 0\n", + "\n", + " .../breakout_sessions/Intro_to_sklearn_2023.ipynb | 0\n", + "\n", + " {thursday => 4_thursday}/Grover_Open_Science/placeholder.rtf | 0\n", + "\n", + " README.md | 2 ++\n", + "\n", + " 4 files changed, 2 insertions(+)\n", + "\n", + " rename {monday => 1_monday}/martin_tabular/TabularSummer2023.ipynb (100%)\n", + "\n", + " rename {wednesday => 3_wednesday}/breakout_sessions/Intro_to_sklearn_2023.ipynb (100%)\n", + "\n", + " rename {thursday => 4_thursday}/Grover_Open_Science/placeholder.rtf (100%)\n", + "~~~\n", + "\n", + "That's it!\n", + "Now you have the latest and greatest material." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}