diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..434917b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +# Ignore documentation +book/ +docs/ + +# Ignore version control folder +.git/ +.gitignore + +# Ignore R stuff +.Rhistory +.Rproj.user +py-rocket-base.Rproj diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml new file mode 100644 index 0000000..cc5676d --- /dev/null +++ b/.github/workflows/test-build.yml @@ -0,0 +1,37 @@ +name: Docker Image test +on: + workflow_dispatch: null + push: + branches: test + +jobs: + build: + runs-on: ubuntu-latest + permissions: write-all + steps: + - uses: actions/checkout@v3 + with: + ref: test + - name: Login to GitHub Container Registry + if: github.repository == 'nmfs-opensci/py-rocket-base' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{github.actor}} + password: ${{secrets.GITHUB_TOKEN}} + - name: Create short_sha tag + shell: bash + run: | + short_sha=$(echo "${{ github.sha }}" | cut -c1-7) + echo "tag=${short_sha}" >> $GITHUB_ENV + - name: Build the Docker image + if: github.repository == 'nmfs-opensci/py-rocket-base' + run: | + docker build . -f Dockerfile \ + --tag ghcr.io/nmfs-opensci/py-rocket-base/test:latest \ + --tag ghcr.io/nmfs-opensci/py-rocket-base/test:${{ env.tag }} + - name: Publish + if: github.repository == 'nmfs-opensci/py-rocket-base' + run: | + docker push ghcr.io/nmfs-opensci/py-rocket-base/test:latest + docker push ghcr.io/nmfs-opensci/py-rocket-base/test:${{ env.tag }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d31e82e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,62 @@ +FROM ghcr.io/nmfs-opensci/py-rocket-base/base-image:latest + +USER root + +# Define environment variables +# DISPLAY Tell applications where to open desktop apps - this allows notebooks to pop open GUIs +ENV REPO_DIR="/srv/repo" \ + DISPLAY=":1.0" \ + R_VERSION="4.4.1" + +# Add NB_USER to staff group (required for rocker script) +# Ensure the staff group exists first +RUN groupadd -f staff && usermod -a -G staff "${NB_USER}" + +COPY --chown=${NB_USER}:${NB_USER} . ${REPO_DIR} +RUN chgrp -R staff ${REPO_DIR} && \ + chmod -R g+rwx ${REPO_DIR} && \ + rm -rf ${REPO_DIR}/book ${REPO_DIR}/docs + +# Copy scripts to /pyrocket_scripts and set permissions +RUN mkdir -p /pyrocket_scripts && \ + cp -r ${REPO_DIR}/scripts/* /pyrocket_scripts/ && \ + chown -R root:staff /pyrocket_scripts && \ + chmod -R 775 /pyrocket_scripts + +# Install extra conda packages +RUN /pyrocket_scripts/install-conda-packages.sh ${REPO_DIR}/environment.yml + +# Install R, RStudio via Rocker scripts +ENV R_DOCKERFILE="verse_${R_VERSION}" +RUN PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin && \ + chmod +x ${REPO_DIR}/rocker.sh && \ + ${REPO_DIR}/rocker.sh + +# Install extra apt packages +# Install linux packages after R installation since the R install scripts get rid of packages +RUN /pyrocket_scripts/install-apt-packages.sh ${REPO_DIR}/apt.txt + +# Re-enable man pages disabled in Ubuntu 18 minimal image +# https://wiki.ubuntu.com/Minimal +RUN yes | unminimize +# NOTE: $NB_PYTHON_PREFIX is the same as $CONDA_PREFIX at run-time. +# $CONDA_PREFIX isn't available in this context. +# NOTE: Prepending ensures a working path; if $MANPATH was previously empty, +# the trailing colon ensures that system paths are searched. +ENV MANPATH="${NB_PYTHON_PREFIX}/share/man:${MANPATH}" +RUN mandb + +# Add custom Jupyter server configurations +RUN mkdir -p ${NB_PYTHON_PREFIX}/etc/jupyter/jupyter_server_config.d/ && \ + mkdir -p ${NB_PYTHON_PREFIX}/etc/jupyter/jupyter_notebook_config.d/ && \ + cp ${REPO_DIR}/custom_jupyter_server_config.json ${NB_PYTHON_PREFIX}/etc/jupyter/jupyter_server_config.d/ && \ + cp ${REPO_DIR}/custom_jupyter_server_config.json ${NB_PYTHON_PREFIX}/etc/jupyter/jupyter_notebook_config.d/ + +# Set up the start command +USER ${NB_USER} +RUN chmod +x ${REPO_DIR}/start \ + && cp ${REPO_DIR}/start /srv/start + +# Revert to default user and home as pwd +USER ${NB_USER} +WORKDIR ${HOME} diff --git a/appendix b/appendix deleted file mode 100644 index 7fc349d..0000000 --- a/appendix +++ /dev/null @@ -1,56 +0,0 @@ -USER root - -# Clean up extra files in ${REPO_DIR} -RUN rm -rf ${REPO_DIR}/book ${REPO_DIR}/docs - -# repo2docker does not set this. This is the default env in repo2docker type images -ENV CONDA_ENV=notebook -# Tell applications where to open desktop apps - this allows notebooks to pop open GUIs -ENV DISPLAY=":1.0" - -# Install R, RStudio via Rocker scripts -ENV R_VERSION="4.4.1" -ENV R_DOCKERFILE="verse_${R_VERSION}" -# This is in the rocker script but will not run since ${NB_USER} already exists -# Needed because rocker scripts set permissions based on the staff group -RUN usermod -a -G staff "${NB_USER}" -RUN PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin && \ - chmod +x ${REPO_DIR}/rocker.sh && \ - ${REPO_DIR}/rocker.sh - -# Install linux packages after R installation since the R install scripts get rid of packages -# The package_list part is reading the file and doing clean-up to just have the list of packages -RUN package_list=$(grep -v '^\s*#' ${REPO_DIR}/apt2.txt | grep -v '^\s*$' | sed 's/\r//g; s/#.*//; s/^[[:space:]]*//; s/[[:space:]]*$//' | awk '{$1=$1};1') && \ - apt-get update && \ - apt-get install --yes --no-install-recommends $package_list && \ - apt-get autoremove --purge && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Re-enable man pages disabled in Ubuntu 18 minimal image -# https://wiki.ubuntu.com/Minimal -RUN yes | unminimize -# NOTE: $NB_PYTHON_PREFIX is the same as $CONDA_PREFIX at run-time. -# $CONDA_PREFIX isn't available in this context. -# NOTE: Prepending ensures a working path; if $MANPATH was previously empty, -# the trailing colon ensures that system paths are searched. -ENV MANPATH="${NB_PYTHON_PREFIX}/share/man:${MANPATH}" -RUN mandb - -# Add custom jupyter config. You can also put config.py files in the same place -RUN cp ${REPO_DIR}/custom_jupyter_server_config.json ${NB_PYTHON_PREFIX}/etc/jupyter/jupyter_server_config.d/ -RUN cp ${REPO_DIR}/custom_jupyter_server_config.json ${NB_PYTHON_PREFIX}/etc/jupyter/jupyter_notebook_config.d/ - -# Copy scripts into /pyrocket_scripts directory in the image -RUN mkdir -p /pyrocket_scripts && cp -r ${REPO_DIR}/scripts/* /pyrocket_scripts/ - -# Set ownership to root and permissions to 755 -RUN chown -R root:staff /pyrocket_scripts && \ - chmod -R 775 /pyrocket_scripts - -# Convert NB_USER to ENV (from ARG) so that it passes to the child dockerfile -ENV NB_USER=${NB_USER} - -# Revert to default user and home as pwd -USER ${NB_USER} -WORKDIR ${HOME} diff --git a/apt2.txt b/apt.txt similarity index 100% rename from apt2.txt rename to apt.txt diff --git a/environment.yml b/environment.yml index e4d6887..5ea7958 100644 --- a/environment.yml +++ b/environment.yml @@ -5,11 +5,11 @@ channels: - nodefaults dependencies: - - python~=3.11.0 - - jupyterlab>=4.0 + # Core JupyterHub packages + - python=3.12 + - pangeo-notebook=2024.11.11 + - pip - jupyter-resource-usage - - jupyterlab-git - # gh-scoped-creds allows users to securely push to GitHub from their repo. - gh-scoped-creds==4.1 # R/RStudio Support @@ -25,8 +25,7 @@ dependencies: #- syncthing~=1.22.1 # Extra Jupyter tools - - ipython - - ipywidgets + - jupyterlab-git - jupyter-ai - jupyter-book - jupyter-offlinenotebook @@ -35,21 +34,15 @@ dependencies: - jupyterlab-geojson - jupyterlab-h5web - jupyterlab-myst - - jupyterlab_pygments>=0.3.0 - jupytext - nbdime + # JupyterBook Addons - sphinx - sphinxcontrib-bibtex - # Interactive apps - - itables - - voila - # nbgitpuller is very helpful when distributing user content - - nbgitpuller + # More git tools from github-cli - gh - # conda-lock - - conda-lock # Resolves warning "No ICDs were found": https://github.com/CryoInTheCloud/hub-image/issues/50 - ocl-icd-system diff --git a/scripts/install-conda-packages.sh b/scripts/install-conda-packages.sh index 937420f..b9dc9e8 100644 --- a/scripts/install-conda-packages.sh +++ b/scripts/install-conda-packages.sh @@ -4,50 +4,68 @@ # Check if a filename argument is provided if [ -z "$1" ]; then echo "Error: install-conda-packages.sh requires a file name (either conda-lock.yml or environment.yml)." >&2 - echo "Usage: RUN /pyrocket_scripts/install-conda-packages.sh " >&2 + echo "Usage: RUN /pyrocket_scripts/install-conda-packages.sh [env_name]" >&2 exit 1 fi +# Set the environment name, defaulting to ${CONDA_ENV} if not provided +ENV_FILE="$1" +ENV_NAME="${2:-${CONDA_ENV}}" + # Check if running as root and switch to NB_USER if needed if [[ $(id -u) -eq 0 ]]; then echo "Switching to ${NB_USER} to run install-conda-packages.sh" - exec su "${NB_USER}" -c "/bin/bash $0 $1" # Pass along the filename argument + exec su "${NB_USER}" -c "/bin/bash $0 $ENV_FILE $ENV_NAME" fi # Main script execution as NB_USER echo "Running install-conda-packages.sh as ${NB_USER}" - -# Set the file variable to the provided argument -ENV_FILE="$1" +echo " Using environment file: $ENV_FILE" +echo " Target environment: $ENV_NAME" # Verify the file exists and is readable if [ ! -f "$ENV_FILE" ]; then echo " Error: File '$ENV_FILE' not found. Please provide a valid file path." >&2 - echo " Usage: RUN /pyrocket_scripts/install-conda-packages.sh " >&2 exit 1 fi -echo " Found file: $ENV_FILE" - -# Determine file type based on content -if grep -q "lock_set" "$ENV_FILE"; then - echo " Detected conda-lock.yml file." - ${NB_PYTHON_PREFIX}/bin/conda-lock install --name ${CONDA_ENV} -f "$ENV_FILE" - INSTALLATION_HAPPENED=true -elif grep -q "name:" "$ENV_FILE"; then - echo " Detected environment.yml file." - ${CONDA_DIR}/condabin/mamba env update --name ${CONDA_ENV} -f "$ENV_FILE" - INSTALLATION_HAPPENED=true +# Check if the Conda environment exists +if ! ${CONDA_DIR}/condabin/conda env list | grep -q "^$ENV_NAME "; then + echo " Environment '$ENV_NAME' not found. Creating it." + + # Create environment based on file type + if grep -q "lock_set" "$ENV_FILE"; then + echo " Detected conda-lock.yml file." + ${NB_PYTHON_PREFIX}/bin/conda-lock install --name $ENV_NAME -f "$ENV_FILE" + elif grep -q "name:" "$ENV_FILE"; then + echo " Detected environment.yml file." + ${CONDA_DIR}/condabin/mamba env create -f "$ENV_FILE" --name $ENV_NAME + INSTALLATION_HAPPENED=true + else + echo " Error: Unrecognized file format in '$ENV_FILE'." + exit 1 + fi else - # If neither condition matches, output a helpful error message - echo "Error: Unrecognized file format in '${env_file}'." - echo " - For an environment.yml file, ensure it includes a 'name:' entry. Any name is acceptable." - echo " - For a conda-lock.yml file, ensure it includes a 'lock_set:' entry." - exit 1 + echo " Environment '$ENV_NAME' exists. Updating it." + + # Update environment based on file type + if grep -q "lock_set" "$ENV_FILE"; then + echo " Detected conda-lock.yml file." + ${NB_PYTHON_PREFIX}/bin/conda-lock install --name $ENV_NAME -f "$ENV_FILE" + elif grep -q "name:" "$ENV_FILE"; then + echo " Detected environment.yml file." + ${CONDA_DIR}/condabin/mamba env update --name $ENV_NAME -f "$ENV_FILE" + else + echo " Error: Unrecognized file format in '${ENV_FILE}'." + echo " - For an environment.yml file, ensure it includes a 'name:' entry. Any name is acceptable." + echo " - For a conda-lock.yml file, ensure it includes a 'lock_set:' entry." + exit 1 + fi fi # Run cleanup if installation occurred if [ "$INSTALLATION_HAPPENED" = true ]; then + echo " Installation clean-up." ${CONDA_DIR}/condabin/mamba clean -yaf find ${CONDA_DIR} -follow -type f -name '*.a' -delete find ${CONDA_DIR} -follow -type f -name '*.js.map' -delete diff --git a/test.yml b/test.yml new file mode 100644 index 0000000..a06a624 --- /dev/null +++ b/test.yml @@ -0,0 +1,9 @@ +name: test + +channels: + - conda-forge + - nodefaults + +dependencies: + - cmocean +