Update Image to Template Version 2.0.0

.github/workflows/docker-publish.yml

@@ -1,62 +1,94 @@
-name: Docker
-
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
+# Build and publish a Docker image.
+name: Build and publish docker images
 
 on:
-  schedule:
-    - cron: '00 03 * * 0'
   push:
-    branches: [ "main" ]
+    branches: ["main"]
     # Publish semver tags as releases.
-    tags: [ 'v*.*.*' ]
+    tags: ["v*.*.*"]
   pull_request:
-    branches: [ "main" ]
+    branches: ["main"]
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
 
+permissions:
+  contents: write
+  packages: write
+  attestations: write
+  id-token: write
 
 jobs:
-  build:
+  # Job 1: Update Lock File
+  update-lockfile:
+    name: Update Python Lock File
     runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
-          submodules: True
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login
-        uses: docker/login-action@v3
+          ref: ${{ github.head_ref || github.ref_name }}
+          submodules: true
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v2
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Update lock file
+        run: uv lock --upgrade-package nomad-lab
+
+      # Commits any changes made to the lockfile
+      - name: Commit lock file changes
+        run: |
+          git config --global user.name github-actions
+          git config --global user.email [email protected]
+          git add uv.lock
+          if [[ `git status --porcelain` ]]; then
+            git commit -m "Update lockfile"
+            git push origin -o ci.skip # prevent triggering the pipeline again
+          fi
+
+  # Job 2: Build and Push Docker Image
+  docker-publish:
+    name: Build Docker Image
+    runs-on: ubuntu-latest
+    needs: update-lockfile
+
+    strategy:
+      fail-fast: false
+      matrix:
+        service: [app, jupyter]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref || github.ref_name }}
+          submodules: true
+
+      - uses: docker/setup-buildx-action@v3
+
+      - uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      
+
       - name: Extract Docker metadata
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-
-      - name: Build and push 
-        id: build-and-push
+          images: ${{ env.REGISTRY }}/${{ github.repository }}${{ matrix.service == 'jupyter' && '/jupyter' || '' }}
+
+      - name: Build and push Docker image
         uses: docker/build-push-action@v5
         with:
           context: .
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          target: ${{ matrix.service == 'jupyter' && 'jupyter' || 'final' }}
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          #cache-from: type=gha
-          #cache-to: type=gha,mode=max

.github/workflows/initialize.yml

@@ -0,0 +1,60 @@
+name: Template Repository Initialization
+
+on:
+  # Triggers the workflow on creation of repository
+  create:
+  workflow_dispatch:
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+permissions:
+  contents: write
+
+jobs:
+  initialize_repository:
+    if: github.repository != 'FAIRmat-NFDI/nomad-distribution-template'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      # Replaces the template repository name in the README with the new one
+      - name: Update README
+        run: |
+          sed -i "/# NOMAD Oasis Distribution/,/button on the right./d" README.md
+          export image_name="${{ github.repository }}"
+          sed -i "s|fairmat-nfdi/nomad-distribution-template|${image_name,,}|g" README.md
+          sed -i "s|FAIRmat-NFDI/nomad-distribution-template|${{ github.repository }}|g" README.md
+          sed -i "s|FAIRmat-NFDI|${{ github.repository_owner }}|g" README.md
+          sed -i "s|nomad-distribution-template|${{ github.event.repository.name }}|g" README.md
+          sed -i "s|template https://github.com/${{ github.repository }}|template https://github.com/FAIRmat-NFDI/nomad-distribution-template|g" README.md
+          sed -i "s|${{ github.repository }}/releases/latest|FAIRmat-NFDI/nomad-distribution-template/releases/latest|g" README.md
+          sed -i "s|@ git+https://github.com/${{ github.repository_owner }}|@ git+https://github.com/FAIRmat-NFDI|g" README.md
+
+      # Replaces the template repository name in the docker config file with the new one
+      - name: Update docker-compose.yaml
+        run: |
+          export image_name="${{ github.repository }}"
+          sed -i "s|fairmat-nfdi/nomad-distribution-template|${image_name,,}|g" docker-compose.yaml
+
+        # Replaces the template repository name in the nomad config with the new one
+      - name: Update nomad.yaml
+        run: |
+          export image_name="${{ github.repository }}"
+          sed -i "s|fairmat-nfdi/nomad-distribution-template|${image_name,,}|g" configs/nomad.yaml
+
+      # Deletes this workflow file to prevent it from running on branch creation
+      - name: Delete initialization workflow
+        run: rm .github/workflows/initialize.yml
+
+      # Commits all changes
+      - name: Commit repository initialization
+        run: |
+          git config --global user.name github-actions
+          git config --global user.email [email protected]
+          git commit -am "Repository initialization"
+          git push

.volumes/fs/north/users/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

.volumes/fs/public/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

.volumes/fs/staging/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

.volumes/fs/tmp/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

.volumes/mongo/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

Dockerfile

@@ -1,8 +1,150 @@
-FROM gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:v1.3.1
-USER root
-RUN apt-get update
-RUN apt-get -y install git
+# syntax=docker/dockerfile:1
+
+# Comments are provided throughout this file to help you get started.
+# If you need more help, visit the Dockerfile reference guide at
+# https://docs.docker.com/engine/reference/builder/
+
+ARG PYTHON_VERSION=3.12
+
+FROM python:${PYTHON_VERSION}-slim AS base
+
+# Keeps Python from buffering stdout and stderr to avoid situations where
+# the application crashes without emitting any logs due to buffering.
+ENV PYTHONUNBUFFERED=1
+ENV VIRTUAL_ENV=/opt/venv \
+    PATH="/opt/venv/bin:$PATH" \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/opt/venv
+
+# Final stage to create the runnable image with minimal size
+FROM base AS base_final
+
+WORKDIR /app
+
+RUN apt-get update \
+ && apt-get install --yes --quiet --no-install-recommends \
+       libgomp1 \
+       libmagic1 \
+       curl \
+       zip \
+       unzip \
+       nodejs \
+       npm \
+       && npm install -g configurable-http-proxy@^4.2.0 \
+       # clean cache and logs
+       && rm -rf /var/lib/apt/lists/* /var/log/* /var/tmp/* ~/.npm
+
+# Activate the virtualenv in the container
+# See here for more information:
+# https://pythonspeed.com/articles/multi-stage-docker-python/
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Create a non-privileged user that the frenrug will run under.
+# See https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#user
+ARG UID=1000
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    nomad
+
+FROM base AS builder
+
+# Prevents Python from writing pyc files.
+ENV PYTHONDONTWRITEBYTECODE=1
+
+ENV RUNTIME=docker
+
+WORKDIR /app
+
+RUN apt-get update \
+ && apt-get install --yes --quiet --no-install-recommends \
+      libgomp1 \
+      libmagic1 \
+      file \
+      gcc \
+      build-essential \
+      curl \
+      zip \
+      unzip \
+      git \
+ && rm -rf /var/lib/apt/lists/*
+
+# Create a non-privileged user that the frenrug will run under.
+# See https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#user
+ARG UID=1000
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    nomad
+
+
+# Install UV
+COPY --from=ghcr.io/astral-sh/uv:0.4 /uv /bin/uv
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --extra plugins --frozen --no-install-project
+
+
+COPY scripts ./scripts
+
+FROM base_final AS final
+
+COPY --chown=nomad:1000 --from=builder /opt/venv /opt/venv
+COPY --chown=nomad:1000 scripts/run.sh .
+COPY --chown=nomad:1000 scripts/run-worker.sh .
+COPY configs/nomad.yaml nomad.yaml
+
+RUN mkdir -p /app/.volumes/fs \
+ && chown -R nomad:1000 /app \
+ && chown -R nomad:1000 /opt/venv \
+ && mkdir nomad \
+ && cp /opt/venv/lib/python3.12/site-packages/nomad/jupyterhub_config.py nomad/
+
 USER nomad
-COPY plugins.txt plugins.txt
-RUN pip install -r plugins.txt
-COPY nomad.yaml nomad.yaml
+
+# The application ports
+EXPOSE 8000
+EXPOSE 9000
+
+VOLUME /app/.volumes/fs
+
+
+FROM jupyter/datascience-notebook:lab-3.6.2 AS jupyter
+
+# Fix: https://github.com/hadolint/hadolint/wiki/DL4006
+# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+USER root
+
+RUN apt-get update \
+ && apt-get install --yes --quiet --no-install-recommends \
+       libmagic1 \
+       # clean cache and logs
+       && rm -rf /var/lib/apt/lists/* /var/log/* /var/tmp/* ~/.npm
+
+# Switch back to jovyan to avoid accidental container runs as root
+USER ${NB_UID}
+WORKDIR "${HOME}"
+
+COPY --from=ghcr.io/astral-sh/uv:0.4 /uv /bin/uv
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv export --extra plugins --extra jupyter | uv pip install -r /dev/stdin --system
+
+
+# Get rid ot the following message when you open a terminal in jupyterlab:
+# groups: cannot find name for group ID 11320
+RUN touch ${HOME}/.hushlogin