Add pawsey container build recipe

docs/development/docker/pawsey/mpich.dockerfile

@@ -0,0 +1,101 @@
+#####################################################################
+# Multi stage Dockerfile structure:
+# 1. runtime 
+# 2. build
+# 3. final == runtime + min. build
+#
+# It begins with layers for runtime execution. 
+# The runtime environment (packages, permissions, ENV vars.) 
+# are consistent accross all layer of this Dockerfile. 
+# The build layer takes the runtime layer and builds the software
+# stack in /usr/local.
+# The final image is a composite of the runtime layer and 
+# minimal sections of the build layer.
+#####################################################################
+
+ARG MPICH_VERSION="4.2.3"
+ARG PYTHON_VERSION="3.11"
+ARG MPI4PY_VERSION="4.0.1"
+
+FROM python:$PYTHON_VERSION-slim as runtime
+LABEL maintainer="https://github.com/underworldcode/"
+
+################
+## 1. Runtime ##
+################
+# Dockerfile ENV vars - for all image stages
+ENV LANG=C.UTF-8
+# openmpi lib will be install at /usr/local/lib
+ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+# add user jovyan
+ENV NB_USER jovyan
+ENV NB_HOME /home/$NB_USER
+RUN useradd -m -s /bin/bash -N $NB_USER
+
+#install runtime packages
+RUN apt-get update -qq \
+&&  DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
+        ssh \
+        bash \
+&&  apt-get clean \
+&&  rm -rf /var/lib/apt/lists/*
+
+################
+## 2. Build   ##
+################
+FROM runtime as build
+
+ARG MPICH_VERSION
+# Build options for Dockerfile
+
+### from https://quay.io/repository/pawsey/mpich-base/manifest/sha256:ca1cf20a4f1a8793ec794b7f927a03289f4ad449b51bc821ac98c5a269ea5e8a
+ARG MPICH_CONFIGURE_OPTIONS="--enable-fast=all,O3 --enable-fortran --enable-romio --prefix=/usr/local --with-device=ch4:ofi CC=gcc CXX=g++ FC=gfortran FFLAGS=-fallow-argument-mismatch"
+
+ARG MPICH_MAKE_OPTIONS="-j4"
+
+RUN apt-get update -qq \
+&&  DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
+        wget \
+        gcc \
+        gfortran \
+        g++ \
+        make \
+        file \
+&&  apt-get clean \
+&&  rm -rf /var/lib/apt/lists/*
+
+# build mpi
+RUN mkdir -p /tmp/src
+WORKDIR /tmp/src
+RUN wget http://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz --no-check-certificate \
+&& tar -zxf mpich-${MPICH_VERSION}.tar.gz 
+WORKDIR /tmp/src/mpich-${MPICH_VERSION}              
+RUN ./configure ${MPICH_CONFIGURE_OPTIONS} \
+ && make ${MPICH_MAKE_OPTIONS} \
+ && make install \
+ && ldconfig \
+ && rm -rf /tmp/src/
+
+### Add mpi4py for testing
+ARG MPI4PY_VERSION
+# RUN pip --no-cache-dir install --no-deps mpi4py==${MPI4PY_VERSION}
+# RUN pip install mpi4py==${MPI4PY_VERSION}
+RUN pip install --break-system-packages mpi4py==${MPI4PY_VERSION}
+# record build packages used
+RUN apt-mark showmanual > /opt/installed.txt
+
+################
+## 3. Final   ##
+################
+FROM runtime as final
+
+COPY --from=build /usr/local /usr/local
+COPY --from=build /opt/installed.txt /opt/installed.txt
+
+# switch to user and workspace
+WORKDIR $NB_HOME
+USER $
+
+### default location
+CMD [ "/bin/bash" ]
+

docs/development/docker/pawsey/pawsey_docker_builder.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Example Usage:
+#   - Run from underworld2 repository head
+#   - mpi and lavavu dockers are automatically generated via github actions
+#   - petsc and underworld2 must be created by runn the following script.
+
+
+MPICH_VERSION=3.4.3 #3.4a2 #3.4.3 #4.0.1
+MPI4PY_VERSION=3.1.6 #3.1.6 #3.1.6 #4.0.1
+PYTHON_VERSION=3.11
+PETSC_VERSION=3.22.2 #3.22.2
+
+ARCH=$(uname -m)
+# ARCH=x86_64 ### pawsey
+echo "Will build docker image locally for architecture type: $ARCH"
+echo "************************************************************"
+echo ""
+
+if [ "$ARCH" = "x86_64" ]
+then
+  PLATFORM="linux/amd64"
+else
+  PLATFORM="linux/arm64"
+fi
+
+
+
+
+echo " .. Now building mpich"
+# The mpi and lavavu images should be automatically made via github actions
+podman build . \
+--rm --squash-all \
+-f ./docs/development/docker/pawsey/mpich.dockerfile \
+--platform $PLATFORM \
+--build-arg PYTHON_VERSION=$PYTHON_VERSION \
+--build-arg MPICH_VERSION=$MPICH_VERSION \
+--build-arg MPI4PY_VERSION=$MPI4PY_VERSION \
+-t underworldcode/mpich:$MPICH_VERSION-$ARCH
+
+echo " .. Finished building mpich"
+
+#podman save -o mpich_$MPICH_VERSION-$ARCH.tar underworldcode/mpich:$MPICH_VERSION-$ARCH
+
+# echo " .. Now building lavavu"
+# podman build . \
+#   --rm --squash-all \
+#   -f ./docs/development/docker/lavavu/Dockerfile \
+#   --platform $PLATFORM \
+#   --build-arg UBUNTU_VERSION=$UBUNTU_VERSION \
+#   --build-arg PYTHON_VERSION=$PYTHON_VERSION \
+#   -t underworldcode/lavavu:$ARCH
+
+echo " .. Now building petsc"
+podman build . \
+ --rm --squash-all \
+ -f ./docs/development/docker/pawsey/petsc.dockerfile \
+ --platform $PLATFORM \
+ --build-arg MPI4PY_VERSION=$MPI4PY_VERSION \
+ --build-arg MPI_IMAGE="underworldcode/mpich:$MPICH_VERSION-$ARCH" \
+ --build-arg PYTHON_VERSION=$PYTHON_VERSION \
+ --build-arg PETSC_VERSION=$PETSC_VERSION \
+ -t underworldcode/petsc:$PETSC_VERSION-$ARCH
+
+echo " .. Finished building petsc"
+# podman save -o petsc_$PETSC_VERSION-$ARCH.tar underworldcode/petsc:$PETSC_VERSION-$ARCH
+
+echo " .. Now building UW2"
+### don't use pull here as we want the petsc image above
+podman build . \
+  --rm --squash-all \
+  -f ./docs/development/docker/pawsey/underworld.dockerfile \
+  --platform $PLATFORM \
+  --build-arg MPI4PY_VERSION=$MPI4PY_VERSION \
+  --build-arg PYTHON_VERSION=$PYTHON_VERSION \
+  --build-arg PETSC_IMAGE="underworldcode/petsc:$PETSC_VERSION-$ARCH" \
+  --build-arg LAVAVU_IMAGE="underworldcode/lavavu:$ARCH" \
+  -t underworldcode/underworld2:2.16.0b-$ARCH
+
+### save to compressed file (tar), which can be converted by singularity on the HPC
+podman save -o underworld2_2.16.0b-$ARCH.tar underworldcode/underworld2:2.16.0b-$ARCH
+
+### transfer to HPC using rsync/scp
+# rsync -P -a underworld2_2.16.0b-$ARCH.tar [email protected]:/container/directory
+
+### how to build from tar on HPC
+# singularity build underworld2_2.16.0b-$ARCH.sif docker-archive://underworld2_2.16.0b-$ARCH.tar

docs/development/docker/pawsey/petsc.dockerfile

@@ -0,0 +1,167 @@
+#####################################################################
+# Multi stage Dockerfile structure:
+# 1. runtime 
+# 2. build
+# 3. final == runtime + min. build
+#
+# It begins with layers for runtime execution. 
+# The runtime environment (packages, permissions, ENV vars.) 
+# are consistent accross all layer of this Dockerfile. 
+# The build layer takes the runtime layer and builds the software
+# stack in /usr/local.
+# The final image is a composite of the runtime layer and 
+# minimal sections of the build layer.
+#####################################################################
+
+ARG MPI_IMAGE="underworldcode/openmpi:4.1.4"
+ARG PYTHON_VERSION="3.11"
+ARG PETSC_VERSION="3.22.2"
+ARG MPI4PY_VERSION="3.1.6"
+
+FROM ${MPI_IMAGE} as mpi-image
+
+FROM python:$PYTHON_VERSION-slim as runtime
+LABEL maintainer="https://github.com/underworldcode/"
+
+# need to repeat ARGS after every FROM 
+ARG PYTHON_VERSION
+
+################
+## 1. Runtime ##
+################
+
+
+#### Dockerfile ENV vars - for all image stages
+ENV LANG=C.UTF-8
+# mpi lib will be install at /usr/local/lib
+ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+
+# python env vars. 
+# prepappending on PATH means all pip install will goto the PYOPT
+ENV PYVER=${PYTHON_VERSION}
+ENV PYOPT=/opt/venv
+ENV PATH=$PYOPT/bin:$PATH
+ENV PYTHONPATH=$PYTHONPATH:$PYOPT/lib/python${PYVER}/site-packages
+ENV PETSC_DIR=/usr/local
+ENV PYTHONPATH=$PYTHONPATH:$PETSC_DIR/lib
+
+# add user jovyan
+ENV NB_USER jovyan
+ENV NB_HOME /home/$NB_USER
+RUN useradd -m -s /bin/bash -N $NB_USER
+
+RUN apt-get update -qq \
+&&  DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
+        bash-completion \
+        ssh \
+        libopenblas0 \
+        python3-full \
+        python3-dev \
+&& apt-get clean \
+&& rm -rf /var/lib/apt/lists/*
+
+# build and open virtual environment
+RUN python3 -m venv $PYOPT \
+&&  chmod ugo+rwx $PYOPT \
+&&  pip3 install -U setuptools \
+                    wheel
+
+################
+## 2. Build   ##
+################
+
+FROM runtime as build
+
+ARG PYTHON_VERSION
+ARG MPI4PY_VERSION
+ARG PETSC_VERSION
+
+RUN apt-get update -qq \
+&&  DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
+        ca-certificates \
+        wget \
+        build-essential \
+        gfortran \
+        g++ \
+        cmake \
+        libopenblas-dev \
+        libz-dev \
+        git \
+&&  apt-get clean \
+&&  rm -rf /var/lib/apt/lists/*
+
+# copy in from mpi container
+COPY --from=mpi-image /opt/installed.txt /opt/installed.txt
+COPY --from=mpi-image /usr/local /usr/local
+RUN apt-get update -qq \
+&&  DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends $(awk '{print $1'} /opt/installed.txt) \
+&&  apt-get clean \
+&&  rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install --no-cache-dir \
+      "cython==3.0.10" \
+      "numpy<2"
+
+RUN pip --no-cache-dir install --no-deps mpi4py==${MPI4PY_VERSION}
+
+# get petsc
+RUN mkdir -p /tmp/src
+WORKDIR /tmp/src
+RUN wget https://web.cels.anl.gov/projects/petsc/download/release-snapshots/petsc-lite-${PETSC_VERSION}.tar.gz --no-check-certificate \
+&& tar -zxf petsc-lite-${PETSC_VERSION}.tar.gz
+WORKDIR /tmp/src/petsc-${PETSC_VERSION}
+
+# install petsc as root
+RUN PETSC_DIR=`pwd` ./configure --with-debugging=0 --prefix=/usr/local \
+                --COPTFLAGS="-g -O3" --CXXOPTFLAGS="-g -O3" --FOPTFLAGS="-g -O3" \
+                --with-shared-libraries         \
+		--with-petsc4py=1               \
+                --with-zlib=1                   \
+                --with-shared-libraries=1       \
+                --with-cxx-dialect=C++11        \
+                --with-make-np=4                \
+                --download-hdf5=1 \
+                --download-mumps=1              \
+                --download-parmetis=1           \
+                --download-metis=1              \
+                --download-superlu=1            \
+                --download-hypre=1              \
+                --download-scalapack=1          \
+                --download-superlu_dist=1       \
+                --download-pragmatic=1          \
+                --download-ctetgen              \
+                --download-eigen                \
+                --download-superlu=1            \
+                --download-triangle             \
+                --useThreads=0                  \
+&&  make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt all \
+&&  make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt install \
+&&  rm -rf /usr/local/share/petsc
+
+# install h5py with MPI enabled, does not currently work in containers on pawsey
+#RUN CC=mpicc HDF5_MPI="ON" HDF5_DIR=${PETSC_DIR} pip3 install --no-cache-dir --no-build-isolation --no-binary=h5py h5py \
+#RUN HDF5_DIR=${PETSC_DIR} pip3 install --no-cache-dir --no-build-isolation --no-binary=h5py h5py \
+RUN pip install --no-cache-dir jupyterlab
+
+# record builder stage packages used
+RUN pip3 freeze > /opt/requirements.txt \
+&&  apt-mark showmanual > /opt/packages.txt
+
+################
+## 3. Final   ##
+################
+
+FROM runtime as minimal
+
+COPY --from=build /opt /opt
+COPY --from=build /usr/local /usr/local
+
+# add LD_LIBRARY_PATH
+ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+
+# switch to not-root user and workspace
+USER $NB_USER
+WORKDIR $NB_HOME
+
+# default command is to run jupyter lab
+CMD ["jupyter-lab", "--no-browser", "--ip='0.0.0.0'"]