alternative method to rebuild & shipping full CUDA runtime

EESSI-determine-rebuilds.sh

@@ -0,0 +1,123 @@
+#!/bin/bash
+#
+# Script to determine which parts of the EESSI software stack (version set through init/eessi_defaults)
+# have to be rebuilt
+
+# see example parsing of command line arguments at
+#   https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop
+#   https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash
+
+display_help() {
+  echo "usage: $0 [OPTIONS]"
+  echo "  -g | --generic         -  instructs script to build for generic architecture target"
+  echo "  -h | --help            -  display this usage information"
+}
+
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -g|--generic)
+      DETECTION_PARAMETERS="--generic"
+      shift
+      ;;
+    -h|--help)
+      display_help  # Call your function
+      # no shifting needed here, we're done.
+      exit 0
+      ;;
+    -*|--*)
+      echo "Error: Unknown option: $1" >&2
+      exit 1
+      ;;
+    *)  # No more options
+      POSITIONAL_ARGS+=("$1") # save positional arg
+      shift
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+TOPDIR=$(dirname $(realpath $0))
+
+export TMPDIR=$(mktemp -d /tmp/eessi-remove.XXXXXXXX)
+
+source $TOPDIR/scripts/utils.sh
+
+echo ">> Determining software subdirectory to use for current build host..."
+if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
+  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
+  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
+else
+  echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
+fi
+
+echo ">> Setting up environment..."
+
+source $TOPDIR/init/bash
+
+if [ -d $EESSI_CVMFS_REPO ]; then
+    echo_green "$EESSI_CVMFS_REPO available, OK!"
+else
+    fatal_error "$EESSI_CVMFS_REPO is not available!"
+fi
+
+if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
+    fatal_error "Failed to determine software subdirectory?!"
+elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then
+    fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!"
+else
+    echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!"
+fi
+
+echo ">> Configuring EasyBuild..."
+EB="eb"
+source $TOPDIR/configure_easybuild
+
+echo ">> Setting up \$MODULEPATH..."
+# make sure no modules are loaded
+module --force purge
+# ignore current $MODULEPATH entirely
+module unuse $MODULEPATH
+module use $EASYBUILD_INSTALLPATH/modules/all
+if [[ -z ${MODULEPATH} ]]; then
+    fatal_error "Failed to set up \$MODULEPATH?!"
+else
+    echo_green ">> MODULEPATH set up: ${MODULEPATH}"
+fi
+
+# assume there's only one diff file that corresponds to the PR patch file
+pr_diff=$(ls [0-9]*.diff | head -1)
+
+# if this script is run as root, use PR patch file to determine if software needs to be removed first
+changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/")
+if [ -z ${changed_easystacks_rebuilds} ]; then
+    echo "No software needs to be removed."
+else
+    for easystack_file in ${changed_easystacks_rebuilds}; do
+        # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
+        eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
+
+        # load EasyBuild module (will be installed if it's not available yet)
+        source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
+
+        if [ -f ${easystack_file} ]; then
+            echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..."
+            # we need to remove existing installation directories first,
+            # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
+            #  * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
+            rebuild_apps=$(eb --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
+            for app in ${rebuild_apps}; do
+                app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
+                app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua
+                echo_yellow "Removing ${app_dir} and ${app_module}..."
+                find ${app_dir} -type d | sed -e 's/^/REMOVE_DIRECTORY /'
+                find ${app_dir} -type f | sed -e 's/^/REMOVE_FILE /'
+                echo "REMOVE_MODULE ${app_module}"
+            done
+        else
+            fatal_error "Easystack file ${easystack_file} not found!"
+        fi
+    done
+fi

EESSI-remove-software.sh

@@ -89,46 +89,35 @@ fi
 # assume there's only one diff file that corresponds to the PR patch file
 pr_diff=$(ls [0-9]*.diff | head -1)
 
-# if this script is run as root, use PR patch file to determine if software needs to be removed first
-# if [ $EUID -eq 0 ]; then
-# working around lacking support for `--fakeroot` and/or user namespaces
-# we only run as non-root
-if [ $EUID -ne 0 ]; then
-    changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/")
-    if [ -z ${changed_easystacks_rebuilds} ]; then
-        echo "No software needs to be removed."
-    else
-        for easystack_file in ${changed_easystacks_rebuilds}; do
-            # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
-            eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
-
-            # load EasyBuild module (will be installed if it's not available yet)
-            source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
-
-            if [ -f ${easystack_file} ]; then
-                echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..."
-                # we need to remove existing installation directories first,
-                # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
-                #  * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
-                # rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
-                # we cannot run as root so we removed `--allow-use-as-root...`
-                rebuild_apps=$(eb --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
-                for app in ${rebuild_apps}; do
-                    app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
-                    app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua
-                    echo_yellow "Removing ${app_dir} and ${app_module}... (just reporting what would have been done)"
-                    # echo_yellow "Removing ${app_dir} and ${app_module}..."
-                    # rm -rf ${app_dir}
-                    # rm -rf ${app_module}
-                done
-            else
-                fatal_error "Easystack file ${easystack_file} not found!"
-            fi
-        done
-    fi
+changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/")
+if [ -z ${changed_easystacks_rebuilds} ]; then
+    echo "No software needs to be removed."
 else
-    fatal_error "This script can NOT be run by root! (lacking support for `--fakeroot` and/or user namespaces)"
+    for easystack_file in ${changed_easystacks_rebuilds}; do
+        # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
+        eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
+
+        # load EasyBuild module (will be installed if it's not available yet)
+        source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
+
+        if [ -f ${easystack_file} ]; then
+            echo_green "Software rebuild(s) requested in ${easystack_file}, so"
+            echo_green "  determining which existing installation have to be removed (assuming contents"
+            echo_green "  have been made writable/deletable)..."
+            # we need to remove existing installation directories first,
+            # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
+            #  * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
+            # rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
+            rebuild_apps=$(eb --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
+            for app in ${rebuild_apps}; do
+                app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
+                app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua
+                echo_yellow "Removing ${app_dir} and ${app_module}..."
+                rm -rdfv ${app_dir}
+                rm -rdfv ${app_module}
+            done
+        else
+            fatal_error "Easystack file ${easystack_file} not found!"
+        fi
+    done
 fi
-# else
-#     fatal_error "This script can only be run by root!"
-# fi

bot/build.sh

@@ -188,6 +188,49 @@ changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed
 if [[ -z "${changed_easystacks_rebuilds}" ]]; then
     echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step."
 else
+    # determine which software packages (and modules) have to be removed
+    TARBALL_TMP_DETERMINE_STEP_DIR=${PREVIOUS_TMP_DIR}/determine_step
+    mkdir -p ${TARBALL_TMP_DETERMINE_STEP_DIR}
+
+    # prepare arguments to eessi_container.sh specific to determine step
+    declare -a DETERMINE_STEP_ARGS=()
+    DETERMINE_STEP_ARGS+=("--save" "${TARBALL_TMP_DETERMINE_STEP_DIR}")
+    DETERMINE_STEP_ARGS+=("--storage" "${STORAGE}")
+
+    # create tmp file for output of determine step
+    determine_outerr=$(mktemp determine.outerr.XXXX)
+
+    echo "Executing command to determine software to be removed:"
+    echo "./eessi_container.sh ${COMMON_ARGS[@]} ${DETERMINE_STEP_ARGS[@]}"
+    echo "                     -- ./EESSI-determine-rebuilds.sh \"${DETERMINE_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${determine_outerr}"
+    ./eessi_container.sh "${COMMON_ARGS[@]}" "${DETERMINE_STEP_ARGS[@]}" \
+                         -- ./EESSI-determine-rebuilds.sh "${DETERMINE_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${determine_outerr}
+
+    # process output file
+    #   for each line containing 'REMOVE_DIRECTORY some_path'
+    #     create a new directory ${STORAGE}/lower_dirs/some_path_stripped
+    #       where the prefix /cvmfs/repo_name is removed from some_path
+    #     set permission of the directory to u+rwx
+    #   for each line containing 'REMOVE_FILE some_file_path'
+    #     touch a new file ${STORAGE}/lower_dirs/some_file_path_stripped
+    #       where the prefix /cvmfs/repo_name is removed from some_file_path
+    #     set permission of the file to u+rw
+
+    LOWER_DIRS="${STORAGE}/lower_dirs"
+    mkdir -p "${LOWER_DIRS}"
+
+    grep ^REMOVE_DIRECTORY ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_dirs
+    cat ${determine_outerr}.rm_dirs | while read remove_dir; do
+        mkdir -p ${STORAGE}/lower_dirs/${remove_dir}
+        chmod u+rwx ${STORAGE}/lower_dirs/${remove_dir}
+    done
+
+    grep ^REMOVE_FILE ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_files
+    cat ${determine_outerr}.rm_files | while read remove_file; do
+        touch ${STORAGE}/lower_dirs/${remove_file}
+        chmod u+rw ${STORAGE}/lower_dirs/${remove_file}
+    done
+
     # prepare directory to store tarball of tmp for removal and build steps
     TARBALL_TMP_REMOVAL_STEP_DIR=${PREVIOUS_TMP_DIR}/removal_step
     mkdir -p ${TARBALL_TMP_REMOVAL_STEP_DIR}
@@ -196,11 +239,9 @@ else
     declare -a REMOVAL_STEP_ARGS=()
     REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_REMOVAL_STEP_DIR}")
     REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}")
-    # add fakeroot option in order to be able to remove software, see:
-    # https://github.com/EESSI/software-layer/issues/312
-    # CURRENTLY NOT SUPPORTED; software packages need to be removed from
-    # CernVM-FS repository first
-    # REMOVAL_STEP_ARGS+=("--fakeroot")
+    if [[ ! -z ${LOWER_DIRS} ]]; then
+        REMOVAL_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}")
+    fi
 
     # create tmp file for output of removal step
     removal_outerr=$(mktemp remove.outerr.XXXX)

easystacks/pilot.nessi.no/2023.06/rebuilds/20240506-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml

@@ -0,0 +1,9 @@
+# 2024.05.06
+# Original matching of files we could ship was not done correctly. We were
+# matching the basename for files (e.g., libcudart.so from libcudart.so.12)
+# rather than the name stub (libcudart)
+# See https://github.com/EESSI/software-layer/pull/559
+easyconfigs:
+  - CUDA-12.1.1.eb:
+      options:
+        accept-eula-for: CUDA

eb_hooks.py

@@ -668,8 +668,8 @@ def post_sanitycheck_cuda(self, *args, **kwargs):
                 full_path = os.path.join(dir_path, filename)
                 # we only really care about real files, i.e. not symlinks
                 if not os.path.islink(full_path):
-                    # check if the current file is part of the allowlist
-                    basename = os.path.splitext(filename)[0]
+                    # check if the current file name stub is part of the allowlist
+                    basename =  filename.split('.')[0]
                     if basename in allowlist:
                         self.log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path)
                     else:

eessi_container.sh

@@ -86,6 +86,11 @@ display_help() {
   echo "  -n | --nvidia MODE     - configure the container to work with NVIDIA GPUs,"
   echo "                           MODE==install for a CUDA installation, MODE==run to"
   echo "                           attach a GPU, MODE==all for both [default: false]"
+  echo "  -o | --lower-dirs DIRS - list of ':' separated directories that are used"
+  echo "                           in front of the default lower dir (CVMFS repo);"
+  echo "                           fuse-overlayfs will merge all lower directories;"
+  echo "                           the option can be used to make certain directories"
+  echo "                           in the CVMFS repo writable [default: none]"
   echo "  -r | --repository CFG  - configuration file or identifier defining the"
   echo "                           repository to use [default: EESSI via"
   echo "                           default container, see --container]"
@@ -119,6 +124,7 @@ FAKEROOT=0
 VERBOSE=0
 STORAGE=
 LIST_REPOS=0
+LOWER_DIRS=
 MODE="shell"
 SETUP_NVIDIA=0
 REPOSITORY="EESSI"
@@ -174,6 +180,10 @@ while [[ $# -gt 0 ]]; do
       NVIDIA_MODE="$2"
       shift 2
       ;;
+    -o|--lower-dirs)
+      LOWER_DIRS="$2"
+      shift 2
+      ;;
     -r|--repository)
       REPOSITORY="$2"
       shift 2
@@ -616,6 +626,14 @@ if [[ "${ACCESS}" == "rw" ]]; then
 
   EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs"
   EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}"
+  if [[ ! -z ${LOWER_DIRS} ]]; then
+    # need to convert ':' in LOWER_DIRS to ',' because bind mounts use ',' as
+    # separator while the lowerdir overlayfs option uses ':'
+    export BIND_PATHS="${BIND_PATHS},${LOWER_DIRS/:/,}"
+    EESSI_WRITABLE_OVERLAY+=" -o lowerdir=${LOWER_DIRS}:/cvmfs_ro/${repo_name}"
+  else
+    EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}"
+  fi
   EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper"
   EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work"
   EESSI_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}"