From d27fcf4e516e31fd117497970077ca1d293f403c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 16 Oct 2024 09:55:13 +0200 Subject: [PATCH 01/10] Modify EESSI-install-software.sh so that it loads the EESSI module to set the environment --- EESSI-install-software.sh | 105 ++++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 44 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 44f524ebbc..2e032b35b4 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -112,16 +112,6 @@ fi TMPDIR=$(mktemp -d) -echo ">> Setting up environment..." - -source $TOPDIR/init/minimal_eessi_env - -if [ -d $EESSI_CVMFS_REPO ]; then - echo_green "$EESSI_CVMFS_REPO available, OK!" -else - fatal_error "$EESSI_CVMFS_REPO is not available!" -fi - # make sure we're in Prefix environment by checking $SHELL if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then echo_green ">> It looks like we're in a Gentoo Prefix environment, good!" @@ -129,9 +119,7 @@ else fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!" fi -# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory -export PYTHONPYCACHEPREFIX=$TMPDIR/pycache - +# Get override subdir DETECTION_PARAMETERS='' GENERIC=0 EB='eb' @@ -148,10 +136,68 @@ if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" else echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" - # make sure directory exists (since it's expected by init/eessi_environment_variables when using archdetect) - mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + # Run in a subshell, so that minimal_eessi_env doesn't change the shell environment for the rest of this script + ( + # Make sure EESSI_PREFIX and EESSI_OS_TYPE are set + source $TOPDIR/init/minimal_eessi_env + + # make sure directory exists (since it's expected by init/eessi_environment_variables when using archdetect) + mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + ) fi +echo ">> Setting up environment..." + +# If EESSI_VERSION is not set, source the defaults script to set it +if [ -z ${EESSI_VERSION} ]; then + source $TOPDIR/init/eessi_defaults +fi + +# If module command does not exist, use the one from the compat layer +command -v module +module_cmd_exists=$? +if [[ "$module_cmd_exists" -ne 0 ]]; then + echo_green "No module command found, initializing lmod from the compatibility layer" + # Minimal initalization of the lmod from the compat layer + source $TOPDIR/init/lmod/bash +else + echo_green "Module command found" +fi +ml_version_out=$TMPDIR/ml.out +ml --version &> $ml_version_out +if [[ $? -eq 0 ]]; then + echo_green ">> Found Lmod ${LMOD_VERSION}" +else + fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}" +fi + +# Make sure we start with no modules and clean $MODULEPATH +echo ">> Setting up \$MODULEPATH..." +module --force purge +module unuse $MODULEPATH + +# Initialize the EESSI environment +module use $TOPDIR/init/modules +module load EESSI/$EESSI_VERSION + +if [ -d $EESSI_CVMFS_REPO ]; then + echo_green "$EESSI_CVMFS_REPO available, OK!" +else + fatal_error "$EESSI_CVMFS_REPO is not available!" +fi + +# Check that EESSI_SOFTWARE_SUBDIR now matches EESSI_SOFTWARE_SUBDIR_OVERRIDE +if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then + fatal_error "Failed to determine software subdirectory?!" +elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then + fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!" +else + echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!" +fi + +# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory +export PYTHONPYCACHEPREFIX=$TMPDIR/pycache + # if we run the script for the first time, e.g., to start building for a new # stack, we need to ensure certain files are present in # ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} @@ -170,29 +216,6 @@ if [ ! -f ${_lmod_sitepackage_file} ]; then python3 ${TOPDIR}/create_lmodsitepackage.py ${_eessi_software_path} fi -# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) -# $EESSI_SILENT - don't print any messages -# $EESSI_BASIC_ENV - give a basic set of environment variables -EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables - -if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then - fatal_error "Failed to determine software subdirectory?!" -elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then - fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!" -else - echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!" -fi - -echo ">> Initializing Lmod..." -source $EPREFIX/usr/share/Lmod/init/bash -ml_version_out=$TMPDIR/ml.out -ml --version &> $ml_version_out -if [[ $? -eq 0 ]]; then - echo_green ">> Found Lmod ${LMOD_VERSION}" -else - fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}" -fi - echo ">> Configuring EasyBuild..." source $TOPDIR/configure_easybuild @@ -202,12 +225,6 @@ if [ ! -z "${shared_fs_path}" ]; then export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH} fi -echo ">> Setting up \$MODULEPATH..." -# make sure no modules are loaded -module --force purge -# ignore current $MODULEPATH entirely -module unuse $MODULEPATH - # if an accelerator target is specified, we need to make sure that the CPU-only modules are also still available if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then CPU_ONLY_MODULES_PATH=$(echo $EASYBUILD_INSTALLPATH | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@g")/modules/all From 4d22c13fe7c3b6c9820bf10025cf315a9bd2ef8d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 16 Oct 2024 10:00:06 +0200 Subject: [PATCH 02/10] Also replace source of easybuild configuration by loading the EESSI-extend module --- EESSI-install-software.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 2e032b35b4..68ff4a1927 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -217,7 +217,8 @@ if [ ! -f ${_lmod_sitepackage_file} ]; then fi echo ">> Configuring EasyBuild..." -source $TOPDIR/configure_easybuild +export EESSI_CVMFS_INSTALL=1 +module load EESSI-extend/${EESSI_VERSION}-easybuild if [ ! -z "${shared_fs_path}" ]; then shared_eb_sourcepath=${shared_fs_path}/easybuild/sources From 660697a52edf1afa8162d651934935f8c3ac3a2b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 16 Oct 2024 10:50:08 +0200 Subject: [PATCH 03/10] Add a quick build so we can test the EESSI-install-software.sh script --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml index 5325f2e553..c33d2e1ac6 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml @@ -7,3 +7,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21430 from-commit: 8b509882d03402e2998ff9b22c154a6957e36d6b + - patchelf-0.18.0-GCCcore-13.2.0.eb: From 41f8e028e1269d1b455553257cc66690342467b4 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 16 Oct 2024 16:31:45 +0200 Subject: [PATCH 04/10] Move test if we are in prefix --- EESSI-install-software.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 68ff4a1927..65ab8ec986 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -112,12 +112,6 @@ fi TMPDIR=$(mktemp -d) -# make sure we're in Prefix environment by checking $SHELL -if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then - echo_green ">> It looks like we're in a Gentoo Prefix environment, good!" -else - fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!" -fi # Get override subdir DETECTION_PARAMETERS='' @@ -180,6 +174,14 @@ module unuse $MODULEPATH module use $TOPDIR/init/modules module load EESSI/$EESSI_VERSION +# make sure we're in Prefix environment by checking $SHELL +# We can only do this after loading the EESSI module, as we need ${EPREFIX} +if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then + echo_green ">> It looks like we're in a Gentoo Prefix environment, good!" +else + fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!" +fi + if [ -d $EESSI_CVMFS_REPO ]; then echo_green "$EESSI_CVMFS_REPO available, OK!" else From ab27742e903932214860424ced935b9ef2515f6c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 16 Oct 2024 21:57:42 +0200 Subject: [PATCH 05/10] Unsetting all the other variables that EESSI-extend would consider, we can only set one --- EESSI-install-software.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index add7e7a9be..0cb05efddc 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -221,6 +221,9 @@ if [ ! -f ${_lmod_sitepackage_file} ]; then fi echo ">> Configuring EasyBuild..." +unset EESSI_USER_INSTALL +unset EESSI_PROJECT_INSTALL +unset EESSI_SITE_INSTALL export EESSI_CVMFS_INSTALL=1 module load EESSI-extend/${EESSI_VERSION}-easybuild From c374bf451e11285eef343ed596f11c1c91bd7598 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 21 Oct 2024 15:17:44 +0200 Subject: [PATCH 06/10] Make sure to use the site installation path for install_cuda_libraries.sh --- scripts/gpu_support/nvidia/install_cuda_and_libraries.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index f9d889c1a1..69b11e26da 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -109,6 +109,13 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do # Load EasyBuild version for this easystack file _before_ loading EESSI-extend module avail EasyBuild module load EasyBuild/${eb_version} + # Make sure EESSI-extend does a site install here + # We need to reload it with the current environment variables set + unset EESSI_CVMFS_INSTALL + unset EESSI_PROJECT_INSTALL + unset EESSI_USER_INSTALL + export EESSI_SITE_INSTALL=1 + module unload EESSI-extend module load EESSI-extend/${EESSI_VERSION}-easybuild # Install modules in hidden .modules dir to keep track of what was installed before From 9c48e836aa4464d04f69922dbddeb8ec14d72802 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 21 Oct 2024 15:21:00 +0200 Subject: [PATCH 07/10] Make sure we reload EESSI-extend with the correct env vars set, since we want to install in the CVMFS prefix --- EESSI-install-software.sh | 85 +++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 39 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 0cb05efddc..e3b31d25e8 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -220,45 +220,6 @@ if [ ! -f ${_lmod_sitepackage_file} ]; then python3 ${TOPDIR}/create_lmodsitepackage.py ${_eessi_software_path} fi -echo ">> Configuring EasyBuild..." -unset EESSI_USER_INSTALL -unset EESSI_PROJECT_INSTALL -unset EESSI_SITE_INSTALL -export EESSI_CVMFS_INSTALL=1 -module load EESSI-extend/${EESSI_VERSION}-easybuild - -if [ ! -z "${shared_fs_path}" ]; then - shared_eb_sourcepath=${shared_fs_path}/easybuild/sources - echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path" - export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH} -fi - -# if an accelerator target is specified, we need to make sure that the CPU-only modules are also still available -if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then - CPU_ONLY_MODULES_PATH=$(echo $EASYBUILD_INSTALLPATH | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@g")/modules/all - if [ -d ${CPU_ONLY_MODULES_PATH} ]; then - module use ${CPU_ONLY_MODULES_PATH} - else - fatal_error "Derived path to CPU-only modules does not exist: ${CPU_ONLY_MODULES_PATH}" - fi -fi - -# If in dev.eessi.io, allow building on top of softw -if [[ "${EESSI_CVMFS_REPO}" == /cvmfs/dev.eessi.io ]]; then - module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all -fi - -module use $EASYBUILD_INSTALLPATH/modules/all - -if [[ -z ${MODULEPATH} ]]; then - fatal_error "Failed to set up \$MODULEPATH?!" -else - echo_green ">> MODULEPATH set up: ${MODULEPATH}" -fi - -# assume there's only one diff file that corresponds to the PR patch file -pr_diff=$(ls [0-9]*.diff | head -1) - # install any additional required scripts # order is important: these are needed to install a full CUDA SDK in host_injections # for now, this just reinstalls all scripts. Note the most elegant, but works @@ -301,6 +262,52 @@ if command_exists "nvidia-smi"; then ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh fi + +echo ">> Configuring EasyBuild..." + +# Make sure that we use the EESSI_CVMFS_INSTALL +# Since the path is set when loading EESSI-extend, we reload it to make sure it works - even if it is already loaded +# Note we need to do this after running install_cuda_and_libraries, since that does installations in the EESSI_SITE_INSTALL +unset EESSI_USER_INSTALL +unset EESSI_PROJECT_INSTALL +unset EESSI_SITE_INSTALL +export EESSI_CVMFS_INSTALL=1 +module unload EESSI-extend +module load EESSI-extend/${EESSI_VERSION}-easybuild + +if [ ! -z "${shared_fs_path}" ]; then + shared_eb_sourcepath=${shared_fs_path}/easybuild/sources + echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path" + export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH} +fi + +# if an accelerator target is specified, we need to make sure that the CPU-only modules are also still available +if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then + CPU_ONLY_MODULES_PATH=$(echo $EASYBUILD_INSTALLPATH | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@g")/modules/all + if [ -d ${CPU_ONLY_MODULES_PATH} ]; then + module use ${CPU_ONLY_MODULES_PATH} + else + fatal_error "Derived path to CPU-only modules does not exist: ${CPU_ONLY_MODULES_PATH}" + fi +fi + +# If in dev.eessi.io, allow building on top of softw +if [[ "${EESSI_CVMFS_REPO}" == /cvmfs/dev.eessi.io ]]; then + module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all +fi + +module use $EASYBUILD_INSTALLPATH/modules/all + +if [[ -z ${MODULEPATH} ]]; then + fatal_error "Failed to set up \$MODULEPATH?!" +else + echo_green ">> MODULEPATH set up: ${MODULEPATH}" +fi + +# assume there's only one diff file that corresponds to the PR patch file +pr_diff=$(ls [0-9]*.diff | head -1) + + # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing') if [ -z "${changed_easystacks}" ]; then From ad42fad480c3b49a373069e2f323895bf0aa23e3 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 22 Oct 2024 08:17:06 +0200 Subject: [PATCH 08/10] Remove patchelf, add util-linux. Patchelf is already in the stack, so not suitable as a test package --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml | 1 - .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml index c33d2e1ac6..5325f2e553 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml @@ -7,4 +7,3 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21430 from-commit: 8b509882d03402e2998ff9b22c154a6957e36d6b - - patchelf-0.18.0-GCCcore-13.2.0.eb: diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 170a639064..afeed2404e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -9,3 +9,4 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21600 from-commit: 9b12318bcff1749781d9eb71c23e21bc3a79ed01 + - util-linux-2.39-GCCcore-13.2.0.eb: From ce5f2c8640b60d23cdebb527f4045549c0e1829e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 22 Oct 2024 09:09:09 +0200 Subject: [PATCH 09/10] Remove util-linux again, this was just to prove the PR worked --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index afeed2404e..170a639064 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -9,4 +9,3 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21600 from-commit: 9b12318bcff1749781d9eb71c23e21bc3a79ed01 - - util-linux-2.39-GCCcore-13.2.0.eb: From abf1aff4146f773a0c9e9a92f0c6eaf480e09a72 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Tue, 22 Oct 2024 23:21:22 +0200 Subject: [PATCH 10/10] Update EESSI-install-software.sh Co-authored-by: ocaisa --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index e3b31d25e8..65c43d5ac5 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -291,7 +291,7 @@ if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then fi fi -# If in dev.eessi.io, allow building on top of softw +# If in dev.eessi.io, allow building on top of software.eessi.io if [[ "${EESSI_CVMFS_REPO}" == /cvmfs/dev.eessi.io ]]; then module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all fi