Skip to content

Commit

Permalink
Merge branch '2023.06-software.eessi.io' into eessi-2023.06-Cassiopei…
Browse files Browse the repository at this point in the history
…a/2.0.0-foss/2023a
  • Loading branch information
bedroge authored Nov 11, 2024
2 parents 6caca55 + e6fbee6 commit d12b9ea
Show file tree
Hide file tree
Showing 7 changed files with 77 additions and 43 deletions.
58 changes: 58 additions & 0 deletions .github/workflows/scripts/only_latest_easystacks.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#!/bin/bash
#
# This script figures out the latest version of EasyBuild being used for the installation of easystack
# files.
#
# This file is part of the EESSI software layer, see
# https://github.com/EESSI/software-layer.git
#
# author: Alan O'Cais (CECAM)
#
# license: GPLv2
#

EESSI_VERSION=${EESSI_VERSION:-"2023.06"}

directory="easystacks/software.eessi.io/${EESSI_VERSION}"
# List of example filenames
files=($(find "$directory" -name "*.yml" | grep -e '-eb-'))
[ -n "$DEBUG" ] && echo "${files[@]}"

versions=()
# Loop over each filename
for filename in "${files[@]}"; do
# Extract the semantic version using grep
version=$(echo "$filename" | grep -oP '(?<=eb-)\d+\.\d+\.\d+?(?=-)')

# Output the result
[ -n "$DEBUG" ] && echo "Filename: $filename"
[ -n "$DEBUG" ] && echo "Extracted version: $version"
[ -n "$DEBUG" ] && echo
versions+=("$version")
done
highest_version=$(printf "%s\n" "${versions[@]}" | sort -V | tail -n 1)

[ -n "$DEBUG" ] && echo "Highest version: $highest_version"
[ -n "$DEBUG" ] && echo
[ -n "$DEBUG" ] && echo "Matching files:"
all_latest_easystacks=($(find $directory -type f -name "*eb-$highest_version*.yml"))

accel_latest_easystacks=()
cpu_latest_easystacks=()

# Loop through the array and split based on partial matching of string
accel="/accel/"
for item in "${all_latest_easystacks[@]}"; do
if [[ "$item" == *"$accel"* ]]; then
accel_latest_easystacks+=("$item")
else
cpu_latest_easystacks+=("$item")
fi
done

# Output the results
if [ -n "$ACCEL_EASYSTACKS" ]; then
echo "${accel_latest_easystacks[@]}"
else
echo "${cpu_latest_easystacks[@]}"
fi
21 changes: 13 additions & 8 deletions .github/workflows/test-software.eessi.io.yml
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,20 @@ jobs:
env | grep ^EESSI | sort
# first check the CPU-only builds for this CPU target
echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)"
for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do
echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)"
for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do
if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then
if grep -q 2022b <<<"${easystack_file}"; then
# skip the check of installed software on zen4 for foss/2022b builds
continue
elif grep -q CUDA <<<"${easystack_file}"; then
# skip the check of install CUDA software in the CPU path for zen4
continue
# skip the check of installed software on zen4 for foss/2022b builds
continue
fi
if [[ $easystack_file == *"rebuilds"* ]]; then
# Also handle rebuilds, make a temporary EasyStack file where we clean out all 2022b stuff and use that
new_easystack=$(mktemp pruned_easystackXXX --suffix=.yml)
# first clean out the options then clean out the .eb name
sed '/2022b\|12\.2\.0/,/\.eb/{/\.eb/!d}' "${easystack_file}" | sed '/2022b\|12\.2\.0/d' > $new_easystack
diff --unified=0 "$easystack_file" "$new_easystack" || :
easystack_file="$new_easystack"
fi
fi
echo "check missing installations for ${easystack_file}..."
Expand All @@ -82,7 +87,7 @@ jobs:
for accel in ${accelerators}; do
module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all
echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}"
for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/accel/$(dirname ${accel})/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do
for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do
echo "check missing installations for ${easystack_file}..."
./check_missing_installations.sh ${easystack_file}
ec=$?
Expand Down
36 changes: 1 addition & 35 deletions check_missing_installations.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,17 +25,6 @@ easystack=$1

LOCAL_TMPDIR=$(mktemp -d)

# Clone the develop branch of EasyBuild and use that to search for easyconfigs

if [[ -z ${EASYBUILD_ROBOT_PATHS} ]]; then
git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs
export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs
fi

# All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr
tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack})
grep -v from-pr ${easystack} > ${tmp_easystack}

source $TOPDIR/scripts/utils.sh

source $TOPDIR/configure_easybuild
Expand All @@ -45,34 +34,11 @@ ${EB:-eb} --show-config

echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..."
eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out}
${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out}
exit_code=${PIPESTATUS[0]}

ok_msg="Command 'eb --missing ...' succeeded, analysing output..."
fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'"
if [ "$exit_code" -ne 0 ] && [ ! -z "$pr_exceptions" ]; then
# We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR
# to software-layer, and see if then it passes. If so, we can report a more specific fail_msg
# Note that if no --from-pr's were used in this PR, $pr_exceptions will be empty and we might as
# well skip this check - unmerged PRs can not be the reason for the non-zero exit code in that scenario

# Let's use awk so we can allow for exceptions if we are given a PR diff file
awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack"
awk_command=${awk_command/\\/} # Strip out the backslash we needed for !
eval ${awk_command/EXCEPTIONS/$pr_exceptions} > ${tmp_easystack}

msg=">> Checking for missing installations in ${EASYBUILD_INSTALLPATH},"
msg="${msg} allowing for --from-pr's that were added in this PR..."
echo ${msg}
eb_missing_out=$LOCAL_TMPDIR/eb_missing_with_from_pr.out
${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out}
exit_code_with_from_pr=${PIPESTATUS[0]}

# If now we succeeded, the reason must be that we originally stripped the --from-pr's
if [ "$exit_code_with_from_pr" -eq 0 ]; then
fail_msg="$fail_msg (are you sure all PRs referenced have been merged in EasyBuild?)"
fi
fi

check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ easyconfigs:
options:
# see https://github.com/easybuilders/easybuild-easyconfigs/pull/21650
from-commit: 109998f6adcda7efb4174b1e5f73b41ee82d1f13
- Solids4foam-2.1-foss-2023a.eb:
options:
# see https://github.com/easybuilders/easybuild-easyconfigs/pull/21606
from-commit: 63562c58acf1be64407192b6862c3bd80253d2e0
- Cassiopeia-2.0.0-foss-2023a.eb:
options:
# see https://github.com/easybuilders/easybuild-easyconfigs/pull/21657
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@ easyconfigs:
options:
# see https://github.com/easybuilders/easybuild-easyblocks/pull/3496
include-easyblocks-from-commit: 60633b0acfd41a0732992d9e16800dae71a056eb
- Cython-3.0.10-GCCcore-13.2.0.eb

0 comments on commit d12b9ea

Please sign in to comment.