From 61d70afb366cc206f889aa4fa6b81832838ae188 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Wed, 15 Nov 2023 10:16:30 -0600 Subject: [PATCH 01/22] Remove --deptype=all --- bin/build-spack-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 5c50c4b..2b801c8 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -726,7 +726,7 @@ _maybe_cache_binaries() { ${__debug_spack_buildcache:+-d} \ ${__verbose_spack_buildcache:+-v} \ ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \ - buildcache create --deptype=all \ + buildcache create \ ${buildcache_package_opts[*]:+"${buildcache_package_opts[@]}"} \ ${buildcache_key_opts[*]:+"${buildcache_key_opts[@]}"} \ ${buildcache_rel_arg} "$cache" \ @@ -1347,7 +1347,7 @@ if (( failed )) && (( want_emergency_buildcache )); then \ else \ _cmd $ERROR $PIPE spack \ \${common_spack_opts[*]:+\"\${common_spack_opts[@]}\"} \ - buildcache create --deptype=all \ + buildcache create \ \${buildcache_key_opts[*]:+\"\${buildcache_key_opts[@]}\"} \ \$buildcache_rel_arg --rebuild-index \ \"$working_dir/copyBack/spack-emergency-cache\" \ From 2706819a43bdd6d793274f0f2940647fada593e1 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Wed, 15 Nov 2023 20:27:24 -0600 Subject: [PATCH 02/22] Remove obsolete update-index and keys commands --- bin/make_spack | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/make_spack b/bin/make_spack index 9e11785..1d4da26 100755 --- a/bin/make_spack +++ b/bin/make_spack @@ -20,7 +20,7 @@ install_latest() { hash_pkg_ver=`spack -k buildcache list --long --allarch "$pkg $plat" | tail -1` echo "make_spack: info: latest $pkg is $hash_pkg_ver" hash=`echo $hash_pkg_ver | sed -e 's/ .*//'` - spack -k buildcache install -o -a -m "/$hash" + spack -k buildcache install -o -m "/$hash" else spack -k install "$pkg $plat" fi @@ -146,11 +146,11 @@ create_spack() { ;; esac export plat="target=`uname -i` os=`spack arch --operating-system`" - spack mirror add --scope site fnal $binary_cache - spack buildcache update-index -k fnal - spack mirror add --scope site scisoft $binary_cache_bak - spack buildcache update-index -k scisoft - spack -k buildcache keys --install --trust --force + #spack mirror add --scope site fnal $binary_cache + #spack buildcache update-index -k fnal + #spack mirror add --scope site scisoft $binary_cache_bak + #spack buildcache update-index -k scisoft + #spack -k buildcache keys --install --trust --force add_recipe_repos if $upgrading then From 4144d905732b3c9f7209dc723246191b83b1a3ea Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Wed, 15 Nov 2023 20:32:44 -0600 Subject: [PATCH 03/22] Skip adding mirrors and keys in minimal install --- bin/make_spack | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/bin/make_spack b/bin/make_spack index 1d4da26..2a4d8c2 100755 --- a/bin/make_spack +++ b/bin/make_spack @@ -146,12 +146,17 @@ create_spack() { ;; esac export plat="target=`uname -i` os=`spack arch --operating-system`" - #spack mirror add --scope site fnal $binary_cache - #spack buildcache update-index -k fnal - #spack mirror add --scope site scisoft $binary_cache_bak - #spack buildcache update-index -k scisoft - #spack -k buildcache keys --install --trust --force - add_recipe_repos + if $minimal + then + : + else + spack mirror add --scope site fnal $binary_cache + spack buildcache update-index -k fnal + spack mirror add --scope site scisoft $binary_cache_bak + spack buildcache update-index -k scisoft + spack -k buildcache keys --install --trust --force + fi + add_recipe_repos if $upgrading then spack reindex From a6b99eced5807c54fffc39708aa122ca26638118 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Thu, 16 Nov 2023 13:18:31 -0600 Subject: [PATCH 04/22] Don't use system glib --- templates/packages.yaml.almalinux9 | 5 ----- 1 file changed, 5 deletions(-) diff --git a/templates/packages.yaml.almalinux9 b/templates/packages.yaml.almalinux9 index c74f361..80f0340 100644 --- a/templates/packages.yaml.almalinux9 +++ b/templates/packages.yaml.almalinux9 @@ -136,11 +136,6 @@ packages: - spec: "gdbm @1.19 %gcc@11.3.1 os=almalinux9" prefix: /usr buildable: False - glib: - externals: - - spec: "glib @2.68.4 %gcc@11.3.1 os=almalinux9" - prefix: /usr - buildable: False gettext: externals: - spec: "gettext ~bzip2+curses~git+libunistring+libxml2~tar+xz @0.21 %gcc@11.3.1 os=almalinux9" From 3ef91c19dc70668721a043f6f5d1e9924728ed92 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Mon, 20 Nov 2023 15:56:18 -0600 Subject: [PATCH 05/22] Remove glib on slf7 --- templates/packages.yaml.scientific7 | 5 ----- 1 file changed, 5 deletions(-) diff --git a/templates/packages.yaml.scientific7 b/templates/packages.yaml.scientific7 index 40377c6..d348fed 100644 --- a/templates/packages.yaml.scientific7 +++ b/templates/packages.yaml.scientific7 @@ -156,11 +156,6 @@ packages: externals: - spec: "git @1.8.3.1 %gcc@4.8.5 os=scientific7" prefix: /usr - glib: - externals: - - spec: "glib @2.56.1 %gcc@4.8.5 os=scientific7" - prefix: /usr - buildable: False gmake: externals: - spec: "gmake @3.82 %gcc@4.8.5 os=scientific7" From 834a45931017915c3bf351b40284150e950db03b Mon Sep 17 00:00:00 2001 From: Chris Green Date: Tue, 28 Nov 2023 10:23:17 -0600 Subject: [PATCH 06/22] Support Spack cache type in build-spack-env.sh (#33) * Improve parsing of cache options to allow for cache type * Pass cache type to spack mirror when specified --- bin/build-spack-env.sh | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 2b801c8..0fb5108 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -71,7 +71,7 @@ BRIEF OPTIONS --cache-write-(sources|binaries[= ](all|none|deps|dependencies|(no|non)[_-]roots|roots)) --no-cache-write-(sources|binaries) - --extra-(sources|binaries)-write-cache[= ](\|)?||)(,...)+ + --extra-(sources|binaries)-write-cache[= ](|)(,...)+ --clear-mirrors --color[= ](auto|always|never) --(debug|verbose)-spack-(bootstrap|buildcache|concretize|install) @@ -95,8 +95,8 @@ BRIEF OPTIONS --verbosity[= ](-?[0-9]+|INFO|WARNING|(FATAL_|INTERNAL_)?ERROR|INFO|PROGRESS|DEBUG_[1-9][0-9]*) --no-ups --ups[= ](plain|traditional|unified|-[ptu]) - --with-cache[= ](\|)?||)(,...)+ - --with-concretiz(e|ing|ation)-cache[= ](\|)?||)(,...)+ + --with-cache[= ](\|)?(:)?(|)(,...)+ + --with-concretiz(e|ing|ation)-cache[= ](\|)?(:)?(|)(,...)+ --with-padding --working-dir[= ] @@ -169,7 +169,7 @@ SPACK CONFIGURATION OPTIONS Control whether sources or binary packages are written to local caches under /copyBack. - --extra-(sources|binaries)-write-cache[= ]|)(,...)+ + --extra-(sources|binaries)-write-cache[= ](|)(,...)+ Extra source/binary cache locations for built products. Incompatible with --no-cache-write-(sources|binaries). @@ -178,12 +178,13 @@ SPACK CONFIGURATION OPTIONS Remove bootstrapped mirrors/caches from configuration. - --with-cache[= ](\|)?||)(,...)+ - --with-concretiz(e|ing|ation)-cache[= ](\|)?||)(,...)+ + --with-cache[= ](\|)?(:)?(|)(,...)+ + --with-concretiz(e|ing|ation)-cache[= ](\|)?(:)?(|)(,...)+ Add a read-only mirror/cache. If --safe-concretize is set, added caches will be ignored during the concretizaton process unless the - second form is used. + second form is used. If specified, may be "source," or + "binary." Other Spack Configuration @@ -357,11 +358,12 @@ EOF } _cache_info() { - if [[ "$cache_spec" =~ ^([^|]+)\|(.*)$ ]]; then - cache_name="${BASH_REMATCH[1]}" - cache_url="${BASH_REMATCH[2]}" + if [[ "$cache_spec" =~ ^(([^|]+)\|)?((source|binary):)?(.*)$ ]]; then + cache_name="${BASH_REMATCH[2]:-buildcache_$((++cache_count))}" + cache_type="${BASH_REMATCH[4]}" + cache_url="${BASH_REMATCH[5]}" else - cache_name="buildcache_$((++cache_count))" + _die $EXIT_SPACK_CONFIG_FAILURE "unable to parse cache_spec \"$cache_spec\"" fi } @@ -497,8 +499,8 @@ _configure_spack() { _cache_info "$cache_spec" _cmd $DEBUG_1 spack \ ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \ - mirror add --scope=site "$cache_name" "$cache_url" \ - || _die $EXIT_SPACK_CONFIG_FAILURE "executing spack mirror add --scope=site $cache_name \"$cache_url" + mirror add --scope=site ${cache_type:+--type "${cache_type}"} "$cache_name" "$cache_url" \ + || _die $EXIT_SPACK_CONFIG_FAILURE "executing spack mirror add --scope=site ${cache_type:+--type \"${cache_type}\"} \"$cache_name\" \"$cache_url\"" done # 4. Spack recipe repos. _report $PROGRESS "configuring user-specified recipe repositories" @@ -688,7 +690,7 @@ _make_concretize_mirrors_yaml() { _cache_info "$cache_spec" _cmd $DEBUG_1 spack \ ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \ - mirror add --scope=site $cache_name "$cache_url" || + mirror add --scope=site ${cache_type:+--type "${cache_type}"} "$cache_name" "$cache_url" || _die $EXIT_SPACK_CONFIG_FAILURE \ "unable to add $cache_url to concretization-specific mirrors" done @@ -1212,9 +1214,9 @@ fi # Local cache locations are derived from $working_dir. local_caches=( - "__local_binaries|$working_dir/copyBack/spack-packages/binaries" - "__local_compilers|$working_dir/copyBack/spack-packages/compilers" - "__local_sources|$working_dir/copyBack/spack-packages/sources" + "__local_binaries|binary:$working_dir/copyBack/spack-packages/binaries" + "__local_compilers|binary:$working_dir/copyBack/spack-packages/compilers" + "__local_sources|source:$working_dir/copyBack/spack-packages/sources" ) spack_env_top_dir="$working_dir/spack_env" From 647d808ee729567f2d3b35fa282f8504353cb273 Mon Sep 17 00:00:00 2001 From: Chris Green Date: Tue, 28 Nov 2023 10:23:37 -0600 Subject: [PATCH 07/22] Minor improvements to build-spack-env.sh (#32) * Documentation clarity * Improve trap handling to allow for interruption during copy back * Report when no environments specified --- bin/build-spack-env.sh | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 0fb5108..d93b05d 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -62,7 +62,7 @@ working_dir="${WORKSPACE:=$(pwd)}" usage() { cat < (--)? (|)+ +usage: $prog (--)? [(|)] ... $prog (-[h?]|--help) EOF @@ -560,6 +560,7 @@ _copy_back_logs() { local tar_tmp="$working_dir/copyBack/tmp" local spack_env= env_spec= install_prefix= _report $INFO "end-of-job copy-back..." + trap 'status=$?; _report $INFO "end-of-job copy-back PREEMPTED by signal $((status - 128))"; exit $status' INT mkdir -p "$tar_tmp/"{spack_env,spack-stage} cd "$spack_env_top_dir" _cmd $DEBUG_3 spack clean -dmp @@ -892,8 +893,9 @@ _process_environment() { env create $view_opt $env_name "$env_cfg" \ || _die $EXIT_SPACK_ENV_FAILURE "unable to create environment $env_name from $env_cfg" - # Save logs and attempt to cache successful builds before we're killed. - trap 'interrupt=$?; _report $INFO "user interrupt"; _copy_back_logs' HUP INT QUIT TERM + # Record an intentional stoppage. EXIT trap will take care of + # log/cache preservation. + trap 'interrupt=$?; trap - HUP INT QUIT TERM; _report $INFO "user interrupt"' HUP INT QUIT TERM local is_compiler_env= local is_nonterminal_compiler_env= @@ -1394,13 +1396,17 @@ environment_specs=("$@") num_environments=${#environment_specs[@]} env_idx=0 -#################################### -# Build each specified environment. -for env_cfg in ${environment_specs[*]:+"${environment_specs[@]}"}; do - _report $PROGRESS "processing user-specified environment configuration $env_cfg" - _process_environment "$env_cfg" -done -#################################### +if (( ! num_environments )); then # NOP + _report $INFO "no environment configurations specified: exiting after setup" +else + #################################### + # Build each specified environment. + for env_cfg in ${environment_specs[*]:+"${environment_specs[@]}"}; do + _report $PROGRESS "processing user-specified environment configuration $env_cfg" + _process_environment "$env_cfg" + done + #################################### +fi ### Local Variables: ### mode: sh From 2157d399aa6e1cfc25a1d349db474c2e1f668ac9 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Thu, 23 Nov 2023 20:06:38 -0600 Subject: [PATCH 08/22] Use buildcache create --only options for emergency buildcache too --- bin/build-spack-env.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index d93b05d..7d27d82 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -1352,6 +1352,7 @@ if (( failed )) && (( want_emergency_buildcache )); then \ _cmd $ERROR $PIPE spack \ \${common_spack_opts[*]:+\"\${common_spack_opts[@]}\"} \ buildcache create \ + \${buildcache_package_opts[*]:+\"\${buildcache_package_opts[@]}\"} \ \${buildcache_key_opts[*]:+\"\${buildcache_key_opts[@]}\"} \ \$buildcache_rel_arg --rebuild-index \ \"$working_dir/copyBack/spack-emergency-cache\" \ From 5845957ecdabf4ea98cb82ff312cce1784baf630 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Fri, 1 Dec 2023 08:56:13 -0600 Subject: [PATCH 09/22] Add sync_from_jenkins script --- bin/sync_from_jenkins | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100755 bin/sync_from_jenkins diff --git a/bin/sync_from_jenkins b/bin/sync_from_jenkins new file mode 100755 index 0000000..3f23841 --- /dev/null +++ b/bin/sync_from_jenkins @@ -0,0 +1,26 @@ +#!/bin/bash +curl -OL https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/label1=$1/$2/artifact/copyBack/jenkins-build-spack-env-label1=$1-$2.tar.bz2 +mkdir $1-$2; tar -x -C $1-$2 -f jenkins-build-spack-env-label1=$1-$2.tar.bz2 +#if (curl https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$2/artifact/copyBack/spack-emergency-cache/ 2>/dev/null | grep -q 404 ); then + spack mirror add $1-$2e --type binary https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$1/artifact/copyBack/spack-emergency-cache/ +#fi +#if (curl https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$2/artifact/copyBack/spack-binary-cache/ 2>/dev/null | grep -q 404 ); then + spack mirror add $1-$2b --type binary https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$1/artifact/copyBack/spack-binary-cache/ +#fi +#if (curl https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$2/artifact/copyBack/spack-compiler-cache/ 2>/dev/null | grep -q 404 );then + spack mirror add $1-$2c --type binary https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$1/artifact/copyBack/spack-compiler-cache +#fi +spack mirror list +if [ -d $1-$2/spack_env/var/spack/environments ]; then + for env in $(ls -d $1-$2/spack_env/var/spack/environments/*);do + spack -e $env buildcache sync $1-$2b local-binary-plain + spack -e $env buildcache sync $1-$2e local-binary-plain + done +fi +if [ -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments ]; then + for env in $(ls -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments/*);do + spack -e $env buildcache sync $1-$2b local-binary-unified + spack -e $env buildcache sync $1-$2e local-binary-unified + done +fi + From 2e7e35904c36a430816e3a361c63fe94272b61f4 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Sat, 2 Dec 2023 08:13:58 -0600 Subject: [PATCH 10/22] Copy compiler environment buildcaches to compiler mirror --- bin/sync_from_jenkins | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bin/sync_from_jenkins b/bin/sync_from_jenkins index 3f23841..7536692 100755 --- a/bin/sync_from_jenkins +++ b/bin/sync_from_jenkins @@ -16,11 +16,19 @@ if [ -d $1-$2/spack_env/var/spack/environments ]; then spack -e $env buildcache sync $1-$2b local-binary-plain spack -e $env buildcache sync $1-$2e local-binary-plain done + for env in $(ls -d $1-$2/spack_env/var/spack/environments/{gcc,clang}*);do + spack -e $env buildcache sync $1-$2c local-compiler-plain + spack -e $env buildcache sync $1-$2e local-compiler-plain + done fi if [ -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments ]; then for env in $(ls -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments/*);do spack -e $env buildcache sync $1-$2b local-binary-unified spack -e $env buildcache sync $1-$2e local-binary-unified done + for env in $(ls -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments/{gcc,clang}*);do + spack -e $env buildcache sync $1-$2b local-compiler-unified + spack -e $env buildcache sync $1-$2e local-compiler-unified + done fi From 408d2e6089f455dcbc70630d58302c1719c730de Mon Sep 17 00:00:00 2001 From: Eric Flumerfelt Date: Tue, 5 Dec 2023 11:07:35 -0600 Subject: [PATCH 11/22] Add libtirpc and fix typo. --- templates/packagelist | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/packagelist b/templates/packagelist index 5bb7761..31b5328 100644 --- a/templates/packagelist +++ b/templates/packagelist @@ -54,6 +54,7 @@ libfontenc:libfontenc:$0-devel libICE:libice:$0-devel :libpciaccess: libSM:libsm:$0-devel +libtirpc:libtirpc:$0-devel libtool:libtool:$0 libuuid:libuuid:$0-devel libX11:libx11:$0-devel @@ -101,7 +102,7 @@ xcb-util-image:xcb-util-image:$0-devel xcb-util-keysyms:xcb-util-keysyms:$0-devel xcb-util-renderutil:xcb-util-renderutil:$0-devel xcb-util-wm:xcb-util-wm:$0-devel -xorg-11-util-macros:util-macros:$0 +xorg-x11-util-macros:util-macros:$0 xorg-x11-proto-devel:xextproto:$0 (xorg-x11-server)-Xorg:xorg-server:\1-devel xorg-x11-proto-devel:xproto:$0 From 3268abd1265776955cb5b129fd10b4062387bcf4 Mon Sep 17 00:00:00 2001 From: Chris Green Date: Tue, 5 Dec 2023 17:37:28 -0600 Subject: [PATCH 12/22] Improve variable naming --- bin/build-spack-env.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 7d27d82..4b04a27 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -522,9 +522,9 @@ _configure_spack() { #################################### # Check whether spack buildcache create still needs -r - local buildcache_rel_help="$(spack buildcache create --help | grep -Ee '^[[:space:]]-r\b')" - [ -z "$buildcache_rel_help" ] || - [[ "$buildcache_rel_help" == *"(deprecated)"* ]] || + local buildcache_create_help="$(spack buildcache create --help | grep -Ee '^[[:space:]]-r\b')" + [ -z "$buildcache_create_help" ] || + [[ "$buildcache_create_help" == *"(deprecated)"* ]] || buildcache_rel_arg="-r" #################################### From c1b3a3cc569d8da543b2811ea1fead8b7c2915c6 Mon Sep 17 00:00:00 2001 From: Chris Green Date: Tue, 5 Dec 2023 17:38:15 -0600 Subject: [PATCH 13/22] Detect Spack support of `--type` option to `spack mirror add` --- bin/build-spack-env.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 4b04a27..12c1e6e 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -360,7 +360,7 @@ EOF _cache_info() { if [[ "$cache_spec" =~ ^(([^|]+)\|)?((source|binary):)?(.*)$ ]]; then cache_name="${BASH_REMATCH[2]:-buildcache_$((++cache_count))}" - cache_type="${BASH_REMATCH[4]}" + (( have_mirror_add_type )) && cache_type="${BASH_REMATCH[4]}" cache_url="${BASH_REMATCH[5]}" else _die $EXIT_SPACK_CONFIG_FAILURE "unable to parse cache_spec \"$cache_spec\"" @@ -528,6 +528,12 @@ _configure_spack() { buildcache_rel_arg="-r" #################################### + #################################### + # Check whether spack mirror add supports --type + mirror_add_help="$(spack mirror add --help | grep -Ee '^[[:space:]]--type[[:space:]]+')" + [ -n "$mirror_add_help" ] && have_mirror_add_type=1 + #################################### + #################################### # Make sure we know about compilers. _report $PROGRESS "configuring compilers" From ae7ab189544cf45fb8b22ca80d4cc1b50ff2e51a Mon Sep 17 00:00:00 2001 From: Chris Green Date: Tue, 5 Dec 2023 17:38:42 -0600 Subject: [PATCH 14/22] Support specs for oneapi and dpcpp compilers --- bin/build-spack-env.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 12c1e6e..efcfa07 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -770,6 +770,8 @@ _maybe_register_compiler() { compiler_spec="${compiler_spec/@/@=}" compiler_spec="${compiler_spec/@==/@=}" compiler_build_spec=${compiler_spec/clang/llvm} + compiler_build_spec=${compiler_build_spec/oneapi/intel-oneapi-compilers} + compiler_build_spec=${compiler_build_spec/dpcpp/intel-oneapi-compilers} local compiler_path="$(_cmd $DEBUG_2 $PIPE spack \ -e $env_name \ ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \ From ad2b6b3906d78c1711e5bebbd117e3b64f48e461 Mon Sep 17 00:00:00 2001 From: Chris Green Date: Mon, 11 Dec 2023 17:48:06 -0600 Subject: [PATCH 15/22] Fix typo in documentation --- bin/build-spack-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index efcfa07..ae38e4c 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -306,7 +306,7 @@ CACHING SOURCE AND BINARY PACKAGES `/copyBack/spack-packages/binaries` or `/copyBack/spack-packages/compilers` (as appropriate) after that environment has been built successfully. If - `--cache-write-binaries=no_root` is active, then root packages of + `--cache-write-binaries=no_roots` is active, then root packages of non-compiler environments will not be cached. From 7a5f416bc9982d544da015bfad1e024f4ccb688c Mon Sep 17 00:00:00 2001 From: Chris Green Date: Mon, 11 Dec 2023 17:48:17 -0600 Subject: [PATCH 16/22] Fix indentation --- bin/build-spack-env.sh | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index ae38e4c..4038f9a 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -724,30 +724,30 @@ _maybe_cache_binaries() { if (( ${#hashes_to_cache[@]} )); then local cache= for cache in "$working_dir/copyBack/spack-$binary_mirror-cache" \ - ${extra_sources_write_cache[*]:+"${extra_sources_write_cache[@]}"}; do + ${extra_sources_write_cache[*]:+"${extra_sources_write_cache[@]}"}; do _report $PROGRESS "caching$msg_extra binary packages for environment $env_name to $cache" - for hash in "${hashes_to_cache[@]}";do + for hash in "${hashes_to_cache[@]}"; do if [ -f "$(spack location -i $hash)/.spack/binary_distribution" ]; then - _report $DEBUG_1 "Skipping package installed from buildcache $hash" - else + _report $DEBUG_1 "Skipping package installed from buildcache $hash" + else _cmd $DEBUG_1 $PROGRESS \ - spack \ - ${__debug_spack_buildcache:+-d} \ - ${__verbose_spack_buildcache:+-v} \ - ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \ - buildcache create \ - ${buildcache_package_opts[*]:+"${buildcache_package_opts[@]}"} \ - ${buildcache_key_opts[*]:+"${buildcache_key_opts[@]}"} \ - ${buildcache_rel_arg} "$cache" \ - $hash - fi + spack \ + ${__debug_spack_buildcache:+-d} \ + ${__verbose_spack_buildcache:+-v} \ + ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \ + buildcache create \ + ${buildcache_package_opts[*]:+"${buildcache_package_opts[@]}"} \ + ${buildcache_key_opts[*]:+"${buildcache_key_opts[@]}"} \ + ${buildcache_rel_arg} "$cache" \ + $hash + fi done - _report $PROGRESS "updating build cache index" - _cmd $DEBUG_1 $PROGRESS \ + _report $PROGRESS "updating build cache index" + _cmd $DEBUG_1 $PROGRESS \ spack \ ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \ buildcache update-index -k "$cache" - done + done fi } From 0ac633f2f111662672d1aa5e643007d19a1aa004 Mon Sep 17 00:00:00 2001 From: Chris Green Date: Tue, 12 Dec 2023 10:10:30 -0600 Subject: [PATCH 17/22] `_remove_hash()` can now remove multiple hashes in one pass --- bin/build-spack-env.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 4038f9a..c92f723 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -984,12 +984,14 @@ _quote() { } _remove_hash() { - local hashes_var="$1" handled_hash="$2" + local hashes_var="$1" + shift + local OIFS="$IFS"; IFS=$'\n'; IFS="$OIFS"; handled_hashes=($(echo "$*" | sort -u)); IFS="$OIFS" eval local "hashes=(\${$hashes_var[*]:+\"\${$hashes_var\[@\]}\"})" (( ${#hashes[@]} )) || return local filtered_hashes=() for hash in ${hashes[*]:+"${hashes[@]}"}; do - [ "$handled_hash" = "$hash" ] || filtered_hashes+=("$hash") + _in_sorted_hashlist "$hash" "${handled_hashes[@]}" || filtered_hashes+=("$hash") done eval $hashes_var="(\${filtered_hashes[*]:+\"\${filtered_hashes[@]}\"})" } From 9b38476cd004d0bc10d6df825315eb84ac5e30b0 Mon Sep 17 00:00:00 2001 From: Chris Green Date: Tue, 12 Dec 2023 10:11:47 -0600 Subject: [PATCH 18/22] Filter `non_root_hashes` of root hashes that are also dependencies --- bin/build-spack-env.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index c92f723..185c47c 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -390,6 +390,10 @@ _classify_concretized_specs() { # Sort root hashes for efficient checking. OIFS="$IFS"; IFS=$'\n'; root_hashes=($(echo "${root_hashes[*]}" | sort -u)); IFS="$OIFS" _report $DEBUG_2 "root_hashes=\n ${root_hashes[@]/%/$'\n' }" + # Make sure root hashes that are also dependencies of other roots are + # all removed from non_root_hashes. + _remove_hash non_root_hashes "${root_hashes[@]}" + # Record the number of hashes we need to deal with, and report info. idx=${#hashes[@]} local n_unique=$(IFS=$'\n'; echo "${hashes[*]}" | sort -u | wc -l) _report $DEBUG_1 "examined $specline_idx speclines and found ${#root_hashes[@]} roots and $n_unique unique packages" From f9bb25935cfdfb301c5824bbc4e27e93804a533f Mon Sep 17 00:00:00 2001 From: Chris Green Date: Tue, 12 Dec 2023 10:12:32 -0600 Subject: [PATCH 19/22] Localize `OIFS` where appropriate --- bin/build-spack-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 185c47c..7f573e0 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -388,7 +388,7 @@ _classify_concretized_specs() { levels+=($level) done # Sort root hashes for efficient checking. - OIFS="$IFS"; IFS=$'\n'; root_hashes=($(echo "${root_hashes[*]}" | sort -u)); IFS="$OIFS" + local OIFS="$IFS"; IFS=$'\n'; root_hashes=($(echo "${root_hashes[*]}" | sort -u)); IFS="$OIFS" _report $DEBUG_2 "root_hashes=\n ${root_hashes[@]/%/$'\n' }" # Make sure root hashes that are also dependencies of other roots are # all removed from non_root_hashes. @@ -841,7 +841,7 @@ _piecemeal_build() { _report $DEBUG_2 "finished piecemeal build analysis with $idx entries remaining" # Uniquify hashes. local sorted_buildable_dep_hashes=() - OIFS="$IFS"; IFS=$'\n' + local OIFS="$IFS"; IFS=$'\n' while IFS='' read -r dep_hash; do sorted_buildable_dep_hashes+=("$dep_hash") done < <(echo "${buildable_dep_hashes[*]}" | sort -ut / -k 1nr -k 2,3) From 4d364e40754b68122658ec4ca8493416970758ae Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Tue, 12 Dec 2023 13:51:01 -0600 Subject: [PATCH 20/22] Add compiler projection yaml files for plain and unified UPS layout --- templates/compiler-projections.yaml.plain | 6 ++++++ templates/compiler-projections.yaml.unified | 6 ++++++ 2 files changed, 12 insertions(+) create mode 100644 templates/compiler-projections.yaml.plain create mode 100644 templates/compiler-projections.yaml.unified diff --git a/templates/compiler-projections.yaml.plain b/templates/compiler-projections.yaml.plain new file mode 100644 index 0000000..d7cd273 --- /dev/null +++ b/templates/compiler-projections.yaml.plain @@ -0,0 +1,6 @@ +spack: + config: + install_tree: + projections: + gcc: "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}" + llvm: "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}" diff --git a/templates/compiler-projections.yaml.unified b/templates/compiler-projections.yaml.unified new file mode 100644 index 0000000..b917431 --- /dev/null +++ b/templates/compiler-projections.yaml.unified @@ -0,0 +1,6 @@ +spack: + config: + install_tree: + projections: + gcc: "{package.name}/{version}/{architecture}-{compiler.name}-{compiler.version}" + llvm: "{package.name}/{version}{architecture}-{compiler.name}-{compiler.version}" From 34b31795140298d0e8de390943bce7b34283a253 Mon Sep 17 00:00:00 2001 From: Chris Green Date: Tue, 12 Dec 2023 15:54:04 -0600 Subject: [PATCH 21/22] Remove unneeded escapes from eval --- bin/build-spack-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 7f573e0..36b1496 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -991,7 +991,7 @@ _remove_hash() { local hashes_var="$1" shift local OIFS="$IFS"; IFS=$'\n'; IFS="$OIFS"; handled_hashes=($(echo "$*" | sort -u)); IFS="$OIFS" - eval local "hashes=(\${$hashes_var[*]:+\"\${$hashes_var\[@\]}\"})" + eval local "hashes=(\${$hashes_var[*]:+\"\${$hashes_var[@]}\"})" (( ${#hashes[@]} )) || return local filtered_hashes=() for hash in ${hashes[*]:+"${hashes[@]}"}; do From 9f285b3c017dfd87a748d3ab262dcbcb5b1bd2b0 Mon Sep 17 00:00:00 2001 From: Chris Green Date: Tue, 12 Dec 2023 17:44:22 -0600 Subject: [PATCH 22/22] Use `--only package` for per-package build cache writes --- bin/build-spack-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 36b1496..ec9bc18 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -739,7 +739,7 @@ _maybe_cache_binaries() { ${__debug_spack_buildcache:+-d} \ ${__verbose_spack_buildcache:+-v} \ ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \ - buildcache create \ + buildcache create --only package \ ${buildcache_package_opts[*]:+"${buildcache_package_opts[@]}"} \ ${buildcache_key_opts[*]:+"${buildcache_key_opts[@]}"} \ ${buildcache_rel_arg} "$cache" \