diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 5c50c4b..ec9bc18 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -62,7 +62,7 @@ working_dir="${WORKSPACE:=$(pwd)}" usage() { cat < (--)? (|)+ +usage: $prog (--)? [(|)] ... $prog (-[h?]|--help) EOF @@ -71,7 +71,7 @@ BRIEF OPTIONS --cache-write-(sources|binaries[= ](all|none|deps|dependencies|(no|non)[_-]roots|roots)) --no-cache-write-(sources|binaries) - --extra-(sources|binaries)-write-cache[= ](\|)?||)(,...)+ + --extra-(sources|binaries)-write-cache[= ](|)(,...)+ --clear-mirrors --color[= ](auto|always|never) --(debug|verbose)-spack-(bootstrap|buildcache|concretize|install) @@ -95,8 +95,8 @@ BRIEF OPTIONS --verbosity[= ](-?[0-9]+|INFO|WARNING|(FATAL_|INTERNAL_)?ERROR|INFO|PROGRESS|DEBUG_[1-9][0-9]*) --no-ups --ups[= ](plain|traditional|unified|-[ptu]) - --with-cache[= ](\|)?||)(,...)+ - --with-concretiz(e|ing|ation)-cache[= ](\|)?||)(,...)+ + --with-cache[= ](\|)?(:)?(|)(,...)+ + --with-concretiz(e|ing|ation)-cache[= ](\|)?(:)?(|)(,...)+ --with-padding --working-dir[= ] @@ -169,7 +169,7 @@ SPACK CONFIGURATION OPTIONS Control whether sources or binary packages are written to local caches under /copyBack. - --extra-(sources|binaries)-write-cache[= ]|)(,...)+ + --extra-(sources|binaries)-write-cache[= ](|)(,...)+ Extra source/binary cache locations for built products. Incompatible with --no-cache-write-(sources|binaries). @@ -178,12 +178,13 @@ SPACK CONFIGURATION OPTIONS Remove bootstrapped mirrors/caches from configuration. - --with-cache[= ](\|)?||)(,...)+ - --with-concretiz(e|ing|ation)-cache[= ](\|)?||)(,...)+ + --with-cache[= ](\|)?(:)?(|)(,...)+ + --with-concretiz(e|ing|ation)-cache[= ](\|)?(:)?(|)(,...)+ Add a read-only mirror/cache. If --safe-concretize is set, added caches will be ignored during the concretizaton process unless the - second form is used. + second form is used. If specified, may be "source," or + "binary." Other Spack Configuration @@ -305,7 +306,7 @@ CACHING SOURCE AND BINARY PACKAGES `/copyBack/spack-packages/binaries` or `/copyBack/spack-packages/compilers` (as appropriate) after that environment has been built successfully. If - `--cache-write-binaries=no_root` is active, then root packages of + `--cache-write-binaries=no_roots` is active, then root packages of non-compiler environments will not be cached. @@ -357,11 +358,12 @@ EOF } _cache_info() { - if [[ "$cache_spec" =~ ^([^|]+)\|(.*)$ ]]; then - cache_name="${BASH_REMATCH[1]}" - cache_url="${BASH_REMATCH[2]}" + if [[ "$cache_spec" =~ ^(([^|]+)\|)?((source|binary):)?(.*)$ ]]; then + cache_name="${BASH_REMATCH[2]:-buildcache_$((++cache_count))}" + (( have_mirror_add_type )) && cache_type="${BASH_REMATCH[4]}" + cache_url="${BASH_REMATCH[5]}" else - cache_name="buildcache_$((++cache_count))" + _die $EXIT_SPACK_CONFIG_FAILURE "unable to parse cache_spec \"$cache_spec\"" fi } @@ -386,8 +388,12 @@ _classify_concretized_specs() { levels+=($level) done # Sort root hashes for efficient checking. - OIFS="$IFS"; IFS=$'\n'; root_hashes=($(echo "${root_hashes[*]}" | sort -u)); IFS="$OIFS" + local OIFS="$IFS"; IFS=$'\n'; root_hashes=($(echo "${root_hashes[*]}" | sort -u)); IFS="$OIFS" _report $DEBUG_2 "root_hashes=\n ${root_hashes[@]/%/$'\n' }" + # Make sure root hashes that are also dependencies of other roots are + # all removed from non_root_hashes. + _remove_hash non_root_hashes "${root_hashes[@]}" + # Record the number of hashes we need to deal with, and report info. idx=${#hashes[@]} local n_unique=$(IFS=$'\n'; echo "${hashes[*]}" | sort -u | wc -l) _report $DEBUG_1 "examined $specline_idx speclines and found ${#root_hashes[@]} roots and $n_unique unique packages" @@ -497,8 +503,8 @@ _configure_spack() { _cache_info "$cache_spec" _cmd $DEBUG_1 spack \ ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \ - mirror add --scope=site "$cache_name" "$cache_url" \ - || _die $EXIT_SPACK_CONFIG_FAILURE "executing spack mirror add --scope=site $cache_name \"$cache_url" + mirror add --scope=site ${cache_type:+--type "${cache_type}"} "$cache_name" "$cache_url" \ + || _die $EXIT_SPACK_CONFIG_FAILURE "executing spack mirror add --scope=site ${cache_type:+--type \"${cache_type}\"} \"$cache_name\" \"$cache_url\"" done # 4. Spack recipe repos. _report $PROGRESS "configuring user-specified recipe repositories" @@ -520,12 +526,18 @@ _configure_spack() { #################################### # Check whether spack buildcache create still needs -r - local buildcache_rel_help="$(spack buildcache create --help | grep -Ee '^[[:space:]]-r\b')" - [ -z "$buildcache_rel_help" ] || - [[ "$buildcache_rel_help" == *"(deprecated)"* ]] || + local buildcache_create_help="$(spack buildcache create --help | grep -Ee '^[[:space:]]-r\b')" + [ -z "$buildcache_create_help" ] || + [[ "$buildcache_create_help" == *"(deprecated)"* ]] || buildcache_rel_arg="-r" #################################### + #################################### + # Check whether spack mirror add supports --type + mirror_add_help="$(spack mirror add --help | grep -Ee '^[[:space:]]--type[[:space:]]+')" + [ -n "$mirror_add_help" ] && have_mirror_add_type=1 + #################################### + #################################### # Make sure we know about compilers. _report $PROGRESS "configuring compilers" @@ -558,6 +570,7 @@ _copy_back_logs() { local tar_tmp="$working_dir/copyBack/tmp" local spack_env= env_spec= install_prefix= _report $INFO "end-of-job copy-back..." + trap 'status=$?; _report $INFO "end-of-job copy-back PREEMPTED by signal $((status - 128))"; exit $status' INT mkdir -p "$tar_tmp/"{spack_env,spack-stage} cd "$spack_env_top_dir" _cmd $DEBUG_3 spack clean -dmp @@ -688,7 +701,7 @@ _make_concretize_mirrors_yaml() { _cache_info "$cache_spec" _cmd $DEBUG_1 spack \ ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \ - mirror add --scope=site $cache_name "$cache_url" || + mirror add --scope=site ${cache_type:+--type "${cache_type}"} "$cache_name" "$cache_url" || _die $EXIT_SPACK_CONFIG_FAILURE \ "unable to add $cache_url to concretization-specific mirrors" done @@ -715,30 +728,30 @@ _maybe_cache_binaries() { if (( ${#hashes_to_cache[@]} )); then local cache= for cache in "$working_dir/copyBack/spack-$binary_mirror-cache" \ - ${extra_sources_write_cache[*]:+"${extra_sources_write_cache[@]}"}; do + ${extra_sources_write_cache[*]:+"${extra_sources_write_cache[@]}"}; do _report $PROGRESS "caching$msg_extra binary packages for environment $env_name to $cache" - for hash in "${hashes_to_cache[@]}";do + for hash in "${hashes_to_cache[@]}"; do if [ -f "$(spack location -i $hash)/.spack/binary_distribution" ]; then - _report $DEBUG_1 "Skipping package installed from buildcache $hash" - else + _report $DEBUG_1 "Skipping package installed from buildcache $hash" + else _cmd $DEBUG_1 $PROGRESS \ - spack \ - ${__debug_spack_buildcache:+-d} \ - ${__verbose_spack_buildcache:+-v} \ - ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \ - buildcache create --deptype=all \ - ${buildcache_package_opts[*]:+"${buildcache_package_opts[@]}"} \ - ${buildcache_key_opts[*]:+"${buildcache_key_opts[@]}"} \ - ${buildcache_rel_arg} "$cache" \ - $hash - fi + spack \ + ${__debug_spack_buildcache:+-d} \ + ${__verbose_spack_buildcache:+-v} \ + ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \ + buildcache create --only package \ + ${buildcache_package_opts[*]:+"${buildcache_package_opts[@]}"} \ + ${buildcache_key_opts[*]:+"${buildcache_key_opts[@]}"} \ + ${buildcache_rel_arg} "$cache" \ + $hash + fi done - _report $PROGRESS "updating build cache index" - _cmd $DEBUG_1 $PROGRESS \ + _report $PROGRESS "updating build cache index" + _cmd $DEBUG_1 $PROGRESS \ spack \ ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \ buildcache update-index -k "$cache" - done + done fi } @@ -761,6 +774,8 @@ _maybe_register_compiler() { compiler_spec="${compiler_spec/@/@=}" compiler_spec="${compiler_spec/@==/@=}" compiler_build_spec=${compiler_spec/clang/llvm} + compiler_build_spec=${compiler_build_spec/oneapi/intel-oneapi-compilers} + compiler_build_spec=${compiler_build_spec/dpcpp/intel-oneapi-compilers} local compiler_path="$(_cmd $DEBUG_2 $PIPE spack \ -e $env_name \ ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \ @@ -826,7 +841,7 @@ _piecemeal_build() { _report $DEBUG_2 "finished piecemeal build analysis with $idx entries remaining" # Uniquify hashes. local sorted_buildable_dep_hashes=() - OIFS="$IFS"; IFS=$'\n' + local OIFS="$IFS"; IFS=$'\n' while IFS='' read -r dep_hash; do sorted_buildable_dep_hashes+=("$dep_hash") done < <(echo "${buildable_dep_hashes[*]}" | sort -ut / -k 1nr -k 2,3) @@ -890,8 +905,9 @@ _process_environment() { env create $view_opt $env_name "$env_cfg" \ || _die $EXIT_SPACK_ENV_FAILURE "unable to create environment $env_name from $env_cfg" - # Save logs and attempt to cache successful builds before we're killed. - trap 'interrupt=$?; _report $INFO "user interrupt"; _copy_back_logs' HUP INT QUIT TERM + # Record an intentional stoppage. EXIT trap will take care of + # log/cache preservation. + trap 'interrupt=$?; trap - HUP INT QUIT TERM; _report $INFO "user interrupt"' HUP INT QUIT TERM local is_compiler_env= local is_nonterminal_compiler_env= @@ -972,12 +988,14 @@ _quote() { } _remove_hash() { - local hashes_var="$1" handled_hash="$2" - eval local "hashes=(\${$hashes_var[*]:+\"\${$hashes_var\[@\]}\"})" + local hashes_var="$1" + shift + local OIFS="$IFS"; IFS=$'\n'; IFS="$OIFS"; handled_hashes=($(echo "$*" | sort -u)); IFS="$OIFS" + eval local "hashes=(\${$hashes_var[*]:+\"\${$hashes_var[@]}\"})" (( ${#hashes[@]} )) || return local filtered_hashes=() for hash in ${hashes[*]:+"${hashes[@]}"}; do - [ "$handled_hash" = "$hash" ] || filtered_hashes+=("$hash") + _in_sorted_hashlist "$hash" "${handled_hashes[@]}" || filtered_hashes+=("$hash") done eval $hashes_var="(\${filtered_hashes[*]:+\"\${filtered_hashes[@]}\"})" } @@ -1212,9 +1230,9 @@ fi # Local cache locations are derived from $working_dir. local_caches=( - "__local_binaries|$working_dir/copyBack/spack-packages/binaries" - "__local_compilers|$working_dir/copyBack/spack-packages/compilers" - "__local_sources|$working_dir/copyBack/spack-packages/sources" + "__local_binaries|binary:$working_dir/copyBack/spack-packages/binaries" + "__local_compilers|binary:$working_dir/copyBack/spack-packages/compilers" + "__local_sources|source:$working_dir/copyBack/spack-packages/sources" ) spack_env_top_dir="$working_dir/spack_env" @@ -1347,7 +1365,8 @@ if (( failed )) && (( want_emergency_buildcache )); then \ else \ _cmd $ERROR $PIPE spack \ \${common_spack_opts[*]:+\"\${common_spack_opts[@]}\"} \ - buildcache create --deptype=all \ + buildcache create \ + \${buildcache_package_opts[*]:+\"\${buildcache_package_opts[@]}\"} \ \${buildcache_key_opts[*]:+\"\${buildcache_key_opts[@]}\"} \ \$buildcache_rel_arg --rebuild-index \ \"$working_dir/copyBack/spack-emergency-cache\" \ @@ -1392,13 +1411,17 @@ environment_specs=("$@") num_environments=${#environment_specs[@]} env_idx=0 -#################################### -# Build each specified environment. -for env_cfg in ${environment_specs[*]:+"${environment_specs[@]}"}; do - _report $PROGRESS "processing user-specified environment configuration $env_cfg" - _process_environment "$env_cfg" -done -#################################### +if (( ! num_environments )); then # NOP + _report $INFO "no environment configurations specified: exiting after setup" +else + #################################### + # Build each specified environment. + for env_cfg in ${environment_specs[*]:+"${environment_specs[@]}"}; do + _report $PROGRESS "processing user-specified environment configuration $env_cfg" + _process_environment "$env_cfg" + done + #################################### +fi ### Local Variables: ### mode: sh diff --git a/bin/make_spack b/bin/make_spack index 9e11785..2a4d8c2 100755 --- a/bin/make_spack +++ b/bin/make_spack @@ -20,7 +20,7 @@ install_latest() { hash_pkg_ver=`spack -k buildcache list --long --allarch "$pkg $plat" | tail -1` echo "make_spack: info: latest $pkg is $hash_pkg_ver" hash=`echo $hash_pkg_ver | sed -e 's/ .*//'` - spack -k buildcache install -o -a -m "/$hash" + spack -k buildcache install -o -m "/$hash" else spack -k install "$pkg $plat" fi @@ -146,12 +146,17 @@ create_spack() { ;; esac export plat="target=`uname -i` os=`spack arch --operating-system`" - spack mirror add --scope site fnal $binary_cache - spack buildcache update-index -k fnal - spack mirror add --scope site scisoft $binary_cache_bak - spack buildcache update-index -k scisoft - spack -k buildcache keys --install --trust --force - add_recipe_repos + if $minimal + then + : + else + spack mirror add --scope site fnal $binary_cache + spack buildcache update-index -k fnal + spack mirror add --scope site scisoft $binary_cache_bak + spack buildcache update-index -k scisoft + spack -k buildcache keys --install --trust --force + fi + add_recipe_repos if $upgrading then spack reindex diff --git a/bin/sync_from_jenkins b/bin/sync_from_jenkins new file mode 100755 index 0000000..7536692 --- /dev/null +++ b/bin/sync_from_jenkins @@ -0,0 +1,34 @@ +#!/bin/bash +curl -OL https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/label1=$1/$2/artifact/copyBack/jenkins-build-spack-env-label1=$1-$2.tar.bz2 +mkdir $1-$2; tar -x -C $1-$2 -f jenkins-build-spack-env-label1=$1-$2.tar.bz2 +#if (curl https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$2/artifact/copyBack/spack-emergency-cache/ 2>/dev/null | grep -q 404 ); then + spack mirror add $1-$2e --type binary https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$1/artifact/copyBack/spack-emergency-cache/ +#fi +#if (curl https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$2/artifact/copyBack/spack-binary-cache/ 2>/dev/null | grep -q 404 ); then + spack mirror add $1-$2b --type binary https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$1/artifact/copyBack/spack-binary-cache/ +#fi +#if (curl https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$2/artifact/copyBack/spack-compiler-cache/ 2>/dev/null | grep -q 404 );then + spack mirror add $1-$2c --type binary https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$1/artifact/copyBack/spack-compiler-cache +#fi +spack mirror list +if [ -d $1-$2/spack_env/var/spack/environments ]; then + for env in $(ls -d $1-$2/spack_env/var/spack/environments/*);do + spack -e $env buildcache sync $1-$2b local-binary-plain + spack -e $env buildcache sync $1-$2e local-binary-plain + done + for env in $(ls -d $1-$2/spack_env/var/spack/environments/{gcc,clang}*);do + spack -e $env buildcache sync $1-$2c local-compiler-plain + spack -e $env buildcache sync $1-$2e local-compiler-plain + done +fi +if [ -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments ]; then + for env in $(ls -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments/*);do + spack -e $env buildcache sync $1-$2b local-binary-unified + spack -e $env buildcache sync $1-$2e local-binary-unified + done + for env in $(ls -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments/{gcc,clang}*);do + spack -e $env buildcache sync $1-$2b local-compiler-unified + spack -e $env buildcache sync $1-$2e local-compiler-unified + done +fi + diff --git a/templates/compiler-projections.yaml.plain b/templates/compiler-projections.yaml.plain new file mode 100644 index 0000000..d7cd273 --- /dev/null +++ b/templates/compiler-projections.yaml.plain @@ -0,0 +1,6 @@ +spack: + config: + install_tree: + projections: + gcc: "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}" + llvm: "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}" diff --git a/templates/compiler-projections.yaml.unified b/templates/compiler-projections.yaml.unified new file mode 100644 index 0000000..b917431 --- /dev/null +++ b/templates/compiler-projections.yaml.unified @@ -0,0 +1,6 @@ +spack: + config: + install_tree: + projections: + gcc: "{package.name}/{version}/{architecture}-{compiler.name}-{compiler.version}" + llvm: "{package.name}/{version}{architecture}-{compiler.name}-{compiler.version}" diff --git a/templates/packagelist b/templates/packagelist index 5bb7761..31b5328 100644 --- a/templates/packagelist +++ b/templates/packagelist @@ -54,6 +54,7 @@ libfontenc:libfontenc:$0-devel libICE:libice:$0-devel :libpciaccess: libSM:libsm:$0-devel +libtirpc:libtirpc:$0-devel libtool:libtool:$0 libuuid:libuuid:$0-devel libX11:libx11:$0-devel @@ -101,7 +102,7 @@ xcb-util-image:xcb-util-image:$0-devel xcb-util-keysyms:xcb-util-keysyms:$0-devel xcb-util-renderutil:xcb-util-renderutil:$0-devel xcb-util-wm:xcb-util-wm:$0-devel -xorg-11-util-macros:util-macros:$0 +xorg-x11-util-macros:util-macros:$0 xorg-x11-proto-devel:xextproto:$0 (xorg-x11-server)-Xorg:xorg-server:\1-devel xorg-x11-proto-devel:xproto:$0 diff --git a/templates/packages.yaml.almalinux9 b/templates/packages.yaml.almalinux9 index c74f361..80f0340 100644 --- a/templates/packages.yaml.almalinux9 +++ b/templates/packages.yaml.almalinux9 @@ -136,11 +136,6 @@ packages: - spec: "gdbm @1.19 %gcc@11.3.1 os=almalinux9" prefix: /usr buildable: False - glib: - externals: - - spec: "glib @2.68.4 %gcc@11.3.1 os=almalinux9" - prefix: /usr - buildable: False gettext: externals: - spec: "gettext ~bzip2+curses~git+libunistring+libxml2~tar+xz @0.21 %gcc@11.3.1 os=almalinux9" diff --git a/templates/packages.yaml.scientific7 b/templates/packages.yaml.scientific7 index 40377c6..d348fed 100644 --- a/templates/packages.yaml.scientific7 +++ b/templates/packages.yaml.scientific7 @@ -156,11 +156,6 @@ packages: externals: - spec: "git @1.8.3.1 %gcc@4.8.5 os=scientific7" prefix: /usr - glib: - externals: - - spec: "glib @2.56.1 %gcc@4.8.5 os=scientific7" - prefix: /usr - buildable: False gmake: externals: - spec: "gmake @3.82 %gcc@4.8.5 os=scientific7"