Skip to content

Commit

Permalink
merge
Browse files Browse the repository at this point in the history
  • Loading branch information
@marcmengel
marcmengel committed May 15, 2024
2 parents 3e7e5fd + c22a0fa commit 6fe8951
Show file tree
Hide file tree
Showing 11 changed files with 324 additions and 184 deletions.
13 changes: 10 additions & 3 deletions bin/bootstrap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,15 @@ default_fermi_spack_tools_repo=https://github.com/FNALssi/fermi-spack-tools.git

parse_args() {
with_padding=""
eval set : $(getopt --longoptions with_padding,help,query-packages,fermi_spack_tools_release:,spack_release:,spack_repo:,fermi_spack_tools_repo: -- x "$@")
shift
if x=$(getopt --longoptions with_padding,help,query-packages,fermi_spack_tools_release:,spack_release:,spack_repo:,fermi_spack_tools_repo: -- x "$@")
then
eval set : $x
shift
else
usage
exit 1
fi

spack_repo=$default_spack_repo
fermi_spack_tools_repo=$default_fermi_spack_tools_repo
ver=$default_fermi_spack_tools_version
Expand Down Expand Up @@ -115,7 +122,7 @@ main() {
PATH=$dest/fermi-spack-tools/$ver/NULL/bin:$PATH

message "Setting up with make_spack"
make_spack $query_packages --spack_release $spack_version --spack_repo $spack_repo $with_padding --minimal -u $dest
make_spack --verbose $query_packages --spack_release $spack_version --spack_repo $spack_repo $with_padding --minimal -u $dest

message "Finding compilers"
source $dest/setup-env.sh
Expand Down
316 changes: 178 additions & 138 deletions bin/build-spack-env.sh
View file

Large diffs are not rendered by default.

27 changes: 20 additions & 7 deletions bin/make_packages_yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,32 @@ x*/*) dir=`echo $0 | sed -e 's;\(.*\)/\([^/]*\);\1/..;'` ;;
x*) dir=.. ;;
esac

src=$dir/templates/packagelist
want_cvmfs=false
while :
do
case "x$1" in
x--help|x)
usage() {
echo "Usage: make_packages_yaml [options] spack_root [os]"
echo " options:"
echo " --help print this message"
echo " --with-externals include /cvmfs/fermilab.opensciencegrid.org/packages/external packages"
echo " --with-cvmfs=/a/b/c include other cvmfs externals"
echo " --debug turn on debugging messages"
echo " --packagelist=/a/b/c use custom package list"
}

src=$dir/templates/packagelist
want_cvmfs=false
if x=$(getopt --longoptions --long-options help,with-externals,with-cvmfs,debug,packagelist: -- "$@")
then
eval set : $x
shift
else
usage
exit 1
fi

while :
do
case "x$1" in
x--help|x)
usage
exit 0
;;
x--with-externals)
Expand Down Expand Up @@ -58,7 +71,7 @@ fi
dst=$1/etc/spack/${sd}${os}/packages.yaml

# packages to make not buildable
force_system='^(bdftopcf|damageproto|diffutils|expat|findutils|font-util|gdbm|gettext|libc|libfontenc|libice|libx11|libxau|libxcb|libxdamage|libxdmcp|libxext|libxfixes|libxfont|libxkbcommon|libxmu|libxpm|libxrandr|libxrender|libxshmfence|libxt|libxv|libxvmc|libxxf86vm|mesa-glu|mkfontdir|mkfontscale|openssl|pkg-config|pkgconf|tar|tcl|tk|xcb-util-(image|keysyms|renderutil|wm)|xextproto|xorg-server|xproto|xproxymanagementprotocol|xrandr|xtrans|zlib)$'
force_system='^(bdftopcf|damageproto|diffutils|expat|findutils|font-util|gdbm|gettext|libc|libfontenc|libice|libx11|libxau|libxcb|libxdamage|libxdmcp|libxext|libxfixes|libxfont|libxkbcommon|libxmu|libxpm|libxrandr|libxrender|libxshmfence|libxt|libxv|libxvmc|libxxf86vm|mesa-glu|mkfontdir|mkfontscale|motif|openssl|pkg-config|pkgconf|tar|tcl|tk|xcb-util-(image|keysyms|renderutil|wm)|xextproto|xorg-server|xproto|xproxymanagementprotocol|xrandr|xtrans|zlib)$'

# packages to force target (instead of merely preferred)
force_x86_64='^(hwloc|libpciaccess|libsigsegv)$'
Expand Down
64 changes: 45 additions & 19 deletions bin/make_spack
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,20 @@ bootstrap_patchelf() {
}

find_compilers() {
source $SPACK_ROOT/share/spack/setup-env.sh
spack compiler find --scope=site
source $SPACK_ROOT/share/spack/setup-env.sh
spack compiler find --scope=site

if [ "$(spack arch --o)" = "almalinux9" ]
then
# deal with almalinux 11.3.1/11.4.1 compiler confusion
# put add 11.3.1 in as also living in /usr/bin...
cf=$SPACK_ROOT/etc/spack/compilers.yaml
sed -e '1d' -e 's/11.4.1/11.3.1/' < $cf >> $cf
fi
}

bootstrap_spack() {
bootstrap_patchelf
# bootstrap_patchelf
find_compilers
if echo $spack_release | grep rollout > /dev/null
then
Expand All @@ -95,14 +103,17 @@ bootstrap_spack() {
then
make_packages_yaml $SPACK_ROOT
else
if [ "$(spack --version)" '>' "0.20.0" ]
spack_version="$(spack --version)"
if [ "$spack_version" '>' "0.20.0" ]
then
sd="$(spack arch --platform)/"
else
sd=""
fi
echo "Installing stock packages.yaml in $SPACK_ROOT/etc/spack/${sd}${os}"
cp $spackbindir/../templates/packages.yaml.${os} $SPACK_ROOT/etc/spack/${sd}${os}/packages.yaml
mkdir -p "$SPACK_ROOT/etc/spack/${sd}${os}" &&
cp -v "$spackbindir/../templates/packages.yaml.${os}" \
"$SPACK_ROOT/etc/spack/${sd}${os}/packages.yaml"
fi
}

Expand Down Expand Up @@ -135,7 +146,7 @@ create_spack() {
*github.com*) args="--depth 4" ;;
*) args="";;
esac
git clone $args -b $spack_release $spack_repo $b
git clone $args --depth 2 -b $spack_release $spack_repo $b

if cd $b
then
Expand All @@ -154,17 +165,17 @@ create_spack() {
;;
esac
export plat="target=`uname -i` os=`spack arch --operating-system`"
if $minimal
then
:
else
spack mirror add --scope site fnal $binary_cache
spack buildcache update-index -k fnal
spack mirror add --scope site scisoft $binary_cache_bak
spack buildcache update-index -k scisoft
spack -k buildcache keys --install --trust --force
fi
spack mirror add --scope site fnal $binary_cache
spack mirror add --scope site scisoft $binary_cache_bak
spack gpg list
# sometimes the buildcache keys doesn't work, so try
# a second timde if it fails?
spack buildcache keys --install --trust --force ||
spack buildcache keys --install --trust --force
spack gpg list

add_recipe_repos

if $upgrading
then
spack reindex
Expand Down Expand Up @@ -210,6 +221,7 @@ usage(){
echo " -t fermi 'traditional' layout"
echo " -p plain spack default layout"
echo " -m minimal setup"
echo " -v verbose flag"
echo ""
echo " make a spack instance with given layout"
echo " --query-packages turns on running make_packages_yaml (slow)"
Expand All @@ -231,9 +243,17 @@ parse_args() {
layout=unified
query_packages=false
repovers=""
verbose=false

eval set : $(getopt --longoptions help,depth,with_padding,upgrade,spack_release:,minimal,no_buildcache,spack_repo:,repover:,query-packages --options mptu -- : "$@")
shift
origargs="$*"
if x=$(getopt --longoptions help,depth,with_padding,upgrade,spack_release:,minimal,no_buildcache,repover,spack_repo:,query-packages --options mptuv -- : "$@")
then
eval set : $x
shift
else
usage
exit 1
fi
while echo x$1 | grep x- > /dev/null
do
case "x$1" in
Expand All @@ -250,6 +270,7 @@ parse_args() {
x-t) layout=traditional; shift;;
x-p) layout=plain; shift;;
x-m) minimal=true; shift;;
x-v) verbose=true; shift;;
x--) shift; break;;
esac
done
Expand All @@ -260,6 +281,11 @@ parse_args() {
usage
exit 1
fi
if $verbose
then
echo "$(date --iso-8601=seconds) Starting make_spack $origargs"
set -x
fi
}

main() {
Expand Down Expand Up @@ -321,7 +347,7 @@ main() {
export PATH=$SPACK_ROOT/bin:$PATH
create_spack

if [ -n "$dir" ]; then
if [ -n "$dir" ] && [ -d "$dir/spack" ]; then
rm -f $dir/spack/current
ln -s $dir/spack/$spack_release $dir/spack/current
fi
Expand Down
15 changes: 9 additions & 6 deletions bin/make_subspack
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,14 @@ parse_args() {
verbose=false
plain=false

eval set : $(getopt --longoptions with_padding,spack_release:,spack_repo,local_env:,dev_pkgs: --options vtup -- "$@")
shift
if x=$(getopt --longoptions with_padding,spack_release:,spack_repo,local_env:,dev_pkgs: --options vtup -- "$@")
then
eval set : $x
shift
else
usage
exit 1
fi

while echo x$1 | grep x- > /dev/null
do
Expand Down Expand Up @@ -188,10 +194,7 @@ test -d $SPACK_ROOTd || mkdir -p $SPACK_ROOTd
message git clone-ing spack

cd $SPACK_ROOTd
case "$spack_root" in
*github.com*) args="--depth 4" ;;
*) args="";;
esac
args="--depth 2"

# default to just doing a fs-based clone, which saves lots of space and time
if [ $spack_repo = default ]
Expand Down
40 changes: 40 additions & 0 deletions bin/sync_from_jenkins_local
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/bin/bash
curl -OL https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/label1=$1/$2/artifact/copyBack/jenkins-build-spack-env-label1=$1-$2.tar.bz2
mkdir $1-$2; tar -x -C $1-$2 -f jenkins-build-spack-env-label1=$1-$2.tar.bz2
#if (curl https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$2/artifact/copyBack/spack-emergency-cache/ 2>/dev/null | grep -q 404 ); then
spack mirror add $1-$2e --type binary https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$1/artifact/copyBack/spack-emergency-cache/
#fi
#if (curl https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$2/artifact/copyBack/spack-binary-cache/ 2>/dev/null | grep -q 404 ); then
spack mirror add $1-$2b --type binary https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$1/artifact/copyBack/spack-binary-cache/
#fi
#if (curl https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$2/artifact/copyBack/spack-compiler-cache/ 2>/dev/null | grep -q 404 );then
spack mirror add $1-$2c --type binary https://buildmaster.fnal.gov/buildmaster/view/spack/job/build-spack-env/$2/label1=$1/artifact/copyBack/spack-compiler-cache
#fi
spack mirror list
if [ -d $1-$2/spack_env/var/spack/environments ]; then
mkdir local-binary-plain
spack mirror add local-binary-plain $PWD/local-binary-plain
for env in $(ls -d $1-$2/spack_env/var/spack/environments/*);do
spack -e $env buildcache sync $1-$2b local-binary-plain
spack -e $env buildcache sync $1-$2e local-binary-plain
spack buildcache update-index local-binary-plain
done
mkdir local-compiler-plain
spack mirror add local-compiler-plain $PWD/local-compiler-plain
for env in $(ls -d $1-$2/spack_env/var/spack/environments/{gcc,clang}*);do
spack -e $env buildcache sync $1-$2c local-compiler-plain
done
spack buildcache update-index local-compiler-plain
fi
if [ -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments ]; then
for env in $(ls -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments/*);do
spack -e $env buildcache sync $1-$2b local-binary-unified
spack -e $env buildcache sync $1-$2e local-binary-unified
done
spack buildcache update-index local-binary-unified
for env in $(ls -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments/{gcc,clang}*);do
spack -e $env buildcache sync $1-$2b local-compiler-unified
done
spack buildcache update-index local-compiler-unified
fi

6 changes: 4 additions & 2 deletions bin/sync_from_jenkins → bin/sync_from_jenkins_scisoft
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,21 @@ if [ -d $1-$2/spack_env/var/spack/environments ]; then
spack -e $env buildcache sync $1-$2b local-binary-plain
spack -e $env buildcache sync $1-$2e local-binary-plain
done
spack buildcache update index local-binary-plain
for env in $(ls -d $1-$2/spack_env/var/spack/environments/{gcc,clang}*);do
spack -e $env buildcache sync $1-$2c local-compiler-plain
spack -e $env buildcache sync $1-$2e local-compiler-plain
done
spack buildcache update index local-compiler-plain
fi
if [ -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments ]; then
for env in $(ls -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments/*);do
spack -e $env buildcache sync $1-$2b local-binary-unified
spack -e $env buildcache sync $1-$2e local-binary-unified
done
spack buildcache update index local-binary-unified
for env in $(ls -d $1-$2/spack_env/spack/v0.21.0-fermi/NULL/var/spack/environments/{gcc,clang}*);do
spack -e $env buildcache sync $1-$2b local-compiler-unified
spack -e $env buildcache sync $1-$2e local-compiler-unified
done
spack buildcache update index local-compiler-unified
fi

16 changes: 7 additions & 9 deletions bin/ups_to_spack
View file
Original file line number Diff line number Diff line change
Expand Up @@ -457,11 +457,7 @@ class package:
tf_2 = re.sub(".*-2.6", "scientificfermi5", tf_2)
tf_2 = re.sub("^[0-9].*", "scientificfermi5", tf_2)

# excessive intel-centrism...
if f_os.find("64bit") > 0:
tf_3 = "x86_64_v2"
else:
tf_3 = "x86"
tf_3 = "x86_64_v2"

logging.debug("spack_arch(%s) -> %s-%s-%s" % (self.flav, tf_1, tf_2, tf_3))
return "%s-%s-%s" % (tf_1, tf_2, tf_3)
Expand Down Expand Up @@ -930,7 +926,7 @@ endif()
# check that we have a ups_to_spack repository...
if not os.access(uts_repo_yaml, os.R_OK):
os.system("spack repo create %s ups_to_spack" % uts_repo)
os.system("spack repo add --scope=default %s" % uts_repo )
os.system("spack repo add --scope=site %s" % uts_repo )

self.get_namespace(pkg)
# pick out path parts before/after version...
Expand Down Expand Up @@ -1917,7 +1913,7 @@ use_spack = []
# just pick off any leading "--" options
#
while sys.argv[1][:2] == "--":
print("parsing: %s" % repr(sys.argv))
#print("parsing: %s" % repr(sys.argv))
if sys.argv[1][:6] == "--pdr=" :
pdr = sys.argv[1][6:]
#print("pdr=%s" % pdr)
Expand All @@ -1934,14 +1930,16 @@ while sys.argv[1][:2] == "--":

else:
print("Unknown flag %s" % sys.argv[1])
usage()
exit(1)

sys.argv = sys.argv[:1] + sys.argv[2:]

uts = ups_to_spack(pdr=pdr)

if not os.path.exists("/tmp/empty.tar"):
d = os.path.dirname(os.path.dirname(__FILE__))
os.system("cp {0}/templates/emtpy.tar /tmp/empty.tar .".format(d))
d = os.path.dirname(os.path.dirname(__file__))
os.system("cp {0}/templates/empty.tar /tmp/empty.tar".format(d))

# handle --use_spack args: prepopulate conversion cache
for us in use_spack:
Expand Down
1 change: 1 addition & 0 deletions templates/packagelist
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ mesa-libGL:mesa:$0-devel
meson:meson:$0
(xorg-x11-font-utils|mkfontscale):mkfontdir:$0
(xorg-x11-font-utils|mkfontscale):mkfontscale:$0
motif:motif:$0
mpich(-[0-9.]+)?:mpich:$0-devel
nasm:nasm:$0
ninja-build:ninja:$0
Expand Down
5 changes: 5 additions & 0 deletions templates/packages.yaml.almalinux9
View file
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,11 @@ packages:
- spec: "mkfontscale @1.2.1 %gcc@11 os=almalinux9"
prefix: /usr
buildable: False
motif:
externals:
- spec: "motif @2.3.4 %gcc@11 os=almalinux9"
prefix: /usr
buildable: False
nasm:
externals:
- spec: "nasm @2.15.03 %gcc@11 os=almalinux9"
Expand Down
5 changes: 5 additions & 0 deletions templates/packages.yaml.scientific7
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,11 @@ packages:
- spec: "mkfontscale @7.5 %[email protected] os=scientific7"
prefix: /usr
buildable: False
motif:
externals:
- spec: "motif @2.3.4 %[email protected] os=scientific7"
prefix: /usr
buildable: False
mpich:
externals:
- spec: "mpich @3.2 %[email protected] os=scientific7"
Expand Down

0 comments on commit 6fe8951

Please sign in to comment.