From 83ad8b738a2af3caa990f1d30c30146ca7536e8e Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Wed, 15 Nov 2023 15:30:21 +0000 Subject: [PATCH 01/22] Spack: Remove everything from csd3-icelake environment Most of the Spack compilers and external packages are RHEL/Rocky Linux/Scientific Linux 7 (for the Cascade Lakes) and have issues on the Icelakes (running Rocky Linux 8) hence why this resets the environment to a clean slate. --- .../csd3-icelake/compute-node/spack.yaml | 219 ------------------ 1 file changed, 219 deletions(-) diff --git a/benchmarks/spack/csd3-icelake/compute-node/spack.yaml b/benchmarks/spack/csd3-icelake/compute-node/spack.yaml index feb46354..d4550d07 100644 --- a/benchmarks/spack/csd3-icelake/compute-node/spack.yaml +++ b/benchmarks/spack/csd3-icelake/compute-node/spack.yaml @@ -9,225 +9,6 @@ spack: include: - ../../common.yaml compilers: - - compiler: - spec: intel@17.0.4 - paths: - cc: /usr/local/Cluster-Apps/intel/2017.4/compilers_and_libraries_2017.4.196/linux/bin/intel64/icc - cxx: /usr/local/Cluster-Apps/intel/2017.4/compilers_and_libraries_2017.4.196/linux/bin/intel64/icpc - f77: /usr/local/Cluster-Apps/intel/2017.4/compilers_and_libraries_2017.4.196/linux/bin/intel64/ifort - fc: /usr/local/Cluster-Apps/intel/2017.4/compilers_and_libraries_2017.4.196/linux/bin/intel64/ifort - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - - compiler: - spec: gcc@4.8.5 - paths: - cc: /usr/bin/gcc - cxx: /usr/bin/g++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - - compiler: - spec: gcc@5.5.0 - paths: - cc: /usr/local/software/master/gcc/5/bin/gcc - cxx: /usr/local/software/master/gcc/5/bin/g++ - f77: /usr/local/software/master/gcc/5/bin/gfortran - fc: /usr/local/software/master/gcc/5/bin/gfortran - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - - compiler: - spec: gcc@6.5.0 - paths: - cc: /usr/local/software/master/gcc/6/bin/gcc - cxx: /usr/local/software/master/gcc/6/bin/g++ - f77: /usr/local/software/master/gcc/6/bin/gfortran - fc: /usr/local/software/master/gcc/6/bin/gfortran - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - - compiler: - spec: gcc@7.4.0 - paths: - cc: /usr/local/software/master/gcc/7/bin/gcc - cxx: /usr/local/software/master/gcc/7/bin/g++ - f77: /usr/local/software/master/gcc/7/bin/gfortran - fc: /usr/local/software/master/gcc/7/bin/gfortran - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - - compiler: - spec: gcc@8.4.0 - paths: - cc: /usr/local/software/master/gcc/8/bin/gcc - cxx: /usr/local/software/master/gcc/8/bin/g++ - f77: /usr/local/software/master/gcc/8/bin/gfortran - fc: /usr/local/software/master/gcc/8/bin/gfortran - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - - compiler: - spec: gcc@9.3.0 - paths: - cc: /usr/local/software/master/gcc/9/bin/gcc - cxx: /usr/local/software/master/gcc/9/bin/g++ - f77: /usr/local/software/master/gcc/9/bin/gfortran - fc: /usr/local/software/master/gcc/9/bin/gfortran - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - - compiler: - spec: intel@18.0.5 - paths: - cc: /usr/local/Cluster-Apps/intel/2018.4/compilers_and_libraries_2018.5.274/linux/bin/intel64/icc - cxx: /usr/local/Cluster-Apps/intel/2018.4/compilers_and_libraries_2018.5.274/linux/bin/intel64/icpc - f77: /usr/local/Cluster-Apps/intel/2018.4/compilers_and_libraries_2018.5.274/linux/bin/intel64/ifort - fc: /usr/local/Cluster-Apps/intel/2018.4/compilers_and_libraries_2018.5.274/linux/bin/intel64/ifort - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - - compiler: - spec: intel@19.0.4.243 - paths: - cc: /usr/local/Cluster-Apps/intel/2019.4/compilers_and_libraries_2019.4.243/linux/bin/intel64/icc - cxx: /usr/local/Cluster-Apps/intel/2019.4/compilers_and_libraries_2019.4.243/linux/bin/intel64/icpc - f77: /usr/local/Cluster-Apps/intel/2019.4/compilers_and_libraries_2019.4.243/linux/bin/intel64/ifort - fc: /usr/local/Cluster-Apps/intel/2019.4/compilers_and_libraries_2019.4.243/linux/bin/intel64/ifort - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - - compiler: - spec: intel@19.1.2.254 - paths: - cc: /usr/local/Cluster-Apps/intel/2020.2/compilers_and_libraries_2020.2.254/linux/bin/intel64/icc - cxx: /usr/local/Cluster-Apps/intel/2020.2/compilers_and_libraries_2020.2.254/linux/bin/intel64/icpc - f77: /usr/local/Cluster-Apps/intel/2020.2/compilers_and_libraries_2020.2.254/linux/bin/intel64/ifort - fc: /usr/local/Cluster-Apps/intel/2020.2/compilers_and_libraries_2020.2.254/linux/bin/intel64/ifort - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - - compiler: - spec: intel@19.1.3.304 - paths: - cc: /usr/local/Cluster-Apps/intel/2020.4/compilers_and_libraries_2020.4.304/linux/bin/intel64/icc - cxx: /usr/local/Cluster-Apps/intel/2020.4/compilers_and_libraries_2020.4.304/linux/bin/intel64/icpc - f77: /usr/local/Cluster-Apps/intel/2020.4/compilers_and_libraries_2020.4.304/linux/bin/intel64/ifort - fc: /usr/local/Cluster-Apps/intel/2020.4/compilers_and_libraries_2020.4.304/linux/bin/intel64/ifort - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [ intel/compilers/2020.4 ] - environment: {} - extra_rpaths: [] packages: all: target: [icelake] - autoconf: - externals: - - spec: autoconf@2.69 - prefix: /usr - automake: - externals: - - spec: automake@1.13.4 - prefix: /usr - cmake: - externals: - - spec: cmake@2.8.12.2 - prefix: /usr - - spec: cmake@3.17.0 - prefix: /usr/local/software/master/cmake/latest - gsl: - externals: - - spec: gsl@2.1 - prefix: /usr/local/Cluster-Apps/gsl/2.1 - - spec: gsl@2.4 - prefix: /usr/local/Cluster-Apps/gsl/2.4 - hdf5: - externals: - - spec: hdf5@1.10.3%gcc@4.8.5~mpi - prefix: /usr/local/software/hdf5/1.10.3 - - spec: hdf5@1.10.5%gcc@9.3.0+mpi^openmpi - prefix: /usr/local/Cluster-Apps/hdf5/openmpi/gcc/9.2/1.10.5 - - spec: hdf5@1.10.5%intel+mpi^intel-mpi - prefix: /usr/local/Cluster-Apps/hdf5/mpi/intel/2019.3/1.10.5 - - spec: hdf5@1.12.0%gcc@9.3.0+mpi^openmpi - prefix: /usr/local/Cluster-Apps/hdf5/openmpi/gcc/9.3/1.12.0 - intel-mkl: - externals: - - spec: intel-mkl@2020.4.304 - prefix: /usr/local/Cluster-Apps/intel/2020.4/compilers_and_libraries_2020.4.304/linux/mkl - intel-mpi: - externals: - - spec: intel-mpi@2017.4.239 - prefix: /usr/local/Cluster-Apps/intel/2017.4/compilers_and_libraries_2017.4.196/linux/mpi/intel64 - - spec: intel-mpi@2018.5.274 - prefix: /usr/local/Cluster-Apps/intel/2018.4/compilers_and_libraries_2018.5.274/linux/mpi/intel64 - - spec: intel-mpi@2019.4.243 - prefix: /usr/local/Cluster-Apps/intel/2019.4/compilers_and_libraries_2019.4.243/linux/mpi/intel64 - - spec: intel-mpi@2020.2.254 - prefix: /usr/local/Cluster-Apps/intel/2020.2/compilers_and_libraries_2020.2.254/linux/mpi/intel64 - - spec: intel-mpi@2020.4.304 - prefix: /usr/local/Cluster-Apps/intel/2020.4/compilers_and_libraries_2020.4.304/linux/mpi/intel64 - libtool: - externals: - - spec: libtool@2.4 - prefix: /usr - m4: - externals: - - spec: m4@1.4.16 - prefix: /usr - openmpi: - externals: - - spec: openmpi@4.0.4 - prefix: /usr/local/Cluster-Apps/openmpi/gcc/9.3/4.0.4 - - spec: openmpi@4.0.2 - prefix: /usr/local/Cluster-Apps/openmpi/gnu/4.0.2-gcc-9.2 - - spec: openmpi@4.0.1 - prefix: /usr/local/Cluster-Apps/openmpi/gnu/4.0.1-gcc-9.2 - - spec: openmpi@4.0.0 - prefix: /usr/local/software/spack/spack-0.11.2/opt/spack/linux-rhel7-x86_64/intel-17.0.8/openmpi-4.0.0-tnwn5io24eaoxhdxxcscx5jrsvlneqqx - perl: - externals: - - spec: perl@5.16.3+cpanm+shared+threads - prefix: /usr - python: - externals: - - spec: python@3.8.2+bz2+ctypes+dbm+lzma+nis+pyexpat+readline+sqlite3+ssl+tix+tkinter+uuid+zlib - prefix: /usr/local/software/master/python/3.8 - - spec: python@2.7.5+bz2+ctypes+dbm+lzma+nis+pyexpat+readline+sqlite3+ssl~tix~tkinter+uuid+zlib - prefix: /usr - - spec: python@3.6.8+bz2+ctypes+dbm+lzma+nis+pyexpat+readline+sqlite3+ssl~tix~tkinter+uuid+zlib - prefix: /usr From a31e753079251db30a17ec8266f822d1d6bb5e3d Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Wed, 15 Nov 2023 15:50:22 +0000 Subject: [PATCH 02/22] Spack: Add compilers to csd3-icelake environment --- .../csd3-icelake/compute-node/spack.yaml | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/benchmarks/spack/csd3-icelake/compute-node/spack.yaml b/benchmarks/spack/csd3-icelake/compute-node/spack.yaml index d4550d07..093c9f8f 100644 --- a/benchmarks/spack/csd3-icelake/compute-node/spack.yaml +++ b/benchmarks/spack/csd3-icelake/compute-node/spack.yaml @@ -9,6 +9,84 @@ spack: include: - ../../common.yaml compilers: + - compiler: + spec: clang@=15.0.7 + paths: + cc: /usr/bin/clang + cxx: /usr/bin/clang++ + f77: null + fc: null + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: gcc@=8.5.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: gcc@=11.3.0 + paths: + cc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/gcc-11.3.0/gcc-11.3.0/4zpip55j2rww33vhy62jl4eliwynqfru/bin/gcc + cxx: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/gcc-11.3.0/gcc-11.3.0/4zpip55j2rww33vhy62jl4eliwynqfru/bin/g++ + f77: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/gcc-11.3.0/gcc-11.3.0/4zpip55j2rww33vhy62jl4eliwynqfru/bin/gfortran + fc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/gcc-11.3.0/gcc-11.3.0/4zpip55j2rww33vhy62jl4eliwynqfru/bin/gfortran + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: dpcpp@=2022.1.0 + paths: + cc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/icx + cxx: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/dpcpp + f77: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/ifx + fc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/ifx + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: intel@=2021.6.0 + paths: + cc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/intel64/icc + cxx: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/intel64/icpc + f77: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/intel64/ifort + fc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/intel64/ifort + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: oneapi@=2022.1.0 + paths: + cc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/icx + cxx: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/icpx + f77: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/ifx + fc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/ifx + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] packages: all: target: [icelake] From 982fca21744cfd3d216aa5e772bb99f959f48d89 Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Wed, 15 Nov 2023 15:53:13 +0000 Subject: [PATCH 03/22] Spack: Add system external packages to csd3-icelake environment --- .../csd3-icelake/compute-node/spack.yaml | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/benchmarks/spack/csd3-icelake/compute-node/spack.yaml b/benchmarks/spack/csd3-icelake/compute-node/spack.yaml index 093c9f8f..2a1f2b45 100644 --- a/benchmarks/spack/csd3-icelake/compute-node/spack.yaml +++ b/benchmarks/spack/csd3-icelake/compute-node/spack.yaml @@ -90,3 +90,105 @@ spack: packages: all: target: [icelake] + binutils: + externals: + - spec: binutils@2.30.119 + prefix: /usr + coreutils: + externals: + - spec: coreutils@8.30 + prefix: /usr + diffutils: + externals: + - spec: diffutils@3.6 + prefix: /usr + findutils: + externals: + - spec: findutils@4.6.0 + prefix: /usr + gawk: + externals: + - spec: gawk@4.2.1 + prefix: /usr + openssh: + externals: + - spec: openssh@8.0p1 + prefix: /usr + openssl: + externals: + - spec: openssl@1.1.1k + prefix: /usr + tar: + externals: + - spec: tar@1.30 + prefix: /usr + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr + automake: + externals: + - spec: automake@1.16.1 + prefix: /usr + bison: + externals: + - spec: bison@3.0.4 + prefix: /usr + cmake: + externals: + - spec: cmake@3.20.2 + prefix: /usr + curl: + externals: + - spec: curl@7.61.1+gssapi+ldap+nghttp2 + prefix: /usr + cvs: + externals: + - spec: cvs@1.11.23 + prefix: /usr + doxygen: + externals: + - spec: doxygen@1.8.14+graphviz~mscgen + prefix: /usr + flex: + externals: + - spec: flex@2.6.1+lex + prefix: /usr + git: + externals: + - spec: git@2.39.3+tcltk + prefix: /usr + gmake: + externals: + - spec: gmake@4.2.1 + prefix: /usr + groff: + externals: + - spec: groff@1.22.3 + prefix: /usr + libtool: + externals: + - spec: libtool@2.4.6 + prefix: /usr + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + pkgconf: + externals: + - spec: pkgconf@1.4.2 + prefix: /usr + subversion: + externals: + - spec: subversion@1.10.2 + prefix: /usr + - spec: subversion@1.12.2 + prefix: /usr/local/software/global-rhel8 + swig: + externals: + - spec: swig@4.1.1 + prefix: /usr + texinfo: + externals: + - spec: texinfo@6.5 + prefix: /usr From 64bdc1b8c32c76b7f7fe4c194ea19eeddc089a40 Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Wed, 15 Nov 2023 17:51:23 +0000 Subject: [PATCH 04/22] Spack: Add modules as external packages to csd3-icelake environment --- .../csd3-icelake/compute-node/spack.yaml | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/benchmarks/spack/csd3-icelake/compute-node/spack.yaml b/benchmarks/spack/csd3-icelake/compute-node/spack.yaml index 2a1f2b45..93b1840b 100644 --- a/benchmarks/spack/csd3-icelake/compute-node/spack.yaml +++ b/benchmarks/spack/csd3-icelake/compute-node/spack.yaml @@ -158,6 +158,10 @@ spack: externals: - spec: git@2.39.3+tcltk prefix: /usr + gsl: + externals: + - spec: gsl@2.7.1~external-cblas + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/gcc-11.3.0/gsl-2.7.1-cjb5f4uixxxx6j7i7budicfhdb2rqtwu gmake: externals: - spec: gmake@4.2.1 @@ -166,6 +170,29 @@ spack: externals: - spec: groff@1.22.3 prefix: /usr + hdf5: + externals: + - spec: hdf5@1.10.8~cxx+fortran+hl~ipo~java+mpi+shared+szip~threadsafe+tools + api=default build_type=RelWithDebInfo patches=ee351eb %intel@2021.6.0 ^intel-oneapi-mpi@2021.6.0 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/intel-2021.6.0/hdf5-1.10.8-h75adcalc32r6k5hkaajovxvzvqa6rec + - spec: hdf5@1.10.8~cxx+fortran+hl~ipo~java+mpi+shared+szip~threadsafe+tools + api=default build_type=RelWithDebInfo patches=ee351eb %intel@2021.6.0 ^openmpi@4.1.5 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/intel-2021.6.0/hdf5-1.10.8-liscqv523hlafp5fxpqzxh2salpr4pm5 + - spec: hdf5@1.10.8~cxx+fortran+hl~ipo~java+mpi+shared+szip~threadsafe+tools + api=default build_type=RelWithDebInfo patches=ee351eb %gcc@11.3.0 ^intel-oneapi-mpi@2021.6.0 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/gcc-11.3.0/hdf5-1.10.8-xbjoagf5adm6bbnfczvtbbw7ohgn2glt + intel-oneapi-mkl: + externals: + - spec: intel-oneapi-mkl@2022.1.0~cluster~ilp64+shared %intel@2021.6.0 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/intel-2021.6.0/intel-oneapi-mkl-2022.1.0-mngj3ad6jpozccxhdxtmzexvz6qkjag2 + - spec: intel-oneapi-mkl@2022.1.0~cluster~ilp64+shared %gcc@11.3.0 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/gcc-11.3.0/intel-oneapi-mkl-2022.1.0-wt7gb6hfk7ojls7rj2k7dkyyt36de4n7 + intel-oneapi-mpi: + externals: + - spec: intel-oneapi-mpi@2021.6.0~external-libfabric~ilp64 %intel@2021.6.0 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/intel-2021.6.0/intel-oneapi-mpi-2021.6.0-guxuvcpmykplbrr2e3af2yd7njqhau5e + - spec: intel-oneapi-mpi@2021.6.0~external-libfabric~ilp64 %gcc@11.3.0 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/gcc-11.3.0/intel-oneapi-mpi-2021.6.0-2yn3govbxvgmaanmi4wpwjrqa2b3lthl libtool: externals: - spec: libtool@2.4.6 @@ -174,10 +201,23 @@ spack: externals: - spec: m4@1.4.18 prefix: /usr + openmpi: + externals: + - spec: openmpi@4.1.5~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~java+legacylaunchers~lustre~memchecker+pmi+romio+rsh~singularity+static+vt+wrapper-rpath + fabrics=ucx schedulers=slurm %intel@2021.6.0 + prefix: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/openmpi-4.1.5/intel-2021.6.0/b42idtrxnxnhjeplfsf7xjlghp5kwvkz/ pkgconf: externals: - spec: pkgconf@1.4.2 prefix: /usr + python: + externals: + - spec: python@2.7.18+bz2+crypt+ctypes+dbm~lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tkinter+uuid+zlib + prefix: /usr + - spec: python@3.6.8+bz2+crypt+ctypes+dbm+lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib + prefix: /usr + - spec: python@3.9.12+bz2+crypt+ctypes+dbm+lzma~nis+pyexpat+pythoncmd+readline+sqlite3+ssl~tix+tkinter+uuid+zlib + prefix: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/python-3.9.12/gcc-11.3.0/pdcqf4o5kyppnbgu6s4xkujbzvxmezu5 subversion: externals: - spec: subversion@1.10.2 From 99f3f2b31d222586643c9e14b8c11f84c61ec633 Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Tue, 21 Nov 2023 11:40:33 +0000 Subject: [PATCH 05/22] Spack: Add preferred compiler & MPI for csd3-icelakes --- benchmarks/spack/csd3-icelake/compute-node/spack.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/spack/csd3-icelake/compute-node/spack.yaml b/benchmarks/spack/csd3-icelake/compute-node/spack.yaml index 93b1840b..ed7f5086 100644 --- a/benchmarks/spack/csd3-icelake/compute-node/spack.yaml +++ b/benchmarks/spack/csd3-icelake/compute-node/spack.yaml @@ -89,6 +89,9 @@ spack: extra_rpaths: [] packages: all: + providers: + mpi: [intel-oneapi-mpi] # Prefer Intel MPI + compiler: [intel] # Prefer Intel Classic Compilers target: [icelake] binutils: externals: From 5459536e14487b5cca08e38bdf7505db7747528d Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Tue, 21 Nov 2023 15:16:13 +0000 Subject: [PATCH 06/22] Refactor CSD3 configurations Since we have two different OSs on CSD3, CentOS 7 and Rocky Linux 8, it is important to only use software built for the correct OS in order to avoid issues. --- benchmarks/reframe_config.py | 48 ++++++++----------- .../cascadelake}/spack.yaml | 0 .../icelake}/spack.yaml | 0 3 files changed, 19 insertions(+), 29 deletions(-) rename benchmarks/spack/{csd3-cascadelake/compute-node => csd3-centos7/cascadelake}/spack.yaml (100%) rename benchmarks/spack/{csd3-icelake/compute-node => csd3-rocky8/icelake}/spack.yaml (100%) diff --git a/benchmarks/reframe_config.py b/benchmarks/reframe_config.py index d486b2a2..a652dae9 100644 --- a/benchmarks/reframe_config.py +++ b/benchmarks/reframe_config.py @@ -66,19 +66,21 @@ def spack_root_to_path(): ] }, # end ARCHER2 { - # https://docs.hpc.cam.ac.uk/hpc/user-guide/cclake.html - 'name': 'csd3-icelake', - 'descr': 'CSD3 Icelake', - 'hostnames': ['login-q-[0-9]+'], + # https://www.hpc.cam.ac.uk/index.php/high-performance-computing + 'name': 'csd3-rocky8', + 'descr': 'Cambridge Service for Data Driven Discovery - Rocky Linux 8 (RHEL8 compatible) nodes', + 'hostnames': ['login-q-[0-4]+'], 'modules_system': 'tmod4', 'partitions': [ { - 'name': 'compute-node', - 'descr': 'Icelake compute nodes', + # https://docs.hpc.cam.ac.uk/hpc/user-guide/icelake.html + 'name': 'icelake', + 'descr': 'Ice Lake compute nodes', 'scheduler': 'slurm', 'launcher': 'mpirun', + 'modules': ['rhel8/default-icl'], 'access': ['--partition=icelake', '--exclusive'], - 'environs': ['default', 'intel2020-csd3'], + 'environs': ['default'], 'max_jobs': 64, 'processor': { 'num_cpus': 76, @@ -88,19 +90,21 @@ def spack_root_to_path(): }, }, ] - }, # end CSD3 Icelake + }, # end CSD3 Rocky 8 { - # https://docs.hpc.cam.ac.uk/hpc/user-guide/cclake.html - 'name': 'csd3-cascadelake', - 'descr': 'CSD3 Cascade lake', - 'hostnames': ['login-e-[0-9]+'], + # https://www.hpc.cam.ac.uk/index.php/high-performance-computing + 'name': 'csd3-centos7', + 'descr': 'Cambridge Service for Data Driven Discovery - CentOS 7 (RHEL7 compatible) nodes', + 'hostnames': ['login-p-[0-4]+'], 'modules_system': 'tmod32', 'partitions': [ { - 'name': 'compute-node', - 'descr': 'Skylake compute nodes', + # https://docs.hpc.cam.ac.uk/hpc/user-guide/cclake.html + 'name': 'cascadelake', + 'descr': 'Cascade Lake compute nodes', 'scheduler': 'slurm', 'launcher': 'mpirun', + 'modules': ['rhel7/default-ccl'], 'access': ['--partition=cclake', '--exclusive'], 'environs': ['default'], 'max_jobs': 64, @@ -112,7 +116,7 @@ def spack_root_to_path(): } }, ] - }, # end CSD3 Cascade lake + }, # end CSD3 CentOS 7 { # https://www.rc.ucl.ac.uk/docs/Clusters/Myriad/#node-types 'name': 'myriad', @@ -671,20 +675,6 @@ def spack_root_to_path(): 'cxx': 'mpiicpc', 'ftn': 'mpiifort' }, - { - 'name': 'intel2020-csd3', - 'modules': ["intel/compilers/2020.4", - "intel/mkl/2020.4", - "intel/impi/2020.4/intel", - "intel/libs/idb/2020.4", - "intel/libs/tbb/2020.4", - "intel/libs/ipp/2020.4", - "intel/libs/daal/2020.4", - "intel/bundles/complib/2020.4"], - 'cc': 'mpiicc', - 'cxx': 'mpiicpc', - 'ftn': 'mpiifort' - }, ], 'logging': [ { diff --git a/benchmarks/spack/csd3-cascadelake/compute-node/spack.yaml b/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml similarity index 100% rename from benchmarks/spack/csd3-cascadelake/compute-node/spack.yaml rename to benchmarks/spack/csd3-centos7/cascadelake/spack.yaml diff --git a/benchmarks/spack/csd3-icelake/compute-node/spack.yaml b/benchmarks/spack/csd3-rocky8/icelake/spack.yaml similarity index 100% rename from benchmarks/spack/csd3-icelake/compute-node/spack.yaml rename to benchmarks/spack/csd3-rocky8/icelake/spack.yaml From d347e660238bc31627d689eb85c8ca77711f313d Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Tue, 21 Nov 2023 15:19:14 +0000 Subject: [PATCH 07/22] Spack: Update CSD3 Cascade Lake environment This adds some external Spack packages provided by the OS and specifies a preferred compiler and MPI implementation. --- .../spack/csd3-centos7/cascadelake/spack.yaml | 109 ++++++++++++++++-- 1 file changed, 100 insertions(+), 9 deletions(-) diff --git a/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml b/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml index 265b68b2..2d05edf1 100644 --- a/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml +++ b/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml @@ -159,7 +159,44 @@ spack: - /usr/local/Cluster-Apps/intel/2020.4/compilers_and_libraries_2020.4.304/linux/lib packages: all: - target: [skylake_avx512] + providers: + mpi: [intel-mpi@2020.4.304] # Prefer Intel MPI + compiler: [intel@19.1.3.304] # Prefer Intel Classic Compilers + target: [cascadelake] + binutils: + externals: + - spec: binutils@2.27.44 + prefix: /usr + coreutils: + externals: + - spec: coreutils@8.22 + prefix: /usr + diffutils: + externals: + - spec: diffutils@3.3 + prefix: /usr + findutils: + externals: + - spec: findutils@4.5.11 + prefix: /usr + gawk: + externals: + - spec: gawk@4.0.2 + prefix: /usr + openssh: + externals: + - spec: openssh@7.4p1 + prefix: /usr + openssl: + externals: + - spec: openssl@1.0.2k-fips + prefix: /usr + - spec: openssl@1.1.1k + prefix: /usr + tar: + externals: + - spec: tar@1.26 + prefix: /usr autoconf: externals: - spec: autoconf@2.69 @@ -168,12 +205,74 @@ spack: externals: - spec: automake@1.13.4 prefix: /usr + bison: + externals: + - spec: bison@3.0.4 + prefix: /usr cmake: externals: - spec: cmake@2.8.12.2 prefix: /usr - spec: cmake@3.17.0 prefix: /usr/local/software/master/cmake/latest + curl: + externals: + - spec: curl@7.29.0+ldap + prefix: /usr + cvs: + externals: + - spec: cvs@1.11.23 + prefix: /usr + doxygen: + externals: + - spec: doxygen@1.8.5+graphviz~mscgen + prefix: /usr + flex: + externals: + - spec: flex@2.5.37+lex + prefix: /usr + git: + externals: + - spec: git@1.8.3.1+tcltk + prefix: /usr + - spec: git@2.39.2~tcltk + prefix: /usr/local/software/global + gmake: + externals: + - spec: gmake@3.82 + prefix: /usr + groff: + externals: + - spec: groff@1.22.2 + prefix: /usr + libtool: + externals: + - spec: libtool@2.4.2 + prefix: /usr + - spec: libtool@2.4 + prefix: /usr + m4: + externals: + - spec: m4@1.4.16 + prefix: /usr + pkg-config: + externals: + - spec: pkg-config@0.27.1 + prefix: /usr + subversion: + externals: + - spec: subversion@1.7.14 + prefix: /usr + - spec: subversion@1.12.2 + prefix: /usr/local/software/global + swig: + externals: + - spec: swig@2.0.10 + prefix: /usr + texinfo: + externals: + - spec: texinfo@5.1 + prefix: /usr gsl: externals: - spec: gsl@2.1 @@ -206,14 +305,6 @@ spack: prefix: /usr/local/Cluster-Apps/intel/2020.2/compilers_and_libraries_2020.2.254/linux/mpi/intel64 - spec: intel-mpi@2020.4.304 prefix: /usr/local/Cluster-Apps/intel/2020.4/compilers_and_libraries_2020.4.304/linux/mpi/intel64 - libtool: - externals: - - spec: libtool@2.4 - prefix: /usr - m4: - externals: - - spec: m4@1.4.16 - prefix: /usr openmpi: externals: - spec: openmpi@4.0.4 From 71a7f9ecf09cb428725e197eeb1da7467c8e6eac Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Tue, 21 Nov 2023 15:33:22 +0000 Subject: [PATCH 08/22] Spack: Refactor CSD3 icelake environment --- benchmarks/spack/csd3-rocky8/compilers.yaml | 79 +++++++ .../spack/csd3-rocky8/icelake/spack.yaml | 223 +----------------- benchmarks/spack/csd3-rocky8/packages.yaml | 143 +++++++++++ 3 files changed, 224 insertions(+), 221 deletions(-) create mode 100644 benchmarks/spack/csd3-rocky8/compilers.yaml create mode 100644 benchmarks/spack/csd3-rocky8/packages.yaml diff --git a/benchmarks/spack/csd3-rocky8/compilers.yaml b/benchmarks/spack/csd3-rocky8/compilers.yaml new file mode 100644 index 00000000..61890d33 --- /dev/null +++ b/benchmarks/spack/csd3-rocky8/compilers.yaml @@ -0,0 +1,79 @@ +compilers: +- compiler: + spec: clang@=15.0.7 + paths: + cc: /usr/bin/clang + cxx: /usr/bin/clang++ + f77: null + fc: null + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@=8.5.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@=11.3.0 + paths: + cc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/gcc-11.3.0/gcc-11.3.0/4zpip55j2rww33vhy62jl4eliwynqfru/bin/gcc + cxx: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/gcc-11.3.0/gcc-11.3.0/4zpip55j2rww33vhy62jl4eliwynqfru/bin/g++ + f77: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/gcc-11.3.0/gcc-11.3.0/4zpip55j2rww33vhy62jl4eliwynqfru/bin/gfortran + fc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/gcc-11.3.0/gcc-11.3.0/4zpip55j2rww33vhy62jl4eliwynqfru/bin/gfortran + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: dpcpp@=2022.1.0 + paths: + cc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/icx + cxx: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/dpcpp + f77: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/ifx + fc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/ifx + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: intel@=2021.6.0 + paths: + cc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/intel64/icc + cxx: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/intel64/icpc + f77: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/intel64/ifort + fc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/intel64/ifort + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: oneapi@=2022.1.0 + paths: + cc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/icx + cxx: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/icpx + f77: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/ifx + fc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/ifx + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] diff --git a/benchmarks/spack/csd3-rocky8/icelake/spack.yaml b/benchmarks/spack/csd3-rocky8/icelake/spack.yaml index ed7f5086..453a7960 100644 --- a/benchmarks/spack/csd3-rocky8/icelake/spack.yaml +++ b/benchmarks/spack/csd3-rocky8/icelake/spack.yaml @@ -8,230 +8,11 @@ spack: view: false include: - ../../common.yaml - compilers: - - compiler: - spec: clang@=15.0.7 - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: null - fc: null - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - - compiler: - spec: gcc@=8.5.0 - paths: - cc: /usr/bin/gcc - cxx: /usr/bin/g++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - - compiler: - spec: gcc@=11.3.0 - paths: - cc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/gcc-11.3.0/gcc-11.3.0/4zpip55j2rww33vhy62jl4eliwynqfru/bin/gcc - cxx: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/gcc-11.3.0/gcc-11.3.0/4zpip55j2rww33vhy62jl4eliwynqfru/bin/g++ - f77: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/gcc-11.3.0/gcc-11.3.0/4zpip55j2rww33vhy62jl4eliwynqfru/bin/gfortran - fc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/gcc-11.3.0/gcc-11.3.0/4zpip55j2rww33vhy62jl4eliwynqfru/bin/gfortran - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - - compiler: - spec: dpcpp@=2022.1.0 - paths: - cc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/icx - cxx: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/dpcpp - f77: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/ifx - fc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/ifx - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - - compiler: - spec: intel@=2021.6.0 - paths: - cc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/intel64/icc - cxx: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/intel64/icpc - f77: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/intel64/ifort - fc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/intel64/ifort - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - - compiler: - spec: oneapi@=2022.1.0 - paths: - cc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/icx - cxx: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/icpx - f77: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/ifx - fc: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/intel-oneapi-compilers-2022.1.0/gcc-11.3.0/b6zld2mz7cid27yloxznoidymd7vywwz/compiler/2022.1.0/linux/bin/ifx - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] + - ../compilers.yaml + - ../packages.yaml packages: all: providers: mpi: [intel-oneapi-mpi] # Prefer Intel MPI compiler: [intel] # Prefer Intel Classic Compilers target: [icelake] - binutils: - externals: - - spec: binutils@2.30.119 - prefix: /usr - coreutils: - externals: - - spec: coreutils@8.30 - prefix: /usr - diffutils: - externals: - - spec: diffutils@3.6 - prefix: /usr - findutils: - externals: - - spec: findutils@4.6.0 - prefix: /usr - gawk: - externals: - - spec: gawk@4.2.1 - prefix: /usr - openssh: - externals: - - spec: openssh@8.0p1 - prefix: /usr - openssl: - externals: - - spec: openssl@1.1.1k - prefix: /usr - tar: - externals: - - spec: tar@1.30 - prefix: /usr - autoconf: - externals: - - spec: autoconf@2.69 - prefix: /usr - automake: - externals: - - spec: automake@1.16.1 - prefix: /usr - bison: - externals: - - spec: bison@3.0.4 - prefix: /usr - cmake: - externals: - - spec: cmake@3.20.2 - prefix: /usr - curl: - externals: - - spec: curl@7.61.1+gssapi+ldap+nghttp2 - prefix: /usr - cvs: - externals: - - spec: cvs@1.11.23 - prefix: /usr - doxygen: - externals: - - spec: doxygen@1.8.14+graphviz~mscgen - prefix: /usr - flex: - externals: - - spec: flex@2.6.1+lex - prefix: /usr - git: - externals: - - spec: git@2.39.3+tcltk - prefix: /usr - gsl: - externals: - - spec: gsl@2.7.1~external-cblas - prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/gcc-11.3.0/gsl-2.7.1-cjb5f4uixxxx6j7i7budicfhdb2rqtwu - gmake: - externals: - - spec: gmake@4.2.1 - prefix: /usr - groff: - externals: - - spec: groff@1.22.3 - prefix: /usr - hdf5: - externals: - - spec: hdf5@1.10.8~cxx+fortran+hl~ipo~java+mpi+shared+szip~threadsafe+tools - api=default build_type=RelWithDebInfo patches=ee351eb %intel@2021.6.0 ^intel-oneapi-mpi@2021.6.0 - prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/intel-2021.6.0/hdf5-1.10.8-h75adcalc32r6k5hkaajovxvzvqa6rec - - spec: hdf5@1.10.8~cxx+fortran+hl~ipo~java+mpi+shared+szip~threadsafe+tools - api=default build_type=RelWithDebInfo patches=ee351eb %intel@2021.6.0 ^openmpi@4.1.5 - prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/intel-2021.6.0/hdf5-1.10.8-liscqv523hlafp5fxpqzxh2salpr4pm5 - - spec: hdf5@1.10.8~cxx+fortran+hl~ipo~java+mpi+shared+szip~threadsafe+tools - api=default build_type=RelWithDebInfo patches=ee351eb %gcc@11.3.0 ^intel-oneapi-mpi@2021.6.0 - prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/gcc-11.3.0/hdf5-1.10.8-xbjoagf5adm6bbnfczvtbbw7ohgn2glt - intel-oneapi-mkl: - externals: - - spec: intel-oneapi-mkl@2022.1.0~cluster~ilp64+shared %intel@2021.6.0 - prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/intel-2021.6.0/intel-oneapi-mkl-2022.1.0-mngj3ad6jpozccxhdxtmzexvz6qkjag2 - - spec: intel-oneapi-mkl@2022.1.0~cluster~ilp64+shared %gcc@11.3.0 - prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/gcc-11.3.0/intel-oneapi-mkl-2022.1.0-wt7gb6hfk7ojls7rj2k7dkyyt36de4n7 - intel-oneapi-mpi: - externals: - - spec: intel-oneapi-mpi@2021.6.0~external-libfabric~ilp64 %intel@2021.6.0 - prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/intel-2021.6.0/intel-oneapi-mpi-2021.6.0-guxuvcpmykplbrr2e3af2yd7njqhau5e - - spec: intel-oneapi-mpi@2021.6.0~external-libfabric~ilp64 %gcc@11.3.0 - prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/gcc-11.3.0/intel-oneapi-mpi-2021.6.0-2yn3govbxvgmaanmi4wpwjrqa2b3lthl - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr - m4: - externals: - - spec: m4@1.4.18 - prefix: /usr - openmpi: - externals: - - spec: openmpi@4.1.5~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~java+legacylaunchers~lustre~memchecker+pmi+romio+rsh~singularity+static+vt+wrapper-rpath - fabrics=ucx schedulers=slurm %intel@2021.6.0 - prefix: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/openmpi-4.1.5/intel-2021.6.0/b42idtrxnxnhjeplfsf7xjlghp5kwvkz/ - pkgconf: - externals: - - spec: pkgconf@1.4.2 - prefix: /usr - python: - externals: - - spec: python@2.7.18+bz2+crypt+ctypes+dbm~lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tkinter+uuid+zlib - prefix: /usr - - spec: python@3.6.8+bz2+crypt+ctypes+dbm+lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib - prefix: /usr - - spec: python@3.9.12+bz2+crypt+ctypes+dbm+lzma~nis+pyexpat+pythoncmd+readline+sqlite3+ssl~tix+tkinter+uuid+zlib - prefix: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/python-3.9.12/gcc-11.3.0/pdcqf4o5kyppnbgu6s4xkujbzvxmezu5 - subversion: - externals: - - spec: subversion@1.10.2 - prefix: /usr - - spec: subversion@1.12.2 - prefix: /usr/local/software/global-rhel8 - swig: - externals: - - spec: swig@4.1.1 - prefix: /usr - texinfo: - externals: - - spec: texinfo@6.5 - prefix: /usr diff --git a/benchmarks/spack/csd3-rocky8/packages.yaml b/benchmarks/spack/csd3-rocky8/packages.yaml new file mode 100644 index 00000000..5373b3b0 --- /dev/null +++ b/benchmarks/spack/csd3-rocky8/packages.yaml @@ -0,0 +1,143 @@ +packages: + binutils: + externals: + - spec: binutils@2.30.119 + prefix: /usr + coreutils: + externals: + - spec: coreutils@8.30 + prefix: /usr + diffutils: + externals: + - spec: diffutils@3.6 + prefix: /usr + findutils: + externals: + - spec: findutils@4.6.0 + prefix: /usr + gawk: + externals: + - spec: gawk@4.2.1 + prefix: /usr + openssh: + externals: + - spec: openssh@8.0p1 + prefix: /usr + openssl: + externals: + - spec: openssl@1.1.1k + prefix: /usr + tar: + externals: + - spec: tar@1.30 + prefix: /usr + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr + automake: + externals: + - spec: automake@1.16.1 + prefix: /usr + bison: + externals: + - spec: bison@3.0.4 + prefix: /usr + cmake: + externals: + - spec: cmake@3.20.2 + prefix: /usr + curl: + externals: + - spec: curl@7.61.1+gssapi+ldap+nghttp2 + prefix: /usr + cvs: + externals: + - spec: cvs@1.11.23 + prefix: /usr + doxygen: + externals: + - spec: doxygen@1.8.14+graphviz~mscgen + prefix: /usr + flex: + externals: + - spec: flex@2.6.1+lex + prefix: /usr + git: + externals: + - spec: git@2.39.3+tcltk + prefix: /usr + gsl: + externals: + - spec: gsl@2.7.1~external-cblas + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/gcc-11.3.0/gsl-2.7.1-cjb5f4uixxxx6j7i7budicfhdb2rqtwu + gmake: + externals: + - spec: gmake@4.2.1 + prefix: /usr + groff: + externals: + - spec: groff@1.22.3 + prefix: /usr + hdf5: + externals: + - spec: hdf5@1.10.8~cxx+fortran+hl~ipo~java+mpi+shared+szip~threadsafe+tools + api=default build_type=RelWithDebInfo patches=ee351eb %intel@2021.6.0 ^intel-oneapi-mpi@2021.6.0 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/intel-2021.6.0/hdf5-1.10.8-h75adcalc32r6k5hkaajovxvzvqa6rec + - spec: hdf5@1.10.8~cxx+fortran+hl~ipo~java+mpi+shared+szip~threadsafe+tools + api=default build_type=RelWithDebInfo patches=ee351eb %intel@2021.6.0 ^openmpi@4.1.5 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/intel-2021.6.0/hdf5-1.10.8-liscqv523hlafp5fxpqzxh2salpr4pm5 + - spec: hdf5@1.10.8~cxx+fortran+hl~ipo~java+mpi+shared+szip~threadsafe+tools + api=default build_type=RelWithDebInfo patches=ee351eb %gcc@11.3.0 ^intel-oneapi-mpi@2021.6.0 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/gcc-11.3.0/hdf5-1.10.8-xbjoagf5adm6bbnfczvtbbw7ohgn2glt + intel-oneapi-mkl: + externals: + - spec: intel-oneapi-mkl@2022.1.0~cluster~ilp64+shared %intel@2021.6.0 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/intel-2021.6.0/intel-oneapi-mkl-2022.1.0-mngj3ad6jpozccxhdxtmzexvz6qkjag2 + - spec: intel-oneapi-mkl@2022.1.0~cluster~ilp64+shared %gcc@11.3.0 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/gcc-11.3.0/intel-oneapi-mkl-2022.1.0-wt7gb6hfk7ojls7rj2k7dkyyt36de4n7 + intel-oneapi-mpi: + externals: + - spec: intel-oneapi-mpi@2021.6.0~external-libfabric~ilp64 %intel@2021.6.0 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/intel-2021.6.0/intel-oneapi-mpi-2021.6.0-guxuvcpmykplbrr2e3af2yd7njqhau5e + - spec: intel-oneapi-mpi@2021.6.0~external-libfabric~ilp64 %gcc@11.3.0 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/gcc-11.3.0/intel-oneapi-mpi-2021.6.0-2yn3govbxvgmaanmi4wpwjrqa2b3lthl + libtool: + externals: + - spec: libtool@2.4.6 + prefix: /usr + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + openmpi: + externals: + - spec: openmpi@4.1.5~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~java+legacylaunchers~lustre~memchecker+pmi+romio+rsh~singularity+static+vt+wrapper-rpath + fabrics=ucx schedulers=slurm %intel@2021.6.0 + prefix: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/openmpi-4.1.5/intel-2021.6.0/b42idtrxnxnhjeplfsf7xjlghp5kwvkz/ + pkgconf: + externals: + - spec: pkgconf@1.4.2 + prefix: /usr + python: + externals: + - spec: python@2.7.18+bz2+crypt+ctypes+dbm~lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tkinter+uuid+zlib + prefix: /usr + - spec: python@3.6.8+bz2+crypt+ctypes+dbm+lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib + prefix: /usr + - spec: python@3.9.12+bz2+crypt+ctypes+dbm+lzma~nis+pyexpat+pythoncmd+readline+sqlite3+ssl~tix+tkinter+uuid+zlib + prefix: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/python-3.9.12/gcc-11.3.0/pdcqf4o5kyppnbgu6s4xkujbzvxmezu5 + subversion: + externals: + - spec: subversion@1.10.2 + prefix: /usr + - spec: subversion@1.12.2 + prefix: /usr/local/software/global-rhel8 + swig: + externals: + - spec: swig@4.1.1 + prefix: /usr + texinfo: + externals: + - spec: texinfo@6.5 + prefix: /usr From 10ff908485edd7546ea7b7bd500b5e24c65119cd Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Tue, 21 Nov 2023 16:48:58 +0000 Subject: [PATCH 09/22] CSD3: Add config for Sapphire Rapids partition Note that there is currently a common software stack for the Ice Lakes and Sapphire Rapids and both partitions use the same OS (Rocky Linux 8). --- benchmarks/reframe_config.py | 17 +++++++++++++++++ .../csd3-rocky8/sapphirerapids/spack.yaml | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 benchmarks/spack/csd3-rocky8/sapphirerapids/spack.yaml diff --git a/benchmarks/reframe_config.py b/benchmarks/reframe_config.py index a652dae9..8c91c2fb 100644 --- a/benchmarks/reframe_config.py +++ b/benchmarks/reframe_config.py @@ -89,6 +89,23 @@ def spack_root_to_path(): 'num_cpus_per_socket': 38, }, }, + { + # https://docs.hpc.cam.ac.uk/hpc/user-guide/icelake.html + 'name': 'sapphirerapids', + 'descr': 'Sapphire Rapids compute nodes', + 'scheduler': 'slurm', + 'launcher': 'mpirun', + 'modules': ['rhel8/default-sar'], + 'access': ['--partition=sapphire', '--exclusive'], + 'environs': ['default'], + 'max_jobs': 64, + 'processor': { + 'num_cpus': 112, + 'num_cpus_per_core': 1, + 'num_sockets': 2, + 'num_cpus_per_socket': 56, + }, + }, ] }, # end CSD3 Rocky 8 { diff --git a/benchmarks/spack/csd3-rocky8/sapphirerapids/spack.yaml b/benchmarks/spack/csd3-rocky8/sapphirerapids/spack.yaml new file mode 100644 index 00000000..ebdf8e76 --- /dev/null +++ b/benchmarks/spack/csd3-rocky8/sapphirerapids/spack.yaml @@ -0,0 +1,18 @@ +# This is a Spack Environment file. +# +# It describes a set of packages to be installed, along with +# configuration settings. +spack: + # add package specs to the `specs` list + specs: [] + view: false + include: + - ../../common.yaml + - ../compilers.yaml + - ../packages.yaml + packages: + all: + providers: + mpi: [intel-oneapi-mpi] # Prefer Intel MPI + compiler: [intel] # Prefer Intel Classic Compilers + target: [sapphirerapids] From 60e6274f3c85f84a4273d1194e7b4952ff5711e1 Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Wed, 22 Nov 2023 15:23:52 +0000 Subject: [PATCH 10/22] Spack: Make csd3-rocky8 targets requirements Also allow the concretizer to target sapphirerapids when building on the icelake login nodes. --- benchmarks/spack/csd3-rocky8/icelake/spack.yaml | 2 +- benchmarks/spack/csd3-rocky8/sapphirerapids/spack.yaml | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/benchmarks/spack/csd3-rocky8/icelake/spack.yaml b/benchmarks/spack/csd3-rocky8/icelake/spack.yaml index 453a7960..3a231b2f 100644 --- a/benchmarks/spack/csd3-rocky8/icelake/spack.yaml +++ b/benchmarks/spack/csd3-rocky8/icelake/spack.yaml @@ -15,4 +15,4 @@ spack: providers: mpi: [intel-oneapi-mpi] # Prefer Intel MPI compiler: [intel] # Prefer Intel Classic Compilers - target: [icelake] + require: 'target=icelake' diff --git a/benchmarks/spack/csd3-rocky8/sapphirerapids/spack.yaml b/benchmarks/spack/csd3-rocky8/sapphirerapids/spack.yaml index ebdf8e76..08019898 100644 --- a/benchmarks/spack/csd3-rocky8/sapphirerapids/spack.yaml +++ b/benchmarks/spack/csd3-rocky8/sapphirerapids/spack.yaml @@ -15,4 +15,8 @@ spack: providers: mpi: [intel-oneapi-mpi] # Prefer Intel MPI compiler: [intel] # Prefer Intel Classic Compilers - target: [sapphirerapids] + require: 'target=sapphirerapids' + concretizer: + targets: + # Allow targetting sapphirerapids when building on icelake login node + host_compatible: false From 42c63e2d932de66d78d836c375a9ba6d7cbf8490 Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Thu, 23 Nov 2023 17:47:21 +0000 Subject: [PATCH 11/22] Reframe: Update CSD3 configurations Set the environment variables explicitly rather than using the rhel?/default-* modules so as to avoid any environment pollution from default modules. --- benchmarks/reframe_config.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/benchmarks/reframe_config.py b/benchmarks/reframe_config.py index 8c91c2fb..5131604a 100644 --- a/benchmarks/reframe_config.py +++ b/benchmarks/reframe_config.py @@ -78,7 +78,11 @@ def spack_root_to_path(): 'descr': 'Ice Lake compute nodes', 'scheduler': 'slurm', 'launcher': 'mpirun', - 'modules': ['rhel8/default-icl'], + 'env_vars': [ + ['I_MPI_PMI_LIBRARY', '/usr/local/software/slurm/current-rhel8/lib/libpmi2.so'], + ['I_MPI_OFI_PROVIDER', 'mlx'], + ['UCX_NET_DEVICES', 'mlx5_0:1'], + ], 'access': ['--partition=icelake', '--exclusive'], 'environs': ['default'], 'max_jobs': 64, @@ -95,7 +99,11 @@ def spack_root_to_path(): 'descr': 'Sapphire Rapids compute nodes', 'scheduler': 'slurm', 'launcher': 'mpirun', - 'modules': ['rhel8/default-sar'], + 'env_vars': [ + ['I_MPI_PMI_LIBRARY', '/usr/local/software/slurm/current-rhel8/lib/libpmi2.so'], + ['I_MPI_OFI_PROVIDER', 'mlx'], + ['UCX_NET_DEVICES', 'mlx5_0:1'], + ], 'access': ['--partition=sapphire', '--exclusive'], 'environs': ['default'], 'max_jobs': 64, @@ -121,7 +129,11 @@ def spack_root_to_path(): 'descr': 'Cascade Lake compute nodes', 'scheduler': 'slurm', 'launcher': 'mpirun', - 'modules': ['rhel7/default-ccl'], + 'env_vars': [ + ['I_MPI_PMI_LIBRARY', '/usr/local/software/slurm/current/lib/libpmi2.so'], + ['I_MPI_OFI_PROVIDER', 'mlx'], + ['UCX_NET_DEVICES', 'mlx5_0:1'], + ], 'access': ['--partition=cclake', '--exclusive'], 'environs': ['default'], 'max_jobs': 64, From 5ec019fedff40480f5bb40f624cfbeced092b86e Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Thu, 23 Nov 2023 17:54:30 +0000 Subject: [PATCH 12/22] Spack: Add more external packages to csd3 environments --- .../spack/csd3-centos7/cascadelake/spack.yaml | 57 ++++++++++++++++++- benchmarks/spack/csd3-rocky8/packages.yaml | 8 +++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml b/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml index 2d05edf1..e35bd873 100644 --- a/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml +++ b/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml @@ -157,6 +157,45 @@ spack: environment: {} extra_rpaths: - /usr/local/Cluster-Apps/intel/2020.4/compilers_and_libraries_2020.4.304/linux/lib + - compiler: + spec: dpcpp@=2022.0.0 + paths: + cc: /usr/local/software/intel/oneapi/2022.1/compiler/2022.0.2/linux/bin/icx + cxx: /usr/local/software/intel/oneapi/2022.1/compiler/2022.0.2/linux/bin/dpcpp + f77: /usr/local/software/intel/oneapi/2022.1/compiler/2022.0.2/linux/bin/ifx + fc: /usr/local/software/intel/oneapi/2022.1/compiler/2022.0.2/linux/bin/ifx + flags: {} + operating_system: centos7 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: intel@=2021.5.0 + paths: + cc: /usr/local/software/intel/oneapi/2022.1/compiler/2022.0.2/linux/bin/intel64/icc + cxx: /usr/local/software/intel/oneapi/2022.1/compiler/2022.0.2/linux/bin/intel64/icpc + f77: /usr/local/software/intel/oneapi/2022.1/compiler/2022.0.2/linux/bin/intel64/ifort + fc: /usr/local/software/intel/oneapi/2022.1/compiler/2022.0.2/linux/bin/intel64/ifort + flags: {} + operating_system: centos7 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: oneapi@=2022.0.0 + paths: + cc: /usr/local/software/intel/oneapi/2022.1/compiler/2022.0.2/linux/bin/icx + cxx: /usr/local/software/intel/oneapi/2022.1/compiler/2022.0.2/linux/bin/icpx + f77: /usr/local/software/intel/oneapi/2022.1/compiler/2022.0.2/linux/bin/ifx + fc: /usr/local/software/intel/oneapi/2022.1/compiler/2022.0.2/linux/bin/ifx + flags: {} + operating_system: centos7 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] packages: all: providers: @@ -241,6 +280,10 @@ spack: externals: - spec: gmake@3.82 prefix: /usr + gmp: + externals: + - spec: gmp@6.1.2 + prefix: /usr/local/software/spack/spack-0.11.2/opt/spack/linux-rhel7-x86_64/gcc-5.4.0/gmp-6.1.2-sdvx5boug4isdzywdxgaqzwwor46yden groff: externals: - spec: groff@1.22.2 @@ -255,6 +298,10 @@ spack: externals: - spec: m4@1.4.16 prefix: /usr + mpfr: + externals: + - spec: mpfr@4.0.1 + prefix: /usr/local/software/spack/current/opt/spack/linux-rhel7-x86_64/gcc-5.4.0/mpfr-4.0.1-bernlygfdt6uk3fgrmwvyqssc7zzlwsw pkg-config: externals: - spec: pkg-config@0.27.1 @@ -285,7 +332,7 @@ spack: prefix: /usr/local/software/hdf5/1.10.3 - spec: hdf5@1.10.5%gcc@9.3.0+mpi^openmpi prefix: /usr/local/Cluster-Apps/hdf5/openmpi/gcc/9.2/1.10.5 - - spec: hdf5@1.10.5%intel+mpi^intel-mpi + - spec: hdf5@1.10.5 +mpi^intel-mpi prefix: /usr/local/Cluster-Apps/hdf5/mpi/intel/2019.3/1.10.5 - spec: hdf5@1.12.0%gcc@9.3.0+mpi^openmpi prefix: /usr/local/Cluster-Apps/hdf5/openmpi/gcc/9.3/1.12.0 @@ -293,6 +340,10 @@ spack: externals: - spec: intel-mkl@2020.4.304 prefix: /usr/local/Cluster-Apps/intel/2020.4/compilers_and_libraries_2020.4.304/linux/mkl + intel-oneapi-mkl: + externals: + - spec: intel-oneapi-mkl@2022.0.2 + prefix: /usr/local/software/intel/oneapi/2022.1 intel-mpi: externals: - spec: intel-mpi@2017.4.239 @@ -305,6 +356,10 @@ spack: prefix: /usr/local/Cluster-Apps/intel/2020.2/compilers_and_libraries_2020.2.254/linux/mpi/intel64 - spec: intel-mpi@2020.4.304 prefix: /usr/local/Cluster-Apps/intel/2020.4/compilers_and_libraries_2020.4.304/linux/mpi/intel64 + intel-oneapi-mpi: + externals: + - spec: intel-oneapi-mpi@2021.5.1 + prefix: /usr/local/software/intel/oneapi/2022.1 openmpi: externals: - spec: openmpi@4.0.4 diff --git a/benchmarks/spack/csd3-rocky8/packages.yaml b/benchmarks/spack/csd3-rocky8/packages.yaml index 5373b3b0..cd02b94a 100644 --- a/benchmarks/spack/csd3-rocky8/packages.yaml +++ b/benchmarks/spack/csd3-rocky8/packages.yaml @@ -75,6 +75,10 @@ packages: externals: - spec: gmake@4.2.1 prefix: /usr + gmp: + externals: + - spec: gmp@6.2.1 + prefix: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/gmp-6.2.1/gcc-11.3.0/f3azme3k3yopgg3h7cab2v3ctqvvubng groff: externals: - spec: groff@1.22.3 @@ -110,6 +114,10 @@ packages: externals: - spec: m4@1.4.18 prefix: /usr + mpfr: + externals: + - spec: mpfr@4.1.0 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/gcc-11.3.0/mpfr-4.1.0-cgv6xkpzaje632spokpqhtmacqmjk2bk openmpi: externals: - spec: openmpi@4.1.5~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~java+legacylaunchers~lustre~memchecker+pmi+romio+rsh~singularity+static+vt+wrapper-rpath From 5a0346ab0b0a4eed3d719f21bb199764107aa787 Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Thu, 23 Nov 2023 17:55:45 +0000 Subject: [PATCH 13/22] Swift: Update CSD3 reference values --- benchmarks/apps/swift/swift.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/benchmarks/apps/swift/swift.py b/benchmarks/apps/swift/swift.py index 0e944d42..a4002e7f 100644 --- a/benchmarks/apps/swift/swift.py +++ b/benchmarks/apps/swift/swift.py @@ -23,11 +23,14 @@ class SwiftBenchmark(SpackTest): 'cosma8': { 'duration': (50, None, 0.2, 'seconds'), }, - 'csd3-skylake': { - 'duration': (50, None, 0.2, 'seconds'), + 'csd3-centos7:cascadelake': { + 'duration': (37, None, 0.2, 'seconds'), + }, + 'csd3-rocky8:icelake': { + 'duration': (35, None, 0.2, 'seconds'), }, - 'csd3-icelake': { - 'duration': (350, None, 0.2, 'seconds'), + 'csd3-rocky8:sapphirerapids': { + 'duration': (32, None, 0.2, 'seconds'), }, 'dial3': { 'duration': (150, None, 0.2, 'seconds'), From 064886a8cd82c12f6c14fe6bf9a94e3c2b04200a Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Thu, 23 Nov 2023 17:57:10 +0000 Subject: [PATCH 14/22] HPL: Update with new CSD3 partition names --- .../cascadelake}/1/HPL.dat | 0 .../icelake}/1/HPL.dat | 0 .../hpl/csd3-rocky8/sapphirerapids/1/HPL.dat | 31 +++++++++++++++++++ benchmarks/apps/hpl/hpl.py | 18 ++++++++--- 4 files changed, 44 insertions(+), 5 deletions(-) rename benchmarks/apps/hpl/{csd3-icelake => csd3-centos7/cascadelake}/1/HPL.dat (100%) rename benchmarks/apps/hpl/{csd3-skylake => csd3-rocky8/icelake}/1/HPL.dat (100%) create mode 100644 benchmarks/apps/hpl/csd3-rocky8/sapphirerapids/1/HPL.dat diff --git a/benchmarks/apps/hpl/csd3-icelake/1/HPL.dat b/benchmarks/apps/hpl/csd3-centos7/cascadelake/1/HPL.dat similarity index 100% rename from benchmarks/apps/hpl/csd3-icelake/1/HPL.dat rename to benchmarks/apps/hpl/csd3-centos7/cascadelake/1/HPL.dat diff --git a/benchmarks/apps/hpl/csd3-skylake/1/HPL.dat b/benchmarks/apps/hpl/csd3-rocky8/icelake/1/HPL.dat similarity index 100% rename from benchmarks/apps/hpl/csd3-skylake/1/HPL.dat rename to benchmarks/apps/hpl/csd3-rocky8/icelake/1/HPL.dat diff --git a/benchmarks/apps/hpl/csd3-rocky8/sapphirerapids/1/HPL.dat b/benchmarks/apps/hpl/csd3-rocky8/sapphirerapids/1/HPL.dat new file mode 100644 index 00000000..bcd59616 --- /dev/null +++ b/benchmarks/apps/hpl/csd3-rocky8/sapphirerapids/1/HPL.dat @@ -0,0 +1,31 @@ +HPLinpack benchmark input file +Innovative Computing Laboratory, University of Tennessee +HPL.out output file name (if any) +6 device out (6=stdout,7=stderr,file) +1 # of problems sizes (N) +137472 Ns # ~80% mem +1 # of NBs +384 NBs +1 PMAP process mapping (0=Row-,1=Column-major) +1 # of process grids (P x Q) +1 Ps +1 Qs +16.0 threshold +1 # of panel fact +2 1 0 PFACTs (0=left, 1=Crout, 2=Right) +1 # of recursive stopping criterium +2 NBMINs (>= 1) +1 # of panels in recursion +2 NDIVs +1 # of recursive panel fact. +1 0 2 RFACTs (0=left, 1=Crout, 2=Right) +1 # of broadcast +0 BCASTs (0=1rg,1=1rM,2=2rg,3=2rM,4=Lng,5=LnM) +1 # of lookahead depth +0 DEPTHs (>=0) +0 SWAP (0=bin-exch,1=long,2=mix) +1 swapping threshold +1 L1 in (0=transposed,1=no-transposed) form +1 U in (0=transposed,1=no-transposed) form +0 Equilibration (0=no,1=yes) +8 memory alignment in double (> 0) diff --git a/benchmarks/apps/hpl/hpl.py b/benchmarks/apps/hpl/hpl.py index 1c676931..61be93a0 100644 --- a/benchmarks/apps/hpl/hpl.py +++ b/benchmarks/apps/hpl/hpl.py @@ -28,12 +28,15 @@ class Hpl(SpackTest): # Dictionary of reference values, indexed by number of tasks. full_reference = { 1: { - 'csd3-skylake': { - 'Gflops': (2000, -0.2, None, 'Gflops'), + 'csd3-centos7:cascadelake': { + 'Gflops': (2600, -0.2, None, 'Gflops'), }, - 'csd3-icelake': { + 'csd3-rocky8:icelake': { 'Gflops': (4500, -0.2, None, 'Gflops'), }, + 'csd3-rocky8:sapphirerapids': { + 'Gflops': (6200, -0.2, None, 'Gflops'), + }, }, } @@ -42,7 +45,9 @@ def set_sourcesdir(self): if self.config_dir: self.sourcesdir = self.config_dir else: - self.sourcesdir = path.join(path.dirname(__file__), self.current_system.name, str(self.num_tasks)) + self.sourcesdir = path.join(path.dirname(__file__), + self.current_system.name, self.current_partition.name, + str(self.num_tasks)) @run_before('sanity') def set_sanity_patterns(self): @@ -66,7 +71,10 @@ def set_perf_patterns(self): } # If we have a reference for current combination of system + number of # tasks, use it, otherwise default to generic empty reference. - if self.num_tasks in self.full_reference.keys() and self.current_system.name in self.full_reference[self.num_tasks].keys(): + if (self.num_tasks in self.full_reference.keys() and + (self.current_system.name in self.full_reference[self.num_tasks].keys() + or self.current_system.name + ':' + self.current_partition.name + in self.full_reference[self.num_tasks].keys())): self.reference = self.full_reference[self.num_tasks] else: self.reference = { From e0486a75ba87853c52530d43e5a1693ddacee981 Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Fri, 24 Nov 2023 17:25:58 +0000 Subject: [PATCH 15/22] Spack: Add more external packages to csd3 environments --- benchmarks/spack/csd3-centos7/cascadelake/spack.yaml | 8 ++++++++ benchmarks/spack/csd3-rocky8/packages.yaml | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml b/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml index e35bd873..314973f2 100644 --- a/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml +++ b/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml @@ -336,6 +336,10 @@ spack: prefix: /usr/local/Cluster-Apps/hdf5/mpi/intel/2019.3/1.10.5 - spec: hdf5@1.12.0%gcc@9.3.0+mpi^openmpi prefix: /usr/local/Cluster-Apps/hdf5/openmpi/gcc/9.3/1.12.0 + intel-oneapi-ipp: + externals: + - spec: intel-oneapi-ipp@2021.5.2 + prefix: /usr/local/software/intel/oneapi/2022.1 intel-mkl: externals: - spec: intel-mkl@2020.4.304 @@ -360,6 +364,10 @@ spack: externals: - spec: intel-oneapi-mpi@2021.5.1 prefix: /usr/local/software/intel/oneapi/2022.1 + intel-oneapi-tbb: + externals: + - spec: intel-oneapi-tbb@2021.5.1 + prefix: /usr/local/software/intel/oneapi/2022.1 openmpi: externals: - spec: openmpi@4.0.4 diff --git a/benchmarks/spack/csd3-rocky8/packages.yaml b/benchmarks/spack/csd3-rocky8/packages.yaml index cd02b94a..7d61ac68 100644 --- a/benchmarks/spack/csd3-rocky8/packages.yaml +++ b/benchmarks/spack/csd3-rocky8/packages.yaml @@ -106,6 +106,12 @@ packages: prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/intel-2021.6.0/intel-oneapi-mpi-2021.6.0-guxuvcpmykplbrr2e3af2yd7njqhau5e - spec: intel-oneapi-mpi@2021.6.0~external-libfabric~ilp64 %gcc@11.3.0 prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/gcc-11.3.0/intel-oneapi-mpi-2021.6.0-2yn3govbxvgmaanmi4wpwjrqa2b3lthl + intel-oneapi-tbb: + externals: + - spec: intel-oneapi-tbb@2021.6.0 %intel@2021.6.0 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/intel-2021.6.0/intel-oneapi-tbb-2021.6.0-2kqmsptsinryemz5gyirbg5umohppbev + - spec: intel-oneapi-tbb@2021.6.0 %gcc@11.3.0 + prefix: /usr/local/software/spack/spack-rhel8-20210927/opt/spack/linux-rocky8-icelake/gcc-11.3.0/intel-oneapi-tbb-2021.6.0-wf2uirslvvxv6vb6fsk2cd7dyyvnrvgb libtool: externals: - spec: libtool@2.4.6 From 3f3fe23f17a377de65698b619dc758d787405680 Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Fri, 24 Nov 2023 17:26:58 +0000 Subject: [PATCH 16/22] Grid: Update csd3 reference values --- benchmarks/apps/grid/grid.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/benchmarks/apps/grid/grid.py b/benchmarks/apps/grid/grid.py index a28286f9..430deba0 100644 --- a/benchmarks/apps/grid/grid.py +++ b/benchmarks/apps/grid/grid.py @@ -38,11 +38,14 @@ class GridBenchmark_ITT(GridBenchmark): 'cosma8': { 'Performance': (425000, None, None, 'Mflop/s per node') }, - 'csd3-icelake': { - 'Performance': (450000, None, None, 'Mflop/s per node') + 'csd3-centos7:cascadelake': { + 'Performance': (350000, None, None, 'Mflop/s per node') + }, + 'csd3-rocky8:icelake': { + 'Performance': (500000, None, None, 'Mflop/s per node') }, - 'csd3-skylake': { - 'Performance': (22000, None, None, 'Mflop/s per node') + 'csd3-rocky8:sapphirerapids': { + 'Performance': (1000000, None, None, 'Mflop/s per node') }, 'dial3': { 'Performance': (28000, None, None, 'Mflop/s per node') From 85fde8bf4272cdb52466f7ffb35cdb0ca82ddaff Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Tue, 28 Nov 2023 14:20:11 +0000 Subject: [PATCH 17/22] CSD3: Fix incorrect cascadelake config --- benchmarks/reframe_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/reframe_config.py b/benchmarks/reframe_config.py index 5131604a..6b19653a 100644 --- a/benchmarks/reframe_config.py +++ b/benchmarks/reframe_config.py @@ -139,7 +139,7 @@ def spack_root_to_path(): 'max_jobs': 64, 'processor': { 'num_cpus': 56, - 'num_cpus_per_core': 2, + 'num_cpus_per_core': 1, 'num_sockets': 2, 'num_cpus_per_socket': 28, } From 4980485e04937e0fd92ec142354184331e2acc4e Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Thu, 7 Dec 2023 14:55:59 +0000 Subject: [PATCH 18/22] CSD3: Modify ReFrame configuration Always request all of the memory on a node as `--exclusive` does not imply this according to the current SLURM documentation (https://slurm.schedmd.com/sbatch.html#OPT_exclusive). This can make a difference to benchmark results when not using all of the cores on a node due to the increased memory bandwidth available. Also increase the job_submit_timeout as the SLURM controller can be a bit slow on CSD3. --- benchmarks/reframe_config.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/benchmarks/reframe_config.py b/benchmarks/reframe_config.py index 6b19653a..e40ace5c 100644 --- a/benchmarks/reframe_config.py +++ b/benchmarks/reframe_config.py @@ -83,7 +83,10 @@ def spack_root_to_path(): ['I_MPI_OFI_PROVIDER', 'mlx'], ['UCX_NET_DEVICES', 'mlx5_0:1'], ], - 'access': ['--partition=icelake', '--exclusive'], + 'access': ['--partition=icelake', '--exclusive', '--mem=256120'], + 'sched_options': { + 'job_submit_timeout': 120, + }, 'environs': ['default'], 'max_jobs': 64, 'processor': { @@ -94,7 +97,6 @@ def spack_root_to_path(): }, }, { - # https://docs.hpc.cam.ac.uk/hpc/user-guide/icelake.html 'name': 'sapphirerapids', 'descr': 'Sapphire Rapids compute nodes', 'scheduler': 'slurm', @@ -104,7 +106,10 @@ def spack_root_to_path(): ['I_MPI_OFI_PROVIDER', 'mlx'], ['UCX_NET_DEVICES', 'mlx5_0:1'], ], - 'access': ['--partition=sapphire', '--exclusive'], + 'access': ['--partition=sapphire', '--exclusive', '--mem=512960'], + 'sched_options': { + 'job_submit_timeout': 120, + }, 'environs': ['default'], 'max_jobs': 64, 'processor': { @@ -134,7 +139,10 @@ def spack_root_to_path(): ['I_MPI_OFI_PROVIDER', 'mlx'], ['UCX_NET_DEVICES', 'mlx5_0:1'], ], - 'access': ['--partition=cclake', '--exclusive'], + 'access': ['--partition=cclake', '--exclusive', '--mem=191520'], + 'sched_options': { + 'job_submit_timeout': 120, + }, 'environs': ['default'], 'max_jobs': 64, 'processor': { From 28b717386de56f698ed1a208c2c7120d119c4c55 Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Thu, 7 Dec 2023 15:25:19 +0000 Subject: [PATCH 19/22] Sombrero Example: Update CSD3 reference values Note that these values are when it is built with the Intel Classic Compiler which is preferred in the Spack environments. --- benchmarks/examples/sombrero/sombrero.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/benchmarks/examples/sombrero/sombrero.py b/benchmarks/examples/sombrero/sombrero.py index 3245987c..4c995022 100644 --- a/benchmarks/examples/sombrero/sombrero.py +++ b/benchmarks/examples/sombrero/sombrero.py @@ -69,11 +69,14 @@ class SombreroBenchmark(SpackTest): 'cosma8': { 'flops': (3.8, -0.2, None, 'Gflops/seconds'), }, - 'csd3-skylake': { - 'flops': (1.2, -0.2, None, 'Gflops/seconds'), + 'csd3-centos7:cascadelake': { + 'flops': (8.0, -0.2, None, 'Gflops/seconds'), }, - 'csd3-icelake': { - 'flops': (1.5, -0.2, None, 'Gflops/seconds'), + 'csd3-rocky8:icelake': { + 'flops': (6.0, -0.2, None, 'Gflops/seconds'), + }, + 'csd3-rocky8:sapphirerapids': { + 'flops': (8.0, -0.2, None, 'Gflops/seconds'), }, 'dial3': { 'flops': (1.2, -0.2, None, 'Gflops/seconds'), @@ -101,7 +104,7 @@ class SombreroBenchmark(SpackTest): }, '*': { 'flops': (1, None, None, 'Gflops/seconds'), - }, + }, } def __init__(self): From d2c09dab1e925a94bd395f84e32062f1742986c2 Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Tue, 12 Dec 2023 09:04:53 +0000 Subject: [PATCH 20/22] Spack: Alphabetise CSD3 external packages Also remove libtool@2.4.2 from cascadelake environment as it is deprecated. --- .../spack/csd3-centos7/cascadelake/spack.yaml | 146 +++++++++--------- benchmarks/spack/csd3-rocky8/packages.yaml | 64 ++++---- 2 files changed, 104 insertions(+), 106 deletions(-) diff --git a/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml b/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml index 314973f2..e0d22f40 100644 --- a/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml +++ b/benchmarks/spack/csd3-centos7/cascadelake/spack.yaml @@ -202,40 +202,6 @@ spack: mpi: [intel-mpi@2020.4.304] # Prefer Intel MPI compiler: [intel@19.1.3.304] # Prefer Intel Classic Compilers target: [cascadelake] - binutils: - externals: - - spec: binutils@2.27.44 - prefix: /usr - coreutils: - externals: - - spec: coreutils@8.22 - prefix: /usr - diffutils: - externals: - - spec: diffutils@3.3 - prefix: /usr - findutils: - externals: - - spec: findutils@4.5.11 - prefix: /usr - gawk: - externals: - - spec: gawk@4.0.2 - prefix: /usr - openssh: - externals: - - spec: openssh@7.4p1 - prefix: /usr - openssl: - externals: - - spec: openssl@1.0.2k-fips - prefix: /usr - - spec: openssl@1.1.1k - prefix: /usr - tar: - externals: - - spec: tar@1.26 - prefix: /usr autoconf: externals: - spec: autoconf@2.69 @@ -244,6 +210,10 @@ spack: externals: - spec: automake@1.13.4 prefix: /usr + binutils: + externals: + - spec: binutils@2.27.44 + prefix: /usr bison: externals: - spec: bison@3.0.4 @@ -254,6 +224,10 @@ spack: prefix: /usr - spec: cmake@3.17.0 prefix: /usr/local/software/master/cmake/latest + coreutils: + externals: + - spec: coreutils@8.22 + prefix: /usr curl: externals: - spec: curl@7.29.0+ldap @@ -262,14 +236,26 @@ spack: externals: - spec: cvs@1.11.23 prefix: /usr + diffutils: + externals: + - spec: diffutils@3.3 + prefix: /usr doxygen: externals: - spec: doxygen@1.8.5+graphviz~mscgen prefix: /usr + findutils: + externals: + - spec: findutils@4.5.11 + prefix: /usr flex: externals: - spec: flex@2.5.37+lex prefix: /usr + gawk: + externals: + - spec: gawk@4.0.2 + prefix: /usr git: externals: - spec: git@1.8.3.1+tcltk @@ -288,38 +274,6 @@ spack: externals: - spec: groff@1.22.2 prefix: /usr - libtool: - externals: - - spec: libtool@2.4.2 - prefix: /usr - - spec: libtool@2.4 - prefix: /usr - m4: - externals: - - spec: m4@1.4.16 - prefix: /usr - mpfr: - externals: - - spec: mpfr@4.0.1 - prefix: /usr/local/software/spack/current/opt/spack/linux-rhel7-x86_64/gcc-5.4.0/mpfr-4.0.1-bernlygfdt6uk3fgrmwvyqssc7zzlwsw - pkg-config: - externals: - - spec: pkg-config@0.27.1 - prefix: /usr - subversion: - externals: - - spec: subversion@1.7.14 - prefix: /usr - - spec: subversion@1.12.2 - prefix: /usr/local/software/global - swig: - externals: - - spec: swig@2.0.10 - prefix: /usr - texinfo: - externals: - - spec: texinfo@5.1 - prefix: /usr gsl: externals: - spec: gsl@2.1 @@ -336,18 +290,10 @@ spack: prefix: /usr/local/Cluster-Apps/hdf5/mpi/intel/2019.3/1.10.5 - spec: hdf5@1.12.0%gcc@9.3.0+mpi^openmpi prefix: /usr/local/Cluster-Apps/hdf5/openmpi/gcc/9.3/1.12.0 - intel-oneapi-ipp: - externals: - - spec: intel-oneapi-ipp@2021.5.2 - prefix: /usr/local/software/intel/oneapi/2022.1 intel-mkl: externals: - spec: intel-mkl@2020.4.304 prefix: /usr/local/Cluster-Apps/intel/2020.4/compilers_and_libraries_2020.4.304/linux/mkl - intel-oneapi-mkl: - externals: - - spec: intel-oneapi-mkl@2022.0.2 - prefix: /usr/local/software/intel/oneapi/2022.1 intel-mpi: externals: - spec: intel-mpi@2017.4.239 @@ -360,6 +306,14 @@ spack: prefix: /usr/local/Cluster-Apps/intel/2020.2/compilers_and_libraries_2020.2.254/linux/mpi/intel64 - spec: intel-mpi@2020.4.304 prefix: /usr/local/Cluster-Apps/intel/2020.4/compilers_and_libraries_2020.4.304/linux/mpi/intel64 + intel-oneapi-ipp: + externals: + - spec: intel-oneapi-ipp@2021.5.2 + prefix: /usr/local/software/intel/oneapi/2022.1 + intel-oneapi-mkl: + externals: + - spec: intel-oneapi-mkl@2022.0.2 + prefix: /usr/local/software/intel/oneapi/2022.1 intel-oneapi-mpi: externals: - spec: intel-oneapi-mpi@2021.5.1 @@ -368,6 +322,18 @@ spack: externals: - spec: intel-oneapi-tbb@2021.5.1 prefix: /usr/local/software/intel/oneapi/2022.1 + libtool: + externals: + - spec: libtool@2.4 + prefix: /usr + m4: + externals: + - spec: m4@1.4.16 + prefix: /usr + mpfr: + externals: + - spec: mpfr@4.0.1 + prefix: /usr/local/software/spack/current/opt/spack/linux-rhel7-x86_64/gcc-5.4.0/mpfr-4.0.1-bernlygfdt6uk3fgrmwvyqssc7zzlwsw openmpi: externals: - spec: openmpi@4.0.4 @@ -378,10 +344,24 @@ spack: prefix: /usr/local/Cluster-Apps/openmpi/gnu/4.0.1-gcc-9.2 - spec: openmpi@4.0.0 prefix: /usr/local/software/spack/spack-0.11.2/opt/spack/linux-rhel7-x86_64/intel-17.0.8/openmpi-4.0.0-tnwn5io24eaoxhdxxcscx5jrsvlneqqx + openssh: + externals: + - spec: openssh@7.4p1 + prefix: /usr + openssl: + externals: + - spec: openssl@1.0.2k-fips + prefix: /usr + - spec: openssl@1.1.1k + prefix: /usr perl: externals: - spec: perl@5.16.3+cpanm+shared+threads prefix: /usr + pkg-config: + externals: + - spec: pkg-config@0.27.1 + prefix: /usr python: externals: - spec: python@3.8.2+bz2+ctypes+dbm+lzma+nis+pyexpat+readline+sqlite3+ssl+tix+tkinter+uuid+zlib @@ -390,3 +370,21 @@ spack: prefix: /usr - spec: python@3.6.8+bz2+ctypes+dbm+lzma+nis+pyexpat+readline+sqlite3+ssl~tix~tkinter+uuid+zlib prefix: /usr + subversion: + externals: + - spec: subversion@1.7.14 + prefix: /usr + - spec: subversion@1.12.2 + prefix: /usr/local/software/global + swig: + externals: + - spec: swig@2.0.10 + prefix: /usr + tar: + externals: + - spec: tar@1.26 + prefix: /usr + texinfo: + externals: + - spec: texinfo@5.1 + prefix: /usr diff --git a/benchmarks/spack/csd3-rocky8/packages.yaml b/benchmarks/spack/csd3-rocky8/packages.yaml index 7d61ac68..6859fccb 100644 --- a/benchmarks/spack/csd3-rocky8/packages.yaml +++ b/benchmarks/spack/csd3-rocky8/packages.yaml @@ -1,36 +1,4 @@ packages: - binutils: - externals: - - spec: binutils@2.30.119 - prefix: /usr - coreutils: - externals: - - spec: coreutils@8.30 - prefix: /usr - diffutils: - externals: - - spec: diffutils@3.6 - prefix: /usr - findutils: - externals: - - spec: findutils@4.6.0 - prefix: /usr - gawk: - externals: - - spec: gawk@4.2.1 - prefix: /usr - openssh: - externals: - - spec: openssh@8.0p1 - prefix: /usr - openssl: - externals: - - spec: openssl@1.1.1k - prefix: /usr - tar: - externals: - - spec: tar@1.30 - prefix: /usr autoconf: externals: - spec: autoconf@2.69 @@ -39,6 +7,10 @@ packages: externals: - spec: automake@1.16.1 prefix: /usr + binutils: + externals: + - spec: binutils@2.30.119 + prefix: /usr bison: externals: - spec: bison@3.0.4 @@ -47,6 +19,10 @@ packages: externals: - spec: cmake@3.20.2 prefix: /usr + coreutils: + externals: + - spec: coreutils@8.30 + prefix: /usr curl: externals: - spec: curl@7.61.1+gssapi+ldap+nghttp2 @@ -55,14 +31,26 @@ packages: externals: - spec: cvs@1.11.23 prefix: /usr + diffutils: + externals: + - spec: diffutils@3.6 + prefix: /usr doxygen: externals: - spec: doxygen@1.8.14+graphviz~mscgen prefix: /usr + findutils: + externals: + - spec: findutils@4.6.0 + prefix: /usr flex: externals: - spec: flex@2.6.1+lex prefix: /usr + gawk: + externals: + - spec: gawk@4.2.1 + prefix: /usr git: externals: - spec: git@2.39.3+tcltk @@ -129,6 +117,14 @@ packages: - spec: openmpi@4.1.5~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~java+legacylaunchers~lustre~memchecker+pmi+romio+rsh~singularity+static+vt+wrapper-rpath fabrics=ucx schedulers=slurm %intel@2021.6.0 prefix: /usr/local/software/spack/spack-views/rocky8-icelake-20220710/openmpi-4.1.5/intel-2021.6.0/b42idtrxnxnhjeplfsf7xjlghp5kwvkz/ + openssh: + externals: + - spec: openssh@8.0p1 + prefix: /usr + openssl: + externals: + - spec: openssl@1.1.1k + prefix: /usr pkgconf: externals: - spec: pkgconf@1.4.2 @@ -151,6 +147,10 @@ packages: externals: - spec: swig@4.1.1 prefix: /usr + tar: + externals: + - spec: tar@1.30 + prefix: /usr texinfo: externals: - spec: texinfo@6.5 From 606dcbc8979d83579d4231389c141aed7fc5a6b5 Mon Sep 17 00:00:00 2001 From: Miren Radia Date: Tue, 12 Dec 2023 11:46:49 +0000 Subject: [PATCH 21/22] CSD3: Remove --mem flag Though this can improve performance, it can also lead to problems when submitting some jobs so leave it up to the user to add this if they want. --- benchmarks/reframe_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/reframe_config.py b/benchmarks/reframe_config.py index e40ace5c..002b0e30 100644 --- a/benchmarks/reframe_config.py +++ b/benchmarks/reframe_config.py @@ -83,7 +83,7 @@ def spack_root_to_path(): ['I_MPI_OFI_PROVIDER', 'mlx'], ['UCX_NET_DEVICES', 'mlx5_0:1'], ], - 'access': ['--partition=icelake', '--exclusive', '--mem=256120'], + 'access': ['--partition=icelake', '--exclusive'], 'sched_options': { 'job_submit_timeout': 120, }, @@ -106,7 +106,7 @@ def spack_root_to_path(): ['I_MPI_OFI_PROVIDER', 'mlx'], ['UCX_NET_DEVICES', 'mlx5_0:1'], ], - 'access': ['--partition=sapphire', '--exclusive', '--mem=512960'], + 'access': ['--partition=sapphire', '--exclusive'], 'sched_options': { 'job_submit_timeout': 120, }, @@ -139,7 +139,7 @@ def spack_root_to_path(): ['I_MPI_OFI_PROVIDER', 'mlx'], ['UCX_NET_DEVICES', 'mlx5_0:1'], ], - 'access': ['--partition=cclake', '--exclusive', '--mem=191520'], + 'access': ['--partition=cclake', '--exclusive'], 'sched_options': { 'job_submit_timeout': 120, }, From aced6b9fa6fd45824e2dd39e0e92d25c6c587ec4 Mon Sep 17 00:00:00 2001 From: Miren Radia <32646026+mirenradia@users.noreply.github.com> Date: Tue, 12 Dec 2023 14:02:16 +0000 Subject: [PATCH 22/22] Sombrero: Fix indentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mosè Giordano --- benchmarks/examples/sombrero/sombrero.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/examples/sombrero/sombrero.py b/benchmarks/examples/sombrero/sombrero.py index 4c995022..aa15e40e 100644 --- a/benchmarks/examples/sombrero/sombrero.py +++ b/benchmarks/examples/sombrero/sombrero.py @@ -104,7 +104,7 @@ class SombreroBenchmark(SpackTest): }, '*': { 'flops': (1, None, None, 'Gflops/seconds'), - }, + }, } def __init__(self):