diff --git a/benchmarks/modules/utils.py b/benchmarks/modules/utils.py
index 95b9b336..7b50e227 100644
--- a/benchmarks/modules/utils.py
+++ b/benchmarks/modules/utils.py
@@ -303,7 +303,10 @@ def set_sge_num_slots(self):
num_tasks = self.num_tasks or 1
num_cpus_per_task = self.num_cpus_per_task or 1
# Set the total number of CPUs to be requested for the SGE scheduler.
- self.extra_resources['mpi'] = {'num_slots': num_tasks * num_cpus_per_task}
+ if self.current_system.name == 'kathleen':
+ self.extra_resources['mpi'] = {'num_slots': max(41, num_tasks * num_cpus_per_task)}
+ else:
+ self.extra_resources['mpi'] = {'num_slots': num_tasks * num_cpus_per_task}
@run_before('compile')
def set_isambard_memory(self):
diff --git a/benchmarks/reframe_config.py b/benchmarks/reframe_config.py
index 7818900c..790e18d4 100644
--- a/benchmarks/reframe_config.py
+++ b/benchmarks/reframe_config.py
@@ -188,6 +188,41 @@ def spack_root_to_path():
},
]
}, # end CSD3 CentOS 7
+ {
+ # https://www.rc.ucl.ac.uk/docs/Clusters/Kathleen/#node-types
+ 'name': 'kathleen',
+ 'descr': 'Kathleen',
+ 'hostnames': ['login[0-9]+.kathleen.ucl.ac.uk'],
+ 'max_local_jobs': 1,
+ 'partitions': [
+ {
+ 'name': 'compute-node',
+ 'descr': 'Kathleen compute nodes',
+ 'scheduler': 'sge',
+ 'launcher': 'mpirun',
+ 'environs': ['default'],
+ 'max_jobs': 36,
+ 'processor': {
+ 'num_cpus': 40,
+ 'num_cpus_per_core': 1,
+ 'num_sockets': 2,
+ 'num_cpus_per_socket': 20,
+ },
+ 'resources': [
+ {
+ 'name': 'mpi',
+ 'options': ['-pe mpi {num_slots}'],
+ },
+ {
+ # Disable hyperthreading (default).
+ # See https://www.rc.ucl.ac.uk/docs/Clusters/Kathleen/#hyperthreading for details
+ 'name': 'hyperthreads',
+ 'options': ['-l threads=2'],
+ },
+ ],
+ }
+ ]
+ }, # end Kathleen
{
# https://www.rc.ucl.ac.uk/docs/Clusters/Myriad/#node-types
'name': 'myriad',
diff --git a/benchmarks/spack/kathleen/compute-node/spack.yaml b/benchmarks/spack/kathleen/compute-node/spack.yaml
new file mode 100644
index 00000000..e3b90792
--- /dev/null
+++ b/benchmarks/spack/kathleen/compute-node/spack.yaml
@@ -0,0 +1,320 @@
+# This is a Spack Environment file.
+#
+# It describes a set of packages to be installed, along with
+# configuration settings.
+spack:
+ # add package specs to the `specs` list
+ specs: []
+ view: false
+ include:
+ - ../../common.yaml
+ compilers:
+ - compiler:
+ spec: intel@18.0.3
+ paths:
+ cc: /shared/ucl/apps/intel/2018.Update3/compilers_and_libraries_2018.3.222/linux/bin/intel64/icc
+ cxx: /shared/ucl/apps/intel/2018.Update3/compilers_and_libraries_2018.3.222/linux/bin/intel64/icpc
+ f77: /shared/ucl/apps/intel/2018.Update3/compilers_and_libraries_2018.3.222/linux/bin/intel64/ifort
+ fc: /shared/ucl/apps/intel/2018.Update3/compilers_and_libraries_2018.3.222/linux/bin/intel64/ifort
+ flags: {}
+ operating_system: rhel7
+ target: x86_64
+ modules: []
+ environment: {}
+ extra_rpaths: []
+ - compiler:
+ spec: intel@19.1.0.166
+ paths:
+ cc: /shared/ucl/apps/intel/2020/compilers_and_libraries_2020.0.166/linux/bin/intel64/icc
+ cxx: /shared/ucl/apps/intel/2020/compilers_and_libraries_2020.0.166/linux/bin/intel64/icpc
+ f77: /shared/ucl/apps/intel/2020/compilers_and_libraries_2020.0.166/linux/bin/intel64/ifort
+ fc: /shared/ucl/apps/intel/2020/compilers_and_libraries_2020.0.166/linux/bin/intel64/ifort
+ flags: {}
+ operating_system: rhel7
+ target: x86_64
+ modules: []
+ environment: {}
+ extra_rpaths: []
+ - compiler:
+ spec: gcc@4.8.5
+ paths:
+ cc: /usr/bin/gcc
+ cxx: null
+ f77: /usr/bin/gfortran
+ fc: /usr/bin/gfortran
+ flags: {}
+ operating_system: rhel7
+ target: x86_64
+ modules: []
+ environment: {}
+ extra_rpaths: []
+ - compiler:
+ spec: gcc@4.9.2
+ paths:
+ cc: /shared/ucl/apps/gcc/4.9.2/bin/gcc
+ cxx: /shared/ucl/apps/gcc/4.9.2/bin/g++
+ f77: /shared/ucl/apps/gcc/4.9.2/bin/gfortran
+ fc: /shared/ucl/apps/gcc/4.9.2/bin/gfortran
+ flags: {}
+ operating_system: rhel7
+ target: x86_64
+ modules: []
+ environment:
+ prepend_path:
+ LD_LIBRARY_PATH: /shared/ucl/apps/gcc/4.9.2/lib64
+ extra_rpaths: []
+ - compiler:
+ spec: gcc@7.3.0
+ paths:
+ cc: /shared/ucl/apps/gcc/7.3.0/bin/gcc
+ cxx: /shared/ucl/apps/gcc/7.3.0/bin/g++
+ f77: /shared/ucl/apps/gcc/7.3.0/bin/gfortran
+ fc: /shared/ucl/apps/gcc/7.3.0/bin/gfortran
+ flags: {}
+ operating_system: rhel7
+ target: x86_64
+ modules: []
+ environment:
+ prepend_path:
+ LD_LIBRARY_PATH: /shared/ucl/apps/gcc/7.3.0/lib64
+ extra_rpaths: []
+ - compiler:
+ spec: gcc@8.3.0
+ paths:
+ cc: /shared/ucl/apps/gcc/8.3.0/bin/gcc
+ cxx: /shared/ucl/apps/gcc/8.3.0/bin/g++
+ f77: /shared/ucl/apps/gcc/8.3.0/bin/gfortran
+ fc: /shared/ucl/apps/gcc/8.3.0/bin/gfortran
+ flags: {}
+ operating_system: rhel7
+ target: x86_64
+ modules: []
+ environment:
+ prepend_path:
+ LD_LIBRARY_PATH: /shared/ucl/apps/gcc/8.3.0/lib64
+ extra_rpaths: []
+ - compiler:
+ spec: gcc@9.2.0
+ paths:
+ cc: /shared/ucl/apps/gcc/9.2.0/bin/gcc
+ cxx: /shared/ucl/apps/gcc/9.2.0/bin/g++
+ f77: /shared/ucl/apps/gcc/9.2.0/bin/gfortran
+ fc: /shared/ucl/apps/gcc/9.2.0/bin/gfortran
+ flags: {}
+ operating_system: rhel7
+ target: x86_64
+ modules: []
+ environment:
+ prepend_path:
+ LD_LIBRARY_PATH: /shared/ucl/apps/gcc/9.2.0/lib64
+ extra_rpaths: []
+ - compiler:
+ spec: gcc@10.2.0
+ paths:
+ cc: /shared/ucl/apps/gcc/10.2.0-p95889/bin/gcc
+ cxx: /shared/ucl/apps/gcc/10.2.0-p95889/bin/g++
+ f77: /shared/ucl/apps/gcc/10.2.0-p95889/bin/gfortran
+ fc: /shared/ucl/apps/gcc/10.2.0-p95889/bin/gfortran
+ flags: {}
+ operating_system: rhel7
+ target: x86_64
+ modules: []
+ environment:
+ prepend_path:
+ LD_LIBRARY_PATH: /shared/ucl/apps/gcc/10.2.0-p95889/lib64
+ extra_rpaths: []
+ - compiler:
+ spec: dpcpp@=2022.1.0
+ paths:
+ cc: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/icx
+ cxx: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/dpcpp
+ f77: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/ifx
+ fc: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/ifx
+ flags: {}
+ operating_system: rhel7
+ target: x86_64
+ modules: []
+ environment: {}
+ extra_rpaths: []
+ - compiler:
+ spec: intel@=2021.6.0
+ paths:
+ cc: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/intel64/icc
+ cxx: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/intel64/icpc
+ f77: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/intel64/ifort
+ fc: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/intel64/ifort
+ flags: {}
+ operating_system: rhel7
+ target: x86_64
+ modules: []
+ environment: {}
+ extra_rpaths: []
+ - compiler:
+ spec: oneapi@=2022.1.0
+ paths:
+ cc: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/icx
+ cxx: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/icpx
+ f77: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/ifx
+ fc: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/ifx
+ flags: {}
+ operating_system: rhel7
+ target: x86_64
+ modules: []
+ environment: {}
+ extra_rpaths: []
+ packages:
+ autoconf:
+ externals:
+ - spec: autoconf@2.69
+ prefix: /usr
+ automake:
+ externals:
+ - spec: automake@1.13.4
+ prefix: /usr
+# binutils:
+# externals:
+# - spec: binutils@2.27.43
+# prefix: /usr
+ bzip2:
+ externals:
+ - spec: bzip2@1.0.6
+ prefix: /usr
+ cmake:
+ externals:
+ - spec: cmake@2.8.12.2
+ prefix: /usr
+ - spec: cmake@3.2.1
+ prefix: /shared/ucl/apps/cmake/3.2.1/gnu-4.9.2
+ - spec: cmake@3.7.2
+ prefix: /shared/ucl/apps/cmake/3.7.2/gnu-4.9.2
+ - spec: cmake@3.13.3
+ prefix: /shared/ucl/apps/cmake/3.13.3/gnu-4.9.2
+ - spec: cmake@3.19.1
+ prefix: /shared/ucl/apps/cmake/3.19.1/gnu-4.9.2
+ - spec: cmake@3.21.1
+ prefix: /shared/ucl/apps/cmake/3.21.1/gnu-4.9.2
+# coreutils:
+# externals:
+# - spec: coreutils@8.22
+# prefix: /usr
+ curl:
+ externals:
+ - spec: curl@7.29.0+ldap
+ prefix: /usr
+ diff:
+ externals:
+ - spec: diff@3.3
+ prefix: /usr
+ find:
+ externals:
+ - spec: find@4.5.11
+ prefix: /usr
+ flex:
+ externals:
+ - spec: flex@2.5.39~lex
+ prefix: /shared/ucl/apps/flex/2.5.39/gnu-4.9.2
+ gawk:
+ externals:
+ - spec: gawk@4.0.2
+ prefix: /usr
+ git:
+ externals:
+ - spec: git@2.32.0+tcltk
+ prefix: /shared/ucl/apps/git/2.32.0/gnu-4.9.2
+ - spec: git@1.8.3.1~tcltk
+ prefix: /usr
+ gmake:
+ externals:
+ - spec: gmake@3.82
+ prefix: /usr
+ groff:
+ externals:
+ - spec: groff@1.22.2
+ prefix: /usr
+ hwloc:
+ externals:
+ - spec: hwloc@1.11.12
+ prefix: /shared/ucl/apps/hwloc/1.11.12
+ intel-mkl:
+ externals:
+ - spec: intel-mkl@2018.3.222
+ prefix: /shared/ucl/apps/intel/2018.Update3/compilers_and_libraries_2018.3.222/linux/mkl
+ - spec: intel-mkl@2020.0.166
+ prefix: /shared/ucl/apps/intel/2020/compilers_and_libraries_2020.0.166/linux/mkl
+ intel-mpi:
+ externals:
+ - spec: intel-mpi@2018.3.222
+ prefix: /shared/ucl/apps/intel/2018.Update3
+ - spec: intel-mpi@2020.0.166
+ prefix: /shared/ucl/apps/intel/2020
+ intel-oneapi-mpi:
+ externals:
+ - spec: intel-oneapi-mpi@2021.6.0
+ prefix: /shared/ucl/apps/intel/2022.2/
+ libtool:
+ externals:
+ - spec: libtool@2.4.2
+ prefix: /usr
+ libxml2:
+ externals:
+ - spec: libxml2@2.9.4
+ prefix: /shared/ucl/apps/libxml2/2.9.4/gnu-4.9.2
+ libxsmm:
+ externals:
+ - spec: libxsmm@1.16.1
+ prefix: /shared/ucl/apps/cp2k/8.2/cp2k-8.2/tools/toolchain/install/libxsmm-1.16.1
+ m4:
+ externals:
+ - spec: m4@1.4.16
+ prefix: /usr
+ ninja:
+ externals:
+ - spec: ninja@1.10.2.git.kitware.jobserver-1
+ prefix: /shared/ucl/apps/python/bundles/python39-6.0.0/venv
+ numactl:
+ externals:
+ - spec: numactl@2.0.12
+ prefix: /shared/ucl/apps/numactl/2.0.12
+ openmpi:
+ externals:
+ - spec: openmpi@3.1.4%intel@18.0.3
+ prefix: /shared/ucl/apps/openmpi/3.1.4/intel-2018
+ - spec: openmpi@3.1.4%gcc@7.3.0
+ prefix: /shared/ucl/apps/openmpi/3.1.4/gnu-7.3.0
+ - spec: openmpi@3.1.5%gcc@9.2.0
+ prefix: /shared/ucl/apps/openmpi/3.1.5/gnu-9.2.0
+ - spec: openmpi@3.1.6%gcc@4.9.2
+ prefix: /shared/ucl/apps/openmpi/3.1.6/gnu-4.9.2
+ - spec: openmpi@4.0.3%gcc@4.9.2
+ prefix: /shared/ucl/apps/openmpi/4.0.3/gnu-4.9.2
+ - spec: openmpi@4.0.5%gcc@10.2.0
+ prefix: /shared/ucl/apps/openmpi/4.0.5/gnu-10.2.0
+ # # This is a newer version of OpenMPI but with an older version of GCC,
+ # # so that by default Spack prefers this over above versions, let's hide
+ # # it so that we can use GCC 10.2 out-of-the-box.
+ # - spec: openmpi@4.1.1%gcc@4.9.2
+ # prefix: /shared/ucl/apps/openmpi/4.1.1/gnu-4.9.2
+# openssh:
+# externals:
+# - spec: openssh@7.4p1
+# prefix: /usr
+ openssl:
+ externals:
+ - spec: openssl@1.0.2k-fips
+ prefix: /usr
+ perl:
+ externals:
+ - spec: perl@5.16.3~cpanm+shared+threads
+ prefix: /usr
+ pkg-config:
+ externals:
+ - spec: pkg-config@0.27.1
+ prefix: /usr
+ subversion:
+ externals:
+ - spec: subversion@1.14.1
+ prefix: /shared/ucl/apps/subversion/1.14.1
+ tar:
+ externals:
+ - spec: tar@1.26
+ prefix: /usr
diff --git a/docs/systems.md b/docs/systems.md
index ea036bba..195959d3 100644
--- a/docs/systems.md
+++ b/docs/systems.md
@@ -129,7 +129,7 @@ reframe -c benchmarks/examples/sombrero -r --performance-report --system isambar
You may also need to compile GPU applications on the compute nodes, as the login node does not have any GPUs (this really depends on the build system of the application at hand, whether it needs access to a GPU during the build or it is sufficient to have the GPU toolkit available).
-## Myriad
+## Myriad and Kathleen
### Python3 module