Merge pull request #280 from ukri-excalibur/ic/add-kathleen-config

Add kathleen config

benchmarks/modules/utils.py

@@ -303,7 +303,10 @@ def set_sge_num_slots(self):
             num_tasks = self.num_tasks or 1
             num_cpus_per_task = self.num_cpus_per_task or 1
             # Set the total number of CPUs to be requested for the SGE scheduler.
-            self.extra_resources['mpi'] = {'num_slots': num_tasks * num_cpus_per_task}
+            if self.current_system.name == 'kathleen':
+                self.extra_resources['mpi'] = {'num_slots': max(41, num_tasks * num_cpus_per_task)}
+            else:
+                self.extra_resources['mpi'] = {'num_slots': num_tasks * num_cpus_per_task}
 
     @run_before('compile')
     def set_isambard_memory(self):

benchmarks/reframe_config.py

@@ -188,6 +188,41 @@ def spack_root_to_path():
                 },
             ]
         },  # end CSD3 CentOS 7
+        {
+            # https://www.rc.ucl.ac.uk/docs/Clusters/Kathleen/#node-types
+            'name': 'kathleen',
+            'descr': 'Kathleen',
+            'hostnames': ['login[0-9]+.kathleen.ucl.ac.uk'],
+            'max_local_jobs': 1,
+            'partitions': [
+                {
+                    'name': 'compute-node',
+                    'descr': 'Kathleen compute nodes',
+                    'scheduler': 'sge',
+                    'launcher': 'mpirun',
+                    'environs': ['default'],
+                    'max_jobs': 36,
+                    'processor': {
+                        'num_cpus': 40,
+                        'num_cpus_per_core': 1,
+                        'num_sockets': 2,
+                        'num_cpus_per_socket': 20,
+                    },
+                    'resources': [
+                        {
+                            'name': 'mpi',
+                            'options': ['-pe mpi {num_slots}'],
+                        },
+                        {
+                            # Disable hyperthreading (default).
+                            # See https://www.rc.ucl.ac.uk/docs/Clusters/Kathleen/#hyperthreading for details
+                            'name': 'hyperthreads',
+                            'options': ['-l threads=2'],
+                        },
+                    ],
+                }
+            ]
+        }, # end Kathleen
         {
             # https://www.rc.ucl.ac.uk/docs/Clusters/Myriad/#node-types
             'name': 'myriad',

benchmarks/spack/kathleen/compute-node/spack.yaml

@@ -0,0 +1,320 @@
+# This is a Spack Environment file.
+#
+# It describes a set of packages to be installed, along with
+# configuration settings.
+spack:
+  # add package specs to the `specs` list
+  specs: []
+  view: false
+  include:
+  - ../../common.yaml
+  compilers:
+  - compiler:
+      spec: [email protected]
+      paths:
+        cc: /shared/ucl/apps/intel/2018.Update3/compilers_and_libraries_2018.3.222/linux/bin/intel64/icc
+        cxx: /shared/ucl/apps/intel/2018.Update3/compilers_and_libraries_2018.3.222/linux/bin/intel64/icpc
+        f77: /shared/ucl/apps/intel/2018.Update3/compilers_and_libraries_2018.3.222/linux/bin/intel64/ifort
+        fc: /shared/ucl/apps/intel/2018.Update3/compilers_and_libraries_2018.3.222/linux/bin/intel64/ifort
+      flags: {}
+      operating_system: rhel7
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+  - compiler:
+      spec: [email protected]
+      paths:
+        cc: /shared/ucl/apps/intel/2020/compilers_and_libraries_2020.0.166/linux/bin/intel64/icc
+        cxx: /shared/ucl/apps/intel/2020/compilers_and_libraries_2020.0.166/linux/bin/intel64/icpc
+        f77: /shared/ucl/apps/intel/2020/compilers_and_libraries_2020.0.166/linux/bin/intel64/ifort
+        fc: /shared/ucl/apps/intel/2020/compilers_and_libraries_2020.0.166/linux/bin/intel64/ifort
+      flags: {}
+      operating_system: rhel7
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+  - compiler:
+      spec: [email protected]
+      paths:
+        cc: /usr/bin/gcc
+        cxx: null
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      flags: {}
+      operating_system: rhel7
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+  - compiler:
+      spec: [email protected]
+      paths:
+        cc: /shared/ucl/apps/gcc/4.9.2/bin/gcc
+        cxx: /shared/ucl/apps/gcc/4.9.2/bin/g++
+        f77: /shared/ucl/apps/gcc/4.9.2/bin/gfortran
+        fc: /shared/ucl/apps/gcc/4.9.2/bin/gfortran
+      flags: {}
+      operating_system: rhel7
+      target: x86_64
+      modules: []
+      environment:
+        prepend_path:
+          LD_LIBRARY_PATH: /shared/ucl/apps/gcc/4.9.2/lib64
+      extra_rpaths: []
+  - compiler:
+      spec: [email protected]
+      paths:
+        cc: /shared/ucl/apps/gcc/7.3.0/bin/gcc
+        cxx: /shared/ucl/apps/gcc/7.3.0/bin/g++
+        f77: /shared/ucl/apps/gcc/7.3.0/bin/gfortran
+        fc: /shared/ucl/apps/gcc/7.3.0/bin/gfortran
+      flags: {}
+      operating_system: rhel7
+      target: x86_64
+      modules: []
+      environment:
+        prepend_path:
+          LD_LIBRARY_PATH: /shared/ucl/apps/gcc/7.3.0/lib64
+      extra_rpaths: []
+  - compiler:
+      spec: [email protected]
+      paths:
+        cc: /shared/ucl/apps/gcc/8.3.0/bin/gcc
+        cxx: /shared/ucl/apps/gcc/8.3.0/bin/g++
+        f77: /shared/ucl/apps/gcc/8.3.0/bin/gfortran
+        fc: /shared/ucl/apps/gcc/8.3.0/bin/gfortran
+      flags: {}
+      operating_system: rhel7
+      target: x86_64
+      modules: []
+      environment:
+        prepend_path:
+          LD_LIBRARY_PATH: /shared/ucl/apps/gcc/8.3.0/lib64
+      extra_rpaths: []
+  - compiler:
+      spec: [email protected]
+      paths:
+        cc: /shared/ucl/apps/gcc/9.2.0/bin/gcc
+        cxx: /shared/ucl/apps/gcc/9.2.0/bin/g++
+        f77: /shared/ucl/apps/gcc/9.2.0/bin/gfortran
+        fc: /shared/ucl/apps/gcc/9.2.0/bin/gfortran
+      flags: {}
+      operating_system: rhel7
+      target: x86_64
+      modules: []
+      environment:
+        prepend_path:
+          LD_LIBRARY_PATH: /shared/ucl/apps/gcc/9.2.0/lib64
+      extra_rpaths: []
+  - compiler:
+      spec: [email protected]
+      paths:
+        cc: /shared/ucl/apps/gcc/10.2.0-p95889/bin/gcc
+        cxx: /shared/ucl/apps/gcc/10.2.0-p95889/bin/g++
+        f77: /shared/ucl/apps/gcc/10.2.0-p95889/bin/gfortran
+        fc: /shared/ucl/apps/gcc/10.2.0-p95889/bin/gfortran
+      flags: {}
+      operating_system: rhel7
+      target: x86_64
+      modules: []
+      environment:
+        prepend_path:
+          LD_LIBRARY_PATH: /shared/ucl/apps/gcc/10.2.0-p95889/lib64
+      extra_rpaths: []
+  - compiler:
+      spec: dpcpp@=2022.1.0
+      paths:
+        cc: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/icx
+        cxx: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/dpcpp
+        f77: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/ifx
+        fc: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/ifx
+      flags: {}
+      operating_system: rhel7
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+  - compiler:
+      spec: intel@=2021.6.0
+      paths:
+        cc: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/intel64/icc
+        cxx: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/intel64/icpc
+        f77: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/intel64/ifort
+        fc: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/intel64/ifort
+      flags: {}
+      operating_system: rhel7
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+  - compiler:
+      spec: oneapi@=2022.1.0
+      paths:
+        cc: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/icx
+        cxx: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/icpx
+        f77: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/ifx
+        fc: /shared/ucl/apps/intel/2022.2/compiler/2022.1.0/linux/bin/ifx
+      flags: {}
+      operating_system: rhel7
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+  packages:
+    autoconf:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+    automake:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+#  binutils:
+#    externals:
+#    - spec: [email protected]
+#      prefix: /usr
+    bzip2:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+    cmake:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      - spec: [email protected]
+        prefix: /shared/ucl/apps/cmake/3.2.1/gnu-4.9.2
+      - spec: [email protected]
+        prefix: /shared/ucl/apps/cmake/3.7.2/gnu-4.9.2
+      - spec: [email protected]
+        prefix: /shared/ucl/apps/cmake/3.13.3/gnu-4.9.2
+      - spec: [email protected]
+        prefix: /shared/ucl/apps/cmake/3.19.1/gnu-4.9.2
+      - spec: [email protected]
+        prefix: /shared/ucl/apps/cmake/3.21.1/gnu-4.9.2
+#  coreutils:
+#    externals:
+#    - spec: [email protected]
+#      prefix: /usr
+    curl:
+      externals:
+      - spec: [email protected]+ldap
+        prefix: /usr
+    diff:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+    find:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+    flex:
+      externals:
+      - spec: [email protected]~lex
+        prefix: /shared/ucl/apps/flex/2.5.39/gnu-4.9.2
+    gawk:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+    git:
+      externals:
+      - spec: [email protected]+tcltk
+        prefix: /shared/ucl/apps/git/2.32.0/gnu-4.9.2
+      - spec: [email protected]~tcltk
+        prefix: /usr
+    gmake:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+    groff:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+    hwloc:
+      externals:
+      - spec: [email protected]
+        prefix: /shared/ucl/apps/hwloc/1.11.12
+    intel-mkl:
+      externals:
+      - spec: [email protected]
+        prefix: /shared/ucl/apps/intel/2018.Update3/compilers_and_libraries_2018.3.222/linux/mkl
+      - spec: [email protected]
+        prefix: /shared/ucl/apps/intel/2020/compilers_and_libraries_2020.0.166/linux/mkl
+    intel-mpi:
+      externals:
+      - spec: [email protected]
+        prefix: /shared/ucl/apps/intel/2018.Update3
+      - spec: [email protected]
+        prefix: /shared/ucl/apps/intel/2020
+    intel-oneapi-mpi:
+      externals:
+      - spec: [email protected]
+        prefix: /shared/ucl/apps/intel/2022.2/
+    libtool:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+    libxml2:
+      externals:
+      - spec: [email protected]
+        prefix: /shared/ucl/apps/libxml2/2.9.4/gnu-4.9.2
+    libxsmm:
+      externals:
+      - spec: [email protected]
+        prefix: /shared/ucl/apps/cp2k/8.2/cp2k-8.2/tools/toolchain/install/libxsmm-1.16.1
+    m4:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+    ninja:
+      externals:
+      - spec: [email protected]
+        prefix: /shared/ucl/apps/python/bundles/python39-6.0.0/venv
+    numactl:
+      externals:
+      - spec: [email protected]
+        prefix: /shared/ucl/apps/numactl/2.0.12
+    openmpi:
+      externals:
+      - spec: [email protected]%[email protected]
+        prefix: /shared/ucl/apps/openmpi/3.1.4/intel-2018
+      - spec: [email protected]%[email protected]
+        prefix: /shared/ucl/apps/openmpi/3.1.4/gnu-7.3.0
+      - spec: [email protected]%[email protected]
+        prefix: /shared/ucl/apps/openmpi/3.1.5/gnu-9.2.0
+      - spec: [email protected]%[email protected]
+        prefix: /shared/ucl/apps/openmpi/3.1.6/gnu-4.9.2
+      - spec: [email protected]%[email protected]
+        prefix: /shared/ucl/apps/openmpi/4.0.3/gnu-4.9.2
+      - spec: [email protected]%[email protected]
+        prefix: /shared/ucl/apps/openmpi/4.0.5/gnu-10.2.0
+    # # This is a newer version of OpenMPI but with an older version of GCC,
+    # # so that by default Spack prefers this over above versions, let's hide
+    # # it so that we can use GCC 10.2 out-of-the-box.
+    # - spec: [email protected]%[email protected]
+    #   prefix: /shared/ucl/apps/openmpi/4.1.1/gnu-4.9.2
+#  openssh:
+#    externals:
+#    - spec: [email protected]
+#      prefix: /usr
+    openssl:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+    perl:
+      externals:
+      - spec: [email protected]~cpanm+shared+threads
+        prefix: /usr
+    pkg-config:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+    subversion:
+      externals:
+      - spec: [email protected]
+        prefix: /shared/ucl/apps/subversion/1.14.1
+    tar:
+      externals:
+      - spec: [email protected]
+        prefix: /usr

docs/systems.md

@@ -129,7 +129,7 @@ reframe -c benchmarks/examples/sombrero -r --performance-report --system isambar
 
 You may also need to compile GPU applications on the compute nodes, as the login node does not have any GPUs (this really depends on the build system of the application at hand, whether it needs access to a GPU during the build or it is sufficient to have the GPU toolkit available).
 
-## Myriad
+## Myriad and Kathleen
 
 ### Python3 module