[CSD3] Update configurations and Spack environments

benchmarks/apps/grid/grid.py

@@ -38,11 +38,14 @@ class GridBenchmark_ITT(GridBenchmark):
         'cosma8': {
             'Performance': (425000, None, None, 'Mflop/s per node')
         },
-        'csd3-icelake': {
-            'Performance': (450000, None, None, 'Mflop/s per node')
+        'csd3-centos7:cascadelake': {
+            'Performance': (350000, None, None, 'Mflop/s per node')
+        },
+        'csd3-rocky8:icelake': {
+            'Performance': (500000, None, None, 'Mflop/s per node')
         },
-        'csd3-skylake': {
-            'Performance': (22000, None, None, 'Mflop/s per node')
+        'csd3-rocky8:sapphirerapids': {
+            'Performance': (1000000, None, None, 'Mflop/s per node')
         },
         'dial3': {
             'Performance': (28000, None, None, 'Mflop/s per node')

benchmarks/apps/hpl/csd3-rocky8/sapphirerapids/1/HPL.dat

@@ -0,0 +1,31 @@
+HPLinpack benchmark input file
+Innovative Computing Laboratory, University of Tennessee
+HPL.out      output file name (if any)
+6            device out (6=stdout,7=stderr,file)
+1            # of problems sizes (N)
+137472        Ns # ~80% mem
+1            # of NBs
+384          NBs
+1            PMAP process mapping (0=Row-,1=Column-major)
+1            # of process grids (P x Q)
+1            Ps
+1            Qs
+16.0         threshold
+1            # of panel fact
+2 1 0        PFACTs (0=left, 1=Crout, 2=Right)
+1            # of recursive stopping criterium
+2            NBMINs (>= 1)
+1            # of panels in recursion
+2            NDIVs
+1            # of recursive panel fact.
+1 0 2        RFACTs (0=left, 1=Crout, 2=Right)
+1            # of broadcast
+0            BCASTs (0=1rg,1=1rM,2=2rg,3=2rM,4=Lng,5=LnM)
+1            # of lookahead depth
+0            DEPTHs (>=0)
+0            SWAP (0=bin-exch,1=long,2=mix)
+1            swapping threshold
+1            L1 in (0=transposed,1=no-transposed) form
+1            U  in (0=transposed,1=no-transposed) form
+0            Equilibration (0=no,1=yes)
+8            memory alignment in double (> 0)

benchmarks/apps/hpl/hpl.py

@@ -28,12 +28,15 @@ class Hpl(SpackTest):
     # Dictionary of reference values, indexed by number of tasks.
     full_reference = {
         1: {
-            'csd3-skylake': {
-                'Gflops': (2000, -0.2, None, 'Gflops'),
+            'csd3-centos7:cascadelake': {
+                'Gflops': (2600, -0.2, None, 'Gflops'),
             },
-            'csd3-icelake': {
+            'csd3-rocky8:icelake': {
                 'Gflops': (4500, -0.2, None, 'Gflops'),
             },
+            'csd3-rocky8:sapphirerapids': {
+                'Gflops': (6200, -0.2, None, 'Gflops'),
+            },
         },
     }
 
@@ -42,7 +45,9 @@ def set_sourcesdir(self):
         if self.config_dir:
             self.sourcesdir = self.config_dir
         else:
-            self.sourcesdir = path.join(path.dirname(__file__), self.current_system.name, str(self.num_tasks))
+            self.sourcesdir = path.join(path.dirname(__file__),
+                    self.current_system.name, self.current_partition.name,
+                    str(self.num_tasks))
 
     @run_before('sanity')
     def set_sanity_patterns(self):
@@ -66,7 +71,10 @@ def set_perf_patterns(self):
         }
         # If we have a reference for current combination of system + number of
         # tasks, use it, otherwise default to generic empty reference.
-        if self.num_tasks in self.full_reference.keys() and self.current_system.name in self.full_reference[self.num_tasks].keys():
+        if (self.num_tasks in self.full_reference.keys() and
+            (self.current_system.name in self.full_reference[self.num_tasks].keys()
+            or self.current_system.name + ':' + self.current_partition.name
+            in self.full_reference[self.num_tasks].keys())):
             self.reference = self.full_reference[self.num_tasks]
         else:
             self.reference = {

benchmarks/apps/swift/swift.py

@@ -23,11 +23,14 @@ class SwiftBenchmark(SpackTest):
         'cosma8': {
             'duration': (50, None, 0.2, 'seconds'),
         },
-        'csd3-skylake': {
-            'duration': (50, None, 0.2, 'seconds'),
+        'csd3-centos7:cascadelake': {
+            'duration': (37, None, 0.2, 'seconds'),
+        },
+        'csd3-rocky8:icelake': {
+            'duration': (35, None, 0.2, 'seconds'),
         },
-        'csd3-icelake': {
-            'duration': (350, None, 0.2, 'seconds'),
+        'csd3-rocky8:sapphirerapids': {
+            'duration': (32, None, 0.2, 'seconds'),
         },
         'dial3': {
             'duration': (150, None, 0.2, 'seconds'),

benchmarks/examples/sombrero/sombrero.py

@@ -69,11 +69,14 @@ class SombreroBenchmark(SpackTest):
         'cosma8': {
             'flops': (3.8, -0.2, None, 'Gflops/seconds'),
         },
-        'csd3-skylake': {
-            'flops': (1.2, -0.2, None, 'Gflops/seconds'),
+        'csd3-centos7:cascadelake': {
+            'flops': (8.0, -0.2, None, 'Gflops/seconds'),
         },
-        'csd3-icelake': {
-            'flops': (1.5, -0.2, None, 'Gflops/seconds'),
+        'csd3-rocky8:icelake': {
+            'flops': (6.0, -0.2, None, 'Gflops/seconds'),
+        },
+        'csd3-rocky8:sapphirerapids': {
+            'flops': (8.0, -0.2, None, 'Gflops/seconds'),
         },
         'dial3': {
             'flops': (1.2, -0.2, None, 'Gflops/seconds'),

benchmarks/reframe_config.py

@@ -66,19 +66,28 @@ def spack_root_to_path():
             ]
         },  # end ARCHER2
         {
-            # https://docs.hpc.cam.ac.uk/hpc/user-guide/cclake.html
-            'name': 'csd3-icelake',
-            'descr': 'CSD3 Icelake',
-            'hostnames': ['login-q-[0-9]+'],
+            # https://www.hpc.cam.ac.uk/index.php/high-performance-computing
+            'name': 'csd3-rocky8',
+            'descr': 'Cambridge Service for Data Driven Discovery - Rocky Linux 8 (RHEL8 compatible) nodes',
+            'hostnames': ['login-q-[0-4]+'],
             'modules_system': 'tmod4',
             'partitions': [
                 {
-                    'name': 'compute-node',
-                    'descr': 'Icelake compute nodes',
+                    # https://docs.hpc.cam.ac.uk/hpc/user-guide/icelake.html
+                    'name': 'icelake',
+                    'descr': 'Ice Lake compute nodes',
                     'scheduler': 'slurm',
                     'launcher': 'mpirun',
+                    'env_vars': [
+                        ['I_MPI_PMI_LIBRARY', '/usr/local/software/slurm/current-rhel8/lib/libpmi2.so'],
+                        ['I_MPI_OFI_PROVIDER', 'mlx'],
+                        ['UCX_NET_DEVICES', 'mlx5_0:1'],
+                    ],
                     'access': ['--partition=icelake', '--exclusive'],
-                    'environs': ['default', 'intel2020-csd3'],
+                    'sched_options': {
+                        'job_submit_timeout': 120,
+                    },
+                    'environs': ['default'],
                     'max_jobs': 64,
                     'processor': {
                         'num_cpus': 76,
@@ -87,32 +96,64 @@ def spack_root_to_path():
                         'num_cpus_per_socket': 38,
                     },
                 },
+                {
+                    'name': 'sapphirerapids',
+                    'descr': 'Sapphire Rapids compute nodes',
+                    'scheduler': 'slurm',
+                    'launcher': 'mpirun',
+                    'env_vars': [
+                        ['I_MPI_PMI_LIBRARY', '/usr/local/software/slurm/current-rhel8/lib/libpmi2.so'],
+                        ['I_MPI_OFI_PROVIDER', 'mlx'],
+                        ['UCX_NET_DEVICES', 'mlx5_0:1'],
+                    ],
+                    'access': ['--partition=sapphire', '--exclusive'],
+                    'sched_options': {
+                        'job_submit_timeout': 120,
+                    },
+                    'environs': ['default'],
+                    'max_jobs': 64,
+                    'processor': {
+                        'num_cpus': 112,
+                        'num_cpus_per_core': 1,
+                        'num_sockets': 2,
+                        'num_cpus_per_socket': 56,
+                    },
+                },
             ]
-        },  # end CSD3 Icelake
+        },  # end CSD3 Rocky 8
         {
-            # https://docs.hpc.cam.ac.uk/hpc/user-guide/cclake.html
-            'name': 'csd3-cascadelake',
-            'descr': 'CSD3 Cascade lake',
-            'hostnames': ['login-e-[0-9]+'],
+            # https://www.hpc.cam.ac.uk/index.php/high-performance-computing
+            'name': 'csd3-centos7',
+            'descr': 'Cambridge Service for Data Driven Discovery - CentOS 7 (RHEL7 compatible) nodes',
+            'hostnames': ['login-p-[0-4]+'],
             'modules_system': 'tmod32',
             'partitions': [
                 {
-                    'name': 'compute-node',
-                    'descr': 'Skylake compute nodes',
+                    # https://docs.hpc.cam.ac.uk/hpc/user-guide/cclake.html
+                    'name': 'cascadelake',
+                    'descr': 'Cascade Lake compute nodes',
                     'scheduler': 'slurm',
                     'launcher': 'mpirun',
+                    'env_vars': [
+                        ['I_MPI_PMI_LIBRARY', '/usr/local/software/slurm/current/lib/libpmi2.so'],
+                        ['I_MPI_OFI_PROVIDER', 'mlx'],
+                        ['UCX_NET_DEVICES', 'mlx5_0:1'],
+                    ],
                     'access': ['--partition=cclake', '--exclusive'],
+                    'sched_options': {
+                        'job_submit_timeout': 120,
+                    },
                     'environs': ['default'],
                     'max_jobs': 64,
                     'processor': {
                         'num_cpus': 56,
-                        'num_cpus_per_core': 2,
+                        'num_cpus_per_core': 1,
                         'num_sockets': 2,
                         'num_cpus_per_socket': 28,
                     }
                 },
             ]
-        },  # end CSD3 Cascade lake
+        },  # end CSD3 CentOS 7
         {
             # https://www.rc.ucl.ac.uk/docs/Clusters/Myriad/#node-types
             'name': 'myriad',
@@ -671,20 +712,6 @@ def spack_root_to_path():
             'cxx': 'mpiicpc',
             'ftn': 'mpiifort'
         },
-        {
-            'name': 'intel2020-csd3',
-            'modules': ["intel/compilers/2020.4",
-                        "intel/mkl/2020.4",
-                        "intel/impi/2020.4/intel",
-                        "intel/libs/idb/2020.4",
-                        "intel/libs/tbb/2020.4",
-                        "intel/libs/ipp/2020.4",
-                        "intel/libs/daal/2020.4",
-                        "intel/bundles/complib/2020.4"],
-            'cc': 'mpiicc',
-            'cxx': 'mpiicpc',
-            'ftn': 'mpiifort'
-        },
     ],
     'logging': [
         {