From 90320308402e0769701a77331e4afca0206af390 Mon Sep 17 00:00:00 2001 From: JPRichings Date: Wed, 25 Sep 2024 20:54:52 +0100 Subject: [PATCH 1/6] add new test for xcompact3d on 4096 nodes --- tests/apps/xcompact3d/src/input-4096.i3d | 109 ++++++++++++++++++ .../xcompact3d/xcompact3d_capability_ofi.py | 57 +++++++++ .../xcompact3d/xcompact3d_capability_ucx.py | 67 +++++++++++ 3 files changed, 233 insertions(+) create mode 100644 tests/apps/xcompact3d/src/input-4096.i3d create mode 100644 tests/apps/xcompact3d/xcompact3d_capability_ofi.py create mode 100644 tests/apps/xcompact3d/xcompact3d_capability_ucx.py diff --git a/tests/apps/xcompact3d/src/input-4096.i3d b/tests/apps/xcompact3d/src/input-4096.i3d new file mode 100644 index 0000000..a116aaf --- /dev/null +++ b/tests/apps/xcompact3d/src/input-4096.i3d @@ -0,0 +1,109 @@ +! -*- mode: f90 -*- + +!=================== +&BasicParam +!=================== + +! Flow type (1=Lock-exchange, 2=TGV, 3=Channel, 4=Periodic hill, 5=Cylinder, 6=dbg-schemes) +itype = 2 + +! Domain decomposition +p_row=512 ! Row partition +p_col=1024 ! Column partition + +! Mesh +nx=4097 ! X-direction nodes +ny=4097 ! Y-direction nodes +nz=4097 ! Z-direction nodes +istret = 0 ! y mesh refinement (0:no, 1:center, 2:both sides, 3:bottom) +beta = 0.259065151 ! Refinement parameter (beta) + +! Domain +xlx = 3.14159265358979 ! Lx (Size of the box in x-direction) +yly = 3.14159265358979 ! Ly (Size of the boy in y-direction) +zlz = 3.14159265358979 ! Lz (Size of the boz in z-direction) + +! Boundary conditions +nclx1 = 1 +nclxn = 1 +ncly1 = 1 +nclyn = 1 +nclz1 = 1 +nclzn = 1 + +! Flow parameters +iin = 1 ! Inflow conditions (1: classic, 2: turbinit) +re = 1600. ! nu=1/re (Kinematic Viscosity) +u1 = 8. ! u1 (max velocity) (for inflow condition) +u2 = 8. ! u2 (min velocity) (for inflow condition) +init_noise = 0.0 ! Turbulence intensity (1=100%) !! Initial condition +inflow_noise = 0.0 ! Turbulence intensity (1=100%) !! Inflow condition + +! Time stepping +dt = 0.00005 ! Time step +ifirst = 1 ! First iteration +ilast = 400 ! Last iteration + +! Enable modelling tools +ilesmod=0 ! if 0 then DNS +iscalar=0 ! If iscalar=0 (no scalar), if iscalar=1 (scalar) +iibm=0 ! Flag for immersed boundary method + +! Enable io +ivisu=1 ! Store snapshots +ipost=1 ! Do online postprocessing + +/End + +!==================== +&NumOptions +!==================== + +! Spatial derivatives +ifirstder = 4 ! (1->2nd central, 2->4th central, 3->4th compact, 4-> 6th compact) +isecondder = 4 ! (1->2nd central, 2->4th central, 3->4th compact, 4-> 6th compact, 5->hyperviscous 6th) + +! Time scheme +itimescheme = 5 ! Time integration scheme (1->Euler,2->AB2, 3->AB3, 4->AB4,5->RK3,6->RK4) + +/End + +!================= +&InOutParam +!================= + +! Basic I/O +irestart = 0 ! Read initial flow field ? +icheckpoint = 1000 ! Frequency for writing backup file +ioutput = 10000 ! Frequency for visualization +ilist = 25 ! Frequency for the output +nvisu = 1 ! Size for visualisation collection + +/End + +!================= +&Statistics +!================= + +nstat = 1 ! Size arrays for statistic collection + +/End + +!####################### +! OPTIONAL PARAMETERS +!####################### + +!================= +&LESModel +!================= +jles=1 +smagcst=0.1 +walecst=0.5 +iconserv = 0 ! Formulation SGS divergence (0: non conservative, 1: conservative) +/End + +!================= +&CASE +!================= +/End + diff --git a/tests/apps/xcompact3d/xcompact3d_capability_ofi.py b/tests/apps/xcompact3d/xcompact3d_capability_ofi.py new file mode 100644 index 0000000..3090ba7 --- /dev/null +++ b/tests/apps/xcompact3d/xcompact3d_capability_ofi.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +"""Reframe test for XCompact3D""" + +# Based on original work from: +# Copyright 2016-2022 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class XCompact3DHugeOFITest(rfm.RegressionTest): + """XCompact 3D Huge Test""" + + valid_systems = ["archer2:compute"] + valid_prog_environs = ["PrgEnv-gnu"] + + tags = {"performance", "largescale", "applications"} + + num_nodes = 4096 + num_tasks_per_node = 128 + num_cpus_per_task = 1 + num_tasks = num_nodes * num_tasks_per_node * num_cpus_per_task + + env_vars = {"OMP_NUM_THREADS": str(num_cpus_per_task), "MPICH_ALLTOALLV_THROTTLE": "32"} + + time_limit = "1h" + build_system = "CMake" + build_system.ftn = "ftn" + prebuild_cmds = [ + "git clone https://github.com/xcompact3d/Incompact3d.git", + "cd Incompact3d", + ] + builddir = "Incompact3d" + executable = "Incompact3d/bin/xcompact3d" + executable_opts = ["input-4096.i3d"] + modules = ["cmake/3.29.4"] + + reference = {"archer2:compute": {"steptime": (6.3, -0.2, 0.2, "seconds")}} + + @sanity_function + def assert_finished(self): + """Sanity check that simulation finished successfully""" + return sn.assert_found("Good job", self.stdout) + + @performance_function("seconds", perf_key="performance") + def extract_perf(self): + """Extract performance value to compare with reference value""" + return sn.extractsingle( + r"Averaged time per step \(s\):\s+(?P\S+)", + self.stdout, + "steptime", + float, + ) diff --git a/tests/apps/xcompact3d/xcompact3d_capability_ucx.py b/tests/apps/xcompact3d/xcompact3d_capability_ucx.py new file mode 100644 index 0000000..95e5926 --- /dev/null +++ b/tests/apps/xcompact3d/xcompact3d_capability_ucx.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +"""Reframe test for XCompact3D""" + +# Based on original work from: +# Copyright 2016-2022 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class XCompact3DHugeUCXTest(rfm.RegressionTest): + """XCompact 3D Huge Test""" + + valid_systems = ["archer2:compute"] + valid_prog_environs = ["PrgEnv-gnu"] + modules = ["craype-network-ucx"] + + tags = {"performance", "largescale", "applications"} + + num_nodes = 2048 + num_tasks_per_node = 128 + num_cpus_per_task = 1 + num_tasks = num_nodes * num_tasks_per_node * num_cpus_per_task + + env_vars = { + "OMP_NUM_THREADS": str(num_cpus_per_task), + "UCX_TLS": "dc,self,sm", + "UCX_DC_MLX5_RX_QUEUE_LEN": "20000", + "UCX_DC_RX_QUEUE_LEN": "20000", + "UCX_DC_MLX5_TIMEOUT": "20m", + "UCX_DC_TIMEOUT": "20m", + "UCX_UD_MLX5_TX_NUM_GET_BYTES": "1G", + "UCX_UD_MLX5_MAX_GET_ZCOPY": "128k", + } + + time_limit = "1h" + build_system = "CMake" + build_system.ftn = "ftn" + prebuild_cmds = [ + "git clone https://github.com/xcompact3d/Incompact3d.git", + "cd Incompact3d", + ] + builddir = "Incompact3d" + executable = "Incompact3d/bin/xcompact3d" + executable_opts = ["input-4096.i3d"] + modules = ["cmake/3.29.4"] + + reference = {"archer2:compute": {"steptime": (6.3, -0.2, 0.2, "seconds")}} + + @sanity_function + def assert_finished(self): + """Sanity check that simulation finished successfully""" + return sn.assert_found("Good job", self.stdout) + + @performance_function("seconds", perf_key="performance") + def extract_perf(self): + """Extract performance value to compare with reference value""" + return sn.extractsingle( + r"Averaged time per step \(s\):\s+(?P\S+)", + self.stdout, + "steptime", + float, + ) From 4c2402bd557972439892d002cdb830e9ca87fbe1 Mon Sep 17 00:00:00 2001 From: JPRichings Date: Thu, 26 Sep 2024 12:54:47 +0100 Subject: [PATCH 2/6] added additional test for various scales of xcompact 3d runs --- .../src/{large.i3d => input-1024.i3d} | 0 tests/apps/xcompact3d/src/input-2048.i3d | 109 ++++++++++++++++++ tests/apps/xcompact3d/src/input-64.i3d | 109 ++++++++++++++++++ tests/apps/xcompact3d/src/input-8.i3d | 109 ++++++++++++++++++ tests/apps/xcompact3d/xcompact3d.py | 8 +- tests/apps/xcompact3d/xcompact3dLarge.py | 57 +++++++++ tests/apps/xcompact3d/xcompact3dSmall.py | 57 +++++++++ .../xcompact3d/xcompact3d_capability_ofi.py | 2 +- .../xcompact3d/xcompact3d_capability_ucx.py | 4 +- 9 files changed, 448 insertions(+), 7 deletions(-) rename tests/apps/xcompact3d/src/{large.i3d => input-1024.i3d} (100%) create mode 100644 tests/apps/xcompact3d/src/input-2048.i3d create mode 100644 tests/apps/xcompact3d/src/input-64.i3d create mode 100644 tests/apps/xcompact3d/src/input-8.i3d create mode 100644 tests/apps/xcompact3d/xcompact3dLarge.py create mode 100644 tests/apps/xcompact3d/xcompact3dSmall.py diff --git a/tests/apps/xcompact3d/src/large.i3d b/tests/apps/xcompact3d/src/input-1024.i3d similarity index 100% rename from tests/apps/xcompact3d/src/large.i3d rename to tests/apps/xcompact3d/src/input-1024.i3d diff --git a/tests/apps/xcompact3d/src/input-2048.i3d b/tests/apps/xcompact3d/src/input-2048.i3d new file mode 100644 index 0000000..0c72d63 --- /dev/null +++ b/tests/apps/xcompact3d/src/input-2048.i3d @@ -0,0 +1,109 @@ +! -*- mode: f90 -*- + +!=================== +&BasicParam +!=================== + +! Flow type (1=Lock-exchange, 2=TGV, 3=Channel, 4=Periodic hill, 5=Cylinder, 6=dbg-schemes) +itype = 2 + +! Domain decomposition +p_row=512 ! Row partition +p_col=512 ! Column partition + +! Mesh +nx=4097 ! X-direction nodes +ny=4097 ! Y-direction nodes +nz=4097 ! Z-direction nodes +istret = 0 ! y mesh refinement (0:no, 1:center, 2:both sides, 3:bottom) +beta = 0.259065151 ! Refinement parameter (beta) + +! Domain +xlx = 3.14159265358979 ! Lx (Size of the box in x-direction) +yly = 3.14159265358979 ! Ly (Size of the boy in y-direction) +zlz = 3.14159265358979 ! Lz (Size of the boz in z-direction) + +! Boundary conditions +nclx1 = 1 +nclxn = 1 +ncly1 = 1 +nclyn = 1 +nclz1 = 1 +nclzn = 1 + +! Flow parameters +iin = 1 ! Inflow conditions (1: classic, 2: turbinit) +re = 1600. ! nu=1/re (Kinematic Viscosity) +u1 = 8. ! u1 (max velocity) (for inflow condition) +u2 = 8. ! u2 (min velocity) (for inflow condition) +init_noise = 0.0 ! Turbulence intensity (1=100%) !! Initial condition +inflow_noise = 0.0 ! Turbulence intensity (1=100%) !! Inflow condition + +! Time stepping +dt = 0.00005 ! Time step +ifirst = 1 ! First iteration +ilast = 400 ! Last iteration + +! Enable modelling tools +ilesmod=0 ! if 0 then DNS +iscalar=0 ! If iscalar=0 (no scalar), if iscalar=1 (scalar) +iibm=0 ! Flag for immersed boundary method + +! Enable io +ivisu=1 ! Store snapshots +ipost=1 ! Do online postprocessing + +/End + +!==================== +&NumOptions +!==================== + +! Spatial derivatives +ifirstder = 4 ! (1->2nd central, 2->4th central, 3->4th compact, 4-> 6th compact) +isecondder = 4 ! (1->2nd central, 2->4th central, 3->4th compact, 4-> 6th compact, 5->hyperviscous 6th) + +! Time scheme +itimescheme = 5 ! Time integration scheme (1->Euler,2->AB2, 3->AB3, 4->AB4,5->RK3,6->RK4) + +/End + +!================= +&InOutParam +!================= + +! Basic I/O +irestart = 0 ! Read initial flow field ? +icheckpoint = 1000 ! Frequency for writing backup file +ioutput = 10000 ! Frequency for visualization +ilist = 25 ! Frequency for the output +nvisu = 1 ! Size for visualisation collection + +/End + +!================= +&Statistics +!================= + +nstat = 1 ! Size arrays for statistic collection + +/End + +!####################### +! OPTIONAL PARAMETERS +!####################### + +!================= +&LESModel +!================= +jles=1 +smagcst=0.1 +walecst=0.5 +iconserv = 0 ! Formulation SGS divergence (0: non conservative, 1: conservative) +/End + +!================= +&CASE +!================= +/End + diff --git a/tests/apps/xcompact3d/src/input-64.i3d b/tests/apps/xcompact3d/src/input-64.i3d new file mode 100644 index 0000000..b876e67 --- /dev/null +++ b/tests/apps/xcompact3d/src/input-64.i3d @@ -0,0 +1,109 @@ +! -*- mode: f90 -*- + +!=================== +&BasicParam +!=================== + +! Flow type (1=Lock-exchange, 2=TGV, 3=Channel, 4=Periodic hill, 5=Cylinder, 6=dbg-schemes) +itype = 2 + +! Domain decomposition +p_row=64 ! Row partition +p_col=128 ! Column partition + +! Mesh +nx=4097 ! X-direction nodes +ny=4097 ! Y-direction nodes +nz=4097 ! Z-direction nodes +istret = 0 ! y mesh refinement (0:no, 1:center, 2:both sides, 3:bottom) +beta = 0.259065151 ! Refinement parameter (beta) + +! Domain +xlx = 3.14159265358979 ! Lx (Size of the box in x-direction) +yly = 3.14159265358979 ! Ly (Size of the boy in y-direction) +zlz = 3.14159265358979 ! Lz (Size of the boz in z-direction) + +! Boundary conditions +nclx1 = 1 +nclxn = 1 +ncly1 = 1 +nclyn = 1 +nclz1 = 1 +nclzn = 1 + +! Flow parameters +iin = 1 ! Inflow conditions (1: classic, 2: turbinit) +re = 1600. ! nu=1/re (Kinematic Viscosity) +u1 = 8. ! u1 (max velocity) (for inflow condition) +u2 = 8. ! u2 (min velocity) (for inflow condition) +init_noise = 0.0 ! Turbulence intensity (1=100%) !! Initial condition +inflow_noise = 0.0 ! Turbulence intensity (1=100%) !! Inflow condition + +! Time stepping +dt = 0.00005 ! Time step +ifirst = 1 ! First iteration +ilast = 400 ! Last iteration + +! Enable modelling tools +ilesmod=0 ! if 0 then DNS +iscalar=0 ! If iscalar=0 (no scalar), if iscalar=1 (scalar) +iibm=0 ! Flag for immersed boundary method + +! Enable io +ivisu=1 ! Store snapshots +ipost=1 ! Do online postprocessing + +/End + +!==================== +&NumOptions +!==================== + +! Spatial derivatives +ifirstder = 4 ! (1->2nd central, 2->4th central, 3->4th compact, 4-> 6th compact) +isecondder = 4 ! (1->2nd central, 2->4th central, 3->4th compact, 4-> 6th compact, 5->hyperviscous 6th) + +! Time scheme +itimescheme = 5 ! Time integration scheme (1->Euler,2->AB2, 3->AB3, 4->AB4,5->RK3,6->RK4) + +/End + +!================= +&InOutParam +!================= + +! Basic I/O +irestart = 0 ! Read initial flow field ? +icheckpoint = 1000 ! Frequency for writing backup file +ioutput = 10000 ! Frequency for visualization +ilist = 25 ! Frequency for the output +nvisu = 1 ! Size for visualisation collection + +/End + +!================= +&Statistics +!================= + +nstat = 1 ! Size arrays for statistic collection + +/End + +!####################### +! OPTIONAL PARAMETERS +!####################### + +!================= +&LESModel +!================= +jles=1 +smagcst=0.1 +walecst=0.5 +iconserv = 0 ! Formulation SGS divergence (0: non conservative, 1: conservative) +/End + +!================= +&CASE +!================= +/End + diff --git a/tests/apps/xcompact3d/src/input-8.i3d b/tests/apps/xcompact3d/src/input-8.i3d new file mode 100644 index 0000000..68a606b --- /dev/null +++ b/tests/apps/xcompact3d/src/input-8.i3d @@ -0,0 +1,109 @@ +! -*- mode: f90 -*- + +!=================== +&BasicParam +!=================== + +! Flow type (1=Lock-exchange, 2=TGV, 3=Channel, 4=Periodic hill, 5=Cylinder, 6=dbg-schemes) +itype = 2 + +! Domain decomposition +p_row=32 ! Row partition +p_col=32 ! Column partition + +! Mesh +nx=513 ! X-direction nodes +ny=513 ! Y-direction nodes +nz=513 ! Z-direction nodes +istret = 0 ! y mesh refinement (0:no, 1:center, 2:both sides, 3:bottom) +beta = 0.259065151 ! Refinement parameter (beta) + +! Domain +xlx = 3.14159265358979 ! Lx (Size of the box in x-direction) +yly = 3.14159265358979 ! Ly (Size of the boy in y-direction) +zlz = 3.14159265358979 ! Lz (Size of the boz in z-direction) + +! Boundary conditions +nclx1 = 1 +nclxn = 1 +ncly1 = 1 +nclyn = 1 +nclz1 = 1 +nclzn = 1 + +! Flow parameters +iin = 1 ! Inflow conditions (1: classic, 2: turbinit) +re = 1600. ! nu=1/re (Kinematic Viscosity) +u1 = 8. ! u1 (max velocity) (for inflow condition) +u2 = 8. ! u2 (min velocity) (for inflow condition) +init_noise = 0.0 ! Turbulence intensity (1=100%) !! Initial condition +inflow_noise = 0.0 ! Turbulence intensity (1=100%) !! Inflow condition + +! Time stepping +dt = 0.00005 ! Time step +ifirst = 1 ! First iteration +ilast = 400 ! Last iteration + +! Enable modelling tools +ilesmod=0 ! if 0 then DNS +iscalar=0 ! If iscalar=0 (no scalar), if iscalar=1 (scalar) +iibm=0 ! Flag for immersed boundary method + +! Enable io +ivisu=1 ! Store snapshots +ipost=1 ! Do online postprocessing + +/End + +!==================== +&NumOptions +!==================== + +! Spatial derivatives +ifirstder = 4 ! (1->2nd central, 2->4th central, 3->4th compact, 4-> 6th compact) +isecondder = 4 ! (1->2nd central, 2->4th central, 3->4th compact, 4-> 6th compact, 5->hyperviscous 6th) + +! Time scheme +itimescheme = 5 ! Time integration scheme (1->Euler,2->AB2, 3->AB3, 4->AB4,5->RK3,6->RK4) + +/End + +!================= +&InOutParam +!================= + +! Basic I/O +irestart = 0 ! Read initial flow field ? +icheckpoint = 1000 ! Frequency for writing backup file +ioutput = 10000 ! Frequency for visualization +ilist = 25 ! Frequency for the output +nvisu = 1 ! Size for visualisation collection + +/End + +!================= +&Statistics +!================= + +nstat = 1 ! Size arrays for statistic collection + +/End + +!####################### +! OPTIONAL PARAMETERS +!####################### + +!================= +&LESModel +!================= +jles=1 +smagcst=0.1 +walecst=0.5 +iconserv = 0 ! Formulation SGS divergence (0: non conservative, 1: conservative) +/End + +!================= +&CASE +!================= +/End + diff --git a/tests/apps/xcompact3d/xcompact3d.py b/tests/apps/xcompact3d/xcompact3d.py index c37a226..27e3495 100644 --- a/tests/apps/xcompact3d/xcompact3d.py +++ b/tests/apps/xcompact3d/xcompact3d.py @@ -12,15 +12,15 @@ @rfm.simple_test -class XCompact3DLargeTest(rfm.RegressionTest): - """XCompact 3D Large Test""" +class XCompact3DTest(rfm.RegressionTest): + """XCompact 3D Test""" valid_systems = ["archer2:compute"] valid_prog_environs = ["PrgEnv-gnu"] tags = {"performance", "largescale", "applications"} - num_nodes = 1024 + num_nodes = 64 num_tasks_per_node = 128 num_cpus_per_task = 1 num_tasks = num_nodes * num_tasks_per_node * num_cpus_per_task @@ -36,7 +36,7 @@ class XCompact3DLargeTest(rfm.RegressionTest): ] builddir = "Incompact3d" executable = "Incompact3d/bin/xcompact3d" - executable_opts = ["large.i3d"] + executable_opts = ["input-64.i3d"] modules = ["cmake/3.29.4"] reference = {"archer2:compute": {"steptime": (6.3, -0.2, 0.2, "seconds")}} diff --git a/tests/apps/xcompact3d/xcompact3dLarge.py b/tests/apps/xcompact3d/xcompact3dLarge.py new file mode 100644 index 0000000..86cd043 --- /dev/null +++ b/tests/apps/xcompact3d/xcompact3dLarge.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +"""Reframe test for XCompact3D""" + +# Based on original work from: +# Copyright 2016-2022 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class XCompact3DLargeTest(rfm.RegressionTest): + """XCompact 3D Large Test""" + + valid_systems = ["archer2:compute"] + valid_prog_environs = ["PrgEnv-gnu"] + + tags = {"performance", "largescale", "applications"} + + num_nodes = 1024 + num_tasks_per_node = 128 + num_cpus_per_task = 1 + num_tasks = num_nodes * num_tasks_per_node * num_cpus_per_task + + env_vars = {"OMP_NUM_THREADS": str(num_cpus_per_task)} + + time_limit = "1h" + build_system = "CMake" + build_system.ftn = "ftn" + prebuild_cmds = [ + "git clone https://github.com/xcompact3d/Incompact3d.git", + "cd Incompact3d", + ] + builddir = "Incompact3d" + executable = "Incompact3d/bin/xcompact3d" + executable_opts = ["input-1024.i3d"] + modules = ["cmake/3.29.4"] + + reference = {"archer2:compute": {"steptime": (6.3, -0.2, 0.2, "seconds")}} + + @sanity_function + def assert_finished(self): + """Sanity check that simulation finished successfully""" + return sn.assert_found("Good job", self.stdout) + + @performance_function("seconds", perf_key="performance") + def extract_perf(self): + """Extract performance value to compare with reference value""" + return sn.extractsingle( + r"Averaged time per step \(s\):\s+(?P\S+)", + self.stdout, + "steptime", + float, + ) diff --git a/tests/apps/xcompact3d/xcompact3dSmall.py b/tests/apps/xcompact3d/xcompact3dSmall.py new file mode 100644 index 0000000..be4ffec --- /dev/null +++ b/tests/apps/xcompact3d/xcompact3dSmall.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +"""Reframe test for XCompact3D""" + +# Based on original work from: +# Copyright 2016-2022 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class XCompact3DSmallTest(rfm.RegressionTest): + """XCompact 3D Small Test""" + + valid_systems = ["archer2:compute"] + valid_prog_environs = ["PrgEnv-gnu"] + + tags = {"performance", "largescale", "applications"} + + num_nodes = 8 + num_tasks_per_node = 128 + num_cpus_per_task = 1 + num_tasks = num_nodes * num_tasks_per_node * num_cpus_per_task + + env_vars = {"OMP_NUM_THREADS": str(num_cpus_per_task)} + + time_limit = "1h" + build_system = "CMake" + build_system.ftn = "ftn" + prebuild_cmds = [ + "git clone https://github.com/xcompact3d/Incompact3d.git", + "cd Incompact3d", + ] + builddir = "Incompact3d" + executable = "Incompact3d/bin/xcompact3d" + executable_opts = ["input-8.i3d"] + modules = ["cmake/3.29.4"] + + reference = {"archer2:compute": {"steptime": (6.3, -0.2, 0.2, "seconds")}} + + @sanity_function + def assert_finished(self): + """Sanity check that simulation finished successfully""" + return sn.assert_found("Good job", self.stdout) + + @performance_function("seconds", perf_key="performance") + def extract_perf(self): + """Extract performance value to compare with reference value""" + return sn.extractsingle( + r"Averaged time per step \(s\):\s+(?P\S+)", + self.stdout, + "steptime", + float, + ) diff --git a/tests/apps/xcompact3d/xcompact3d_capability_ofi.py b/tests/apps/xcompact3d/xcompact3d_capability_ofi.py index 3090ba7..c38cf5d 100644 --- a/tests/apps/xcompact3d/xcompact3d_capability_ofi.py +++ b/tests/apps/xcompact3d/xcompact3d_capability_ofi.py @@ -15,7 +15,7 @@ class XCompact3DHugeOFITest(rfm.RegressionTest): """XCompact 3D Huge Test""" - valid_systems = ["archer2:compute"] + valid_systems = ["archer2:compute-capability"] valid_prog_environs = ["PrgEnv-gnu"] tags = {"performance", "largescale", "applications"} diff --git a/tests/apps/xcompact3d/xcompact3d_capability_ucx.py b/tests/apps/xcompact3d/xcompact3d_capability_ucx.py index 95e5926..f90cde2 100644 --- a/tests/apps/xcompact3d/xcompact3d_capability_ucx.py +++ b/tests/apps/xcompact3d/xcompact3d_capability_ucx.py @@ -15,7 +15,7 @@ class XCompact3DHugeUCXTest(rfm.RegressionTest): """XCompact 3D Huge Test""" - valid_systems = ["archer2:compute"] + valid_systems = ["archer2:compute-capability"] valid_prog_environs = ["PrgEnv-gnu"] modules = ["craype-network-ucx"] @@ -46,7 +46,7 @@ class XCompact3DHugeUCXTest(rfm.RegressionTest): ] builddir = "Incompact3d" executable = "Incompact3d/bin/xcompact3d" - executable_opts = ["input-4096.i3d"] + executable_opts = ["input-2048.i3d"] modules = ["cmake/3.29.4"] reference = {"archer2:compute": {"steptime": (6.3, -0.2, 0.2, "seconds")}} From 09afd275831c7ee00d3d8fc0d14bc40aef55e735 Mon Sep 17 00:00:00 2001 From: JPRichings Date: Thu, 26 Sep 2024 14:46:33 +0100 Subject: [PATCH 3/6] update tests based on job submission tests --- tests/apps/xcompact3d/src/input-64.i3d | 6 +++--- tests/apps/xcompact3d/xcompact3d.py | 3 +-- tests/apps/xcompact3d/xcompact3dSmall.py | 3 +-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/apps/xcompact3d/src/input-64.i3d b/tests/apps/xcompact3d/src/input-64.i3d index b876e67..d8b5f8c 100644 --- a/tests/apps/xcompact3d/src/input-64.i3d +++ b/tests/apps/xcompact3d/src/input-64.i3d @@ -12,9 +12,9 @@ p_row=64 ! Row partition p_col=128 ! Column partition ! Mesh -nx=4097 ! X-direction nodes -ny=4097 ! Y-direction nodes -nz=4097 ! Z-direction nodes +nx=1025 ! X-direction nodes +ny=1025 ! Y-direction nodes +nz=1025 ! Z-direction nodes istret = 0 ! y mesh refinement (0:no, 1:center, 2:both sides, 3:bottom) beta = 0.259065151 ! Refinement parameter (beta) diff --git a/tests/apps/xcompact3d/xcompact3d.py b/tests/apps/xcompact3d/xcompact3d.py index 27e3495..21c64eb 100644 --- a/tests/apps/xcompact3d/xcompact3d.py +++ b/tests/apps/xcompact3d/xcompact3d.py @@ -18,7 +18,7 @@ class XCompact3DTest(rfm.RegressionTest): valid_systems = ["archer2:compute"] valid_prog_environs = ["PrgEnv-gnu"] - tags = {"performance", "largescale", "applications"} + tags = {"performance", "applications"} num_nodes = 64 num_tasks_per_node = 128 @@ -29,7 +29,6 @@ class XCompact3DTest(rfm.RegressionTest): time_limit = "1h" build_system = "CMake" - build_system.ftn = "ftn" prebuild_cmds = [ "git clone https://github.com/xcompact3d/Incompact3d.git", "cd Incompact3d", diff --git a/tests/apps/xcompact3d/xcompact3dSmall.py b/tests/apps/xcompact3d/xcompact3dSmall.py index be4ffec..f902c81 100644 --- a/tests/apps/xcompact3d/xcompact3dSmall.py +++ b/tests/apps/xcompact3d/xcompact3dSmall.py @@ -18,7 +18,7 @@ class XCompact3DSmallTest(rfm.RegressionTest): valid_systems = ["archer2:compute"] valid_prog_environs = ["PrgEnv-gnu"] - tags = {"performance", "largescale", "applications"} + tags = {"performance", "applications"} num_nodes = 8 num_tasks_per_node = 128 @@ -29,7 +29,6 @@ class XCompact3DSmallTest(rfm.RegressionTest): time_limit = "1h" build_system = "CMake" - build_system.ftn = "ftn" prebuild_cmds = [ "git clone https://github.com/xcompact3d/Incompact3d.git", "cd Incompact3d", From ea478a31310b85db9c5861d95259c917c37052a8 Mon Sep 17 00:00:00 2001 From: JPRichings Date: Thu, 26 Sep 2024 15:03:49 +0100 Subject: [PATCH 4/6] updates to capability tests for compilation phase --- tests/apps/xcompact3d/xcompact3d_capability_ofi.py | 1 - tests/apps/xcompact3d/xcompact3d_capability_ucx.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/apps/xcompact3d/xcompact3d_capability_ofi.py b/tests/apps/xcompact3d/xcompact3d_capability_ofi.py index c38cf5d..a754152 100644 --- a/tests/apps/xcompact3d/xcompact3d_capability_ofi.py +++ b/tests/apps/xcompact3d/xcompact3d_capability_ofi.py @@ -29,7 +29,6 @@ class XCompact3DHugeOFITest(rfm.RegressionTest): time_limit = "1h" build_system = "CMake" - build_system.ftn = "ftn" prebuild_cmds = [ "git clone https://github.com/xcompact3d/Incompact3d.git", "cd Incompact3d", diff --git a/tests/apps/xcompact3d/xcompact3d_capability_ucx.py b/tests/apps/xcompact3d/xcompact3d_capability_ucx.py index f90cde2..4bcd407 100644 --- a/tests/apps/xcompact3d/xcompact3d_capability_ucx.py +++ b/tests/apps/xcompact3d/xcompact3d_capability_ucx.py @@ -39,7 +39,6 @@ class XCompact3DHugeUCXTest(rfm.RegressionTest): time_limit = "1h" build_system = "CMake" - build_system.ftn = "ftn" prebuild_cmds = [ "git clone https://github.com/xcompact3d/Incompact3d.git", "cd Incompact3d", From dbdce91f462107707d7f8e1ddf5d99db680b1775 Mon Sep 17 00:00:00 2001 From: JPRichings Date: Thu, 26 Sep 2024 15:11:07 +0100 Subject: [PATCH 5/6] correct name style --- tests/apps/xcompact3d/{xcompact3dLarge.py => xcompact3d_large.py} | 0 tests/apps/xcompact3d/{xcompact3dSmall.py => xcompact3d_small.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/apps/xcompact3d/{xcompact3dLarge.py => xcompact3d_large.py} (100%) rename tests/apps/xcompact3d/{xcompact3dSmall.py => xcompact3d_small.py} (100%) diff --git a/tests/apps/xcompact3d/xcompact3dLarge.py b/tests/apps/xcompact3d/xcompact3d_large.py similarity index 100% rename from tests/apps/xcompact3d/xcompact3dLarge.py rename to tests/apps/xcompact3d/xcompact3d_large.py diff --git a/tests/apps/xcompact3d/xcompact3dSmall.py b/tests/apps/xcompact3d/xcompact3d_small.py similarity index 100% rename from tests/apps/xcompact3d/xcompact3dSmall.py rename to tests/apps/xcompact3d/xcompact3d_small.py From c4d2059077c942f3f7bec13c2f667d987c72e1f7 Mon Sep 17 00:00:00 2001 From: JPRichings Date: Tue, 29 Oct 2024 09:33:07 +0000 Subject: [PATCH 6/6] updates to frequency setting --- configuration/archer2.py | 31 +++++- tests/apps/apps_base.py | 133 ++++++++++++++++++++++++ tests/apps/cp2k/cp2k_check.py | 18 ++-- tests/apps/xcompact3d/xcompact3d_app.py | 84 +++++++++++++++ 4 files changed, 257 insertions(+), 9 deletions(-) create mode 100644 tests/apps/apps_base.py create mode 100644 tests/apps/xcompact3d/xcompact3d_app.py diff --git a/configuration/archer2.py b/configuration/archer2.py index 783c205..1536368 100644 --- a/configuration/archer2.py +++ b/configuration/archer2.py @@ -25,7 +25,7 @@ def command(self, job): "descr": "Login nodes", "scheduler": "local", "launcher": "local", - "environs": ["Default", "PrgEnv-gnu", "PrgEnv-cray", "PrgEnv-aocc"], + "environs": ["Default", "PrgEnv-gnu", "PrgEnv-cray", "PrgEnv-aocc", "PrgEnv-gnu-hf", "PrgEnv-cray-hf", "PrgEnv-aocc-hf"], }, { "name": "compute", @@ -38,7 +38,7 @@ def command(self, job): "--partition=standard", "--qos=standard", ], - "environs": ["PrgEnv-gnu", "PrgEnv-cray", "PrgEnv-aocc"], + "environs": ["PrgEnv-gnu", "PrgEnv-cray", "PrgEnv-aocc", "PrgEnv-gnu-hf", "PrgEnv-cray-hf", "PrgEnv-aocc-hf"], "max_jobs": 64, "processor": { "num_cpus": 128, @@ -100,6 +100,15 @@ def command(self, job): "ftn": "ftn", "target_systems": ["archer2"], }, + { + "name": "PrgEnv-gnu-hf", + "modules": ["PrgEnv-gnu"], + "cc": "cc", + "cxx": "CC", + "ftn": "ftn", + "env_vars": [["SLURM_CPU_FREQ_REQ", 2250000]], + "target_systems": ["archer2"], + }, { "name": "PrgEnv-cray", "modules": ["PrgEnv-cray"], @@ -108,6 +117,15 @@ def command(self, job): "ftn": "ftn", "target_systems": ["archer2"], }, + { + "name": "PrgEnv-cray-hf", + "modules": ["PrgEnv-cray"], + "cc": "cc", + "cxx": "CC", + "ftn": "ftn", + "env_vars": [["SLURM_CPU_FREQ_REQ", 2250000]], + "target_systems": ["archer2"], + }, { "name": "PrgEnv-aocc", "modules": ["PrgEnv-aocc"], @@ -116,6 +134,15 @@ def command(self, job): "ftn": "ftn", "target_systems": ["archer2"], }, + { + "name": "PrgEnv-aocc-hf", + "modules": ["PrgEnv-aocc"], + "cc": "cc", + "cxx": "CC", + "ftn": "ftn", + "env_vars": [["SLURM_CPU_FREQ_REQ", 2250000]], + "target_systems": ["archer2"], + }, { "name": "rocm-PrgEnv-gnu", "modules": [ diff --git a/tests/apps/apps_base.py b/tests/apps/apps_base.py new file mode 100644 index 0000000..109152c --- /dev/null +++ b/tests/apps/apps_base.py @@ -0,0 +1,133 @@ +"""ReFrame base module for applications tests""" + +import reframe as rfm +import reframe.utility.sanity as sn +import abc + + + +@rfm.simple_test +class AppsFetchBase(rfm.RunonlyRegressionTest): + """Reframe base class for accessing application code""" + descr = 'Fetch app code' + version = variable(str, value='7.3') + executable = 'wget' + executable_opts = [ + f'app-{version}' + ] + local = True + valid_systems = [''] + valid_prog_environs = [''] + + @sanity_function + def validate_download(self): + return sn.assert_eq(self.job.exitcode, 0) + + +@rfm.simple_test +class AppsCompileBase(rfm.CompileOnlyRegressionTest, metaclass=abc.ABCMeta): + """Reframe base class for application compilation tests""" + descr = 'Build app' + build_system = '' + valid_systems = [''] + valid_prog_environs = [''] + + @abc.abstractmethod + @run_after('init') + def add_dependencies(self): + self.depends_on('', udeps.fully) + + @sanity_function + def validate_compilation_no_errors(self): + """Sanity check that software compiled correctly""" + return sn.assert_eq(self.job.exitcode, 0) + + @abc.abstractmethod + @sanity_function + def validate_compilation_executable_produced(self): + """Sanity check that software compiled correctly""" + pass + + +@rfm.simple_test +class AppsRunBase(rfm.RunOnlyRegressionTest, metaclass=abc.ABCMeta): + """ReFrame base class for applications execution tests""" + + # Test classifications + tags = {""} # { "applications", "performance", "largescale", "hugescale"} + + # Test environments + valid_prog_environs = [""] + + #Test executables + executable = "" + + # + extra_resources = {""} # {"qos": {"qos": "standard"}} + strict_check = True + + # Depency + appbinary = fixture(AppsCompileBase, scope="environment") + + # Outputs + keep_files = [""] + + # Info + maintainers =[""] + + + # Sanity checks + + @sanity_function + def validate_run_finished_no_error(self) -> bool: + """Sanity check that simulation finished successfully""" + return sn.assert_eq(self.job.exitcode, 0) + + @abc.abstractmethod + @sanity_function + def validate_run_finished(self) -> bool: + """Sanity check that simulation finished successfully""" + pass + + @abc.abstractmethod + @sanity_function + def assert_correctness(self) -> bool: + """Sanity check that simulation finished correctly""" + pass + + # Application performance check + + @abc.abstractmethod + @performance_function("app", perf_key="performance") + def extract_performance(self): + """Extract performance value to compare with reference value""" + #return sn.extractsingle("","") + pass + + + # Job performance tests + + @performance_function("J", perf_key="job_energy") + def extract_job_energy(self): + """Extract value of energy used in job from slurm""" + jobid = self.job.jobid + + slurm = rfm.utility.osext.run_command( + "sacct -j " + str(jobid) + " --format=JobID,ConsumedEnergy --noconvert | tr '\n' ' ' ", + check=True, + shell=True, + ) + + energy_slurm = sn.extractall_s( + r"JobID\s+ConsumedEnergy\s+------------ --------------\s+[0-9]+\s+[0-9]+\s+[0-9]+.bat\+\s+[0-9]+\s+[0-9]+.ext\+\s+[0-9]+\s+[0-9]+.0\s+(?P[0-9]+)", + str(slurm.stdout), + "energy", + ) + return int(str(energy_slurm[0])) + + @performance_function("sec", perf_key="job_time") + def extract_job_time(self): + """Extract value of the duration of the job from slurm""" + return self.completion_time + + diff --git a/tests/apps/cp2k/cp2k_check.py b/tests/apps/cp2k/cp2k_check.py index 9c5568d..4a8cafd 100644 --- a/tests/apps/cp2k/cp2k_check.py +++ b/tests/apps/cp2k/cp2k_check.py @@ -18,6 +18,10 @@ class CP2KBaseCheck(rfm.RunOnlyRegressionTest): # Output files to be retained keep_files = ["cp2k.out"] + # Set time limit + + time_limit = "20m" + maintainers = ["j.richings@epcc.ed.ac.uk"] use_multithreading = False tags = {"applications", "performance"} @@ -68,13 +72,13 @@ class CP2KARCHER2(CP2KBaseCheck): # Select system to use valid_systems = ["archer2:compute"] # Set Programming Environment - valid_prog_environs = ["PrgEnv-gnu"] + valid_prog_environs = ["PrgEnv-gnu", "PrgEnv-gnu-hf"] # Description of test descr = "CP2K " # Command line options for executable executable_opts = ("-i input_bulk_HFX_3.inp -o cp2k.out ").split() # different cpu frequencies - freq = parameter(["2250000", "2000000"]) + freq = ["2250000", "2000000"] # slurm parameters num_tasks = 384 num_tasks_per_node = 16 @@ -89,20 +93,20 @@ class CP2KARCHER2(CP2KBaseCheck): @run_after("init") def setup_params(self): """sets up extra parameters""" - self.descr += self.freq + #self.descr += self.freq if self.current_system.name in ["archer2"]: self.env_vars = { "OMP_NUM_THREADS": str(self.num_cpus_per_task), - "OMP_PLACES": "cores", - "SLURM_CPU_FREQ_REQ": self.freq, - } + "OMP_PLACES": "cores"} +# "SLURM_CPU_FREQ_REQ": self.freq, +# } @run_before("performance") def set_reference(self): """Changes reference values""" if self.current_system.name in ["archer2"]: # https://reframe-hpc.readthedocs.io/en/stable/utility_functions_reference.html#reframe.utility.ScopedDict - self.reference["archer2:compute:performance"] = self.reference_performance[self.freq] + self.reference["archer2:compute:performance"] = self.reference_performance["2250000" if self.current_environ.name[-3:] =='-hf' else "2000000"] @rfm.simple_test diff --git a/tests/apps/xcompact3d/xcompact3d_app.py b/tests/apps/xcompact3d/xcompact3d_app.py new file mode 100644 index 0000000..8c89d24 --- /dev/null +++ b/tests/apps/xcompact3d/xcompact3d_app.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 + +"""Reframe test for XCompact3D""" + +# Based on original work from: +# Copyright 2016-2022 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + + +class XCompact3dFetchStable(AppsFetchBase): + """Test to fetch xcompact3d from github""" + executable = 'git clone' + executable_opts = [ + f'https://github.com/xcompact3d/Incompact3d.git' + ] + + local = True + + valid_systems = ['*'] + valid_prog_environs = [''] + + +class Xcompact3dBuildStable(AppsCompileBase): + """Reframe base class for application compilation tests""" + descr = 'Build app' + build_system = '' + valid_systems = [''] + valid_prog_environs = [''] + + + + + + + + +@rfm.simple_test +class XCompact3DTest(rfm.RegressionTest): + """XCompact 3D Test""" + + valid_systems = ["archer2:compute"] + valid_prog_environs = ["PrgEnv-gnu"] + + tags = {"performance", "applications"} + + num_nodes = 64 + num_tasks_per_node = 128 + num_cpus_per_task = 1 + num_tasks = num_nodes * num_tasks_per_node * num_cpus_per_task + + env_vars = {"OMP_NUM_THREADS": str(num_cpus_per_task)} + + time_limit = "1h" + build_system = "CMake" + prebuild_cmds = [ + "git clone https://github.com/xcompact3d/Incompact3d.git", + "cd Incompact3d", + ] + builddir = "Incompact3d" + executable = "Incompact3d/bin/xcompact3d" + executable_opts = ["input-64.i3d"] + modules = ["cmake/3.29.4"] + + reference = {"archer2:compute": {"steptime": (6.3, -0.2, 0.2, "seconds")}} + + @sanity_function + def assert_finished(self): + """Sanity check that simulation finished successfully""" + return sn.assert_found("Good job", self.stdout) + + @performance_function("seconds", perf_key="performance") + def extract_perf(self): + """Extract performance value to compare with reference value""" + return sn.extractsingle( + r"Averaged time per step \(s\):\s+(?P\S+)", + self.stdout, + "steptime", + float, + )