Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

20240925 xcompact3d capability #51

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 29 additions & 2 deletions configuration/archer2.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ def command(self, job):
"descr": "Login nodes",
"scheduler": "local",
"launcher": "local",
"environs": ["Default", "PrgEnv-gnu", "PrgEnv-cray", "PrgEnv-aocc"],
"environs": ["Default", "PrgEnv-gnu", "PrgEnv-cray", "PrgEnv-aocc", "PrgEnv-gnu-hf", "PrgEnv-cray-hf", "PrgEnv-aocc-hf"],
},
{
"name": "compute",
Expand All @@ -38,7 +38,7 @@ def command(self, job):
"--partition=standard",
"--qos=standard",
],
"environs": ["PrgEnv-gnu", "PrgEnv-cray", "PrgEnv-aocc"],
"environs": ["PrgEnv-gnu", "PrgEnv-cray", "PrgEnv-aocc", "PrgEnv-gnu-hf", "PrgEnv-cray-hf", "PrgEnv-aocc-hf"],
"max_jobs": 64,
"processor": {
"num_cpus": 128,
Expand Down Expand Up @@ -100,6 +100,15 @@ def command(self, job):
"ftn": "ftn",
"target_systems": ["archer2"],
},
{
"name": "PrgEnv-gnu-hf",
"modules": ["PrgEnv-gnu"],
"cc": "cc",
"cxx": "CC",
"ftn": "ftn",
"env_vars": [["SLURM_CPU_FREQ_REQ", 2250000]],
"target_systems": ["archer2"],
},
{
"name": "PrgEnv-cray",
"modules": ["PrgEnv-cray"],
Expand All @@ -108,6 +117,15 @@ def command(self, job):
"ftn": "ftn",
"target_systems": ["archer2"],
},
{
"name": "PrgEnv-cray-hf",
"modules": ["PrgEnv-cray"],
"cc": "cc",
"cxx": "CC",
"ftn": "ftn",
"env_vars": [["SLURM_CPU_FREQ_REQ", 2250000]],
"target_systems": ["archer2"],
},
{
"name": "PrgEnv-aocc",
"modules": ["PrgEnv-aocc"],
Expand All @@ -116,6 +134,15 @@ def command(self, job):
"ftn": "ftn",
"target_systems": ["archer2"],
},
{
"name": "PrgEnv-aocc-hf",
"modules": ["PrgEnv-aocc"],
"cc": "cc",
"cxx": "CC",
"ftn": "ftn",
"env_vars": [["SLURM_CPU_FREQ_REQ", 2250000]],
"target_systems": ["archer2"],
},
{
"name": "rocm-PrgEnv-gnu",
"modules": [
Expand Down
133 changes: 133 additions & 0 deletions tests/apps/apps_base.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
"""ReFrame base module for applications tests"""

import reframe as rfm
import reframe.utility.sanity as sn
import abc



@rfm.simple_test
class AppsFetchBase(rfm.RunonlyRegressionTest):
"""Reframe base class for accessing application code"""
descr = 'Fetch app code'
version = variable(str, value='7.3')
executable = 'wget'
executable_opts = [
f'app-{version}'
]
local = True
valid_systems = ['']
valid_prog_environs = ['']

@sanity_function
def validate_download(self):
return sn.assert_eq(self.job.exitcode, 0)


@rfm.simple_test
class AppsCompileBase(rfm.CompileOnlyRegressionTest, metaclass=abc.ABCMeta):
"""Reframe base class for application compilation tests"""
descr = 'Build app'
build_system = ''
valid_systems = ['']
valid_prog_environs = ['']

@abc.abstractmethod
@run_after('init')
def add_dependencies(self):
self.depends_on('', udeps.fully)

@sanity_function
def validate_compilation_no_errors(self):
"""Sanity check that software compiled correctly"""
return sn.assert_eq(self.job.exitcode, 0)

@abc.abstractmethod
@sanity_function
def validate_compilation_executable_produced(self):
"""Sanity check that software compiled correctly"""
pass


@rfm.simple_test
class AppsRunBase(rfm.RunOnlyRegressionTest, metaclass=abc.ABCMeta):
"""ReFrame base class for applications execution tests"""

# Test classifications
tags = {""} # { "applications", "performance", "largescale", "hugescale"}

# Test environments
valid_prog_environs = [""]

#Test executables
executable = ""

#
extra_resources = {""} # {"qos": {"qos": "standard"}}
strict_check = True

# Depency
appbinary = fixture(AppsCompileBase, scope="environment")

# Outputs
keep_files = [""]

# Info
maintainers =[""]


# Sanity checks

@sanity_function
def validate_run_finished_no_error(self) -> bool:
"""Sanity check that simulation finished successfully"""
return sn.assert_eq(self.job.exitcode, 0)

@abc.abstractmethod
@sanity_function
def validate_run_finished(self) -> bool:
"""Sanity check that simulation finished successfully"""
pass

@abc.abstractmethod
@sanity_function
def assert_correctness(self) -> bool:
"""Sanity check that simulation finished correctly"""
pass

# Application performance check

@abc.abstractmethod
@performance_function("app", perf_key="performance")
def extract_performance(self):
"""Extract performance value to compare with reference value"""
#return sn.extractsingle("","")
pass


# Job performance tests

@performance_function("J", perf_key="job_energy")
def extract_job_energy(self):
"""Extract value of energy used in job from slurm"""
jobid = self.job.jobid

slurm = rfm.utility.osext.run_command(
"sacct -j " + str(jobid) + " --format=JobID,ConsumedEnergy --noconvert | tr '\n' ' ' ",
check=True,
shell=True,
)

energy_slurm = sn.extractall_s(
r"JobID\s+ConsumedEnergy\s+------------ --------------\s+[0-9]+\s+[0-9]+\s+[0-9]+.bat\+\s+[0-9]+\s+[0-9]+.ext\+\s+[0-9]+\s+[0-9]+.0\s+(?P<energy>[0-9]+)",
str(slurm.stdout),
"energy",
)
return int(str(energy_slurm[0]))

@performance_function("sec", perf_key="job_time")
def extract_job_time(self):
"""Extract value of the duration of the job from slurm"""
return self.completion_time


18 changes: 11 additions & 7 deletions tests/apps/cp2k/cp2k_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ class CP2KBaseCheck(rfm.RunOnlyRegressionTest):
# Output files to be retained
keep_files = ["cp2k.out"]

# Set time limit

time_limit = "20m"

maintainers = ["[email protected]"]
use_multithreading = False
tags = {"applications", "performance"}
Expand Down Expand Up @@ -68,13 +72,13 @@ class CP2KARCHER2(CP2KBaseCheck):
# Select system to use
valid_systems = ["archer2:compute"]
# Set Programming Environment
valid_prog_environs = ["PrgEnv-gnu"]
valid_prog_environs = ["PrgEnv-gnu", "PrgEnv-gnu-hf"]
# Description of test
descr = "CP2K "
# Command line options for executable
executable_opts = ("-i input_bulk_HFX_3.inp -o cp2k.out ").split()
# different cpu frequencies
freq = parameter(["2250000", "2000000"])
freq = ["2250000", "2000000"]
# slurm parameters
num_tasks = 384
num_tasks_per_node = 16
Expand All @@ -89,20 +93,20 @@ class CP2KARCHER2(CP2KBaseCheck):
@run_after("init")
def setup_params(self):
"""sets up extra parameters"""
self.descr += self.freq
#self.descr += self.freq
if self.current_system.name in ["archer2"]:
self.env_vars = {
"OMP_NUM_THREADS": str(self.num_cpus_per_task),
"OMP_PLACES": "cores",
"SLURM_CPU_FREQ_REQ": self.freq,
}
"OMP_PLACES": "cores"}
# "SLURM_CPU_FREQ_REQ": self.freq,
# }

@run_before("performance")
def set_reference(self):
"""Changes reference values"""
if self.current_system.name in ["archer2"]:
# https://reframe-hpc.readthedocs.io/en/stable/utility_functions_reference.html#reframe.utility.ScopedDict
self.reference["archer2:compute:performance"] = self.reference_performance[self.freq]
self.reference["archer2:compute:performance"] = self.reference_performance["2250000" if self.current_environ.name[-3:] =='-hf' else "2000000"]


@rfm.simple_test
Expand Down
109 changes: 109 additions & 0 deletions tests/apps/xcompact3d/src/input-2048.i3d
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
! -*- mode: f90 -*-

!===================
&BasicParam
!===================

! Flow type (1=Lock-exchange, 2=TGV, 3=Channel, 4=Periodic hill, 5=Cylinder, 6=dbg-schemes)
itype = 2

! Domain decomposition
p_row=512 ! Row partition
p_col=512 ! Column partition

! Mesh
nx=4097 ! X-direction nodes
ny=4097 ! Y-direction nodes
nz=4097 ! Z-direction nodes
istret = 0 ! y mesh refinement (0:no, 1:center, 2:both sides, 3:bottom)
beta = 0.259065151 ! Refinement parameter (beta)

! Domain
xlx = 3.14159265358979 ! Lx (Size of the box in x-direction)
yly = 3.14159265358979 ! Ly (Size of the boy in y-direction)
zlz = 3.14159265358979 ! Lz (Size of the boz in z-direction)

! Boundary conditions
nclx1 = 1
nclxn = 1
ncly1 = 1
nclyn = 1
nclz1 = 1
nclzn = 1

! Flow parameters
iin = 1 ! Inflow conditions (1: classic, 2: turbinit)
re = 1600. ! nu=1/re (Kinematic Viscosity)
u1 = 8. ! u1 (max velocity) (for inflow condition)
u2 = 8. ! u2 (min velocity) (for inflow condition)
init_noise = 0.0 ! Turbulence intensity (1=100%) !! Initial condition
inflow_noise = 0.0 ! Turbulence intensity (1=100%) !! Inflow condition

! Time stepping
dt = 0.00005 ! Time step
ifirst = 1 ! First iteration
ilast = 400 ! Last iteration

! Enable modelling tools
ilesmod=0 ! if 0 then DNS
iscalar=0 ! If iscalar=0 (no scalar), if iscalar=1 (scalar)
iibm=0 ! Flag for immersed boundary method

! Enable io
ivisu=1 ! Store snapshots
ipost=1 ! Do online postprocessing

/End

!====================
&NumOptions
!====================

! Spatial derivatives
ifirstder = 4 ! (1->2nd central, 2->4th central, 3->4th compact, 4-> 6th compact)
isecondder = 4 ! (1->2nd central, 2->4th central, 3->4th compact, 4-> 6th compact, 5->hyperviscous 6th)

! Time scheme
itimescheme = 5 ! Time integration scheme (1->Euler,2->AB2, 3->AB3, 4->AB4,5->RK3,6->RK4)

/End

!=================
&InOutParam
!=================

! Basic I/O
irestart = 0 ! Read initial flow field ?
icheckpoint = 1000 ! Frequency for writing backup file
ioutput = 10000 ! Frequency for visualization
ilist = 25 ! Frequency for the output
nvisu = 1 ! Size for visualisation collection

/End

!=================
&Statistics
!=================

nstat = 1 ! Size arrays for statistic collection

/End

!#######################
! OPTIONAL PARAMETERS
!#######################

!=================
&LESModel
!=================
jles=1
smagcst=0.1
walecst=0.5
iconserv = 0 ! Formulation SGS divergence (0: non conservative, 1: conservative)
/End

!=================
&CASE
!=================
/End

Loading
Loading