Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement UPP output() method #639

Merged
merged 10 commits into from
Nov 1, 2024
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions docs/sections/user_guide/cli/drivers/upp/show-schema.out
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
"upp": {
"additionalProperties": false,
"properties": {
"control_file": {
"type": "string"
},
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
"execution": {
"additionalProperties": false,
"properties": {
Expand All @@ -15,6 +18,3 @@
"debug": {
"type": "boolean"
},
"exclusive": {
"type": "boolean"
},
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
3 changes: 1 addition & 2 deletions docs/shared/upp.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
upp:
control_file: /path/to/postxconfig-NT.txt
execution:
batchargs:
export: NONE
Expand All @@ -12,8 +13,6 @@ upp:
mpiargs:
- "--ntasks $SLURM_CPUS_ON_NODE"
mpicmd: srun
files_to_copy:
postxconfig-NT.txt: /path/to/postxconfig-NT.txt
files_to_link:
eta_micro_lookup.dat: /path/to/nam_micro_lookup.dat
params_grib2_tbl_new: /path/to/params_grib2_tbl_new
Expand Down
9 changes: 7 additions & 2 deletions src/uwtools/drivers/driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
from uwtools.utils.file import writable
from uwtools.utils.processing import run_shell_cmd

OutputT = dict[str, Union[str, list[str]]]

# NB: Class docstrings are programmatically defined.


Expand Down Expand Up @@ -399,7 +401,10 @@ def show_output(self):
Show the output to be created by this component.
"""
yield self.taskname("expected output")
print(json.dumps(self.output, indent=2, sort_keys=True))
try:
print(json.dumps(self.output, indent=2, sort_keys=True))
except UWConfigError as e:
log.error(e)
yield asset(None, lambda: True)

@task
Expand Down Expand Up @@ -428,7 +433,7 @@ def _run_via_local_execution(self):
# Public methods

@property
def output(self) -> dict[str, Union[str, list[str]]]:
def output(self) -> OutputT:
"""
Returns a description of the file(s) created when this component runs.
"""
Expand Down
55 changes: 54 additions & 1 deletion src/uwtools/drivers/upp.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@
A driver for UPP.
"""

from math import log10
from pathlib import Path

from iotaa import asset, task, tasks

from uwtools.config.formats.nml import NMLConfig
from uwtools.drivers.driver import DriverCycleLeadtimeBased
from uwtools.drivers.driver import DriverCycleLeadtimeBased, OutputT
from uwtools.drivers.support import set_driver_docstring
from uwtools.exceptions import UWConfigError
from uwtools.strings import STR
from uwtools.utils.tasks import file, filecopy, symlink

Expand All @@ -18,8 +20,24 @@ class UPP(DriverCycleLeadtimeBased):
A driver for UPP.
"""

# Facts specific to the supported UPP version:

GENPROCTYPE_IDX = 8
NFIELDS = 16
NPARAMS = 42

# Workflow tasks

@tasks
def control_file(self):
"""
The GRIB control file.
"""
yield self.taskname("GRIB control file")
yield filecopy(
src=Path(self.config["control_file"]), dst=self.rundir / "postxconfig-NT.txt"
)

@tasks
def files_copied(self):
"""
Expand Down Expand Up @@ -66,6 +84,7 @@ def provisioned_rundir(self):
"""
yield self.taskname("provisioned run directory")
yield [
self.control_file(),
self.files_copied(),
self.files_linked(),
self.namelist_file(),
Expand All @@ -81,6 +100,40 @@ def driver_name(cls) -> str:
"""
return STR.upp

@property
def output(self) -> OutputT:
"""
Returns a description of the file(s) created when this component runs.
"""
# Derive values from the current driver config. GRIB output filename suffixes include the
# forecast leadtime, zero-padded to at least 2 digits (more if necessary). Avoid taking the
# log of zero.
cf = self.config["control_file"]
leadtime = int(self.leadtime.total_seconds() / 3600)
suffix = ".GrbF%0{}d".format(max(2, int(log10(leadtime or 1)) + 1)) % leadtime
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
# Read the control file into an array of lines. Get the number of blocks (one per output
# GRIB file) and the number of variables per block. For each block, construct a filename
# from the block's identifier and the suffix defined above.
try:
with open(cf, "r", encoding="utf-8") as f:
lines = f.read().split("\n")
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
except (FileNotFoundError, PermissionError) as e:
raise UWConfigError(f"Could not open UPP control file {cf}") from e
nblocks, lines = int(lines[0]), lines[1:]
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
nvars, lines = list(map(int, lines[:nblocks])), lines[nblocks:]
paths = []
for _ in range(nblocks):
identifier = lines[0]
paths.append(str(self.rundir / (identifier + suffix)))
fields, lines = lines[: self.NFIELDS], lines[self.NFIELDS :]
_, lines = (
(lines[0], lines[1:])
if fields[self.GENPROCTYPE_IDX] == "ens_fcst"
else (None, lines)
)
lines = lines[self.NPARAMS * nvars.pop() :]
return {"gribfiles": paths}
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved

# Private helper methods

@property
Expand Down
4 changes: 4 additions & 0 deletions src/uwtools/resources/jsonschema/upp.jsonschema
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
"upp": {
"additionalProperties": false,
"properties": {
"control_file": {
"type": "string"
},
"execution": {
"$ref": "urn:uwtools:execution-parallel"
},
Expand Down Expand Up @@ -187,6 +190,7 @@
}
},
"required": [
"control_file",
"execution",
"namelist",
"rundir"
Expand Down
7 changes: 7 additions & 0 deletions src/uwtools/tests/drivers/test_driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -457,6 +457,13 @@ def test_driver_show_output(capsys, config):
assert capsys.readouterr().out.strip() == dedent(expected).strip()


def test_driver_show_output_fail(caplog, config):
with patch.object(ConcreteDriverTimeInvariant, "output", new_callable=PropertyMock) as output:
output.side_effect = UWConfigError("FAIL")
ConcreteDriverTimeInvariant(config).show_output()
assert "FAIL" in caplog.messages


@mark.parametrize(
"base_file,update_values,expected",
[
Expand Down
34 changes: 29 additions & 5 deletions src/uwtools/tests/drivers/test_upp.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from uwtools.drivers.driver import Driver
from uwtools.drivers.upp import UPP
from uwtools.exceptions import UWNotImplementedError
from uwtools.exceptions import UWConfigError
from uwtools.logging import log
from uwtools.tests.support import logged, regex_logged

Expand All @@ -25,6 +25,7 @@
def config(tmp_path):
return {
"upp": {
"control_file": "/path/to/postxconfig-NT.txt",
"execution": {
"batchargs": {
"cores": 1,
Expand Down Expand Up @@ -90,7 +91,6 @@ def leadtime():
"_scheduler",
"_validate",
"_write_runscript",
"output",
"run",
"runscript",
],
Expand Down Expand Up @@ -160,10 +160,34 @@ def test_UPP_namelist_file_missing_base_file(caplog, driverobj):
assert regex_logged(caplog, "missing.nml: State: Not Ready (external asset)")


def test_UPP_output(driverobj):
with raises(UWNotImplementedError) as e:
def test_UPP_output(driverobj, tmp_path):
fields = ["?"] * (UPP.NFIELDS - 1)
parameters = ["?"] * UPP.NPARAMS
# fmt: off
control_data = [
"2", # number of blocks
"1", # number variables in 2nd block
"2", # number variables in 1st block
"FOO", # identifier of 1st block
*fields, # fields of 1st block
*(parameters * 2) , # variable parameters of 1st block
"BAR", # identifier of 2nd block
*fields, # fields of 2nd block
*parameters, # variable parameters of 2nd block
]
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
# fmt: on
control_file = tmp_path / "postxconfig-NT.txt"
with open(control_file, "w", encoding="utf-8") as f:
print("\n".join(control_data), file=f)
driverobj._config["control_file"] = str(control_file)
expected = {"gribfiles": [str(driverobj.rundir / ("%s.GrbF24" % x)) for x in ("FOO", "BAR")]}
assert driverobj.output == expected


def test_UPP_output_fail(driverobj):
with raises(UWConfigError) as e:
assert driverobj.output
assert str(e.value) == "The output() method is not yet implemented for this driver"
assert str(e.value) == "Could not open UPP control file %s" % driverobj.config["control_file"]


def test_UPP_provisioned_rundir(driverobj):
Expand Down
9 changes: 8 additions & 1 deletion src/uwtools/tests/test_schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -1952,6 +1952,7 @@ def test_schema_ungrib_rundir(ungrib_prop):

def test_schema_upp():
config = {
"control_file": "/path/to/postxconfig-NT.txt",
"execution": {
"batchargs": {
"cores": 1,
Expand All @@ -1977,7 +1978,7 @@ def test_schema_upp():
# Basic correctness:
assert not errors(config)
# Some top-level keys are required:
for key in ("execution", "namelist", "rundir"):
for key in ("control_file", "execution", "namelist", "rundir"):
assert f"'{key}' is a required property" in errors(with_del(config, key))
# Other top-level keys are optional:
assert not errors({**config, "files_to_copy": {"dst": "src"}})
Expand All @@ -1986,6 +1987,12 @@ def test_schema_upp():
assert "Additional properties are not allowed" in errors({**config, "foo": "bar"})


def test_schema_upp_control_file(upp_prop):
errors = upp_prop("control_file")
# A string value is required:
assert "is not of type 'string'" in errors(None)


def test_schema_upp_namelist(upp_prop):
maxpathlen = 256
errors = upp_prop("namelist")
Expand Down
Loading