From 2626b6dfa0e368c12e455c8ddf4147eca7b99946 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 30 Jul 2024 19:58:57 +0000 Subject: [PATCH 01/18] Move str2path() from utils.api to utils.file --- src/uwtools/api/config.py | 2 +- src/uwtools/api/file.py | 2 +- src/uwtools/api/rocoto.py | 2 +- src/uwtools/api/template.py | 2 +- src/uwtools/tests/utils/test_api.py | 12 ------------ src/uwtools/tests/utils/test_file.py | 12 ++++++++++++ src/uwtools/utils/api.py | 13 ++----------- src/uwtools/utils/file.py | 9 +++++++++ 8 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/uwtools/api/config.py b/src/uwtools/api/config.py index 810b046a1..c1683bfc8 100644 --- a/src/uwtools/api/config.py +++ b/src/uwtools/api/config.py @@ -16,8 +16,8 @@ from uwtools.config.tools import realize_config as _realize from uwtools.config.validator import validate_external as _validate_external from uwtools.utils.api import ensure_data_source as _ensure_data_source -from uwtools.utils.api import str2path as _str2path from uwtools.utils.file import FORMAT as _FORMAT +from uwtools.utils.file import str2path as _str2path # Public diff --git a/src/uwtools/api/file.py b/src/uwtools/api/file.py index 725836f7d..cfae21dd0 100644 --- a/src/uwtools/api/file.py +++ b/src/uwtools/api/file.py @@ -10,7 +10,7 @@ from uwtools.file import Copier, Linker from uwtools.utils.api import ensure_data_source as _ensure_data_source -from uwtools.utils.api import str2path as _str2path +from uwtools.utils.file import str2path as _str2path def copy( diff --git a/src/uwtools/api/rocoto.py b/src/uwtools/api/rocoto.py index e3486dd3f..3b0406e4b 100644 --- a/src/uwtools/api/rocoto.py +++ b/src/uwtools/api/rocoto.py @@ -9,7 +9,7 @@ from uwtools.rocoto import realize_rocoto_xml as _realize from uwtools.rocoto import validate_rocoto_xml_file as _validate from uwtools.utils.api import ensure_data_source as _ensure_data_source -from uwtools.utils.api import str2path as _str2path +from uwtools.utils.file import str2path as _str2path def realize( diff --git a/src/uwtools/api/template.py b/src/uwtools/api/template.py index 0379b2960..f8611935d 100644 --- a/src/uwtools/api/template.py +++ b/src/uwtools/api/template.py @@ -10,7 +10,7 @@ from uwtools.config.jinja2 import render as _render from uwtools.exceptions import UWTemplateRenderError from uwtools.utils.api import ensure_data_source as _ensure_data_source -from uwtools.utils.api import str2path as _str2path +from uwtools.utils.file import str2path as _str2path def render( diff --git a/src/uwtools/tests/utils/test_api.py b/src/uwtools/tests/utils/test_api.py index fb6faa552..debb95a5b 100644 --- a/src/uwtools/tests/utils/test_api.py +++ b/src/uwtools/tests/utils/test_api.py @@ -89,18 +89,6 @@ def test_make_execute_leadtime_no_cycle_error(execute_kwargs): assert "When leadtime is specified, cycle is required" in str(e) -@mark.parametrize("val", [Path("/some/path"), {"foo": 88}]) -def test_str2path_passthrough(val): - assert api.str2path(val) == val - - -def test_str2path_convert(): - val = "/some/path" - result = api.str2path(val) - assert isinstance(result, Path) - assert result == Path(val) - - @mark.parametrize("hours", [0, 24, 168]) def test__execute(execute_kwargs, hours, tmp_path): graph_file = tmp_path / "g.dot" diff --git a/src/uwtools/tests/utils/test_file.py b/src/uwtools/tests/utils/test_file.py index d33282a61..032e15e98 100644 --- a/src/uwtools/tests/utils/test_file.py +++ b/src/uwtools/tests/utils/test_file.py @@ -112,6 +112,18 @@ def test_resource_path(): assert file.resource_path().is_dir() +@mark.parametrize("val", [Path("/some/path"), {"foo": 88}]) +def test_str2path_passthrough(val): + assert file.str2path(val) == val + + +def test_str2path_convert(): + val = "/some/path" + result = file.str2path(val) + assert isinstance(result, Path) + assert result == Path(val) + + def test_writable_file(tmp_path): apath = tmp_path / "afile" with file.writable(filepath=apath) as f: diff --git a/src/uwtools/utils/api.py b/src/uwtools/utils/api.py index e4044d913..dfe4b8932 100644 --- a/src/uwtools/utils/api.py +++ b/src/uwtools/utils/api.py @@ -6,12 +6,12 @@ import re from inspect import getfullargspec from pathlib import Path -from typing import Any, Callable, Optional, TypeVar, Union +from typing import Callable, Optional, TypeVar, Union from uwtools.drivers.driver import DriverT from uwtools.drivers.support import graph -from uwtools.drivers.support import tasks as _tasks from uwtools.exceptions import UWError +from uwtools.utils.file import str2path T = TypeVar("T") @@ -133,15 +133,6 @@ def execute_cycle_leadtime( # pylint: disable=unused-argument return execute -def str2path(val: Any) -> Any: - """ - Return str value as Path, other types unmodified. - - :param val: Any value. - """ - return Path(val) if isinstance(val, str) else val - - # Private diff --git a/src/uwtools/utils/file.py b/src/uwtools/utils/file.py index 2b2eb9237..5b0d27e15 100644 --- a/src/uwtools/utils/file.py +++ b/src/uwtools/utils/file.py @@ -102,6 +102,15 @@ def resource_path(suffix: str = "") -> Path: return prefix / suffix +def str2path(val: Any) -> Any: + """ + Return str value as Path, other types unmodified. + + :param val: Any value. + """ + return Path(val) if isinstance(val, str) else val + + @contextmanager def writable(filepath: Optional[Path] = None, mode: str = "w") -> Generator[IO, None, None]: """ From 4387e92ed3965a5388f0bf507c5f217713193c91 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 30 Jul 2024 20:25:34 +0000 Subject: [PATCH 02/18] Let Config() take str, coerce to Path --- src/uwtools/config/formats/base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/uwtools/config/formats/base.py b/src/uwtools/config/formats/base.py index 857384b79..854449e11 100644 --- a/src/uwtools/config/formats/base.py +++ b/src/uwtools/config/formats/base.py @@ -14,6 +14,7 @@ from uwtools.config.support import INCLUDE_TAG, depth, log_and_error, yaml_to_str from uwtools.exceptions import UWConfigError from uwtools.logging import INDENT, log +from uwtools.utils.file import str2path class Config(ABC, UserDict): @@ -22,7 +23,7 @@ class Config(ABC, UserDict): several configuration-file formats. """ - def __init__(self, config: Optional[Union[dict, Path]] = None) -> None: + def __init__(self, config: Optional[Union[dict, str, Path]] = None) -> None: """ Construct a Config object. @@ -33,7 +34,7 @@ def __init__(self, config: Optional[Union[dict, Path]] = None) -> None: self._config_file = None self.update(config) else: - self._config_file = config if config else None + self._config_file = str2path(config) if config else None self.data = self._load(self._config_file) if self.get_depth_threshold() and self.depth != self.get_depth_threshold(): raise UWConfigError( From 516cbf3422f758288dcee713c239e22d8e2c2bb4 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 30 Jul 2024 20:28:16 +0000 Subject: [PATCH 03/18] Let Assets et al. take str as well as Path --- src/uwtools/drivers/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 1a5ff081f..1f4d1001b 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -36,7 +36,7 @@ def __init__( self, cycle: Optional[datetime] = None, leadtime: Optional[timedelta] = None, - config: Optional[Union[dict, Path]] = None, + config: Optional[Union[dict, str, Path]] = None, dry_run: bool = False, key_path: Optional[list[str]] = None, schema_file: Optional[Path] = None, From a72d788103e46f91add0204d958445fb78572598 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 30 Jul 2024 20:45:48 +0000 Subject: [PATCH 04/18] Let Assets et al. take YAMLConfig --- src/uwtools/drivers/driver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 1f4d1001b..a50de18e4 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -36,12 +36,12 @@ def __init__( self, cycle: Optional[datetime] = None, leadtime: Optional[timedelta] = None, - config: Optional[Union[dict, str, Path]] = None, + config: Optional[Union[dict, str, YAMLConfig, Path]] = None, dry_run: bool = False, key_path: Optional[list[str]] = None, schema_file: Optional[Path] = None, ) -> None: - self._config = YAMLConfig(config=config) + self._config = config if isinstance(config, YAMLConfig) else YAMLConfig(config=config) self._config.dereference( context={ **({"cycle": cycle} if cycle else {}), From 865d0093ac68ce75b06f41ef4ceeb186227244a6 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 30 Jul 2024 22:22:14 +0000 Subject: [PATCH 05/18] Tweak prototype configs --- docs/shared/cdeps.yaml | 6 +++--- docs/shared/ww3.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/shared/cdeps.yaml b/docs/shared/cdeps.yaml index d4b8882dc..16683aee8 100644 --- a/docs/shared/cdeps.yaml +++ b/docs/shared/cdeps.yaml @@ -24,7 +24,7 @@ cdeps: stream_data_files: - /path/to/data_file1.nc stream_data_variables: - - 'temperature Sa_tbot' + - temperature Sa_tbot stream_lev_dimname: 'null' stream_mesh_file: /path/to/mesh1.nc stream_offset: 0 @@ -59,8 +59,8 @@ cdeps: - /path/to/data_file2.nc - /path/to/data_file3.nc stream_data_variables: - - 'sst So_t' - - 'mask So_omask' + - sst So_t + - mask So_omask stream_lev_dimname: 'null' stream_mesh_file: /path/to/mesh2.nc stream_offset: 0 diff --git a/docs/shared/ww3.yaml b/docs/shared/ww3.yaml index 8f4917e42..ac1cefa83 100644 --- a/docs/shared/ww3.yaml +++ b/docs/shared/ww3.yaml @@ -2,5 +2,5 @@ ww3: namelist: template_file: /path/to/ww3/ww3_shel.nml.IN template_values: - input_forcing_winds: "C" + input_forcing_winds: C rundir: /path/to/run/directory From 30682f062697793f46285142d191e6c76af5a0d4 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 30 Jul 2024 23:06:52 +0000 Subject: [PATCH 06/18] Use STR in drivers --- src/uwtools/drivers/chgres_cube.py | 8 +-- src/uwtools/drivers/driver.py | 81 ++++++++++++----------- src/uwtools/drivers/esg_grid.py | 4 +- src/uwtools/drivers/filter_topo.py | 4 +- src/uwtools/drivers/fv3.py | 10 +-- src/uwtools/drivers/global_equiv_resol.py | 2 +- src/uwtools/drivers/ioda.py | 2 +- src/uwtools/drivers/jedi.py | 12 ++-- src/uwtools/drivers/jedi_base.py | 5 +- src/uwtools/drivers/make_hgrid.py | 2 +- src/uwtools/drivers/make_solo_mosaic.py | 2 +- src/uwtools/drivers/mpas.py | 8 +-- src/uwtools/drivers/mpas_init.py | 8 +-- src/uwtools/drivers/orog_gsl.py | 8 +-- src/uwtools/drivers/schism.py | 4 +- src/uwtools/drivers/sfc_climo_gen.py | 4 +- src/uwtools/drivers/shave.py | 2 +- src/uwtools/drivers/upp.py | 12 ++-- src/uwtools/drivers/ww3.py | 4 +- src/uwtools/strings.py | 16 +++++ 20 files changed, 110 insertions(+), 88 deletions(-) diff --git a/src/uwtools/drivers/chgres_cube.py b/src/uwtools/drivers/chgres_cube.py index 83f432813..9291ff68e 100644 --- a/src/uwtools/drivers/chgres_cube.py +++ b/src/uwtools/drivers/chgres_cube.py @@ -29,10 +29,10 @@ def namelist_file(self): path = self._rundir / fn yield asset(path, path.is_file) input_files = [] - namelist = self._driver_config["namelist"] - if base_file := namelist.get("base_file"): + namelist = self._driver_config[STR.namelist] + if base_file := namelist.get(STR.basefile): input_files.append(base_file) - if update_values := namelist.get("update_values"): + if update_values := namelist.get(STR.updatevalues): config_files = update_values["config"] for k in ["mosaic_file_target_grid", "vcoord_file_target_grid"]: input_files.append(config_files[k]) @@ -83,7 +83,7 @@ def runscript(self): yield None envvars = { "KMP_AFFINITY": "scatter", - "OMP_NUM_THREADS": self._driver_config.get("execution", {}).get("threads", 1), + "OMP_NUM_THREADS": self._driver_config.get(STR.execution, {}).get(STR.threads, 1), "OMP_STACKSIZE": "1024m", } self._write_runscript(path=path, envvars=envvars) diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index a50de18e4..241be88b4 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -22,6 +22,7 @@ from uwtools.exceptions import UWConfigError from uwtools.logging import log from uwtools.scheduler import JobScheduler +from uwtools.strings import STR from uwtools.utils.processing import execute # NB: Class docstrings are programmatically defined. @@ -40,17 +41,21 @@ def __init__( dry_run: bool = False, key_path: Optional[list[str]] = None, schema_file: Optional[Path] = None, + coupler: Optional[str] = None, ) -> None: - self._config = config if isinstance(config, YAMLConfig) else YAMLConfig(config=config) - self._config.dereference( + config = config if isinstance(config, YAMLConfig) else YAMLConfig(config=config) + config.dereference( context={ - **({"cycle": cycle} if cycle else {}), - **({"leadtime": leadtime} if leadtime is not None else {}), - **self._config.data, + **({STR.cycle: cycle} if cycle else {}), + **({STR.leadtime: leadtime} if leadtime is not None else {}), + **config.data, } ) + self._config = config for key in key_path or []: self._config = self._config[key] + if coupler: + self._config[STR.rundir] = config[coupler][STR.rundir] self._validate(schema_file) dryrun(enable=dry_run) @@ -62,7 +67,7 @@ def __repr__(self) -> str: m, s = divmod(r, 60) leadtime = "%02d:%02d:%02d" % (h, m, s) return " ".join( - filter(None, [str(self), cycle, leadtime, "in", self._driver_config["rundir"]]) + filter(None, [str(self), cycle, leadtime, "in", self._driver_config[STR.rundir]]) ) def __str__(self) -> str: @@ -107,8 +112,8 @@ def _create_user_updated_config( :param path: Path to dump file to. :param schema: Schema to validate final config against. """ - user_values = config_values.get("update_values", {}) - if base_file := config_values.get("base_file"): + user_values = config_values.get(STR.updatevalues, {}) + if base_file := config_values.get(STR.basefile): cfgobj = config_class(base_file) cfgobj.update_values(user_values) cfgobj.dereference() @@ -154,19 +159,19 @@ def _namelist_schema( """ schema: dict = {"type": "object"} nmlcfg = self._driver_config - for config_key in config_keys or ["namelist"]: + for config_key in config_keys or [STR.namelist]: nmlcfg = nmlcfg[config_key] - if nmlcfg.get("validate", True): + if nmlcfg.get(STR.validate, True): schema_file = get_schema_file(schema_name=self._driver_name.replace("_", "-")) with open(schema_file, "r", encoding="utf-8") as f: schema = json.load(f) for schema_key in schema_keys or [ - "properties", + STR.properties, self._driver_name, - "properties", - "namelist", - "properties", - "update_values", + STR.properties, + STR.namelist, + STR.properties, + STR.updatevalues, ]: schema = schema[schema_key] return schema @@ -176,7 +181,7 @@ def _rundir(self) -> Path: """ The path to the component's run directory. """ - return Path(self._driver_config["rundir"]) + return Path(self._driver_config[STR.rundir]) def _taskname(self, suffix: str) -> str: """ @@ -312,7 +317,7 @@ def run(self): """ A run. """ - yield self._taskname("run") + yield self._taskname(STR.run) yield (self._run_via_batch_submission() if self._batch else self._run_via_local_execution()) @task @@ -357,17 +362,17 @@ def _run_resources(self) -> dict[str, Any]: Returns platform configuration data. """ try: - platform = self._config["platform"] + platform = self._config[STR.platform] except KeyError as e: raise UWConfigError("Required 'platform' block missing in config") from e - threads = self._driver_config.get("execution", {}).get("threads") + threads = self._driver_config.get(STR.execution, {}).get(STR.threads) return { - "account": platform["account"], - "rundir": self._rundir, - "scheduler": platform["scheduler"], - "stdout": "%s.out" % self._runscript_path.name, # config may override - **({"threads": threads} if threads else {}), - **self._driver_config.get("execution", {}).get("batchargs", {}), + STR.account: platform[STR.account], + STR.rundir: self._rundir, + STR.scheduler: platform[STR.scheduler], + STR.stdout: "%s.out" % self._runscript_path.name, # config may override + **({STR.threads: threads} if threads else {}), + **self._driver_config.get(STR.execution, {}).get(STR.batchargs, {}), } @property @@ -375,12 +380,12 @@ def _runcmd(self) -> str: """ Returns the full command-line component invocation. """ - execution = self._driver_config.get("execution", {}) - mpiargs = execution.get("mpiargs", []) + execution = self._driver_config.get(STR.execution, {}) + mpiargs = execution.get(STR.mpiargs, []) components = [ - execution.get("mpicmd"), # MPI run program + execution.get(STR.mpicmd), # MPI run program *[str(x) for x in mpiargs], # MPI arguments - execution["executable"], # component executable name + execution[STR.executable], # component executable name ] return " ".join(filter(None, components)) @@ -449,7 +454,7 @@ def _validate(self, schema_file: Optional[Path] = None) -> None: validate_external(schema_file=schema_file, config=self._config) else: validate_internal(schema_name=self._driver_name.replace("_", "-"), config=self._config) - validate_internal(schema_name="platform", config=self._config) + validate_internal(schema_name=STR.platform, config=self._config) def _write_runscript(self, path: Path, envvars: Optional[dict[str, str]] = None) -> None: """ @@ -457,11 +462,11 @@ def _write_runscript(self, path: Path, envvars: Optional[dict[str, str]] = None) """ path.parent.mkdir(parents=True, exist_ok=True) envvars = envvars or {} - threads = self._driver_config.get("execution", {}).get("threads") + threads = self._driver_config.get(STR.execution, {}).get(STR.threads) if threads and "OMP_NUM_THREADS" not in envvars: raise UWConfigError("Config specified threads but driver does not set OMP_NUM_THREADS") rs = self._runscript( - envcmds=self._driver_config.get("execution", {}).get("envcmds", []), + envcmds=self._driver_config.get(STR.execution, {}).get(STR.envcmds, []), envvars=envvars, execution=[ "time %s" % self._runcmd, @@ -581,11 +586,11 @@ def _add_docstring(class_: type, omit: Optional[list[str]] = None) -> None: ) -_add_docstring(Assets, omit=["batch"]) -_add_docstring(AssetsCycleBased, omit=["batch", "leadtime"]) -_add_docstring(AssetsCycleLeadtimeBased, omit=["batch"]) -_add_docstring(AssetsTimeInvariant, omit=["batch", "cycle", "leadtime"]) +_add_docstring(Assets, omit=[STR.batch]) +_add_docstring(AssetsCycleBased, omit=[STR.batch, STR.leadtime]) +_add_docstring(AssetsCycleLeadtimeBased, omit=[STR.batch]) +_add_docstring(AssetsTimeInvariant, omit=[STR.batch, STR.cycle, STR.leadtime]) _add_docstring(Driver) -_add_docstring(DriverCycleBased, omit=["leadtime"]) +_add_docstring(DriverCycleBased, omit=[STR.leadtime]) _add_docstring(DriverCycleLeadtimeBased) -_add_docstring(DriverTimeInvariant, omit=["cycle", "leadtime"]) +_add_docstring(DriverTimeInvariant, omit=[STR.cycle, STR.leadtime]) diff --git a/src/uwtools/drivers/esg_grid.py b/src/uwtools/drivers/esg_grid.py index e7eca1670..8b9ed6033 100644 --- a/src/uwtools/drivers/esg_grid.py +++ b/src/uwtools/drivers/esg_grid.py @@ -28,11 +28,11 @@ def namelist_file(self): yield self._taskname(fn) path = self._rundir / fn yield asset(path, path.is_file) - base_file = self._driver_config["namelist"].get("base_file") + base_file = self._driver_config[STR.namelist].get(STR.basefile) yield file(Path(base_file)) if base_file else None self._create_user_updated_config( config_class=NMLConfig, - config_values=self._driver_config["namelist"], + config_values=self._driver_config[STR.namelist], path=path, schema=self._namelist_schema(schema_keys=["$defs", "namelist_content"]), ) diff --git a/src/uwtools/drivers/filter_topo.py b/src/uwtools/drivers/filter_topo.py index 0cbb720a2..2fc8def8a 100644 --- a/src/uwtools/drivers/filter_topo.py +++ b/src/uwtools/drivers/filter_topo.py @@ -25,7 +25,7 @@ def input_grid_file(self): The input grid file. """ src = Path(self._driver_config["config"]["input_grid_file"]) - dst = Path(self._driver_config["rundir"]) / src.name + dst = Path(self._driver_config[STR.rundir]) / src.name yield self._taskname("Input grid") yield asset(dst, dst.is_file) yield symlink(target=src, linkname=dst) @@ -42,7 +42,7 @@ def namelist_file(self): yield None self._create_user_updated_config( config_class=NMLConfig, - config_values=self._driver_config["namelist"], + config_values=self._driver_config[STR.namelist], path=path, schema=self._namelist_schema(), ) diff --git a/src/uwtools/drivers/fv3.py b/src/uwtools/drivers/fv3.py index 6d6647376..745e0be58 100644 --- a/src/uwtools/drivers/fv3.py +++ b/src/uwtools/drivers/fv3.py @@ -67,7 +67,7 @@ def field_table(self): yield self._taskname(fn) path = self._rundir / fn yield asset(path, path.is_file) - yield filecopy(src=Path(self._driver_config["field_table"]["base_file"]), dst=path) + yield filecopy(src=Path(self._driver_config["field_table"][STR.basefile]), dst=path) @tasks def files_copied(self): @@ -100,7 +100,7 @@ def model_configure(self): yield self._taskname(fn) path = self._rundir / fn yield asset(path, path.is_file) - base_file = self._driver_config["model_configure"].get("base_file") + base_file = self._driver_config["model_configure"].get(STR.basefile) yield file(Path(base_file)) if base_file else None self._create_user_updated_config( config_class=YAMLConfig, @@ -117,11 +117,11 @@ def namelist_file(self): yield self._taskname(fn) path = self._rundir / fn yield asset(path, path.is_file) - base_file = self._driver_config["namelist"].get("base_file") + base_file = self._driver_config[STR.namelist].get(STR.basefile) yield file(Path(base_file)) if base_file else None self._create_user_updated_config( config_class=NMLConfig, - config_values=self._driver_config["namelist"], + config_values=self._driver_config[STR.namelist], path=path, schema=self._namelist_schema(), ) @@ -170,7 +170,7 @@ def runscript(self): "ESMF_RUNTIME_COMPLIANCECHECK": "OFF:depth=4", "KMP_AFFINITY": "scatter", "MPI_TYPE_DEPTH": 20, - "OMP_NUM_THREADS": self._driver_config.get("execution", {}).get("threads", 1), + "OMP_NUM_THREADS": self._driver_config.get(STR.execution, {}).get(STR.threads, 1), "OMP_STACKSIZE": "512m", } self._write_runscript(path=path, envvars=envvars) diff --git a/src/uwtools/drivers/global_equiv_resol.py b/src/uwtools/drivers/global_equiv_resol.py index c946febf9..034c067b0 100644 --- a/src/uwtools/drivers/global_equiv_resol.py +++ b/src/uwtools/drivers/global_equiv_resol.py @@ -51,6 +51,6 @@ def _runcmd(self): """ Returns the full command-line component invocation. """ - executable = self._driver_config["execution"]["executable"] + executable = self._driver_config[STR.execution][STR.executable] input_file_path = self._driver_config["input_grid_file"] return f"{executable} {input_file_path}" diff --git a/src/uwtools/drivers/ioda.py b/src/uwtools/drivers/ioda.py index cb120bf08..973bb29e9 100644 --- a/src/uwtools/drivers/ioda.py +++ b/src/uwtools/drivers/ioda.py @@ -49,6 +49,6 @@ def _runcmd(self) -> str: """ Returns the full command-line component invocation. """ - executable = self._driver_config["execution"]["executable"] + executable = self._driver_config[STR.execution][STR.executable] jedi_config = str(self._rundir / self._config_fn) return " ".join([executable, jedi_config]) diff --git a/src/uwtools/drivers/jedi.py b/src/uwtools/drivers/jedi.py index 1f2df52e7..52caa65c2 100644 --- a/src/uwtools/drivers/jedi.py +++ b/src/uwtools/drivers/jedi.py @@ -42,11 +42,11 @@ def validate_only(self): yield taskname a = asset(None, lambda: False) yield a - executable = file(Path(self._driver_config["execution"]["executable"])) + executable = file(Path(self._driver_config[STR.execution][STR.executable])) config = self.configuration_file() yield [executable, config] cmd = "time %s --validate-only %s 2>&1" % (refs(executable), refs(config)) - if envcmds := self._driver_config["execution"].get("envcmds"): + if envcmds := self._driver_config[STR.execution].get(STR.envcmds): cmd = " && ".join([*envcmds, cmd]) result = run(taskname, cmd) if result.success: @@ -74,13 +74,13 @@ def _runcmd(self) -> str: """ Returns the full command-line component invocation. """ - execution = self._driver_config["execution"] + execution = self._driver_config[STR.execution] jedi_config = self._rundir / self._config_fn - mpiargs = execution.get("mpiargs", []) + mpiargs = execution.get(STR.mpiargs, []) components = [ - execution.get("mpicmd"), # MPI run program + execution.get(STR.mpicmd), # MPI run program *[str(x) for x in mpiargs], # MPI arguments - execution["executable"], # component executable name + execution[STR.executable], # component executable name str(jedi_config), # JEDI config file ] return " ".join(filter(None, components)) diff --git a/src/uwtools/drivers/jedi_base.py b/src/uwtools/drivers/jedi_base.py index 3c965bab6..11d029177 100644 --- a/src/uwtools/drivers/jedi_base.py +++ b/src/uwtools/drivers/jedi_base.py @@ -1,5 +1,5 @@ """ -A base class for jedi-based drivers. +A base class for JEDI-based drivers. """ from abc import abstractmethod @@ -9,6 +9,7 @@ from uwtools.config.formats.yaml import YAMLConfig from uwtools.drivers.driver import DriverCycleBased +from uwtools.strings import STR from uwtools.utils.tasks import file, filecopy, symlink @@ -28,7 +29,7 @@ def configuration_file(self): yield self._taskname(fn) path = self._rundir / fn yield asset(path, path.is_file) - base_file = self._driver_config["configuration_file"].get("base_file") + base_file = self._driver_config["configuration_file"].get(STR.basefile) yield file(Path(base_file)) if base_file else None self._create_user_updated_config( config_class=YAMLConfig, diff --git a/src/uwtools/drivers/make_hgrid.py b/src/uwtools/drivers/make_hgrid.py index 1d3ead8cd..765db2346 100644 --- a/src/uwtools/drivers/make_hgrid.py +++ b/src/uwtools/drivers/make_hgrid.py @@ -37,7 +37,7 @@ def _runcmd(self): """ Returns the full command-line component invocation. """ - executable = self._driver_config["execution"]["executable"] + executable = self._driver_config[STR.execution][STR.executable] config = self._driver_config["config"] flags = [] for k, v in config.items(): diff --git a/src/uwtools/drivers/make_solo_mosaic.py b/src/uwtools/drivers/make_solo_mosaic.py index 34bc83587..5d7776a74 100644 --- a/src/uwtools/drivers/make_solo_mosaic.py +++ b/src/uwtools/drivers/make_solo_mosaic.py @@ -37,7 +37,7 @@ def _runcmd(self): """ Returns the full command-line component invocation. """ - executable = self._driver_config["execution"]["executable"] + executable = self._driver_config[STR.execution][STR.executable] flags = " ".join(f"--{k} {v}" for k, v in self._driver_config["config"].items()) return f"{executable} {flags}" diff --git a/src/uwtools/drivers/mpas.py b/src/uwtools/drivers/mpas.py index 914908e54..3ff22e716 100644 --- a/src/uwtools/drivers/mpas.py +++ b/src/uwtools/drivers/mpas.py @@ -45,19 +45,19 @@ def namelist_file(self): path = self._rundir / "namelist.atmosphere" yield self._taskname(str(path)) yield asset(path, path.is_file) - base_file = self._driver_config["namelist"].get("base_file") + base_file = self._driver_config[STR.namelist].get(STR.basefile) yield file(Path(base_file)) if base_file else None duration = timedelta(hours=self._driver_config["length"]) str_duration = str(duration).replace(" days, ", "_") - namelist = self._driver_config["namelist"] - update_values = namelist.get("update_values", {}) + namelist = self._driver_config[STR.namelist] + update_values = namelist.get(STR.updatevalues, {}) update_values.setdefault("nhyd_model", {}).update( { "config_start_time": self._cycle.strftime("%Y-%m-%d_%H:00:00"), "config_run_duration": str_duration, } ) - namelist["update_values"] = update_values + namelist[STR.updatevalues] = update_values self._create_user_updated_config( config_class=NMLConfig, config_values=namelist, diff --git a/src/uwtools/drivers/mpas_init.py b/src/uwtools/drivers/mpas_init.py index e7acb7d95..d0e768828 100644 --- a/src/uwtools/drivers/mpas_init.py +++ b/src/uwtools/drivers/mpas_init.py @@ -48,20 +48,20 @@ def namelist_file(self): yield self._taskname(fn) path = self._rundir / fn yield asset(path, path.is_file) - base_file = self._driver_config["namelist"].get("base_file") + base_file = self._driver_config[STR.namelist].get(STR.basefile) yield file(Path(base_file)) if base_file else None stop_time = self._cycle + timedelta( hours=self._driver_config["boundary_conditions"]["length"] ) - namelist = self._driver_config["namelist"] - update_values = namelist.get("update_values", {}) + namelist = self._driver_config[STR.namelist] + update_values = namelist.get(STR.updatevalues, {}) update_values.setdefault("nhyd_model", {}).update( { "config_start_time": self._cycle.strftime("%Y-%m-%d_%H:00:00"), "config_stop_time": stop_time.strftime("%Y-%m-%d_%H:00:00"), } ) - namelist["update_values"] = update_values + namelist[STR.updatevalues] = update_values self._create_user_updated_config( config_class=NMLConfig, config_values=namelist, diff --git a/src/uwtools/drivers/orog_gsl.py b/src/uwtools/drivers/orog_gsl.py index 9af84a59c..12823032c 100644 --- a/src/uwtools/drivers/orog_gsl.py +++ b/src/uwtools/drivers/orog_gsl.py @@ -27,7 +27,7 @@ def input_grid_file(self): self._driver_config["config"][k] for k in ["resolution", "tile", "halo"] ) src = Path(self._driver_config["config"]["input_grid_file"]) - dst = Path(self._driver_config["rundir"]) / fn + dst = Path(self._driver_config[STR.rundir]) / fn yield self._taskname("Input grid") yield asset(dst, dst.is_file) yield symlink(target=src, linkname=dst) @@ -52,7 +52,7 @@ def topo_data_2p5m(self): """ fn = "geo_em.d01.lat-lon.2.5m.HGT_M.nc" src = Path(self._driver_config["config"]["topo_data_2p5m"]) - dst = Path(self._driver_config["rundir"]) / fn + dst = Path(self._driver_config[STR.rundir]) / fn yield self._taskname("Input grid") yield asset(dst, dst.is_file) yield symlink(target=src, linkname=dst) @@ -64,7 +64,7 @@ def topo_data_30s(self): """ fn = "HGT.Beljaars_filtered.lat-lon.30s_res.nc" src = Path(self._driver_config["config"]["topo_data_30s"]) - dst = Path(self._driver_config["rundir"]) / fn + dst = Path(self._driver_config[STR.rundir]) / fn yield self._taskname("Input grid") yield asset(dst, dst.is_file) yield symlink(target=src, linkname=dst) @@ -84,5 +84,5 @@ def _runcmd(self): Returns the full command-line component invocation. """ inputs = [str(self._driver_config["config"][k]) for k in ("tile", "resolution", "halo")] - executable = self._driver_config["execution"]["executable"] + executable = self._driver_config[STR.execution][STR.executable] return "echo '%s' | %s" % ("\n".join(inputs), executable) diff --git a/src/uwtools/drivers/schism.py b/src/uwtools/drivers/schism.py index bc6806e09..575b78ba4 100644 --- a/src/uwtools/drivers/schism.py +++ b/src/uwtools/drivers/schism.py @@ -28,12 +28,12 @@ def namelist_file(self): yield self._taskname(fn) path = self._rundir / fn yield asset(path, path.is_file) - template_file = Path(self._driver_config["namelist"]["template_file"]) + template_file = Path(self._driver_config[STR.namelist]["template_file"]) yield file(path=template_file) render( input_file=template_file, output_file=path, - overrides=self._driver_config["namelist"]["template_values"], + overrides=self._driver_config[STR.namelist]["template_values"], ) @tasks diff --git a/src/uwtools/drivers/sfc_climo_gen.py b/src/uwtools/drivers/sfc_climo_gen.py index 3cf6bca38..02090ade5 100644 --- a/src/uwtools/drivers/sfc_climo_gen.py +++ b/src/uwtools/drivers/sfc_climo_gen.py @@ -28,14 +28,14 @@ def namelist_file(self): yield self._taskname(f"namelist file {fn}") path = self._rundir / fn yield asset(path, path.is_file) - vals = self._driver_config["namelist"]["update_values"]["config"] + vals = self._driver_config[STR.namelist][STR.updatevalues]["config"] input_paths = [Path(v) for k, v in vals.items() if k.startswith("input_")] input_paths += [Path(vals["mosaic_file_mdl"])] input_paths += [Path(vals["orog_dir_mdl"]) / fn for fn in vals["orog_files_mdl"]] yield [file(input_path) for input_path in input_paths] self._create_user_updated_config( config_class=NMLConfig, - config_values=self._driver_config["namelist"], + config_values=self._driver_config[STR.namelist], path=path, schema=self._namelist_schema(), ) diff --git a/src/uwtools/drivers/shave.py b/src/uwtools/drivers/shave.py index af41306f3..a949add77 100644 --- a/src/uwtools/drivers/shave.py +++ b/src/uwtools/drivers/shave.py @@ -37,7 +37,7 @@ def _runcmd(self): """ Returns the full command-line component invocation. """ - executable = self._driver_config["execution"]["executable"] + executable = self._driver_config[STR.execution][STR.executable] config = self._driver_config["config"] input_file = config["input_grid_file"] output_file = input_file.replace(".nc", "_NH0.nc") diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index 2b56eb649..9b4f95d7c 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -49,12 +49,12 @@ def namelist_file(self): path = self._namelist_path yield self._taskname(str(path)) yield asset(path, path.is_file) - base_file = self._driver_config["namelist"].get("base_file") + base_file = self._driver_config[STR.namelist].get(STR.basefile) yield file(Path(base_file)) if base_file else None path.parent.mkdir(parents=True, exist_ok=True) self._create_user_updated_config( config_class=NMLConfig, - config_values=self._driver_config["namelist"], + config_values=self._driver_config[STR.namelist], path=path, schema=self._namelist_schema(), ) @@ -93,11 +93,11 @@ def _runcmd(self) -> str: """ Returns the full command-line component invocation. """ - execution = self._driver_config.get("execution", {}) - mpiargs = execution.get("mpiargs", []) + execution = self._driver_config.get(STR.execution, {}) + mpiargs = execution.get(STR.mpiargs, []) components = [ - execution.get("mpicmd"), + execution.get(STR.mpicmd), *[str(x) for x in mpiargs], - "%s < %s" % (execution["executable"], self._namelist_path.name), + "%s < %s" % (execution[STR.executable], self._namelist_path.name), ] return " ".join(filter(None, components)) diff --git a/src/uwtools/drivers/ww3.py b/src/uwtools/drivers/ww3.py index 2faf6d64c..54d957e47 100644 --- a/src/uwtools/drivers/ww3.py +++ b/src/uwtools/drivers/ww3.py @@ -28,12 +28,12 @@ def namelist_file(self): yield self._taskname(fn) path = self._rundir / fn yield asset(path, path.is_file) - template_file = Path(self._driver_config["namelist"]["template_file"]) + template_file = Path(self._driver_config[STR.namelist]["template_file"]) yield file(template_file) render( input_file=template_file, output_file=path, - overrides=self._driver_config["namelist"]["template_values"], + overrides=self._driver_config[STR.namelist]["template_values"], ) @tasks diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index fcd53851f..3319eb592 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -60,8 +60,11 @@ class STR: String lookup map. """ + account: str = "account" action: str = "action" + basefile: str = "base_file" batch: str = "batch" + batchargs: str = "batchargs" cdeps: str = "cdeps" cfgfile: str = "config_file" chgrescube: str = "chgres_cube" @@ -72,8 +75,11 @@ class STR: cycle: str = "cycle" dryrun: str = "dry_run" env: str = "env" + envcmds: str = "envcmds" esggrid: str = "esg_grid" + executable: str = "executable" execute: str = "execute" + execution: str = "execution" file1fmt: str = "file_1_format" file1path: str = "file_1_path" file2fmt: str = "file_2_format" @@ -100,28 +106,38 @@ class STR: module: str = "module" mpas: str = "mpas" mpasinit: str = "mpas_init" + mpiargs: str = "mpiargs" + mpicmd: str = "mpicmd" + namelist: str = "namelist" oroggsl: str = "orog_gsl" outfile: str = "output_file" outfmt: str = "output_format" + platform: str = "platform" + properties: str = "properties" quiet: str = "quiet" realize: str = "realize" render: str = "render" rocoto: str = "rocoto" run: str = "run" + rundir: str = "rundir" + scheduler: str = "scheduler" schemafile: str = "schema_file" schism: str = "schism" searchpath: str = "search_path" sfcclimogen: str = "sfc_climo_gen" shave: str = "shave" + stdout: str = "stdout" targetdir: str = "target_dir" task: str = "task" tasks: str = "tasks" template: str = "template" + threads: str = "threads" total: str = "total" translate: str = "translate" ungrib: str = "ungrib" updatefile: str = "update_file" updatefmt: str = "update_format" + updatevalues: str = "update_values" upp: str = "upp" validate: str = "validate" valsfile: str = "values_file" From 7ba20cf77eafc433f1f056cb9748a42ab54c3067 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 31 Jul 2024 00:24:59 +0000 Subject: [PATCH 07/18] Honor coupler --- src/uwtools/drivers/driver.py | 40 ++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 241be88b4..5988257e8 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -43,19 +43,19 @@ def __init__( schema_file: Optional[Path] = None, coupler: Optional[str] = None, ) -> None: - config = config if isinstance(config, YAMLConfig) else YAMLConfig(config=config) - config.dereference( + self._config_full = config if isinstance(config, YAMLConfig) else YAMLConfig(config=config) + self._config_full.dereference( context={ **({STR.cycle: cycle} if cycle else {}), **({STR.leadtime: leadtime} if leadtime is not None else {}), - **config.data, + **self._config_full.data, } ) - self._config = config + self._config = self._config_full for key in key_path or []: self._config = self._config[key] if coupler: - self._config[STR.rundir] = config[coupler][STR.rundir] + self._config[STR.rundir] = self._config_full[coupler][STR.rundir] self._validate(schema_file) dryrun(enable=dry_run) @@ -217,10 +217,11 @@ class AssetsCycleBased(Assets): def __init__( self, cycle: datetime, - config: Optional[Union[dict, Path]] = None, + config: Optional[Union[dict, str, YAMLConfig, Path]] = None, dry_run: bool = False, key_path: Optional[list[str]] = None, schema_file: Optional[Path] = None, + coupler: Optional[str] = None, ): super().__init__( cycle=cycle, @@ -228,6 +229,7 @@ def __init__( dry_run=dry_run, key_path=key_path, schema_file=schema_file, + coupler=coupler, ) self._cycle = cycle @@ -241,10 +243,11 @@ def __init__( self, cycle: datetime, leadtime: timedelta, - config: Optional[Union[dict, Path]] = None, + config: Optional[Union[dict, str, YAMLConfig, Path]] = None, dry_run: bool = False, key_path: Optional[list[str]] = None, schema_file: Optional[Path] = None, + coupler: Optional[str] = None, ): super().__init__( cycle=cycle, @@ -253,6 +256,7 @@ def __init__( dry_run=dry_run, key_path=key_path, schema_file=schema_file, + coupler=coupler, ) self._cycle = cycle self._leadtime = leadtime @@ -265,16 +269,18 @@ class AssetsTimeInvariant(Assets): def __init__( self, - config: Optional[Union[dict, Path]] = None, + config: Optional[Union[dict, str, YAMLConfig, Path]] = None, dry_run: bool = False, key_path: Optional[list[str]] = None, schema_file: Optional[Path] = None, + coupler: Optional[str] = None, ): super().__init__( config=config, dry_run=dry_run, key_path=key_path, schema_file=schema_file, + coupler=coupler, ) @@ -287,11 +293,12 @@ def __init__( self, cycle: Optional[datetime] = None, leadtime: Optional[timedelta] = None, - config: Optional[Union[dict, Path]] = None, + config: Optional[Union[dict, str, YAMLConfig, Path]] = None, dry_run: bool = False, key_path: Optional[list[str]] = None, batch: bool = False, schema_file: Optional[Path] = None, + coupler: Optional[str] = None, ): super().__init__( cycle=cycle, @@ -300,8 +307,11 @@ def __init__( dry_run=dry_run, key_path=key_path, schema_file=schema_file, + coupler=coupler, ) self._batch = batch + if coupler: + self._config[STR.execution] = self._config_full[coupler][STR.execution] # Workflow tasks @@ -487,11 +497,12 @@ class DriverCycleBased(Driver): def __init__( self, cycle: datetime, - config: Optional[Union[dict, Path]] = None, + config: Optional[Union[dict, str, YAMLConfig, Path]] = None, dry_run: bool = False, key_path: Optional[list[str]] = None, batch: bool = False, schema_file: Optional[Path] = None, + coupler: Optional[str] = None, ): super().__init__( cycle=cycle, @@ -500,6 +511,7 @@ def __init__( key_path=key_path, batch=batch, schema_file=schema_file, + coupler=coupler, ) self._cycle = cycle @@ -513,11 +525,12 @@ def __init__( self, cycle: datetime, leadtime: timedelta, - config: Optional[Union[dict, Path]] = None, + config: Optional[Union[dict, str, YAMLConfig, Path]] = None, dry_run: bool = False, key_path: Optional[list[str]] = None, batch: bool = False, schema_file: Optional[Path] = None, + coupler: Optional[str] = None, ): super().__init__( cycle=cycle, @@ -527,6 +540,7 @@ def __init__( key_path=key_path, batch=batch, schema_file=schema_file, + coupler=coupler, ) self._cycle = cycle self._leadtime = leadtime @@ -539,11 +553,12 @@ class DriverTimeInvariant(Driver): def __init__( self, - config: Optional[Union[dict, Path]] = None, + config: Optional[Union[dict, str, YAMLConfig, Path]] = None, dry_run: bool = False, key_path: Optional[list[str]] = None, batch: bool = False, schema_file: Optional[Path] = None, + coupler: Optional[str] = None, ): super().__init__( config=config, @@ -551,6 +566,7 @@ def __init__( key_path=key_path, batch=batch, schema_file=schema_file, + coupler=coupler, ) From 1b7cf020b8eb04f29f4e9cb69388dd87a2f01af8 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 31 Jul 2024 04:09:57 +0000 Subject: [PATCH 08/18] WIP --- src/uwtools/api/config.py | 12 ++++-- src/uwtools/config/validator.py | 14 +++---- src/uwtools/drivers/driver.py | 12 ++++-- src/uwtools/rocoto.py | 4 +- src/uwtools/tests/drivers/test_driver.py | 52 ++++++++++++++++++++++-- 5 files changed, 74 insertions(+), 20 deletions(-) diff --git a/src/uwtools/api/config.py b/src/uwtools/api/config.py index c1683bfc8..2d402b1be 100644 --- a/src/uwtools/api/config.py +++ b/src/uwtools/api/config.py @@ -15,6 +15,7 @@ from uwtools.config.tools import compare_configs as _compare from uwtools.config.tools import realize_config as _realize from uwtools.config.validator import validate_external as _validate_external +from uwtools.exceptions import UWConfigError from uwtools.utils.api import ensure_data_source as _ensure_data_source from uwtools.utils.file import FORMAT as _FORMAT from uwtools.utils.file import str2path as _str2path @@ -173,9 +174,14 @@ def validate( :param stdin_ok: OK to read from ``stdin``? :return: ``True`` if the YAML file conforms to the schema, ``False`` otherwise. """ - return _validate_external( - schema_file=_str2path(schema_file), config=_ensure_data_source(_str2path(config), stdin_ok) - ) + try: + _validate_external( + schema_file=_str2path(schema_file), + config=_ensure_data_source(_str2path(config), stdin_ok), + ) + except UWConfigError: + return False + return True # Import-time code diff --git a/src/uwtools/config/validator.py b/src/uwtools/config/validator.py index 8edf2999e..16de42ed0 100644 --- a/src/uwtools/config/validator.py +++ b/src/uwtools/config/validator.py @@ -47,37 +47,37 @@ def validate(schema: dict, config: dict) -> bool: def validate_internal( - schema_name: str, config: Union[dict, YAMLConfig, Optional[Path]] = None + schema_name: str, config: Optional[Union[dict, YAMLConfig, Path]] = None ) -> None: """ Validate a config against a uwtools-internal schema. - :param config: The config to validate. :param schema_name: Name of uwtools schema to validate the config against. + :param config: The config to validate. :raises: UWConfigError if config fails validation. """ log.info("Validating config against internal schema %s", schema_name) schema_file = get_schema_file(schema_name) log.debug("Using schema file: %s", schema_file) - if not validate_external(config=config, schema_file=schema_file): - raise UWConfigError("YAML validation errors") + validate_external(config=config, schema_file=schema_file) def validate_external( schema_file: Path, config: Union[dict, YAMLConfig, Optional[Path]] = None -) -> bool: +) -> None: """ Validate a YAML config against the JSON Schema in the given schema file. :param schema_file: The JSON Schema file to use for validation. :param config: The config to validate. - :return: Did the YAML file conform to the schema? + :raises: UWConfigError if config fails validation. """ with open(schema_file, "r", encoding="utf-8") as f: schema = json.load(f) cfgobj = _prep_config(config) - return validate(schema=schema, config=cfgobj.data) + if not validate(schema=schema, config=cfgobj.data): + raise UWConfigError("YAML validation errors") # Private functions diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 5988257e8..3e193c92a 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -55,7 +55,7 @@ def __init__( for key in key_path or []: self._config = self._config[key] if coupler: - self._config[STR.rundir] = self._config_full[coupler][STR.rundir] + self._config[self._driver_name][STR.rundir] = self._config_full[coupler][STR.rundir] self._validate(schema_file) dryrun(enable=dry_run) @@ -201,12 +201,14 @@ def _taskname(self, suffix: str) -> str: def _validate(self, schema_file: Optional[Path] = None) -> None: """ Perform all necessary schema validation. + + :param schema_file: The JSON Schema file to use for validation. + :raises: UWConfigError if config fails validation. """ - schema_name = self._driver_name.replace("_", "-") if schema_file: validate_external(schema_file=schema_file, config=self._config) else: - validate_internal(schema_name=schema_name, config=self._config) + validate_internal(schema_name=self._driver_name.replace("_", "-"), config=self._config) class AssetsCycleBased(Assets): @@ -311,7 +313,9 @@ def __init__( ) self._batch = batch if coupler: - self._config[STR.execution] = self._config_full[coupler][STR.execution] + self._config[self._driver_name][STR.execution] = self._config_full[coupler][ + STR.execution + ] # Workflow tasks diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index ffd9953ad..a04cda373 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -348,9 +348,7 @@ def _config_validate(self, config: Union[dict, YAMLConfig, Optional[Path]]) -> N :raises: UWConfigError if config fails validation. """ schema_file = resource_path("jsonschema/rocoto.jsonschema") - ok = validate_yaml(schema_file=schema_file, config=config) - if not ok: - raise UWConfigError("YAML validation errors") + validate_yaml(schema_file=schema_file, config=config) @property def _doctype(self) -> Optional[str]: diff --git a/src/uwtools/tests/drivers/test_driver.py b/src/uwtools/tests/drivers/test_driver.py index 6eb29431e..c0cdc7ba8 100644 --- a/src/uwtools/tests/drivers/test_driver.py +++ b/src/uwtools/tests/drivers/test_driver.py @@ -71,9 +71,9 @@ class ConcreteDriverTimeInvariant(Common, driver.DriverTimeInvariant): pass -def write(path, s): +def write(path, x): with open(path, "w", encoding="utf-8") as f: - json.dump(s, f) + json.dump(x, f) return path @@ -103,10 +103,30 @@ def config(tmp_path): "account": "me", "scheduler": "slurm", }, - "rootdir": "/path/to", } +@fixture +def coupler_schema(tmp_path): + return write( + tmp_path / "concrete.jsonschema", + { + "properties": { + "concrete": { + "properties": { + "execution": {"type": "object"}, + "rundir": {"type": "string"}, + }, + "required": ["rundir"], + "type": "object", + }, + }, + "required": ["concrete"], + "type": "object", + }, + ) + + @fixture def assetsobj(config): return ConcreteAssetsTimeInvariant(config=config, dry_run=False) @@ -124,6 +144,17 @@ def test_Assets(assetsobj): assert Path(assetsobj._driver_config["base_file"]).name == "base.yaml" +def test_Assets_coupler(config, coupler_schema): + config["coupler"] = {"rundir": "/coupler/run/dir"} + del config["concrete"]["rundir"] + with patch.object(ConcreteAssetsTimeInvariant, "_validate", driver.Assets._validate): + with raises(UWConfigError): + ConcreteAssetsTimeInvariant(config=config, schema_file=coupler_schema) + assert ConcreteAssetsTimeInvariant( + config=config, schema_file=coupler_schema, coupler="coupler" + ) + + def test_Assets_repr_cycle_based(config): obj = ConcreteAssetsCycleBased(config=config, cycle=dt.datetime(2024, 7, 2, 12)) expected = "concrete 2024-07-02T12:00 in %s" % obj._driver_config["rundir"] @@ -265,6 +296,21 @@ def test_Driver(driverobj): assert driverobj._batch is True +def test_Driver_coupler(config, coupler_schema): + config["coupler"] = { + "execution": {"executable": "/path/to/coupled.exe"}, + "rundir": "/coupler/run/dir", + } + del config["concrete"]["rundir"] + del config["concrete"]["execution"] + with patch.object(ConcreteDriverTimeInvariant, "_validate", driver.Driver._validate): + with raises(UWConfigError): + ConcreteDriverTimeInvariant(config=config, schema_file=coupler_schema) + assert ConcreteDriverTimeInvariant( + config=config, schema_file=coupler_schema, coupler="coupler" + ) + + # Tests for workflow methods From 12fa5ddf3e866c368a83b23cd2095fe18f64c2ec Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 31 Jul 2024 04:18:20 +0000 Subject: [PATCH 09/18] WIP --- src/uwtools/tests/api/test_config.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/uwtools/tests/api/test_config.py b/src/uwtools/tests/api/test_config.py index f9ae27a73..abe750442 100644 --- a/src/uwtools/tests/api/test_config.py +++ b/src/uwtools/tests/api/test_config.py @@ -9,6 +9,7 @@ from uwtools.api import config from uwtools.config.formats.yaml import YAMLConfig +from uwtools.exceptions import UWConfigError from uwtools.utils.file import FORMAT @@ -94,18 +95,22 @@ def test_realize_to_dict(): @mark.parametrize("cfg", [{"foo": "bar"}, YAMLConfig(config={})]) def test_validate(cfg): kwargs: dict = {"schema_file": "schema-file", "config": cfg} - with patch.object(config, "_validate_external", return_value=True) as _validate_external: - assert config.validate(**kwargs) - _validate_external.assert_called_once_with( - schema_file=Path(kwargs["schema_file"]), config=kwargs["config"] + with patch.object(config, "_validate_external") as _validate_external: + assert config.validate(**kwargs) is True + _validate_external.side_effect = UWConfigError() + assert config.validate(**kwargs) is False + _validate_external.assert_called_with( + schema_file=Path(kwargs["schema_file"]), + config=kwargs["config"], ) -def test_validate_config_file(tmp_path): +@mark.parametrize("cast", (str, Path)) +def test_validate_config_file(cast, tmp_path): cfg = tmp_path / "config.yaml" with open(cfg, "w", encoding="utf-8") as f: yaml.dump({}, f) - kwargs: dict = {"schema_file": "schema-file", "config": cfg} + kwargs: dict = {"schema_file": "schema-file", "config": cast(cfg)} with patch.object(config, "_validate_external", return_value=True) as _validate_external: assert config.validate(**kwargs) _validate_external.assert_called_once_with(schema_file=Path(kwargs["schema_file"]), config=cfg) From e4ec2997184b4b7e835a69a735969508478e31b3 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 31 Jul 2024 05:08:18 +0000 Subject: [PATCH 10/18] Add inherited parameter descriptions to driver docstrings --- docs/conf.py | 3 ++- src/uwtools/drivers/cdeps.py | 4 ++++ src/uwtools/drivers/driver.py | 3 ++- src/uwtools/drivers/support.py | 13 +++++++++++++ src/uwtools/tests/drivers/test_driver.py | 13 +++++++++++-- 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 00cb58d5f..db432053b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ autoclass_content = "both" autodoc_mock_imports = ["f90nml", "iotaa", "jsonschema", "lxml", "referencing"] +autodoc_typehints = "description" copyright = str(dt.datetime.now().year) extensions = ["sphinx.ext.autodoc", "sphinx.ext.extlinks", "sphinx.ext.intersphinx"] extlinks_detect_hardcoded_links = True @@ -24,8 +25,8 @@ numfig_format = {"figure": "Figure %s"} project = "Unified Workflow Tools" release = _metadata["version"] -version = _metadata["version"] user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36" +version = _metadata["version"] extlinks = { "anaconda": ("https://www.anaconda.com/%s", "%s"), diff --git a/src/uwtools/drivers/cdeps.py b/src/uwtools/drivers/cdeps.py index d3ad16c66..fc76a347a 100644 --- a/src/uwtools/drivers/cdeps.py +++ b/src/uwtools/drivers/cdeps.py @@ -9,6 +9,7 @@ from uwtools.api.template import _render from uwtools.config.formats.nml import NMLConfig from uwtools.drivers.driver import AssetsCycleBased +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR from uwtools.utils.tasks import file @@ -127,3 +128,6 @@ def _model_stream_file(self, group: str, path: Path, template_file: str) -> None output_file=path, values_src=self._driver_config[group], ) + + +set_driver_docstring(CDEPS) diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 3e193c92a..9cbdb4076 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -585,7 +585,7 @@ def _add_docstring(class_: type, omit: Optional[list[str]] = None) -> None: :param omit: Parameters to omit from the docstring. """ base = """ - The driver. + A driver. :param cycle: The cycle. :param leadtime: The leadtime. @@ -594,6 +594,7 @@ def _add_docstring(class_: type, omit: Optional[list[str]] = None) -> None: :param key_path: Keys leading through the config to the driver's configuration block. :param batch: Run component via the batch system? :param schema_file: Path to schema file to use to validate an external driver. + :param coupler: Name of coupler block in config providing shared values. """ setattr( class_, diff --git a/src/uwtools/drivers/support.py b/src/uwtools/drivers/support.py index 83b78b181..dc0ed14b2 100644 --- a/src/uwtools/drivers/support.py +++ b/src/uwtools/drivers/support.py @@ -2,6 +2,8 @@ Driver support. """ +from typing import Type + import iotaa as _iotaa from uwtools.drivers.driver import DriverT @@ -14,6 +16,17 @@ def graph() -> str: return _iotaa.graph() +def set_driver_docstring(driver_class: Type) -> None: + """ + Appends inherited parameter descriptions to the driver's own docstring. + """ + header = driver_class.__doc__ + body = driver_class.__mro__[1].__doc__ + assert header is not None + assert body is not None + setattr(driver_class, "__doc__", "\n".join([header.strip(), *body.split("\n")[1:]])) + + def tasks(driver_class: DriverT) -> dict[str, str]: """ Returns a mapping from task names to their one-line descriptions. diff --git a/src/uwtools/tests/drivers/test_driver.py b/src/uwtools/tests/drivers/test_driver.py index c0cdc7ba8..3d9a254cd 100644 --- a/src/uwtools/tests/drivers/test_driver.py +++ b/src/uwtools/tests/drivers/test_driver.py @@ -606,6 +606,15 @@ class C: assert getattr(C, "__doc__") is None with patch.object(driver, "C", C, create=True): class_ = driver.C # type: ignore # pylint: disable=no-member - omit = ["cycle", "leadtime", "config", "dry_run", "key_path", "batch", "schema_file"] + omit = [ + "batch", + "config", + "coupler", + "cycle", + "dry_run", + "key_path", + "leadtime", + "schema_file", + ] driver._add_docstring(class_=class_, omit=omit) - assert getattr(C, "__doc__").strip() == "The driver." + assert getattr(C, "__doc__").strip() == "A driver." From af1b31c148b53bbb259af8d978a62a4a03ae8285 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 31 Jul 2024 05:14:38 +0000 Subject: [PATCH 11/18] Add inherited parameter descriptions to driver docstrings --- src/uwtools/drivers/chgres_cube.py | 4 ++++ src/uwtools/drivers/esg_grid.py | 4 ++++ src/uwtools/drivers/filter_topo.py | 4 ++++ src/uwtools/drivers/fv3.py | 4 ++++ src/uwtools/drivers/global_equiv_resol.py | 4 ++++ src/uwtools/drivers/ioda.py | 4 ++++ src/uwtools/drivers/jedi.py | 4 ++++ src/uwtools/drivers/make_hgrid.py | 4 ++++ src/uwtools/drivers/make_solo_mosaic.py | 4 ++++ src/uwtools/drivers/mpas.py | 4 ++++ src/uwtools/drivers/mpas_init.py | 4 ++++ src/uwtools/drivers/orog_gsl.py | 4 ++++ src/uwtools/drivers/schism.py | 4 ++++ src/uwtools/drivers/sfc_climo_gen.py | 4 ++++ src/uwtools/drivers/shave.py | 4 ++++ src/uwtools/drivers/ungrib.py | 4 ++++ src/uwtools/drivers/upp.py | 4 ++++ src/uwtools/drivers/ww3.py | 4 ++++ 18 files changed, 72 insertions(+) diff --git a/src/uwtools/drivers/chgres_cube.py b/src/uwtools/drivers/chgres_cube.py index 9291ff68e..eacc8998b 100644 --- a/src/uwtools/drivers/chgres_cube.py +++ b/src/uwtools/drivers/chgres_cube.py @@ -8,6 +8,7 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.drivers.driver import DriverCycleBased +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR from uwtools.utils.tasks import file @@ -96,3 +97,6 @@ def _driver_name(self) -> str: Returns the name of this driver. """ return STR.chgrescube + + +set_driver_docstring(ChgresCube) diff --git a/src/uwtools/drivers/esg_grid.py b/src/uwtools/drivers/esg_grid.py index 8b9ed6033..e3f86b037 100644 --- a/src/uwtools/drivers/esg_grid.py +++ b/src/uwtools/drivers/esg_grid.py @@ -8,6 +8,7 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.drivers.driver import DriverTimeInvariant +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR from uwtools.utils.tasks import file @@ -56,3 +57,6 @@ def _driver_name(self) -> str: Returns the name of this driver. """ return STR.esggrid + + +set_driver_docstring(ESGGrid) diff --git a/src/uwtools/drivers/filter_topo.py b/src/uwtools/drivers/filter_topo.py index 2fc8def8a..18a12b7c8 100644 --- a/src/uwtools/drivers/filter_topo.py +++ b/src/uwtools/drivers/filter_topo.py @@ -8,6 +8,7 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.drivers.driver import DriverTimeInvariant +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR from uwtools.utils.tasks import symlink @@ -67,3 +68,6 @@ def _driver_name(self) -> str: Returns the name of this driver. """ return STR.filtertopo + + +set_driver_docstring(FilterTopo) diff --git a/src/uwtools/drivers/fv3.py b/src/uwtools/drivers/fv3.py index 745e0be58..d7ab65c7e 100644 --- a/src/uwtools/drivers/fv3.py +++ b/src/uwtools/drivers/fv3.py @@ -10,6 +10,7 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.config.formats.yaml import YAMLConfig from uwtools.drivers.driver import DriverCycleBased +from uwtools.drivers.support import set_driver_docstring from uwtools.logging import log from uwtools.strings import STR from uwtools.utils.tasks import file, filecopy, symlink @@ -183,3 +184,6 @@ def _driver_name(self) -> str: Returns the name of this driver. """ return STR.fv3 + + +set_driver_docstring(FV3) diff --git a/src/uwtools/drivers/global_equiv_resol.py b/src/uwtools/drivers/global_equiv_resol.py index 034c067b0..59eff0149 100644 --- a/src/uwtools/drivers/global_equiv_resol.py +++ b/src/uwtools/drivers/global_equiv_resol.py @@ -7,6 +7,7 @@ from iotaa import asset, external, tasks from uwtools.drivers.driver import DriverTimeInvariant +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR @@ -54,3 +55,6 @@ def _runcmd(self): executable = self._driver_config[STR.execution][STR.executable] input_file_path = self._driver_config["input_grid_file"] return f"{executable} {input_file_path}" + + +set_driver_docstring(GlobalEquivResol) diff --git a/src/uwtools/drivers/ioda.py b/src/uwtools/drivers/ioda.py index 973bb29e9..2be609535 100644 --- a/src/uwtools/drivers/ioda.py +++ b/src/uwtools/drivers/ioda.py @@ -5,6 +5,7 @@ from iotaa import tasks from uwtools.drivers.jedi_base import JEDIBase +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR @@ -52,3 +53,6 @@ def _runcmd(self) -> str: executable = self._driver_config[STR.execution][STR.executable] jedi_config = str(self._rundir / self._config_fn) return " ".join([executable, jedi_config]) + + +set_driver_docstring(IODA) diff --git a/src/uwtools/drivers/jedi.py b/src/uwtools/drivers/jedi.py index 52caa65c2..4b7e57210 100644 --- a/src/uwtools/drivers/jedi.py +++ b/src/uwtools/drivers/jedi.py @@ -8,6 +8,7 @@ from iotaa import asset, refs, run, task, tasks from uwtools.drivers.jedi_base import JEDIBase +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR from uwtools.utils.tasks import file @@ -84,3 +85,6 @@ def _runcmd(self) -> str: str(jedi_config), # JEDI config file ] return " ".join(filter(None, components)) + + +set_driver_docstring(JEDI) diff --git a/src/uwtools/drivers/make_hgrid.py b/src/uwtools/drivers/make_hgrid.py index 765db2346..821e39e40 100644 --- a/src/uwtools/drivers/make_hgrid.py +++ b/src/uwtools/drivers/make_hgrid.py @@ -5,6 +5,7 @@ from iotaa import tasks from uwtools.drivers.driver import DriverTimeInvariant +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR @@ -48,3 +49,6 @@ def _runcmd(self): else: flags.append("--%s %s" % (k, v)) return f"{executable} " + " ".join(flags) + + +set_driver_docstring(MakeHgrid) diff --git a/src/uwtools/drivers/make_solo_mosaic.py b/src/uwtools/drivers/make_solo_mosaic.py index 5d7776a74..57abb6a3d 100644 --- a/src/uwtools/drivers/make_solo_mosaic.py +++ b/src/uwtools/drivers/make_solo_mosaic.py @@ -5,6 +5,7 @@ from iotaa import tasks from uwtools.drivers.driver import DriverTimeInvariant +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR @@ -48,3 +49,6 @@ def _taskname(self, suffix: str) -> str: :param suffix: Log-string suffix. """ return "%s %s" % (self._driver_name, suffix) + + +set_driver_docstring(MakeSoloMosaic) diff --git a/src/uwtools/drivers/mpas.py b/src/uwtools/drivers/mpas.py index 3ff22e716..e99a5b1ce 100644 --- a/src/uwtools/drivers/mpas.py +++ b/src/uwtools/drivers/mpas.py @@ -9,6 +9,7 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.drivers.mpas_base import MPASBase +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR from uwtools.utils.tasks import file, symlink @@ -80,3 +81,6 @@ def _streams_fn(self) -> str: The streams filename. """ return "streams.atmosphere" + + +set_driver_docstring(MPAS) diff --git a/src/uwtools/drivers/mpas_init.py b/src/uwtools/drivers/mpas_init.py index d0e768828..f7b7b7cc7 100644 --- a/src/uwtools/drivers/mpas_init.py +++ b/src/uwtools/drivers/mpas_init.py @@ -9,6 +9,7 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.drivers.mpas_base import MPASBase +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR from uwtools.utils.tasks import file, symlink @@ -84,3 +85,6 @@ def _streams_fn(self) -> str: The streams filename. """ return "streams.init_atmosphere" + + +set_driver_docstring(MPASInit) diff --git a/src/uwtools/drivers/orog_gsl.py b/src/uwtools/drivers/orog_gsl.py index 12823032c..19b03feba 100644 --- a/src/uwtools/drivers/orog_gsl.py +++ b/src/uwtools/drivers/orog_gsl.py @@ -7,6 +7,7 @@ from iotaa import asset, task, tasks from uwtools.drivers.driver import DriverTimeInvariant +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR from uwtools.utils.tasks import symlink @@ -86,3 +87,6 @@ def _runcmd(self): inputs = [str(self._driver_config["config"][k]) for k in ("tile", "resolution", "halo")] executable = self._driver_config[STR.execution][STR.executable] return "echo '%s' | %s" % ("\n".join(inputs), executable) + + +set_driver_docstring(OrogGSL) diff --git a/src/uwtools/drivers/schism.py b/src/uwtools/drivers/schism.py index 575b78ba4..9877c9239 100644 --- a/src/uwtools/drivers/schism.py +++ b/src/uwtools/drivers/schism.py @@ -8,6 +8,7 @@ from uwtools.api.template import render from uwtools.drivers.driver import AssetsCycleBased +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR from uwtools.utils.tasks import file @@ -52,3 +53,6 @@ def _driver_name(self) -> str: Returns the name of this driver. """ return STR.schism + + +set_driver_docstring(SCHISM) diff --git a/src/uwtools/drivers/sfc_climo_gen.py b/src/uwtools/drivers/sfc_climo_gen.py index 02090ade5..0a34928c8 100644 --- a/src/uwtools/drivers/sfc_climo_gen.py +++ b/src/uwtools/drivers/sfc_climo_gen.py @@ -8,6 +8,7 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.drivers.driver import DriverTimeInvariant +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR from uwtools.utils.tasks import file @@ -59,3 +60,6 @@ def _driver_name(self) -> str: Returns the name of this driver. """ return STR.sfcclimogen + + +set_driver_docstring(SfcClimoGen) diff --git a/src/uwtools/drivers/shave.py b/src/uwtools/drivers/shave.py index a949add77..39ffb50be 100644 --- a/src/uwtools/drivers/shave.py +++ b/src/uwtools/drivers/shave.py @@ -5,6 +5,7 @@ from iotaa import tasks from uwtools.drivers.driver import DriverTimeInvariant +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR @@ -44,3 +45,6 @@ def _runcmd(self): flags = [config[key] for key in ["nx", "ny", "nh4", "input_grid_file"]] flags.append(output_file) return f"{executable} {' '.join(str(flag) for flag in flags)}" + + +set_driver_docstring(Shave) diff --git a/src/uwtools/drivers/ungrib.py b/src/uwtools/drivers/ungrib.py index 1b79871a0..ca2cbbcdb 100644 --- a/src/uwtools/drivers/ungrib.py +++ b/src/uwtools/drivers/ungrib.py @@ -9,6 +9,7 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.drivers.driver import DriverCycleBased +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR from uwtools.utils.tasks import file @@ -132,3 +133,6 @@ def _ext(n): """ b = 26 return "{:A>3}".format(("" if n < b else _ext(n // b)) + chr(65 + n % b))[-3:] + + +set_driver_docstring(Ungrib) diff --git a/src/uwtools/drivers/upp.py b/src/uwtools/drivers/upp.py index 9b4f95d7c..313fb372b 100644 --- a/src/uwtools/drivers/upp.py +++ b/src/uwtools/drivers/upp.py @@ -8,6 +8,7 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.drivers.driver import DriverCycleLeadtimeBased +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR from uwtools.utils.tasks import file, filecopy, symlink @@ -101,3 +102,6 @@ def _runcmd(self) -> str: "%s < %s" % (execution[STR.executable], self._namelist_path.name), ] return " ".join(filter(None, components)) + + +set_driver_docstring(UPP) diff --git a/src/uwtools/drivers/ww3.py b/src/uwtools/drivers/ww3.py index 54d957e47..5b7c7cc7e 100644 --- a/src/uwtools/drivers/ww3.py +++ b/src/uwtools/drivers/ww3.py @@ -8,6 +8,7 @@ from uwtools.api.template import render from uwtools.drivers.driver import AssetsCycleBased +from uwtools.drivers.support import set_driver_docstring from uwtools.strings import STR from uwtools.utils.tasks import file @@ -66,3 +67,6 @@ def _driver_name(self) -> str: Returns the name of this driver. """ return STR.ww3 + + +set_driver_docstring(WaveWatchIII) From 2a9501307402cb0a958ae318b32241e89f125b96 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 31 Jul 2024 05:44:06 +0000 Subject: [PATCH 12/18] Update unit tests --- src/uwtools/drivers/support.py | 16 +++++++++++----- src/uwtools/tests/drivers/test_support.py | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/uwtools/drivers/support.py b/src/uwtools/drivers/support.py index dc0ed14b2..65b567e88 100644 --- a/src/uwtools/drivers/support.py +++ b/src/uwtools/drivers/support.py @@ -2,6 +2,7 @@ Driver support. """ +import re from typing import Type import iotaa as _iotaa @@ -19,12 +20,17 @@ def graph() -> str: def set_driver_docstring(driver_class: Type) -> None: """ Appends inherited parameter descriptions to the driver's own docstring. + + :param driver_class: The class whose docstring to update. """ - header = driver_class.__doc__ - body = driver_class.__mro__[1].__doc__ - assert header is not None - assert body is not None - setattr(driver_class, "__doc__", "\n".join([header.strip(), *body.split("\n")[1:]])) + head_old = driver_class.__doc__ + body_old = driver_class.__mro__[1].__doc__ + assert head_old is not None + assert body_old is not None + head = head_old.strip() + body = re.sub(r"\n\n+", "\n", body_old.strip()).split("\n")[1:] + new = "\n".join([f"{head}\n", *body]) + setattr(driver_class, "__doc__", new) def tasks(driver_class: DriverT) -> dict[str, str]: diff --git a/src/uwtools/tests/drivers/test_support.py b/src/uwtools/tests/drivers/test_support.py index d3026e407..421abaae0 100644 --- a/src/uwtools/tests/drivers/test_support.py +++ b/src/uwtools/tests/drivers/test_support.py @@ -20,6 +20,26 @@ def ready(): ready() assert support.graph().startswith("digraph") + +def test_set_driver_docstring(): + class Parent: + """ + This will be discarded. + + body + """ + + class Child(Parent): + """ + head. + """ + + support.set_driver_docstring(Child) + assert Child.__doc__ == "head.\n\n body" + + +def test_tasks(): + class SomeDriver(DriverTimeInvariant): def provisioned_rundir(self): pass From 7b6f70bb9691bfc7c5cf8b852f2dc880a3d313de Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 31 Jul 2024 15:49:30 +0000 Subject: [PATCH 13/18] Improve docstring param description for 'coupler' --- src/uwtools/drivers/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 9cbdb4076..4a488b273 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -594,7 +594,7 @@ def _add_docstring(class_: type, omit: Optional[list[str]] = None) -> None: :param key_path: Keys leading through the config to the driver's configuration block. :param batch: Run component via the batch system? :param schema_file: Path to schema file to use to validate an external driver. - :param coupler: Name of coupler block in config providing shared values. + :param coupler: Name of coupler block in config providing run-time values. """ setattr( class_, From a30d2b6f1572dbe3510b2debd40adcf1210f0e44 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 31 Jul 2024 18:16:13 +0000 Subject: [PATCH 14/18] Improvement in set_driver_docstring() --- recipe/run_test.sh | 2 +- src/uwtools/drivers/support.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/recipe/run_test.sh b/recipe/run_test.sh index defba8696..8f4753643 100755 --- a/recipe/run_test.sh +++ b/recipe/run_test.sh @@ -23,7 +23,7 @@ lint() { msg Running linter ( set -eux - pylint -j 4 . + pylint -j 1 . ) msg OK } diff --git a/src/uwtools/drivers/support.py b/src/uwtools/drivers/support.py index 65b567e88..e01707835 100644 --- a/src/uwtools/drivers/support.py +++ b/src/uwtools/drivers/support.py @@ -24,10 +24,11 @@ def set_driver_docstring(driver_class: Type) -> None: :param driver_class: The class whose docstring to update. """ head_old = driver_class.__doc__ - body_old = driver_class.__mro__[1].__doc__ assert head_old is not None - assert body_old is not None head = head_old.strip() + parent_class = driver_class.__mro__[1] + body_old = parent_class.__doc__ + assert body_old is not None body = re.sub(r"\n\n+", "\n", body_old.strip()).split("\n")[1:] new = "\n".join([f"{head}\n", *body]) setattr(driver_class, "__doc__", new) From 079b506dc2ec9c6752ce5c8d1adfb1d0aa287f52 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 2 Aug 2024 00:10:02 +0000 Subject: [PATCH 15/18] Fix type annotation --- src/uwtools/config/validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/config/validator.py b/src/uwtools/config/validator.py index 16de42ed0..0467efe8d 100644 --- a/src/uwtools/config/validator.py +++ b/src/uwtools/config/validator.py @@ -64,7 +64,7 @@ def validate_internal( def validate_external( - schema_file: Path, config: Union[dict, YAMLConfig, Optional[Path]] = None + schema_file: Path, config: Optional[Union[dict, YAMLConfig, Path]] = None ) -> None: """ Validate a YAML config against the JSON Schema in the given schema file. From 3a72b0e9996d5894ddb6c9f50c15310f13b66955 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 2 Aug 2024 00:23:08 +0000 Subject: [PATCH 16/18] 'coupler' -> 'controller' --- src/uwtools/drivers/driver.py | 40 ++++++++++++------------ src/uwtools/tests/drivers/test_driver.py | 22 ++++++------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 4a488b273..c3a549eac 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -41,7 +41,7 @@ def __init__( dry_run: bool = False, key_path: Optional[list[str]] = None, schema_file: Optional[Path] = None, - coupler: Optional[str] = None, + controller: Optional[str] = None, ) -> None: self._config_full = config if isinstance(config, YAMLConfig) else YAMLConfig(config=config) self._config_full.dereference( @@ -54,8 +54,8 @@ def __init__( self._config = self._config_full for key in key_path or []: self._config = self._config[key] - if coupler: - self._config[self._driver_name][STR.rundir] = self._config_full[coupler][STR.rundir] + if controller: + self._config[self._driver_name][STR.rundir] = self._config_full[controller][STR.rundir] self._validate(schema_file) dryrun(enable=dry_run) @@ -223,7 +223,7 @@ def __init__( dry_run: bool = False, key_path: Optional[list[str]] = None, schema_file: Optional[Path] = None, - coupler: Optional[str] = None, + controller: Optional[str] = None, ): super().__init__( cycle=cycle, @@ -231,7 +231,7 @@ def __init__( dry_run=dry_run, key_path=key_path, schema_file=schema_file, - coupler=coupler, + controller=controller, ) self._cycle = cycle @@ -249,7 +249,7 @@ def __init__( dry_run: bool = False, key_path: Optional[list[str]] = None, schema_file: Optional[Path] = None, - coupler: Optional[str] = None, + controller: Optional[str] = None, ): super().__init__( cycle=cycle, @@ -258,7 +258,7 @@ def __init__( dry_run=dry_run, key_path=key_path, schema_file=schema_file, - coupler=coupler, + controller=controller, ) self._cycle = cycle self._leadtime = leadtime @@ -275,14 +275,14 @@ def __init__( dry_run: bool = False, key_path: Optional[list[str]] = None, schema_file: Optional[Path] = None, - coupler: Optional[str] = None, + controller: Optional[str] = None, ): super().__init__( config=config, dry_run=dry_run, key_path=key_path, schema_file=schema_file, - coupler=coupler, + controller=controller, ) @@ -300,7 +300,7 @@ def __init__( key_path: Optional[list[str]] = None, batch: bool = False, schema_file: Optional[Path] = None, - coupler: Optional[str] = None, + controller: Optional[str] = None, ): super().__init__( cycle=cycle, @@ -309,11 +309,11 @@ def __init__( dry_run=dry_run, key_path=key_path, schema_file=schema_file, - coupler=coupler, + controller=controller, ) self._batch = batch - if coupler: - self._config[self._driver_name][STR.execution] = self._config_full[coupler][ + if controller: + self._config[self._driver_name][STR.execution] = self._config_full[controller][ STR.execution ] @@ -506,7 +506,7 @@ def __init__( key_path: Optional[list[str]] = None, batch: bool = False, schema_file: Optional[Path] = None, - coupler: Optional[str] = None, + controller: Optional[str] = None, ): super().__init__( cycle=cycle, @@ -515,7 +515,7 @@ def __init__( key_path=key_path, batch=batch, schema_file=schema_file, - coupler=coupler, + controller=controller, ) self._cycle = cycle @@ -534,7 +534,7 @@ def __init__( key_path: Optional[list[str]] = None, batch: bool = False, schema_file: Optional[Path] = None, - coupler: Optional[str] = None, + controller: Optional[str] = None, ): super().__init__( cycle=cycle, @@ -544,7 +544,7 @@ def __init__( key_path=key_path, batch=batch, schema_file=schema_file, - coupler=coupler, + controller=controller, ) self._cycle = cycle self._leadtime = leadtime @@ -562,7 +562,7 @@ def __init__( key_path: Optional[list[str]] = None, batch: bool = False, schema_file: Optional[Path] = None, - coupler: Optional[str] = None, + controller: Optional[str] = None, ): super().__init__( config=config, @@ -570,7 +570,7 @@ def __init__( key_path=key_path, batch=batch, schema_file=schema_file, - coupler=coupler, + controller=controller, ) @@ -594,7 +594,7 @@ def _add_docstring(class_: type, omit: Optional[list[str]] = None) -> None: :param key_path: Keys leading through the config to the driver's configuration block. :param batch: Run component via the batch system? :param schema_file: Path to schema file to use to validate an external driver. - :param coupler: Name of coupler block in config providing run-time values. + :param controller: Name of block in config controlling run-time values. """ setattr( class_, diff --git a/src/uwtools/tests/drivers/test_driver.py b/src/uwtools/tests/drivers/test_driver.py index 3d9a254cd..b7a8d21eb 100644 --- a/src/uwtools/tests/drivers/test_driver.py +++ b/src/uwtools/tests/drivers/test_driver.py @@ -107,7 +107,7 @@ def config(tmp_path): @fixture -def coupler_schema(tmp_path): +def controller_schema(tmp_path): return write( tmp_path / "concrete.jsonschema", { @@ -144,14 +144,14 @@ def test_Assets(assetsobj): assert Path(assetsobj._driver_config["base_file"]).name == "base.yaml" -def test_Assets_coupler(config, coupler_schema): - config["coupler"] = {"rundir": "/coupler/run/dir"} +def test_Assets_controller(config, controller_schema): + config["controller"] = {"rundir": "/controller/run/dir"} del config["concrete"]["rundir"] with patch.object(ConcreteAssetsTimeInvariant, "_validate", driver.Assets._validate): with raises(UWConfigError): - ConcreteAssetsTimeInvariant(config=config, schema_file=coupler_schema) + ConcreteAssetsTimeInvariant(config=config, schema_file=controller_schema) assert ConcreteAssetsTimeInvariant( - config=config, schema_file=coupler_schema, coupler="coupler" + config=config, schema_file=controller_schema, controller="controller" ) @@ -296,18 +296,18 @@ def test_Driver(driverobj): assert driverobj._batch is True -def test_Driver_coupler(config, coupler_schema): - config["coupler"] = { +def test_Driver_controller(config, controller_schema): + config["controller"] = { "execution": {"executable": "/path/to/coupled.exe"}, - "rundir": "/coupler/run/dir", + "rundir": "/controller/run/dir", } del config["concrete"]["rundir"] del config["concrete"]["execution"] with patch.object(ConcreteDriverTimeInvariant, "_validate", driver.Driver._validate): with raises(UWConfigError): - ConcreteDriverTimeInvariant(config=config, schema_file=coupler_schema) + ConcreteDriverTimeInvariant(config=config, schema_file=controller_schema) assert ConcreteDriverTimeInvariant( - config=config, schema_file=coupler_schema, coupler="coupler" + config=config, schema_file=controller_schema, controller="controller" ) @@ -609,7 +609,7 @@ class C: omit = [ "batch", "config", - "coupler", + "controller", "cycle", "dry_run", "key_path", From e3613265eb163ff258d0d5fcf75adff5304b1237 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 2 Aug 2024 00:42:59 +0000 Subject: [PATCH 17/18] Tweak log message --- src/uwtools/config/validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/config/validator.py b/src/uwtools/config/validator.py index 0467efe8d..0803598dd 100644 --- a/src/uwtools/config/validator.py +++ b/src/uwtools/config/validator.py @@ -57,7 +57,7 @@ def validate_internal( :raises: UWConfigError if config fails validation. """ - log.info("Validating config against internal schema %s", schema_name) + log.info("Validating config against internal schema: %s", schema_name) schema_file = get_schema_file(schema_name) log.debug("Using schema file: %s", schema_file) validate_external(config=config, schema_file=schema_file) From 99a6cb97cce7b203a66d871897b441a6dc5ee6f9 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Fri, 2 Aug 2024 00:49:10 +0000 Subject: [PATCH 18/18] Doc updates --- .../cli/tools/file/copy-exec-timedep.out | 20 +++++----- .../user_guide/cli/tools/file/copy-exec.out | 30 +++++++------- .../cli/tools/file/link-exec-timedep.out | 20 +++++----- .../user_guide/cli/tools/file/link-exec.out | 30 +++++++------- .../cli/tools/rocoto/realize-exec-file.out | 5 ++- .../rocoto/realize-exec-stdin-stdout.out | 5 ++- .../rocoto/realize-exec-stdout-verbose.out | 40 +++++++++---------- .../cli/tools/rocoto/realize-exec-stdout.out | 5 ++- 8 files changed, 79 insertions(+), 76 deletions(-) diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec-timedep.out b/docs/sections/user_guide/cli/tools/file/copy-exec-timedep.out index 56be1a29f..30b4605fa 100644 --- a/docs/sections/user_guide/cli/tools/file/copy-exec-timedep.out +++ b/docs/sections/user_guide/cli/tools/file/copy-exec-timedep.out @@ -1,13 +1,13 @@ -[2024-06-05T15:24:49] INFO Validating config against internal schema files-to-stage -[2024-06-05T15:24:49] INFO 0 UW schema-validation errors found -[2024-06-05T15:24:49] INFO File copies: Initial state: Not Ready -[2024-06-05T15:24:49] INFO File copies: Checking requirements -[2024-06-05T15:24:49] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Initial state: Not Ready -[2024-06-05T15:24:49] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Checking requirements -[2024-06-05T15:24:49] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Requirement(s) ready -[2024-06-05T15:24:49] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Executing -[2024-06-05T15:24:49] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Final state: Ready -[2024-06-05T15:24:49] INFO File copies: Final state: Ready +[2024-08-02T00:43:08] INFO Validating config against internal schema: files-to-stage +[2024-08-02T00:43:08] INFO 0 UW schema-validation errors found +[2024-08-02T00:43:08] INFO File copies: Initial state: Not Ready +[2024-08-02T00:43:08] INFO File copies: Checking requirements +[2024-08-02T00:43:08] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Initial state: Not Ready +[2024-08-02T00:43:08] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Checking requirements +[2024-08-02T00:43:08] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Requirement(s) ready +[2024-08-02T00:43:08] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Executing +[2024-08-02T00:43:08] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Final state: Ready +[2024-08-02T00:43:08] INFO File copies: Final state: Ready copy-dst-timedep └── baz-2024-05-29T18 diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec.out b/docs/sections/user_guide/cli/tools/file/copy-exec.out index 2c9c63ff0..7c5b50333 100644 --- a/docs/sections/user_guide/cli/tools/file/copy-exec.out +++ b/docs/sections/user_guide/cli/tools/file/copy-exec.out @@ -1,18 +1,18 @@ -[2024-06-05T15:24:50] INFO Validating config against internal schema files-to-stage -[2024-06-05T15:24:50] INFO 0 UW schema-validation errors found -[2024-06-05T15:24:50] INFO File copies: Initial state: Not Ready -[2024-06-05T15:24:50] INFO File copies: Checking requirements -[2024-06-05T15:24:50] INFO Copy src/foo -> copy-dst/foo: Initial state: Not Ready -[2024-06-05T15:24:50] INFO Copy src/foo -> copy-dst/foo: Checking requirements -[2024-06-05T15:24:50] INFO Copy src/foo -> copy-dst/foo: Requirement(s) ready -[2024-06-05T15:24:50] INFO Copy src/foo -> copy-dst/foo: Executing -[2024-06-05T15:24:50] INFO Copy src/foo -> copy-dst/foo: Final state: Ready -[2024-06-05T15:24:50] INFO Copy src/bar -> copy-dst/subdir/bar: Initial state: Not Ready -[2024-06-05T15:24:50] INFO Copy src/bar -> copy-dst/subdir/bar: Checking requirements -[2024-06-05T15:24:50] INFO Copy src/bar -> copy-dst/subdir/bar: Requirement(s) ready -[2024-06-05T15:24:50] INFO Copy src/bar -> copy-dst/subdir/bar: Executing -[2024-06-05T15:24:50] INFO Copy src/bar -> copy-dst/subdir/bar: Final state: Ready -[2024-06-05T15:24:50] INFO File copies: Final state: Ready +[2024-08-02T00:43:08] INFO Validating config against internal schema: files-to-stage +[2024-08-02T00:43:08] INFO 0 UW schema-validation errors found +[2024-08-02T00:43:08] INFO File copies: Initial state: Not Ready +[2024-08-02T00:43:08] INFO File copies: Checking requirements +[2024-08-02T00:43:08] INFO Copy src/foo -> copy-dst/foo: Initial state: Not Ready +[2024-08-02T00:43:08] INFO Copy src/foo -> copy-dst/foo: Checking requirements +[2024-08-02T00:43:08] INFO Copy src/foo -> copy-dst/foo: Requirement(s) ready +[2024-08-02T00:43:08] INFO Copy src/foo -> copy-dst/foo: Executing +[2024-08-02T00:43:08] INFO Copy src/foo -> copy-dst/foo: Final state: Ready +[2024-08-02T00:43:08] INFO Copy src/bar -> copy-dst/subdir/bar: Initial state: Not Ready +[2024-08-02T00:43:08] INFO Copy src/bar -> copy-dst/subdir/bar: Checking requirements +[2024-08-02T00:43:08] INFO Copy src/bar -> copy-dst/subdir/bar: Requirement(s) ready +[2024-08-02T00:43:08] INFO Copy src/bar -> copy-dst/subdir/bar: Executing +[2024-08-02T00:43:08] INFO Copy src/bar -> copy-dst/subdir/bar: Final state: Ready +[2024-08-02T00:43:08] INFO File copies: Final state: Ready copy-dst ├── foo diff --git a/docs/sections/user_guide/cli/tools/file/link-exec-timedep.out b/docs/sections/user_guide/cli/tools/file/link-exec-timedep.out index b0f76f803..6256a41df 100644 --- a/docs/sections/user_guide/cli/tools/file/link-exec-timedep.out +++ b/docs/sections/user_guide/cli/tools/file/link-exec-timedep.out @@ -1,13 +1,13 @@ -[2024-06-05T15:24:50] INFO Validating config against internal schema files-to-stage -[2024-06-05T15:24:50] INFO 0 UW schema-validation errors found -[2024-06-05T15:24:50] INFO File links: Initial state: Not Ready -[2024-06-05T15:24:50] INFO File links: Checking requirements -[2024-06-05T15:24:50] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Initial state: Not Ready -[2024-06-05T15:24:50] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Checking requirements -[2024-06-05T15:24:50] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Requirement(s) ready -[2024-06-05T15:24:50] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Executing -[2024-06-05T15:24:50] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Final state: Ready -[2024-06-05T15:24:50] INFO File links: Final state: Ready +[2024-08-02T00:43:09] INFO Validating config against internal schema: files-to-stage +[2024-08-02T00:43:09] INFO 0 UW schema-validation errors found +[2024-08-02T00:43:09] INFO File links: Initial state: Not Ready +[2024-08-02T00:43:09] INFO File links: Checking requirements +[2024-08-02T00:43:09] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Initial state: Not Ready +[2024-08-02T00:43:09] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Checking requirements +[2024-08-02T00:43:09] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Requirement(s) ready +[2024-08-02T00:43:09] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Executing +[2024-08-02T00:43:09] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Final state: Ready +[2024-08-02T00:43:09] INFO File links: Final state: Ready link-dst-timedep └── baz-2024-05-29T18 -> ../src/20240529/12/006/baz diff --git a/docs/sections/user_guide/cli/tools/file/link-exec.out b/docs/sections/user_guide/cli/tools/file/link-exec.out index 8a056aef4..2247f979c 100644 --- a/docs/sections/user_guide/cli/tools/file/link-exec.out +++ b/docs/sections/user_guide/cli/tools/file/link-exec.out @@ -1,18 +1,18 @@ -[2024-06-05T15:24:49] INFO Validating config against internal schema files-to-stage -[2024-06-05T15:24:49] INFO 0 UW schema-validation errors found -[2024-06-05T15:24:49] INFO File links: Initial state: Not Ready -[2024-06-05T15:24:49] INFO File links: Checking requirements -[2024-06-05T15:24:49] INFO Link link-dst/foo -> src/foo: Initial state: Not Ready -[2024-06-05T15:24:49] INFO Link link-dst/foo -> src/foo: Checking requirements -[2024-06-05T15:24:49] INFO Link link-dst/foo -> src/foo: Requirement(s) ready -[2024-06-05T15:24:49] INFO Link link-dst/foo -> src/foo: Executing -[2024-06-05T15:24:49] INFO Link link-dst/foo -> src/foo: Final state: Ready -[2024-06-05T15:24:49] INFO Link link-dst/subdir/bar -> src/bar: Initial state: Not Ready -[2024-06-05T15:24:49] INFO Link link-dst/subdir/bar -> src/bar: Checking requirements -[2024-06-05T15:24:49] INFO Link link-dst/subdir/bar -> src/bar: Requirement(s) ready -[2024-06-05T15:24:49] INFO Link link-dst/subdir/bar -> src/bar: Executing -[2024-06-05T15:24:49] INFO Link link-dst/subdir/bar -> src/bar: Final state: Ready -[2024-06-05T15:24:49] INFO File links: Final state: Ready +[2024-08-02T00:43:08] INFO Validating config against internal schema: files-to-stage +[2024-08-02T00:43:08] INFO 0 UW schema-validation errors found +[2024-08-02T00:43:08] INFO File links: Initial state: Not Ready +[2024-08-02T00:43:08] INFO File links: Checking requirements +[2024-08-02T00:43:08] INFO Link link-dst/foo -> src/foo: Initial state: Not Ready +[2024-08-02T00:43:08] INFO Link link-dst/foo -> src/foo: Checking requirements +[2024-08-02T00:43:08] INFO Link link-dst/foo -> src/foo: Requirement(s) ready +[2024-08-02T00:43:08] INFO Link link-dst/foo -> src/foo: Executing +[2024-08-02T00:43:08] INFO Link link-dst/foo -> src/foo: Final state: Ready +[2024-08-02T00:43:08] INFO Link link-dst/subdir/bar -> src/bar: Initial state: Not Ready +[2024-08-02T00:43:08] INFO Link link-dst/subdir/bar -> src/bar: Checking requirements +[2024-08-02T00:43:08] INFO Link link-dst/subdir/bar -> src/bar: Requirement(s) ready +[2024-08-02T00:43:08] INFO Link link-dst/subdir/bar -> src/bar: Executing +[2024-08-02T00:43:08] INFO Link link-dst/subdir/bar -> src/bar: Final state: Ready +[2024-08-02T00:43:08] INFO File links: Final state: Ready link-dst ├── foo -> ../src/foo diff --git a/docs/sections/user_guide/cli/tools/rocoto/realize-exec-file.out b/docs/sections/user_guide/cli/tools/rocoto/realize-exec-file.out index 8a5125f26..894f75605 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/realize-exec-file.out +++ b/docs/sections/user_guide/cli/tools/rocoto/realize-exec-file.out @@ -1,5 +1,5 @@ -[2024-05-23T19:39:16] INFO 0 UW schema-validation errors found -[2024-05-23T19:39:16] INFO 0 Rocoto validation errors found +[2024-08-02T00:43:07] INFO 0 UW schema-validation errors found +[2024-08-02T00:43:07] INFO 0 Rocoto validation errors found 00:01:00 echo hello $person hello + --reservation my_reservation person siri diff --git a/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdin-stdout.out b/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdin-stdout.out index b1a091c82..865b4367d 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdin-stdout.out +++ b/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdin-stdout.out @@ -1,5 +1,5 @@ -[2024-05-23T19:39:16] INFO 0 UW schema-validation errors found -[2024-05-23T19:39:16] INFO 0 Rocoto validation errors found +[2024-08-02T00:43:08] INFO 0 UW schema-validation errors found +[2024-08-02T00:43:08] INFO 0 Rocoto validation errors found @@ -14,6 +14,7 @@ 00:01:00 echo hello $person hello + --reservation my_reservation person siri diff --git a/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout-verbose.out b/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout-verbose.out index 02a29dca1..2a055c04a 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout-verbose.out +++ b/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout-verbose.out @@ -1,21 +1,21 @@ -[2024-07-09T00:31:39] DEBUG Command: uw rocoto realize --config-file rocoto.yaml --verbose -[2024-07-09T00:31:39] DEBUG Dereferencing, current value: -[2024-07-09T00:31:39] DEBUG workflow: -[2024-07-09T00:31:39] DEBUG attrs: -[2024-07-09T00:31:39] DEBUG realtime: false -[2024-07-09T00:31:39] DEBUG scheduler: slurm -[2024-07-09T00:31:39] DEBUG cycledef: -[2024-07-09T00:31:39] DEBUG - attrs: -[2024-07-09T00:31:39] DEBUG group: howdy -[2024-07-09T00:31:39] DEBUG spec: 202209290000 202209300000 06:00:00 +[2024-08-02T00:43:09] DEBUG Command: uw rocoto realize --config-file rocoto.yaml --verbose +[2024-08-02T00:43:09] DEBUG Dereferencing, current value: +[2024-08-02T00:43:09] DEBUG workflow: +[2024-08-02T00:43:09] DEBUG attrs: +[2024-08-02T00:43:09] DEBUG realtime: false +[2024-08-02T00:43:09] DEBUG scheduler: slurm +[2024-08-02T00:43:09] DEBUG cycledef: +[2024-08-02T00:43:09] DEBUG - attrs: +[2024-08-02T00:43:09] DEBUG group: howdy +[2024-08-02T00:43:09] DEBUG spec: 202209290000 202209300000 06:00:00 ... -[2024-07-09T00:31:39] DEBUG attrs: -[2024-07-09T00:31:39] DEBUG cycledefs: howdy -[2024-07-09T00:31:39] DEBUG account: '&ACCOUNT;' -[2024-07-09T00:31:39] DEBUG command: echo hello $person -[2024-07-09T00:31:39] DEBUG jobname: hello -[2024-07-09T00:31:39] DEBUG nodes: 1:ppn=1 -[2024-07-09T00:31:39] DEBUG walltime: 00:01:00 -[2024-07-09T00:31:39] DEBUG envars: -[2024-07-09T00:31:39] DEBUG person: siri -[2024-07-09T00:31:39] INFO 0 Rocoto validation errors found +[2024-08-02T00:43:09] DEBUG cycledefs: howdy +[2024-08-02T00:43:09] DEBUG account: '&ACCOUNT;' +[2024-08-02T00:43:09] DEBUG command: echo hello $person +[2024-08-02T00:43:09] DEBUG jobname: hello +[2024-08-02T00:43:09] DEBUG native: --reservation my_reservation +[2024-08-02T00:43:09] DEBUG nodes: 1:ppn=1 +[2024-08-02T00:43:09] DEBUG walltime: 00:01:00 +[2024-08-02T00:43:09] DEBUG envars: +[2024-08-02T00:43:09] DEBUG person: siri +[2024-08-02T00:43:09] INFO 0 Rocoto validation errors found diff --git a/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout.out b/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout.out index b1a091c82..865b4367d 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout.out +++ b/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout.out @@ -1,5 +1,5 @@ -[2024-05-23T19:39:16] INFO 0 UW schema-validation errors found -[2024-05-23T19:39:16] INFO 0 Rocoto validation errors found +[2024-08-02T00:43:08] INFO 0 UW schema-validation errors found +[2024-08-02T00:43:08] INFO 0 Rocoto validation errors found @@ -14,6 +14,7 @@ 00:01:00 echo hello $person hello + --reservation my_reservation person siri