From 7f0dc6ee7de5e81c8e160fb4ec7eb445b39f2554 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Mon, 17 Jun 2024 16:15:52 -0600 Subject: [PATCH 1/7] First pass at adding IODA driver. --- docs/index.rst | 6 + docs/sections/user_guide/api/index.rst | 1 + docs/sections/user_guide/api/ioda.rst | 5 + .../sections/user_guide/cli/drivers/index.rst | 1 + .../user_guide/yaml/components/index.rst | 1 + .../user_guide/yaml/components/ioda.rst | 41 +++++++ docs/shared/ioda.yaml | 25 ++++ docs/shared/jedi.yaml | 2 +- docs/shared/sfc_climo_gen.yaml | 2 +- docs/shared/upp.yaml | 2 +- src/uwtools/api/ioda.py | 12 ++ src/uwtools/cli.py | 2 + src/uwtools/drivers/ioda.py | 77 ++++++++++++ .../resources/jsonschema/ioda.jsonschema | 53 +++++++++ src/uwtools/strings.py | 1 + src/uwtools/tests/drivers/test_ioda.py | 112 ++++++++++++++++++ src/uwtools/tests/test_schemas.py | 49 ++++++++ 17 files changed, 389 insertions(+), 3 deletions(-) create mode 100644 docs/sections/user_guide/api/ioda.rst create mode 100644 docs/sections/user_guide/yaml/components/ioda.rst create mode 100644 docs/shared/ioda.yaml create mode 100644 src/uwtools/api/ioda.py create mode 100644 src/uwtools/drivers/ioda.py create mode 100644 src/uwtools/resources/jsonschema/ioda.jsonschema create mode 100644 src/uwtools/tests/drivers/test_ioda.py diff --git a/docs/index.rst b/docs/index.rst index 5dfc63e79..3cbb280e9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -178,6 +178,12 @@ UPP Driver for JEDI ^^^^^^^^^^^^^^^ +IODA +"""" + +| **CLI**: ``uw ioda -h`` +| **API**: ``import uwtools.api.ioda`` + JEDI """" diff --git a/docs/sections/user_guide/api/index.rst b/docs/sections/user_guide/api/index.rst index f4666de9c..15bbbd764 100644 --- a/docs/sections/user_guide/api/index.rst +++ b/docs/sections/user_guide/api/index.rst @@ -9,6 +9,7 @@ API filter_topo fv3 global_equiv_resol + ioda jedi logging make_hgrid diff --git a/docs/sections/user_guide/api/ioda.rst b/docs/sections/user_guide/api/ioda.rst new file mode 100644 index 000000000..a5e5ae2dd --- /dev/null +++ b/docs/sections/user_guide/api/ioda.rst @@ -0,0 +1,5 @@ +``uwtools.api.ioda`` +==================== + +.. automodule:: uwtools.api.ioda + :members: diff --git a/docs/sections/user_guide/cli/drivers/index.rst b/docs/sections/user_guide/cli/drivers/index.rst index 5a32da3a0..45e97ffb6 100644 --- a/docs/sections/user_guide/cli/drivers/index.rst +++ b/docs/sections/user_guide/cli/drivers/index.rst @@ -9,6 +9,7 @@ Drivers filter_topo fv3 global_equiv_resol + ioda jedi make_hgrid make_solo_mosaic diff --git a/docs/sections/user_guide/yaml/components/index.rst b/docs/sections/user_guide/yaml/components/index.rst index 7c53512cc..5ea9f9050 100644 --- a/docs/sections/user_guide/yaml/components/index.rst +++ b/docs/sections/user_guide/yaml/components/index.rst @@ -9,6 +9,7 @@ UW YAML for Components filter_topo fv3 global_equiv_resol + ioda jedi make_hgrid make_solo_mosaic diff --git a/docs/sections/user_guide/yaml/components/ioda.rst b/docs/sections/user_guide/yaml/components/ioda.rst new file mode 100644 index 000000000..ea36366c8 --- /dev/null +++ b/docs/sections/user_guide/yaml/components/ioda.rst @@ -0,0 +1,41 @@ +.. _ioda_yaml: + +ioda +==== + +Structured YAML to run IODA is validated by JSON Schema and requires the ``ioda:`` block, described below. If ``ioda`` is to be run via a batch system, the ``platform:`` block, described :ref:`here `, is also required. + +.. include:: /shared/injected_cycle.rst + +Here is a prototype UW YAML ioda: block, explained in detail below: + +.. highlight:: yaml +.. literalinclude:: /shared/ioda.yaml + +UW YAML for the ``ioda:`` Block +------------------------------- + +execution: +^^^^^^^^^^ + +See :ref:`this page ` for details. + +configuration_file: +^^^^^^^^^^^^^^^^^^^ + +Supports ``base_file:`` and ``update_values:`` blocks (see :ref:`updating_values` for details). + +files_to_copy: +^^^^^^^^^^^^^^ + +See :ref:`this page ` for details. + +files_to_link: +^^^^^^^^^^^^^^ + +Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies. + +run_dir: +^^^^^^^^ + +The path to the run directory. diff --git a/docs/shared/ioda.yaml b/docs/shared/ioda.yaml new file mode 100644 index 000000000..ac4f4c351 --- /dev/null +++ b/docs/shared/ioda.yaml @@ -0,0 +1,25 @@ +ioda: + configuration_file: + base_file: path/to/config.yaml + update_values: + baz: qux + execution: + batchargs: + nodes: 1 + stdout: path/to/runscript.out + walltime: "00:05:00" + envcmds: + - module load some-module + - module load ioda-module + executable: /path/to/a/ioda/exe + mpicmd: time + files_to_copy: + d/f2: /path/to/f2 + f1: /path/to/f1 + files_to_link: + f3: /path/to/f3 + f4: d/f4 + run_dir: /path/to/run/dir +platform: + account: me + scheduler: slurm diff --git a/docs/shared/jedi.yaml b/docs/shared/jedi.yaml index 19215ef73..088281934 100644 --- a/docs/shared/jedi.yaml +++ b/docs/shared/jedi.yaml @@ -22,7 +22,7 @@ jedi: files_to_link: f3: /path/to/f3 f4: d/f4 - run_dir: /path/to/run + run_dir: /path/to/run/dir platform: account: me scheduler: slurm diff --git a/docs/shared/sfc_climo_gen.yaml b/docs/shared/sfc_climo_gen.yaml index 4d231ba55..8ba912c26 100644 --- a/docs/shared/sfc_climo_gen.yaml +++ b/docs/shared/sfc_climo_gen.yaml @@ -33,7 +33,7 @@ sfc_climo_gen: snowfree_albedo_method: bilinear vegetation_greenness_method: bilinear validate: true - run_dir: /path/to/run + run_dir: /path/to/run/dir platform: account: me scheduler: slurm diff --git a/docs/shared/upp.yaml b/docs/shared/upp.yaml index 7bb92efa3..a28f60e4f 100644 --- a/docs/shared/upp.yaml +++ b/docs/shared/upp.yaml @@ -35,7 +35,7 @@ upp: - 100 - 1 validate: true - run_dir: /path/to/run + run_dir: /path/to/run/dir platform: account: me scheduler: slurm diff --git a/src/uwtools/api/ioda.py b/src/uwtools/api/ioda.py new file mode 100644 index 000000000..cc3a5174e --- /dev/null +++ b/src/uwtools/api/ioda.py @@ -0,0 +1,12 @@ +""" +API access to the ``uwtools`` ``ioda`` driver. +""" + +from uwtools.drivers.ioda import IODA as _Driver +from uwtools.drivers.support import graph +from uwtools.utils.api import make_execute as _make_execute +from uwtools.utils.api import make_tasks as _make_tasks + +execute = _make_execute(_Driver, with_cycle=True) +tasks = _make_tasks(_Driver) +__all__ = ["execute", "graph", "tasks"] diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index a9f82308e..8298f3ce6 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -69,6 +69,7 @@ def main() -> None: STR.filtertopo, STR.fv3, STR.globalequivresol, + STR.ioda, STR.jedi, STR.makehgrid, STR.makesolomosaic, @@ -1046,6 +1047,7 @@ def _parse_args(raw_args: List[str]) -> Tuple[Args, Checks]: for component in [ STR.chgrescube, STR.fv3, + STR.ioda, STR.jedi, STR.mpas, STR.mpasinit, diff --git a/src/uwtools/drivers/ioda.py b/src/uwtools/drivers/ioda.py new file mode 100644 index 000000000..6b309b1c6 --- /dev/null +++ b/src/uwtools/drivers/ioda.py @@ -0,0 +1,77 @@ +""" +A driver for the ioda component. +""" + +from datetime import datetime +from pathlib import Path +from typing import List, Optional + +from iotaa import tasks + +from uwtools.drivers.jedi import JEDI +from uwtools.strings import STR + + +class IODA(JEDI): + """ + A driver for the IODA component. + """ + + def __init__( + self, + cycle: datetime, + config: Optional[Path] = None, + dry_run: bool = False, + batch: bool = False, + key_path: Optional[List[str]] = None, + ): + """ + The driver. + + :param cycle: The forecast cycle. + :param config: Path to config file. + :param dry_run: Run in dry-run mode? + :param batch: Run component via the batch system? + :param key_path: Keys leading through the config to the driver's configuration block. + """ + super().__init__( + config=config, dry_run=dry_run, batch=batch, cycle=cycle, key_path=key_path + ) + self._cycle = cycle + + @tasks + def provisioned_run_directory(self): + """ + Run directory provisioned with all required content. + """ + yield self._taskname("provisioned run directory") + yield [ + self.configuration_file(), + self.files_copied(), + self.files_linked(), + self.runscript(), + ] + + # Private helper methods + + @property + def _config_fn(self) -> str: + """ + Returns the name of the config file used in execution. + """ + return "ioda.yaml" + + @property + def _driver_name(self) -> str: + """ + Returns the name of this driver. + """ + return STR.ioda + + def _taskname(self, suffix: str) -> str: + """ + Returns a common tag for graph-task log messages. + + :param suffix: Log-string suffix. + """ + return self._taskname_with_cycle(self._cycle, suffix) diff --git a/src/uwtools/resources/jsonschema/ioda.jsonschema b/src/uwtools/resources/jsonschema/ioda.jsonschema new file mode 100644 index 000000000..a49e6c5cf --- /dev/null +++ b/src/uwtools/resources/jsonschema/ioda.jsonschema @@ -0,0 +1,53 @@ +{ + "properties": { + "ioda": { + "additionalProperties": false, + "properties": { + "configuration_file": { + "additionalProperties": false, + "anyOf": [ + { + "required": [ + "base_file" + ] + }, + { + "required": [ + "update_values" + ] + } + ], + "properties": { + "base_file": { + "type": "string" + }, + "update_values": { + "minProperties": 1, + "type": "object" + } + }, + "type": "object" + }, + "execution": { + "$ref": "urn:uwtools:execution" + }, + "files_to_copy": { + "$ref": "urn:uwtools:files-to-stage" + }, + "files_to_link": { + "$ref": "urn:uwtools:files-to-stage" + }, + "run_dir": { + "type": "string" + } + }, + "required": [ + "configuration_file", + "execution", + "run_dir" + ], + "type": "object" + } + }, + "type": "object" +} diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index ac76ce501..ffd14bb67 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -84,6 +84,7 @@ class STR: help: str = "help" infile: str = "input_file" infmt: str = "input_format" + ioda: str = "ioda" jedi: str = "jedi" keypath: str = "key_path" keys: str = "keys" diff --git a/src/uwtools/tests/drivers/test_ioda.py b/src/uwtools/tests/drivers/test_ioda.py new file mode 100644 index 000000000..16406428b --- /dev/null +++ b/src/uwtools/tests/drivers/test_ioda.py @@ -0,0 +1,112 @@ +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name +""" +IODA driver tests. +""" +import datetime as dt +from unittest.mock import DEFAULT as D +from unittest.mock import patch + +from pytest import fixture + +from uwtools.drivers.driver import Driver +from uwtools.drivers.ioda import IODA + +# Fixtures + + +@fixture +def config(tmp_path): + base_file = tmp_path / "base.yaml" + base_file.write_text("foo: bar") + return { + "ioda": { + "execution": { + "batchargs": { + "export": "NONE", + "cores": 1, + "stdout": "/path/to/file", + "walltime": "00:02:00", + }, + "envcmds": [ + "module load some-module", + "module load jedi-module", + ], + "executable": "/path/to/bufr2ioda.x", + "mpiargs": ["--export=ALL", "--ntasks $SLURM_CPUS_ON_NODE"], + "mpicmd": "srun", + }, + "configuration_file": { + "base_file": str(base_file), + "update_values": {"baz": "qux"}, + }, + "files_to_copy": { + "foo": "/path/to/foo", + "bar/baz": "/path/to/baz", + }, + "files_to_link": { + "foo": "/path/to/foo", + "bar/baz": "/path/to/baz", + }, + "run_dir": str(tmp_path), + }, + "platform": { + "account": "me", + "scheduler": "slurm", + }, + } + + +@fixture +def cycle(): + return dt.datetime(2024, 5, 1, 6) + + +@fixture +def driverobj(config, cycle): + return IODA(config=config, cycle=cycle, batch=True) + + +# Tests + + +def test_IODA(): + for method in [ + "_driver_config", + "_resources", + "_run_via_batch_submission", + "_run_via_local_execution", + "_runscript", + "_runscript_done_file", + "_runscript_path", + "_scheduler", + "_validate", + "_write_runscript", + "run", + "runscript", + ]: + assert getattr(IODA, method) is getattr(Driver, method) + + +def test_IODA_provisioned_run_directory(driverobj): + with patch.multiple( + driverobj, + configuration_file=D, + files_copied=D, + files_linked=D, + runscript=D, + ) as mocks: + driverobj.provisioned_run_directory() + for m in mocks: + mocks[m].assert_called_once_with() + + +def test_IODA__config_fn(driverobj): + assert driverobj._config_fn == "ioda.yaml" + + +def test_IODA__driver_name(driverobj): + assert driverobj._driver_name == "ioda" + + +def test_IODA__taskname(driverobj): + assert driverobj._taskname("foo") == "20240501 06Z ioda foo" diff --git a/src/uwtools/tests/test_schemas.py b/src/uwtools/tests/test_schemas.py index 0b187ff55..a99e8d152 100644 --- a/src/uwtools/tests/test_schemas.py +++ b/src/uwtools/tests/test_schemas.py @@ -54,6 +54,11 @@ def global_equiv_resol_prop(): ) +@fixture +def ioda_prop(): + return partial(schema_validator, "ioda", "properties", "ioda", "properties") + + @fixture def jedi_prop(): return partial(schema_validator, "jedi", "properties", "jedi", "properties") @@ -642,6 +647,50 @@ def test_schema_global_equiv_resol_paths(global_equiv_resol_prop, schema_entry): assert "88 is not of type 'string'" in errors(88) +# ioda + + +def test_schema_ioda(): + config = { + "configuration_file": { + "base_file": "/path/to/ioda.yaml", + "update_values": {"foo": "bar", "baz": "qux"}, + }, + "execution": {"executable": "/tmp/ioda.exe"}, + "files_to_copy": {"file1": "src1", "file2": "src2"}, + "files_to_link": {"link1": "src3", "link2": "src4"}, + "run_dir": "/tmp", + } + errors = schema_validator("ioda", "properties", "ioda") + # Basic correctness: + assert not errors(config) + # All top-level keys are required: + for key in ("configuration_file", "execution", "run_dir"): + assert f"'{key}' is a required property" in errors(with_del(config, key)) + # Additional top-level keys are not allowed: + assert "Additional properties are not allowed" in errors({**config, "foo": "bar"}) + + +def test_schema_ioda_configuration_file(ioda_prop): + bf = {"base_file": "/path/to/ioda.yaml"} + uv = {"update_values": {"foo": "bar", "baz": "qux"}} + errors = ioda_prop("configuration_file") + # base_file and update_values are ok together: + assert not errors({**bf, **uv}) + # And either is ok alone: + assert not errors(bf) + assert not errors(uv) + # update_values cannot be empty: + assert "should be non-empty" in errors({"update_values": {}}) + + +def test_schema_ioda_run_dir(ioda_prop): + errors = ioda_prop("run_dir") + # Must be a string: + assert not errors("/some/path") + assert "88 is not of type 'string'" in errors(88) + + # jedi From 8d5f69074c5eff125a52527e2597b134ac58d360 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Mon, 17 Jun 2024 16:19:33 -0600 Subject: [PATCH 2/7] Adding CLI docs. --- docs/sections/user_guide/cli/drivers/ioda.rst | 52 +++++++++++++++++++ .../user_guide/cli/drivers/ioda/Makefile | 1 + .../user_guide/cli/drivers/ioda/help.cmd | 1 + .../user_guide/cli/drivers/ioda/help.out | 28 ++++++++++ .../user_guide/cli/drivers/ioda/run-help.cmd | 1 + .../user_guide/cli/drivers/ioda/run-help.out | 30 +++++++++++ 6 files changed, 113 insertions(+) create mode 100644 docs/sections/user_guide/cli/drivers/ioda.rst create mode 120000 docs/sections/user_guide/cli/drivers/ioda/Makefile create mode 100644 docs/sections/user_guide/cli/drivers/ioda/help.cmd create mode 100644 docs/sections/user_guide/cli/drivers/ioda/help.out create mode 100644 docs/sections/user_guide/cli/drivers/ioda/run-help.cmd create mode 100644 docs/sections/user_guide/cli/drivers/ioda/run-help.out diff --git a/docs/sections/user_guide/cli/drivers/ioda.rst b/docs/sections/user_guide/cli/drivers/ioda.rst new file mode 100644 index 000000000..160d15ab6 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ioda.rst @@ -0,0 +1,52 @@ +``ioda`` +======== + +The ``uw`` mode for configuring and running the IODA components of the JEDI framework. + +.. literalinclude:: ioda/help.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: ioda/help.out + :language: text + +All tasks take the same arguments. For example: + +.. literalinclude:: ioda/run-help.cmd + :language: text + :emphasize-lines: 1 +.. literalinclude:: ioda/run-help.out + :language: text + +Examples +^^^^^^^^ + +The examples use a configuration file named ``config.yaml`` with contents similar to: + +.. highlight:: yaml +.. literalinclude:: /shared/ioda.yaml + +Its contents are described in section :ref:`ioda_yaml`. + +* Run ``ioda`` on an interactive node + + .. code-block:: text + + $ uw ioda run --config-file config.yaml --cycle 2024-05-22T12 + +The driver creates a ``runscript.ioda`` file in the directory specified by ``run_dir:`` in the config and runs it, executing ``ioda``. + +* Run ``ioda`` via a batch job + + .. code-block:: text + + $ uw ioda run --config-file config.yaml --cycle 2024-05-22T12 --batch + +The driver creates a ``runscript.ioda`` file in the directory specified by ``run_dir:`` in the config and submits it to the batch system. Running with ``--batch`` requires a correctly configured ``platform:`` block in ``config.yaml``, as well as appropriate settings in the ``execution:`` block under ``ioda:``. + +* Specifying the ``--dry-run`` flag results in the driver logging messages about actions it would have taken, without actually taking any. + + .. code-block:: text + + $ uw ioda run --config-file config.yaml --cycle 2024-05-22T12 --batch --dry-run + +.. include:: /shared/key_path.rst diff --git a/docs/sections/user_guide/cli/drivers/ioda/Makefile b/docs/sections/user_guide/cli/drivers/ioda/Makefile new file mode 120000 index 000000000..2486334a6 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ioda/Makefile @@ -0,0 +1 @@ +../../Makefile.outputs \ No newline at end of file diff --git a/docs/sections/user_guide/cli/drivers/ioda/help.cmd b/docs/sections/user_guide/cli/drivers/ioda/help.cmd new file mode 100644 index 000000000..6950774cf --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ioda/help.cmd @@ -0,0 +1 @@ +uw ioda --help diff --git a/docs/sections/user_guide/cli/drivers/ioda/help.out b/docs/sections/user_guide/cli/drivers/ioda/help.out new file mode 100644 index 000000000..ba762a87c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ioda/help.out @@ -0,0 +1,28 @@ +usage: uw jedi [-h] [--version] TASK ... + +Execute jedi tasks + +Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + +Positional arguments: + TASK + configuration_file + The JEDI YAML configuration file + files_copied + Files copied for run + files_linked + Files linked for run + provisioned_run_directory + Run directory provisioned with all required content + run + A run + runscript + The runscript + validate + Validate the UW driver config + validate_only + Validate JEDI config YAML diff --git a/docs/sections/user_guide/cli/drivers/ioda/run-help.cmd b/docs/sections/user_guide/cli/drivers/ioda/run-help.cmd new file mode 100644 index 000000000..94ff27e02 --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ioda/run-help.cmd @@ -0,0 +1 @@ +uw ioda run --help diff --git a/docs/sections/user_guide/cli/drivers/ioda/run-help.out b/docs/sections/user_guide/cli/drivers/ioda/run-help.out new file mode 100644 index 000000000..242e6185c --- /dev/null +++ b/docs/sections/user_guide/cli/drivers/ioda/run-help.out @@ -0,0 +1,30 @@ +usage: uw jedi run --cycle CYCLE [-h] [--version] [--config-file PATH] + [--batch] [--dry-run] [--graph-file PATH] + [--key-path KEY[.KEY...]] [--quiet] [--verbose] + +A run + +Required arguments: + --cycle CYCLE + The cycle in ISO8601 format (e.g. 2024-05-23T18) + +Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) + --batch + Submit run to batch scheduler + --dry-run + Only log info, making no changes + --graph-file PATH + Path to Graphviz DOT output [experimental] + --key-path KEY[.KEY...] + Dot-separated path of keys leading through the config to the driver's + configuration block + --quiet, -q + Print no logging messages + --verbose, -v + Print all logging messages From e9f0be18373b2ae27280b2f5127fac71d95e48fc Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Mon, 17 Jun 2024 16:58:34 -0600 Subject: [PATCH 3/7] Update IODA output. --- docs/Makefile | 2 +- docs/sections/user_guide/cli/drivers/ioda/help.out | 4 ++-- docs/sections/user_guide/cli/drivers/ioda/run-help.out | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index fcbe5c56f..5f68e1b41 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,7 +17,7 @@ docs: $(MAKE) html examples: - $(MAKE) -C sections + COLUMNS=80 $(MAKE) -C sections linkcheck: $(SPHINXBUILD) -b linkcheck $(SPHINXOPTS) -c $(CURDIR) $(CURDIR) build/linkcheck diff --git a/docs/sections/user_guide/cli/drivers/ioda/help.out b/docs/sections/user_guide/cli/drivers/ioda/help.out index ba762a87c..a0ff25d7b 100644 --- a/docs/sections/user_guide/cli/drivers/ioda/help.out +++ b/docs/sections/user_guide/cli/drivers/ioda/help.out @@ -1,6 +1,6 @@ -usage: uw jedi [-h] [--version] TASK ... +usage: uw ioda [-h] [--version] TASK ... -Execute jedi tasks +Execute ioda tasks Optional arguments: -h, --help diff --git a/docs/sections/user_guide/cli/drivers/ioda/run-help.out b/docs/sections/user_guide/cli/drivers/ioda/run-help.out index 242e6185c..fcbe6dd72 100644 --- a/docs/sections/user_guide/cli/drivers/ioda/run-help.out +++ b/docs/sections/user_guide/cli/drivers/ioda/run-help.out @@ -1,4 +1,4 @@ -usage: uw jedi run --cycle CYCLE [-h] [--version] [--config-file PATH] +usage: uw ioda run --cycle CYCLE [-h] [--version] [--config-file PATH] [--batch] [--dry-run] [--graph-file PATH] [--key-path KEY[.KEY...]] [--quiet] [--verbose] @@ -6,7 +6,7 @@ A run Required arguments: --cycle CYCLE - The cycle in ISO8601 format (e.g. 2024-05-23T18) + The cycle in ISO8601 format (e.g. 2024-06-17T18) Optional arguments: -h, --help From b17d2e475953fa4e8ab6059879f81410ae33c55f Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Tue, 18 Jun 2024 14:12:19 -0600 Subject: [PATCH 4/7] Apply suggestions from code review Co-authored-by: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> --- docs/sections/user_guide/yaml/components/ioda.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sections/user_guide/yaml/components/ioda.rst b/docs/sections/user_guide/yaml/components/ioda.rst index ea36366c8..2cfa1c32b 100644 --- a/docs/sections/user_guide/yaml/components/ioda.rst +++ b/docs/sections/user_guide/yaml/components/ioda.rst @@ -7,7 +7,7 @@ Structured YAML to run IODA is validated by JSON Schema and requires the ``ioda: .. include:: /shared/injected_cycle.rst -Here is a prototype UW YAML ioda: block, explained in detail below: +Here is a prototype UW YAML ``ioda:`` block, explained in detail below: .. highlight:: yaml .. literalinclude:: /shared/ioda.yaml From 2e88ce544d4138bddfa430a1a325bed1a09accc3 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 18 Jun 2024 14:13:40 -0600 Subject: [PATCH 5/7] Adding changes from review comments. --- src/uwtools/drivers/ioda.py | 37 ++---- src/uwtools/drivers/jedi.py | 75 +----------- src/uwtools/drivers/jedi_base.py | 108 ++++++++++++++++++ src/uwtools/drivers/mpas_base.py | 7 -- .../resources/jsonschema/ioda.jsonschema | 2 +- src/uwtools/tests/drivers/test_ioda.py | 2 - 6 files changed, 120 insertions(+), 111 deletions(-) create mode 100644 src/uwtools/drivers/jedi_base.py diff --git a/src/uwtools/drivers/ioda.py b/src/uwtools/drivers/ioda.py index 6b309b1c6..547bdf9ea 100644 --- a/src/uwtools/drivers/ioda.py +++ b/src/uwtools/drivers/ioda.py @@ -8,37 +8,15 @@ from iotaa import tasks -from uwtools.drivers.jedi import JEDI +from uwtools.drivers.jedi_base import JEDIBase from uwtools.strings import STR -class IODA(JEDI): +class IODA(JEDIBase): """ A driver for the IODA component. """ - def __init__( - self, - cycle: datetime, - config: Optional[Path] = None, - dry_run: bool = False, - batch: bool = False, - key_path: Optional[List[str]] = None, - ): - """ - The driver. - - :param cycle: The forecast cycle. - :param config: Path to config file. - :param dry_run: Run in dry-run mode? - :param batch: Run component via the batch system? - :param key_path: Keys leading through the config to the driver's configuration block. - """ - super().__init__( - config=config, dry_run=dry_run, batch=batch, cycle=cycle, key_path=key_path - ) - self._cycle = cycle - @tasks def provisioned_run_directory(self): """ @@ -68,10 +46,11 @@ def _driver_name(self) -> str: """ return STR.ioda - def _taskname(self, suffix: str) -> str: + @property + def _runcmd(self) -> str: """ - Returns a common tag for graph-task log messages. - - :param suffix: Log-string suffix. + Returns the full command-line component invocation. """ - return self._taskname_with_cycle(self._cycle, suffix) + executable = self._driver_config["execution"]["executable"] + jedi_config = self._rundir / self._config_fn + return " ".join(executable, jedi_config) diff --git a/src/uwtools/drivers/jedi.py b/src/uwtools/drivers/jedi.py index b44a2ed10..09d51b83a 100644 --- a/src/uwtools/drivers/jedi.py +++ b/src/uwtools/drivers/jedi.py @@ -10,79 +10,18 @@ from iotaa import asset, refs, run, task, tasks from uwtools.config.formats.yaml import YAMLConfig -from uwtools.drivers.driver import Driver +from uwtools.drivers.jedi_base import JEDIBase from uwtools.strings import STR from uwtools.utils.tasks import file, filecopy, symlink -class JEDI(Driver): +class JEDI(JEDIBase): """ A driver for the JEDI component. """ - def __init__( - self, - cycle: datetime, - config: Optional[Path] = None, - dry_run: bool = False, - batch: bool = False, - key_path: Optional[List[str]] = None, - ): - """ - The driver. - - :param cycle: The forecast cycle. - :param config: Path to config file. - :param dry_run: Run in dry-run mode? - :param batch: Run component via the batch system? - :param key_path: Keys leading through the config to the driver's configuration block. - """ - super().__init__( - config=config, dry_run=dry_run, batch=batch, cycle=cycle, key_path=key_path - ) - self._cycle = cycle - # Workflow tasks - @task - def configuration_file(self): - """ - The JEDI YAML configuration file. - """ - fn = self._config_fn - yield self._taskname(fn) - path = self._rundir / fn - yield asset(path, path.is_file) - base_file = self._driver_config["configuration_file"].get("base_file") - yield file(Path(base_file)) if base_file else None - self._create_user_updated_config( - config_class=YAMLConfig, - config_values=self._driver_config["configuration_file"], - path=path, - ) - - @tasks - def files_copied(self): - """ - Files copied for run. - """ - yield self._taskname("files copied") - yield [ - filecopy(src=Path(src), dst=self._rundir / dst) - for dst, src in self._driver_config.get("files_to_copy", {}).items() - ] - - @tasks - def files_linked(self): - """ - Files linked for run. - """ - yield self._taskname("files linked") - yield [ - symlink(target=Path(target), linkname=self._rundir / linkname) - for linkname, target in self._driver_config.get("files_to_link", {}).items() - ] - @tasks def provisioned_run_directory(self): """ @@ -138,7 +77,7 @@ def _runcmd(self) -> str: """ Returns the full command-line component invocation. """ - execution = self._driver_config.get("execution", {}) + execution = self._driver_config["execution"] jedi_config = self._rundir / self._config_fn mpiargs = execution.get("mpiargs", []) components = [ @@ -148,11 +87,3 @@ def _runcmd(self) -> str: str(jedi_config), # JEDI config file ] return " ".join(filter(None, components)) - - def _taskname(self, suffix: str) -> str: - """ - Returns a common tag for graph-task log messages. - - :param suffix: Log-string suffix. - """ - return self._taskname_with_cycle(self._cycle, suffix) diff --git a/src/uwtools/drivers/jedi_base.py b/src/uwtools/drivers/jedi_base.py new file mode 100644 index 000000000..e1ed747ff --- /dev/null +++ b/src/uwtools/drivers/jedi_base.py @@ -0,0 +1,108 @@ +""" +A base class for jedi-based drivers. +""" + +import logging +from datetime import datetime +from pathlib import Path +from typing import List, Optional + +from iotaa import asset, refs, run, task, tasks + +from uwtools.config.formats.yaml import YAMLConfig +from uwtools.drivers.driver import Driver +from uwtools.strings import STR +from uwtools.utils.tasks import file, filecopy, symlink + + +class JEDIBase(Driver): + """ + A base class for the JEDI-like drivers. + """ + + def __init__( + self, + cycle: datetime, + config: Optional[Path] = None, + dry_run: bool = False, + batch: bool = False, + key_path: Optional[List[str]] = None, + ): + """ + The driver. + + :param cycle: The forecast cycle. + :param config: Path to config file. + :param dry_run: Run in dry-run mode? + :param batch: Run component via the batch system? + :param key_path: Keys leading through the config to the driver's configuration block. + """ + super().__init__( + config=config, dry_run=dry_run, batch=batch, cycle=cycle, key_path=key_path + ) + self._cycle = cycle + + # Workflow tasks + + @task + def configuration_file(self): + """ + The executable's YAML configuration file. + """ + fn = self._config_fn + yield self._taskname(fn) + path = self._rundir / fn + yield asset(path, path.is_file) + base_file = self._driver_config["configuration_file"].get("base_file") + yield file(Path(base_file)) if base_file else None + self._create_user_updated_config( + config_class=YAMLConfig, + config_values=self._driver_config["configuration_file"], + path=path, + ) + + @tasks + def files_copied(self): + """ + Files copied for run. + """ + yield self._taskname("files copied") + yield [ + filecopy(src=Path(src), dst=self._rundir / dst) + for dst, src in self._driver_config.get("files_to_copy", {}).items() + ] + + @tasks + def files_linked(self): + """ + Files linked for run. + """ + yield self._taskname("files linked") + yield [ + symlink(target=Path(target), linkname=self._rundir / linkname) + for linkname, target in self._driver_config.get("files_to_link", {}).items() + ] + + @tasks + @abstractmethod + def provisioned_run_directory(self): + """ + Run directory provisioned with all required content. + """ + + # Private helper methods + + @property + @abstractmethod + def _config_fn(self) -> str: + """ + Returns the name of the config file used in execution. + """ + + def _taskname(self, suffix: str) -> str: + """ + Returns a common tag for graph-task log messages. + + :param suffix: Log-string suffix. + """ + return self._taskname_with_cycle(self._cycle, suffix) diff --git a/src/uwtools/drivers/mpas_base.py b/src/uwtools/drivers/mpas_base.py index 253635218..a72be4dd7 100644 --- a/src/uwtools/drivers/mpas_base.py +++ b/src/uwtools/drivers/mpas_base.py @@ -134,13 +134,6 @@ def streams_file(self): # Private helper methods - @property - @abstractmethod - def _driver_name(self) -> str: - """ - Returns the name of this driver. - """ - @property @abstractmethod def _streams_fn(self) -> str: diff --git a/src/uwtools/resources/jsonschema/ioda.jsonschema b/src/uwtools/resources/jsonschema/ioda.jsonschema index a49e6c5cf..be74a2635 100644 --- a/src/uwtools/resources/jsonschema/ioda.jsonschema +++ b/src/uwtools/resources/jsonschema/ioda.jsonschema @@ -29,7 +29,7 @@ "type": "object" }, "execution": { - "$ref": "urn:uwtools:execution" + "$ref": "urn:uwtools:execution-serial" }, "files_to_copy": { "$ref": "urn:uwtools:files-to-stage" diff --git a/src/uwtools/tests/drivers/test_ioda.py b/src/uwtools/tests/drivers/test_ioda.py index 16406428b..682302d5b 100644 --- a/src/uwtools/tests/drivers/test_ioda.py +++ b/src/uwtools/tests/drivers/test_ioda.py @@ -32,8 +32,6 @@ def config(tmp_path): "module load jedi-module", ], "executable": "/path/to/bufr2ioda.x", - "mpiargs": ["--export=ALL", "--ntasks $SLURM_CPUS_ON_NODE"], - "mpicmd": "srun", }, "configuration_file": { "base_file": str(base_file), From d83fda9746988590592585f8a9e8dfe46df2f3a9 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 18 Jun 2024 14:33:16 -0600 Subject: [PATCH 6/7] Passing all tests. --- src/uwtools/drivers/ioda.py | 8 ++------ src/uwtools/drivers/jedi.py | 5 +---- src/uwtools/drivers/jedi_base.py | 5 ++--- src/uwtools/tests/drivers/test_ioda.py | 9 +++++++-- src/uwtools/tests/drivers/test_jedi.py | 10 +++++----- 5 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/uwtools/drivers/ioda.py b/src/uwtools/drivers/ioda.py index 547bdf9ea..24dff3c41 100644 --- a/src/uwtools/drivers/ioda.py +++ b/src/uwtools/drivers/ioda.py @@ -2,10 +2,6 @@ A driver for the ioda component. """ -from datetime import datetime -from pathlib import Path -from typing import List, Optional - from iotaa import tasks from uwtools.drivers.jedi_base import JEDIBase @@ -52,5 +48,5 @@ def _runcmd(self) -> str: Returns the full command-line component invocation. """ executable = self._driver_config["execution"]["executable"] - jedi_config = self._rundir / self._config_fn - return " ".join(executable, jedi_config) + jedi_config = str(self._rundir / self._config_fn) + return " ".join([executable, jedi_config]) diff --git a/src/uwtools/drivers/jedi.py b/src/uwtools/drivers/jedi.py index 09d51b83a..7e83c5941 100644 --- a/src/uwtools/drivers/jedi.py +++ b/src/uwtools/drivers/jedi.py @@ -3,16 +3,13 @@ """ import logging -from datetime import datetime from pathlib import Path -from typing import List, Optional from iotaa import asset, refs, run, task, tasks -from uwtools.config.formats.yaml import YAMLConfig from uwtools.drivers.jedi_base import JEDIBase from uwtools.strings import STR -from uwtools.utils.tasks import file, filecopy, symlink +from uwtools.utils.tasks import file class JEDI(JEDIBase): diff --git a/src/uwtools/drivers/jedi_base.py b/src/uwtools/drivers/jedi_base.py index e1ed747ff..9418ec482 100644 --- a/src/uwtools/drivers/jedi_base.py +++ b/src/uwtools/drivers/jedi_base.py @@ -2,16 +2,15 @@ A base class for jedi-based drivers. """ -import logging +from abc import abstractmethod from datetime import datetime from pathlib import Path from typing import List, Optional -from iotaa import asset, refs, run, task, tasks +from iotaa import asset, task, tasks from uwtools.config.formats.yaml import YAMLConfig from uwtools.drivers.driver import Driver -from uwtools.strings import STR from uwtools.utils.tasks import file, filecopy, symlink diff --git a/src/uwtools/tests/drivers/test_ioda.py b/src/uwtools/tests/drivers/test_ioda.py index 682302d5b..1394322c5 100644 --- a/src/uwtools/tests/drivers/test_ioda.py +++ b/src/uwtools/tests/drivers/test_ioda.py @@ -8,8 +8,8 @@ from pytest import fixture -from uwtools.drivers.driver import Driver from uwtools.drivers.ioda import IODA +from uwtools.drivers.jedi_base import JEDIBase # Fixtures @@ -82,7 +82,7 @@ def test_IODA(): "run", "runscript", ]: - assert getattr(IODA, method) is getattr(Driver, method) + assert getattr(IODA, method) is getattr(JEDIBase, method) def test_IODA_provisioned_run_directory(driverobj): @@ -106,5 +106,10 @@ def test_IODA__driver_name(driverobj): assert driverobj._driver_name == "ioda" +def test_IODA__runcmd(driverobj): + config = str(driverobj._rundir / driverobj._config_fn) + assert driverobj._runcmd == f"/path/to/bufr2ioda.x {config}" + + def test_IODA__taskname(driverobj): assert driverobj._taskname("foo") == "20240501 06Z ioda foo" diff --git a/src/uwtools/tests/drivers/test_jedi.py b/src/uwtools/tests/drivers/test_jedi.py index 3a0853225..26a79929f 100644 --- a/src/uwtools/tests/drivers/test_jedi.py +++ b/src/uwtools/tests/drivers/test_jedi.py @@ -13,9 +13,9 @@ from pytest import fixture from uwtools.config.formats.yaml import YAMLConfig -from uwtools.drivers import jedi -from uwtools.drivers.driver import Driver +from uwtools.drivers import jedi, jedi_base from uwtools.drivers.jedi import JEDI +from uwtools.drivers.jedi_base import JEDIBase from uwtools.logging import log from uwtools.tests.support import regex_logged @@ -92,7 +92,7 @@ def test_JEDI(): "run", "runscript", ]: - assert getattr(JEDI, method) is getattr(Driver, method) + assert getattr(JEDI, method) is getattr(JEDIBase, method) def test_JEDI_configuration_file(driverobj): @@ -120,7 +120,7 @@ def test_JEDI_configuration_file_missing_base_file(caplog, driverobj): def test_JEDI_files_copied(driverobj): - with patch.object(jedi, "filecopy") as filecopy: + with patch.object(jedi_base, "filecopy") as filecopy: driverobj._driver_config["run_dir"] = "/path/to/run" driverobj.files_copied() assert filecopy.call_count == 2 @@ -134,7 +134,7 @@ def test_JEDI_files_copied(driverobj): def test_JEDI_files_linked(driverobj): - with patch.object(jedi, "symlink") as symlink: + with patch.object(jedi_base, "symlink") as symlink: driverobj._driver_config["run_dir"] = "/path/to/run" driverobj.files_linked() assert symlink.call_count == 2 From 6b33b46af80c36c5d3e98e5177021f16179533f5 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 18 Jun 2024 14:58:53 -0600 Subject: [PATCH 7/7] Changes to output examples. --- docs/sections/user_guide/cli/drivers/ioda/help.out | 4 +--- docs/sections/user_guide/cli/drivers/jedi/help.out | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/sections/user_guide/cli/drivers/ioda/help.out b/docs/sections/user_guide/cli/drivers/ioda/help.out index a0ff25d7b..6ab9994ba 100644 --- a/docs/sections/user_guide/cli/drivers/ioda/help.out +++ b/docs/sections/user_guide/cli/drivers/ioda/help.out @@ -11,7 +11,7 @@ Optional arguments: Positional arguments: TASK configuration_file - The JEDI YAML configuration file + The executable's YAML configuration file files_copied Files copied for run files_linked @@ -24,5 +24,3 @@ Positional arguments: The runscript validate Validate the UW driver config - validate_only - Validate JEDI config YAML diff --git a/docs/sections/user_guide/cli/drivers/jedi/help.out b/docs/sections/user_guide/cli/drivers/jedi/help.out index ba762a87c..847c99fb1 100644 --- a/docs/sections/user_guide/cli/drivers/jedi/help.out +++ b/docs/sections/user_guide/cli/drivers/jedi/help.out @@ -11,7 +11,7 @@ Optional arguments: Positional arguments: TASK configuration_file - The JEDI YAML configuration file + The executable's YAML configuration file files_copied Files copied for run files_linked