From 3ac1185f156c113bfcf34a7a5aa03b3b9e30046c Mon Sep 17 00:00:00 2001 From: WeirAE Date: Mon, 10 Jun 2024 09:22:38 -0500 Subject: [PATCH 1/8] Build Schism driver framework --- .../user_guide/yaml/components/index.rst | 1 + .../user_guide/yaml/components/schism.rst | 32 ++++++++ docs/shared/schism.yaml | 9 ++ src/uwtools/drivers/schism.py | 78 ++++++++++++++++++ .../resources/jsonschema/schism.jsonschema | 35 ++++++++ src/uwtools/strings.py | 1 + src/uwtools/tests/drivers/test_schism.py | 82 +++++++++++++++++++ src/uwtools/tests/test_schemas.py | 51 +++++++++++- 8 files changed, 288 insertions(+), 1 deletion(-) create mode 100644 docs/sections/user_guide/yaml/components/schism.rst create mode 100644 docs/shared/schism.yaml create mode 100644 src/uwtools/drivers/schism.py create mode 100644 src/uwtools/resources/jsonschema/schism.jsonschema create mode 100644 src/uwtools/tests/drivers/test_schism.py diff --git a/docs/sections/user_guide/yaml/components/index.rst b/docs/sections/user_guide/yaml/components/index.rst index 7c0b0d2f3..0ae0e7f56 100644 --- a/docs/sections/user_guide/yaml/components/index.rst +++ b/docs/sections/user_guide/yaml/components/index.rst @@ -13,6 +13,7 @@ UW YAML for Components make_solo_mosaic mpas mpas_init + schism sfc_climo_gen shave ungrib diff --git a/docs/sections/user_guide/yaml/components/schism.rst b/docs/sections/user_guide/yaml/components/schism.rst new file mode 100644 index 000000000..e76d8f2f2 --- /dev/null +++ b/docs/sections/user_guide/yaml/components/schism.rst @@ -0,0 +1,32 @@ +.. _schism_yaml: + +schism +====== + +Structured YAML to configure SCHISM as part of a compiled coupled executable is validated by JSON Schema and requires the ``schism:`` block, described below. + +Here is a prototype UW YAML ``schism:`` block, explained in detail below: + +.. highlight:: yaml +.. literalinclude:: ../../../../shared/schism.yaml + +UW YAML for the ``schism:`` Block +--------------------------------- + +namelist: +^^^^^^^^^ + + .. important:: The SCHISM namelist file is provisioned by rendering an input template file containing Jinja2 expressions. Unlike namelist files provisioned by ``uwtools`` for other components, the SCHISM namelist file will not be validated. + + **template_file:** + + The path to the input template file containing Jinja2 expressions (perhaps named ``param.nml.IN``), based on the ``param.nml`` file from the SCHISM build. + + **template_values:** + + Key-value pairs necessary to render all Jinja2 expressions in the input template file named by ``template_file:``. + +run_dir: +^^^^^^^^ + +The path to the run directory. diff --git a/docs/shared/schism.yaml b/docs/shared/schism.yaml new file mode 100644 index 000000000..e5ff15480 --- /dev/null +++ b/docs/shared/schism.yaml @@ -0,0 +1,9 @@ +schism: + namelist: + template_file: /path/to/schism/param.nml.IN + template_values: + dt: 100 + run_dir: /path/to/run/directory +platform: + account: me + scheduler: slurm diff --git a/src/uwtools/drivers/schism.py b/src/uwtools/drivers/schism.py new file mode 100644 index 000000000..995d1c0cd --- /dev/null +++ b/src/uwtools/drivers/schism.py @@ -0,0 +1,78 @@ +""" +An assets driver for SCHISM. +""" + +from datetime import datetime +from pathlib import Path +from typing import List, Optional + +from iotaa import asset, task, tasks + +from uwtools.api.template import render +from uwtools.drivers.driver import Assets +from uwtools.strings import STR +from uwtools.utils.tasks import file + + +class SCHISM(Assets): + """ + A library driver for SCHISM. + """ + + def __init__( + self, + cycle: datetime, + config: Optional[Path] = None, + dry_run: bool = False, + batch: bool = False, + key_path: Optional[List[str]] = None, + ): + """ + The driver. + + :param cycle: The cycle. + :param config: Path to config file (read stdin if missing or None). + :param dry_run: Run in dry-run mode? + :param batch: Run component via the batch system? + :param key_path: Keys leading through the config to the driver's configuration block. + """ + super().__init__( + config=config, dry_run=dry_run, batch=batch, cycle=cycle, key_path=key_path + ) + self._cycle = cycle + + # Workflow tasks + + @task + def namelist_file(self): + """ + Render the namelist from the template file. + """ + fn = "param.nml" + yield self._taskname(fn) + path = self._rundir / fn + yield asset(path, path.is_file) + yield file(path=Path(self._driver_config["namelist"]["template_file"])) + render( + input_file=Path(self._driver_config["namelist"]["template_file"]), + output_file=path, + overrides=self._driver_config["namelist"]["template_values"], + ) + + @tasks + def provisioned_run_directory(self): + """ + Run directory provisioned with all required content. + """ + yield self._taskname("provisioned run directory") + yield [self.namelist_file()] + + + # Private helper methods + + @property + def _driver_name(self) -> str: + """ + Returns the name of this driver. + """ + return STR.schism diff --git a/src/uwtools/resources/jsonschema/schism.jsonschema b/src/uwtools/resources/jsonschema/schism.jsonschema new file mode 100644 index 000000000..97ae27123 --- /dev/null +++ b/src/uwtools/resources/jsonschema/schism.jsonschema @@ -0,0 +1,35 @@ +{ + "properties": { + "schism": { + "additionalProperties": false, + "properties": { + "namelist": { + "additionalProperties": false, + "properties": { + "template_file": { + "type": "string" + }, + "template_values": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + }, + "required": [ + "template_file" + ], + "type": "object" + }, + "run_dir": { + "type": "string" + } + }, + "required": [ + "namelist", + "run_dir" + ], + "type": "object" + } + } +} diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index cef7a9cf7..42af18122 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -103,6 +103,7 @@ class STR: rocoto: str = "rocoto" run: str = "run" schemafile: str = "schema_file" + schism: str = "schism" searchpath: str = "search_path" sfcclimogen: str = "sfc_climo_gen" shave: str = "shave" diff --git a/src/uwtools/tests/drivers/test_schism.py b/src/uwtools/tests/drivers/test_schism.py new file mode 100644 index 000000000..cc46e5f84 --- /dev/null +++ b/src/uwtools/tests/drivers/test_schism.py @@ -0,0 +1,82 @@ +# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name +""" +SCHISM driver tests. +""" +import datetime as dt +from unittest.mock import DEFAULT as D +from unittest.mock import patch + +import yaml +from pytest import fixture + +from uwtools.drivers import schism + +# Fixtures + + +@fixture +def config(tmp_path): + return { + "schism": { + "namelist": { + "template_file": str(tmp_path / "param.nml.IN"), + "template_values": { + "dt": 100, + }, + }, + "run_dir": str(tmp_path), + }, + } + + +@fixture +def config_file(config, tmp_path): + path = tmp_path / "config.yaml" + with open(path, "w", encoding="utf-8") as f: + yaml.dump(config, f) + return path + + +@fixture +def cycle(): + return dt.datetime(2024, 2, 1, 18) + + +@fixture +def driverobj(config_file, cycle): + return schism.SCHISM(config=config_file, cycle=cycle, batch=True) + + +# Tests + + +def test_SCHISM(driverobj): + assert isinstance(driverobj, schism.SCHISM) + + +def test_WaveWatchIII_namelist_file(driverobj): + src = driverobj._driver_config["namelist"]["template_file"] + with open(src, "w", encoding="utf-8") as f: + yaml.dump({}, f) + dst = driverobj._rundir / "param.nml" + assert not dst.is_file() + driverobj.namelist_file() + assert dst.is_file() + + +def test_SCHISM_provisioned_run_directory(driverobj): + with patch.multiple( + driverobj, + namelist_file=D, + ) as mocks: + driverobj.provisioned_run_directory() + for m in mocks: + mocks[m].assert_called_once_with() + + +def test_SCHISM__driver_config(driverobj): + assert driverobj._driver_config == driverobj._config["schism"] + + +def test_SCHISM__validate(driverobj): + driverobj._validate() diff --git a/src/uwtools/tests/test_schemas.py b/src/uwtools/tests/test_schemas.py index da37c46d2..6a7883264 100644 --- a/src/uwtools/tests/test_schemas.py +++ b/src/uwtools/tests/test_schemas.py @@ -115,6 +115,11 @@ def sfc_climo_gen_prop(): return partial(schema_validator, "sfc-climo-gen", "properties", "sfc_climo_gen", "properties") +@fixture +def schism_prop(): + return partial(schema_validator, "schism", "properties", "schism", "properties") + + @fixture def shave_prop(): return partial(schema_validator, "shave", "properties", "shave", "properties") @@ -1131,6 +1136,50 @@ def test_schema_rocoto_workflow_cycledef(): assert "'foo' is not valid" in errors([{"attrs": {"activation_offset": "foo"}, "spec": spec}]) +# schism + + +def test_schema_schism(): + config = { + "namelist": { + "template_file": "/tmp/param.nml", + "template_values": { + "dt": 100, + }, + }, + "run_dir": "/tmp", + } + errors = schema_validator("schism", "properties", "schism") + # Basic correctness: + assert not errors(config) + # All top-level keys are required: + for key in ("namelist", "run_dir"): + assert f"'{key}' is a required property" in errors(with_del(config, key)) + # Additional top-level keys are not allowed: + assert "Additional properties are not allowed" in errors({**config, "foo": "bar"}) + + +def test_schema_schism_namelist(schism_prop): + errors = schism_prop("namelist") + # At least template_file is required: + assert "'template_file' is a required property" in errors({}) + # Just template_file is ok: + assert not errors({"template_file": "/path/to/param.nml"}) + # Both template_file and template_values are ok: + assert not errors( + { + "template_file": "/path/to/param.nml", + "template_values": {"dt": 100}, + } + ) + + +def test_schema_schism_run_dir(schism_prop): + errors = schism_prop("run_dir") + # Must be a string: + assert not errors("/some/path") + assert "88 is not of type 'string'" in errors(88) + # sfc-climo-gen @@ -1399,7 +1448,7 @@ def test_schema_upp_run_dir(upp_prop): assert "88 is not of type 'string'" in errors(88) -# ungrib +# wavewatch III def test_schema_ww3(): From 99a414639605ac0741bb2bb7c54a26ff2b6073ca Mon Sep 17 00:00:00 2001 From: WeirAE Date: Mon, 10 Jun 2024 09:23:49 -0500 Subject: [PATCH 2/8] Formatting fix --- src/uwtools/drivers/schism.py | 1 - src/uwtools/tests/test_schemas.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/drivers/schism.py b/src/uwtools/drivers/schism.py index 995d1c0cd..14d12fb95 100644 --- a/src/uwtools/drivers/schism.py +++ b/src/uwtools/drivers/schism.py @@ -67,7 +67,6 @@ def provisioned_run_directory(self): yield self._taskname("provisioned run directory") yield [self.namelist_file()] - # Private helper methods @property diff --git a/src/uwtools/tests/test_schemas.py b/src/uwtools/tests/test_schemas.py index 6a7883264..f1bb7d51b 100644 --- a/src/uwtools/tests/test_schemas.py +++ b/src/uwtools/tests/test_schemas.py @@ -1180,6 +1180,7 @@ def test_schema_schism_run_dir(schism_prop): assert not errors("/some/path") assert "88 is not of type 'string'" in errors(88) + # sfc-climo-gen From a613ba7bc08e0d558f9009e61a0fc8b1da854665 Mon Sep 17 00:00:00 2001 From: WeirAE Date: Mon, 10 Jun 2024 09:40:52 -0500 Subject: [PATCH 3/8] fix schema errors --- src/uwtools/resources/jsonschema/schism.jsonschema | 8 +++++++- src/uwtools/resources/jsonschema/ww3.jsonschema | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/uwtools/resources/jsonschema/schism.jsonschema b/src/uwtools/resources/jsonschema/schism.jsonschema index 97ae27123..7b108afd9 100644 --- a/src/uwtools/resources/jsonschema/schism.jsonschema +++ b/src/uwtools/resources/jsonschema/schism.jsonschema @@ -11,8 +11,14 @@ }, "template_values": { "additionalProperties": { - "type": "string" + "type": [ + "array", + "boolean", + "number", + "string" + ] }, + "minProperties": 1, "type": "object" } }, diff --git a/src/uwtools/resources/jsonschema/ww3.jsonschema b/src/uwtools/resources/jsonschema/ww3.jsonschema index 21ddcbe82..c5c585660 100644 --- a/src/uwtools/resources/jsonschema/ww3.jsonschema +++ b/src/uwtools/resources/jsonschema/ww3.jsonschema @@ -11,8 +11,14 @@ }, "template_values": { "additionalProperties": { - "type": "string" + "type": [ + "array", + "boolean", + "number", + "string" + ] }, + "minProperties": 1, "type": "object" } }, From 870b3a952bee81562ce8d09420e8fe0fcf54be15 Mon Sep 17 00:00:00 2001 From: Brian Weir <94982354+WeirAE@users.noreply.github.com> Date: Mon, 10 Jun 2024 10:38:55 -0500 Subject: [PATCH 4/8] Update src/uwtools/drivers/schism.py Co-authored-by: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> --- src/uwtools/drivers/schism.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/drivers/schism.py b/src/uwtools/drivers/schism.py index 14d12fb95..8ec817080 100644 --- a/src/uwtools/drivers/schism.py +++ b/src/uwtools/drivers/schism.py @@ -65,7 +65,7 @@ def provisioned_run_directory(self): Run directory provisioned with all required content. """ yield self._taskname("provisioned run directory") - yield [self.namelist_file()] + yield self.namelist_file() # Private helper methods From d830b09ed09f46ca6d2385b04a84c4666c48efa3 Mon Sep 17 00:00:00 2001 From: WeirAE Date: Wed, 12 Jun 2024 17:04:11 -0500 Subject: [PATCH 5/8] had to add logging to failing test --- src/uwtools/tests/config/test_jinja2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/uwtools/tests/config/test_jinja2.py b/src/uwtools/tests/config/test_jinja2.py index e29a9a8de..c7f9c3e1b 100644 --- a/src/uwtools/tests/config/test_jinja2.py +++ b/src/uwtools/tests/config/test_jinja2.py @@ -433,6 +433,7 @@ def test__supplement_values_priority(supplemental_values): def test__values_needed(caplog): + log.setLevel(logging.DEBUG) undeclared_variables = {"roses_color", "lavender_smell"} jinja2._values_needed(undeclared_variables) assert logged(caplog, "Value(s) needed to render this template are:") From 4c1ec6c2b81a98d9a5351ca23326dabf8f363847 Mon Sep 17 00:00:00 2001 From: WeirAE Date: Wed, 12 Jun 2024 17:16:06 -0500 Subject: [PATCH 6/8] fix copy/paste name error --- src/uwtools/tests/drivers/test_schism.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uwtools/tests/drivers/test_schism.py b/src/uwtools/tests/drivers/test_schism.py index cc46e5f84..1bb1c14f8 100644 --- a/src/uwtools/tests/drivers/test_schism.py +++ b/src/uwtools/tests/drivers/test_schism.py @@ -54,7 +54,7 @@ def test_SCHISM(driverobj): assert isinstance(driverobj, schism.SCHISM) -def test_WaveWatchIII_namelist_file(driverobj): +def test_SCHISM_namelist_file(driverobj): src = driverobj._driver_config["namelist"]["template_file"] with open(src, "w", encoding="utf-8") as f: yaml.dump({}, f) From 2942fb61f6ed0fa5651a008874ddcf6d0ac00982 Mon Sep 17 00:00:00 2001 From: Brian Weir <94982354+WeirAE@users.noreply.github.com> Date: Thu, 13 Jun 2024 08:42:51 -0500 Subject: [PATCH 7/8] Apply suggestions from code review Co-authored-by: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> --- docs/sections/user_guide/yaml/components/schism.rst | 2 +- src/uwtools/drivers/schism.py | 7 ++++--- src/uwtools/tests/drivers/test_schism.py | 12 ++---------- src/uwtools/tests/test_schemas.py | 2 +- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/docs/sections/user_guide/yaml/components/schism.rst b/docs/sections/user_guide/yaml/components/schism.rst index e76d8f2f2..c5febd86e 100644 --- a/docs/sections/user_guide/yaml/components/schism.rst +++ b/docs/sections/user_guide/yaml/components/schism.rst @@ -8,7 +8,7 @@ Structured YAML to configure SCHISM as part of a compiled coupled executable is Here is a prototype UW YAML ``schism:`` block, explained in detail below: .. highlight:: yaml -.. literalinclude:: ../../../../shared/schism.yaml +.. literalinclude:: /shared/schism.yaml UW YAML for the ``schism:`` Block --------------------------------- diff --git a/src/uwtools/drivers/schism.py b/src/uwtools/drivers/schism.py index 8ec817080..899eb6f40 100644 --- a/src/uwtools/drivers/schism.py +++ b/src/uwtools/drivers/schism.py @@ -16,7 +16,7 @@ class SCHISM(Assets): """ - A library driver for SCHISM. + An assets driver for SCHISM. """ def __init__( @@ -52,9 +52,10 @@ def namelist_file(self): yield self._taskname(fn) path = self._rundir / fn yield asset(path, path.is_file) - yield file(path=Path(self._driver_config["namelist"]["template_file"])) + template_file = Path(self._driver_config["namelist"]["template_file"]) + yield file(path=template_file) render( - input_file=Path(self._driver_config["namelist"]["template_file"]), + input_file=template_file, output_file=path, overrides=self._driver_config["namelist"]["template_values"], ) diff --git a/src/uwtools/tests/drivers/test_schism.py b/src/uwtools/tests/drivers/test_schism.py index 1bb1c14f8..693f79b9b 100644 --- a/src/uwtools/tests/drivers/test_schism.py +++ b/src/uwtools/tests/drivers/test_schism.py @@ -29,22 +29,14 @@ def config(tmp_path): } -@fixture -def config_file(config, tmp_path): - path = tmp_path / "config.yaml" - with open(path, "w", encoding="utf-8") as f: - yaml.dump(config, f) - return path - - @fixture def cycle(): return dt.datetime(2024, 2, 1, 18) @fixture -def driverobj(config_file, cycle): - return schism.SCHISM(config=config_file, cycle=cycle, batch=True) +def driverobj(config, cycle): + return schism.SCHISM(config=config, cycle=cycle, batch=True) # Tests diff --git a/src/uwtools/tests/test_schemas.py b/src/uwtools/tests/test_schemas.py index f1bb7d51b..c6ddb7477 100644 --- a/src/uwtools/tests/test_schemas.py +++ b/src/uwtools/tests/test_schemas.py @@ -1449,7 +1449,7 @@ def test_schema_upp_run_dir(upp_prop): assert "88 is not of type 'string'" in errors(88) -# wavewatch III +# ww3 def test_schema_ww3(): From 5b20dbc2bc4d06826ec77d98c25e6b21bea26805 Mon Sep 17 00:00:00 2001 From: WeirAE Date: Thu, 13 Jun 2024 09:15:38 -0500 Subject: [PATCH 8/8] open template_values to all JSON types --- src/uwtools/resources/jsonschema/schism.jsonschema | 8 -------- src/uwtools/resources/jsonschema/ww3.jsonschema | 8 -------- 2 files changed, 16 deletions(-) diff --git a/src/uwtools/resources/jsonschema/schism.jsonschema b/src/uwtools/resources/jsonschema/schism.jsonschema index 7b108afd9..0a70561f8 100644 --- a/src/uwtools/resources/jsonschema/schism.jsonschema +++ b/src/uwtools/resources/jsonschema/schism.jsonschema @@ -10,14 +10,6 @@ "type": "string" }, "template_values": { - "additionalProperties": { - "type": [ - "array", - "boolean", - "number", - "string" - ] - }, "minProperties": 1, "type": "object" } diff --git a/src/uwtools/resources/jsonschema/ww3.jsonschema b/src/uwtools/resources/jsonschema/ww3.jsonschema index c5c585660..b8912ccee 100644 --- a/src/uwtools/resources/jsonschema/ww3.jsonschema +++ b/src/uwtools/resources/jsonschema/ww3.jsonschema @@ -10,14 +10,6 @@ "type": "string" }, "template_values": { - "additionalProperties": { - "type": [ - "array", - "boolean", - "number", - "string" - ] - }, "minProperties": 1, "type": "object" }