diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index 3cac4774f..9f4acbdab 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -49,6 +49,7 @@ def main() -> None: modes = { STR.config: _dispatch_config, STR.forecast: _dispatch_forecast, + STR.rocoto: _dispatch_rocoto, STR.template: _dispatch_template, } sys.exit(0 if modes[args.mode](args) else 1) @@ -305,6 +306,89 @@ def _dispatch_forecast_run(args: Namespace) -> bool: ) +# Mode rocoto + + +def _add_subparser_rocoto(subparsers: Subparsers) -> ModeChecks: + """ + Subparser for mode: rocoto + + :param subparsers: Parent parser's subparsers, to add this subparser to. + """ + parser = _add_subparser(subparsers, STR.rocoto, "Realize and validate Rocoto XML Documents") + _basic_setup(parser) + subparsers = _add_subparsers(parser, STR.submode) + return { + STR.realize: _add_subparser_rocoto_realize(subparsers), + STR.validate: _add_subparser_rocoto_validate(subparsers), + } + + +def _add_subparser_rocoto_realize(subparsers: Subparsers) -> SubmodeChecks: + """ + Subparser for mode: rocoto realize + + :param subparsers: Parent parser's subparsers, to add this subparser to. + """ + parser = _add_subparser(subparsers, STR.realize, "Realize a Rocoto XML workflow document") + required = parser.add_argument_group(TITLE_REQ_ARG) + _add_arg_output_file(required) + optional = _basic_setup(parser) + _add_arg_input_file(optional) + checks = _add_args_quiet_and_verbose(optional) + return checks + + +def _add_subparser_rocoto_validate(subparsers: Subparsers) -> SubmodeChecks: + """ + Subparser for mode: rocoto validate + + :param subparsers: Parent parser's subparsers, to add this subparser to. + """ + parser = _add_subparser(subparsers, STR.validate, "Validate Rocoto XML") + optional = _basic_setup(parser) + _add_arg_input_file(optional) + checks = _add_args_quiet_and_verbose(optional) + return checks + + +def _dispatch_rocoto(args: Namespace) -> bool: + """ + Dispatch logic for rocoto mode. + + :param args: Parsed command-line args. + """ + return { + STR.realize: _dispatch_rocoto_realize, + STR.validate: _dispatch_rocoto_validate, + }[ + args.submode + ](args) + + +def _dispatch_rocoto_realize(args: Namespace) -> bool: + """ + Dispatch logic for rocoto realize submode. Validate input and output. + + :param args: Parsed command-line args. + """ + success = uwtools.rocoto.realize_rocoto_xml( + config_file=args.input_file, rendered_output=args.output_file + ) + return success + + +def _dispatch_rocoto_validate(args: Namespace) -> bool: + """ + Dispatch logic for rocoto validate submode. + + :param args: Parsed command-line args. + """ + + success = uwtools.rocoto.validate_rocoto_xml(input_xml=args.input_file) + return success + + # Mode template @@ -675,6 +759,7 @@ def _parse_args(raw_args: List[str]) -> Tuple[Namespace, Checks]: checks = { STR.config: _add_subparser_config(subparsers), STR.forecast: _add_subparser_forecast(subparsers), + STR.rocoto: _add_subparser_rocoto(subparsers), STR.template: _add_subparser_template(subparsers), } return parser.parse_args(raw_args), checks @@ -717,6 +802,7 @@ class _STR: quiet: str = "quiet" realize: str = "realize" render: str = "render" + rocoto: str = "rocoto" run: str = "run" schemafile: str = "schema_file" submode: str = "submode" diff --git a/src/uwtools/config/j2template.py b/src/uwtools/config/j2template.py index 44cd7b86c..870d532b5 100644 --- a/src/uwtools/config/j2template.py +++ b/src/uwtools/config/j2template.py @@ -8,6 +8,9 @@ from jinja2 import BaseLoader, Environment, FileSystemLoader, Template, meta +from uwtools.types import DefinitePath, OptionalPath +from uwtools.utils.file import readable + class J2Template: """ @@ -17,7 +20,7 @@ class J2Template: def __init__( self, values: dict, - template_path: Optional[str] = None, + template_path: OptionalPath = None, template_str: Optional[str] = None, **kwargs, ) -> None: @@ -40,7 +43,7 @@ def __init__( # Public methods - def dump(self, output_path: str) -> None: + def dump(self, output_path: DefinitePath) -> None: """ Write rendered template to the path provided. @@ -70,13 +73,13 @@ def undeclared_variables(self) -> Set[str]: j2_parsed = self._j2env.parse(self._template_str) else: assert self._template_path is not None - with open(self._template_path, encoding="utf-8") as file_: + with readable(self._template_path) as file_: j2_parsed = self._j2env.parse(file_.read()) return meta.find_undeclared_variables(j2_parsed) # Private methods - def _load_file(self, template_path: str) -> Template: + def _load_file(self, template_path: OptionalPath) -> Template: """ Load the Jinja2 template from the file provided. @@ -85,7 +88,7 @@ def _load_file(self, template_path: str) -> Template: """ self._j2env = Environment(loader=FileSystemLoader(searchpath="/")) _register_filters(self._j2env) - return self._j2env.get_template(template_path) + return self._j2env.get_template(str(template_path)) def _load_string(self, template: str) -> Template: """ diff --git a/src/uwtools/resources/rocoto.jinja2 b/src/uwtools/resources/rocoto.jinja2 index f6e30313c..3de1adadf 100644 --- a/src/uwtools/resources/rocoto.jinja2 +++ b/src/uwtools/resources/rocoto.jinja2 @@ -62,22 +62,22 @@ {%- endfor %} ]> - + - {%- for group, cdefs in cycledefs.items() %} + {%- for group, cdefs in workflow.cycledefs.items() %} {%- for cdef in cdefs %} {{ cdef }} {%- endfor %} {%- endfor %} - {{ log }} + {{ workflow.log }} -{%- for item, settings in tasks.items() %} +{%- for item, settings in workflow.tasks.items() %} {%- if item.split("_", 1)[0] == "task" %} {{ task(name=item.split("_", 1)[-1], settings=settings ) }} {%- elif item.split("_", 1)[0] == "metatask" %} diff --git a/src/uwtools/resources/rocoto.jsonschema b/src/uwtools/resources/rocoto.jsonschema index 5c192c02e..aeb1a41a8 100644 --- a/src/uwtools/resources/rocoto.jsonschema +++ b/src/uwtools/resources/rocoto.jsonschema @@ -171,10 +171,10 @@ "maxProperties": 3, "minProperties": 2, "patternProperties": { - "^metatask(_[a-z0-9_]+)?$": { + "^metatask(_.*)?$": { "$ref": "#/$defs/metatask" }, - "^task(_[a-z0-9_]+)?$": { + "^task(_.*)?$": { "$ref": "#/$defs/task" } }, @@ -279,7 +279,7 @@ "dependency": { "$ref": "#/$defs/dependency" }, - "envar": { + "envars": { "type": "object" }, "exclusive": { @@ -407,10 +407,10 @@ "additionalProperties": false, "minProperties": 1, "patternProperties": { - "^metatask(_[a-z0-9_]+)?$": { + "^metatask(_.*)?$": { "$ref": "#/$defs/metatask" }, - "^task(_[a-z0-9_]+)?$": { + "^task(_.*)?$": { "$ref": "#/$defs/task" } }, diff --git a/src/uwtools/rocoto.py b/src/uwtools/rocoto.py index 4bf329c4d..95ae8e0ca 100644 --- a/src/uwtools/rocoto.py +++ b/src/uwtools/rocoto.py @@ -2,8 +2,18 @@ Support for creating Rocoto XML workflow documents. """ +import logging +import shutil +import tempfile +from importlib import resources + +from lxml import etree + +import uwtools.config.validator from uwtools.config.core import YAMLConfig from uwtools.config.j2template import J2Template +from uwtools.types import DefinitePath, OptionalPath +from uwtools.utils.file import readable # Private functions @@ -12,7 +22,7 @@ def _add_jobname(tree: dict) -> None: """ Add a "jobname" attribute to each "task" element in the given config tree. - :param tree: A config tree containing "task" elements.. + :param tree: A config tree containing "task" elements. """ for element, subtree in tree.items(): element_parts = element.split("_", maxsplit=1) @@ -25,20 +35,119 @@ def _add_jobname(tree: dict) -> None: _add_jobname(subtree) -# Public functions -def write_rocoto_xml(input_yaml: str, input_template: str, rendered_output: str) -> None: +def _add_jobname_to_tasks( + input_yaml: OptionalPath = None, +) -> YAMLConfig: """ - Main entry point. + Load YAML config and add job names to each defined workflow task. :param input_yaml: Path to YAML input file. - :param input_template: Path to input template file. - :param rendered_output: Path to write rendered XML file. """ values = YAMLConfig(input_yaml) - tasks = values["tasks"] + tasks = values["workflow"]["tasks"] if isinstance(tasks, dict): _add_jobname(tasks) + return values + + +def _rocoto_schema_xml() -> DefinitePath: + """ + The path to the file containing the schema to validate the XML file against. + """ + with resources.as_file(resources.files("uwtools.resources")) as path: + return path / "schema_with_metatasks.rng" + + +def _rocoto_schema_yaml() -> DefinitePath: + """ + The path to the file containing the schema to validate the YAML file against. + """ + with resources.as_file(resources.files("uwtools.resources")) as path: + return path / "rocoto.jsonschema" + + +def _rocoto_template_xml() -> DefinitePath: + """ + The path to the file containing the Rocoto workflow document template to render. + """ + with resources.as_file(resources.files("uwtools.resources")) as path: + return path / "rocoto.jinja2" + + +def _write_rocoto_xml( + config_file: OptionalPath, + rendered_output: DefinitePath, +) -> None: + """ + Render the Rocoto workflow defined in the given YAML to XML. + + :param config_file: Path to YAML input file. + :param rendered_output: Path to write rendered XML file. + """ + + values = _add_jobname_to_tasks(config_file) # Render the template. - template = J2Template(values=values.data, template_path=input_template) - template.dump(output_path=rendered_output) + template = J2Template(values=values.data, template_path=_rocoto_template_xml()) + template.dump(output_path=str(rendered_output)) + + +# Public functions +def realize_rocoto_xml( + config_file: OptionalPath, + rendered_output: DefinitePath, +) -> bool: + """ + Realize the Rocoto workflow defined in the given YAML as XML. Validate both the YAML input and + XML output. + + :param config_file: Path to YAML input file. + :param rendered_output: Path to write rendered XML file. + :return: Did the input and output files conform to theirr schemas? + """ + + # Validate the YAML. + if uwtools.config.validator.validate_yaml( + config_file=config_file, schema_file=_rocoto_schema_yaml() + ): + # Render the template to a temporary file. + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + _write_rocoto_xml( + config_file=config_file, + rendered_output=temp_file.name, + ) + # Validate the XML. + if validate_rocoto_xml(input_xml=temp_file.name): + # If no issues were detected, save temp file and report success. + shutil.move(temp_file.name, rendered_output) + return True + logging.error("Rocoto validation errors identified in %s", temp_file.name) + return False + logging.error("YAML validation errors identified in %s", config_file) + return False + + +def validate_rocoto_xml(input_xml: OptionalPath) -> bool: + """ + Given a rendered XML file, validate it against the Rocoto schema. + + :param input_xml: Path to rendered XML file. + :return: Did the XML file conform to the schema? + """ + + # Validate the XML. + with open(_rocoto_schema_xml(), "r", encoding="utf-8") as f: + schema = etree.RelaxNG(etree.parse(f)) + with readable(input_xml) as f: + xml = f.read() + tree = etree.fromstring(bytes(xml, encoding="utf-8")) + success = schema.validate(tree) + + # Log validation errors. + errors = str(etree.RelaxNG.error_log).split("\n") + log_method = logging.error if len(errors) else logging.info + log_method("%s Rocoto validation error%s found", len(errors), "" if len(errors) == 1 else "s") + for line in errors: + logging.error(line) + + return success diff --git a/src/uwtools/tests/config/test_j2template.py b/src/uwtools/tests/config/test_j2template.py index c9ad33f81..a893314d2 100644 --- a/src/uwtools/tests/config/test_j2template.py +++ b/src/uwtools/tests/config/test_j2template.py @@ -32,7 +32,7 @@ def test_bad_args(testdata): def test_dump(testdata, tmp_path): - path = str(tmp_path / "rendered.txt") + path = tmp_path / "rendered.txt" j2template = J2Template(testdata.config, template_str=testdata.template) j2template.dump(output_path=path) with open(path, "r", encoding="utf-8") as f: @@ -43,7 +43,7 @@ def test_render_file(testdata, tmp_path): path = tmp_path / "template.jinja2" with path.open("w", encoding="utf-8") as f: print(testdata.template, file=f) - validate(J2Template(testdata.config, template_path=str(path))) + validate(J2Template(testdata.config, template_path=path)) def test_render_string(testdata): diff --git a/src/uwtools/tests/fixtures/hello_workflow.yaml b/src/uwtools/tests/fixtures/hello_workflow.yaml index 9db8657d7..be42a83fb 100644 --- a/src/uwtools/tests/fixtures/hello_workflow.yaml +++ b/src/uwtools/tests/fixtures/hello_workflow.yaml @@ -1,3 +1,4 @@ +workflow: attrs: realtime: false scheduler: slurm diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index 060586d8f..7dbcf6141 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -192,17 +192,17 @@ def test__dispatch_config(params): submode, funcname = params args = ns() vars(args).update({STR.submode: submode}) - with patch.object(cli, funcname) as m: + with patch.object(cli, funcname) as func: cli._dispatch_config(args) - assert m.called_once_with(args) + assert func.called_once_with(args) def test__dispatch_config_compare(): args = ns() vars(args).update({STR.file1path: 1, STR.file1fmt: 2, STR.file2path: 3, STR.file2fmt: 4}) - with patch.object(cli.uwtools.config.core, "compare_configs") as m: + with patch.object(cli.uwtools.config.core, "compare_configs") as compare_configs: cli._dispatch_config_compare(args) - assert m.called_once_with(args) + assert compare_configs.called_once_with(args) def test__dispatch_config_realize(): @@ -219,9 +219,9 @@ def test__dispatch_config_realize(): STR.dryrun: 8, } ) - with patch.object(cli.uwtools.config.core, "realize_config") as m: + with patch.object(cli.uwtools.config.core, "realize_config") as realize_config: cli._dispatch_config_realize(args) - assert m.called_once_with(args) + assert realize_config.called_once_with(args) def test__dispatch_config_translate_arparse_to_jinja2(): @@ -235,12 +235,12 @@ def test__dispatch_config_translate_arparse_to_jinja2(): STR.dryrun: 5, } ) - with patch.object(cli.uwtools.config.atparse_to_jinja2, "convert") as m: + with patch.object(cli.uwtools.config.atparse_to_jinja2, "convert") as convert: cli._dispatch_config_translate(args) - assert m.called_once_with(args) + assert convert.called_once_with(args) -def test_dispath_config_translate_unsupported(): +def test__dispatch_config_translate_unsupported(): args = ns() vars(args).update( {STR.infile: 1, STR.infmt: "jpg", STR.outfile: 3, STR.outfmt: "png", STR.dryrun: 5} @@ -251,12 +251,12 @@ def test_dispath_config_translate_unsupported(): def test__dispatch_config_validate_yaml(): args = ns() vars(args).update({STR.infile: 1, STR.infmt: FORMAT.yaml, STR.schemafile: 3}) - with patch.object(cli.uwtools.config.validator, "validate_yaml") as m: + with patch.object(cli.uwtools.config.validator, "validate_yaml") as validate_yaml: cli._dispatch_config_validate(args) - assert m.called_once_with(args) + assert validate_yaml.called_once_with(args) -def test_dispath_config_validate_unsupported(): +def test__dispatch_config_validate_unsupported(): args = ns() vars(args).update({STR.infile: 1, STR.infmt: "jpg", STR.schemafile: 3}) assert cli._dispatch_config_validate(args) is False @@ -267,9 +267,9 @@ def test__dispatch_forecast(params): submode, funcname = params args = ns() vars(args).update({STR.submode: submode}) - with patch.object(cli, funcname) as m: + with patch.object(cli, funcname) as module: cli._dispatch_forecast(args) - assert m.called_once_with(args) + assert module.called_once_with(args) def test__dispatch_forecast_run(): @@ -281,12 +281,63 @@ def test__dispatch_forecast_run(): forecast_model="foo", ) vars(args).update({STR.cfgfile: 1, "forecast_model": "foo"}) - with patch.object(cli.uwtools.drivers.forecast, "FooForecast", create=True) as m: + with patch.object(cli.uwtools.drivers.forecast, "FooForecast", create=True) as FooForecast: CLASSES = {"foo": getattr(cli.uwtools.drivers.forecast, "FooForecast")} with patch.object(cli.uwtools.drivers.forecast, "CLASSES", new=CLASSES): cli._dispatch_forecast_run(args) - assert m.called_once_with(args) - m().run.assert_called_once_with(cycle="2023-01-01T00:00:00") + assert FooForecast.called_once_with(args) + FooForecast().run.assert_called_once_with(cycle="2023-01-01T00:00:00") + + +@pytest.mark.parametrize( + "params", + [ + (STR.realize, "_dispatch_rocoto_realize"), + (STR.validate, "_dispatch_rocoto_validate"), + ], +) +def test__dispatch_rocoto(params): + submode, funcname = params + args = ns() + vars(args).update({STR.submode: submode}) + with patch.object(cli, funcname) as module: + cli._dispatch_rocoto(args) + assert module.called_once_with(args) + + +def test__dispatch_rocoto_realize(): + args = ns() + vars(args).update({STR.infile: 1, STR.outfile: 2}) + with patch.object(cli.uwtools.rocoto, "realize_rocoto_xml") as module: + cli._dispatch_rocoto_realize(args) + assert module.called_once_with(args) + + +def test__dispatch_rocoto_realize_invalid(): + args = ns() + vars(args).update( + { + STR.infile: 1, + STR.outfile: 2, + } + ) + with patch.object(cli.uwtools.rocoto, "realize_rocoto_xml", return_value=False): + assert cli._dispatch_rocoto_realize(args) is False + + +def test__dispatch_rocoto_validate_xml(): + args = ns() + vars(args).update({STR.infile: 1}) + with patch.object(cli.uwtools.rocoto, "validate_rocoto_xml") as validate: + cli._dispatch_rocoto_validate(args) + assert validate.called_once_with(args) + + +def test__dispatch_rocoto_validate_xml_invalid(): + args = ns() + vars(args).update({STR.infile: 1, STR.verbose: False}) + with patch.object(cli.uwtools.rocoto, "validate_rocoto_xml", return_value=False): + assert cli._dispatch_rocoto_validate(args) is False @pytest.mark.parametrize("params", [(STR.render, "_dispatch_template_render")]) @@ -294,9 +345,9 @@ def test__dispatch_template(params): submode, funcname = params args = ns() vars(args).update({STR.submode: submode}) - with patch.object(cli, funcname) as m: + with patch.object(cli, funcname) as func: cli._dispatch_template(args) - assert m.called_once_with(args) + assert func.called_once_with(args) def test__dispatch_template_render_yaml(): @@ -312,9 +363,9 @@ def test__dispatch_template_render_yaml(): STR.dryrun: 7, } ) - with patch.object(cli.uwtools.config.templater, STR.render) as m: + with patch.object(cli.uwtools.config.templater, STR.render) as templater: cli._dispatch_template_render(args) - assert m.called_once_with(args) + assert templater.called_once_with(args) @pytest.mark.parametrize("quiet", [True]) diff --git a/src/uwtools/tests/test_rocoto.py b/src/uwtools/tests/test_rocoto.py index f00b74040..a0031ff05 100644 --- a/src/uwtools/tests/test_rocoto.py +++ b/src/uwtools/tests/test_rocoto.py @@ -3,19 +3,21 @@ Tests for uwtools.rocoto module. """ +import tempfile from importlib import resources +from unittest.mock import patch import pytest import yaml -from lxml import etree from uwtools import rocoto +from uwtools.config.core import YAMLConfig from uwtools.tests import support # Test functions -def test_add_jobname(): +def test__add_jobname(): expected = yaml.safe_load( """ task_hello: @@ -44,26 +46,67 @@ def test_add_jobname(): assert expected == tree -def test_write_rocoto_xml(tmp_path): - input_yaml = support.fixture_path("hello_workflow.yaml") - with resources.as_file(resources.files("uwtools.resources")) as resc: - input_template = resc / "rocoto.jinja2" +def test__add_jobname_to_tasks(): + with resources.as_file(resources.files("uwtools.tests.fixtures")) as path: + input_yaml = path / "hello_workflow.yaml" + + values = YAMLConfig(input_yaml) + tasks = values["workflow"]["tasks"] + with patch.object(rocoto, "_add_jobname") as module: + rocoto._add_jobname_to_tasks(input_yaml) + assert module.called_once_with(tasks) + + +def test__rocoto_schema_yaml(): + with resources.as_file(resources.files("uwtools.resources")) as path: + expected = path / "rocoto.jsonschema" + assert rocoto._rocoto_schema_yaml() == expected + + +def test__rocoto_schema_xml(): + with resources.as_file(resources.files("uwtools.resources")) as path: + expected = path / "schema_with_metatasks.rng" + assert rocoto._rocoto_schema_xml() == expected + + +@pytest.mark.parametrize("vals", [("hello_workflow.yaml", True), ("fruit_config.yaml", False)]) +def test_realize_rocoto_xml(vals, tmp_path): + fn, validity = vals output = tmp_path / "rendered.xml" - rocoto.write_rocoto_xml( - input_yaml=input_yaml, input_template=str(input_template), rendered_output=str(output) - ) - expected = support.fixture_path("hello_workflow.xml") - support.compare_files(expected, output) + with patch.object(rocoto, "validate_rocoto_xml", value=True): + with patch.object(rocoto.uwtools.config.validator, "_bad_paths", return_value=None): + with resources.as_file(resources.files("uwtools.tests.fixtures")) as path: + config_file = path / fn + result = rocoto.realize_rocoto_xml(config_file=config_file, rendered_output=output) + assert result is validity + + +def test_realize_rocoto_invalid_xml(): + config_file = support.fixture_path("hello_workflow.yaml") + xml = support.fixture_path("rocoto_invalid.xml") + with patch.object(rocoto, "_write_rocoto_xml", return_value=None): + with patch.object(rocoto.uwtools.config.validator, "_bad_paths", return_value=None): + with patch.object(tempfile, "NamedTemporaryFile") as context_manager: + context_manager.return_value.__enter__.return_value.name = xml + result = rocoto.realize_rocoto_xml(config_file=config_file, rendered_output=xml) + assert result is False @pytest.mark.parametrize("vals", [("hello_workflow.xml", True), ("rocoto_invalid.xml", False)]) def test_rocoto_xml_is_valid(vals): fn, validity = vals - with resources.as_file(resources.files("uwtools.resources")) as resc: - with open(resc / "schema_with_metatasks.rng", "r", encoding="utf-8") as f: - schema = etree.RelaxNG(etree.parse(f)) - xml = support.fixture_path(fn) - tree = etree.parse(xml) - assert schema.validate(tree) is validity + result = rocoto.validate_rocoto_xml(input_xml=xml) + + assert result is validity + + +def test__write_rocoto_xml(tmp_path): + config_file = support.fixture_path("hello_workflow.yaml") + output = tmp_path / "rendered.xml" + + rocoto._write_rocoto_xml(config_file=config_file, rendered_output=output) + + expected = support.fixture_path("hello_workflow.xml") + assert support.compare_files(expected, output) is True diff --git a/src/uwtools/utils/file.py b/src/uwtools/utils/file.py index c00784c7d..87c20973f 100644 --- a/src/uwtools/utils/file.py +++ b/src/uwtools/utils/file.py @@ -28,6 +28,7 @@ class _FORMAT: _ini: str = "ini" _jinja2: str = "jinja2" _nml: str = "nml" + _xml: str = "xml" _yaml: str = "yaml" # Variants: