UW 322 as a user, I would like to validate my rocoto xml workflow, gi…

…ven the rocoto native schema (ufs-community#317)

* Start of CLI, need inline validation and tests

* updates to fix issues
Tests in progress
Will recheck file.py edits

* updates to testing

* add inline validation of input and output

* fixed tests and logic
reverted file.py changes

* Incorporating multiple suggestions
Pending rocoto.py edits and log handling

* added realize_rocoto_xml(), tests in progress

* Added verbose logging, tests in progress

* fixes done, need rocoto tests and paths

* Major suggestions added, fixing minor issues

* all 'as m' clarified to 'as module'

* fixed path handling

* Incorporating feedback and correcting validation
still need to fix coverage of invalid XML

* removed pragma, added temp output handling
note: coverage still an issue

* change naming from input_yaml to config_file

* Resolving feedback, still fixing coverage

* Fixing residual naming issues

* invalid_xml test coverage fixed

* fixed  importers to handle OptionalPath

* Docstrings fixed, investigating rocoto.jinja2

* Clarified schema vs template; current write error

* Fixed template and task handling return
Current test issue with passing over temp xml

* fixed tests

* Removed unnecessary declarations

* Several fixes; still an issue in rocoto.jinja2

* unpatched write

src/uwtools/cli.py

@@ -49,6 +49,7 @@ def main() -> None:
         modes = {
             STR.config: _dispatch_config,
             STR.forecast: _dispatch_forecast,
+            STR.rocoto: _dispatch_rocoto,
             STR.template: _dispatch_template,
         }
         sys.exit(0 if modes[args.mode](args) else 1)
@@ -305,6 +306,89 @@ def _dispatch_forecast_run(args: Namespace) -> bool:
     )
 
 
+# Mode rocoto
+
+
+def _add_subparser_rocoto(subparsers: Subparsers) -> ModeChecks:
+    """
+    Subparser for mode: rocoto
+
+    :param subparsers: Parent parser's subparsers, to add this subparser to.
+    """
+    parser = _add_subparser(subparsers, STR.rocoto, "Realize and validate Rocoto XML Documents")
+    _basic_setup(parser)
+    subparsers = _add_subparsers(parser, STR.submode)
+    return {
+        STR.realize: _add_subparser_rocoto_realize(subparsers),
+        STR.validate: _add_subparser_rocoto_validate(subparsers),
+    }
+
+
+def _add_subparser_rocoto_realize(subparsers: Subparsers) -> SubmodeChecks:
+    """
+    Subparser for mode: rocoto realize
+
+    :param subparsers: Parent parser's subparsers, to add this subparser to.
+    """
+    parser = _add_subparser(subparsers, STR.realize, "Realize a Rocoto XML workflow document")
+    required = parser.add_argument_group(TITLE_REQ_ARG)
+    _add_arg_output_file(required)
+    optional = _basic_setup(parser)
+    _add_arg_input_file(optional)
+    checks = _add_args_quiet_and_verbose(optional)
+    return checks
+
+
+def _add_subparser_rocoto_validate(subparsers: Subparsers) -> SubmodeChecks:
+    """
+    Subparser for mode: rocoto validate
+
+    :param subparsers: Parent parser's subparsers, to add this subparser to.
+    """
+    parser = _add_subparser(subparsers, STR.validate, "Validate Rocoto XML")
+    optional = _basic_setup(parser)
+    _add_arg_input_file(optional)
+    checks = _add_args_quiet_and_verbose(optional)
+    return checks
+
+
+def _dispatch_rocoto(args: Namespace) -> bool:
+    """
+    Dispatch logic for rocoto mode.
+
+    :param args: Parsed command-line args.
+    """
+    return {
+        STR.realize: _dispatch_rocoto_realize,
+        STR.validate: _dispatch_rocoto_validate,
+    }[
+        args.submode
+    ](args)
+
+
+def _dispatch_rocoto_realize(args: Namespace) -> bool:
+    """
+    Dispatch logic for rocoto realize submode. Validate input and output.
+
+    :param args: Parsed command-line args.
+    """
+    success = uwtools.rocoto.realize_rocoto_xml(
+        config_file=args.input_file, rendered_output=args.output_file
+    )
+    return success
+
+
+def _dispatch_rocoto_validate(args: Namespace) -> bool:
+    """
+    Dispatch logic for rocoto validate submode.
+
+    :param args: Parsed command-line args.
+    """
+
+    success = uwtools.rocoto.validate_rocoto_xml(input_xml=args.input_file)
+    return success
+
+
 # Mode template
 
 
@@ -675,6 +759,7 @@ def _parse_args(raw_args: List[str]) -> Tuple[Namespace, Checks]:
     checks = {
         STR.config: _add_subparser_config(subparsers),
         STR.forecast: _add_subparser_forecast(subparsers),
+        STR.rocoto: _add_subparser_rocoto(subparsers),
         STR.template: _add_subparser_template(subparsers),
     }
     return parser.parse_args(raw_args), checks
@@ -717,6 +802,7 @@ class _STR:
     quiet: str = "quiet"
     realize: str = "realize"
     render: str = "render"
+    rocoto: str = "rocoto"
     run: str = "run"
     schemafile: str = "schema_file"
     submode: str = "submode"

src/uwtools/config/j2template.py

@@ -8,6 +8,9 @@
 
 from jinja2 import BaseLoader, Environment, FileSystemLoader, Template, meta
 
+from uwtools.types import DefinitePath, OptionalPath
+from uwtools.utils.file import readable
+
 
 class J2Template:
     """
@@ -17,7 +20,7 @@ class J2Template:
     def __init__(
         self,
         values: dict,
-        template_path: Optional[str] = None,
+        template_path: OptionalPath = None,
         template_str: Optional[str] = None,
         **kwargs,
     ) -> None:
@@ -40,7 +43,7 @@ def __init__(
 
     # Public methods
 
-    def dump(self, output_path: str) -> None:
+    def dump(self, output_path: DefinitePath) -> None:
         """
         Write rendered template to the path provided.
 
@@ -70,13 +73,13 @@ def undeclared_variables(self) -> Set[str]:
             j2_parsed = self._j2env.parse(self._template_str)
         else:
             assert self._template_path is not None
-            with open(self._template_path, encoding="utf-8") as file_:
+            with readable(self._template_path) as file_:
                 j2_parsed = self._j2env.parse(file_.read())
         return meta.find_undeclared_variables(j2_parsed)
 
     # Private methods
 
-    def _load_file(self, template_path: str) -> Template:
+    def _load_file(self, template_path: OptionalPath) -> Template:
         """
         Load the Jinja2 template from the file provided.
 
@@ -85,7 +88,7 @@ def _load_file(self, template_path: str) -> Template:
         """
         self._j2env = Environment(loader=FileSystemLoader(searchpath="/"))
         _register_filters(self._j2env)
-        return self._j2env.get_template(template_path)
+        return self._j2env.get_template(str(template_path))
 
     def _load_string(self, template: str) -> Template:
         """

src/uwtools/resources/rocoto.jinja2

@@ -62,22 +62,22 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE workflow [
 
-{%- for entity, value in entities.items() %}
+{%- for entity, value in workflow.entities.items() %}
   <!ENTITY {{ entity }} "{{ value }}">
 {%- endfor %}
 
 ]>
-<workflow {% for attr, val in attrs.items() %}{{ attr }}="{{ val }}" {% endfor %}>
+<workflow {% for attr, val in workflow.attrs.items() %}{{ attr }}="{{ val }}" {% endfor %}>
 
-  {%- for group, cdefs in cycledefs.items() %}
+  {%- for group, cdefs in workflow.cycledefs.items() %}
     {%- for cdef in cdefs %}
   <cycledef group="{{ group }}">{{ cdef }}</cycledef>
     {%- endfor %}
   {%- endfor %}
 
-  <log>{{ log }}</log>
+  <log>{{ workflow.log }}</log>
 
-{%- for item, settings in tasks.items() %}
+{%- for item, settings in workflow.tasks.items() %}
   {%- if item.split("_", 1)[0] == "task" %}
   {{ task(name=item.split("_", 1)[-1], settings=settings ) }}
   {%- elif item.split("_", 1)[0] == "metatask" %}

src/uwtools/resources/rocoto.jsonschema

@@ -171,10 +171,10 @@
       "maxProperties": 3,
       "minProperties": 2,
       "patternProperties": {
-        "^metatask(_[a-z0-9_]+)?$": {
+        "^metatask(_.*)?$": {
           "$ref": "#/$defs/metatask"
         },
-        "^task(_[a-z0-9_]+)?$": {
+        "^task(_.*)?$": {
           "$ref": "#/$defs/task"
         }
       },
@@ -279,7 +279,7 @@
         "dependency": {
           "$ref": "#/$defs/dependency"
         },
-        "envar": {
+        "envars": {
           "type": "object"
         },
         "exclusive": {
@@ -407,10 +407,10 @@
           "additionalProperties": false,
           "minProperties": 1,
           "patternProperties": {
-            "^metatask(_[a-z0-9_]+)?$": {
+            "^metatask(_.*)?$": {
               "$ref": "#/$defs/metatask"
             },
-            "^task(_[a-z0-9_]+)?$": {
+            "^task(_.*)?$": {
               "$ref": "#/$defs/task"
             }
           },

src/uwtools/rocoto.py

@@ -2,8 +2,18 @@
 Support for creating Rocoto XML workflow documents.
 """
 
+import logging
+import shutil
+import tempfile
+from importlib import resources
+
+from lxml import etree
+
+import uwtools.config.validator
 from uwtools.config.core import YAMLConfig
 from uwtools.config.j2template import J2Template
+from uwtools.types import DefinitePath, OptionalPath
+from uwtools.utils.file import readable
 
 # Private functions
 
@@ -12,7 +22,7 @@ def _add_jobname(tree: dict) -> None:
     """
     Add a "jobname" attribute to each "task" element in the given config tree.
 
-    :param tree: A config tree containing "task" elements..
+    :param tree: A config tree containing "task" elements.
     """
     for element, subtree in tree.items():
         element_parts = element.split("_", maxsplit=1)
@@ -25,20 +35,119 @@ def _add_jobname(tree: dict) -> None:
             _add_jobname(subtree)
 
 
-# Public functions
-def write_rocoto_xml(input_yaml: str, input_template: str, rendered_output: str) -> None:
+def _add_jobname_to_tasks(
+    input_yaml: OptionalPath = None,
+) -> YAMLConfig:
     """
-    Main entry point.
+    Load YAML config and add job names to each defined workflow task.
 
     :param input_yaml: Path to YAML input file.
-    :param input_template: Path to input template file.
-    :param rendered_output: Path to write rendered XML file.
     """
     values = YAMLConfig(input_yaml)
-    tasks = values["tasks"]
+    tasks = values["workflow"]["tasks"]
     if isinstance(tasks, dict):
         _add_jobname(tasks)
+    return values
+
+
+def _rocoto_schema_xml() -> DefinitePath:
+    """
+    The path to the file containing the schema to validate the XML file against.
+    """
+    with resources.as_file(resources.files("uwtools.resources")) as path:
+        return path / "schema_with_metatasks.rng"
+
+
+def _rocoto_schema_yaml() -> DefinitePath:
+    """
+    The path to the file containing the schema to validate the YAML file against.
+    """
+    with resources.as_file(resources.files("uwtools.resources")) as path:
+        return path / "rocoto.jsonschema"
+
+
+def _rocoto_template_xml() -> DefinitePath:
+    """
+    The path to the file containing the Rocoto workflow document template to render.
+    """
+    with resources.as_file(resources.files("uwtools.resources")) as path:
+        return path / "rocoto.jinja2"
+
+
+def _write_rocoto_xml(
+    config_file: OptionalPath,
+    rendered_output: DefinitePath,
+) -> None:
+    """
+    Render the Rocoto workflow defined in the given YAML to XML.
+
+    :param config_file: Path to YAML input file.
+    :param rendered_output: Path to write rendered XML file.
+    """
+
+    values = _add_jobname_to_tasks(config_file)
 
     # Render the template.
-    template = J2Template(values=values.data, template_path=input_template)
-    template.dump(output_path=rendered_output)
+    template = J2Template(values=values.data, template_path=_rocoto_template_xml())
+    template.dump(output_path=str(rendered_output))
+
+
+# Public functions
+def realize_rocoto_xml(
+    config_file: OptionalPath,
+    rendered_output: DefinitePath,
+) -> bool:
+    """
+    Realize the Rocoto workflow defined in the given YAML as XML. Validate both the YAML input and
+    XML output.
+
+    :param config_file: Path to YAML input file.
+    :param rendered_output: Path to write rendered XML file.
+    :return: Did the input and output files conform to theirr schemas?
+    """
+
+    # Validate the YAML.
+    if uwtools.config.validator.validate_yaml(
+        config_file=config_file, schema_file=_rocoto_schema_yaml()
+    ):
+        # Render the template to a temporary file.
+        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+            _write_rocoto_xml(
+                config_file=config_file,
+                rendered_output=temp_file.name,
+            )
+            # Validate the XML.
+            if validate_rocoto_xml(input_xml=temp_file.name):
+                # If no issues were detected, save temp file and report success.
+                shutil.move(temp_file.name, rendered_output)
+                return True
+        logging.error("Rocoto validation errors identified in %s", temp_file.name)
+        return False
+    logging.error("YAML validation errors identified in %s", config_file)
+    return False
+
+
+def validate_rocoto_xml(input_xml: OptionalPath) -> bool:
+    """
+    Given a rendered XML file, validate it against the Rocoto schema.
+
+    :param input_xml: Path to rendered XML file.
+    :return: Did the XML file conform to the schema?
+    """
+
+    # Validate the XML.
+    with open(_rocoto_schema_xml(), "r", encoding="utf-8") as f:
+        schema = etree.RelaxNG(etree.parse(f))
+    with readable(input_xml) as f:
+        xml = f.read()
+    tree = etree.fromstring(bytes(xml, encoding="utf-8"))
+    success = schema.validate(tree)
+
+    # Log validation errors.
+    errors = str(etree.RelaxNG.error_log).split("\n")
+    log_method = logging.error if len(errors) else logging.info
+    log_method("%s Rocoto validation error%s found", len(errors), "" if len(errors) == 1 else "s")
+    for line in errors:
+        logging.error(line)
+
+    return success