UW-391 Add IODA driver (#511)

Adds IODA driver, docs, and schema

docs/Makefile

@@ -17,7 +17,7 @@ docs:
 	$(MAKE) html
 
 examples:
-	$(MAKE) -C sections
+	COLUMNS=80 $(MAKE) -C sections
 
 linkcheck:
 	$(SPHINXBUILD) -b linkcheck $(SPHINXOPTS) -c $(CURDIR) $(CURDIR) build/linkcheck

docs/index.rst

@@ -178,6 +178,12 @@ UPP
 Driver for JEDI
 ^^^^^^^^^^^^^^^
 
+IODA
+""""
+
+| **CLI**: ``uw ioda -h``
+| **API**: ``import uwtools.api.ioda``
+
 JEDI
 """"
 

docs/sections/user_guide/api/index.rst

@@ -9,6 +9,7 @@ API
    filter_topo
    fv3
    global_equiv_resol
+   ioda
    jedi
    logging
    make_hgrid

docs/sections/user_guide/api/ioda.rst

@@ -0,0 +1,5 @@
+``uwtools.api.ioda``
+====================
+
+.. automodule:: uwtools.api.ioda
+    :members:

docs/sections/user_guide/cli/drivers/index.rst

@@ -9,6 +9,7 @@ Drivers
    filter_topo
    fv3
    global_equiv_resol
+   ioda
    jedi
    make_hgrid
    make_solo_mosaic

docs/sections/user_guide/cli/drivers/ioda.rst

@@ -0,0 +1,52 @@
+``ioda``
+========
+
+The ``uw`` mode for configuring and running the IODA components of the JEDI framework.
+
+.. literalinclude:: ioda/help.cmd
+   :language: text
+   :emphasize-lines: 1
+.. literalinclude:: ioda/help.out
+   :language: text
+
+All tasks take the same arguments. For example:
+
+.. literalinclude:: ioda/run-help.cmd
+   :language: text
+   :emphasize-lines: 1
+.. literalinclude:: ioda/run-help.out
+   :language: text
+
+Examples
+^^^^^^^^
+
+The examples use a configuration file named ``config.yaml`` with contents similar to:
+
+.. highlight:: yaml
+.. literalinclude:: /shared/ioda.yaml
+
+Its contents are described in section :ref:`ioda_yaml`.
+
+* Run ``ioda`` on an interactive node
+
+   .. code-block:: text
+
+      $ uw ioda run --config-file config.yaml --cycle 2024-05-22T12
+
+The driver creates a ``runscript.ioda`` file in the directory specified by ``run_dir:`` in the config and runs it, executing ``ioda``.
+
+* Run ``ioda`` via a batch job
+
+   .. code-block:: text
+
+      $ uw ioda run --config-file config.yaml --cycle 2024-05-22T12 --batch
+
+The driver creates a ``runscript.ioda`` file in the directory specified by ``run_dir:`` in the config and submits it to the batch system. Running with ``--batch`` requires a correctly configured ``platform:`` block in ``config.yaml``, as well as appropriate settings in the ``execution:`` block under ``ioda:``.
+
+* Specifying the ``--dry-run`` flag results in the driver logging messages about actions it would have taken, without actually taking any.
+
+  .. code-block:: text
+
+     $ uw ioda run --config-file config.yaml --cycle 2024-05-22T12 --batch --dry-run
+
+.. include:: /shared/key_path.rst

docs/sections/user_guide/cli/drivers/ioda/Makefile

@@ -0,0 +1 @@
+../../Makefile.outputs

docs/sections/user_guide/cli/drivers/ioda/help.cmd

@@ -0,0 +1 @@
+uw ioda --help

docs/sections/user_guide/cli/drivers/ioda/help.out

@@ -0,0 +1,26 @@
+usage: uw ioda [-h] [--version] TASK ...
+
+Execute ioda tasks
+
+Optional arguments:
+  -h, --help
+      Show help and exit
+  --version
+      Show version info and exit
+
+Positional arguments:
+  TASK
+    configuration_file
+      The executable's YAML configuration file
+    files_copied
+      Files copied for run
+    files_linked
+      Files linked for run
+    provisioned_run_directory
+      Run directory provisioned with all required content
+    run
+      A run
+    runscript
+      The runscript
+    validate
+      Validate the UW driver config

docs/sections/user_guide/cli/drivers/ioda/run-help.cmd

@@ -0,0 +1 @@
+uw ioda run --help

docs/sections/user_guide/cli/drivers/ioda/run-help.out

@@ -0,0 +1,30 @@
+usage: uw ioda run --cycle CYCLE [-h] [--version] [--config-file PATH]
+                   [--batch] [--dry-run] [--graph-file PATH]
+                   [--key-path KEY[.KEY...]] [--quiet] [--verbose]
+
+A run
+
+Required arguments:
+  --cycle CYCLE
+      The cycle in ISO8601 format (e.g. 2024-06-17T18)
+
+Optional arguments:
+  -h, --help
+      Show help and exit
+  --version
+      Show version info and exit
+  --config-file PATH, -c PATH
+      Path to UW YAML config file (default: read from stdin)
+  --batch
+      Submit run to batch scheduler
+  --dry-run
+      Only log info, making no changes
+  --graph-file PATH
+      Path to Graphviz DOT output [experimental]
+  --key-path KEY[.KEY...]
+      Dot-separated path of keys leading through the config to the driver's
+      configuration block
+  --quiet, -q
+      Print no logging messages
+  --verbose, -v
+      Print all logging messages

docs/sections/user_guide/cli/drivers/jedi/help.out

@@ -11,7 +11,7 @@ Optional arguments:
 Positional arguments:
   TASK
     configuration_file
-      The JEDI YAML configuration file
+      The executable's YAML configuration file
     files_copied
       Files copied for run
     files_linked

docs/sections/user_guide/yaml/components/index.rst

@@ -9,6 +9,7 @@ UW YAML for Components
    filter_topo
    fv3
    global_equiv_resol
+   ioda
    jedi
    make_hgrid
    make_solo_mosaic

docs/sections/user_guide/yaml/components/ioda.rst

@@ -0,0 +1,41 @@
+.. _ioda_yaml:
+
+ioda
+====
+
+Structured YAML to run IODA is validated by JSON Schema and requires the ``ioda:`` block, described below. If ``ioda`` is to be run via a batch system, the ``platform:`` block, described :ref:`here <platform_yaml>`, is also required.
+
+.. include:: /shared/injected_cycle.rst
+
+Here is a prototype UW YAML ``ioda:`` block, explained in detail below:
+
+.. highlight:: yaml
+.. literalinclude:: /shared/ioda.yaml
+
+UW YAML for the ``ioda:`` Block
+-------------------------------
+
+execution:
+^^^^^^^^^^
+
+See :ref:`this page <execution_yaml>` for details.
+
+configuration_file:
+^^^^^^^^^^^^^^^^^^^
+
+Supports ``base_file:`` and ``update_values:`` blocks (see :ref:`updating_values` for details).
+
+files_to_copy:
+^^^^^^^^^^^^^^
+
+See :ref:`this page <files_yaml>` for details.
+
+files_to_link:
+^^^^^^^^^^^^^^
+
+Identical to ``files_to_copy:`` except that symbolic links will be created in the run directory instead of copies.
+
+run_dir:
+^^^^^^^^
+
+The path to the run directory.

docs/shared/ioda.yaml

@@ -0,0 +1,25 @@
+ioda:
+  configuration_file:
+    base_file: path/to/config.yaml
+    update_values:
+      baz: qux
+  execution:
+    batchargs:
+      nodes: 1
+      stdout: path/to/runscript.out
+      walltime: "00:05:00"
+    envcmds:
+      - module load some-module
+      - module load ioda-module
+    executable: /path/to/a/ioda/exe
+    mpicmd: time
+  files_to_copy:
+    d/f2: /path/to/f2
+    f1: /path/to/f1
+  files_to_link:
+    f3: /path/to/f3
+    f4: d/f4
+  run_dir: /path/to/run/dir
+platform:
+  account: me
+  scheduler: slurm

docs/shared/jedi.yaml

@@ -22,7 +22,7 @@ jedi:
   files_to_link:
     f3: /path/to/f3
     f4: d/f4
-  run_dir: /path/to/run
+  run_dir: /path/to/run/dir
 platform:
   account: me
   scheduler: slurm

docs/shared/sfc_climo_gen.yaml

@@ -33,7 +33,7 @@ sfc_climo_gen:
         snowfree_albedo_method: bilinear
         vegetation_greenness_method: bilinear
     validate: true
-  run_dir: /path/to/run
+  run_dir: /path/to/run/dir
 platform:
   account: me
   scheduler: slurm

docs/shared/upp.yaml

@@ -35,7 +35,7 @@ upp:
           - 100
           - 1
     validate: true
-  run_dir: /path/to/run
+  run_dir: /path/to/run/dir
 platform:
   account: me
   scheduler: slurm

src/uwtools/api/ioda.py

@@ -0,0 +1,12 @@
+"""
+API access to the ``uwtools`` ``ioda`` driver.
+"""
+
+from uwtools.drivers.ioda import IODA as _Driver
+from uwtools.drivers.support import graph
+from uwtools.utils.api import make_execute as _make_execute
+from uwtools.utils.api import make_tasks as _make_tasks
+
+execute = _make_execute(_Driver, with_cycle=True)
+tasks = _make_tasks(_Driver)
+__all__ = ["execute", "graph", "tasks"]

src/uwtools/cli.py

@@ -69,6 +69,7 @@ def main() -> None:
                 STR.filtertopo,
                 STR.fv3,
                 STR.globalequivresol,
+                STR.ioda,
                 STR.jedi,
                 STR.makehgrid,
                 STR.makesolomosaic,
@@ -1046,6 +1047,7 @@ def _parse_args(raw_args: List[str]) -> Tuple[Args, Checks]:
         for component in [
             STR.chgrescube,
             STR.fv3,
+            STR.ioda,
             STR.jedi,
             STR.mpas,
             STR.mpasinit,

src/uwtools/drivers/ioda.py

@@ -0,0 +1,52 @@
+"""
+A driver for the ioda component.
+"""
+
+from iotaa import tasks
+
+from uwtools.drivers.jedi_base import JEDIBase
+from uwtools.strings import STR
+
+
+class IODA(JEDIBase):
+    """
+    A driver for the IODA component.
+    """
+
+    @tasks
+    def provisioned_run_directory(self):
+        """
+        Run directory provisioned with all required content.
+        """
+        yield self._taskname("provisioned run directory")
+        yield [
+            self.configuration_file(),
+            self.files_copied(),
+            self.files_linked(),
+            self.runscript(),
+        ]
+
+    # Private helper methods
+
+    @property
+    def _config_fn(self) -> str:
+        """
+        Returns the name of the config file used in execution.
+        """
+        return "ioda.yaml"
+
+    @property
+    def _driver_name(self) -> str:
+        """
+        Returns the name of this driver.
+        """
+        return STR.ioda
+
+    @property
+    def _runcmd(self) -> str:
+        """
+        Returns the full command-line component invocation.
+        """
+        executable = self._driver_config["execution"]["executable"]
+        jedi_config = str(self._rundir / self._config_fn)
+        return " ".join([executable, jedi_config])