Skip to content

Commit

Permalink
UW-632 Schema Injection (#551)
Browse files Browse the repository at this point in the history
  • Loading branch information
maddenp-noaa authored Aug 2, 2024
1 parent a918dbd commit 01ad7f3
Show file tree
Hide file tree
Showing 47 changed files with 462 additions and 234 deletions.
3 changes: 2 additions & 1 deletion docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

autoclass_content = "both"
autodoc_mock_imports = ["f90nml", "iotaa", "jsonschema", "lxml", "referencing"]
autodoc_typehints = "description"
copyright = str(dt.datetime.now().year)
extensions = ["sphinx.ext.autodoc", "sphinx.ext.extlinks", "sphinx.ext.intersphinx"]
extlinks_detect_hardcoded_links = True
Expand All @@ -24,8 +25,8 @@
numfig_format = {"figure": "Figure %s"}
project = "Unified Workflow Tools"
release = _metadata["version"]
version = _metadata["version"]
user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"
version = _metadata["version"]

extlinks = {
"anaconda": ("https://www.anaconda.com/%s", "%s"),
Expand Down
20 changes: 10 additions & 10 deletions docs/sections/user_guide/cli/tools/file/copy-exec-timedep.out
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
[2024-06-05T15:24:49] INFO Validating config against internal schema files-to-stage
[2024-06-05T15:24:49] INFO 0 UW schema-validation errors found
[2024-06-05T15:24:49] INFO File copies: Initial state: Not Ready
[2024-06-05T15:24:49] INFO File copies: Checking requirements
[2024-06-05T15:24:49] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Initial state: Not Ready
[2024-06-05T15:24:49] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Checking requirements
[2024-06-05T15:24:49] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Requirement(s) ready
[2024-06-05T15:24:49] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Executing
[2024-06-05T15:24:49] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Final state: Ready
[2024-06-05T15:24:49] INFO File copies: Final state: Ready
[2024-08-02T00:43:08] INFO Validating config against internal schema: files-to-stage
[2024-08-02T00:43:08] INFO 0 UW schema-validation errors found
[2024-08-02T00:43:08] INFO File copies: Initial state: Not Ready
[2024-08-02T00:43:08] INFO File copies: Checking requirements
[2024-08-02T00:43:08] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Initial state: Not Ready
[2024-08-02T00:43:08] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Checking requirements
[2024-08-02T00:43:08] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Requirement(s) ready
[2024-08-02T00:43:08] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Executing
[2024-08-02T00:43:08] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Final state: Ready
[2024-08-02T00:43:08] INFO File copies: Final state: Ready

copy-dst-timedep
└── baz-2024-05-29T18
Expand Down
30 changes: 15 additions & 15 deletions docs/sections/user_guide/cli/tools/file/copy-exec.out
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
[2024-06-05T15:24:50] INFO Validating config against internal schema files-to-stage
[2024-06-05T15:24:50] INFO 0 UW schema-validation errors found
[2024-06-05T15:24:50] INFO File copies: Initial state: Not Ready
[2024-06-05T15:24:50] INFO File copies: Checking requirements
[2024-06-05T15:24:50] INFO Copy src/foo -> copy-dst/foo: Initial state: Not Ready
[2024-06-05T15:24:50] INFO Copy src/foo -> copy-dst/foo: Checking requirements
[2024-06-05T15:24:50] INFO Copy src/foo -> copy-dst/foo: Requirement(s) ready
[2024-06-05T15:24:50] INFO Copy src/foo -> copy-dst/foo: Executing
[2024-06-05T15:24:50] INFO Copy src/foo -> copy-dst/foo: Final state: Ready
[2024-06-05T15:24:50] INFO Copy src/bar -> copy-dst/subdir/bar: Initial state: Not Ready
[2024-06-05T15:24:50] INFO Copy src/bar -> copy-dst/subdir/bar: Checking requirements
[2024-06-05T15:24:50] INFO Copy src/bar -> copy-dst/subdir/bar: Requirement(s) ready
[2024-06-05T15:24:50] INFO Copy src/bar -> copy-dst/subdir/bar: Executing
[2024-06-05T15:24:50] INFO Copy src/bar -> copy-dst/subdir/bar: Final state: Ready
[2024-06-05T15:24:50] INFO File copies: Final state: Ready
[2024-08-02T00:43:08] INFO Validating config against internal schema: files-to-stage
[2024-08-02T00:43:08] INFO 0 UW schema-validation errors found
[2024-08-02T00:43:08] INFO File copies: Initial state: Not Ready
[2024-08-02T00:43:08] INFO File copies: Checking requirements
[2024-08-02T00:43:08] INFO Copy src/foo -> copy-dst/foo: Initial state: Not Ready
[2024-08-02T00:43:08] INFO Copy src/foo -> copy-dst/foo: Checking requirements
[2024-08-02T00:43:08] INFO Copy src/foo -> copy-dst/foo: Requirement(s) ready
[2024-08-02T00:43:08] INFO Copy src/foo -> copy-dst/foo: Executing
[2024-08-02T00:43:08] INFO Copy src/foo -> copy-dst/foo: Final state: Ready
[2024-08-02T00:43:08] INFO Copy src/bar -> copy-dst/subdir/bar: Initial state: Not Ready
[2024-08-02T00:43:08] INFO Copy src/bar -> copy-dst/subdir/bar: Checking requirements
[2024-08-02T00:43:08] INFO Copy src/bar -> copy-dst/subdir/bar: Requirement(s) ready
[2024-08-02T00:43:08] INFO Copy src/bar -> copy-dst/subdir/bar: Executing
[2024-08-02T00:43:08] INFO Copy src/bar -> copy-dst/subdir/bar: Final state: Ready
[2024-08-02T00:43:08] INFO File copies: Final state: Ready

copy-dst
├── foo
Expand Down
20 changes: 10 additions & 10 deletions docs/sections/user_guide/cli/tools/file/link-exec-timedep.out
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
[2024-06-05T15:24:50] INFO Validating config against internal schema files-to-stage
[2024-06-05T15:24:50] INFO 0 UW schema-validation errors found
[2024-06-05T15:24:50] INFO File links: Initial state: Not Ready
[2024-06-05T15:24:50] INFO File links: Checking requirements
[2024-06-05T15:24:50] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Initial state: Not Ready
[2024-06-05T15:24:50] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Checking requirements
[2024-06-05T15:24:50] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Requirement(s) ready
[2024-06-05T15:24:50] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Executing
[2024-06-05T15:24:50] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Final state: Ready
[2024-06-05T15:24:50] INFO File links: Final state: Ready
[2024-08-02T00:43:09] INFO Validating config against internal schema: files-to-stage
[2024-08-02T00:43:09] INFO 0 UW schema-validation errors found
[2024-08-02T00:43:09] INFO File links: Initial state: Not Ready
[2024-08-02T00:43:09] INFO File links: Checking requirements
[2024-08-02T00:43:09] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Initial state: Not Ready
[2024-08-02T00:43:09] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Checking requirements
[2024-08-02T00:43:09] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Requirement(s) ready
[2024-08-02T00:43:09] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Executing
[2024-08-02T00:43:09] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Final state: Ready
[2024-08-02T00:43:09] INFO File links: Final state: Ready

link-dst-timedep
└── baz-2024-05-29T18 -> ../src/20240529/12/006/baz
Expand Down
30 changes: 15 additions & 15 deletions docs/sections/user_guide/cli/tools/file/link-exec.out
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
[2024-06-05T15:24:49] INFO Validating config against internal schema files-to-stage
[2024-06-05T15:24:49] INFO 0 UW schema-validation errors found
[2024-06-05T15:24:49] INFO File links: Initial state: Not Ready
[2024-06-05T15:24:49] INFO File links: Checking requirements
[2024-06-05T15:24:49] INFO Link link-dst/foo -> src/foo: Initial state: Not Ready
[2024-06-05T15:24:49] INFO Link link-dst/foo -> src/foo: Checking requirements
[2024-06-05T15:24:49] INFO Link link-dst/foo -> src/foo: Requirement(s) ready
[2024-06-05T15:24:49] INFO Link link-dst/foo -> src/foo: Executing
[2024-06-05T15:24:49] INFO Link link-dst/foo -> src/foo: Final state: Ready
[2024-06-05T15:24:49] INFO Link link-dst/subdir/bar -> src/bar: Initial state: Not Ready
[2024-06-05T15:24:49] INFO Link link-dst/subdir/bar -> src/bar: Checking requirements
[2024-06-05T15:24:49] INFO Link link-dst/subdir/bar -> src/bar: Requirement(s) ready
[2024-06-05T15:24:49] INFO Link link-dst/subdir/bar -> src/bar: Executing
[2024-06-05T15:24:49] INFO Link link-dst/subdir/bar -> src/bar: Final state: Ready
[2024-06-05T15:24:49] INFO File links: Final state: Ready
[2024-08-02T00:43:08] INFO Validating config against internal schema: files-to-stage
[2024-08-02T00:43:08] INFO 0 UW schema-validation errors found
[2024-08-02T00:43:08] INFO File links: Initial state: Not Ready
[2024-08-02T00:43:08] INFO File links: Checking requirements
[2024-08-02T00:43:08] INFO Link link-dst/foo -> src/foo: Initial state: Not Ready
[2024-08-02T00:43:08] INFO Link link-dst/foo -> src/foo: Checking requirements
[2024-08-02T00:43:08] INFO Link link-dst/foo -> src/foo: Requirement(s) ready
[2024-08-02T00:43:08] INFO Link link-dst/foo -> src/foo: Executing
[2024-08-02T00:43:08] INFO Link link-dst/foo -> src/foo: Final state: Ready
[2024-08-02T00:43:08] INFO Link link-dst/subdir/bar -> src/bar: Initial state: Not Ready
[2024-08-02T00:43:08] INFO Link link-dst/subdir/bar -> src/bar: Checking requirements
[2024-08-02T00:43:08] INFO Link link-dst/subdir/bar -> src/bar: Requirement(s) ready
[2024-08-02T00:43:08] INFO Link link-dst/subdir/bar -> src/bar: Executing
[2024-08-02T00:43:08] INFO Link link-dst/subdir/bar -> src/bar: Final state: Ready
[2024-08-02T00:43:08] INFO File links: Final state: Ready

link-dst
├── foo -> ../src/foo
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[2024-05-23T19:39:16] INFO 0 UW schema-validation errors found
[2024-05-23T19:39:16] INFO 0 Rocoto validation errors found
[2024-08-02T00:43:07] INFO 0 UW schema-validation errors found
[2024-08-02T00:43:07] INFO 0 Rocoto validation errors found

<?xml version='1.0' encoding='utf-8'?>
<!DOCTYPE workflow [
Expand All @@ -15,6 +15,7 @@
<walltime>00:01:00</walltime>
<command>echo hello $person</command>
<jobname>hello</jobname>
<native>--reservation my_reservation</native>
<envar>
<name>person</name>
<value>siri</value>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[2024-05-23T19:39:16] INFO 0 UW schema-validation errors found
[2024-05-23T19:39:16] INFO 0 Rocoto validation errors found
[2024-08-02T00:43:08] INFO 0 UW schema-validation errors found
[2024-08-02T00:43:08] INFO 0 Rocoto validation errors found
<?xml version='1.0' encoding='utf-8'?>
<!DOCTYPE workflow [
<!ENTITY ACCOUNT "myaccount">
Expand All @@ -14,6 +14,7 @@
<walltime>00:01:00</walltime>
<command>echo hello $person</command>
<jobname>hello</jobname>
<native>--reservation my_reservation</native>
<envar>
<name>person</name>
<value>siri</value>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,21 +1,21 @@
[2024-07-09T00:31:39] DEBUG Command: uw rocoto realize --config-file rocoto.yaml --verbose
[2024-07-09T00:31:39] DEBUG Dereferencing, current value:
[2024-07-09T00:31:39] DEBUG workflow:
[2024-07-09T00:31:39] DEBUG attrs:
[2024-07-09T00:31:39] DEBUG realtime: false
[2024-07-09T00:31:39] DEBUG scheduler: slurm
[2024-07-09T00:31:39] DEBUG cycledef:
[2024-07-09T00:31:39] DEBUG - attrs:
[2024-07-09T00:31:39] DEBUG group: howdy
[2024-07-09T00:31:39] DEBUG spec: 202209290000 202209300000 06:00:00
[2024-08-02T00:43:09] DEBUG Command: uw rocoto realize --config-file rocoto.yaml --verbose
[2024-08-02T00:43:09] DEBUG Dereferencing, current value:
[2024-08-02T00:43:09] DEBUG workflow:
[2024-08-02T00:43:09] DEBUG attrs:
[2024-08-02T00:43:09] DEBUG realtime: false
[2024-08-02T00:43:09] DEBUG scheduler: slurm
[2024-08-02T00:43:09] DEBUG cycledef:
[2024-08-02T00:43:09] DEBUG - attrs:
[2024-08-02T00:43:09] DEBUG group: howdy
[2024-08-02T00:43:09] DEBUG spec: 202209290000 202209300000 06:00:00
...
[2024-07-09T00:31:39] DEBUG attrs:
[2024-07-09T00:31:39] DEBUG cycledefs: howdy
[2024-07-09T00:31:39] DEBUG account: '&ACCOUNT;'
[2024-07-09T00:31:39] DEBUG command: echo hello $person
[2024-07-09T00:31:39] DEBUG jobname: hello
[2024-07-09T00:31:39] DEBUG nodes: 1:ppn=1
[2024-07-09T00:31:39] DEBUG walltime: 00:01:00
[2024-07-09T00:31:39] DEBUG envars:
[2024-07-09T00:31:39] DEBUG person: siri
[2024-07-09T00:31:39] INFO 0 Rocoto validation errors found
[2024-08-02T00:43:09] DEBUG cycledefs: howdy
[2024-08-02T00:43:09] DEBUG account: '&ACCOUNT;'
[2024-08-02T00:43:09] DEBUG command: echo hello $person
[2024-08-02T00:43:09] DEBUG jobname: hello
[2024-08-02T00:43:09] DEBUG native: --reservation my_reservation
[2024-08-02T00:43:09] DEBUG nodes: 1:ppn=1
[2024-08-02T00:43:09] DEBUG walltime: 00:01:00
[2024-08-02T00:43:09] DEBUG envars:
[2024-08-02T00:43:09] DEBUG person: siri
[2024-08-02T00:43:09] INFO 0 Rocoto validation errors found
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[2024-05-23T19:39:16] INFO 0 UW schema-validation errors found
[2024-05-23T19:39:16] INFO 0 Rocoto validation errors found
[2024-08-02T00:43:08] INFO 0 UW schema-validation errors found
[2024-08-02T00:43:08] INFO 0 Rocoto validation errors found
<?xml version='1.0' encoding='utf-8'?>
<!DOCTYPE workflow [
<!ENTITY ACCOUNT "myaccount">
Expand All @@ -14,6 +14,7 @@
<walltime>00:01:00</walltime>
<command>echo hello $person</command>
<jobname>hello</jobname>
<native>--reservation my_reservation</native>
<envar>
<name>person</name>
<value>siri</value>
Expand Down
6 changes: 3 additions & 3 deletions docs/shared/cdeps.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ cdeps:
stream_data_files:
- /path/to/data_file1.nc
stream_data_variables:
- 'temperature Sa_tbot'
- temperature Sa_tbot
stream_lev_dimname: 'null'
stream_mesh_file: /path/to/mesh1.nc
stream_offset: 0
Expand Down Expand Up @@ -59,8 +59,8 @@ cdeps:
- /path/to/data_file2.nc
- /path/to/data_file3.nc
stream_data_variables:
- 'sst So_t'
- 'mask So_omask'
- sst So_t
- mask So_omask
stream_lev_dimname: 'null'
stream_mesh_file: /path/to/mesh2.nc
stream_offset: 0
Expand Down
2 changes: 1 addition & 1 deletion docs/shared/ww3.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@ ww3:
namelist:
template_file: /path/to/ww3/ww3_shel.nml.IN
template_values:
input_forcing_winds: "C"
input_forcing_winds: C
rundir: /path/to/run/directory
14 changes: 10 additions & 4 deletions src/uwtools/api/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,10 @@
from uwtools.config.tools import compare_configs as _compare
from uwtools.config.tools import realize_config as _realize
from uwtools.config.validator import validate_external as _validate_external
from uwtools.exceptions import UWConfigError
from uwtools.utils.api import ensure_data_source as _ensure_data_source
from uwtools.utils.api import str2path as _str2path
from uwtools.utils.file import FORMAT as _FORMAT
from uwtools.utils.file import str2path as _str2path

# Public

Expand Down Expand Up @@ -173,9 +174,14 @@ def validate(
:param stdin_ok: OK to read from ``stdin``?
:return: ``True`` if the YAML file conforms to the schema, ``False`` otherwise.
"""
return _validate_external(
schema_file=_str2path(schema_file), config=_ensure_data_source(_str2path(config), stdin_ok)
)
try:
_validate_external(
schema_file=_str2path(schema_file),
config=_ensure_data_source(_str2path(config), stdin_ok),
)
except UWConfigError:
return False
return True


# Import-time code
Expand Down
2 changes: 1 addition & 1 deletion src/uwtools/api/rocoto.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from uwtools.rocoto import realize_rocoto_xml as _realize
from uwtools.rocoto import validate_rocoto_xml_file as _validate
from uwtools.utils.api import ensure_data_source as _ensure_data_source
from uwtools.utils.api import str2path as _str2path
from uwtools.utils.file import str2path as _str2path


def realize(
Expand Down
2 changes: 1 addition & 1 deletion src/uwtools/api/template.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from uwtools.config.jinja2 import render as _render
from uwtools.exceptions import UWTemplateRenderError
from uwtools.utils.api import ensure_data_source as _ensure_data_source
from uwtools.utils.api import str2path as _str2path
from uwtools.utils.file import str2path as _str2path


def render(
Expand Down
5 changes: 3 additions & 2 deletions src/uwtools/config/formats/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from uwtools.config.support import INCLUDE_TAG, depth, log_and_error, yaml_to_str
from uwtools.exceptions import UWConfigError
from uwtools.logging import INDENT, log
from uwtools.utils.file import str2path


class Config(ABC, UserDict):
Expand All @@ -22,7 +23,7 @@ class Config(ABC, UserDict):
several configuration-file formats.
"""

def __init__(self, config: Optional[Union[dict, Path]] = None) -> None:
def __init__(self, config: Optional[Union[dict, str, Path]] = None) -> None:
"""
Construct a Config object.
Expand All @@ -33,7 +34,7 @@ def __init__(self, config: Optional[Union[dict, Path]] = None) -> None:
self._config_file = None
self.update(config)
else:
self._config_file = config if config else None
self._config_file = str2path(config) if config else None
self.data = self._load(self._config_file)
if self.get_depth_threshold() and self.depth != self.get_depth_threshold():
raise UWConfigError(
Expand Down
18 changes: 9 additions & 9 deletions src/uwtools/config/validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,37 +47,37 @@ def validate(schema: dict, config: dict) -> bool:


def validate_internal(
schema_name: str, config: Union[dict, YAMLConfig, Optional[Path]] = None
schema_name: str, config: Optional[Union[dict, YAMLConfig, Path]] = None
) -> None:
"""
Validate a config against a uwtools-internal schema.
:param config: The config to validate.
:param schema_name: Name of uwtools schema to validate the config against.
:param config: The config to validate.
:raises: UWConfigError if config fails validation.
"""

log.info("Validating config against internal schema %s", schema_name)
log.info("Validating config against internal schema: %s", schema_name)
schema_file = get_schema_file(schema_name)
log.debug("Using schema file: %s", schema_file)
if not validate_external(config=config, schema_file=schema_file):
raise UWConfigError("YAML validation errors")
validate_external(config=config, schema_file=schema_file)


def validate_external(
schema_file: Path, config: Union[dict, YAMLConfig, Optional[Path]] = None
) -> bool:
schema_file: Path, config: Optional[Union[dict, YAMLConfig, Path]] = None
) -> None:
"""
Validate a YAML config against the JSON Schema in the given schema file.
:param schema_file: The JSON Schema file to use for validation.
:param config: The config to validate.
:return: Did the YAML file conform to the schema?
:raises: UWConfigError if config fails validation.
"""
with open(schema_file, "r", encoding="utf-8") as f:
schema = json.load(f)
cfgobj = _prep_config(config)
return validate(schema=schema, config=cfgobj.data)
if not validate(schema=schema, config=cfgobj.data):
raise UWConfigError("YAML validation errors")


# Private functions
Expand Down
4 changes: 4 additions & 0 deletions src/uwtools/drivers/cdeps.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from uwtools.api.template import _render
from uwtools.config.formats.nml import NMLConfig
from uwtools.drivers.driver import AssetsCycleBased
from uwtools.drivers.support import set_driver_docstring
from uwtools.strings import STR
from uwtools.utils.tasks import file

Expand Down Expand Up @@ -127,3 +128,6 @@ def _model_stream_file(self, group: str, path: Path, template_file: str) -> None
output_file=path,
values_src=self._driver_config[group],
)


set_driver_docstring(CDEPS)
Loading

0 comments on commit 01ad7f3

Please sign in to comment.