Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

UW-632 Schema Injection #551

Merged
merged 23 commits into from
Aug 2, 2024
Merged
Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
2626b6d
Move str2path() from utils.api to utils.file
maddenp-noaa Jul 30, 2024
fc56038
Merge branch 'main' into uw-632-schema-injection
maddenp-noaa Jul 30, 2024
4387e92
Let Config() take str, coerce to Path
maddenp-noaa Jul 30, 2024
516cbf3
Let Assets et al. take str as well as Path
maddenp-noaa Jul 30, 2024
a72d788
Let Assets et al. take YAMLConfig
maddenp-noaa Jul 30, 2024
858e32d
Merge branch 'main' into uw-632-schema-injection
maddenp-noaa Jul 30, 2024
865d009
Tweak prototype configs
maddenp-noaa Jul 30, 2024
30682f0
Use STR in drivers
maddenp-noaa Jul 30, 2024
7ba20cf
Honor coupler
maddenp-noaa Jul 31, 2024
1b7cf02
WIP
maddenp-noaa Jul 31, 2024
12fa5dd
WIP
maddenp-noaa Jul 31, 2024
e4ec299
Add inherited parameter descriptions to driver docstrings
maddenp-noaa Jul 31, 2024
af1b31c
Add inherited parameter descriptions to driver docstrings
maddenp-noaa Jul 31, 2024
2a95013
Update unit tests
maddenp-noaa Jul 31, 2024
7b6f70b
Improve docstring param description for 'coupler'
maddenp-noaa Jul 31, 2024
77b8348
Merge branch 'main' into uw-632-schema-injection
maddenp-noaa Jul 31, 2024
a30d2b6
Improvement in set_driver_docstring()
maddenp-noaa Jul 31, 2024
ecda55c
Merge branch 'main' into uw-632-schema-injection
maddenp-noaa Aug 1, 2024
9b7fcec
Merge branch 'main' into uw-632-schema-injection
maddenp-noaa Aug 2, 2024
079b506
Fix type annotation
maddenp-noaa Aug 2, 2024
3a72b0e
'coupler' -> 'controller'
maddenp-noaa Aug 2, 2024
e361326
Tweak log message
maddenp-noaa Aug 2, 2024
99a6cb9
Doc updates
maddenp-noaa Aug 2, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

autoclass_content = "both"
autodoc_mock_imports = ["f90nml", "iotaa", "jsonschema", "lxml", "referencing"]
autodoc_typehints = "description"
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
copyright = str(dt.datetime.now().year)
extensions = ["sphinx.ext.autodoc", "sphinx.ext.extlinks", "sphinx.ext.intersphinx"]
extlinks_detect_hardcoded_links = True
Expand All @@ -24,8 +25,8 @@
numfig_format = {"figure": "Figure %s"}
project = "Unified Workflow Tools"
release = _metadata["version"]
version = _metadata["version"]
user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"
version = _metadata["version"]

extlinks = {
"anaconda": ("https://www.anaconda.com/%s", "%s"),
Expand Down
6 changes: 3 additions & 3 deletions docs/shared/cdeps.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ cdeps:
stream_data_files:
- /path/to/data_file1.nc
stream_data_variables:
- 'temperature Sa_tbot'
- temperature Sa_tbot
stream_lev_dimname: 'null'
stream_mesh_file: /path/to/mesh1.nc
stream_offset: 0
Expand Down Expand Up @@ -59,8 +59,8 @@ cdeps:
- /path/to/data_file2.nc
- /path/to/data_file3.nc
stream_data_variables:
- 'sst So_t'
- 'mask So_omask'
- sst So_t
- mask So_omask
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
stream_lev_dimname: 'null'
stream_mesh_file: /path/to/mesh2.nc
stream_offset: 0
Expand Down
2 changes: 1 addition & 1 deletion docs/shared/ww3.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@ ww3:
namelist:
template_file: /path/to/ww3/ww3_shel.nml.IN
template_values:
input_forcing_winds: "C"
input_forcing_winds: C
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
rundir: /path/to/run/directory
2 changes: 1 addition & 1 deletion recipe/run_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ lint() {
msg Running linter
(
set -eux
pylint -j 4 .
pylint -j 1 .
)
msg OK
}
Expand Down
14 changes: 10 additions & 4 deletions src/uwtools/api/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,10 @@
from uwtools.config.tools import compare_configs as _compare
from uwtools.config.tools import realize_config as _realize
from uwtools.config.validator import validate_external as _validate_external
from uwtools.exceptions import UWConfigError
from uwtools.utils.api import ensure_data_source as _ensure_data_source
from uwtools.utils.api import str2path as _str2path
from uwtools.utils.file import FORMAT as _FORMAT
from uwtools.utils.file import str2path as _str2path
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved

# Public

Expand Down Expand Up @@ -173,9 +174,14 @@ def validate(
:param stdin_ok: OK to read from ``stdin``?
:return: ``True`` if the YAML file conforms to the schema, ``False`` otherwise.
"""
return _validate_external(
schema_file=_str2path(schema_file), config=_ensure_data_source(_str2path(config), stdin_ok)
)
try:
_validate_external(
schema_file=_str2path(schema_file),
config=_ensure_data_source(_str2path(config), stdin_ok),
)
except UWConfigError:
return False
return True
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved


# Import-time code
Expand Down
2 changes: 1 addition & 1 deletion src/uwtools/api/file.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

from uwtools.file import Copier, Linker
from uwtools.utils.api import ensure_data_source as _ensure_data_source
from uwtools.utils.api import str2path as _str2path
from uwtools.utils.file import str2path as _str2path


def copy(
Expand Down
2 changes: 1 addition & 1 deletion src/uwtools/api/rocoto.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from uwtools.rocoto import realize_rocoto_xml as _realize
from uwtools.rocoto import validate_rocoto_xml_file as _validate
from uwtools.utils.api import ensure_data_source as _ensure_data_source
from uwtools.utils.api import str2path as _str2path
from uwtools.utils.file import str2path as _str2path


def realize(
Expand Down
2 changes: 1 addition & 1 deletion src/uwtools/api/template.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from uwtools.config.jinja2 import render as _render
from uwtools.exceptions import UWTemplateRenderError
from uwtools.utils.api import ensure_data_source as _ensure_data_source
from uwtools.utils.api import str2path as _str2path
from uwtools.utils.file import str2path as _str2path


def render(
Expand Down
5 changes: 3 additions & 2 deletions src/uwtools/config/formats/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from uwtools.config.support import INCLUDE_TAG, depth, log_and_error, yaml_to_str
from uwtools.exceptions import UWConfigError
from uwtools.logging import INDENT, log
from uwtools.utils.file import str2path


class Config(ABC, UserDict):
Expand All @@ -22,7 +23,7 @@ class Config(ABC, UserDict):
several configuration-file formats.
"""

def __init__(self, config: Optional[Union[dict, Path]] = None) -> None:
def __init__(self, config: Optional[Union[dict, str, Path]] = None) -> None:
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
"""
Construct a Config object.

Expand All @@ -33,7 +34,7 @@ def __init__(self, config: Optional[Union[dict, Path]] = None) -> None:
self._config_file = None
self.update(config)
else:
self._config_file = config if config else None
self._config_file = str2path(config) if config else None
self.data = self._load(self._config_file)
if self.get_depth_threshold() and self.depth != self.get_depth_threshold():
raise UWConfigError(
Expand Down
14 changes: 7 additions & 7 deletions src/uwtools/config/validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,37 +47,37 @@ def validate(schema: dict, config: dict) -> bool:


def validate_internal(
schema_name: str, config: Union[dict, YAMLConfig, Optional[Path]] = None
schema_name: str, config: Optional[Union[dict, YAMLConfig, Path]] = None
) -> None:
"""
Validate a config against a uwtools-internal schema.

:param config: The config to validate.
:param schema_name: Name of uwtools schema to validate the config against.
:param config: The config to validate.
:raises: UWConfigError if config fails validation.
"""

log.info("Validating config against internal schema %s", schema_name)
schema_file = get_schema_file(schema_name)
log.debug("Using schema file: %s", schema_file)
if not validate_external(config=config, schema_file=schema_file):
raise UWConfigError("YAML validation errors")
validate_external(config=config, schema_file=schema_file)
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved


def validate_external(
schema_file: Path, config: Union[dict, YAMLConfig, Optional[Path]] = None
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
) -> bool:
) -> None:
"""
Validate a YAML config against the JSON Schema in the given schema file.

:param schema_file: The JSON Schema file to use for validation.
:param config: The config to validate.
:return: Did the YAML file conform to the schema?
:raises: UWConfigError if config fails validation.
"""
with open(schema_file, "r", encoding="utf-8") as f:
schema = json.load(f)
cfgobj = _prep_config(config)
return validate(schema=schema, config=cfgobj.data)
if not validate(schema=schema, config=cfgobj.data):
raise UWConfigError("YAML validation errors")


# Private functions
Expand Down
4 changes: 4 additions & 0 deletions src/uwtools/drivers/cdeps.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from uwtools.api.template import _render
from uwtools.config.formats.nml import NMLConfig
from uwtools.drivers.driver import AssetsCycleBased
from uwtools.drivers.support import set_driver_docstring
from uwtools.strings import STR
from uwtools.utils.tasks import file

Expand Down Expand Up @@ -127,3 +128,6 @@ def _model_stream_file(self, group: str, path: Path, template_file: str) -> None
output_file=path,
values_src=self._driver_config[group],
)


set_driver_docstring(CDEPS)
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
12 changes: 8 additions & 4 deletions src/uwtools/drivers/chgres_cube.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from uwtools.config.formats.nml import NMLConfig
from uwtools.drivers.driver import DriverCycleBased
from uwtools.drivers.support import set_driver_docstring
from uwtools.strings import STR
from uwtools.utils.tasks import file

Expand All @@ -29,10 +30,10 @@ def namelist_file(self):
path = self._rundir / fn
yield asset(path, path.is_file)
input_files = []
namelist = self._driver_config["namelist"]
if base_file := namelist.get("base_file"):
namelist = self._driver_config[STR.namelist]
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
if base_file := namelist.get(STR.basefile):
input_files.append(base_file)
if update_values := namelist.get("update_values"):
if update_values := namelist.get(STR.updatevalues):
config_files = update_values["config"]
for k in ["mosaic_file_target_grid", "vcoord_file_target_grid"]:
input_files.append(config_files[k])
Expand Down Expand Up @@ -83,7 +84,7 @@ def runscript(self):
yield None
envvars = {
"KMP_AFFINITY": "scatter",
"OMP_NUM_THREADS": self._driver_config.get("execution", {}).get("threads", 1),
"OMP_NUM_THREADS": self._driver_config.get(STR.execution, {}).get(STR.threads, 1),
"OMP_STACKSIZE": "1024m",
}
self._write_runscript(path=path, envvars=envvars)
Expand All @@ -96,3 +97,6 @@ def _driver_name(self) -> str:
Returns the name of this driver.
"""
return STR.chgrescube


set_driver_docstring(ChgresCube)
Loading
Loading