Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fv3-schema-update #405

Merged
merged 23 commits into from
Feb 7, 2024
Merged

fv3-schema-update #405

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
c14f5ce
Updates from UW-449 branch
maddenp-noaa Jan 31, 2024
01a81e7
lfs -> lsf
maddenp-noaa Feb 1, 2024
ed7c77b
Make mpicmd required in schema and code
maddenp-noaa Feb 1, 2024
90ddc22
Update schema
maddenp-noaa Feb 6, 2024
8fa2d91
WIP
maddenp-noaa Feb 6, 2024
8547612
WIP
maddenp-noaa Feb 6, 2024
6e2f898
WIP
maddenp-noaa Feb 6, 2024
d6072f1
More schema tests
maddenp-noaa Feb 6, 2024
ca7aed4
WIP
maddenp-noaa Feb 6, 2024
b946c97
field_table is required in YAML
maddenp-noaa Feb 6, 2024
f4fab21
Move account to platform block
maddenp-noaa Feb 6, 2024
06582e1
Fix account lookup
maddenp-noaa Feb 6, 2024
419d282
Fix
maddenp-noaa Feb 6, 2024
045ce5a
Sort
maddenp-noaa Feb 6, 2024
608909a
WIP on file lists
maddenp-noaa Feb 6, 2024
a9e0efb
Only str->str mapping for filesToStage
maddenp-noaa Feb 6, 2024
753ad6b
More granular schema tests
maddenp-noaa Feb 6, 2024
32fb275
More granular schema tests
maddenp-noaa Feb 6, 2024
4be66c2
More granular schema tests
maddenp-noaa Feb 6, 2024
1cf6d48
More granular schema tests
maddenp-noaa Feb 7, 2024
144e5fb
Fix required top-level keys
maddenp-noaa Feb 7, 2024
fa7e5c4
Extra schema test
maddenp-noaa Feb 7, 2024
bda2eb2
Merge branch 'main' into fv3-schema-update
maddenp-noaa Feb 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
242 changes: 193 additions & 49 deletions src/uwtools/resources/FV3Forecast.jsonschema
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,38 +1,33 @@
{
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
"$defs": {
"updatable_config": {
"additionalProperties": false,
"properties": {
"properties": {
"base_file": {
"format": "uri",
"type": "string"
},
"update_values": {
"type": "object"
}
},
"required": [
"base_file"
],
"type": "object"
}
"filesToStage": {
"minProperties": 1,
"patternProperties": {
"^.*$": {
"oneOf": [
{
"type": "string"
},
{
"items": {
"type": "string"
},
"type": "array"
}
]
}
},
"type": "object"
}
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
},
"description": "This document is to validate user-defined FV3 forecast config files",
"properties": {
"forecast": {
"additionalProperties": false,
"description": "parameters of the forecast",
"properties": {
"cycle_dependent|static": {
"propertyNames": {
"type": "string"
},
"type": "object"
"cycle_dependent": {
"$ref": "#/$defs/filesToStage"
},
"diag_table": {
"format": "uri",
"type": "string"
},
"domain": {
Expand All @@ -43,88 +38,237 @@
"type": "string"
},
"executable": {
"format": "uri",
"type": "string"
},
"fd_ufs": {
"$ref": "#/$defs/updatable_config"
},
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
"field_table": {
"$ref": "#/$defs/updatable_config"
"additionalProperties": false,
"properties": {
"base_file": {
"type": "string"
},
"update_values": {
"minProperties": 1,
"patternProperties": {
"^.*$": {
"additionalProperties": false,
"properties": {
"longname": {
"type": "string"
},
"profile_type": {
"oneOf": [
{
"additionalProperties": false,
"properties": {
"name": {
"const": "fixed"
},
"surface_value": {
"type": "number"
}
},
"required": [
"name",
"surface_value"
],
"type": "object"
},
{
"additionalProperties": false,
"properties": {
"name": {
"const": "profile"
},
"surface_value": {
"type": "number"
},
"top_value": {
"type": "number"
}
},
"required": [
"name",
"surface_value",
"top_value"
],
"type": "object"
}
]
},
"units": {
"type": "string"
}
},
"required": [
"longname",
"profile_type",
"units"
],
"type": "object"
}
},
"type": "object"
}
},
"required": [
"base_file"
],
"type": "object"
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
},
"length": {
"minimum": 1,
"type": "integer"
},
"model_configure": {
"$ref": "#/$defs/updatable_config"
},
"mpicmd": {
"type": "string"
"additionalProperties": false,
"properties": {
"base_file": {
"type": "string"
},
"update_values": {
"minProperties": 1,
"patternProperties": {
"^.*$": {
"type": [
"boolean",
"number",
"string"
]
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
}
},
"type": "object"
}
},
"required": [
"base_file"
],
"type": "object"
},
"namelist": {
"$ref": "#/$defs/updatable_config"
"additionalProperties": false,
"properties": {
"base_file": {
"type": "string"
},
"update_values": {
"minProperties": 1,
"patternProperties": {
"^.*$": {
"minProperties": 1,
"patternProperties": {
"^.*$": {
"minProperties": 1,
"type": [
"array",
"boolean",
"number",
"string"
]
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
},
"type": "object"
},
"type": "object"
}
},
"type": "object"
}
},
"required": [
"base_file"
],
"type": "object"
},
"run_dir": {
"format": "uri",
"type": "string"
},
"ufs_configure": {
"format": "uri",
"type": "string"
"runtime_info": {
"additionalProperties": false,
"properties": {
"mpi_args": {
"items": {
"type": "string"
},
"type": "array"
},
"threads": {
"type": "integer"
}
},
"type": "object"
},
"static": {
"$ref": "#/$defs/filesToStage"
}
},
"required": [
"run_dir",
"runtime_info"
],
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
"type": "object"
},
"platform": {
"additionalProperties": false,
"properties": {
"required": [
"mpicmd"
],
"mpicmd": {
"type": "string"
},
"scheduler": {
"enum": [
"lsf",
"lfs",
"pbs",
"slurm"
],
"type": "string"
}
},
"required": [
"scheduler"
],
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
"type": "object"
},
"preprocessing": {
"additionalProperties": false,
"properties": {
"lateral_boundary_conditions": {
"additionalProperties": false,
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
"properties": {
"interval_hours": {
"default": 3,
"minimum": 1,
"type": "number"
"type": "integer"
},
"offset": {
"default": 0,
"minimum": 0,
"type": "number"
"type": "integer"
},
"output_file_path": {
"format": "uri",
"type": "string"
}
},
"required": [
"interval_hours",
"offset",
"output_file_path"
],
"type": "object"
}
},
"required": [
"lateral_boundary_conditions"
],
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
"type": "object"
},
"user": {
"additionalProperties": false,
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
"properties": {
"account": {
"type": "string"
}
},
"required": [
"account"
],
"type": "object"
}
},
"title": "FV3 Forecast config",
"type": "object"
}
21 changes: 20 additions & 1 deletion src/uwtools/tests/drivers/test_forecast.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from uwtools.drivers.driver import Driver
from uwtools.drivers.forecast import FV3Forecast
from uwtools.logging import log
from uwtools.tests.support import compare_files, fixture_path, logged
from uwtools.tests.support import compare_files, fixture_path, logged, validator
from uwtools.types import ExistAct


Expand Down Expand Up @@ -404,3 +404,22 @@ def test_FV3Forecast__run_via_local_execution(fv3_run_assets):
assert success is True
assert lines[0] == "Command:"
execute.assert_called_once_with(cmd=ANY, cwd=ANY, log_output=True)


# Schema tests
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved


def test_FV3Forecast_schema_filesToStage():
errors = validator("FV3Forecast.jsonschema", "$defs", "filesToStage")
# The input must be an dict:
assert "is not of type 'object'" in errors([])
# A str -> str dict is ok:
assert not errors({"file1": "/path/to/file1", "file2": "/path/to/file2"})
# A str -> List[str] dict is ok:
assert not errors({"dir": ["/path/to/file1", "/path/to/file2"]})
# An empty dict is not allowed:
assert "does not have enough properties" in errors({})
# Non-string values are not allowed:
assert "not valid" in errors({"file1": True})
# Non-string list elements are not allowed:
assert "not valid" in errors({"dir": [88]})
21 changes: 21 additions & 0 deletions src/uwtools/tests/support.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,14 @@
import re
from importlib import resources
from pathlib import Path
from typing import Callable

import yaml
from _pytest.logging import LogCaptureFixture

from uwtools.config.validator import _validation_errors
from uwtools.utils.file import resource_pathobj


def compare_files(path1: str, path2: str) -> bool:
"""
Expand Down Expand Up @@ -86,3 +91,19 @@ def regex_logged(caplog: LogCaptureFixture, msg: str) -> bool:
"""
pattern = re.compile(re.escape(msg))
return any(pattern.search(record.message) for record in caplog.records)


def validator(schema_fn: str, *args) -> Callable:
"""
Create a lambda that returns errors from validating a config input.

:param schema_fn: The schema filename, relative to package resources.
:param args: Keys leading to sub-schema to be used to validate eventual input.
:returns: A lambda that, when called with an input to test, returns a string (possibly empty)
containing the validation errors.
"""
with open(resource_pathobj(schema_fn), "r", encoding="utf-8") as f:
schema = yaml.safe_load(f)
for arg in args:
schema = {"$defs": schema["$defs"], **schema[arg]}
return lambda config: "\n".join(str(x) for x in _validation_errors(config, schema))
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
Loading
Loading