Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Do not wrap YAML lines #524

Merged
merged 15 commits into from
Jul 10, 2024
Original file line number Diff line number Diff line change
@@ -1,21 +1,21 @@
[2024-05-23T19:39:16] DEBUG Command: uw rocoto realize --config-file rocoto.yaml --verbose
[2024-05-23T19:39:16] DEBUG Dereferencing, current value:
[2024-05-23T19:39:16] DEBUG workflow:
[2024-05-23T19:39:16] DEBUG attrs:
[2024-05-23T19:39:16] DEBUG realtime: false
[2024-05-23T19:39:16] DEBUG scheduler: slurm
[2024-05-23T19:39:16] DEBUG cycledef:
[2024-05-23T19:39:16] DEBUG - attrs:
[2024-05-23T19:39:16] DEBUG group: howdy
[2024-05-23T19:39:16] DEBUG spec: 202209290000 202209300000 06:00:00
[2024-07-09T00:31:39] DEBUG Command: uw rocoto realize --config-file rocoto.yaml --verbose
[2024-07-09T00:31:39] DEBUG Dereferencing, current value:
[2024-07-09T00:31:39] DEBUG workflow:
[2024-07-09T00:31:39] DEBUG attrs:
[2024-07-09T00:31:39] DEBUG realtime: false
[2024-07-09T00:31:39] DEBUG scheduler: slurm
[2024-07-09T00:31:39] DEBUG cycledef:
[2024-07-09T00:31:39] DEBUG - attrs:
[2024-07-09T00:31:39] DEBUG group: howdy
[2024-07-09T00:31:39] DEBUG spec: 202209290000 202209300000 06:00:00
...
[2024-05-23T19:39:16] DEBUG account: '&ACCOUNT;'
[2024-05-23T19:39:16] DEBUG attrs:
[2024-05-23T19:39:16] DEBUG cycledefs: howdy
[2024-05-23T19:39:16] DEBUG command: echo hello $person
[2024-05-23T19:39:16] DEBUG envars:
[2024-05-23T19:39:16] DEBUG person: siri
[2024-05-23T19:39:16] DEBUG jobname: hello
[2024-05-23T19:39:16] DEBUG nodes: 1:ppn=1
[2024-05-23T19:39:16] DEBUG walltime: 00:01:00
[2024-05-23T19:39:16] INFO 0 Rocoto validation errors found
[2024-07-09T00:31:39] DEBUG attrs:
[2024-07-09T00:31:39] DEBUG cycledefs: howdy
[2024-07-09T00:31:39] DEBUG account: '&ACCOUNT;'
[2024-07-09T00:31:39] DEBUG command: echo hello $person
[2024-07-09T00:31:39] DEBUG jobname: hello
[2024-07-09T00:31:39] DEBUG nodes: 1:ppn=1
[2024-07-09T00:31:39] DEBUG walltime: 00:01:00
[2024-07-09T00:31:39] DEBUG envars:
[2024-07-09T00:31:39] DEBUG person: siri
[2024-07-09T00:31:39] INFO 0 Rocoto validation errors found
20 changes: 13 additions & 7 deletions src/uwtools/config/formats/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,13 @@
from abc import ABC, abstractmethod
from collections import UserDict
from copy import deepcopy
from io import StringIO
from pathlib import Path
from typing import Optional, Union

import yaml

from uwtools.config import jinja2
from uwtools.config.support import INCLUDE_TAG, depth, log_and_error
from uwtools.config.support import INCLUDE_TAG, depth, log_and_error, yaml_to_str
from uwtools.exceptions import UWConfigError
from uwtools.logging import INDENT, log

Expand Down Expand Up @@ -44,14 +43,21 @@ def __init__(self, config: Optional[Union[dict, Path]] = None) -> None:

def __repr__(self) -> str:
"""
Returns the YAML string representation of a Config object.
Returns the string representation of a Config object.
"""
s = StringIO()
yaml.dump(self.data, s)
return s.getvalue()
return self._dict_to_str(self.data)
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved

# Private methods

@classmethod
@abstractmethod
def _dict_to_str(cls, cfg: dict) -> str:
"""
Returns the string representation of the given dict.

:param cfg: A dict object.
"""

@abstractmethod
def _load(self, config_file: Optional[Path]) -> dict:
"""
Expand Down Expand Up @@ -165,7 +171,7 @@ def dereference(self, context: Optional[dict] = None) -> None:

def logstate(state: str) -> None:
log.debug("Dereferencing, %s value:", state)
for line in str(self).split("\n"):
for line in yaml_to_str(self.data).split("\n"):
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved
log.debug("%s%s", INDENT, line)

while True:
Expand Down
45 changes: 28 additions & 17 deletions src/uwtools/config/formats/fieldtable.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,31 @@ class FieldTableConfig(YAMLConfig):
an input YAML file.
"""

# Private methods

@classmethod
def _dict_to_str(cls, cfg: dict) -> str:
"""
Returns the field-table representation of the given dict.

:param cfg: A dict object.
"""
lines = []
for field, settings in cfg.items():
lines.append(f' "TRACER", "atmos_mod", "{field}"')
for key, value in settings.items():
if isinstance(value, dict):
method_string = f'{" ":7}"{key}", "{value.pop("name")}"'
# All control vars go into one set of quotes.
control_vars = [f"{method}={val}" for method, val in value.items()]
# Whitespace after the comma matters.
lines.append(f'{method_string}, "{", ".join(control_vars)}"')
else:
# Formatting of variable spacing dependent on key length.
lines.append(f'{" ":11}"{key}", "{value}"')
lines[-1] += " /"
return "\n".join(lines)
maddenp-noaa marked this conversation as resolved.
Show resolved Hide resolved

# Public methods

def dump(self, path: Optional[Path] = None) -> None:
Expand All @@ -22,8 +47,8 @@ def dump(self, path: Optional[Path] = None) -> None:
"""
self.dump_dict(self.data, path)

@staticmethod
def dump_dict(cfg: dict, path: Optional[Path] = None) -> None:
@classmethod
def dump_dict(cls, cfg: dict, path: Optional[Path] = None) -> None:
"""
Dumps a provided config dictionary in Field Table format.

Expand All @@ -45,22 +70,8 @@ def dump_dict(cfg: dict, path: Optional[Path] = None) -> None:
:param cfg: The in-memory config object to dump.
:param path: Path to dump config to.
"""
lines = []
for field, settings in cfg.items():
lines.append(f' "TRACER", "atmos_mod", "{field}"')
for key, value in settings.items():
if isinstance(value, dict):
method_string = f'{" ":7}"{key}", "{value.pop("name")}"'
# All control vars go into one set of quotes.
control_vars = [f"{method}={val}" for method, val in value.items()]
# Whitespace after the comma matters.
lines.append(f'{method_string}, "{", ".join(control_vars)}"')
else:
# Formatting of variable spacing dependent on key length.
lines.append(f'{" ":11}"{key}", "{value}"')
lines[-1] += " /"
with writable(path) as f:
print("\n".join(lines), file=f)
print(cls._dict_to_str(cfg), file=f)

@staticmethod
def get_depth_threshold() -> Optional[int]:
Expand Down
41 changes: 25 additions & 16 deletions src/uwtools/config/formats/ini.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,28 @@ def __init__(self, config: Union[dict, Optional[Path]] = None):

# Private methods

@classmethod
def _dict_to_str(cls, cfg: dict) -> str:
"""
Returns the INI representation of the given dict.

:param cfg: A dict object.
"""

# Configparser adds a newline after each section, presumably to create nice-looking output
# when an INI contains multiple sections. Unfortunately, it also adds a newline after the
# _final_ section, resulting in an anomalous trailing newline. To avoid this, write first to
# memory, then strip the trailing newline.

config_check_depths_dump(config_obj=cfg, target_format=FORMAT.ini)
parser = configparser.ConfigParser()
sio = StringIO()
parser.read_dict(cfg)
parser.write(sio)
s = sio.getvalue().strip()
sio.close()
return s

def _load(self, config_file: Optional[Path]) -> dict:
"""
Reads and parses an INI file.
Expand All @@ -46,31 +68,18 @@ def dump(self, path: Optional[Path] = None) -> None:

:param path: Path to dump config to.
"""
config_check_depths_dump(config_obj=self, target_format=FORMAT.ini)
self.dump_dict(self.data, path)

@staticmethod
def dump_dict(cfg: dict, path: Optional[Path] = None) -> None:
@classmethod
def dump_dict(cls, cfg: dict, path: Optional[Path] = None) -> None:
"""
Dumps a provided config dictionary in INI format.

:param cfg: The in-memory config object to dump.
:param path: Path to dump config to.
"""

# Configparser adds a newline after each section, presumably to create nice-looking output
# when an INI contains multiple sections. Unfortunately, it also adds a newline after the
# _final_ section, resulting in an anomalous trailing newline. To avoid this, write first to
# memory, then strip the trailing newline.

config_check_depths_dump(config_obj=cfg, target_format=FORMAT.ini)
parser = configparser.ConfigParser()
s = StringIO()
parser.read_dict(cfg)
parser.write(s)
with writable(path) as f:
print(s.getvalue().strip(), file=f)
s.close()
print(cls._dict_to_str(cfg), file=f)

@staticmethod
def get_depth_threshold() -> Optional[int]:
Expand Down
36 changes: 25 additions & 11 deletions src/uwtools/config/formats/nml.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from collections import OrderedDict
from io import StringIO
from pathlib import Path
from typing import Optional, Union

Expand Down Expand Up @@ -27,6 +28,27 @@ def __init__(self, config: Union[dict, Optional[Path]] = None) -> None:

# Private methods

@classmethod
def _dict_to_str(cls, cfg: dict) -> str:
"""
Returns the field-table representation of the given dict.

:param cfg: A dict object.
"""

def to_od(d):
return OrderedDict(
{key: to_od(val) if isinstance(val, dict) else val for key, val in d.items()}
)

config_check_depths_dump(config_obj=cfg, target_format=FORMAT.nml)
nml: Namelist = Namelist(to_od(cfg)) if not isinstance(cfg, Namelist) else cfg
sio = StringIO()
nml.write(sio, sort=False)
s = sio.getvalue()
sio.close()
return s.strip()

def _load(self, config_file: Optional[Path]) -> dict:
"""
Reads and parses a Fortran namelist file.
Expand All @@ -49,24 +71,16 @@ def dump(self, path: Optional[Path]) -> None:
"""
self.dump_dict(cfg=self.data, path=path)

@staticmethod
def dump_dict(cfg: Union[dict, Namelist], path: Optional[Path] = None) -> None:
@classmethod
def dump_dict(cls, cfg: Union[dict, Namelist], path: Optional[Path] = None) -> None:
"""
Dumps a provided config dictionary in Fortran namelist format.

:param cfg: The in-memory config object to dump.
:param path: Path to dump config to.
"""

def to_od(d):
return OrderedDict(
{key: to_od(val) if isinstance(val, dict) else val for key, val in d.items()}
)

config_check_depths_dump(config_obj=cfg, target_format=FORMAT.nml)
nml: Namelist = Namelist(to_od(cfg)) if not isinstance(cfg, Namelist) else cfg
with writable(path) as f:
nml.write(f, sort=False)
print(cls._dict_to_str(cfg), file=f)

@staticmethod
def get_depth_threshold() -> Optional[int]:
Expand Down
29 changes: 16 additions & 13 deletions src/uwtools/config/formats/sh.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import re
import shlex
from io import StringIO
from pathlib import Path
from typing import Optional, Union

Expand All @@ -27,6 +26,19 @@ def __init__(self, config: Union[dict, Optional[Path]] = None):

# Private methods

@classmethod
def _dict_to_str(cls, cfg: dict) -> str:
"""
Returns the field-table representation of the given dict.

:param cfg: A dict object.
"""
config_check_depths_dump(config_obj=cfg, target_format=FORMAT.sh)
lines = []
for key, value in cfg.items():
lines.append("%s=%s" % (key, shlex.quote(str(value))))
return "\n".join(lines)

def _load(self, config_file: Optional[Path]) -> dict:
"""
Reads and parses key=value lines from shell code.
Expand Down Expand Up @@ -58,25 +70,16 @@ def dump(self, path: Optional[Path]) -> None:
config_check_depths_dump(config_obj=self, target_format=FORMAT.sh)
self.dump_dict(self.data, path)

@staticmethod
def dump_dict(cfg: dict, path: Optional[Path] = None) -> None:
@classmethod
def dump_dict(cls, cfg: dict, path: Optional[Path] = None) -> None:
"""
Dumps a provided config dictionary in bash format.

:param cfg: The in-memory config object to dump.
:param path: Path to dump config to.
"""

# Write first to a StringIO object to avoid creating a partial file in case of problems
# rendering or quoting config values.

config_check_depths_dump(config_obj=cfg, target_format=FORMAT.sh)
s = StringIO()
for key, value in cfg.items():
print("%s=%s" % (key, shlex.quote(str(value))), file=s)
with writable(path) as f:
print(s.getvalue().strip(), file=f)
s.close()
print(cls._dict_to_str(cfg), file=f)

@staticmethod
def get_depth_threshold() -> Optional[int]:
Expand Down
27 changes: 18 additions & 9 deletions src/uwtools/config/formats/yaml.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,14 @@
from f90nml import Namelist # type: ignore

from uwtools.config.formats.base import Config
from uwtools.config.support import INCLUDE_TAG, UWYAMLConvert, UWYAMLRemove, from_od, log_and_error
from uwtools.config.support import (
INCLUDE_TAG,
UWYAMLConvert,
UWYAMLRemove,
from_od,
log_and_error,
yaml_to_str,
)
from uwtools.exceptions import UWConfigError
from uwtools.strings import FORMAT
from uwtools.utils.file import readable, writable
Expand Down Expand Up @@ -44,14 +51,17 @@ class YAMLConfig(Config):
Concrete class to handle YAML config files.
"""

def __repr__(self) -> str:
"""
The string representation of a YAMLConfig object.
# Private methods

@classmethod
def _dict_to_str(cls, cfg: dict) -> str:
"""
self._add_yaml_representers()
return yaml.dump(self.data, default_flow_style=False).strip()
Returns the YAML representation of the given dict.

# Private methods
:param cfg: The in-memory config object.
"""
cls._add_yaml_representers()
return yaml_to_str(cfg)

def _load(self, config_file: Optional[Path]) -> dict:
"""
Expand Down Expand Up @@ -124,9 +134,8 @@ def dump_dict(cls, cfg: dict, path: Optional[Path] = None) -> None:
:param cfg: The in-memory config object to dump.
:param path: Path to dump config to.
"""
cls._add_yaml_representers()
with writable(path) as f:
yaml.dump(cfg, f, sort_keys=False)
print(cls._dict_to_str(cfg), file=f)

@staticmethod
def get_depth_threshold() -> Optional[int]:
Expand Down
Loading
Loading