Revamp config compare for compatibility with more configs (#654)

docs/sections/user_guide/cli/drivers/upp/help.out

@@ -12,6 +12,8 @@ Optional arguments:
 
 Positional arguments:
   TASK
+    control_file
+      The GRIB control file
     files_copied
       Files copied for run
     files_linked

docs/sections/user_guide/cli/tools/config/compare-bad-extension-fix.out

@@ -1,4 +1,8 @@
-[2024-05-23T19:39:15]     INFO - a.txt
-[2024-05-23T19:39:15]     INFO + c.nml
-[2024-05-23T19:39:15]     INFO ---------------------------------------------------------------------
-[2024-05-23T19:39:15]     INFO values:       recipient:  - World + None
+[2024-11-14T23:27:44]     INFO - a.txt
+[2024-11-14T23:27:44]     INFO + c.nml
+[2024-11-14T23:27:44]     INFO ---------------------------------------------------------------------
+[2024-11-14T23:27:44]     INFO ↓ ? = info | -/+ = line unique to - or + file | blank = matching line
+[2024-11-14T23:27:44]     INFO ---------------------------------------------------------------------
+[2024-11-14T23:27:44]     INFO   values:
+[2024-11-14T23:27:44]     INFO     greeting: Hello
+[2024-11-14T23:27:44]     INFO -   recipient: World

docs/sections/user_guide/cli/tools/config/compare-diff.out

@@ -1,4 +1,8 @@
-[2024-05-23T19:39:16]     INFO - a.nml
-[2024-05-23T19:39:16]     INFO + c.nml
-[2024-05-23T19:39:16]     INFO ---------------------------------------------------------------------
-[2024-05-23T19:39:16]     INFO values:       recipient:  - World + None
+[2024-11-14T23:27:44]     INFO - a.nml
+[2024-11-14T23:27:44]     INFO + c.nml
+[2024-11-14T23:27:44]     INFO ---------------------------------------------------------------------
+[2024-11-14T23:27:44]     INFO ↓ ? = info | -/+ = line unique to - or + file | blank = matching line
+[2024-11-14T23:27:44]     INFO ---------------------------------------------------------------------
+[2024-11-14T23:27:44]     INFO   values:
+[2024-11-14T23:27:44]     INFO     greeting: Hello
+[2024-11-14T23:27:44]     INFO -   recipient: World

docs/sections/user_guide/cli/tools/config/compare-match.out

@@ -1,3 +1,2 @@
-[2024-05-23T19:39:15]     INFO - a.nml
-[2024-05-23T19:39:15]     INFO + b.nml
-[2024-05-23T19:39:15]     INFO ---------------------------------------------------------------------
+[2024-11-14T23:27:45]     INFO - a.nml
+[2024-11-14T23:27:45]     INFO + b.nml

docs/sections/user_guide/cli/tools/config/compare-verbose.out

@@ -1,5 +1,9 @@
-[2024-05-23T19:39:15]    DEBUG Command: uw config compare --file-1-path a.nml --file-2-path c.nml --verbose
-[2024-05-23T19:39:15]     INFO - a.nml
-[2024-05-23T19:39:15]     INFO + c.nml
-[2024-05-23T19:39:15]     INFO ---------------------------------------------------------------------
-[2024-05-23T19:39:15]     INFO values:       recipient:  - World + None
+[2024-11-14T23:27:45]    DEBUG Command: uw config compare --file-1-path a.nml --file-2-path c.nml --verbose
+[2024-11-14T23:27:45]     INFO - a.nml
+[2024-11-14T23:27:45]     INFO + c.nml
+[2024-11-14T23:27:45]     INFO ---------------------------------------------------------------------
+[2024-11-14T23:27:45]     INFO ↓ ? = info | -/+ = line unique to - or + file | blank = matching line
+[2024-11-14T23:27:45]     INFO ---------------------------------------------------------------------
+[2024-11-14T23:27:45]     INFO   values:
+[2024-11-14T23:27:45]     INFO     greeting: Hello
+[2024-11-14T23:27:45]     INFO -   recipient: World

src/uwtools/config/formats/base.py

@@ -1,8 +1,11 @@
+import difflib
 import os
 import re
 from abc import ABC, abstractmethod
 from collections import UserDict
 from copy import deepcopy
+from io import StringIO
+from math import inf
 from pathlib import Path
 from typing import Optional, Union
 
@@ -11,7 +14,7 @@
 from uwtools.config import jinja2
 from uwtools.config.support import INCLUDE_TAG, depth, log_and_error, yaml_to_str
 from uwtools.exceptions import UWConfigError
-from uwtools.logging import INDENT, log
+from uwtools.logging import INDENT, MSGWIDTH, log
 from uwtools.utils.file import str2path
 
 
@@ -76,6 +79,26 @@ def _characterize_values(self, values: dict, parent: str) -> tuple[list, list]:
                 complete.append(f"{INDENT}{parent}{key}")
         return complete, template
 
+    @staticmethod
+    def _compare_config_get_lines(d: dict) -> list[str]:
+        """
+        Returns a line-by-line YAML representation of the given dict.
+
+        :param d: A dict object.
+        """
+        sio = StringIO()
+        yaml.safe_dump(d, stream=sio, default_flow_style=False, indent=2, width=inf)
+        return sio.getvalue().splitlines(keepends=True)
+
+    @staticmethod
+    def _compare_config_log_header() -> None:
+        """
+        Log a visual header and description of diff markers.
+        """
+        log.info("-" * MSGWIDTH)
+        log.info("↓ ? = info | -/+ = line unique to - or + file | blank = matching line")
+        log.info("-" * MSGWIDTH)
+
     @property
     def _depth(self) -> int:
         """
@@ -158,7 +181,15 @@ def _parse_include(self, ref_dict: Optional[dict] = None) -> None:
 
     # Public methods
 
-    def compare_config(self, dict1: dict, dict2: Optional[dict] = None) -> bool:
+    @abstractmethod
+    def as_dict(self) -> dict:
+        """
+        Returns a pure dict version of the config.
+        """
+
+    def compare_config(
+        self, dict1: dict, dict2: Optional[dict] = None, header: Optional[bool] = True
+    ) -> bool:
         """
         Compare two config dictionaries.
 
@@ -168,33 +199,16 @@ def compare_config(self, dict1: dict, dict2: Optional[dict] = None) -> bool:
         :param dict2: The second dictionary (default: this config).
         :return: True if the configs are identical, False otherwise.
         """
-        dict2 = self.data if dict2 is None else dict2
-        diffs: dict = {}
-
-        for sect, items in dict2.items():
-            for key, val in items.items():
-                if val != dict1.get(sect, {}).get(key, ""):
-                    try:
-                        diffs[sect][key] = f" - {val} + {dict1.get(sect, {}).get(key)}"
-                    except KeyError:
-                        diffs[sect] = {}
-                        diffs[sect][key] = f" - {val} + {dict1.get(sect, {}).get(key)}"
-
-        for sect, items in dict1.items():
-            for key, val in items.items():
-                if val != dict2.get(sect, {}).get(key, "") and diffs.get(sect, {}).get(key) is None:
-                    try:
-                        diffs[sect][key] = f" - {dict2.get(sect, {}).get(key)} + {val}"
-                    except KeyError:
-                        diffs[sect] = {}
-                        diffs[sect][key] = f" - {dict2.get(sect, {}).get(key)} + {val}"
-
-        for sect, keys in diffs.items():
-            for key in keys:
-                msg = f"{sect}: {key:>15}: {keys[key]}"
-                log.info(msg)
-
-        return not diffs
+        dict2 = self.as_dict() if dict2 is None else dict2
+        lines1, lines2 = map(self._compare_config_get_lines, [dict1, dict2])
+        difflines = list(difflib.ndiff(lines2, lines1))
+        if all(line[0] == " " for line in difflines):  # i.e. no +/-/? lines
+            return True
+        if header:
+            self._compare_config_log_header()
+        for diffline in difflines:
+            log.info(diffline.rstrip())
+        return False
 
     def dereference(self, context: Optional[dict] = None) -> None:
         """

src/uwtools/config/formats/fieldtable.py

@@ -52,6 +52,12 @@ def _get_format() -> str:
 
     # Public methods
 
+    def as_dict(self) -> dict:
+        """
+        Returns a pure dict version of the config.
+        """
+        return self.data
+
     def dump(self, path: Optional[Path] = None) -> None:
         """
         Dump the config in Field Table format.

src/uwtools/config/formats/ini.py

@@ -72,6 +72,12 @@ def _load(self, config_file: Optional[Path]) -> dict:
 
     # Public methods
 
+    def as_dict(self) -> dict:
+        """
+        Returns a pure dict version of the config.
+        """
+        return self.data
+
     def dump(self, path: Optional[Path] = None) -> None:
         """
         Dump the config in INI format.

src/uwtools/config/formats/nml.py

@@ -7,6 +7,7 @@
 from f90nml import Namelist
 
 from uwtools.config.formats.base import Config
+from uwtools.config.support import from_od
 from uwtools.config.tools import config_check_depths_dump
 from uwtools.strings import FORMAT
 from uwtools.utils.file import readable, writable
@@ -76,6 +77,13 @@ def _load(self, config_file: Optional[Path]) -> dict:
 
     # Public methods
 
+    def as_dict(self) -> dict:
+        """
+        Returns a pure dict version of the config.
+        """
+        d = self.data
+        return from_od(d.todict()) if isinstance(d, Namelist) else d
+
     def dump(self, path: Optional[Path]) -> None:
         """
         Dump the config in Fortran namelist format.

src/uwtools/config/formats/sh.py

@@ -73,6 +73,12 @@ def _load(self, config_file: Optional[Path]) -> dict:
 
     # Public methods
 
+    def as_dict(self) -> dict:
+        """
+        Returns a pure dict version of the config.
+        """
+        return self.data
+
     def dump(self, path: Optional[Path]) -> None:
         """
         Dump the config as key=value lines.

src/uwtools/config/formats/yaml.py

@@ -168,6 +168,12 @@ def _yaml_loader(self) -> type[yaml.SafeLoader]:
 
     # Public methods
 
+    def as_dict(self) -> dict:
+        """
+        Returns a pure dict version of the config.
+        """
+        return self.data
+
     def dump(self, path: Optional[Path] = None) -> None:
         """
         Dump the config in YAML format.

src/uwtools/config/tools.py

@@ -9,7 +9,7 @@
 from uwtools.config.jinja2 import unrendered
 from uwtools.config.support import depth, format_to_config, log_and_error
 from uwtools.exceptions import UWConfigError, UWConfigRealizeError, UWError
-from uwtools.logging import MSGWIDTH, log
+from uwtools.logging import log
 from uwtools.strings import FORMAT
 from uwtools.utils.file import get_file_format
 
@@ -34,8 +34,7 @@ def compare_configs(
     cfg_2: Config = format_to_config(config_2_format)(config_2_path)
     log.info("- %s", config_1_path)
     log.info("+ %s", config_2_path)
-    log.info("-" * MSGWIDTH)
-    return cfg_1.compare_config(cfg_2.data)
+    return cfg_1.compare_config(cfg_2.as_dict())
 
 
 def config_check_depths_dump(config_obj: Union[Config, dict], target_format: str) -> None:

src/uwtools/tests/config/formats/test_base.py

@@ -6,6 +6,7 @@
 import logging
 import os
 from datetime import datetime
+from textwrap import dedent
 from unittest.mock import patch
 
 import yaml
@@ -56,6 +57,9 @@ def _load(self, config_file):
         with readable(config_file) as f:
             return yaml.safe_load(f.read())
 
+    def as_dict(self):
+        return self.data
+
     def dump(self, path=None):
         pass
 
@@ -112,7 +116,7 @@ def test__parse_include(config):
     assert len(config["config"]) == 2
 
 
-@mark.parametrize("fmt", [FORMAT.ini, FORMAT.nml, FORMAT.yaml])
+@mark.parametrize("fmt", [FORMAT.nml, FORMAT.yaml])
 def test_compare_config(caplog, fmt, salad_base):
     """
     Compare two config objects.
@@ -129,15 +133,65 @@ def test_compare_config(caplog, fmt, salad_base):
     salad_base["salad"]["dressing"] = "italian"
     salad_base["salad"]["size"] = "large"
     del salad_base["salad"]["how_many"]
-    assert not cfgobj.compare_config(cfgobj, salad_base)
     assert not cfgobj.compare_config(salad_base)
     # Expect to see the following differences logged:
-    for msg in [
-        "salad:        how_many:  - 12 + None",
-        "salad:        dressing:  - balsamic + italian",
-        "salad:            size:  - None + large",
-    ]:
-        assert logged(caplog, msg)
+    expected = """
+    ---------------------------------------------------------------------
+    ↓ ? = info | -/+ = line unique to - or + file | blank = matching line
+    ---------------------------------------------------------------------
+      salad:
+        base: kale
+    -   dressing: balsamic
+    ?             ^  ^ ^^^
+    +   dressing: italian
+    ?             ^^  ^ ^
+        fruit: banana
+    -   how_many: 12
+    +   size: large
+        vegetable: tomato
+    """
+    for line in dedent(expected).strip("\n").split("\n"):
+        assert logged(caplog, line)
+
+
+def test_compare_config_ini(caplog, salad_base):
+    """
+    Compare two config objects.
+    """
+    log.setLevel(logging.INFO)
+    cfgobj = tools.format_to_config("ini")(fixture_path("simple.ini"))
+    salad_base["salad"]["how_many"] = "12"  # str "12" (not int 12) for ini
+    assert cfgobj.compare_config(salad_base) is True
+    # Expect no differences:
+    assert not caplog.records
+    caplog.clear()
+    # Create differences in base dict:
+    salad_base["salad"]["dressing"] = "italian"
+    salad_base["salad"]["size"] = "large"
+    del salad_base["salad"]["how_many"]
+    assert not cfgobj.compare_config(cfgobj.as_dict(), salad_base, header=False)
+    # Expect to see the following differences logged:
+    expected = """
+      salad:
+        base: kale
+    -   dressing: italian
+    ?             ^^   ^^
+    +   dressing: balsamic
+    ?             ^  +++ ^
+        fruit: banana
+    -   size: large
+    +   how_many: '12'
+        vegetable: tomato
+    """
+    for line in dedent(expected).strip("\n").split("\n"):
+        assert logged(caplog, line)
+    anomalous = """
+    ---------------------------------------------------------------------
+    ↓ ? = info | -/+ = line unique to - or + file | blank = matching line
+    ---------------------------------------------------------------------
+    """
+    for line in dedent(anomalous).strip("\n").split("\n"):
+        assert not logged(caplog, line)
 
 
 def test_dereference(tmp_path):