Fix/version bugfix (#19)

* Bugfix for multiple allow options. Signed-off-by: Caroline Russell <[email protected]> * Note if there's a difference in metadata. Signed-off-by: Caroline Russell <[email protected]> * Add some logging. Signed-off-by: Caroline Russell <[email protected]> * Strip leading "v" for semver. Signed-off-by: Caroline Russell <[email protected]> * Update README.md for --debug, bump minor version. Signed-off-by: Caroline Russell <[email protected]> --------- Signed-off-by: Caroline Russell <[email protected]>
AppThreat · Jun 7, 2024 · 08277f3 · 08277f3
1 parent 5d99187
commit 08277f3
Show file tree

Hide file tree

Showing 7 changed files with 70 additions and 3,660 deletions.
diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@ ignore in the comparison and sorts all fields.
 Note, you may use `cjd` rather than `custom-json-diff` to run.
 
 ```
-usage: custom-json-diff [-h] [-v] -i INPUT INPUT [-o OUTPUT] [-c CONFIG] [-x EXCLUDE] {bom-diff} ...
+usage: custom-json-diff [-h] [-v] -i INPUT INPUT [-o OUTPUT] [-c CONFIG] [-x EXCLUDE] [--debug] {bom-diff} ...
 
 positional arguments:
   {bom-diff}            subcommand help
@@ -32,6 +32,7 @@ options:
                         Import TOML configuration file (overrides commandline options).
   -x EXCLUDE, --exclude EXCLUDE
                         Exclude field(s) from comparison.
+  --debug               Print debug messages.
 
 ```
 

diff --git a/custom_json_diff/bom_diff_template.j2 b/custom_json_diff/bom_diff_template.j2
@@ -39,6 +39,16 @@
 
                 </tr>
             {% endfor %}
+            <tr>
+                <td>Metadata matched</td>
+                {% if metadata %}
+                    <td style="text-align: right">✖</td>
+                {% endif %}
+                {% if not metadata %}
+                    <td style="text-align: right">✔</td>
+                {% endif %}
+
+            </tr>
         </table>
 
     </div>
@@ -264,7 +274,7 @@
             </tr>
             {% if not (common_apps or common_frameworks or common_lib or common_other) %}
                 <tr>
-                    <td colspan="2" style="text-align: center">No commonalities found.</td>
+                    <td colspan="2" style="text-align: center">No commonalities.</td>
                 </tr>
             {% endif %}
             {% if common_apps %}

diff --git a/custom_json_diff/cli.py b/custom_json_diff/cli.py
@@ -1,4 +1,5 @@
 import argparse
+import logging
 
 from importlib.metadata import version
 
@@ -7,6 +8,9 @@
 from custom_json_diff.custom_diff_classes import Options
 
 
+logger = logging.getLogger(__name__)
+
+
 def build_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(prog="custom-json-diff")
     parser.add_argument(
@@ -94,12 +98,20 @@ def build_args() -> argparse.Namespace:
         help="Exclude field(s) from comparison.",
         dest="exclude",
     )
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="Print debug messages.",
+        dest="debug",
+    )
 
     return parser.parse_args()
 
 
 def main():
     args = build_args()
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
     exclude = args.exclude.split(",") if args.exclude else []
     include = args.include.split(",") if args.include else []
     options = Options(

diff --git a/custom_json_diff/custom_diff.py b/custom_json_diff/custom_diff.py
@@ -12,6 +12,9 @@
 from custom_json_diff.custom_diff_classes import BomDicts, FlatDicts, Options
 
 
+logger = logging.getLogger(__name__)
+
+
 def calculate_pcts(diff_stats: Dict, j1: BomDicts, j2: BomDicts) -> List[list[str]]:
     j1_counts = j1.generate_counts()
     j2_counts = j2.generate_counts()
@@ -27,7 +30,7 @@ def calculate_pcts(diff_stats: Dict, j1: BomDicts, j2: BomDicts) -> List[list[st
         [f"BOM 2 {key} not matched: ", f"{j2_counts[key] - value}/{j2_counts[key]}"]
         for key, value in diff_stats["common"].items()
     ])
-    return [i for i in result if i[1] not in ["0", "0/0"]]
+    return [i for i in result if not i[1].startswith("0")]
 
 
 def check_regex(regex_keys: Set[re.Pattern], key: str) -> bool:
@@ -62,6 +65,10 @@ def export_html_report(outfile: str, diffs: Dict, j1: BomDicts, j2: BomDicts, op
     diffs["common_summary"]["dependencies"] = parse_purls(
         diffs["common_summary"].get("dependencies", []), purl_regex)
     stats_summary = calculate_pcts(generate_diff_counts(diffs, j1.options.file_2), j1, j2)
+    metadata_results = bool(
+        diffs["diff_summary"][options.file_1].get("misc_data", {}) or
+        diffs["diff_summary"][options.file_2].get("misc_data", {})
+    )
     report_result = jinja_tmpl.render(
         common_lib=diffs["common_summary"].get("components", {}).get("libraries", []),
         common_frameworks=diffs["common_summary"].get("components", {}).get("frameworks", []),
@@ -84,17 +91,18 @@ def export_html_report(outfile: str, diffs: Dict, j1: BomDicts, j2: BomDicts, op
         bom_1=options.file_1,
         bom_2=options.file_2,
         stats=stats_summary,
-        comp_only=options.comp_only
+        comp_only=options.comp_only,
+        metadata=metadata_results,
     )
     with open(outfile, "w", encoding="utf-8") as f:
         f.write(report_result)
-    print(f"HTML report generated: {outfile}")
+    logger.debug(f"HTML report generated: {outfile}")
 
 
 def export_results(outfile: str, diffs: Dict) -> None:
     with open(outfile, "w", encoding="utf-8") as f:
         f.write(json.dumps(diffs, indent=2))
-    print(f"JSON report generated: {outfile}")
+    logger.debug(f"JSON report generated: {outfile}")
 
 
 def filter_dict(data: Dict, options: Options) -> FlatDicts:
@@ -135,8 +143,6 @@ def get_sort_key(data: Dict, sort_keys: List[str]) -> str | bool:
 def handle_results(outfile: str, diffs: Dict) -> None:
     if outfile:
         export_results(outfile, diffs)
-    if not outfile:
-        print(json.dumps(diffs, indent=2))
 
 
 def load_json(json_file: str, options: Options) -> FlatDicts | BomDicts:
@@ -145,10 +151,10 @@ def load_json(json_file: str, options: Options) -> FlatDicts | BomDicts:
             data = json.load(f)
             data = json.loads(json.dumps(data, sort_keys=True))
     except FileNotFoundError:
-        logging.error("File not found: %s", json_file)
+        logger.error("File not found: %s", json_file)
         sys.exit(1)
     except json.JSONDecodeError:
-        logging.error("Invalid JSON: %s", json_file)
+        logger.error("Invalid JSON: %s", json_file)
         sys.exit(1)
     if options.bom_diff:
         data = sort_dict_lists(data, ["url", "content", "ref", "name", "value"])
@@ -180,7 +186,10 @@ def report_results(status: int, diffs: Dict, options: Options, j1: BomDicts | No
     else:
         print("Differences found.")
         handle_results(options.output, diffs)
-    if options.bom_diff and options.output:
+    if not options.output:
+        logger.warning("No output file specified. No reports generated.")
+        return
+    elif options.bom_diff:
         report_file = options.output.replace(".json", "") + ".html"
         export_html_report(report_file, diffs, j1, j2, options)  # type: ignore
 
@@ -202,7 +211,7 @@ def sort_list(lst: List, sort_keys: List[str]) -> List:
     if isinstance(lst[0], dict):
         if sort_key := get_sort_key(lst[0], sort_keys):
             return sorted(lst, key=lambda x: x[sort_key])
-        logging.debug("No key(s) specified for sorting. Cannot sort list of dictionaries.")
+        logger.debug("No key(s) specified for sorting. Cannot sort list of dictionaries.")
         return lst
     if isinstance(lst[0], (str, int)):
         lst.sort()

diff --git a/custom_json_diff/custom_diff_classes.py b/custom_json_diff/custom_diff_classes.py
@@ -1,4 +1,3 @@
-import contextlib
 import logging
 import re
 import sys
@@ -10,7 +9,7 @@
 from json_flatten import unflatten  # type: ignore
 
 
-log = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
 
 
 class BomComponent:
@@ -32,13 +31,15 @@ def __init__(self, comp: Dict, options: "Options"):
         self.hashes = comp.get("hashes", [])
         self.scope = comp.get("scope", [])
         self.description = comp.get("description", "")
+        self.external_references = comp.get("externalReferences", [])
 
     def __eq__(self, other):
+        if self.options.allow_new_versions and self.options.allow_new_data:
+            return self._advanced_eq(other) and self._check_new_versions(other)
+        if self.options.allow_new_versions:
+            return self._check_new_versions(other) and self._check_list_eq(other) and self.search_key == other.search_key
         if self.options.allow_new_data:
             return self._advanced_eq(other)
-        if self.options.allow_new_versions:
-            return self._check_new_versions(other)
-
         else:
             return self.search_key == other.search_key and self._check_list_eq(other)
 
@@ -60,10 +61,8 @@ def _check_list_eq(self, other):
 
     def _check_new_versions(self, other):
         if self.options.bom_num == 1:
-            return (self.search_key == other.search_key and self.version <= other.version and
-                    self._check_list_eq(other))
-        return (self.search_key == other.search_key and self.version >= other.version and
-                self._check_list_eq(other))
+            return self.version <= other.version
+        return self.version >= other.version
 
 
 class BomService:
@@ -304,28 +303,29 @@ def check_for_empty_eq(bom_1: BomComponent, bom_2: BomComponent) -> bool:
         return False
     if bom_1.publisher and bom_1.publisher != bom_2.publisher:
         return False
-    if bom_1.bom_ref and bom_1.bom_ref != bom_2.bom_ref:
-        return False
-    if bom_1.purl and bom_1.purl != bom_2.purl:
-        return False
     if bom_1.author and bom_1.author != bom_2.author:
         return False
     if bom_1.component_type and bom_1.component_type != bom_2.component_type:
         return False
-    if bom_1.options.allow_new_versions and bom_1.version and not bom_1.version >= bom_2.version:
-        return False
-    elif bom_1.version and bom_1.version != bom_2.version:
-        return False
     if bom_1.properties and bom_1.properties != bom_2.properties:
         return False
     if bom_1.evidence and bom_1.evidence != bom_2.evidence:
         return False
     if bom_1.licenses and bom_1.licenses != bom_2.licenses:
         return False
-    if bom_1.hashes and bom_1.hashes != bom_2.hashes:
-        return False
     if bom_1.scope and bom_1.scope != bom_2.scope:
         return False
+    if bom_1.external_references and bom_1.external_references != bom_2.external_references:
+        return False
+    if not bom_1.options.allow_new_versions:
+        if bom_1.version and bom_1.version != bom_2.version:
+            return False
+        if bom_1.bom_ref and bom_1.bom_ref != bom_2.bom_ref:
+            return False
+        if bom_1.purl and bom_1.purl != bom_2.purl:
+            return False
+        if bom_1.hashes and bom_1.hashes != bom_2.hashes:
+            return False
     return not bom_1.description or bom_1.description == bom_2.description
 
 
@@ -402,7 +402,7 @@ def import_config(config: str) -> Dict:
         try:
             toml_data = toml.load(f)
         except toml.TomlDecodeError:
-            logging.error("Invalid TOML.")
+            logger.error("Invalid TOML.")
             sys.exit(1)
     return toml_data
 
@@ -436,10 +436,10 @@ def parse_bom_dict(data: Dict, options: "Options") -> Tuple[List, List, List, Li
 
 
 def set_version(version: str, allow_new_versions: bool = False) -> semver.Version | str:
-    with contextlib.suppress(ValueError):
+    try:
         if allow_new_versions and version:
-            version = version.rstrip(".RELEASE")
+            version = version.lstrip("v").rstrip(".RELEASE")
             return semver.Version.parse(version, True)
-    # except ValueError:
-    #     log.warning("Could not parse version: %s", version)
+    except ValueError:
+        logger.debug("Could not parse version: %s", version)
     return version
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "custom-json-diff"
-version = "1.2.1"
+version = "1.3.0"
 description = "Custom JSON and CycloneDx BOM diffing and comparison tool."
 authors = [
   { name = "Caroline Russell", email = "[email protected]" },