Restore access to exclude to bom-diff, options bugfix. (#16)

Signed-off-by: Caroline Russell <[email protected]>
AppThreat · Jun 7, 2024 · 64f1386 · 64f1386
1 parent 3f21e02
commit 64f1386
Show file tree

Hide file tree

Showing 6 changed files with 3,752 additions and 85 deletions.
diff --git a/README.md b/README.md
@@ -13,12 +13,11 @@ ignore in the comparison and sorts all fields.
 ## CLI Usage
 
 ```
-usage: custom-json-diff [-h] [-v] -i INPUT INPUT [-o OUTPUT] [-c CONFIG] {bom-diff,json-diff} ...
+usage: custom-json-diff [-h] [-v] -i INPUT INPUT [-o OUTPUT] [-c CONFIG] [-x EXCLUDE [EXCLUDE ...]] {bom-diff} ...
 
 positional arguments:
-  {bom-diff,json-diff}  subcommand help
+  {bom-diff}            subcommand help
     bom-diff            compare CycloneDX BOMs
-    json-diff           compare two JSON files
 
 options:
   -h, --help            show this help message and exit
@@ -29,6 +28,8 @@ options:
                         Export JSON of differences to this file.
   -c CONFIG, --config-file CONFIG
                         Import TOML configuration file (overrides commandline options).
+  -x EXCLUDE [EXCLUDE ...], --exclude EXCLUDE [EXCLUDE ...]
+                        Exclude field(s) from comparison.
 
 ```
 
@@ -49,52 +50,6 @@ options:
                         Include properties/evidence/licenses/hashes in comparison (list which with space inbetween).
 ```
 
-json-diff usage
-```
-usage: cjd json-diff [-h] [-x EXCLUDE [EXCLUDE ...]]
-
-options:
-  -h, --help            show this help message and exit
-  -x EXCLUDE [EXCLUDE ...], --exclude EXCLUDE [EXCLUDE ...]
-                        Exclude field(s) from comparison.
-```
-
-## Specifying fields to exclude
-
-To exclude fields from comparison, use the `-x` or `--exclude` flag and specify the field name(s) 
-to exclude. The json will be flattened, so fields are specified using dot notation. For example:
-
-```json
-{
-    "field1": {
-        "field2": "value", 
-        "field3": [
-            {"a": "val1", "b": "val2"}, 
-            {"a": "val3", "b": "val4"}
-        ]
-    }
-}
-```
-
-is flattened to:
-```json
-{
-    "field1.field2": "value",
-    "field1.field3.[0].a": "val1",
-    "field1.field3.[0].b": "val2",
-    "field1.field3.[1].a": "val3",
-    "field1.field3.[1].b": "val4"
-}
-```
-
-To exclude field2, you would specify `field1.field2`. To exclude the `a` field in the array of 
-objects, you would specify `field1.field3.[].a` (do NOT include the array index, just do `[]`). 
-Multiple fields may be specified separated by a space. To better understand what your fields should
-be, check out json-flatten, which is the package used for this function.
-
->Note: In the context of BOM diffing, this list is only used for the metadata, not the components, 
-> services, or dependencies.
-
 ## Bom Diff
 
 The bom-diff command compares CycloneDx BOM components, services, and dependencies, as well as data 
@@ -106,6 +61,10 @@ for inclusion using `bom-diff --include-extra` and whichever field(s) you wish t
 - evidence
 - licenses
 - hashes
+- externalReferences
+
+You can use the -x --exclude switch before the bom-diff command to exclude any of these 
+(see [Specifying fields to exclude](#specifying-fields-to-exclude) ).
 
 Default included fields:
 
@@ -141,6 +100,39 @@ fields included by default.
 
 The --components-only option only analyzes components, not services, dependencies, or other data.
 
+## Specifying fields to exclude
+
+To exclude fields from comparison, use the `-x` or `--exclude` flag and specify the field name(s) 
+to exclude. The json will be flattened, so fields are specified using dot notation. For example:
+
+```json
+{
+    "field1": {
+        "field2": "value", 
+        "field3": [
+            {"a": "val1", "b": "val2"}, 
+            {"a": "val3", "b": "val4"}
+        ]
+    }
+}
+```
+
+is flattened to:
+```json
+{
+    "field1.field2": "value",
+    "field1.field3.[0].a": "val1",
+    "field1.field3.[0].b": "val2",
+    "field1.field3.[1].a": "val3",
+    "field1.field3.[1].b": "val4"
+}
+```
+
+To exclude field2, you would specify `field1.field2`. To exclude the `a` field in the array of 
+objects, you would specify `field1.field3.[].a` (do NOT include the array index, just do `[]`). 
+Multiple fields may be specified separated by a space. To better understand what your fields should
+be, check out json-flatten, which is the package used for this function.
+
 ## Sorting
 
 custom-json-diff will sort the imported JSON alphabetically. If your JSON document contains arrays 
@@ -158,6 +150,6 @@ sort_keys = ["url", "content", "ref", "name", "value"]
 allow_new_data = false
 allow_new_versions = true
 components_only = false
-include_extra = ["licenses", "properties", "hashes", "evidence"]
+include_extra = ["licenses", "properties", "hashes", "evidence", "externalReferences"]
 report_template = "custom_json_diff/bom_diff_template.j2"
 ```
diff --git a/custom_json_diff/cli.py b/custom_json_diff/cli.py
@@ -84,13 +84,12 @@ def build_args() -> argparse.Namespace:
     parser_bom_diff.add_argument(
         "--include-extra",
         action="store",
-        help="Include properties/evidence/licenses/hashes (list which with space inbetween).",
+        help="Include properties/evidence/licenses/hashes/externalReferences (list which with space inbetween).",
         default=[],
         dest="include",
         nargs="+",
     )
-    parser_json_diff = subparsers.add_parser("json-diff", help="compare two JSON files")
-    parser_json_diff.add_argument(
+    parser.add_argument(
         "-x",
         "--exclude",
         action="store",

diff --git a/custom_json_diff/custom_diff.py b/custom_json_diff/custom_diff.py
@@ -3,6 +3,7 @@
 import os
 import re
 import sys
+from copy import deepcopy
 from typing import Dict, List, Set, Tuple
 
 from jinja2 import Environment
@@ -23,8 +24,9 @@ def check_regex(regex_keys: Set[re.Pattern], key: str) -> bool:
 
 
 def compare_dicts(options: Options) -> Tuple[int, FlatDicts | BomDicts, FlatDicts | BomDicts]:
+    options2 = deepcopy(options)
     json_1_data = load_json(options.file_1, options)
-    json_2_data = load_json(options.file_2, options)
+    json_2_data = load_json(options.file_2, options2)
     if json_1_data == json_2_data:
         return 0, json_1_data, json_2_data
     else:

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "custom-json-diff"
-version = "1.0.0"
+version = "1.1.0"
 description = "Custom JSON and CycloneDx BOM diffing and comparison tool."
 authors = [
   { name = "Caroline Russell", email = "[email protected]" },

diff --git a/test/test_bom_diff.py b/test/test_bom_diff.py
@@ -6,30 +6,6 @@
 from custom_json_diff.custom_diff_classes import BomComponent, BomDicts, Options
 
 
-@pytest.fixture
-def java_1_bom():
-    options = Options(file_1="test/sbom-java.json", file_2="test/sbom-java2.json", bom_diff=True, include=["hashes", "evidence", "licenses", "properties"])
-    return load_json("test/sbom-java.json", options)
-
-
-@pytest.fixture
-def java_2_bom():
-    options = Options(file_1="test/sbom-java.json", file_2="test/sbom-java2.json", bom_diff=True, include=["hashes", "evidence", "licenses", "properties"])
-    return load_json("test/sbom-java2.json", options)
-
-
-@pytest.fixture
-def python_1_bom():
-    options = Options(file_1="test/sbom-python.json", file_2="test/sbom-python2.json", bom_diff=True, include=["licenses", "hashes", "evidence", "properties"])
-    return load_json("test/sbom-python.json", options)
-
-
-@pytest.fixture
-def python_2_bom():
-    options = Options(file_1="test/sbom-python.json", file_2="test/sbom-python2.json", bom_diff=True, include=["licenses", "hashes", "evidence", "properties"])
-    return load_json("test/sbom-python2.json", options)
-
-
 @pytest.fixture
 def options_1():
     return Options(file_1="test/sbom-java.json", file_2="test/sbom-java2.json", bom_diff=True, include=["hashes", "evidence", "licenses"])
@@ -44,6 +20,7 @@ def options_2():
 def options_3():
     return Options(file_1="bom_1.json", file_2="bom_2.json", bom_diff=True, allow_new_data=True)
 
+
 @pytest.fixture
 def bom_dicts_1():
     options = Options(file_1="bom_1.json", file_2="bom_2.json",
@@ -185,22 +162,93 @@ def bom_dicts_6():
     return bom_dicts
 
 
+@pytest.fixture
+def bom_dicts_7():
+    options = Options(file_1="bom_1.json", file_2="bom_2.json",
+                      bom_diff=True, allow_new_data=True, allow_new_versions=True)
+    bom_dicts = BomDicts(options, "bom_1.json", {}, {})
+    bom_dicts.components = [
+    BomComponent({
+      "bom-ref": "pkg:pypi/[email protected]",
+      "evidence": {
+        "identity": {
+          "confidence": 0.8,
+          "field": "purl",
+          "methods": [
+            {
+              "confidence": 0.8,
+              "technique": "manifest-analysis",
+              "value": "/home/runner/work/src_repos/python/django-goat/requirements_tests.txt"
+            }
+          ]
+        }
+      },
+      "group": "",
+      "name": "requests",
+      "properties": [
+        {
+          "name": "SrcFile",
+          "value": "/home/runner/work/src_repos/python/django-goat/requirements_tests.txt"
+        }
+      ],
+      "purl": "pkg:pypi/[email protected]",
+      "type": "library",
+      "version": "2.31.0"
+    }, options),
+]
+    return bom_dicts
+
+
+@pytest.fixture
+def bom_dicts_8():
+    options = Options(file_1="bom_1.json", file_2="bom_2.json",
+                      bom_diff=True, allow_new_data=True, allow_new_versions=True)
+    bom_dicts = BomDicts(options, "bom_2.json", {}, {})
+    bom_dicts.components = [
+    BomComponent({
+      "bom-ref": "pkg:pypi/[email protected]",
+      "evidence": {
+        "identity": {
+          "confidence": 0.8,
+          "field": "purl",
+          "methods": [
+            {
+              "confidence": 0.8,
+              "technique": "manifest-analysis",
+              "value": "/home/runner/work/src_repos/python/django-goat/requirements_tests.txt"
+            }
+          ]
+        }
+      },
+      "group": "",
+      "name": "requests",
+      "properties": [
+        {
+          "name": "SrcFile",
+          "value": "/home/runner/work/src_repos/python/django-goat/requirements_tests.txt"
+        }
+      ],
+      "purl": "pkg:pypi/[email protected]",
+      "type": "library",
+      "version": "2.32.3"
+    }, options),
+]
+    return bom_dicts
+
+
 @pytest.fixture
 def results():
     with open("test/test_data.json", "r", encoding="utf-8") as f:
         return json.load(f)
 
 
-def test_bom_diff(java_1_bom, java_2_bom, python_1_bom, python_2_bom, results, options_1, options_2):
+def test_bom_diff(results, options_1):
     result, j1, j2 = compare_dicts(options_1)
     result_summary = perform_bom_diff(j1, j2)
     assert result_summary == results["result_4"]
-    result, p1, p2 = compare_dicts(options_2)
-    result_summary2 = perform_bom_diff(p1, p2)
-    assert result_summary2 == results["result_5"]
 
 
-def test_bom_diff_options(results, bom_dicts_1, bom_dicts_2, bom_dicts_3, bom_dicts_4, bom_dicts_5, bom_dicts_6):
+def test_bom_diff_options(results, bom_dicts_1, bom_dicts_2, bom_dicts_3, bom_dicts_4, bom_dicts_5, bom_dicts_6, bom_dicts_7, bom_dicts_8):
     # test --allow-new-data
     result_summary = perform_bom_diff(bom_dicts_1, bom_dicts_2)
     assert result_summary == results["result_1"]
@@ -210,5 +258,9 @@ def test_bom_diff_options(results, bom_dicts_1, bom_dicts_2, bom_dicts_3, bom_di
     assert result_summary == results["result_2"]
 
     # test --allow-new-data and --allow-new-versions
+    result_summary = perform_bom_diff(bom_dicts_7, bom_dicts_8)
+    assert result_summary == results["result_5"]
+
     result_summary = perform_bom_diff(bom_dicts_5, bom_dicts_6)
     assert result_summary == results["result_3"]
+
diff --git a/test/test_data.json b/test/test_data.json