From 5d991877d7f73a77d8de89998276772e007294c6 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Fri, 7 Jun 2024 10:58:48 -0400 Subject: [PATCH] Bugfixes, update html report summary. (#18) Signed-off-by: Caroline Russell --- custom_json_diff/bom_diff_template.j2 | 23 ++++--------- custom_json_diff/cli.py | 12 +++---- custom_json_diff/custom_diff.py | 45 ++++++++++++++++++------- custom_json_diff/custom_diff_classes.py | 23 +++++++------ pyproject.toml | 2 +- 5 files changed, 58 insertions(+), 47 deletions(-) diff --git a/custom_json_diff/bom_diff_template.j2 b/custom_json_diff/bom_diff_template.j2 index a7d8587..6cba002 100644 --- a/custom_json_diff/bom_diff_template.j2 +++ b/custom_json_diff/bom_diff_template.j2 @@ -32,22 +32,13 @@ Summary - {% if not comp_only %} - {% for item in stats %} - - {{ item[0] }} - {{ item[1] }} - - - {% endfor %} - {% endif %} - {% if comp_only %} + {% for item in stats %} - {{ stats[0][0] }} - {{ stats[0][1] }} + {{ item[0] }} + {{ item[1] }} - {% endif %} + {% endfor %} @@ -160,10 +151,10 @@ {% endfor %} {% endif %} - {% if diff_other_comp_1 or diff_other_comp_2 %} + {% if diff_other_1 or diff_other_2 %} other types - {% for item in diff_other_comp_1 %} + {% for item in diff_other_1 %}
{{ item['name'] }}@{{ item['version'] }}
    @@ -175,7 +166,7 @@
{% endfor %} - {% for item in diff_other_comp_2 %} + {% for item in diff_other_2 %}
{{ item['name'] }}@{{ item['version'] }}
    diff --git a/custom_json_diff/cli.py b/custom_json_diff/cli.py index f072780..5ea149d 100644 --- a/custom_json_diff/cli.py +++ b/custom_json_diff/cli.py @@ -100,18 +100,16 @@ def build_args() -> argparse.Namespace: def main(): args = build_args() - if args.exclude: - args.exclude = args.exclude.split(",") - if args.include: - args.include = args.include.split(",") + exclude = args.exclude.split(",") if args.exclude else [] + include = args.include.split(",") if args.include else [] options = Options( allow_new_versions=args.allow_new_versions, allow_new_data=args.allow_new_data, config=args.config, comp_only=args.components_only, bom_diff=args.bom_diff, - include=args.include, - exclude=args.exclude, + include=include, + exclude=exclude, file_1=args.input[0], file_2=args.input[1], output=args.output, @@ -123,7 +121,7 @@ def main(): result_summary = perform_bom_diff(j1, j2) else: result_summary = get_diff(j1, j2, options) - report_results(result, result_summary, j1, options) + report_results(result, result_summary, options, j1, j2) if __name__ == "__main__": diff --git a/custom_json_diff/custom_diff.py b/custom_json_diff/custom_diff.py index 28166c9..96429cf 100644 --- a/custom_json_diff/custom_diff.py +++ b/custom_json_diff/custom_diff.py @@ -12,11 +12,22 @@ from custom_json_diff.custom_diff_classes import BomDicts, FlatDicts, Options -def calculate_pcts(diff_stats: Dict, j1_counts: Dict) -> List[list[str]]: - return [ - [f"Common {key} found: ", f"{value}/{j1_counts[key]}"] - for key, value in diff_stats.items() +def calculate_pcts(diff_stats: Dict, j1: BomDicts, j2: BomDicts) -> List[list[str]]: + j1_counts = j1.generate_counts() + j2_counts = j2.generate_counts() + result = [ + [f"Common {key} matched: ", f"{value}"] + for key, value in diff_stats["common"].items() ] + result.extend([ + [f"BOM 1 {key} not matched: ", f"{j1_counts[key] - value}/{j1_counts[key]}"] + for key, value in diff_stats["common"].items() + ]) + result.extend([ + [f"BOM 2 {key} not matched: ", f"{j2_counts[key] - value}/{j2_counts[key]}"] + for key, value in diff_stats["common"].items() + ]) + return [i for i in result if i[1] not in ["0", "0/0"]] def check_regex(regex_keys: Set[re.Pattern], key: str) -> bool: @@ -33,7 +44,7 @@ def compare_dicts(options: Options) -> Tuple[int, FlatDicts | BomDicts, FlatDict return 1, json_1_data, json_2_data -def export_html_report(outfile: str, diffs: Dict, j1: BomDicts, options: Options) -> None: +def export_html_report(outfile: str, diffs: Dict, j1: BomDicts, j2: BomDicts, options: Options) -> None: if options.report_template: template_file = options.report_template else: @@ -50,7 +61,7 @@ def export_html_report(outfile: str, diffs: Dict, j1: BomDicts, options: Options diffs["diff_summary"][options.file_2].get("dependencies", []), purl_regex) diffs["common_summary"]["dependencies"] = parse_purls( diffs["common_summary"].get("dependencies", []), purl_regex) - stats_summary = calculate_pcts(generate_diff_counts(diffs), j1.generate_counts()) + stats_summary = calculate_pcts(generate_diff_counts(diffs, j1.options.file_2), j1, j2) report_result = jinja_tmpl.render( common_lib=diffs["common_summary"].get("components", {}).get("libraries", []), common_frameworks=diffs["common_summary"].get("components", {}).get("frameworks", []), @@ -91,14 +102,24 @@ def filter_dict(data: Dict, options: Options) -> FlatDicts: return FlatDicts(data).filter_out_keys(options.exclude) -def generate_diff_counts(diffs) -> Dict: - return {"components": len( +def generate_diff_counts(diffs, f2: str) -> Dict: + return {"common": {"components": len( diffs["common_summary"].get("components", {}).get("libraries", [])) + len( diffs["common_summary"].get("components", {}).get("frameworks", [])) + len( diffs["common_summary"].get("components", {}).get("applications", [])) + len( diffs["common_summary"].get("components", {}).get("other_types", [])), - "services": len(diffs["common_summary"].get("services", [])), - "dependencies": len(diffs["common_summary"].get("dependencies", []))} + "services": len(diffs["common_summary"].get("services", [])), + "dependencies": len(diffs["common_summary"].get("dependencies", []))}, + "diff": {"components": len( + diffs["diff_summary"].get(f2, {}).get("components", {}).get("libraries", + [])) + len( + diffs["diff_summary"].get(f2, {}).get("components", {}).get("frameworks", + [])) + len( + diffs["diff_summary"].get(f2, {}).get("components", {}).get("applications", + [])) + len( + diffs["diff_summary"].get(f2, {}).get("components", {}).get("other_types", [])), }, + "services": len(diffs["diff_summary"].get(f2, {}).get("services", [])), + "dependencies": len(diffs["diff_summary"].get(f2, {}).get("dependencies", []))} def get_diff(j1: FlatDicts, j2: FlatDicts, options: Options) -> Dict: @@ -153,7 +174,7 @@ def perform_bom_diff(bom_1: BomDicts, bom_2: BomDicts) -> Dict: return output -def report_results(status: int, diffs: Dict, j1: BomDicts | FlatDicts, options: Options) -> None: +def report_results(status: int, diffs: Dict, options: Options, j1: BomDicts | None = None, j2: BomDicts | None = None) -> None: if status == 0: print("No differences found.") else: @@ -161,7 +182,7 @@ def report_results(status: int, diffs: Dict, j1: BomDicts | FlatDicts, options: handle_results(options.output, diffs) if options.bom_diff and options.output: report_file = options.output.replace(".json", "") + ".html" - export_html_report(report_file, diffs, j1, options) # type: ignore + export_html_report(report_file, diffs, j1, j2, options) # type: ignore def sort_dict_lists(result: Dict, sort_keys: List[str]) -> Dict: diff --git a/custom_json_diff/custom_diff_classes.py b/custom_json_diff/custom_diff_classes.py index 49c5381..b07cf9c 100644 --- a/custom_json_diff/custom_diff_classes.py +++ b/custom_json_diff/custom_diff_classes.py @@ -9,6 +9,7 @@ import toml from json_flatten import unflatten # type: ignore + log = logging.getLogger(__name__) @@ -343,31 +344,31 @@ def create_search_key(key: str, value: str) -> str: def get_cdxgen_excludes(includes: List[str], comp_only: bool, allow_new_versions: bool, - allow_new_data: bool) -> Tuple[List[str], Set[str], Set[str], bool]: + allow_new_data: bool) -> Tuple[List[str], List[str], List[str], bool]: excludes = {'metadata.timestamp': 'metadata.timestamp', 'serialNumber': 'serialNumber', 'metadata.tools.components.[].version': 'metadata.tools.components.[].version', 'metadata.tools.components.[].purl': 'metadata.tools.components.[].purl', 'metadata.tools.components.[].bom-ref': 'metadata.tools.components.[].bom-ref', 'properties': 'components.[].properties', 'evidence': 'components.[].evidence', - 'licenses': 'components.[].licenses', 'hashes': 'components.[].hashes'} + 'licenses': 'components.[].licenses', 'hashes': 'components.[].hashes', + 'externalReferences': 'components.[].externalReferences', + 'externalreferences': 'components.[].externalReferences'} if comp_only: excludes |= {'services': 'services', 'dependencies': 'dependencies'} if allow_new_data: - component_keys = set() - service_keys = set() + component_keys = [] + service_keys = [] else: - component_keys = {'name', 'author', 'publisher', 'group', 'type', 'scope', 'description'} - service_keys = {'name', 'authenticated', 'x-trust-boundary', 'endpoints'} + component_keys = ['name', 'author', 'publisher', 'group', 'type', 'scope', 'description'] + service_keys = ['name', 'authenticated', 'x-trust-boundary', 'endpoints'] if not allow_new_versions: - component_keys.add('version') - component_keys.add('bom-ref') - component_keys.add('purl') + component_keys.extend([i for i in ('version', 'purl', 'bom-ref', 'version') if i not in excludes]) return ( [v for k, v in excludes.items() if k not in includes], - component_keys, - service_keys, + [v for v in component_keys if v not in excludes], + [v for v in service_keys if v not in excludes], allow_new_data, ) diff --git a/pyproject.toml b/pyproject.toml index 75cf67d..2b42a64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "custom-json-diff" -version = "1.2.0" +version = "1.2.1" description = "Custom JSON and CycloneDx BOM diffing and comparison tool." authors = [ { name = "Caroline Russell", email = "caroline@appthreat.dev" },