From ccc9807b428a9aa7364651d77067d71cf03e0a9a Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Sun, 24 Nov 2024 22:19:48 -0500 Subject: [PATCH] Feat: Add --bom-profile argument for limited comparison. Signed-off-by: Caroline Russell --- README.md | 5 ++++- custom_json_diff/cli.py | 23 ++++++++++++++------- custom_json_diff/lib/custom_diff.py | 22 ++++++++++++++++++++ custom_json_diff/lib/custom_diff_classes.py | 15 ++++++-------- custom_json_diff/lib/utils.py | 6 ++++-- pyproject.toml | 2 +- test/test_custom_json_diff.py | 17 ++++++++++++++- 7 files changed, 68 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 0eb43cf..de22ecc 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ options: preset-diff usage ``` -usage: custom-json-diff preset-diff [-h] [--allow-new-versions] [--allow-new-data] [--type PRESET_TYPE] [-r REPORT_TEMPLATE] [--include-extra INCLUDE] +usage: custom-json-diff preset-diff [-h] [--allow-new-versions] [--allow-new-data] [--type PRESET_TYPE] [-r REPORT_TEMPLATE] [--include-extra INCLUDE] [--include-empty] [--bom-profile BOM_PROFILE] options: -h, --help show this help message and exit @@ -50,6 +50,9 @@ options: Jinja2 template to use for report generation. --include-extra INCLUDE BOM only - include properties/evidence/licenses/hashes/externalReferences (list which with comma, no space, inbetween). + --include-empty, -e Include keys with empty values in summary. + --bom-profile BOM_PROFILE, -b BOM_PROFILE + Beta feature. Options: gn, gnv, nv -> only compare bom group/name/version. ``` ## Preset Diffs diff --git a/custom_json_diff/cli.py b/custom_json_diff/cli.py index f0794b0..2fbb9ad 100644 --- a/custom_json_diff/cli.py +++ b/custom_json_diff/cli.py @@ -52,9 +52,9 @@ def build_args() -> argparse.Namespace: dest="config" ) subparsers = parser.add_subparsers(help="subcommand help") - parser_pc_diff = subparsers.add_parser("preset-diff", help="Compare CycloneDX BOMs or Oasis CSAFs") - parser_pc_diff.set_defaults(preset_type="") - parser_pc_diff.add_argument( + parser_ps_diff = subparsers.add_parser("preset-diff", help="Compare CycloneDX BOMs or Oasis CSAFs") + parser_ps_diff.set_defaults(preset_type="") + parser_ps_diff.add_argument( "--allow-new-versions", "-anv", action="store_true", @@ -62,7 +62,7 @@ def build_args() -> argparse.Namespace: dest="allow_new_versions", default=False, ) - parser_pc_diff.add_argument( + parser_ps_diff.add_argument( "--allow-new-data", "-and", action="store_true", @@ -70,13 +70,13 @@ def build_args() -> argparse.Namespace: dest="allow_new_data", default=False, ) - parser_pc_diff.add_argument( + parser_ps_diff.add_argument( "--type", action="store", help="Either bom or csaf", dest="preset_type", ) - parser_pc_diff.add_argument( + parser_ps_diff.add_argument( "-r", "--report-template", action="store", @@ -84,13 +84,13 @@ def build_args() -> argparse.Namespace: dest="report_template", default="", ) - parser_pc_diff.add_argument( + parser_ps_diff.add_argument( "--include-extra", action="store", help="BOM only - include properties/evidence/licenses/hashes/externalReferences (list which with comma, no space, inbetween).", dest="include", ) - parser_pc_diff.add_argument( + parser_ps_diff.add_argument( "--include-empty", "-e", action="store_true", @@ -98,6 +98,11 @@ def build_args() -> argparse.Namespace: dest="include_empty", help="Include keys with empty values in summary.", ) + parser_ps_diff.add_argument( + "--bom-profile", + "-b", + help="Beta feature. Options: gn, gnv, nv -> only compare bom group/name/version." + ) parser.add_argument( "-x", "--exclude", @@ -124,6 +129,8 @@ def main(): preset_type = args.preset_type.lower() if preset_type and preset_type not in ("bom", "csaf"): raise ValueError("Preconfigured type must be either bom or csaf.") + if args.bom_profile and args.bom_profile not in ("gn", "gnv", "nv"): + raise ValueError("BOM profile must be either gn, gnv, or nv.") options = Options( allow_new_versions=args.allow_new_versions, allow_new_data=args.allow_new_data, diff --git a/custom_json_diff/lib/custom_diff.py b/custom_json_diff/lib/custom_diff.py index 95c7a00..6bc2339 100644 --- a/custom_json_diff/lib/custom_diff.py +++ b/custom_json_diff/lib/custom_diff.py @@ -71,10 +71,32 @@ def compare_dicts(options: "Options") -> Tuple[int, "BomDicts|CsafDicts|FlatDict def filter_dict(data: Dict, options: "Options") -> FlatDicts: + if options.bom_profile: + match options.bom_profile: + case "gnv": + data = filter_on_bom_profile(data, {"group", "name", "version"}) + case "gn": + data = filter_on_bom_profile(data, {"group", "name"}) + case "nv": + data = filter_on_bom_profile(data, {"name", "version"}) data = flatten(sort_dict_lists(data, options.sort_keys)) return FlatDicts(data).filter_out_keys(options.exclude) +def filter_on_bom_profile(data: Dict, profile_fields: Set) -> Dict: + if not data.get("components"): + return data + new_components = [] + for comp in data["components"]: + ncomp = {} + for key, value in comp.items(): + if key in profile_fields: + ncomp[key] = value + new_components.append(ncomp) + data["components"] = new_components + return data + + def generate_counts(data: Dict) -> Dict: return {"libraries": len(data.get("components", {}).get("libraries", [])), "frameworks": len(data.get("components", {}).get("frameworks", [])), diff --git a/custom_json_diff/lib/custom_diff_classes.py b/custom_json_diff/lib/custom_diff_classes.py index 5a53714..ef6b6c9 100644 --- a/custom_json_diff/lib/custom_diff_classes.py +++ b/custom_json_diff/lib/custom_diff_classes.py @@ -46,6 +46,7 @@ class Options: # type: ignore svc_keys: List = field(default_factory=list) doc_num: int = 1 include_empty: bool = False + bom_profile: str = "" def __post_init__(self): if self.config: @@ -60,9 +61,9 @@ def __post_init__(self): self.include = toml_data.get("settings", {}).get("include_extra", []) self.include_empty = toml_data.get("settings", {}).get("include_empty", False) if self.preconfig_type == "bom": - tmp_exclude, tmp_bom_key_fields, tmp_service_key_fields, self.do_advanced = ( - get_cdxgen_excludes(self.include, self.allow_new_versions, self.allow_new_data)) - self.comp_keys.extend(tmp_bom_key_fields) + tmp_exclude, tmp_service_key_fields, self.do_advanced = ( + get_cdxgen_excludes(self.include, self.allow_new_data)) + # self.comp_keys.extend(tmp_bom_key_fields) self.svc_keys.extend(tmp_service_key_fields) self.exclude.extend(tmp_exclude) self.sort_keys.extend(["purl", "bom-ref", "content", "cve", "id", "url", "text", "ref", "name", "value", "location"]) @@ -71,6 +72,7 @@ def __post_init__(self): self.sort_keys.extend(["text", "title", "product_id", "url"]) self.exclude = list(set(self.exclude)) self.include = list(set(self.include)) + # deprecated self.comp_keys = list(set(self.comp_keys)) self.svc_keys = list(set(self.svc_keys)) self.sort_keys = list(set(self.sort_keys)) @@ -980,7 +982,7 @@ def create_search_key(key: str, value: str) -> str: return combined_key -def get_cdxgen_excludes(includes: List[str], allow_new_versions: bool, allow_new_data: bool) -> Tuple[List[str], List[str], List[str], bool]: +def get_cdxgen_excludes(includes: List[str], allow_new_data: bool) -> Tuple[List[str], List[str], bool]: excludes = {'metadata.timestamp': 'metadata.timestamp', 'serialNumber': 'serialNumber', 'metadata.tools.components.[].version': 'metadata.tools.components.[].version', 'metadata.tools.components.[].purl': 'metadata.tools.components.[].purl', @@ -990,17 +992,12 @@ def get_cdxgen_excludes(includes: List[str], allow_new_versions: bool, allow_new 'externalReferences': 'components.[].externalReferences', 'externalreferences': 'components.[].externalReferences'} if allow_new_data: - component_keys = [] service_keys = [] else: - component_keys = ['name', 'author', 'publisher', 'group', 'type', 'scope', 'description'] service_keys = ['name', 'authenticated', 'x-trust-boundary', 'endpoints'] - if not allow_new_versions: - component_keys.extend([i for i in ('version', 'purl', 'bom-ref', 'version') if i not in excludes]) return ( [v for k, v in excludes.items() if k not in includes], - [v for v in component_keys if v not in excludes], [v for v in service_keys if v not in excludes], allow_new_data, ) diff --git a/custom_json_diff/lib/utils.py b/custom_json_diff/lib/utils.py index 6eca48e..7387dc0 100644 --- a/custom_json_diff/lib/utils.py +++ b/custom_json_diff/lib/utils.py @@ -101,8 +101,10 @@ def export_html_report(outfile: str, diffs: Dict, options: "Options", status: in stats_summary: Dict | None = None) -> None: if options.report_template: template_file = options.report_template + elif options.bom_profile: + template_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "bom_diff_template_minimal.j2") else: - template_file = options.report_template or os.path.join(os.path.dirname(os.path.realpath(__file__)), f"{options.preconfig_type}_diff_template.j2") + template_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), f"{options.preconfig_type}_diff_template.j2") template = file_read(template_file) jinja_env = Environment(autoescape=True) jinja_tmpl = jinja_env.from_string(str(template)) @@ -112,7 +114,7 @@ def export_html_report(outfile: str, diffs: Dict, options: "Options", status: in else: report_result = render_csaf_template(diffs, jinja_tmpl, options, status) except TypeError: - logger.warning(f"Could not render html report for {options.file_1} and {options.file_2} BOM diff. Likely an expected key is missing.") + logger.warning(f"Could not render html report for {options.file_1} and {options.file_2} {options.preconfig_type} diff. Likely an expected key is missing.") return file_write(outfile, report_result, error_msg=f"Unable to generate HTML report at {outfile}.", success_msg=f"HTML report generated: {outfile}") diff --git a/pyproject.toml b/pyproject.toml index 11592b5..1bf28e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "custom-json-diff" -version = "2.1.2" +version = "2.1.3" description = "CycloneDx BOM and Oasis CSAF diffing and comparison tool." authors = [ { name = "Caroline Russell", email = "caroline@appthreat.dev" }, diff --git a/test/test_custom_json_diff.py b/test/test_custom_json_diff.py index ecb2f38..6865c59 100644 --- a/test/test_custom_json_diff.py +++ b/test/test_custom_json_diff.py @@ -3,7 +3,7 @@ import pytest from custom_json_diff.lib.custom_diff import ( - compare_dicts, get_bom_status, get_diff, json_to_class + compare_dicts, filter_on_bom_profile, get_bom_status, get_diff, json_to_class ) from custom_json_diff.lib.custom_diff_classes import Options @@ -90,3 +90,18 @@ def test_get_bom_status(): assert max(get_bom_status(diff_summary_1), get_bom_status(diff_summary_2)) == 2 diff_summary_1["services"] = [{"name": "test"}] assert max(get_bom_status(diff_summary_1), get_bom_status(diff_summary_2)) == 3 + + +def test_filter_on_bom_profile(): + data = {"components": [{"name": "component1", "version": "1.0", "group": "group1"}, + {"name": "component2", "version": "2.0"}]} + assert filter_on_bom_profile(data, {"name", "version"}) == {'components': [{'name': 'component1', 'version': '1.0'}, + {'name': 'component2', 'version': '2.0'}]} + data = {"components": [{"name": "component1", "version": "1.0", "group": "group1"}]} + assert filter_on_bom_profile(data, {"name", "group"}) == {"components": [{"name": "component1", "group": "group1"}]} + assert filter_on_bom_profile({"components": []}, {"name"}) == {"components": []} + assert filter_on_bom_profile({}, {"name"}) == {} + assert filter_on_bom_profile( {"components": []}, {"name"}) == {"components": []} + data = {"metadata": {"author": "test"}, + "components": [{"name": "component1", "version": "1.0"}]} + assert filter_on_bom_profile(data, {"name"}) == {"metadata": {"author": "test"}, "components": [{"name": "component1"}]}