diff --git a/ChangeLog.md b/ChangeLog.md index a914cfc..394da76 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -5,6 +5,11 @@ # CaPyCli - Clearing Automation Python Command Line Tool for SW360 +## UNRELEASED + +* `project createbom` stores release relations (`CONTAINED`, `SIDE_BY_SIDE` etc.) as capycli:projectRelation +* fix slowdown/crash in `project update` for large projects (#121) introduced in 2.7.0 + ## 2.7.0 * fix for `bom findsources` for some JavaScript SBOMs. diff --git a/capycli/common/capycli_bom_support.py b/capycli/common/capycli_bom_support.py index ea3c1e8..952ccd2 100644 --- a/capycli/common/capycli_bom_support.py +++ b/capycli/common/capycli_bom_support.py @@ -66,6 +66,7 @@ class CycloneDxSupport(): CDX_PROP_CLEARING_STATE = "capycli:clearingState" CDX_PROP_CATEGORIES = "capycli:categories" CDX_PROP_PROJ_STATE = "capycli:projectClearingState" + CDX_PROP_PROJ_RELATION = "capycli:projectRelation" CDX_PROP_PROFILE = "siemens:profile" @staticmethod diff --git a/capycli/project/create_bom.py b/capycli/project/create_bom.py index 86d9aa7..54319fe 100644 --- a/capycli/project/create_bom.py +++ b/capycli/project/create_bom.py @@ -8,7 +8,7 @@ import logging import sys -from typing import Any, Dict, List +from typing import Any, Dict, List, Tuple from cyclonedx.model import ExternalReferenceType, HashAlgorithm from cyclonedx.model.bom import Bom @@ -36,14 +36,13 @@ def get_external_id(self, name: str, release_details: Dict[str, Any]) -> str: return release_details["externalIds"].get(name, "") - def get_clearing_state(self, proj: Dict[str, Any], href: str) -> str: - """Returns the clearing state of the given component/release""" + def get_linked_state(self, proj: Dict[str, Any], href: str) -> Tuple[str, str]: + """Returns project mainline state and relation of the given release""" rel = proj["linkedReleases"] for key in rel: if key["release"] == href: - return key["mainlineState"] - - return "" + return (key["mainlineState"], key["relation"]) + return ("", "") def create_project_bom(self, project: Dict[str, Any]) -> List[Component]: bom: List[Component] = [] @@ -112,9 +111,10 @@ def create_project_bom(self, project: Dict[str, Any]) -> List[Component]: print_red(" ERROR: unable to access project:" + repr(swex)) sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360) - state = self.get_clearing_state(project, href) - if state: - CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_PROJ_STATE, state) + mainline_state, relation = self.get_linked_state(project, href) + if mainline_state and relation: + CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_PROJ_STATE, mainline_state) + CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_PROJ_RELATION, relation) sw360_id = self.client.get_id_from_href(href) CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_SW360ID, sw360_id) diff --git a/capycli/project/create_project.py b/capycli/project/create_project.py index bf1b4da..3641366 100644 --- a/capycli/project/create_project.py +++ b/capycli/project/create_project.py @@ -33,9 +33,9 @@ def __init__(self, onlyUpdateProject: bool = False) -> None: self.onlyUpdateProject = onlyUpdateProject self.project_mainline_state: str = "" - def bom_to_release_list(self, sbom: Bom) -> List[str]: - """Creates a list with linked releases""" - linkedReleases = [] + def bom_to_release_list(self, sbom: Bom) -> Dict[str, Any]: + """Creates a list with linked releases from the SBOM.""" + linkedReleases: Dict[str, Any] = {} for cx_comp in sbom.components: rid = CycloneDxSupport.get_property_value(cx_comp, CycloneDxSupport.CDX_PROP_SW360ID) @@ -45,31 +45,17 @@ def bom_to_release_list(self, sbom: Bom) -> List[str]: + ", " + str(cx_comp.version)) continue - linkedReleases.append(rid) - - return linkedReleases - - def get_release_project_mainline_states(self, project: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]: - pms: List[Dict[str, Any]] = [] - if not project: - return pms - - if "linkedReleases" not in project: - return pms + linkedReleases[rid] = {} - for release in project["linkedReleases"]: # NOT ["sw360:releases"] - pms_release = release.get("release", "") - if not pms_release: - continue - pms_state = release.get("mainlineState", "OPEN") - pms_relation = release.get("relation", "UNKNOWN") - pms_entry: Dict[str, Any] = {} - pms_entry["release"] = pms_release - pms_entry["mainlineState"] = pms_state - pms_entry["new_relation"] = pms_relation - pms.append(pms_entry) + mainlineState = CycloneDxSupport.get_property_value(cx_comp, CycloneDxSupport.CDX_PROP_PROJ_STATE) + if mainlineState: + linkedReleases[rid]["mainlineState"] = mainlineState + relation = CycloneDxSupport.get_property_value(cx_comp, CycloneDxSupport.CDX_PROP_PROJ_RELATION) + if relation: + # No typo. In project structure, it's "relation", while release update API uses "releaseRelation". + linkedReleases[rid]["releaseRelation"] = relation - return pms + return linkedReleases def update_project(self, project_id: str, project: Optional[Dict[str, Any]], sbom: Bom, project_info: Dict[str, Any]) -> None: @@ -79,7 +65,6 @@ def update_project(self, project_id: str, project: Optional[Dict[str, Any]], sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360) data = self.bom_to_release_list(sbom) - pms = self.get_release_project_mainline_states(project) ignore_update_elements = ["name", "version"] # remove elements from list because they are handled separately @@ -90,13 +75,17 @@ def update_project(self, project_id: str, project: Optional[Dict[str, Any]], try: print_text(" " + str(len(data)) + " releases in SBOM") + update_mode = self.onlyUpdateProject if project and "_embedded" in project and "sw360:releases" in project["_embedded"]: print_text( " " + str(len(project["_embedded"]["sw360:releases"])) + " releases in project before update") + else: + # Workaround for SW360 API bug: add releases will hang forever for empty projects + update_mode = False # note: type in sw360python, 1.4.0 is wrong - we are using the correct one! - result = self.client.update_project_releases(data, project_id, add=self.onlyUpdateProject) # type: ignore + result = self.client.update_project_releases(data, project_id, add=update_mode) # type: ignore if not result: print_red(" Error updating project releases!") project = self.client.get_project(project_id) @@ -114,20 +103,6 @@ def update_project(self, project_id: str, project: Optional[Dict[str, Any]], if not result2: print_red(" Error updating project!") - if pms and project: - print_text(" Restoring original project mainline states...") - for pms_entry in pms: - update_release = False - for r in project.get("linkedReleases", []): - if r["release"] == pms_entry["release"]: - update_release = True - break - - if update_release: - rid = self.client.get_id_from_href(pms_entry["release"]) - self.client.update_project_release_relationship( - project_id, rid, pms_entry["mainlineState"], pms_entry["new_relation"], "") - except SW360Error as swex: if swex.response is None: print_red(" Unknown error: " + swex.message) @@ -353,6 +328,7 @@ def run(self, args: Any) -> None: sys.exit(ResultCode.RESULT_FILE_NOT_FOUND) is_update_version = False + project = None if args.old_version and args.old_version != "": print_text("Project version will be updated with version: " + args.old_version) @@ -413,7 +389,8 @@ def run(self, args: Any) -> None: if self.project_id: print("Updating project...") try: - project = self.client.get_project(self.project_id) + if project is None: + project = self.client.get_project(self.project_id) except SW360Error as swex: print_red(" ERROR: unable to access project:" + repr(swex)) sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360) diff --git a/tests/test_create_project.py b/tests/test_create_project.py index be6943c..fee828a 100644 --- a/tests/test_create_project.py +++ b/tests/test_create_project.py @@ -426,9 +426,14 @@ def test_project_update(self) -> None: "visibility": "EVERYONE", "_links": { "self": { - "href": TestBase.MYURL + "resource/api/projects/376576" + "href": TestBase.MYURL + "resource/api/projects/007" } }, + "linkedReleases": [{ + "release": "https://sw360.org/api/releases/3765276512", + "mainlineState": "SPECIFIC", + "relation": "UNKNOWN", + }], "_embedded": { "sw360:releases": [{ "name": "Angular 2.3.0", @@ -545,6 +550,172 @@ def test_project_update(self) -> None: out = self.capture_stdout(sut.run, args) self.assertTrue(self.INPUTFILE in out) + @responses.activate + def test_project_copy_from(self) -> None: + sut = CreateProject() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("project") + args.command.append("create") + args.sw360_token = TestBase.MYTOKEN + args.sw360_url = TestBase.MYURL + args.version = "2.0.0" + args.copy_from = "007" + args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", self.INPUTFILE) + args.verbose = True + args.debug = True + + self.add_login_response() + + new_project_json = { + "name": "CaPyCLI", + "version": "2.0.0", + "securityResponsibles": [], + "considerReleasesFromExternalList": False, + "projectType": "PRODUCT", + "visibility": "EVERYONE", + "_links": { + "self": { + "href": TestBase.MYURL + "resource/api/projects/017" + } + }, + "linkedReleases": [{ + "release": "https://sw360.org/api/releases/3765276512", + "mainlineState": "SPECIFIC", + "relation": "UNKNOWN", + }], + "_embedded": { + "sw360:releases": [{ + "name": "Angular 2.3.0", + "version": "2.3.0", + "_links": { + "self": { + "href": "https://sw360.org/api/releases/3765276512" + } + } + }] + } + } + + responses.add( + responses.POST, + url=self.MYURL + "resource/api/projects/duplicate/007", + json=new_project_json, + status=200, + content_type="application/json", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + responses.add( + responses.GET, + url=self.MYURL + "resource/api/projects/017", + json=new_project_json, + status=200, + content_type="application/json", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + # update project releases + responses.add( + responses.POST, + url=self.MYURL + "resource/api/projects/017/releases", + json={ + # server returns complete project, here we only mock a part of it + "name": "CaPyCLI", + "veraion": "1.9.0", + "businessUnit": "SI", + "description": "CaPyCLI", + "linkedReleases": { + "a5cae39f39db4e2587a7d760f59ce3d0": { + "mainlineState": "SPECIFIC", + "releaseRelation": "DYNAMICALLY_LINKED", + "setMainlineState": True, + "setReleaseRelation": True + } + }, + "_links": { + "self": { + "href": self.MYURL + "resource/api/projects/007" + } + }, + "_embedded": { + "sw360:releases": [{ + "name": "Angular 2.3.0", + "version": "2.3.0", + "_links": { + "self": { + "href": "https://sw360.org/api/releases/3765276512" + } + } + }] + } + }, + match=[ + update_release_matcher(["a5cae39f39db4e2587a7d760f59ce3d0"]) + ], + status=201, + content_type="application/json", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + # update project + responses.add( + responses.PATCH, + url=self.MYURL + "resource/api/projects/017", + json={ + # server returns complete project, here we only mock a part of it + "name": "CaPyCLI", + "veraion": "1.9.0", + "businessUnit": "SI", + "description": "CaPyCLI", + "linkedReleases": { + "a5cae39f39db4e2587a7d760f59ce3d0": { + "mainlineState": "SPECIFIC", + "releaseRelation": "DYNAMICALLY_LINKED", + "setMainlineState": True, + "setReleaseRelation": True + } + }, + "_links": { + "self": { + "href": self.MYURL + "resource/api/projects/007" + } + }, + "_embedded": { + "sw360:releases": [{ + "name": "Angular 2.3.0", + "version": "2.3.0", + "_links": { + "self": { + "href": "https://sw360.org/api/releases/3765276512" + } + } + }] + } + }, + match=[ + min_json_matcher( + { + "businessUnit": "SI", + "description": "CaPyCLI", + "ownerGroup": "SI", + "projectOwner": "thomas.graf@siemens.com", + "projectResponsible": "thomas.graf@siemens.com", + "projectType": "INNER_SOURCE", + "tag": "SI BP DB Demo", + "visibility": "EVERYONE" + }) + ], + status=201, + content_type="application/json", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + out = self.capture_stdout(sut.run, args) + self.assertTrue(self.INPUTFILE in out) + @responses.activate def xtest_project_update_old_version(self) -> None: sut = CreateProject()