avoid "Restoring original project mainline states" during "project update"

ChangeLog.md

@@ -5,6 +5,11 @@
 
 # CaPyCli - Clearing Automation Python Command Line Tool for SW360
 
+## UNRELEASED
+
+* `project createbom` stores release relations (`CONTAINED`, `SIDE_BY_SIDE` etc.) as capycli:projectRelation
+* fix slowdown/crash in `project update` for large projects (#121) introduced in 2.7.0
+
 ## 2.7.0
 
 * fix for `bom findsources` for some JavaScript SBOMs.

capycli/common/capycli_bom_support.py

@@ -66,6 +66,7 @@ class CycloneDxSupport():
     CDX_PROP_CLEARING_STATE = "capycli:clearingState"
     CDX_PROP_CATEGORIES = "capycli:categories"
     CDX_PROP_PROJ_STATE = "capycli:projectClearingState"
+    CDX_PROP_PROJ_RELATION = "capycli:projectRelation"
     CDX_PROP_PROFILE = "siemens:profile"
 
     @staticmethod

capycli/project/create_bom.py

@@ -8,7 +8,7 @@
 
 import logging
 import sys
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Tuple
 
 from cyclonedx.model import ExternalReferenceType, HashAlgorithm
 from cyclonedx.model.bom import Bom
@@ -36,14 +36,13 @@ def get_external_id(self, name: str, release_details: Dict[str, Any]) -> str:
 
         return release_details["externalIds"].get(name, "")
 
-    def get_clearing_state(self, proj: Dict[str, Any], href: str) -> str:
-        """Returns the clearing state of the given component/release"""
+    def get_linked_state(self, proj: Dict[str, Any], href: str) -> Tuple[str, str]:
+        """Returns project mainline state and relation of the given release"""
         rel = proj["linkedReleases"]
         for key in rel:
             if key["release"] == href:
-                return key["mainlineState"]
-
-        return ""
+                return (key["mainlineState"], key["relation"])
+        return ("", "")
 
     def create_project_bom(self, project: Dict[str, Any]) -> List[Component]:
         bom: List[Component] = []
@@ -112,9 +111,10 @@ def create_project_bom(self, project: Dict[str, Any]) -> List[Component]:
                 print_red("    ERROR: unable to access project:" + repr(swex))
                 sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)
 
-            state = self.get_clearing_state(project, href)
-            if state:
-                CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_PROJ_STATE, state)
+            mainline_state, relation = self.get_linked_state(project, href)
+            if mainline_state and relation:
+                CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_PROJ_STATE, mainline_state)
+                CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_PROJ_RELATION, relation)
 
             sw360_id = self.client.get_id_from_href(href)
             CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_SW360ID, sw360_id)

capycli/project/create_project.py

@@ -33,9 +33,9 @@ def __init__(self, onlyUpdateProject: bool = False) -> None:
         self.onlyUpdateProject = onlyUpdateProject
         self.project_mainline_state: str = ""
 
-    def bom_to_release_list(self, sbom: Bom) -> List[str]:
-        """Creates a list with linked releases"""
-        linkedReleases = []
+    def bom_to_release_list(self, sbom: Bom) -> Dict[str, Any]:
+        """Creates a list with linked releases from the SBOM."""
+        linkedReleases: Dict[str, Any] = {}
 
         for cx_comp in sbom.components:
             rid = CycloneDxSupport.get_property_value(cx_comp, CycloneDxSupport.CDX_PROP_SW360ID)
@@ -45,31 +45,17 @@ def bom_to_release_list(self, sbom: Bom) -> List[str]:
                     + ", " + str(cx_comp.version))
                 continue
 
-            linkedReleases.append(rid)
-
-        return linkedReleases
-
-    def get_release_project_mainline_states(self, project: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]:
-        pms: List[Dict[str, Any]] = []
-        if not project:
-            return pms
-
-        if "linkedReleases" not in project:
-            return pms
+            linkedReleases[rid] = {}
 
-        for release in project["linkedReleases"]:  # NOT ["sw360:releases"]
-            pms_release = release.get("release", "")
-            if not pms_release:
-                continue
-            pms_state = release.get("mainlineState", "OPEN")
-            pms_relation = release.get("relation", "UNKNOWN")
-            pms_entry: Dict[str, Any] = {}
-            pms_entry["release"] = pms_release
-            pms_entry["mainlineState"] = pms_state
-            pms_entry["new_relation"] = pms_relation
-            pms.append(pms_entry)
+            mainlineState = CycloneDxSupport.get_property_value(cx_comp, CycloneDxSupport.CDX_PROP_PROJ_STATE)
+            if mainlineState:
+                linkedReleases[rid]["mainlineState"] = mainlineState
+            relation = CycloneDxSupport.get_property_value(cx_comp, CycloneDxSupport.CDX_PROP_PROJ_RELATION)
+            if relation:
+                # No typo. In project structure, it's "relation", while release update API uses "releaseRelation".
+                linkedReleases[rid]["releaseRelation"] = relation
 
-        return pms
+        return linkedReleases
 
     def update_project(self, project_id: str, project: Optional[Dict[str, Any]],
                        sbom: Bom, project_info: Dict[str, Any]) -> None:
@@ -79,7 +65,6 @@ def update_project(self, project_id: str, project: Optional[Dict[str, Any]],
             sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)
 
         data = self.bom_to_release_list(sbom)
-        pms = self.get_release_project_mainline_states(project)
 
         ignore_update_elements = ["name", "version"]
         # remove elements from list because they are handled separately
@@ -90,13 +75,17 @@ def update_project(self, project_id: str, project: Optional[Dict[str, Any]],
         try:
             print_text("  " + str(len(data)) + " releases in SBOM")
 
+            update_mode = self.onlyUpdateProject
             if project and "_embedded" in project and "sw360:releases" in project["_embedded"]:
                 print_text(
                     "  " + str(len(project["_embedded"]["sw360:releases"])) +
                     " releases in project before update")
+            else:
+                # Workaround for SW360 API bug: add releases will hang forever for empty projects
+                update_mode = False
 
             # note: type in sw360python, 1.4.0 is wrong - we are using the correct one!
-            result = self.client.update_project_releases(data, project_id, add=self.onlyUpdateProject)  # type: ignore
+            result = self.client.update_project_releases(data, project_id, add=update_mode)  # type: ignore
             if not result:
                 print_red("  Error updating project releases!")
             project = self.client.get_project(project_id)
@@ -114,20 +103,6 @@ def update_project(self, project_id: str, project: Optional[Dict[str, Any]],
                 if not result2:
                     print_red("  Error updating project!")
 
-            if pms and project:
-                print_text("  Restoring original project mainline states...")
-                for pms_entry in pms:
-                    update_release = False
-                    for r in project.get("linkedReleases", []):
-                        if r["release"] == pms_entry["release"]:
-                            update_release = True
-                            break
-
-                    if update_release:
-                        rid = self.client.get_id_from_href(pms_entry["release"])
-                        self.client.update_project_release_relationship(
-                            project_id, rid, pms_entry["mainlineState"], pms_entry["new_relation"], "")
-
         except SW360Error as swex:
             if swex.response is None:
                 print_red("  Unknown error: " + swex.message)
@@ -353,6 +328,7 @@ def run(self, args: Any) -> None:
             sys.exit(ResultCode.RESULT_FILE_NOT_FOUND)
 
         is_update_version = False
+        project = None
 
         if args.old_version and args.old_version != "":
             print_text("Project version will be updated with version: " + args.old_version)
@@ -413,7 +389,8 @@ def run(self, args: Any) -> None:
         if self.project_id:
             print("Updating project...")
             try:
-                project = self.client.get_project(self.project_id)
+                if project is None:
+                    project = self.client.get_project(self.project_id)
             except SW360Error as swex:
                 print_red("  ERROR: unable to access project:" + repr(swex))
                 sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)

tests/test_create_project.py

@@ -426,9 +426,14 @@ def test_project_update(self) -> None:
                 "visibility": "EVERYONE",
                 "_links": {
                     "self": {
-                        "href": TestBase.MYURL + "resource/api/projects/376576"
+                        "href": TestBase.MYURL + "resource/api/projects/007"
                     }
                 },
+                "linkedReleases": [{
+                    "release": "https://sw360.org/api/releases/3765276512",
+                    "mainlineState": "SPECIFIC",
+                    "relation": "UNKNOWN",
+                }],
                 "_embedded": {
                     "sw360:releases": [{
                         "name": "Angular 2.3.0",
@@ -545,6 +550,172 @@ def test_project_update(self) -> None:
         out = self.capture_stdout(sut.run, args)
         self.assertTrue(self.INPUTFILE in out)
 
+    @responses.activate
+    def test_project_copy_from(self) -> None:
+        sut = CreateProject()
+
+        # create argparse command line argument object
+        args = AppArguments()
+        args.command = []
+        args.command.append("project")
+        args.command.append("create")
+        args.sw360_token = TestBase.MYTOKEN
+        args.sw360_url = TestBase.MYURL
+        args.version = "2.0.0"
+        args.copy_from = "007"
+        args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", self.INPUTFILE)
+        args.verbose = True
+        args.debug = True
+
+        self.add_login_response()
+
+        new_project_json = {
+            "name": "CaPyCLI",
+            "version": "2.0.0",
+            "securityResponsibles": [],
+            "considerReleasesFromExternalList": False,
+            "projectType": "PRODUCT",
+            "visibility": "EVERYONE",
+            "_links": {
+                "self": {
+                    "href": TestBase.MYURL + "resource/api/projects/017"
+                }
+            },
+            "linkedReleases": [{
+                "release": "https://sw360.org/api/releases/3765276512",
+                "mainlineState": "SPECIFIC",
+                "relation": "UNKNOWN",
+            }],
+            "_embedded": {
+                "sw360:releases": [{
+                    "name": "Angular 2.3.0",
+                    "version": "2.3.0",
+                    "_links": {
+                        "self": {
+                            "href": "https://sw360.org/api/releases/3765276512"
+                        }
+                    }
+                }]
+            }
+        }
+
+        responses.add(
+            responses.POST,
+            url=self.MYURL + "resource/api/projects/duplicate/007",
+            json=new_project_json,
+            status=200,
+            content_type="application/json",
+            adding_headers={"Authorization": "Token " + self.MYTOKEN},
+        )
+
+        responses.add(
+            responses.GET,
+            url=self.MYURL + "resource/api/projects/017",
+            json=new_project_json,
+            status=200,
+            content_type="application/json",
+            adding_headers={"Authorization": "Token " + self.MYTOKEN},
+        )
+
+        # update project releases
+        responses.add(
+            responses.POST,
+            url=self.MYURL + "resource/api/projects/017/releases",
+            json={
+                # server returns complete project, here we only mock a part of it
+                "name": "CaPyCLI",
+                "veraion": "1.9.0",
+                "businessUnit": "SI",
+                "description": "CaPyCLI",
+                "linkedReleases": {
+                    "a5cae39f39db4e2587a7d760f59ce3d0": {
+                        "mainlineState": "SPECIFIC",
+                        "releaseRelation": "DYNAMICALLY_LINKED",
+                        "setMainlineState": True,
+                        "setReleaseRelation": True
+                    }
+                },
+                "_links": {
+                    "self": {
+                        "href": self.MYURL + "resource/api/projects/007"
+                    }
+                },
+                "_embedded": {
+                    "sw360:releases": [{
+                        "name": "Angular 2.3.0",
+                        "version": "2.3.0",
+                        "_links": {
+                            "self": {
+                                "href": "https://sw360.org/api/releases/3765276512"
+                            }
+                        }
+                    }]
+                }
+            },
+            match=[
+                update_release_matcher(["a5cae39f39db4e2587a7d760f59ce3d0"])
+            ],
+            status=201,
+            content_type="application/json",
+            adding_headers={"Authorization": "Token " + self.MYTOKEN},
+        )
+
+        # update project
+        responses.add(
+            responses.PATCH,
+            url=self.MYURL + "resource/api/projects/017",
+            json={
+                # server returns complete project, here we only mock a part of it
+                "name": "CaPyCLI",
+                "veraion": "1.9.0",
+                "businessUnit": "SI",
+                "description": "CaPyCLI",
+                "linkedReleases": {
+                    "a5cae39f39db4e2587a7d760f59ce3d0": {
+                        "mainlineState": "SPECIFIC",
+                        "releaseRelation": "DYNAMICALLY_LINKED",
+                        "setMainlineState": True,
+                        "setReleaseRelation": True
+                    }
+                },
+                "_links": {
+                    "self": {
+                        "href": self.MYURL + "resource/api/projects/007"
+                    }
+                },
+                "_embedded": {
+                    "sw360:releases": [{
+                        "name": "Angular 2.3.0",
+                        "version": "2.3.0",
+                        "_links": {
+                            "self": {
+                                "href": "https://sw360.org/api/releases/3765276512"
+                            }
+                        }
+                    }]
+                }
+            },
+            match=[
+                min_json_matcher(
+                    {
+                        "businessUnit": "SI",
+                        "description": "CaPyCLI",
+                        "ownerGroup": "SI",
+                        "projectOwner": "[email protected]",
+                        "projectResponsible": "[email protected]",
+                        "projectType": "INNER_SOURCE",
+                        "tag": "SI BP DB Demo",
+                        "visibility": "EVERYONE"
+                    })
+            ],
+            status=201,
+            content_type="application/json",
+            adding_headers={"Authorization": "Token " + self.MYTOKEN},
+        )
+
+        out = self.capture_stdout(sut.run, args)
+        self.assertTrue(self.INPUTFILE in out)
+
     @responses.activate
     def xtest_project_update_old_version(self) -> None:
         sut = CreateProject()