Add dependency-track-upload policy

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Added
 
+- Added policy `dependency_track_upload` to upload SBOM data from workflows to a dependency track instance.
 - Added operations `list-blueprints` and `approve-blueprints` to list and approve remediation PRs created for specific organizations.
 - Added support for teams.
 - Use asyncer to speed up retrieval of live settings. ([#209](https://github.com/eclipse-csi/otterdog/issues/209))

DEPENDENCIES

@@ -46,7 +46,7 @@ pypi/pypi/-/markdown-it-py/3.0.0
 pypi/pypi/-/markupsafe/3.0.2
 pypi/pypi/-/mdurl/0.1.2
 pypi/pypi/-/mintotp/0.3.0
-pypi/pypi/-/motor/3.6.1
+pypi/pypi/-/motor/3.7.0
 pypi/pypi/-/multidict/6.1.0
 pypi/pypi/-/odmantic/1.0.2
 pypi/pypi/-/playwright/1.49.1

docs/reference/policies/dependency-track-upload.md

@@ -0,0 +1,29 @@
+# Upload SBOM data to a dependency track instance
+
+This policy will upload SBOM data as generated by workflows to a dependency track instance.
+
+## Configuration
+
+- `type` - `dependency_track_upload`
+
+### Settings
+
+| Setting         | Necessity | Value type | Description                                                         |
+|-----------------|-----------|------------|---------------------------------------------------------------------|
+| artifact_name   | mandatory | string     | The artifact to look for in workflow runs that contains SBOM data   |
+| base_url        | mandatory | string     | The base url to the dependency track instance                       |
+| workflow_filter | mandatory | string     | Only consider workflows runs that reference the specified workflows |
+
+
+## Example
+
+``` yaml
+name: Upload sbom data to sbom.eclipse.org
+description: |-
+  This policy uploads generated sbom data to sbom.eclipse.org.
+type: dependency_track_upload
+config:
+  artifact_name: "eclipse-sbom-data"
+  base_url: "https://sbom.eclipse.org"
+  workflow_filter: ".*/store-sbom-data.yml.*"
+```

mkdocs.yml

@@ -72,6 +72,7 @@ nav:
            - Status Check: reference/organization/repository/status-check.md
     - Policies:
         - reference/policies/index.md
+        - Upload SBOM data to dependency track: reference/policies/dependency-track-upload.md
         - Usage of macOS large runners: reference/policies/macos-large-runners.md
     - Blueprints:
         - reference/blueprints/index.md

otterdog/providers/github/rest/action_client.py

@@ -1,5 +1,5 @@
 #  *******************************************************************************
-#  Copyright (c) 2023-2024 Eclipse Foundation and others.
+#  Copyright (c) 2023-2025 Eclipse Foundation and others.
 #  This program and the accompanying materials are made available
 #  under the terms of the Eclipse Public License 2.0
 #  which is available at http://www.eclipse.org/legal/epl-v20.html
@@ -24,8 +24,9 @@ async def get_workflows(self, org_id: str, repo: str) -> list[dict[str, Any]]:
         _logger.debug("retrieving workflows for repo '%s/%s'", org_id, repo)
 
         try:
-            result = await self.requester.request_json("GET", f"/repos/{org_id}/{repo}/actions/workflows")
-            return result["workflows"]
+            return await self.requester.request_paged_json(
+                "GET", f"/repos/{org_id}/{repo}/actions/workflows", entries_key="workflows"
+            )
         except GitHubException as ex:
             raise RuntimeError(f"failed retrieving workflows for '{org_id}/{repo}':\n{ex}") from ex
 
@@ -44,3 +45,25 @@ async def cancel_workflow_run(self, org_id: str, repo_name: str, run_id: str) ->
             raise RuntimeError(
                 f"failed cancelling workflow run #{run_id} in repo '{org_id}/{repo_name}'\n{status}: {body}"
             )
+
+    async def get_artifacts(self, org_id: str, repo: str, run_id: int) -> list[dict[str, Any]]:
+        _logger.debug("list artifacts for workflow run #%d in repo '%s/%s'", run_id, org_id, repo)
+
+        try:
+            return await self.requester.request_paged_json(
+                "GET", f"/repos/{org_id}/{repo}/actions/runs/{run_id}/artifacts", entries_key="artifacts"
+            )
+        except GitHubException as ex:
+            raise RuntimeError(f"failed retrieving artifacts for '{org_id}/{repo}':\n{ex}") from ex
+
+    async def download_artifact(self, file, org_id: str, repo_name: str, artifact_id: int) -> None:
+        _logger.debug("downloading workflow artifact in repo '%s/%s'", org_id, repo_name)
+
+        try:
+            async for data in self.requester.request_stream(
+                "GET", f"/repos/{org_id}/{repo_name}/actions/artifacts/{artifact_id}/zip"
+            ):
+                await file.write(data)
+
+        except GitHubException as ex:
+            raise RuntimeError(f"failed downloading workflow artifact from repo '{org_id}/{repo_name}':\n{ex}") from ex

otterdog/providers/github/rest/repo_client.py

@@ -5,6 +5,7 @@
 #  which is available at http://www.eclipse.org/legal/epl-v20.html
 #  SPDX-License-Identifier: EPL-2.0
 #  *******************************************************************************
+
 import asyncio
 import json
 import os

otterdog/providers/github/rest/requester.py

@@ -81,6 +81,7 @@ async def request_paged_json(
         url_path: str,
         data: dict[str, Any] | None = None,
         params: dict[str, str] | None = None,
+        entries_key: str | None = None,
     ) -> list[dict[str, Any]]:
         from urllib import parse
 
@@ -101,12 +102,14 @@ async def request_paged_json(
             self._check_response(url_path, status, body)
             response = json.loads(body)
 
+            entries = response[entries_key] if entries_key is not None else response
+
             if next_url is None:
                 query_params = None
             else:
                 query_params = {k: v[0] for k, v in parse.parse_qs(parse.urlparse(next_url).query).items()}
 
-            for item in response:
+            for item in entries:
                 result.append(item)
 
         return result

otterdog/webapp/config.py

@@ -1,5 +1,5 @@
 #  *******************************************************************************
-#  Copyright (c) 2023-2024 Eclipse Foundation and others.
+#  Copyright (c) 2023-2025 Eclipse Foundation and others.
 #  This program and the accompanying materials are made available
 #  under the terms of the Eclipse Public License 2.0
 #  which is available at http://www.eclipse.org/legal/epl-v20.html
@@ -54,6 +54,8 @@ class AppConfig:
     GITHUB_APP_PRIVATE_KEY = config("GITHUB_APP_PRIVATE_KEY")
 
     PROJECTS_BASE_URL = config("PROJECTS_BASE_URL", default="https://projects.eclipse.org/projects/")
+    DEPENDENCY_TRACK_URL = config("DEPENDENCY_TRACK_URL", default="https://sbom.eclipse.org")
+    DEPENDENCY_TRACK_TOKEN = config("DEPENDENCY_TRACK_TOKEN")
 
 
 class ProductionConfig(AppConfig):

otterdog/webapp/policies/__init__.py

@@ -1,5 +1,5 @@
 #  *******************************************************************************
-#  Copyright (c) 2024 Eclipse Foundation and others.
+#  Copyright (c) 2024-2025 Eclipse Foundation and others.
 #  This program and the accompanying materials are made available
 #  under the terms of the Eclipse Public License 2.0
 #  which is available at http://www.eclipse.org/legal/epl-v20.html
@@ -22,6 +22,7 @@
 
 
 class PolicyType(str, Enum):
+    DEPENDENCY_TRACK_UPLOAD = "dependency_track_upload"
     MACOS_LARGE_RUNNERS_USAGE = "macos_large_runners"
 
 
@@ -92,6 +93,11 @@ def create_policy(
 
             return MacOSLargeRunnersUsagePolicy.model_validate(data)
 
+        case PolicyType.DEPENDENCY_TRACK_UPLOAD:
+            from otterdog.webapp.policies.dependency_track_upload import DependencyTrackUploadPolicy
+
+            return DependencyTrackUploadPolicy.model_validate(data)
+
         case _:
             raise RuntimeError(f"unknown policy type '{policy_type}'")
 

otterdog/webapp/policies/dependency_track_upload.py

@@ -0,0 +1,73 @@
+#  *******************************************************************************
+#  Copyright (c) 2024-2025 Eclipse Foundation and others.
+#  This program and the accompanying materials are made available
+#  under the terms of the Eclipse Public License 2.0
+#  which is available at http://www.eclipse.org/legal/epl-v20.html
+#  SPDX-License-Identifier: EPL-2.0
+#  *******************************************************************************
+
+import re
+from logging import getLogger
+from typing import Any, Self
+
+from quart import current_app
+
+from otterdog.utils import expect_type, unwrap
+from otterdog.webapp.webhook.github_models import WorkflowRun
+
+from . import Policy, PolicyType
+
+logger = getLogger(__name__)
+
+
+class DependencyTrackUploadPolicy(Policy):
+    """
+    A policy to upload sbom data from workflow runs to dependency track.
+    """
+
+    artifact_name: str
+    base_url: str
+    workflow_filter: str
+
+    @property
+    def type(self) -> PolicyType:
+        return PolicyType.DEPENDENCY_TRACK_UPLOAD
+
+    def matches_workflow(self, workflow: str) -> bool:
+        return re.search(self.workflow_filter, workflow) is not None
+
+    def merge(self, other: Self) -> Self:
+        copy = super().merge(other)
+        copy.workflow_filter = other.workflow_filter
+        return copy
+
+    async def evaluate(
+        self,
+        installation_id: int,
+        github_id: str,
+        repo_name: str | None = None,
+        payload: Any | None = None,
+    ) -> None:
+        repo_name = unwrap(repo_name)
+        payload = expect_type(payload, WorkflowRun)
+
+        if (
+            payload.conclusion == "success"
+            and payload.referenced_workflows is not None
+            and any(self.matches_workflow(x.path) for x in payload.referenced_workflows)
+        ):
+            from otterdog.webapp.tasks.policies.upload_sbom import UploadSBOMTask
+
+            logger.info(
+                f"workflow run {payload.name}/#{payload.id} in repo '{github_id}/{repo_name}' contains sbom data"
+            )
+
+            current_app.add_background_task(
+                UploadSBOMTask(
+                    installation_id,
+                    github_id,
+                    repo_name,
+                    self,
+                    payload.id,
+                )
+            )

otterdog/webapp/tasks/blueprints/__init__.py

@@ -1,5 +1,5 @@
 #  *******************************************************************************
-#  Copyright (c) 2024 Eclipse Foundation and others.
+#  Copyright (c) 2024-2025 Eclipse Foundation and others.
 #  This program and the accompanying materials are made available
 #  under the terms of the Eclipse Public License 2.0
 #  which is available at http://www.eclipse.org/legal/epl-v20.html
@@ -52,7 +52,7 @@ async def _pre_execute(self) -> bool:
         if blueprint_status is None:
             return True
 
-        match blueprint_status:
+        match blueprint_status.status:
             case BlueprintStatus.DISMISSED:
                 self.logger.debug(
                     f"Blueprint '{self.blueprint.id}' dismissed for " f"repo '{self.org_id}/{self.repo_name}', skipping"

otterdog/webapp/tasks/policies/__init__.py

@@ -0,0 +1,31 @@
+#  *******************************************************************************
+#  Copyright (c) 2025 Eclipse Foundation and others.
+#  This program and the accompanying materials are made available
+#  under the terms of the Eclipse Public License 2.0
+#  which is available at http://www.eclipse.org/legal/epl-v20.html
+#  SPDX-License-Identifier: EPL-2.0
+#  *******************************************************************************
+
+from __future__ import annotations
+
+from abc import ABC
+from typing import TYPE_CHECKING
+
+from otterdog.webapp.db.models import TaskModel
+from otterdog.webapp.tasks import InstallationBasedTask, Task
+
+if TYPE_CHECKING:
+    from otterdog.webapp.policies import Policy
+
+
+class PolicyTask(InstallationBasedTask, Task[bool], ABC):
+    org_id: str
+    repo_name: str
+    policy: Policy
+
+    def create_task_model(self):
+        return TaskModel(
+            type=type(self).__name__,
+            org_id=self.org_id,
+            repo_name=self.repo_name,
+        )