From a347207baf6c1199fe4f4cf92b3ab48e5730a08f Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Mon, 27 Jan 2025 21:22:14 +0100 Subject: [PATCH 1/4] feat: add dependency track upload policy --- otterdog/webapp/policies/__init__.py | 8 ++- .../policies/dependency_track_upload.py | 53 +++++++++++++++++++ otterdog/webapp/webhook/__init__.py | 38 ++++++++++++- otterdog/webapp/webhook/github_models.py | 30 ++++++++++- otterdog/webapp/webhook/github_webhook.py | 39 +++++++------- 5 files changed, 146 insertions(+), 22 deletions(-) create mode 100644 otterdog/webapp/policies/dependency_track_upload.py diff --git a/otterdog/webapp/policies/__init__.py b/otterdog/webapp/policies/__init__.py index ca7a50a7..057ac256 100644 --- a/otterdog/webapp/policies/__init__.py +++ b/otterdog/webapp/policies/__init__.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2024 Eclipse Foundation and others. +# Copyright (c) 2024-2025 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html @@ -22,6 +22,7 @@ class PolicyType(str, Enum): + DEPENDENCY_TRACK_UPLOAD = "dependency_track_upload" MACOS_LARGE_RUNNERS_USAGE = "macos_large_runners" @@ -92,6 +93,11 @@ def create_policy( return MacOSLargeRunnersUsagePolicy.model_validate(data) + case PolicyType.DEPENDENCY_TRACK_UPLOAD: + from otterdog.webapp.policies.dependency_track_upload import DependencyTrackUploadPolicy + + return DependencyTrackUploadPolicy.model_validate(data) + case _: raise RuntimeError(f"unknown policy type '{policy_type}'") diff --git a/otterdog/webapp/policies/dependency_track_upload.py b/otterdog/webapp/policies/dependency_track_upload.py new file mode 100644 index 00000000..06a1984c --- /dev/null +++ b/otterdog/webapp/policies/dependency_track_upload.py @@ -0,0 +1,53 @@ +# ******************************************************************************* +# Copyright (c) 2024-2025 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +import re +from logging import getLogger +from typing import Any, Self + +from otterdog.utils import expect_type, unwrap +from otterdog.webapp.webhook.github_models import WorkflowRun + +from . import Policy, PolicyType + +logger = getLogger(__name__) + + +class DependencyTrackUploadPolicy(Policy): + """ + A policy to upload sbom data from workflow runs to dependency track. + """ + + workflow_filter: str + + @property + def type(self) -> PolicyType: + return PolicyType.DEPENDENCY_TRACK_UPLOAD + + def matches_workflow(self, workflow: str) -> bool: + return re.search(self.workflow_filter, workflow) is not None + + def merge(self, other: Self) -> Self: + copy = super().merge(other) + copy.workflow_filter = other.workflow_filter + return copy + + async def evaluate( + self, + installation_id: int, + github_id: str, + repo_name: str | None = None, + payload: Any | None = None, + ) -> None: + repo_name = unwrap(repo_name) + payload = expect_type(payload, WorkflowRun) + + if payload.referenced_workflows is not None and any( + self.matches_workflow(x.path) for x in payload.referenced_workflows + ): + logger.info(f"workflow run #{payload.id} in repo '{github_id}/{repo_name}' contains sbom data") diff --git a/otterdog/webapp/webhook/__init__.py b/otterdog/webapp/webhook/__init__.py index 44d17d2b..3dfca6af 100644 --- a/otterdog/webapp/webhook/__init__.py +++ b/otterdog/webapp/webhook/__init__.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2023-2024 Eclipse Foundation and others. +# Copyright (c) 2023-2025 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html @@ -51,6 +51,7 @@ PullRequestReviewEvent, PushEvent, WorkflowJobEvent, + WorkflowRunEvent, touched_by_commits, ) from .github_webhook import GitHubWebhook @@ -384,6 +385,41 @@ async def on_workflow_job_received(data): return success() +@webhook.hook("workflow_run") +async def on_workflow_run_received(data): + try: + event = WorkflowRunEvent.model_validate(data) + except ValidationError: + logger.error("failed to load workflow run event data", exc_info=True) + return success() + + if event.installation is None or event.organization is None: + return success() + + installation_id = event.installation.id + owner = event.organization.login + + if event.action in ["completed"]: + logger.debug(f"workflow run completed in repo: {event.repository.full_name}") + + from otterdog.webapp.db.service import find_policy + from otterdog.webapp.policies import PolicyType + from otterdog.webapp.policies.dependency_track_upload import ( + DependencyTrackUploadPolicy, + ) + + policy_model = await find_policy(owner, PolicyType.DEPENDENCY_TRACK_UPLOAD.value) + if policy_model is not None: + policy = create_policy_from_model(policy_model) + expect_type(policy, DependencyTrackUploadPolicy) + await policy.evaluate( + installation_id, + owner, + event.repository.name, + event.workflow_run, + ) + + async def targets_config_repo(repo_name: str, installation_id: int) -> bool: installation = await get_installation(installation_id) if installation is None: diff --git a/otterdog/webapp/webhook/github_models.py b/otterdog/webapp/webhook/github_models.py index 86ae0a76..3df6aa7d 100644 --- a/otterdog/webapp/webhook/github_models.py +++ b/otterdog/webapp/webhook/github_models.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2023-2024 Eclipse Foundation and others. +# Copyright (c) 2023-2025 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html @@ -176,6 +176,26 @@ class WorkflowJob(BaseModel): labels: list[str] +class WorkflowRef(BaseModel): + path: str + ref: str | None + sha: str + + +class WorkflowRun(BaseModel): + name: str + id: int + run_number: int + run_attempt: int + status: str + head_sha: str + head_branch: str | None = None + created_at: str + run_started_at: str + conclusion: str | None = None + referenced_workflows: list[WorkflowRef] | None + + class Commit(BaseModel): added: list[str] modified: list[str] @@ -247,6 +267,14 @@ class WorkflowJobEvent(Event): repository: Repository +class WorkflowRunEvent(Event): + """A payload sent for workflow run events.""" + + action: str + workflow_run: WorkflowRun + repository: Repository + + def touched_by_commits(predicate: Callable[[str], bool], commits: list[Commit]): def touched(commit: Commit) -> bool: return ( diff --git a/otterdog/webapp/webhook/github_webhook.py b/otterdog/webapp/webhook/github_webhook.py index a0f5aca2..d68ff138 100644 --- a/otterdog/webapp/webhook/github_webhook.py +++ b/otterdog/webapp/webhook/github_webhook.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2024 Eclipse Foundation and others. +# Copyright (c) 2024-2025 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html @@ -113,27 +113,24 @@ def _get_header(key): EVENT_DESCRIPTIONS = { - "commit_comment": "{comment[user][login]} commented on " "{comment[commit_id]} in {repository[full_name]}", - "create": "{sender[login]} created {ref_type} ({ref}) in " "{repository[full_name]}", - "delete": "{sender[login]} deleted {ref_type} ({ref}) in " "{repository[full_name]}", - "deployment": "{sender[login]} deployed {deployment[ref]} to " - "{deployment[environment]} in {repository[full_name]}", + "commit_comment": "{comment[user][login]} commented on {comment[commit_id]} in {repository[full_name]}", + "create": "{sender[login]} created {ref_type} ({ref}) in {repository[full_name]}", + "delete": "{sender[login]} deleted {ref_type} ({ref}) in {repository[full_name]}", + "deployment": "{sender[login]} deployed {deployment[ref]} to {deployment[environment]} in {repository[full_name]}", "deployment_status": "deployment of {deployement[ref]} to " "{deployment[environment]} " "{deployment_status[state]} in " "{repository[full_name]}", "fork": "{forkee[owner][login]} forked {forkee[name]}", "gollum": "{sender[login]} edited wiki pages in {repository[full_name]}", - "issue_comment": "{sender[login]} commented on issue #{issue[number]} " "in {repository[full_name]}", - "issues": "{sender[login]} {action} issue #{issue[number]} in " "{repository[full_name]}", - "member": "{sender[login]} {action} member {member[login]} in " "{repository[full_name]}", - "membership": "{sender[login]} {action} member {member[login]} to team " - "{team[name]} in " - "{repository[full_name]}", + "issue_comment": "{sender[login]} commented on issue #{issue[number]} in {repository[full_name]}", + "issues": "{sender[login]} {action} issue #{issue[number]} in {repository[full_name]}", + "member": "{sender[login]} {action} member {member[login]} in {repository[full_name]}", + "membership": "{sender[login]} {action} member {member[login]} to team {team[name]} in {repository[full_name]}", "page_build": "{sender[login]} built pages in {repository[full_name]}", "ping": "ping from {sender[login]}", "public": "{sender[login]} publicized {repository[full_name]}", - "pull_request": "{sender[login]} {action} pull #{pull_request[number]} in " "{repository[full_name]}", + "pull_request": "{sender[login]} {action} pull #{pull_request[number]} in {repository[full_name]}", "pull_request_review": "{sender[login]} {action} {review[state]} " "review on pull #{pull_request[number]} in " "{repository[full_name]}", @@ -141,17 +138,21 @@ def _get_header(key): "on pull #{pull_request[number]} in " "{repository[full_name]}", "push": "{pusher[name]} pushed {ref} in {repository[full_name]}", - "release": "{release[author][login]} {action} {release[tag_name]} in " "{repository[full_name]}", - "repository": "{sender[login]} {action} repository " "{repository[full_name]}", - "status": "{sender[login]} set {sha} status to {state} in " "{repository[full_name]}", - "team_add": "{sender[login]} added repository {repository[full_name]} to " "team {team[name]}", - "watch": "{sender[login]} {action} watch in repository " "{repository[full_name]}", + "release": "{release[author][login]} {action} {release[tag_name]} in {repository[full_name]}", + "repository": "{sender[login]} {action} repository {repository[full_name]}", + "status": "{sender[login]} set {sha} status to {state} in {repository[full_name]}", + "team_add": "{sender[login]} added repository {repository[full_name]} to team {team[name]}", + "watch": "{sender[login]} {action} watch in repository {repository[full_name]}", "workflow_job": "workflow '{workflow_job[name]}' on '{workflow_job[labels]}' {action} in repository " "{repository[full_name]}", + "workflow_run": "workflow '{workflow_run[name]}' {action} in repository {repository[full_name]}", } -EVENT_FILTER = {"workflow_job": "'{action}' == 'queued'"} +EVENT_FILTER = { + "workflow_job": "'{action}' == 'queued'", + "workflow_run": "'{action}' == 'completed'", +} def _log_event(event_type, data) -> bool: From ba8201d45eed3891535ffff6fae32cc2321ac8b1 Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Thu, 30 Jan 2025 16:53:07 +0100 Subject: [PATCH 2/4] add upload sbom task, add documentation, changelog entry --- CHANGELOG.md | 1 + .../policies/dependency-track-upload.md | 29 +++++ mkdocs.yml | 1 + .../providers/github/rest/action_client.py | 29 ++++- otterdog/providers/github/rest/repo_client.py | 1 + otterdog/providers/github/rest/requester.py | 5 +- .../policies/dependency_track_upload.py | 26 ++++- otterdog/webapp/tasks/blueprints/__init__.py | 4 +- otterdog/webapp/tasks/policies/__init__.py | 31 +++++ otterdog/webapp/tasks/policies/upload_sbom.py | 109 ++++++++++++++++++ otterdog/webapp/webhook/github_models.py | 1 + poetry.lock | 12 +- 12 files changed, 234 insertions(+), 15 deletions(-) create mode 100644 docs/reference/policies/dependency-track-upload.md create mode 100644 otterdog/webapp/tasks/policies/__init__.py create mode 100644 otterdog/webapp/tasks/policies/upload_sbom.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6127a912..f9e4ee73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- Added policy `dependency_track_upload` to upload SBOM data from workflows to a dependency track instance. - Added operations `list-blueprints` and `approve-blueprints` to list and approve remediation PRs created for specific organizations. - Added support for teams. - Use asyncer to speed up retrieval of live settings. ([#209](https://github.com/eclipse-csi/otterdog/issues/209)) diff --git a/docs/reference/policies/dependency-track-upload.md b/docs/reference/policies/dependency-track-upload.md new file mode 100644 index 00000000..60075627 --- /dev/null +++ b/docs/reference/policies/dependency-track-upload.md @@ -0,0 +1,29 @@ +# Upload SBOM data to a dependency track instance + +This policy will upload SBOM data as generated by workflows to a dependency track instance. + +## Configuration + +- `type` - `dependency_track_upload` + +### Settings + +| Setting | Necessity | Value type | Description | +|-----------------|-----------|------------|---------------------------------------------------------------------| +| artifact_name | mandatory | string | The artifact to look for in workflow runs that contains SBOM data | +| base_url | mandatory | string | The base url to the dependency track instance | +| workflow_filter | mandatory | string | Only consider workflows runs that reference the specified workflows | + + +## Example + +``` yaml +name: Upload sbom data to sbom.eclipse.org +description: |- + This policy uploads generated sbom data to sbom.eclipse.org. +type: dependency_track_upload +config: + artifact_name: "eclipse-sbom-data" + base_url: "https://sbom.eclipse.org" + workflow_filter: ".*/store-sbom-data.yml.*" +``` diff --git a/mkdocs.yml b/mkdocs.yml index e110f68c..ab41c022 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -72,6 +72,7 @@ nav: - Status Check: reference/organization/repository/status-check.md - Policies: - reference/policies/index.md + - Upload SBOM data to dependency track: reference/policies/dependency-track-upload.md - Usage of macOS large runners: reference/policies/macos-large-runners.md - Blueprints: - reference/blueprints/index.md diff --git a/otterdog/providers/github/rest/action_client.py b/otterdog/providers/github/rest/action_client.py index fc754806..8cf587c7 100644 --- a/otterdog/providers/github/rest/action_client.py +++ b/otterdog/providers/github/rest/action_client.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2023-2024 Eclipse Foundation and others. +# Copyright (c) 2023-2025 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html @@ -24,8 +24,9 @@ async def get_workflows(self, org_id: str, repo: str) -> list[dict[str, Any]]: _logger.debug("retrieving workflows for repo '%s/%s'", org_id, repo) try: - result = await self.requester.request_json("GET", f"/repos/{org_id}/{repo}/actions/workflows") - return result["workflows"] + return await self.requester.request_paged_json( + "GET", f"/repos/{org_id}/{repo}/actions/workflows", entries_key="workflows" + ) except GitHubException as ex: raise RuntimeError(f"failed retrieving workflows for '{org_id}/{repo}':\n{ex}") from ex @@ -44,3 +45,25 @@ async def cancel_workflow_run(self, org_id: str, repo_name: str, run_id: str) -> raise RuntimeError( f"failed cancelling workflow run #{run_id} in repo '{org_id}/{repo_name}'\n{status}: {body}" ) + + async def get_artifacts(self, org_id: str, repo: str, run_id: int) -> list[dict[str, Any]]: + _logger.debug("list artifacts for workflow run #%d in repo '%s/%s'", run_id, org_id, repo) + + try: + return await self.requester.request_paged_json( + "GET", f"/repos/{org_id}/{repo}/actions/runs/{run_id}/artifacts", entries_key="artifacts" + ) + except GitHubException as ex: + raise RuntimeError(f"failed retrieving artifacts for '{org_id}/{repo}':\n{ex}") from ex + + async def download_artifact(self, file, org_id: str, repo_name: str, artifact_id: int) -> None: + _logger.debug("downloading workflow artifact in repo '%s/%s'", org_id, repo_name) + + try: + async for data in self.requester.request_stream( + "GET", f"/repos/{org_id}/{repo_name}/actions/artifacts/{artifact_id}/zip" + ): + await file.write(data) + + except GitHubException as ex: + raise RuntimeError(f"failed downloading workflow artifact from repo '{org_id}/{repo_name}':\n{ex}") from ex diff --git a/otterdog/providers/github/rest/repo_client.py b/otterdog/providers/github/rest/repo_client.py index 7ef2ce12..beceb742 100644 --- a/otterdog/providers/github/rest/repo_client.py +++ b/otterdog/providers/github/rest/repo_client.py @@ -5,6 +5,7 @@ # which is available at http://www.eclipse.org/legal/epl-v20.html # SPDX-License-Identifier: EPL-2.0 # ******************************************************************************* + import asyncio import json import os diff --git a/otterdog/providers/github/rest/requester.py b/otterdog/providers/github/rest/requester.py index 165a3002..e90f21ed 100644 --- a/otterdog/providers/github/rest/requester.py +++ b/otterdog/providers/github/rest/requester.py @@ -81,6 +81,7 @@ async def request_paged_json( url_path: str, data: dict[str, Any] | None = None, params: dict[str, str] | None = None, + entries_key: str | None = None, ) -> list[dict[str, Any]]: from urllib import parse @@ -101,12 +102,14 @@ async def request_paged_json( self._check_response(url_path, status, body) response = json.loads(body) + entries = response[entries_key] if entries_key is not None else response + if next_url is None: query_params = None else: query_params = {k: v[0] for k, v in parse.parse_qs(parse.urlparse(next_url).query).items()} - for item in response: + for item in entries: result.append(item) return result diff --git a/otterdog/webapp/policies/dependency_track_upload.py b/otterdog/webapp/policies/dependency_track_upload.py index 06a1984c..16342324 100644 --- a/otterdog/webapp/policies/dependency_track_upload.py +++ b/otterdog/webapp/policies/dependency_track_upload.py @@ -10,6 +10,8 @@ from logging import getLogger from typing import Any, Self +from quart import current_app + from otterdog.utils import expect_type, unwrap from otterdog.webapp.webhook.github_models import WorkflowRun @@ -23,6 +25,8 @@ class DependencyTrackUploadPolicy(Policy): A policy to upload sbom data from workflow runs to dependency track. """ + artifact_name: str + base_url: str workflow_filter: str @property @@ -47,7 +51,23 @@ async def evaluate( repo_name = unwrap(repo_name) payload = expect_type(payload, WorkflowRun) - if payload.referenced_workflows is not None and any( - self.matches_workflow(x.path) for x in payload.referenced_workflows + if ( + payload.conclusion == "success" + and payload.referenced_workflows is not None + and any(self.matches_workflow(x.path) for x in payload.referenced_workflows) ): - logger.info(f"workflow run #{payload.id} in repo '{github_id}/{repo_name}' contains sbom data") + from otterdog.webapp.tasks.policies.upload_sbom import UploadSBOMTask + + logger.info( + f"workflow run {payload.name}/#{payload.id} in repo '{github_id}/{repo_name}' contains sbom data" + ) + + current_app.add_background_task( + UploadSBOMTask( + installation_id, + github_id, + repo_name, + self, + payload.id, + ) + ) diff --git a/otterdog/webapp/tasks/blueprints/__init__.py b/otterdog/webapp/tasks/blueprints/__init__.py index c712837d..d3386398 100644 --- a/otterdog/webapp/tasks/blueprints/__init__.py +++ b/otterdog/webapp/tasks/blueprints/__init__.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2024 Eclipse Foundation and others. +# Copyright (c) 2024-2025 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html @@ -52,7 +52,7 @@ async def _pre_execute(self) -> bool: if blueprint_status is None: return True - match blueprint_status: + match blueprint_status.status: case BlueprintStatus.DISMISSED: self.logger.debug( f"Blueprint '{self.blueprint.id}' dismissed for " f"repo '{self.org_id}/{self.repo_name}', skipping" diff --git a/otterdog/webapp/tasks/policies/__init__.py b/otterdog/webapp/tasks/policies/__init__.py new file mode 100644 index 00000000..99707e96 --- /dev/null +++ b/otterdog/webapp/tasks/policies/__init__.py @@ -0,0 +1,31 @@ +# ******************************************************************************* +# Copyright (c) 2025 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from __future__ import annotations + +from abc import ABC +from typing import TYPE_CHECKING + +from otterdog.webapp.db.models import TaskModel +from otterdog.webapp.tasks import InstallationBasedTask, Task + +if TYPE_CHECKING: + from otterdog.webapp.policies import Policy + + +class PolicyTask(InstallationBasedTask, Task[bool], ABC): + org_id: str + repo_name: str + policy: Policy + + def create_task_model(self): + return TaskModel( + type=type(self).__name__, + org_id=self.org_id, + repo_name=self.repo_name, + ) diff --git a/otterdog/webapp/tasks/policies/upload_sbom.py b/otterdog/webapp/tasks/policies/upload_sbom.py new file mode 100644 index 00000000..232afdb0 --- /dev/null +++ b/otterdog/webapp/tasks/policies/upload_sbom.py @@ -0,0 +1,109 @@ +# ******************************************************************************* +# Copyright (c) 2024-2025 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from __future__ import annotations + +import base64 +import json +import os +import tempfile +import zipfile +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import aiofiles +import aiohttp +from pydantic import BaseModel + +from otterdog.webapp import get_temporary_base_directory +from otterdog.webapp.tasks.policies import PolicyTask + +if TYPE_CHECKING: + from otterdog.providers.github.rest import RestApi + from otterdog.webapp.policies.dependency_track_upload import DependencyTrackUploadPolicy + + +@dataclass(repr=False) +class UploadSBOMTask(PolicyTask): + installation_id: int + org_id: str + repo_name: str + policy: DependencyTrackUploadPolicy + workflow_run_id: int + + async def _execute(self) -> bool: + self.logger.info( + "uploading sbom for repo '%s/%s'", + self.org_id, + self.repo_name, + ) + + async with self.get_organization_config() as _: + rest_api = await self.rest_api + + artifacts = await rest_api.action.get_artifacts(self.org_id, self.repo_name, self.workflow_run_id) + for artifact in artifacts: + if artifact["name"] == self.policy.artifact_name: + artifact_id = artifact["id"] + await self._process_artifact(rest_api, artifact_id) + + return True + + async def _process_artifact(self, rest_api: RestApi, artifact_id: int) -> None: + with tempfile.TemporaryDirectory(dir=get_temporary_base_directory()) as tmp_dir: + artifact_file_name = os.path.join(tmp_dir, "artifact.zip") + async with aiofiles.open(artifact_file_name, "wb") as artifact_file: + await rest_api.action.download_artifact(artifact_file, self.org_id, self.repo_name, artifact_id) + + with zipfile.ZipFile(artifact_file_name, "r") as zip_file: + zip_file.extractall(tmp_dir) + + bom_file_name = os.path.join(tmp_dir, "bom.json") + metadata_file_name = os.path.join(tmp_dir, "metadata.json") + + with open(bom_file_name) as bom_file: + bom = json.load(bom_file) + + with open(metadata_file_name) as metadata_file: + metadata = Metadata.model_validate(json.load(metadata_file)) + + await self._upload_bom(bom, metadata) + + async def _upload_bom(self, bom: dict[str, Any], meta_data: Metadata) -> None: + async with aiohttp.ClientSession() as session: + headers = { + "Content-Type": "application/json", + "X-Api-Key": "odt_rKEj8Mq0vFwEE2LmFLgIe766BWRo892k", + } + + data = { + "projectName": meta_data.projectName, + "projectVersion": meta_data.projectVersion, + "parentUUID": meta_data.parentProject, + "autoCreate": True, + "bom": base64.b64encode(json.dumps(bom).encode("utf-8")).decode("utf-8"), + } + + self.logger.info( + f"uploading sbom for '{meta_data.projectName}@{meta_data.projectVersion}' to '{self.policy.base_url}'" + ) + + upload_url = f"{self.policy.base_url}/api/v1/bom" + async with session.put(upload_url, headers=headers, json=data) as response: + if response.status != 200: + error = await response.text() + raise RuntimeError(f"failed to upload SBOM: {error}") + + def __repr__(self) -> str: + return f"UploadSBOMTask(repo={self.org_id}/{self.repo_name}, run_id={self.workflow_run_id})" + + +class Metadata(BaseModel): + projectName: str # noqa + projectVersion: str # noqa + parentProject: str # noqa diff --git a/otterdog/webapp/webhook/github_models.py b/otterdog/webapp/webhook/github_models.py index 3df6aa7d..7fc3d20f 100644 --- a/otterdog/webapp/webhook/github_models.py +++ b/otterdog/webapp/webhook/github_models.py @@ -163,6 +163,7 @@ class Issue(BaseModel): class WorkflowJob(BaseModel): name: str id: int + workflow_name: str | None run_id: int runner_id: int | None = None runner_name: str | None = None diff --git a/poetry.lock b/poetry.lock index 292443f5..181f47c9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2166,14 +2166,14 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pymdown-extensions" -version = "10.14.1" +version = "10.14.2" description = "Extension pack for Python Markdown." optional = false python-versions = ">=3.8" groups = ["docs"] files = [ - {file = "pymdown_extensions-10.14.1-py3-none-any.whl", hash = "sha256:637951cbfbe9874ba28134fb3ce4b8bcadd6aca89ac4998ec29dcbafd554ae08"}, - {file = "pymdown_extensions-10.14.1.tar.gz", hash = "sha256:b65801996a0cd4f42a3110810c306c45b7313c09b0610a6f773730f2a9e3c96b"}, + {file = "pymdown_extensions-10.14.2-py3-none-any.whl", hash = "sha256:f45bc5892410e54fd738ab8ccd736098b7ff0cb27fdb4bf24d0a0c6584bc90e1"}, + {file = "pymdown_extensions-10.14.2.tar.gz", hash = "sha256:7a77b8116dc04193f2c01143760a43387bd9dc4aa05efacb7d838885a7791253"}, ] [package.dependencies] @@ -2334,14 +2334,14 @@ dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments [[package]] name = "pytest-asyncio" -version = "0.25.2" +version = "0.25.3" description = "Pytest support for asyncio" optional = false python-versions = ">=3.9" groups = ["test"] files = [ - {file = "pytest_asyncio-0.25.2-py3-none-any.whl", hash = "sha256:0d0bb693f7b99da304a0634afc0a4b19e49d5e0de2d670f38dc4bfa5727c5075"}, - {file = "pytest_asyncio-0.25.2.tar.gz", hash = "sha256:3f8ef9a98f45948ea91a0ed3dc4268b5326c0e7bce73892acc654df4262ad45f"}, + {file = "pytest_asyncio-0.25.3-py3-none-any.whl", hash = "sha256:9e89518e0f9bd08928f97a3482fdc4e244df17529460bc038291ccaf8f85c7c3"}, + {file = "pytest_asyncio-0.25.3.tar.gz", hash = "sha256:fc1da2cf9f125ada7e710b4ddad05518d4cee187ae9412e9ac9271003497f07a"}, ] [package.dependencies] From edfe6502b4a663e870dc9b7bcb6ceda058591e11 Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Thu, 30 Jan 2025 17:02:02 +0100 Subject: [PATCH 3/4] remove dp token --- otterdog/webapp/config.py | 4 +++- otterdog/webapp/tasks/policies/upload_sbom.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/otterdog/webapp/config.py b/otterdog/webapp/config.py index a6653368..6a0e4744 100644 --- a/otterdog/webapp/config.py +++ b/otterdog/webapp/config.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2023-2024 Eclipse Foundation and others. +# Copyright (c) 2023-2025 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html @@ -54,6 +54,8 @@ class AppConfig: GITHUB_APP_PRIVATE_KEY = config("GITHUB_APP_PRIVATE_KEY") PROJECTS_BASE_URL = config("PROJECTS_BASE_URL", default="https://projects.eclipse.org/projects/") + DEPENDENCY_TRACK_URL = config("DEPENDENCY_TRACK_URL", default="https://sbom.eclipse.org") + DEPENDENCY_TRACK_TOKEN = config("DEPENDENCY_TRACK_TOKEN") class ProductionConfig(AppConfig): diff --git a/otterdog/webapp/tasks/policies/upload_sbom.py b/otterdog/webapp/tasks/policies/upload_sbom.py index 232afdb0..3fcc2388 100644 --- a/otterdog/webapp/tasks/policies/upload_sbom.py +++ b/otterdog/webapp/tasks/policies/upload_sbom.py @@ -19,6 +19,7 @@ import aiofiles import aiohttp from pydantic import BaseModel +from quart import current_app from otterdog.webapp import get_temporary_base_directory from otterdog.webapp.tasks.policies import PolicyTask @@ -36,6 +37,10 @@ class UploadSBOMTask(PolicyTask): policy: DependencyTrackUploadPolicy workflow_run_id: int + @property + def _dependency_track_token(self) -> str: + return current_app.config["DEPENDENCY_TRACK_TOKEN"] + async def _execute(self) -> bool: self.logger.info( "uploading sbom for repo '%s/%s'", @@ -78,7 +83,7 @@ async def _upload_bom(self, bom: dict[str, Any], meta_data: Metadata) -> None: async with aiohttp.ClientSession() as session: headers = { "Content-Type": "application/json", - "X-Api-Key": "odt_rKEj8Mq0vFwEE2LmFLgIe766BWRo892k", + "X-Api-Key": self._dependency_track_token, } data = { From 064b293e281a0d3cbd9cdf3420377381d1590748 Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Thu, 30 Jan 2025 17:02:24 +0100 Subject: [PATCH 4/4] update deps --- DEPENDENCIES | 2 +- poetry.lock | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/DEPENDENCIES b/DEPENDENCIES index 1eb95c92..1021654e 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -46,7 +46,7 @@ pypi/pypi/-/markdown-it-py/3.0.0 pypi/pypi/-/markupsafe/3.0.2 pypi/pypi/-/mdurl/0.1.2 pypi/pypi/-/mintotp/0.3.0 -pypi/pypi/-/motor/3.6.1 +pypi/pypi/-/motor/3.7.0 pypi/pypi/-/multidict/6.1.0 pypi/pypi/-/odmantic/1.0.2 pypi/pypi/-/playwright/1.49.1 diff --git a/poetry.lock b/poetry.lock index 181f47c9..b3b07eb1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1501,18 +1501,18 @@ files = [ [[package]] name = "motor" -version = "3.6.1" +version = "3.7.0" description = "Non-blocking MongoDB driver for Tornado or asyncio" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["app"] files = [ - {file = "motor-3.6.1-py3-none-any.whl", hash = "sha256:7fe552353aded4fa9f05ae515a179df5b1d192b1da56726f422dbb2d8c3b5962"}, - {file = "motor-3.6.1.tar.gz", hash = "sha256:ee2b18386292f9ceb3cc8279a4cd34e4c641c5ac8de3500c30374081c76a9d03"}, + {file = "motor-3.7.0-py3-none-any.whl", hash = "sha256:61bdf1afded179f008d423f98066348157686f25a90776ea155db5f47f57d605"}, + {file = "motor-3.7.0.tar.gz", hash = "sha256:0dfa1f12c812bd90819c519b78bed626b5a9dbb29bba079ccff2bfa8627e0fec"}, ] [package.dependencies] -pymongo = ">=4.9,<4.10" +pymongo = ">=4.9,<5.0" [package.extras] aws = ["pymongo[aws] (>=4.5,<5)"]