From ce7f47909ea9ac9fc6e961bc1affbc8fda44323f Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Mon, 27 Jan 2025 14:33:03 +0100 Subject: [PATCH] feat: add list-blueprints and approve-blueprints operations --- CHANGELOG.md | 1 + otterdog/cli.py | 81 +++++++++- otterdog/config.py | 8 +- otterdog/models/repository.py | 14 +- otterdog/operations/approve_blueprints.py | 138 ++++++++++++++++++ otterdog/operations/list_blueprints.py | 110 ++++++++++++++ .../github/rest/pull_request_client.py | 19 ++- otterdog/providers/github/rest/repo_client.py | 60 +++++--- 8 files changed, 399 insertions(+), 32 deletions(-) create mode 100644 otterdog/operations/approve_blueprints.py create mode 100644 otterdog/operations/list_blueprints.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 73e47bdf..6127a912 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- Added operations `list-blueprints` and `approve-blueprints` to list and approve remediation PRs created for specific organizations. - Added support for teams. - Use asyncer to speed up retrieval of live settings. ([#209](https://github.com/eclipse-csi/otterdog/issues/209)) diff --git a/otterdog/cli.py b/otterdog/cli.py index 976c7319..b8703070 100644 --- a/otterdog/cli.py +++ b/otterdog/cli.py @@ -100,6 +100,59 @@ def invoke(self, ctx: click.Context) -> Any: return super().invoke(ctx) +class SingletonCommand(click.Command): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.context_settings = _CONTEXT_SETTINGS + self.params.insert( + 0, + click.Option( + ["-v", "--verbose"], + count=True, + help="enable verbose output (-vvv for more verbose output)", + ), + ) + + self.params.insert( + 0, + click.Option( + ["-c", "--config"], + default=_CONFIG_FILE, + show_default=True, + type=click.Path(True, True, False), + help="configuration file to use", + ), + ) + + self.params.insert( + 0, + click.Option( + ["--local"], + is_flag=True, + default=False, + show_default=True, + help="work in local mode, not updating the referenced default config", + ), + ) + + def invoke(self, ctx: click.Context) -> Any: + global _CONFIG + + verbose = ctx.params.pop("verbose") + init_logging(verbose) + + config_file = ctx.params.pop("config") + local_mode = ctx.params.pop("local") + + try: + _CONFIG = OtterdogConfig.from_file(config_file, local_mode) + except Exception as exc: + print_exception(exc) + sys.exit(2) + + return super().invoke(ctx) + + @click.group(context_settings=_CONTEXT_SETTINGS) @click.version_option(version=__version__, prog_name="otterdog.sh") def cli(): @@ -297,14 +350,36 @@ def list_members(organizations: list[str], two_factor_disabled: bool): _execute_operation(organizations, ListMembersOperation(two_factor_disabled)) -@cli.command(cls=StdCommand) -def list_projects(organizations: list[str]): +@cli.command(cls=SingletonCommand) +def list_projects(): """ Lists all configured projects and their corresponding GitHub id. """ from otterdog.operations.list_projects import ListProjectsOperation - _execute_operation(organizations, ListProjectsOperation()) + _execute_operation([], ListProjectsOperation()) + + +@cli.command(cls=StdCommand) +@click.option("-b", "--blueprint-id", required=False, help="blueprint id") +def list_blueprints(organizations: list[str], blueprint_id): + """ + List blueprints. + """ + from otterdog.operations.list_blueprints import ListBlueprintsOperation + + _execute_operation(organizations, ListBlueprintsOperation(blueprint_id)) + + +@cli.command(cls=StdCommand) +@click.option("-b", "--blueprint-id", required=False, help="blueprint id") +def approve_blueprints(organizations: list[str], blueprint_id): + """ + Approved remediation PRs for blueprints. + """ + from otterdog.operations.approve_blueprints import ApproveBlueprintsOperation + + _execute_operation(organizations, ApproveBlueprintsOperation(blueprint_id)) @cli.command(cls=StdCommand, name="import") diff --git a/otterdog/config.py b/otterdog/config.py index 532e3e43..70b86e57 100644 --- a/otterdog/config.py +++ b/otterdog/config.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2023-2024 Eclipse Foundation and others. +# Copyright (c) 2023-2025 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html @@ -190,6 +190,7 @@ class OtterdogConfig: local_mode: bool working_dir: str + _base_url: str = dataclasses.field(init=False) _jsonnet_config: Mapping[str, Any] = dataclasses.field(init=False) _github_config: Mapping[str, Any] = dataclasses.field(init=False) _default_credential_provider: str = dataclasses.field(init=False) @@ -199,6 +200,7 @@ class OtterdogConfig: _organizations: list[OrganizationConfig] = dataclasses.field(init=False, default_factory=list) def __post_init__(self): + object.__setattr__(self, "_base_url", query_json("defaults.base_url", self.configuration) or None) object.__setattr__(self, "_jsonnet_config", query_json("defaults.jsonnet", self.configuration) or {}) object.__setattr__(self, "_github_config", query_json("defaults.github", self.configuration) or {}) object.__setattr__( @@ -220,6 +222,10 @@ def __post_init__(self): self._organizations_map[org_config.name.lower()] = org_config self._organizations_map[org_config.github_id.lower()] = org_config + @property + def base_url(self) -> str: + return self._base_url + @property def jsonnet_base_dir(self) -> str: return self._jsonnet_base_dir diff --git a/otterdog/models/repository.py b/otterdog/models/repository.py index c2e2ca93..22da7fd0 100644 --- a/otterdog/models/repository.py +++ b/otterdog/models/repository.py @@ -1089,12 +1089,10 @@ def generate_live_patch( ) ) - parent_repo = coerced_object if current_object is None else current_object - RepositoryWebhook.generate_live_patch_of_list( coerced_object.webhooks, current_object.webhooks if current_object is not None else [], - parent_repo, + coerced_object, context, handler, ) @@ -1102,7 +1100,7 @@ def generate_live_patch( RepositorySecret.generate_live_patch_of_list( coerced_object.secrets, current_object.secrets if current_object is not None else [], - parent_repo, + coerced_object, context, handler, ) @@ -1110,7 +1108,7 @@ def generate_live_patch( RepositoryVariable.generate_live_patch_of_list( coerced_object.variables, current_object.variables if current_object is not None else [], - parent_repo, + coerced_object, context, handler, ) @@ -1118,7 +1116,7 @@ def generate_live_patch( Environment.generate_live_patch_of_list( coerced_object.environments, current_object.environments if current_object is not None else [], - parent_repo, + coerced_object, context, handler, ) @@ -1130,7 +1128,7 @@ def generate_live_patch( BranchProtectionRule.generate_live_patch_of_list( coerced_object.branch_protection_rules, current_object.branch_protection_rules if current_object is not None else [], - parent_repo, + coerced_object, context, handler, ) @@ -1138,7 +1136,7 @@ def generate_live_patch( RepositoryRuleset.generate_live_patch_of_list( coerced_object.rulesets, current_object.rulesets if current_object is not None else [], - parent_repo, + coerced_object, context, handler, ) diff --git a/otterdog/operations/approve_blueprints.py b/otterdog/operations/approve_blueprints.py new file mode 100644 index 00000000..fbe9b3b9 --- /dev/null +++ b/otterdog/operations/approve_blueprints.py @@ -0,0 +1,138 @@ +# ******************************************************************************* +# Copyright (c) 2025 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from otterdog.models.github_organization import GitHubOrganization +from otterdog.providers.github import GitHubProvider +from otterdog.utils import unwrap +from otterdog.webapp.db.models import BlueprintStatusModel + +from . import Operation + +if TYPE_CHECKING: + from otterdog.config import OrganizationConfig + + +class ApproveBlueprintsOperation(Operation): + """ + Approves and merges remediation PRs for blueprints. + """ + + def __init__(self, blueprint_id: str | None): + super().__init__() + self._blueprint_id = blueprint_id + self._blueprints_by_org: dict[str, list[BlueprintStatusModel]] = {} + + @property + def blueprint_id(self) -> str | None: + return self._blueprint_id + + def pre_execute(self) -> None: + import requests + + items = [] + params = { + "pageIndex": 1, + "pageSize": 50, + } + + base_url = unwrap(self._config).base_url + if base_url is None: + raise RuntimeError("no base_url set which is required when using operation 'approve-blueprints'") + + while True: + response = requests.get(f"{base_url}/api/blueprints/remediations", params=params, timeout=10) + response_json = response.json() + items.extend(response_json["data"]) + if len(items) >= response_json["itemsCount"]: + break + else: + params["pageIndex"] = params["pageIndex"] + 1 + + for item in items: + blueprint_status = BlueprintStatusModel.model_validate(item) + + if self._skip_blueprint(blueprint_status): + continue + + blueprint_list = self._blueprints_by_org.setdefault(blueprint_status.id.org_id, []) + blueprint_list.append(blueprint_status) + + def _skip_blueprint(self, blueprint: BlueprintStatusModel) -> bool: + if self.blueprint_id is not None and blueprint.id.blueprint_id != self.blueprint_id: + return True + + return False + + async def execute( + self, + org_config: OrganizationConfig, + org_index: int | None = None, + org_count: int | None = None, + ) -> int: + blueprints = self._blueprints_by_org.get(org_config.github_id, []) + + if len(blueprints) > 0: + github_id = org_config.github_id + jsonnet_config = org_config.jsonnet_config + await jsonnet_config.init_template() + + self._print_project_header(org_config, org_index, org_count) + self.printer.level_up() + + try: + org_file_name = jsonnet_config.org_config_file + if not await self.check_config_file_exists(org_file_name): + return 1 + + try: + organization = GitHubOrganization.load_from_file(github_id, org_file_name) + except RuntimeError as ex: + self.printer.print_error(f"failed to load configuration: {ex!s}") + return 1 + + try: + credentials = self.get_credentials(org_config, only_token=True) + except RuntimeError as e: + self.printer.print_error(f"invalid credentials\n{e!s}") + return 1 + + async with GitHubProvider(credentials) as provider: + rest_api = provider.rest_api + + for blueprint in blueprints: + self.printer.print(f"Merging PR #{blueprint.remediation_pr}: ") + + repo = organization.get_repository(blueprint.id.repo_name) + if repo is None: + self.printer.println("no repo found.") + continue + + if repo.allow_merge_commit is True: + merge_method = "merge" + if repo.allow_squash_merge is True: + merge_method = "squash" + if repo.allow_rebase_merge is True: + merge_method = "rebase" + + result = await rest_api.pull_request.merge_pull_request( + blueprint.id.org_id, blueprint.id.repo_name, f"{blueprint.remediation_pr}", merge_method + ) + + if result["merged"] is True: + self.printer.println("[green]merged[/].") + else: + self.printer.println(f"[red]failed[/]: {result['message']}.") + + finally: + self.printer.level_down() + + return 0 diff --git a/otterdog/operations/list_blueprints.py b/otterdog/operations/list_blueprints.py new file mode 100644 index 00000000..ed65e230 --- /dev/null +++ b/otterdog/operations/list_blueprints.py @@ -0,0 +1,110 @@ +# ******************************************************************************* +# Copyright (c) 2025 Eclipse Foundation and others. +# This program and the accompanying materials are made available +# under the terms of the Eclipse Public License 2.0 +# which is available at http://www.eclipse.org/legal/epl-v20.html +# SPDX-License-Identifier: EPL-2.0 +# ******************************************************************************* + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from rich import box +from rich.table import Table + +from otterdog.utils import unwrap +from otterdog.webapp.db.models import BlueprintStatusModel + +from . import Operation + +if TYPE_CHECKING: + from otterdog.config import OrganizationConfig + + +class ListBlueprintsOperation(Operation): + """ + Lists remediation PRs for blueprints. + """ + + def __init__(self, blueprint_id: str | None): + super().__init__() + self._blueprint_id = blueprint_id + self._table = Table(title="Projects", box=box.ROUNDED) + self._blueprints_by_org: dict[str, list[BlueprintStatusModel]] = {} + + @property + def blueprint_id(self) -> str | None: + return self._blueprint_id + + @property + def table(self) -> Table: + return self._table + + def pre_execute(self) -> None: + import requests + + items = [] + params = { + "pageIndex": 1, + "pageSize": 50, + } + + base_url = unwrap(self._config).base_url + if base_url is None: + raise RuntimeError("no base_url set which is required when using operation 'list-blueprints'") + + while True: + response = requests.get(f"{base_url}/api/blueprints/remediations", params=params, timeout=10) + response_json = response.json() + items.extend(response_json["data"]) + if len(items) >= response_json["itemsCount"]: + break + else: + params["pageIndex"] = params["pageIndex"] + 1 + + self.table.add_column("Index", justify="left", style="cyan") + self.table.add_column("Blueprint ID", justify="left", style="cyan", no_wrap=True) + self.table.add_column("GitHub ID", style="magenta") + self.table.add_column("Repo", justify="left", style="green") + self.table.add_column("PR", justify="left", style="red") + + for item in items: + blueprint_status = BlueprintStatusModel.model_validate(item) + + if self._skip_blueprint(blueprint_status): + continue + + blueprints = self._blueprints_by_org.setdefault(blueprint_status.id.org_id, []) + blueprints.append(blueprint_status) + + def _skip_blueprint(self, blueprint: BlueprintStatusModel) -> bool: + if self.blueprint_id is not None and blueprint.id.blueprint_id != self.blueprint_id: + return True + + return False + + def post_execute(self) -> None: + self.printer.console.print(self.table) + + async def execute( + self, + org_config: OrganizationConfig, + org_index: int | None = None, + org_count: int | None = None, + ) -> int: + blueprints = self._blueprints_by_org.get(org_config.github_id, []) + for blueprint in blueprints: + pr_url = ( + f"https://github.com/{blueprint.id.org_id}/{blueprint.id.repo_name}/pull/{blueprint.remediation_pr}" + ) + + self.table.add_row( + f"{len(self.table.rows) + 1}", + blueprint.id.blueprint_id, + blueprint.id.org_id, + blueprint.id.repo_name, + pr_url, + ) + + return 0 diff --git a/otterdog/providers/github/rest/pull_request_client.py b/otterdog/providers/github/rest/pull_request_client.py index dee35c01..50aa31a2 100644 --- a/otterdog/providers/github/rest/pull_request_client.py +++ b/otterdog/providers/github/rest/pull_request_client.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2024 Eclipse Foundation and others. +# Copyright (c) 2024-2025 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html @@ -77,6 +77,23 @@ async def get_pull_requests( except GitHubException as ex: raise RuntimeError(f"failed retrieving pull requests:\n{ex}") from ex + async def merge_pull_request( + self, + org_id: str, + repo_name: str, + pull_request_number: str, + method: str, + ) -> dict[str, Any]: + _logger.debug("merging pull request #%s for repo '%s/%s'", pull_request_number, org_id, repo_name) + + try: + data = {"merge_method": method} + return await self.requester.request_json( + "PUT", f"/repos/{org_id}/{repo_name}/pulls/{pull_request_number}/merge", data=data + ) + except GitHubException as ex: + raise RuntimeError(f"failed merging pull request:\n{ex}") from ex + async def get_commits( self, org_id: str, diff --git a/otterdog/providers/github/rest/repo_client.py b/otterdog/providers/github/rest/repo_client.py index 254c5aad..a5ee4a49 100644 --- a/otterdog/providers/github/rest/repo_client.py +++ b/otterdog/providers/github/rest/repo_client.py @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2023-2024 Eclipse Foundation and others. +# Copyright (c) 2023-2025 Eclipse Foundation and others. # This program and the accompanying materials are made available # under the terms of the Eclipse Public License 2.0 # which is available at http://www.eclipse.org/legal/epl-v20.html @@ -17,6 +17,7 @@ import aiofiles import chevron +from otterdog.logging import is_trace_enabled from otterdog.providers.github.exception import GitHubException from otterdog.providers.github.rest import RestApi, RestClient, encrypt_value from otterdog.utils import ( @@ -47,6 +48,27 @@ async def get_default_branch(self, org_id: str, repo_name: str) -> str: repo_data = await self.get_simple_repo_data(org_id, repo_name) return repo_data["default_branch"] + async def get_branch(self, org_id: str, repo_name: str, branch_name: str) -> dict[str, Any]: + _logger.debug("retrieving data for branch '%s' in repo '%s/%s'", branch_name, org_id, repo_name) + + try: + return await self.requester.request_json("GET", f"/repos/{org_id}/{repo_name}/branches/{branch_name}") + except GitHubException as ex: + raise RuntimeError( + f"failed retrieving data for branch '{branch_name}' in repo '{org_id}/{repo_name}':\n{ex}" + ) from ex + + async def rename_branch(self, org_id: str, repo_name: str, branch: str, new_name: str) -> dict[str, Any]: + _logger.debug("renaming branch '%s' to '%s' in repo '%s/%s'", branch, new_name, org_id, repo_name) + + try: + data = {"new_name": new_name} + return await self.requester.request_json( + "POST", f"/repos/{org_id}/{repo_name}/branches/{branch}/rename", data + ) + except GitHubException as ex: + raise RuntimeError(f"failed renaming branch '{branch}' in repo '{org_id}/{repo_name}':\n{ex}") from ex + async def get_repo_data(self, org_id: str, repo_name: str) -> dict[str, Any]: _logger.debug("retrieving repo data for '%s/%s'", org_id, repo_name) @@ -200,9 +222,7 @@ async def add_repo( initialized = True break except RuntimeError: - _logger.trace( - f"waiting for repo '{org_id}/{repo_name}' to be initialized, " f"try {i} of 10" - ) + _logger.trace(f"waiting for repo '{org_id}/{repo_name}' to be initialized, try {i} of 10") import time time.sleep(1) @@ -428,6 +448,13 @@ async def _fill_github_pages_config(self, org_id: str, repo_name: str, repo_data status, body = await self.requester.request_raw("GET", f"/repos/{org_id}/{repo_name}/pages") if status == 200: + if is_trace_enabled(): + _logger.trace( + "'%s' url = %s, json = %s", + "GET", + f"/repos/{org_id}/{repo_name}/pages", + json.dumps(json.loads(body), indent=2), + ) repo_data["gh_pages"] = json.loads(body) async def _update_github_pages_config(self, org_id: str, repo_name: str, gh_pages: dict[str, Any]) -> None: @@ -441,7 +468,7 @@ async def _update_github_pages_config(self, org_id: str, repo_name: str, gh_page if "gh_pages" in current_repo_data: break - _logger.trace(f"waiting for repo '{org_id}/{repo_name}' to be initialized, " f"try {i} of 3") + _logger.trace(f"waiting for repo '{org_id}/{repo_name}' to be initialized, try {i} of 3") import time time.sleep(1) @@ -532,14 +559,11 @@ async def _update_default_branch(self, org_id: str, repo_name: str, new_default_ if new_default_branch in existing_branch_names: data = {"default_branch": new_default_branch} await self.requester.request_json("PATCH", f"/repos/{org_id}/{repo_name}", data) - _logger.debug("updated default branch for '%s/%s'", org_id, repo_name) + _logger.debug("updated default branch in repo '%s/%s'", org_id, repo_name) else: default_branch = await self.get_default_branch(org_id, repo_name) - data = {"new_name": new_default_branch} - await self.requester.request_json( - "POST", f"/repos/{org_id}/{repo_name}/branches/{default_branch}/rename", data - ) - _logger.debug("renamed default branch for '%s/%s'", org_id, repo_name) + await self.rename_branch(org_id, repo_name, default_branch, new_default_branch) + _logger.debug("renamed default branch in repo '%s/%s'", org_id, repo_name) except GitHubException as ex: raise RuntimeError(f"failed to update default branch for repo '{org_id}/{repo_name}':\n{ex}") from ex @@ -715,7 +739,7 @@ async def _update_deployment_branch_policies( try: current_branch_policies_by_name = associate_by_key( await self._get_deployment_branch_policies(org_id, repo_name, env_name), - lambda x: f'{x.get("type", "branch")}:{x["name"]}', + lambda x: f"{x.get('type', 'branch')}:{x['name']}", ) except RuntimeError: current_branch_policies_by_name = {} @@ -765,7 +789,7 @@ async def _delete_deployment_branch_policy( status, body = await self.requester.request_raw("DELETE", url) if status != 204: - raise RuntimeError(f"failed deleting deployment branch policy" f"\n{status}: {body}") + raise RuntimeError(f"failed deleting deployment branch policy\n{status}: {body}") _logger.debug("deleted deployment branch policy for env '%s'", env_name) @@ -921,7 +945,7 @@ async def update_workflow_settings(self, org_id: str, repo_name: str, data: dict if status != 204: raise RuntimeError( - f"failed to update workflow settings for repo '{org_id}/{repo_name}'" f"\n{status}: {body}" + f"failed to update workflow settings for repo '{org_id}/{repo_name}'\n{status}: {body}" ) _logger.debug("updated workflow settings for repo '%s/%s'", org_id, repo_name) @@ -962,7 +986,7 @@ async def _update_selected_actions_for_workflow_settings( ) if status != 204: - raise RuntimeError(f"failed updating allowed actions for repo '{org_id}/{repo_name}'" f"\n{status}: {body}") + raise RuntimeError(f"failed updating allowed actions for repo '{org_id}/{repo_name}'\n{status}: {body}") _logger.debug("updated allowed actions for repo '%s/%s'", org_id, repo_name) @@ -985,7 +1009,7 @@ async def _update_default_workflow_permissions(self, org_id: str, repo_name: str if status != 204: raise RuntimeError( - f"failed updating default workflow permissions for repo '{org_id}/{repo_name}'" f"\n{status}: {body}" + f"failed updating default workflow permissions for repo '{org_id}/{repo_name}'\n{status}: {body}" ) _logger.debug("updated default workflow permissions for repo '%s/%s'", org_id, repo_name) @@ -1083,6 +1107,4 @@ async def _download_repository_archive(self, file, org_id: str, repo_name: str, await file.write(data) except GitHubException as ex: - raise RuntimeError( - f"failed retrieving repository archive from " f"repo '{org_id}/{repo_name}':\n{ex}" - ) from ex + raise RuntimeError(f"failed retrieving repository archive from repo '{org_id}/{repo_name}':\n{ex}") from ex