From ae86b54a364d87b15a27f7252eea649c1fc6a722 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 29 Oct 2024 16:57:01 +0100 Subject: [PATCH] [CI] Fix project build & deploy job (#196562) ## Summary In #195581 we've added the option to deploy through the clickable triggers. But in it's current state, it's broken in several aspects. (1) It's not starting on click. Triggers was resulting in a 422 on Buildkite's side, and after digging more into it, this was the error: Screenshot 2024-10-16 at 16 53 13 Apparently, building PRs needs to be enabled on jobs that want to be triggered through the PR bot. (2) It is set up to run regardless of the labels (3) There's no feedback on runs ## Changes This PR: - enables buildability in the pipeline's config - exits early if deploy labels are missing - adds a comment on the PR if a deploy job is started or finished - removes the kibana build step, it's not needed, as we have a step to build the docker image TODO: - [x] Add feedback about a started job (either through a non-required check, or a github comment) - [x] Early exit if a label is missing There are several other builds started right now, because the logic that would trigger a build on changing a draft to ready. To be fixed in https://github.com/elastic/buildkite-pr-bot/issues/78 Tested after manually by enabling the option on the UI, and triggering through the checkbox: https://buildkite.com/elastic/kibana-deploy-project-from-pr/builds/23 --- .../kibana-deploy-project.yml | 2 + .buildkite/pipeline-utils/github/github.ts | 20 +++ .../project-build-and-deploy-pr.yml | 121 ++++++++++-------- .buildkite/pull_requests.json | 7 +- .buildkite/scripts/lifecycle/comment_on_pr.ts | 64 +++++++++ 5 files changed, 160 insertions(+), 54 deletions(-) create mode 100644 .buildkite/scripts/lifecycle/comment_on_pr.ts diff --git a/.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml b/.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml index 3c1bdc00ba371..490c9d9afc4e4 100644 --- a/.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml +++ b/.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml @@ -28,9 +28,11 @@ spec: pipeline_file: .buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml skip_intermediate_builds: true provider_settings: + build_pull_requests: true prefix_pull_request_fork_branch_names: false skip_pull_request_builds_for_existing_commits: true trigger_mode: none + cancel_intermediate_builds: true teams: kibana-operations: access_level: MANAGE_BUILD_AND_READ diff --git a/.buildkite/pipeline-utils/github/github.ts b/.buildkite/pipeline-utils/github/github.ts index 0a7970d750598..eb9a240386bbc 100644 --- a/.buildkite/pipeline-utils/github/github.ts +++ b/.buildkite/pipeline-utils/github/github.ts @@ -93,6 +93,26 @@ export const doAnyChangesMatch = async ( return anyFilesMatchRequired; }; +export function addComment( + comment: string, + owner = process.env.GITHUB_PR_BASE_OWNER, + repo = process.env.GITHUB_PR_BASE_REPO, + prNumber: undefined | string | number = process.env.GITHUB_PR_NUMBER +) { + if (!owner || !repo || !prNumber) { + throw Error( + "Couldn't retrieve Github PR info from environment variables in order to add a comment" + ); + } + + return github.issues.createComment({ + owner, + repo, + issue_number: typeof prNumber === 'number' ? prNumber : parseInt(prNumber, 10), + body: comment, + }); +} + export function getGithubClient() { return github; } diff --git a/.buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml b/.buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml index f7fc94ac444e1..04b738ff363e1 100644 --- a/.buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml +++ b/.buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml @@ -1,53 +1,72 @@ -agents: - provider: gcp - image: family/kibana-ubuntu-2004 - imageProject: elastic-images-prod +env: + ELASTIC_PR_COMMENTS_ENABLED: 'true' + GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true' + GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-deploy-project-from-pr steps: - - command: .buildkite/scripts/lifecycle/pre_build.sh - label: Pre-Build - timeout_in_minutes: 10 - agents: - machineType: n2-standard-2 - retry: - automatic: - - exit_status: '*' - limit: 1 - - - wait: ~ - - - command: .buildkite/scripts/steps/build_kibana.sh - label: Build Kibana Distribution and Plugins - agents: - machineType: n2-standard-16 - preemptible: true - key: build - if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" - timeout_in_minutes: 90 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - - wait: ~ - - - command: .buildkite/scripts/steps/artifacts/docker_image.sh - label: 'Build Project Image' - key: build_project_image - agents: - machineType: n2-standard-16 - preemptible: true - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: '-1' - limit: 3 - - - wait: ~ - - - command: .buildkite/scripts/steps/serverless/deploy.sh - label: 'Deploy Project' - agents: - machineType: n2-standard-4 - preemptible: true - timeout_in_minutes: 10 + - group: 'Project Deployment' + if: "build.env('GITHUB_PR_LABELS') =~ /ci:project-deploy-(elasticsearch|observability|security)/" + + steps: + - command: .buildkite/scripts/lifecycle/pre_build.sh + label: Pre-Build + timeout_in_minutes: 10 + agents: + provider: gcp + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + machineType: n2-standard-2 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: | + ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts "PR Project deployment started at: $BUILDKITE_BUILD_URL" + label: Comment with job URL + agents: + provider: gcp + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + machineType: n2-standard-2 + timeout_in_minutes: 5 + + - wait: ~ + + - command: .buildkite/scripts/steps/artifacts/docker_image.sh + label: 'Build Project Image' + key: build_project_image + agents: + provider: gcp + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + machineType: n2-standard-16 + preemptible: true + timeout_in_minutes: 60 + retry: + automatic: + - exit_status: '-1' + limit: 3 + + - wait: ~ + - command: .buildkite/scripts/steps/serverless/deploy.sh + label: 'Deploy Project' + agents: + provider: gcp + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + machineType: n2-standard-4 + preemptible: true + timeout_in_minutes: 10 + + - wait: ~ + + - command: | + ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts "Project deployed, see credentials at: $BUILDKITE_BUILD_URL" + label: Comment with job URL + agents: + provider: gcp + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + machineType: n2-standard-2 + timeout_in_minutes: 5 diff --git a/.buildkite/pull_requests.json b/.buildkite/pull_requests.json index 20785e92be1b5..cbc0e9df03dc8 100644 --- a/.buildkite/pull_requests.json +++ b/.buildkite/pull_requests.json @@ -49,14 +49,15 @@ "repoOwner": "elastic", "repoName": "kibana", "pipelineSlug": "kibana-deploy-project-from-pr", - + "skip_ci_labels": [], "enabled": true, "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], "allowed_list": ["elastic-vault-github-plugin-prod[bot]"], - "set_commit_status": false, + "set_commit_status": true, + "commit_status_context": "kibana-deploy-project-from-pr", "build_on_commit": false, - "build_on_comment": false, + "build_on_comment": true, "build_drafts": false, "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:deploy)\\W+(?:project))$", "kibana_versions_check": true, diff --git a/.buildkite/scripts/lifecycle/comment_on_pr.ts b/.buildkite/scripts/lifecycle/comment_on_pr.ts new file mode 100644 index 0000000000000..39ebd511d8410 --- /dev/null +++ b/.buildkite/scripts/lifecycle/comment_on_pr.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { addComment } from '#pipeline-utils'; + +const ALLOWED_ENV_VARS = [ + 'BUILDKITE_BRANCH', + 'BUILDKITE_BUILD_ID', + 'BUILDKITE_BUILD_NUMBER', + 'BUILDKITE_BUILD_URL', + 'BUILDKITE_COMMIT', + 'BUILDKITE_PIPELINE_NAME', + 'BUILDKITE_PIPELINE_SLUG', + 'GITHUB_PR_BASE_OWNER', + 'GITHUB_PR_BASE_REPO', + 'GITHUB_PR_BRANCH', + 'GITHUB_PR_HEAD_SHA', + 'GITHUB_PR_HEAD_USER', + 'GITHUB_PR_LABELS', + 'GITHUB_PR_NUMBER', + 'GITHUB_PR_OWNER', + 'GITHUB_PR_REPO', + 'GITHUB_PR_TARGET_BRANCH', + 'GITHUB_PR_TRIGGERED_SHA', + 'GITHUB_PR_TRIGGER_USER', + 'GITHUB_PR_USER', +]; +const DEFAULT_MESSAGE_TEMPLATE = + '🚀 Buildkite job started for PR #${GITHUB_PR_NUMBER}: ${BUILDKITE_BUILD_URL}'; + +export function commentOnPR() { + const messageTemplate = + process.argv.slice(2)?.join(' ') || + process.env.JOB_START_COMMENT_TEMPLATE || + DEFAULT_MESSAGE_TEMPLATE; + if (messageTemplate === DEFAULT_MESSAGE_TEMPLATE) { + console.log('No message template provided, using default message'); + } else { + console.log(`Using message template: ${messageTemplate}`); + } + + const message = messageTemplate.replace(/\${([^}]+)}/g, (_, envVar) => { + if (ALLOWED_ENV_VARS.includes(envVar)) { + return process.env[envVar] || ''; + } else { + return '${' + envVar + '}'; + } + }); + + return addComment(message); +} + +if (require.main === module) { + commentOnPR().catch((error) => { + console.error(error); + process.exit(1); + }); +}