From 8880d60280e2da71424b278737ed1a086fb21ead Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 18:04:32 +0100 Subject: [PATCH 01/11] github-action: support check/lint --- .github/workflows/check-default.yml | 10 +++++++ .github/workflows/check-deploy-kubernetes.yml | 28 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 .github/workflows/check-deploy-kubernetes.yml diff --git a/.github/workflows/check-default.yml b/.github/workflows/check-default.yml index 56bf10ab779..051d27b6540 100644 --- a/.github/workflows/check-default.yml +++ b/.github/workflows/check-default.yml @@ -24,3 +24,13 @@ jobs: run: | go install github.com/magefile/mage make check-default + + unified-release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run check/update + run: | + GO_VERSION=$(cat .go-version) ./dev-tools/run_with_go_ver make test-mage + env: + GOFLAGS: '-mod=readonly' diff --git a/.github/workflows/check-deploy-kubernetes.yml b/.github/workflows/check-deploy-kubernetes.yml new file mode 100644 index 00000000000..3482fe026e4 --- /dev/null +++ b/.github/workflows/check-deploy-kubernetes.yml @@ -0,0 +1,28 @@ +name: check-deploy-kubernetes + +on: + pull_request: + paths: + - "^deploy/kubernetes/.*" + - "^libbeat/docs/version.asciidoc" + push: + branches: + - main + - 7.1* + - 8.* + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version-file: .go-version + - name: Run deploy/kubernetes all + run: make -C deploy/kubernetes all + - name: Run check-no-changes + run: make check-no-changes \ No newline at end of file From 242691b75a77a3080f8a4092b7a9116c82b08b18 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 18:08:37 +0100 Subject: [PATCH 02/11] github-action: enable precommit --- .github/workflows/pre-commit.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/pre-commit.yml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 00000000000..595ac345053 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,19 @@ +name: pre-commit + +on: + # TODO: skip if doc changes only (.*\\.(asciidoc|md)|deploy/kubernetes/.*-kubernetes\\.yaml) + pull_request: + push: + branches: + - main + - 7.1* + - 8.* + +permissions: + contents: read + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: elastic/apm-pipeline-library/.github/actions/pre-commit@current From e2b8d36a38c18c2c9ea1ae4b7eecdaa3257e6fb5 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 18:08:59 +0100 Subject: [PATCH 03/11] remove unrequired steps use github actions instead --- .pre-commit-config.yaml | 7 ----- Jenkinsfile | 45 ------------------------------- deploy/kubernetes/Jenkinsfile.yml | 5 ---- 3 files changed, 57 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 200d0b9277a..cf3ceb27625 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,10 +3,3 @@ repos: rev: v4.4.0 hooks: - id: check-merge-conflict - -- repo: https://github.com/elastic/apm-pipeline-library.git - rev: v1.1.397 - hooks: - - id: check-jenkins-pipelines - files: ^(.ci/(.*\.groovy|Jenkinsfile)|Jenkinsfile)$ - - id: check-jjbb diff --git a/Jenkinsfile b/Jenkinsfile index 9733b7b74c9..4cfe547b02c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -80,28 +80,6 @@ pipeline { } } } - stage('Checks'){ - options { skipDefaultCheckout() } - environment { - GOFLAGS = '-mod=readonly' - } - steps { - withGithubNotify(context: "Checks") { - stageStatusCache(id: 'Checks'){ - // test the ./dev-tools/run_with_go_ver used by the Unified Release process - dir("${BASE_DIR}") { - sh "HOME=${WORKSPACE} GO_VERSION=${GO_VERSION} ./dev-tools/run_with_go_ver make test-mage" - } - withBeatsEnv(archive: false, id: "checks") { - dumpVariables() - whenTrue(env.ONLY_DOCS == 'false') { - runChecks() - } - } - } - } - } - } stage('Build&Test') { options { skipDefaultCheckout() } when { @@ -218,29 +196,6 @@ def isGitHubIssueEnabled() { return isBranch() && currentBuild.currentResult != "SUCCESS" && currentBuild.currentResult != "ABORTED" } -def runChecks() { - def mapParallelTasks = [:] - def content = readYaml(file: 'Jenkinsfile.yml') - content['projects'].each { projectName -> - generateStages(project: projectName, changeset: content['changeset'], filterStage: 'checks').each { k,v -> - mapParallelTasks["${k}"] = v - } - } - // Run pre-commit within the current node and in Jenkins - // hence there is no need to use docker login in the GitHub actions - // some docker images are hosted in an internal docker registry. - mapParallelTasks['pre-commit'] = runPreCommit() - parallel(mapParallelTasks) -} - -def runPreCommit() { - return { - withGithubNotify(context: 'Check pre-commit', tab: 'tests') { - preCommit(commit: "${GIT_BASE_COMMIT}", junit: true) - } - } -} - def runBuildAndTest(Map args = [:]) { def filterStage = args.get('filterStage', 'mandatory') deleteDir() diff --git a/deploy/kubernetes/Jenkinsfile.yml b/deploy/kubernetes/Jenkinsfile.yml index 4e6c799e0de..06a2a1e99eb 100644 --- a/deploy/kubernetes/Jenkinsfile.yml +++ b/deploy/kubernetes/Jenkinsfile.yml @@ -12,11 +12,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-22" ## default label for all the stages stages: - checks: - make: | - make -C deploy/kubernetes all; - make check-no-changes; - stage: checks k8sTest: k8sTest: "v1.29.0,v1.28.0,v1.27.3,v1.26.6" stage: mandatory From b502de3d1ea7d95ee019ba440777eebfb891bd2b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 18:14:22 +0100 Subject: [PATCH 04/11] test --- .github/workflows/check-default.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-default.yml b/.github/workflows/check-default.yml index 051d27b6540..b4e8f7d35d2 100644 --- a/.github/workflows/check-default.yml +++ b/.github/workflows/check-default.yml @@ -31,6 +31,7 @@ jobs: - uses: actions/checkout@v4 - name: Run check/update run: | - GO_VERSION=$(cat .go-version) ./dev-tools/run_with_go_ver make test-mage + #GO_VERSION=$(cat .go-version) ./dev-tools/run_with_go_ver make test-mage + GO_VERSION='1.21.6' ./dev-tools/run_with_go_ver make test-mage env: GOFLAGS: '-mod=readonly' From 786a42cbfd4feb411ab460036e6a2e76fba88c90 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 18:22:03 +0100 Subject: [PATCH 05/11] DRA --- .github/workflows/check-default.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/check-default.yml b/.github/workflows/check-default.yml index b4e8f7d35d2..56bf10ab779 100644 --- a/.github/workflows/check-default.yml +++ b/.github/workflows/check-default.yml @@ -24,14 +24,3 @@ jobs: run: | go install github.com/magefile/mage make check-default - - unified-release: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Run check/update - run: | - #GO_VERSION=$(cat .go-version) ./dev-tools/run_with_go_ver make test-mage - GO_VERSION='1.21.6' ./dev-tools/run_with_go_ver make test-mage - env: - GOFLAGS: '-mod=readonly' From 4e42dc799dc208fa73e585fcab9cf2bc071a9fb8 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 18:22:50 +0100 Subject: [PATCH 06/11] if changes in the workflow --- .github/workflows/check-deploy-kubernetes.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check-deploy-kubernetes.yml b/.github/workflows/check-deploy-kubernetes.yml index 3482fe026e4..d1974fd673a 100644 --- a/.github/workflows/check-deploy-kubernetes.yml +++ b/.github/workflows/check-deploy-kubernetes.yml @@ -5,6 +5,7 @@ on: paths: - "^deploy/kubernetes/.*" - "^libbeat/docs/version.asciidoc" + - "^.github/workflows/check-deploy-kubernetes.yml" push: branches: - main From 5df2fe06b8c2cf34e2900032eab9b5229d4dca21 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 18:23:51 +0100 Subject: [PATCH 07/11] fix regex --- .github/workflows/check-deploy-kubernetes.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check-deploy-kubernetes.yml b/.github/workflows/check-deploy-kubernetes.yml index d1974fd673a..4005defab49 100644 --- a/.github/workflows/check-deploy-kubernetes.yml +++ b/.github/workflows/check-deploy-kubernetes.yml @@ -3,9 +3,9 @@ name: check-deploy-kubernetes on: pull_request: paths: - - "^deploy/kubernetes/.*" - - "^libbeat/docs/version.asciidoc" - - "^.github/workflows/check-deploy-kubernetes.yml" + - "deploy/kubernetes/.*" + - "libbeat/docs/version.asciidoc" + - ".github/workflows/check-deploy-kubernetes.yml" push: branches: - main From 7a8d2f34b9ad1e743368195ebc39db2b3512aea2 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 18:59:22 +0100 Subject: [PATCH 08/11] Update .github/workflows/pre-commit.yml --- .github/workflows/pre-commit.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 595ac345053..096dee6bea5 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -1,7 +1,6 @@ name: pre-commit on: - # TODO: skip if doc changes only (.*\\.(asciidoc|md)|deploy/kubernetes/.*-kubernetes\\.yaml) pull_request: push: branches: From 2505ea4ab68512c77724ea7cd588c80982c37d1c Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 15 May 2024 16:40:41 +0200 Subject: [PATCH 09/11] keep previous setup --- .github/workflows/check-deploy-kubernetes.yml | 29 ---------- Jenkinsfile | 58 +++++++++++++++---- deploy/kubernetes/Jenkinsfile.yml | 5 ++ 3 files changed, 53 insertions(+), 39 deletions(-) delete mode 100644 .github/workflows/check-deploy-kubernetes.yml diff --git a/.github/workflows/check-deploy-kubernetes.yml b/.github/workflows/check-deploy-kubernetes.yml deleted file mode 100644 index 4005defab49..00000000000 --- a/.github/workflows/check-deploy-kubernetes.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: check-deploy-kubernetes - -on: - pull_request: - paths: - - "deploy/kubernetes/.*" - - "libbeat/docs/version.asciidoc" - - ".github/workflows/check-deploy-kubernetes.yml" - push: - branches: - - main - - 7.1* - - 8.* - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version-file: .go-version - - name: Run deploy/kubernetes all - run: make -C deploy/kubernetes all - - name: Run check-no-changes - run: make check-no-changes \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index 4cfe547b02c..04df94f4b25 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -80,6 +80,28 @@ pipeline { } } } + stage('Checks'){ + options { skipDefaultCheckout() } + environment { + GOFLAGS = '-mod=readonly' + } + steps { + withGithubNotify(context: "Checks") { + stageStatusCache(id: 'Checks'){ + // test the ./dev-tools/run_with_go_ver used by the Unified Release process + dir("${BASE_DIR}") { + sh "HOME=${WORKSPACE} GO_VERSION=${GO_VERSION} ./dev-tools/run_with_go_ver make test-mage" + } + withBeatsEnv(archive: false, id: "checks") { + dumpVariables() + whenTrue(env.ONLY_DOCS == 'false') { + runChecks() + } + } + } + } + } + } stage('Build&Test') { options { skipDefaultCheckout() } when { @@ -178,16 +200,6 @@ COMMIT=${env.GIT_BASE_COMMIT} VERSION=${env.VERSION}-SNAPSHOT""") archiveArtifacts artifacts: 'packaging.properties' } - cleanup { - // Required to enable the flaky test reporting with GitHub. Workspace exists since the post/always runs earlier - dir("${BASE_DIR}"){ - notifyBuildResult(prComment: true, - slackComment: true, - analyzeFlakey: !isTag(), jobName: getFlakyJobName(withBranch: getFlakyBranch()), - githubIssue: isGitHubIssueEnabled(), - githubLabels: 'Team:Elastic-Agent-Data-Plane') - } - } } } @@ -196,6 +208,29 @@ def isGitHubIssueEnabled() { return isBranch() && currentBuild.currentResult != "SUCCESS" && currentBuild.currentResult != "ABORTED" } +def runChecks() { + def mapParallelTasks = [:] + def content = readYaml(file: 'Jenkinsfile.yml') + content['projects'].each { projectName -> + generateStages(project: projectName, changeset: content['changeset'], filterStage: 'checks').each { k,v -> + mapParallelTasks["${k}"] = v + } + } + // Run pre-commit within the current node and in Jenkins + // hence there is no need to use docker login in the GitHub actions + // some docker images are hosted in an internal docker registry. + mapParallelTasks['pre-commit'] = runPreCommit() + parallel(mapParallelTasks) +} + +def runPreCommit() { + return { + withGithubNotify(context: 'Check pre-commit', tab: 'tests') { + preCommit(commit: "${GIT_BASE_COMMIT}", junit: true) + } + } +} + def runBuildAndTest(Map args = [:]) { def filterStage = args.get('filterStage', 'mandatory') deleteDir() @@ -563,6 +598,9 @@ def targetWithoutNode(Map args = [:]) { } } withTools(k8s: installK8s, gcp: withGCP, nodejs: withNodejs) { + if (isPackaging && (directory.equals('x-pack/agentbeat') || directory.equals('x-pack/osquerybeat'))) { + sh(label: 'install msitools', script: '.buildkite/scripts/install-msitools.sh') + } // make commands use -C while mage commands require the dir(folder) // let's support this scenario with the location variable. dir(isMage ? directory : '') { diff --git a/deploy/kubernetes/Jenkinsfile.yml b/deploy/kubernetes/Jenkinsfile.yml index 06a2a1e99eb..4e6c799e0de 100644 --- a/deploy/kubernetes/Jenkinsfile.yml +++ b/deploy/kubernetes/Jenkinsfile.yml @@ -12,6 +12,11 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-22" ## default label for all the stages stages: + checks: + make: | + make -C deploy/kubernetes all; + make check-no-changes; + stage: checks k8sTest: k8sTest: "v1.29.0,v1.28.0,v1.27.3,v1.26.6" stage: mandatory From d243bf9580de7a25ed27456d93c7ab0850fb09ee Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 15 May 2024 16:41:33 +0200 Subject: [PATCH 10/11] Update Jenkinsfile --- Jenkinsfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 04df94f4b25..a93f61a0e22 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -598,9 +598,6 @@ def targetWithoutNode(Map args = [:]) { } } withTools(k8s: installK8s, gcp: withGCP, nodejs: withNodejs) { - if (isPackaging && (directory.equals('x-pack/agentbeat') || directory.equals('x-pack/osquerybeat'))) { - sh(label: 'install msitools', script: '.buildkite/scripts/install-msitools.sh') - } // make commands use -C while mage commands require the dir(folder) // let's support this scenario with the location variable. dir(isMage ? directory : '') { From 0b74c3b55d59a9d601a973e50b2b42c5543b73df Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 15 May 2024 16:42:11 +0200 Subject: [PATCH 11/11] Update Jenkinsfile --- Jenkinsfile | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index a93f61a0e22..9733b7b74c9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -200,6 +200,16 @@ COMMIT=${env.GIT_BASE_COMMIT} VERSION=${env.VERSION}-SNAPSHOT""") archiveArtifacts artifacts: 'packaging.properties' } + cleanup { + // Required to enable the flaky test reporting with GitHub. Workspace exists since the post/always runs earlier + dir("${BASE_DIR}"){ + notifyBuildResult(prComment: true, + slackComment: true, + analyzeFlakey: !isTag(), jobName: getFlakyJobName(withBranch: getFlakyBranch()), + githubIssue: isGitHubIssueEnabled(), + githubLabels: 'Team:Elastic-Agent-Data-Plane') + } + } } }