From c9428eb12f81c49cb4521e18f3c3a329fa27fa5f Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 17:34:55 +0300 Subject: [PATCH] Buildkite Xpack/Winlogbeat pipeline fixes (#39007) (#39102) * updated common.sh * fixed typo * updated common.sh * added changeset ps1 script and updated xpack-winlogbeat pipeline * pr fixes * handle backslashes in changeset directories (cherry picked from commit 243f9c33d8aab9e5aba7584d7b3eb64cb28bc18c) Co-authored-by: Olga Naydyonock --- .buildkite/scripts/changesets.psm1 | 72 +++++++++++++++++++ .buildkite/scripts/common.sh | 9 +-- .../x-pack/pipeline.xpack.winlogbeat.yml | 38 +++++----- 3 files changed, 93 insertions(+), 26 deletions(-) create mode 100644 .buildkite/scripts/changesets.psm1 diff --git a/.buildkite/scripts/changesets.psm1 b/.buildkite/scripts/changesets.psm1 new file mode 100644 index 00000000000..15fabd3eba5 --- /dev/null +++ b/.buildkite/scripts/changesets.psm1 @@ -0,0 +1,72 @@ +function ArePathsChanged($patterns) { + $changedlist = @() + foreach ($pattern in $patterns) { + $changedFiles = & git diff --name-only "HEAD@{1}" HEAD | Select-String -Pattern $pattern -SimpleMatch + if ($changedFiles) { + $changedlist += $changedFiles + } + } + if ($changedlist) { + Write-Host "--- Files changed: $changedlist" + return $true + } + else { + Write-Host "--- No files changed within specified changeset: $patterns" + return $false + } +} + +function AreChangedOnlyPaths($patterns) { + $changedFiles = & git diff --name-only "HEAD@{1}" HEAD + Write-Host "--- Git Diff result:" + Write-Host "$changedFiles" + + $matchedFiles = @() + foreach ($pattern in $patterns) { + $matched = $changedFiles | Select-String -Pattern $pattern -SimpleMatch + if ($matched) { + $matchedFiles += $matched + } + } + if (($matchedFiles.Count -eq $changedFiles.Count) -or ($changedFiles.Count -eq 0)) { + return $true + } + return $false +} + +# This function sets a `MODULE` env var, required by IT tests, containing a comma separated list of modules for a given beats project (specified via the first argument). +# The list is built depending on directories that have changed under `modules/` excluding anything else such as asciidoc and png files. +# `MODULE` will empty if no changes apply. +function DefineModuleFromTheChangeSet($projectPath) { + $projectPathTransformed = $projectPath -replace '/', '\\' + $projectPathExclusion = "((?!^$projectPathTransformed\\\/).)*\$" + $exclude = @("^($projectPathExclusion|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") + + $changedModules = '' + + $moduleDirs = Get-ChildItem -Directory "$projectPath\module" + foreach($moduleDir in $moduleDirs) { + if((ArePathsChanged($moduleDir)) -and !(AreChangedOnlyPaths($exclude))) { + if(!$changedModules) { + $changedModules = $moduleDir.Name + } + else { + $changedModules += ',' + $moduleDir.Name + } + } + } + + # TODO: remove this conditional when issue https://github.com/elastic/ingest-dev/issues/2993 gets resolved + if(!$changedModules) { + if($Env:BUILDKITE_PIPELINE_SLUG -eq 'beats-xpack-metricbeat') { + $Env:MODULE = "aws" + } + else { + $Env:MODULE = "kubernetes" + } + } + else { + # TODO: once https://github.com/elastic/ingest-dev/issues/2993 gets resolved, this should be the only thing we export + $Env:MODULE = $changedModules + } +} diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 857ca17681e..3355dcc0aa4 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -39,10 +39,6 @@ xpack_packetbeat_changeset=( "^x-pack/packetbeat/.*" ) -xpack_winlogbeat_changeset=( - "^x-pack/winlogbeat/.*" - ) - ci_changeset=( "^.buildkite/.*" ) @@ -74,9 +70,6 @@ packaging_changeset=( ) case "${BUILDKITE_PIPELINE_SLUG}" in - "beats-winlogbeat") - BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]} - ;; "beats-xpack-metricbeat") BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]} ;; @@ -497,7 +490,7 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi -if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then +if [[ "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" fi diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index e1e2ec3e045..c07e537adf0 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -15,13 +15,16 @@ env: ASDF_MAGE_VERSION: 1.15.0 steps: - - group: "Mandatory Tests" - key: "xpack-winlogbeat-mandatory-tests" + - group: "x-pack/Winlogbeat Mandatory Tests" + key: "x-pack-winlogbeat-mandatory-tests" steps: - - label: ":windows: Xpack/Winlogbeat Win-2019 Unit (MODULE) Tests" + - label: ":windows: x-pack/Winlogbeat Win-2019 Unit (MODULE) Tests" key: "mandatory-win-2019-module-unit-tests" command: | + Import-Module ./.buildkite/scripts/changesets.psm1 + defineModuleFromTheChangeSet 'x-pack/winlogbeat' + Write-Output "~~~ Will run tests with env var MODULE=$$Env:MODULE" Set-Location -Path x-pack/winlogbeat mage build unitTest env: @@ -37,9 +40,9 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "Xpack/Winlogbeat Win-2019 Unit (MODULE) Tests" + context: "x-pack/Winlogbeat Win-2019 Unit (MODULE) Tests" - - label: ":windows: Xpack/Winlogbeat Win-2016 Unit Tests" + - label: ":windows: x-pack/Winlogbeat Win-2016 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -55,9 +58,9 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "Xpack/Winlogbeat Win-2016 Unit Tests" + context: "x-pack/Winlogbeat Win-2016 Unit Tests" - - label: ":windows: Xpack/Winlogbeat Win-2022 Unit Tests" + - label: ":windows: x-pack/Winlogbeat Win-2022 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -73,14 +76,14 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "Xpack/Winlogbeat Win-2022 Unit Tests" + context: "x-pack/Winlogbeat Win-2022 Unit Tests" - group: "Extended Windows Tests" key: "extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - - label: ":windows: Xpack/Winlogbeat Win-10 Unit Tests" + - label: ":windows: x-pack/Winlogbeat Win-10 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -96,9 +99,9 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "Xpack/Winlogbeat Win-10 Unit Tests" + context: "x-pack/Winlogbeat Win-10 Unit Tests" - - label: ":windows: Xpack/Winlogbeat Win-11 Unit Tests" + - label: ":windows: x-pack/Winlogbeat Win-11 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -114,9 +117,9 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "Xpack/Winlogbeat Win-11 Unit Tests" + context: "x-pack/Winlogbeat Win-11 Unit Tests" - - label: ":windows: Xpack/Winlogbeat Win-2019 Unit Tests" + - label: ":windows: x-pack/Winlogbeat Win-2019 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -140,10 +143,10 @@ steps: # this allows building DRA artifacts even if there is flakiness in mandatory tests if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: - - "xpack-winlogbeat-mandatory-tests" + - "x-pack-winlogbeat-mandatory-tests" - - group: "Xpack/Winlogbeat Packaging" - key: "xpack-winlogbeat-packaging" + - group: "x-pack/Winlogbeat Packaging" + key: "x-pack-winlogbeat-packaging" steps: - label: ":ubuntu: Packaging Linux" @@ -159,5 +162,4 @@ steps: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: - context: "Xpack/Winlogbeat Packaging" - + context: "x-pack/Winlogbeat Packaging"