[8.12](backport #37926) migrate winlogbeat pipeline

.buildkite/hooks/pre-command

@@ -12,8 +12,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
   fi
 fi
 
-
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" ]]; then
   source .buildkite/scripts/setenv.sh
   if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then
     echo "Skipped pre-command when running the Upload pipeline"

.buildkite/libbeat/pipeline.libbeat.yml

@@ -39,7 +39,3 @@ steps:
   - label: ":linux: Load dynamic Libbeat pipeline"
     key: "libbeat-pipeline"
     command: ".buildkite/scripts/generate_libbeat_pipeline.sh"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_DEFAULT_MACHINE_TYPE}"

.buildkite/metricbeat/pipeline.yml

@@ -8,7 +8,6 @@ env:
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
-  GO_AGENT_IMAGE: "golang:${GO_VERSION}"
   BEATS_PROJECT_NAME: "metricbeat"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
@@ -46,7 +45,3 @@ steps:
   - label: ":linux: Load dynamic metricbeat pipeline"
     key: "metricbeat-pipeline"
     command: ".buildkite/scripts/generate_metricbeat_pipeline.sh"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_DEFAULT_MACHINE_TYPE}"

.buildkite/packetbeat/pipeline.packetbeat.yml

@@ -53,6 +53,3 @@ steps:
   - label: ":linux: Load dynamic packetbeat pipeline"
     key: "packetbeat-pipeline"
     command: ".buildkite/scripts/generate_packetbeat_pipeline.sh"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"

.buildkite/scripts/common.sh

@@ -7,11 +7,12 @@ platform_type="$(uname)"
 platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]')
 arch_type="$(uname -m)"
 GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""}
+GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""}
 ONLY_DOCS=${ONLY_DOCS:-"true"}
-
 [ -z "${runLibbeat+x}" ] && runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
 [ -z "${runMetricbeat+x}" ] && runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})"
 [ -z "${runPacketbeat+x}" ] && runPacketbeat="$(buildkite-agent meta-data get runPacketbeat --default ${runPacketbeat:-"false"})"
+[ -z "${runWinlogbeat+x}" ] && runWinlogbeat="$(buildkite-agent meta-data get runWinlogbeat --default ${runWinlogbeat:-"false"})"
 [ -z "${runLibBeatArmTest+x}" ] && runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
 [ -z "${runPacketbeatArmTest+x}" ] && runPacketbeatArmTest="$(buildkite-agent meta-data get runPacketbeatArmTest --default ${runPacketbeatArmTest:-"false"})"
 [ -z "${runMetricbeatMacOsTests+x}" ] && runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})"
@@ -29,6 +30,10 @@ packetbeat_changeset=(
   "^packetbeat/.*"
   )
 
+winlogbeat_changeset=(
+  "^winlogbeat/.*"
+  )
+
 oss_changeset=(
   "^go.mod"
   "^pytest.ini"
@@ -231,18 +236,23 @@ are_changed_only_paths() {
 
 are_conditions_met_mandatory_tests() {
   if are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
-    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
-      if are_paths_changed "${metricbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then
-        return 0
-      fi
-    elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
-      if are_paths_changed "${libbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then
-        return 0
-      fi
-    elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
-      if are_paths_changed "${packetbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat" || "${GITHUB_PR_LABELS}" =~ Packetbeat || "${runPacketbeat}" == "true" ]]; then
-        return 0
-      fi
+    return 0
+  fi
+  if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
+    if are_paths_changed "${metricbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then
+      return 0
+    fi
+  elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
+    if are_paths_changed "${libbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then
+      return 0
+    fi
+  elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
+    if are_paths_changed "${packetbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat" || "${GITHUB_PR_LABELS}" =~ Packetbeat || "${runPacketbeat}" == "true" ]]; then
+      return 0
+    fi
+  elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" ]]; then
+    if are_paths_changed "${winlogbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test winlogbeat" || "${GITHUB_PR_LABELS}" =~ Winlogbeat || "${runWinlogbeat}" == "true" ]]; then
+      return 0
     fi
   fi
   return 1
@@ -280,14 +290,8 @@ are_conditions_met_macos_tests() {
 
 are_conditions_met_packaging() {
   if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
-    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
-      if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103
-        return 0
-      fi
-    elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
-      if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then
-        return 0
-      fi
+    if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then
+      return 0
     fi
   fi
   return 1

.buildkite/scripts/generate_winlogbeat_pipeline.sh

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+pipelineName="pipeline.winlogbeat-dynamic.yml"
+
+echo "Add the mandatory and extended tests without additional conditions into the pipeline"
+if are_conditions_met_mandatory_tests; then
+  cat > $pipelineName <<- YAML
+
+steps:
+
+  - group: "Mandatory Tests"
+    key: "mandatory-tests"
+    steps:
+
+      - label: ":negative_squared_cross_mark: Cross compile"
+        key: "mandatory-cross-compile"
+        command: ".buildkite/scripts/crosscompile.sh"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows: Windows 2016/2019/2022 Unit Tests - {{matrix.image}}"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
+        key: "mandatory-win-unit-tests"
+        agents:
+          provider: "gcp"
+          image: "{{matrix.image}}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_WIN_2016}"
+              - "${IMAGE_WIN_2019}"
+              - "${IMAGE_WIN_2022}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+# echo "Add the extended windows tests into the pipeline"
+# TODO: ADD conditions from the main pipeline
+
+  - group: "Extended Windows Tests"
+    key: "extended-win-tests"
+    steps:
+
+      - label: ":windows: Windows 10 Unit Tests"
+        key: "extended-win-10-unit-tests"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_10}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows: Windows 11 Unit Tests"
+        key: "extended-win-11-unit-tests"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_11}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+YAML
+else
+  echo "The conditions don't match to requirements for generating pipeline steps."
+  exit 0
+fi
+
+echo "Check and add the Packaging into the pipeline"
+if are_conditions_met_packaging; then
+  cat >> $pipelineName <<- YAML
+
+  - wait: ~
+    depends_on:
+      - step: "mandatory-tests"
+        allow_failure: false
+
+  - group: "Packaging"    # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589
+    key: "packaging"
+    steps:
+      - label: ":linux: Packaging Linux"
+        key: "packaging-linux"
+        command: ".buildkite/scripts/packaging.sh"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+
+
+YAML
+fi
+
+echo "--- Printing dynamic steps"     #TODO: remove if the pipeline is public
+cat $pipelineName
+
+echo "--- Loading dynamic steps"
+buildkite-agent pipeline upload $pipelineName

.buildkite/scripts/win_unit_tests.ps1

@@ -14,9 +14,12 @@ function withChoco {
     Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
 }
 function withGolang($version) {
-    Write-Host "-- Install golang $version --"
-    choco install -y golang --version=$version
-    refreshenv
+    $downloadPath = Join-Path $env:TEMP "go_installer.msi"
+    $goInstallerUrl = "https://golang.org/dl/go$version.windows-amd64.msi"
+    Invoke-WebRequest -Uri $goInstallerUrl -OutFile $downloadPath
+    Start-Process -FilePath "msiexec.exe" -ArgumentList "/i $downloadPath /quiet" -Wait
+    $goBinPath = "${env:ProgramFiles}\Go\bin"
+    $env:Path += ";$goBinPath"
     go version
 }
 function withPython($version) {

.buildkite/winlogbeat/pipeline.winlogbeat.yml

@@ -1,5 +1,35 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+env:
+  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_WIN_10: "family/general-windows-10"
+  IMAGE_WIN_11: "family/general-windows-11"
+  IMAGE_WIN_2016: "family/core-windows-2016"
+  IMAGE_WIN_2019: "family/core-windows-2019"
+  IMAGE_WIN_2022: "family/core-windows-2022"
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+  BEATS_PROJECT_NAME: "winlogbeat"
 
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+
+  - input: "Input Parameters"
+    key: "input-run-all-stages"
+    fields:
+    - select: "Packetbeat - runWinlogbeat"
+      key: "runWinlogbeat"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+    if: "build.source == 'ui'"
+
+  - wait: ~
+    if: "build.source == 'ui'"
+    allow_dependency_failure: false
+
+  - label: ":linux: Load dynamic winlogbeat pipeline"
+    key: "winlogbeat-pipeline"
+    command: ".buildkite/scripts/generate_winlogbeat_pipeline.sh"