[7.17](backport #38659) Deploy K8S Pipeline migration (#38712)

* Deploy K8S Pipeline migration (#38659)

* updated auditbeat & deploy k8s pipelines

* added kind & kuve env vars, set higer ulimit

* added k8s env setup

(cherry picked from commit aaa4829)

* fixed conflicts n fileinfo test

* updated auditbeat and filebeat pipelines

* added macos arm image

---------

Co-authored-by: Olga Naydyonock <[email protected]>

.buildkite/auditbeat/auditbeat-pipeline.yml

@@ -3,20 +3,20 @@ name: "beats-auditbeat"
 
 env:
   BEATS_PROJECT_NAME: "auditbeat"
-  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
   IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
   IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
   IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
   IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016"
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
-  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
 
 steps:
   - input: "Input Parameters"
@@ -62,7 +62,7 @@ steps:
 
   - label: ":linux: Load dynamic auditbeat pipeline"
     key: "auditbeat-pipeline"
-    command: ".buildkite/scripts/generate_auditbeat_pipeline.sh"
+    command: ".buildkite/auditbeat/generate_auditbeat_pipeline.sh"
     notify:
       - github_commit_status:
           context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"

.buildkite/scripts/generate_auditbeat_pipeline.sh → .buildkite/auditbeat/generate_auditbeat_pipeline.sh

@@ -6,77 +6,69 @@ set -euo pipefail
 
 pipelineName="pipeline.auditbeat-dynamic.yml"
 
-# TODO: steps: must be always included 
+# TODO: steps: must be always included
 echo "Add the mandatory and extended tests without additional conditions into the pipeline"
 if are_conditions_met_mandatory_tests; then
   cat > $pipelineName <<- YAML
 
 steps:
   - group: "Auditbeat Mandatory Testing"
-    key: "mandatory-tests"    
+    key: "mandatory-tests"
 
     steps:
       - label: ":ubuntu: Unit Tests"
-        command:
-          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
         notify:
           - github_commit_status:
               context: "Auditbeat: linux/Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        artifact_paths:
-          - "auditbeat/build/*.xml"
-          - "auditbeat/build/*.json"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
 
       - label: ":rhel: Unit Tests"
-        command:
-          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
         notify:
           - github_commit_status:
               context: "Auditbeat: rhel/Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        artifact_paths:
-          - "auditbeat/build/*.xml"
-          - "auditbeat/build/*.json"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
 
       - label: ":windows:-2016 Unit Tests"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
+        key: "windows-2016"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
         notify:
           - github_commit_status:
-              context: "Auditbeat: windows 2016/Unit Tests"
+              context: "Auditbeat: windows-2016/Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 200
-          disk_type: "pd-ssd"        
-        artifact_paths:
-          - "auditbeat/build/*.xml"
-          - "auditbeat/build/*.json"    
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
 
       - label: ":windows:-2022 Unit Tests"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
+        key: "windows-2022"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
         notify:
           - github_commit_status:
-              context: "Auditbeat: windows 2022/Unit Tests"
+              context: "Auditbeat: windows-2022/Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 200
-          disk_type: "pd-ssd"        
-        artifact_paths:
-          - "auditbeat/build/*.xml"
-          - "auditbeat/build/*.json"              
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
 
       - label: ":linux: Crosscompile"
         command:
-          - ".buildkite/auditbeat/scripts/crosscompile.sh"
+          - "make -C auditbeat crosscompile"
         env:
           GOX_FLAGS: "-arch amd64"
         notify:
@@ -94,89 +86,126 @@ fi
 
 echo "Check and add the Extended Tests into the pipeline"
 
-if are_conditions_met_arm_tests; then
+if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then
   cat >> $pipelineName <<- YAML
+
   - group: "Extended Tests"
-    key: "extended-tests-arm"
+    key: "extended-tests"
     steps:
-      - label: ":arm: ARM64 Unit Tests"
-        key: "extended-arm64-unit-tests"
-        command: ".buildkite/scripts/unit_tests.sh"
+
+YAML
+fi
+
+if are_conditions_met_macos_tests; then
+  cat >> $pipelineName <<- YAML
+
+      - label: ":mac: MacOS Unit Tests"
+        key: "macos-unit-tests-extended"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: MacOS Unit Tests"
         agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_ARM64}"
-          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":mac: MacOS ARM Unit Tests"
+        key: "macos-arm64-unit-tests-extended"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: MacOS ARM Unit Tests"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_ARM}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+YAML
+fi
+
+if are_conditions_met_arm_tests; then
+  cat >> $pipelineName <<- YAML
+      - label: ":linux: ARM Ubuntu Unit Tests"
+        key: "extended-arm64-unit-test"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
         notify:
           - github_commit_status:
-              context: "Auditbeat: ARM Unit tests"
+              context: "Auditbeat: Unit Tests ARM"
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
 YAML
 fi
 
 if are_conditions_met_win_tests; then
   cat >> $pipelineName <<- YAML
   - group: "Windows Extended Testing"
     key: "extended-tests-win"
+
     steps:
-      - label: ":windows: Windows 2019 Unit Tests"
-        key: "extended-win-2019-unit-tests"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
+      - label: ":windows:-2019 Unit Tests"
+        key: "windows-2019-extended"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: Win-2019 Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
+          disk_size: 200
           disk_type: "pd-ssd"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows:-11 Unit Tests"
+        key: "windows-11-extended"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
         notify:
           - github_commit_status:
-              context: "Auditbeat: Windows 2019 Unit Tests"
-
-      - label: ":windows: Windows 10 Unit Tests"
-        key: "extended-win-10-unit-tests"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
+              context: "Auditbeat: Win-11 Unit Tests"
         agents:
           provider: "gcp"
-          image: "${IMAGE_WIN_10}"
+          image: "${IMAGE_WIN_11}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
+          disk_size: 200
           disk_type: "pd-ssd"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows:-10 Unit Tests"
+        key: "windows-10-extended"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
         notify:
           - github_commit_status:
-              context: "Auditbeat: Windows 10 Unit Tests"
-
-      - label: ":windows: Windows 11 Unit Tests"
-        key: "extended-win-11-unit-tests"
-        command: ".buildkite/scripts/win_unit_tests.ps1"
+              context: "Auditbeat: Win-10 Unit Tests"
         agents:
           provider: "gcp"
-          image: "${IMAGE_WIN_11}"
+          image: "${IMAGE_WIN_10}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
+          disk_size: 200
           disk_type: "pd-ssd"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
-        notify:
-          - github_commit_status:
-              context: "Auditbeat: Windows 11 Unit Tests"
 YAML
 fi
 
 echo "Check and add the Packaging into the pipeline"
 if are_conditions_met_packaging; then
 cat >> $pipelineName <<- YAML
   - group: "Packaging"
-    key: "packaging"      
+    key: "packaging"
     depends_on:
       - "mandatory-tests"
+
     steps:
       - label: Package pipeline
         commands: ".buildkite/scripts/packaging/package-step.sh"
         notify:
           - github_commit_status:
               context: "Auditbeat: Packaging"
 
-
 YAML
 fi
 

.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml

@@ -1,5 +1,65 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
+env:
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+  - label: "Checks"
+    command: ".buildkite/deploy/kubernetes/scripts/make.sh"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") == "kubernetes"
+    notify:
+      - github_commit_status:
+          context: "Deploy K8S/Checks"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - label: "K8S Test/K8S version: v1.29.0"
+    key: "k8s-test-129"
+    env:
+      K8S_VERSION: "v1.29.0"
+    commands:
+      - "MODULE=kubernetes make -C metricbeat integration-tests"
+      - "make -C deploy/kubernetes test"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - label: "K8S Test/K8S version: v1.28.0"
+    key: "k8s-test-128"
+    env:
+      K8S_VERSION: "v1.28.0"
+    commands:
+      - "MODULE=kubernetes make -C metricbeat integration-tests"
+      - "make -C deploy/kubernetes test"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - label: "K8S Test/K8S version: v1.27.3"
+    key: "k8s-test-1273"
+    env:
+      K8S_VERSION: "v1.27.3"
+    commands:
+      - "MODULE=kubernetes make -C metricbeat integration-tests"
+      - "make -C deploy/kubernetes test"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - label: "K8S Test/K8S version: v1.26.6"
+    key: "k8s-test-1266"
+    env:
+      K8S_VERSION: "v1.26.6"
+    commands:
+      - "MODULE=kubernetes make -C metricbeat integration-tests"
+      - "make -C deploy/kubernetes test"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"