Deploy K8S Pipeline migration (#38659)

* updated auditbeat & deploy k8s pipelines

* added kind & kuve env vars, set higer ulimit

* added k8s env setup

(cherry picked from commit aaa4829)

.buildkite/auditbeat/auditbeat-pipeline.yml

@@ -2,21 +2,21 @@
 name: "beats-auditbeat"
 
 env:
-  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
+  BEATS_PROJECT_NAME: "auditbeat"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
   IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
   IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
   IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
   IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016"
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
-  BEATS_PROJECT_NAME: "auditbeat"
+  IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
-  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
 
 steps:
   - input: "Input Parameters"
@@ -62,7 +62,7 @@ steps:
 
   - label: ":linux: Load dynamic auditbeat pipeline"
     key: "auditbeat-pipeline"
-    command: ".buildkite/scripts/generate_auditbeat_pipeline.sh"
+    command: ".buildkite/auditbeat/generate_auditbeat_pipeline.sh"
     notify:
       - github_commit_status:
           context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"

.buildkite/auditbeat/generate_auditbeat_pipeline.sh

@@ -0,0 +1,216 @@
+#!/usr/bin/env bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+pipelineName="pipeline.auditbeat-dynamic.yml"
+
+# TODO: steps: must be always included
+echo "Add the mandatory and extended tests without additional conditions into the pipeline"
+if are_conditions_met_mandatory_tests; then
+  cat > $pipelineName <<- YAML
+
+steps:
+  - group: "Auditbeat Mandatory Testing"
+    key: "mandatory-tests"
+
+    steps:
+      - label: ":ubuntu: Unit Tests"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: linux/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":rhel: Unit Tests"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: rhel/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_RHEL9}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows:-2016 Unit Tests"
+        key: "windows-2016"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: windows-2016/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2016}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows:-2022 Unit Tests"
+        key: "windows-2022"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: windows-2022/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2022}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":linux: Crosscompile"
+        command:
+          - "make -C auditbeat crosscompile"
+        env:
+          GOX_FLAGS: "-arch amd64"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: Crosscompile"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+YAML
+else
+  echo "The conditions don't match to requirements for generating pipeline steps."
+  exit 0
+fi
+
+echo "Check and add the Extended Tests into the pipeline"
+
+if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then
+  cat >> $pipelineName <<- YAML
+
+  - group: "Extended Tests"
+    key: "extended-tests"
+    steps:
+
+YAML
+fi
+
+if are_conditions_met_macos_tests; then
+  cat >> $pipelineName <<- YAML
+
+      - label: ":mac: MacOS Unit Tests"
+        key: "macos-unit-tests-extended"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: MacOS Unit Tests"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":mac: MacOS ARM Unit Tests"
+        key: "macos-arm64-unit-tests-extended"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: MacOS ARM Unit Tests"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_ARM}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+YAML
+fi
+
+if are_conditions_met_arm_tests; then
+  cat >> $pipelineName <<- YAML
+      - label: ":linux: ARM Ubuntu Unit Tests"
+        key: "extended-arm64-unit-test"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: Unit Tests ARM"
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+YAML
+fi
+
+if are_conditions_met_win_tests; then
+  cat >> $pipelineName <<- YAML
+  - group: "Windows Extended Testing"
+    key: "extended-tests-win"
+
+    steps:
+      - label: ":windows:-2019 Unit Tests"
+        key: "windows-2019-extended"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: Win-2019 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2019}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows:-11 Unit Tests"
+        key: "windows-11-extended"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: Win-11 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_11}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows:-10 Unit Tests"
+        key: "windows-10-extended"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: Win-10 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_10}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+YAML
+fi
+
+echo "Check and add the Packaging into the pipeline"
+if are_conditions_met_packaging; then
+cat >> $pipelineName <<- YAML
+  - group: "Packaging"
+    key: "packaging"
+    depends_on:
+      - "mandatory-tests"
+
+    steps:
+      - label: Package pipeline
+        commands: ".buildkite/scripts/packaging/package-step.sh"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat: Packaging"
+
+YAML
+fi
+
+echo "--- Printing dynamic steps"     #TODO: remove if the pipeline is public
+cat $pipelineName
+
+echo "--- Loading dynamic steps"
+buildkite-agent pipeline upload $pipelineName

.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml

@@ -1,5 +1,65 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
+env:
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+  - label: "Checks"
+    command: ".buildkite/deploy/kubernetes/scripts/make.sh"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") == "kubernetes"
+    notify:
+      - github_commit_status:
+          context: "Deploy K8S/Checks"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - label: "K8S Test/K8S version: v1.29.0"
+    key: "k8s-test-129"
+    env:
+      K8S_VERSION: "v1.29.0"
+    commands:
+      - "MODULE=kubernetes make -C metricbeat integration-tests"
+      - "make -C deploy/kubernetes test"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - label: "K8S Test/K8S version: v1.28.0"
+    key: "k8s-test-128"
+    env:
+      K8S_VERSION: "v1.28.0"
+    commands:
+      - "MODULE=kubernetes make -C metricbeat integration-tests"
+      - "make -C deploy/kubernetes test"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - label: "K8S Test/K8S version: v1.27.3"
+    key: "k8s-test-1273"
+    env:
+      K8S_VERSION: "v1.27.3"
+    commands:
+      - "MODULE=kubernetes make -C metricbeat integration-tests"
+      - "make -C deploy/kubernetes test"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - label: "K8S Test/K8S version: v1.26.6"
+    key: "k8s-test-1266"
+    env:
+      K8S_VERSION: "v1.26.6"
+    commands:
+      - "MODULE=kubernetes make -C metricbeat integration-tests"
+      - "make -C deploy/kubernetes test"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"

.buildkite/deploy/kubernetes/scripts/install-kind.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MSG="environment variable missing."
+KIND_VERSION=${KIND_VERSION:?$MSG}
+KIND_BINARY="${BIN}/kind"
+
+if command -v kind
+then
+    set +e
+    echo "Found Kind. Checking version.."
+    FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}')
+    if [ "$FOUND_KIND_VERSION" == "$KIND_VERSION" ]
+    then
+        echo "--- Versions match. No need to install Kind. Exiting."
+        exit 0
+    fi
+    set -e
+fi
+
+echo "UNMET DEP: Installing Kind"
+
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
+if [ "${ARCH}" == "aarch64" ] ; then
+    ARCH_SUFFIX=arm64
+else
+    ARCH_SUFFIX=amd64
+fi
+
+if curl -sSLo "${KIND_BINARY}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${OS}-${ARCH_SUFFIX}" ; then
+    chmod +x "${KIND_BINARY}"
+    echo "Kind installed: ${KIND_VERSION}"
+else
+    echo "Something bad with the download, let's delete the corrupted binary"
+    if [ -e "${KIND_BINARY}" ] ; then
+        rm "${KIND_BINARY}"
+    fi
+    exit 1
+fi