Deploy/K8S pipeline post-migration fixes (#39295)

* removed make.sh, updated check step

* updated steps with env setup & removed setup from hook

* updated setenv.sh

* moved module to env

* updated PR based on comments

* updated common.sh

* updated k8s env setup

* removed tmp folder

* added section for tests in BK

.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml

@@ -3,29 +3,40 @@
 env:
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  MODULE: "kubernetes"
+
+  # Other deps
+  ASDF_KIND_VERSION: "0.20.0"
 
 steps:
   - group: "Deploy/K8S"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") =~ /.*kubernetes.*/
 
     steps:
     - label: "Checks"
-      command: ".buildkite/deploy/kubernetes/scripts/make.sh"
+      command: |
+        set -euo pipefail
+        make -C deploy/kubernetes all
+        make check-no-changes
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "Deploy/k8s-checks"
+            context: "deploy/k8s checks"
 
     - label: "K8S Test/K8S version: v1.29.0"
       key: "k8s-test-129"
       env:
         K8S_VERSION: "v1.29.0"
-      commands:
-        - "MODULE=kubernetes make -C metricbeat integration-tests"
-        - "make -C deploy/kubernetes test"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
       retry:
         automatic:
           - limit: 3
@@ -35,15 +46,19 @@ steps:
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "Deploy/k8s-test v1.29.0"
+            context: "deploy/k8s test v1.29.0"
 
     - label: "K8S Test/K8S version: v1.28.0"
       key: "k8s-test-128"
       env:
         K8S_VERSION: "v1.28.0"
-      commands:
-        - "MODULE=kubernetes make -C metricbeat integration-tests"
-        - "make -C deploy/kubernetes test"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
       retry:
         automatic:
           - limit: 3
@@ -53,15 +68,19 @@ steps:
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "Deploy/k8s-test v1.28.0"
+            context: "deploy/k8s test v1.28.0"
 
     - label: "K8S Test/K8S version: v1.27.3"
       key: "k8s-test-1273"
       env:
         K8S_VERSION: "v1.27.3"
-      commands:
-        - "MODULE=kubernetes make -C metricbeat integration-tests"
-        - "make -C deploy/kubernetes test"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
       retry:
         automatic:
           - limit: 3
@@ -71,15 +90,19 @@ steps:
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "Deploy/k8s-test v1.27.3"
+            context: "deploy/k8s test v1.27.3"
 
     - label: "K8S Test/K8S version: v1.26.6"
       key: "k8s-test-1266"
       env:
         K8S_VERSION: "v1.26.6"
-      commands:
-        - "MODULE=kubernetes make -C metricbeat integration-tests"
-        - "make -C deploy/kubernetes test"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
       retry:
         automatic:
           - limit: 3
@@ -89,4 +112,4 @@ steps:
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "Deploy/k8s-test v1.26.6"
+            context: "deploy/k8s test v1.26.6"

.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh

@@ -4,11 +4,15 @@ set -euo pipefail
 
 source .buildkite/env-scripts/util.sh
 
+export KUBECONFIG="${WORKSPACE}/kubecfg"
+export BIN="${WORKSPACE}/bin"
+add_bin_path
+
 echo "--- Installing kind & kubectl"
-retry_with_count 5 .buildkite/deploy/kubernetes/scripts/install-kind.sh
-retry_with_count 5 .buildkite/deploy/kubernetes/scripts/install-kubectl.sh
+asdf plugin add kind
+asdf install kind $ASDF_KIND_VERSION
 
-echo "--- Setting up kind"
+echo "~~~ Setting up kind"
 max_retries=3
 timeout=5
 retries=0

.buildkite/env-scripts/env.sh

@@ -12,7 +12,6 @@ WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
 HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
-TMP_FOLDER="tmp.${REPO}"
 SNAPSHOT="true"
 PYTEST_ADDOPTS=""
 OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
@@ -38,7 +37,6 @@ export WORKSPACE
 export BIN
 export HW_TYPE
 export PLATFORM_TYPE
-export TMP_FOLDER
 export SNAPSHOT
 export PYTEST_ADDOPTS
 export OSS_MODULE_PATTERN

.buildkite/hooks/pre-command

@@ -91,9 +91,3 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then
   fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then
-  source .buildkite/env-scripts/env.sh
-  if [[ "$BUILDKITE_STEP_KEY" == k8s-test* ]]; then
-    .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
-  fi
-fi

.buildkite/hooks/scripts/util.sh

@@ -27,8 +27,8 @@ google_cloud_logout_active_account() {
 cleanup() {
   if [[ "$BUILDKITE_COMMAND" != *"buildkite-agent pipeline upload"* ]]; then
   echo "Deleting temporary files..."
-    if [[ -n "${BIN:-}" ]] && [[ -e "${BIN}/${TMP_FOLDER}" ]]; then
-      rm -rf "${BIN}/${TMP_FOLDER}.*"
+    if [[ -n "${BIN:-}" ]]; then
+      rm -rf "${BIN}"
     fi
   echo "Done."
   fi

.buildkite/scripts/setenv.sh

@@ -2,20 +2,21 @@
 
 set -euo pipefail
 
+WORKSPACE=${WORKSPACE:-"$(pwd)"}
+GO_VERSION=$(cat .go-version)
+
 export REPO="beats"
 export DOCKER_REGISTRY="docker.elastic.co"
 export SETUP_GVM_VERSION="v0.5.1"
 export DOCKER_COMPOSE_VERSION="1.21.0"
 export DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0"
+
 export ASDF_NODEJS_VERSION="18.17.1"
 export AWS_REGION="eu-central-1"
 
-WORKSPACE=${WORKSPACE:-"$(pwd)"}
 export WORKSPACE
-GO_VERSION=$(cat .go-version)
 export GO_VERSION
 
-
 exportVars() {
   local platform_type="$(uname)"
   local arch_type="$(uname -m)"