Merge branch '7.17' into update-version-next-7.17.23

.buildkite/deploy/docker/docker-compose.yml

@@ -0,0 +1,23 @@
+version: '2.3'
+services:
+  # This is a proxy used to block beats until all services are healthy.
+  # See: https://github.com/docker/compose/issues/4369
+  proxy_dep:
+    image: busybox
+    depends_on:
+      localstack: { condition: service_healthy }
+
+  localstack:
+    container_name: "${localstack_integration_test_container}"
+    image: localstack/localstack:3.1.0 # Latest stable release
+    ports:
+      - "127.0.0.1:4566:4566"            # LocalStack Gateway
+    environment:
+      - DEBUG=1
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - LOCALSTACK_HOST=localhost
+      - S3_HOSTNAME=localhost
+      - PROVIDER_OVERRIDE_S3=asf
+    volumes:
+      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
+      - "/var/run/docker.sock:/var/run/docker.sock"

.buildkite/libbeat/pipeline.libbeat.yml

@@ -17,7 +17,33 @@ env:
   TEST_COVERAGE: "true"
 
 steps:
-  - group: "Libbeat Mandatory Tests"
+  - group: "libbeat Check/Update"
+    key: "libbeat-check-update"
+    steps:
+      - label: "Run check/update"
+        command: |
+          make -C libbeat check update
+          make check-no-changes
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          cpu: "4000m"
+          memory: "8Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "libbeat: check/update"
+
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "libbeat-check-update"
+
+  - group: "Mandatory Tests"
     key: "mandatory-tests"
 
     steps:

.buildkite/scripts/common.sh

@@ -378,7 +378,7 @@ startCloudTestEnv() {
   local dir=$1
   withAWS
   echo "--- Run docker-compose services for emulated cloud env"
-  docker-compose -f .ci/jobs/docker-compose.yml up -d                     #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
+  docker-compose -f .buildkite/deploy/docker/docker-compose.yml up -d
   with_Terraform
   terraformInit "$dir"
   export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g')
@@ -440,10 +440,10 @@ installNodeJsDependencies() {
 teardown() {
   # Teardown resources after using them
   echo "---Terraform Cleanup"
-  .ci/scripts/terraform-cleanup.sh "${MODULE_DIR}"              #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
+  .buildkite/scripts/terraform-cleanup.sh "${MODULE_DIR}"
 
   echo "---Docker Compose Cleanup"
-  docker-compose -f .ci/jobs/docker-compose.yml down -v         #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
+  docker-compose -f .buildkite/deploy/docker/docker-compose.yml down -v
 }
 
 unset_secrets () {

.buildkite/scripts/setup_cloud_env.sh

@@ -11,7 +11,7 @@ teardown() {
   tf_cleanup "${MODULE_DIR}"              #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
 
   echo "~~~ Docker Compose Cleanup"
-  docker-compose -f .ci/jobs/docker-compose.yml down -v         #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
+  docker-compose -f .buildkite/deploy/docker/docker-compose.yml down -v
 }
 
 tf_cleanup() {
@@ -38,7 +38,7 @@ export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws"
 set -o xtrace
 
 echo "~~~ Run docker-compose services for emulated cloud env"
-docker-compose -f .ci/jobs/docker-compose.yml up -d        #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
+docker-compose -f .buildkite/deploy/docker/docker-compose.yml up -d
 echo "~~~ Initialize TF cloud resources"
 cd "$MODULE_DIR"
 export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g')

.buildkite/x-pack/pipeline.xpack.libbeat.yml

@@ -19,6 +19,32 @@ env:
   TEST_COVERAGE: "true"
 
 steps:
+  - group: "x-pack/libbeat Check/Update"
+    key: "x-pack-libbeat-check-update"
+    steps:
+      - label: "Run check/update"
+        command: |
+          make -C x-pack/libbeat check update
+          make check-no-changes
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          cpu: "4000m"
+          memory: "8Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "x-pack/libbeat: check/update"
+
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "x-pack-libbeat-check-update"
+
   - group: "x-pack/libbeat Mandatory Tests"
     key: "x-pack-libbeat-mandatory-tests"