From 1b59d076f25ce1aeae6de3785db1e0eb7ad016b1 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:57:34 +0300 Subject: [PATCH 1/2] Migrate needed .ci/* scripts into .buildkite/* ( elastic#3353 ) (#39661) (#39748) * Migrate files and change path to them * Delete files that was migrated to new path (cherry picked from commit f411b0624c17a034d78636c7be9c97486e75e8d8) # Conflicts: # .buildkite/deploy/docker/docker-compose.yml Co-authored-by: ev1yehor <146825775+ev1yehor@users.noreply.github.com> --- .buildkite/deploy/docker/docker-compose.yml | 23 +++++++++++++++++++ .buildkite/scripts/common.sh | 6 ++--- .buildkite/scripts/setup_cloud_env.sh | 4 ++-- .../scripts/terraform-cleanup.sh | 0 4 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 .buildkite/deploy/docker/docker-compose.yml rename {.ci => .buildkite}/scripts/terraform-cleanup.sh (100%) mode change 100755 => 100644 diff --git a/.buildkite/deploy/docker/docker-compose.yml b/.buildkite/deploy/docker/docker-compose.yml new file mode 100644 index 00000000000..c27417158a3 --- /dev/null +++ b/.buildkite/deploy/docker/docker-compose.yml @@ -0,0 +1,23 @@ +version: '2.3' +services: + # This is a proxy used to block beats until all services are healthy. + # See: https://github.com/docker/compose/issues/4369 + proxy_dep: + image: busybox + depends_on: + localstack: { condition: service_healthy } + + localstack: + container_name: "${localstack_integration_test_container}" + image: localstack/localstack:3.1.0 # Latest stable release + ports: + - "127.0.0.1:4566:4566" # LocalStack Gateway + environment: + - DEBUG=1 + - DOCKER_HOST=unix:///var/run/docker.sock + - LOCALSTACK_HOST=localhost + - S3_HOSTNAME=localhost + - PROVIDER_OVERRIDE_S3=asf + volumes: + - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 6ce329d35a3..dc7a61feea1 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -378,7 +378,7 @@ startCloudTestEnv() { local dir=$1 withAWS echo "--- Run docker-compose services for emulated cloud env" - docker-compose -f .ci/jobs/docker-compose.yml up -d #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK + docker-compose -f .buildkite/deploy/docker/docker-compose.yml up -d with_Terraform terraformInit "$dir" export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') @@ -440,10 +440,10 @@ installNodeJsDependencies() { teardown() { # Teardown resources after using them echo "---Terraform Cleanup" - .ci/scripts/terraform-cleanup.sh "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK + .buildkite/scripts/terraform-cleanup.sh "${MODULE_DIR}" echo "---Docker Compose Cleanup" - docker-compose -f .ci/jobs/docker-compose.yml down -v #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK + docker-compose -f .buildkite/deploy/docker/docker-compose.yml down -v } unset_secrets () { diff --git a/.buildkite/scripts/setup_cloud_env.sh b/.buildkite/scripts/setup_cloud_env.sh index 3c23d71573f..a13a8446dc7 100644 --- a/.buildkite/scripts/setup_cloud_env.sh +++ b/.buildkite/scripts/setup_cloud_env.sh @@ -11,7 +11,7 @@ teardown() { tf_cleanup "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK echo "~~~ Docker Compose Cleanup" - docker-compose -f .ci/jobs/docker-compose.yml down -v #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK + docker-compose -f .buildkite/deploy/docker/docker-compose.yml down -v } tf_cleanup() { @@ -38,7 +38,7 @@ export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws" set -o xtrace echo "~~~ Run docker-compose services for emulated cloud env" -docker-compose -f .ci/jobs/docker-compose.yml up -d #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK +docker-compose -f .buildkite/deploy/docker/docker-compose.yml up -d echo "~~~ Initialize TF cloud resources" cd "$MODULE_DIR" export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') diff --git a/.ci/scripts/terraform-cleanup.sh b/.buildkite/scripts/terraform-cleanup.sh old mode 100755 new mode 100644 similarity index 100% rename from .ci/scripts/terraform-cleanup.sh rename to .buildkite/scripts/terraform-cleanup.sh From 18d20572de18469a2a980eba594549b0582c8cc2 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 16:14:09 +0300 Subject: [PATCH 2/2] [7.17](backport #39869) Migrate github checks to buildkite for libbeat and x-pack/libbeat (#39877) * Migrate github checks to buildkite for libbeat and x-pack/libbeat (#39869) * Update * Update pipeline.xpack.libbeat.yml * Update pipeline.xpack.libbeat.yml * update (cherry picked from commit 02a57f76489ac349d24fd2a36818559ff314a624) # Conflicts: # .buildkite/libbeat/pipeline.libbeat.yml # .github/workflows/check-libbeat.yml * Update pipeline.libbeat.yml * Delete check-libbeat.yml --------- Co-authored-by: ev1yehor <146825775+ev1yehor@users.noreply.github.com> --- .buildkite/libbeat/pipeline.libbeat.yml | 28 ++++++++++++++++++- .buildkite/x-pack/pipeline.xpack.libbeat.yml | 26 ++++++++++++++++++ .github/workflows/check-libbeat.yml | 29 -------------------- 3 files changed, 53 insertions(+), 30 deletions(-) delete mode 100644 .github/workflows/check-libbeat.yml diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 91c93716aab..3d3247b9da9 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -17,7 +17,33 @@ env: TEST_COVERAGE: "true" steps: - - group: "Libbeat Mandatory Tests" + - group: "libbeat Check/Update" + key: "libbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C libbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "libbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "libbeat-check-update" + + - group: "Mandatory Tests" key: "mandatory-tests" steps: diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index e48433b0e05..6cb1825b7a1 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -19,6 +19,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/libbeat Check/Update" + key: "x-pack-libbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C x-pack/libbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack/libbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-libbeat-check-update" + - group: "x-pack/libbeat Mandatory Tests" key: "x-pack-libbeat-mandatory-tests" diff --git a/.github/workflows/check-libbeat.yml b/.github/workflows/check-libbeat.yml deleted file mode 100644 index 38b04932a86..00000000000 --- a/.github/workflows/check-libbeat.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: check-libbeat - -on: - pull_request: - paths: - - '.github/workflows/check-packetbeat.yml' - - 'libbeat/**' - - 'x-pack/libbeat/**' - -env: - BEAT_MODULE: 'libbeat' - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Update package lists - run: sudo apt-get update - - name: Install libpcap-dev - run: sudo apt-get install -y libpcap-dev - - name: Run check/update - run: | - go install github.com/magefile/mage - make -C ${{ env.BEAT_MODULE }} check update - make check-no-changes