From a9a7c97ef3c8b7262c2f224f64c5716b2b553d17 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Thu, 2 May 2024 16:31:48 +0300 Subject: [PATCH] Remove python dynamic pipeline scripts (#39187) Remove python dynamic pipeline scripts. Signed-off-by: Alexandros Sapranidis (cherry picked from commit 7d475f543ad4fe2a73adf52782b71be864e61f49) # Conflicts: # .buildkite/pipeline.py --- .buildkite/buildkite.yml | 44 ------------ .buildkite/pytest.ini | 11 --- .../scripts/run_dynamic_pipeline_tests.sh | 43 ----------- .buildkite/test_pipeline.py | 71 ------------------- auditbeat/buildkite.yml | 56 --------------- filebeat/buildkite.yml | 60 ---------------- 6 files changed, 285 deletions(-) delete mode 100644 .buildkite/buildkite.yml delete mode 100644 .buildkite/pytest.ini delete mode 100755 .buildkite/scripts/run_dynamic_pipeline_tests.sh delete mode 100644 .buildkite/test_pipeline.py delete mode 100644 auditbeat/buildkite.yml delete mode 100644 filebeat/buildkite.yml diff --git a/.buildkite/buildkite.yml b/.buildkite/buildkite.yml deleted file mode 100644 index 4707707e07c..00000000000 --- a/.buildkite/buildkite.yml +++ /dev/null @@ -1,44 +0,0 @@ -projects: - - "auditbeat" - - "deploy/kubernetes" - - "filebeat" - - "heartbeat" - - "libbeat" - - "metricbeat" - - "packetbeat" - - "winlogbeat" - - "x-pack/auditbeat" - - "x-pack/dockerlogbeat" - - "x-pack/filebeat" - - "x-pack/functionbeat" - - "x-pack/heartbeat" - - "x-pack/libbeat" - - "x-pack/metricbeat" - - "x-pack/osquerybeat" - - "x-pack/packetbeat" - - "x-pack/winlogbeat" - -## Changeset macros that are defined here and used in each specific 3.0 pipeline. -changeset: - ci: - - "^Jenkinsfile" - - "^\\.ci/scripts/.*" - oss: - - "^go.mod" - - "^pytest.ini" - - "^dev-tools/.*" - - "^libbeat/.*" - - "^testing/.*" - xpack: - - "^go.mod" - - "^pytest.ini" - - "^dev-tools/.*" - - "^libbeat/.*" - - "^testing/.*" - - "^x-pack/libbeat/.*" - -disabled: - when: - labels: ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR. - - skip-ci - draft: true ## Skip the GitHub Pull Request builds with Draft PRs. diff --git a/.buildkite/pytest.ini b/.buildkite/pytest.ini deleted file mode 100644 index 3eff7473d9f..00000000000 --- a/.buildkite/pytest.ini +++ /dev/null @@ -1,11 +0,0 @@ -[pytest] -junit_family=xunit1 - -addopts = --strict-markers -markers = - load: Load tests - tag(name): Tag tests with Go-like semantics - -# Ignore setup and teardown for the timeout -#timeout_func_only = True - diff --git a/.buildkite/scripts/run_dynamic_pipeline_tests.sh b/.buildkite/scripts/run_dynamic_pipeline_tests.sh deleted file mode 100755 index 8eb72d7a96b..00000000000 --- a/.buildkite/scripts/run_dynamic_pipeline_tests.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Run tests for the dynamic pipeline generator only if it's a PR and related files have been changed -# this will allow us to fail fast, if e.g. a PR has broken the generator - -set -euo pipefail - -are_paths_changed() { - local patterns=("${@}") - local changelist=() - for pattern in "${patterns[@]}"; do - changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) - if [ "${#changed_files[@]}" -gt 0 ]; then - changelist+=("${changed_files[@]}") - fi - done - - if [ "${#changelist[@]}" -gt 0 ]; then - echo "Files changed:" - echo "${changelist[*]}" - return 0 - else - echo "No files changed within specified changeset:" - echo "${patterns[*]}" - return 1 - fi -} - -pipeline_generator_changeset=( - "^.buildkite/pipeline.py" - "^*/buildkite.yml" -) - -if ! are_paths_changed "${pipeline_generator_changeset[@]}" || [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then - echo "~~~ Skipping pipeline generator tests" - exit -fi - -echo "~~~ Running pipeline generator tests" - -python3 -mpip install --quiet "pytest" -pushd .buildkite -pytest . -popd diff --git a/.buildkite/test_pipeline.py b/.buildkite/test_pipeline.py deleted file mode 100644 index 75fd949ccc8..00000000000 --- a/.buildkite/test_pipeline.py +++ /dev/null @@ -1,71 +0,0 @@ -import os - -import pytest -import pipeline - - -@pytest.fixture -def ubuntu2204_aws_agent(): - return { - "command": "fake-cmd", - "platform": "platform-ingest-beats-ubuntu-2204-aarch64", - "provider": "aws" - } - - -@pytest.fixture() -def fake_simple_group(): - return { - "unitTest": { - "command": "fake-cmd", - "platform": "family/platform-ingest-beats-ubuntu-2204", - }, - "integrationTest": { - "command": "fake-integration", - "platform": "family/platform-ingest-beats-ubuntu-2204", - "env": { - "FOO": "BAR", - }, - }, - } - - -def test_fetch_stage(ubuntu2204_aws_agent): - step = pipeline.fetch_stage("test", ubuntu2204_aws_agent, "fake", "fake-category") - assert step.create_entity() == { - "label": "fake test", - "command": ["cd fake", "fake-cmd"], - "notify": [ - { - "github_commit_status": { - "context": "Fake: test", - } - } - ], - "agents": { - "provider": "aws", - "imagePrefix": "platform-ingest-beats-ubuntu-2204-aarch64", - "instanceType": "t4g.large", - }, - "artifact_paths": [ - "fake/build/*.xml", - "fake/build/*.json", - ], - } - - -def test_fetch_group(fake_simple_group): - group = pipeline.fetch_group(fake_simple_group, "fake-project", "testing") - assert len(group.steps) == 2 - for step in group.steps: - assert "testing" == step.category - assert "gcp" == step.agent.provider - - assert group.steps[1].env.get("FOO") == "BAR" - - -def test_is_pr(): - os.environ["BUILDKITE_PULL_REQUEST"] = "1234" - assert pipeline.is_pr() is True - os.environ["BUILDKITE_PULL_REQUEST"] = "false" - assert pipeline.is_pr() is False diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml deleted file mode 100644 index 2abf9d68407..00000000000 --- a/auditbeat/buildkite.yml +++ /dev/null @@ -1,56 +0,0 @@ -when: - changeset: ## when PR contains any of those entries in the changeset - - "auditbeat/**" - - "@ci" ## special token regarding the changeset for the ci - - "@oss" ## special token regarding the changeset for the oss -stages: - # mandatory stage - it runs always for: - # - branches/tags - # - on PRs - # - GitHub comment /test auditbeat - # - GitHub label auditbeat - mandatory: - # NOTE: stage name should be unique! - unitTest: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-ubuntu-2204" - crosscompile: - command: "make crosscompile" - platform: "family/platform-ingest-beats-ubuntu-2204" - env: - GOX_FLAGS: "-arch amd64" - unitTest-rhel-9: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-rhel-9" - unitTest-windows-2022: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2022" - unitTest-windows-2016: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2016" - # optional stage - it runs on: - # - branches/tags - # - on PRs if: - # - GitHub comment /test auditbeat . i.e: /test auditbeat integTest - # - GitHub label . i.e: integTest or unitTest-arm or unitTest-macos ... - extended: - # NOTE: stage name should be unique! - integTest: - command: "mage build integTest" - platform: "platform-ingest-beats-ubuntu-2204-aarch64" - provider: "aws" - integTest-arm: - command: "mage build integTest" - platform: "platform-ingest-beats-ubuntu-2204-aarch64" - provider: "aws" - unitTest-arm: - command: "mage build unitTest" - platform: "platform-ingest-beats-ubuntu-2204-aarch64" - provider: "aws" - unitTest-macos: - command: "mage build unitTest" - platform: "generic-13-ventura-x64" - provider: "orka" - unitTest-windows-2019: - command: "mage build unitTest" - platform: "family/core-windows-2019" diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml deleted file mode 100644 index 3fcabc5f1ce..00000000000 --- a/filebeat/buildkite.yml +++ /dev/null @@ -1,60 +0,0 @@ -when: - branches: true ## for all the branches - changeset: ## when PR contains any of those entries in the changeset - - "filebeat/**" - - "@ci" ## special token regarding the changeset for the ci - - "@oss" ## special token regarding the changeset for the oss -stages: - mandatory: - unitTest: - command: "mage unitTest" - platform: "family/platform-ingest-beats-ubuntu-2204" - goIntegTest: - command: "mage goIntegTest" - platform: "family/platform-ingest-beats-ubuntu-2204" - pythonIntegTest: - command: "mage pythonIntegTest" - platform: "family/platform-ingest-beats-ubuntu-2204" - unitTest-windows-2022: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2022" - unitTest-windows-2016: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2016" - - extended_win: - unitTest-windows-2019: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2019" - unitTest-windows-11: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-11" - unitTest-windows-10: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-10" - extended: - unitTest-arm: - command: "mage build unitTest" - platform: "platform-ingest-beats-ubuntu-2204-aarch64" - provider: "aws" # move this inside the platform leaf - when: - comments: - - "/test filebeat for arm" - labels: - - "arm" - parameters: - - "armTest" - branches: true ## for all the branches - tags: true ## for all the tags - unitTest-macos: - command: ".buildkite/filebeat/scripts/unit-tests.sh" - platform: "generic-13-ventura-x64" - provider: "orka" - when: - comments: - - "/test filebeat for macos" - labels: - - "macOS" - parameters: - - "macosTest" - tags: true ## for all the tags