diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
index 147ca45ced1..62c0d86e259 100644
--- a/.buildkite/auditbeat/auditbeat-pipeline.yml
+++ b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -20,7 +20,7 @@ steps:
           - ".buildkite/auditbeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
-              context: "auditbeat: Unit Tests"
+              context: "Auditbeat: linux/Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -33,7 +33,7 @@ steps:
           - ".buildkite/auditbeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
-              context: "auditbeat: Unit Tests"
+              context: "Auditbeat: rhel/Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
@@ -45,7 +45,7 @@ steps:
         command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
         notify:
           - github_commit_status:
-              context: "auditbeat: Unit Tests"
+              context: "Auditbeat: windows/Unit Tests"
         agents:
           provider: "gcp"
           image: "{{matrix.image}}"
@@ -68,7 +68,7 @@ steps:
           GOX_FLAGS: "-arch amd64"
         notify:
           - github_commit_status:
-              context: "auditbeat: Unit Tests"
+              context: "Auditbeat: Crosscompile"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -85,7 +85,7 @@ steps:
           - ".buildkite/auditbeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
-              context: "auditbeat/Extended: Unit Tests ARM"
+              context: "Auditbeat/Extended: Unit Tests ARM"
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
@@ -99,7 +99,7 @@ steps:
           - ".buildkite/auditbeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
-              context: "auditbeat/Extended: MacOS Unit Tests"
+              context: "Auditbeat/Extended: MacOS Unit Tests"
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -115,7 +115,7 @@ steps:
         command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
         notify:
           - github_commit_status:
-              context: "auditbeat/Extended: Win-2019 Unit Tests"
+              context: "Auditbeat/Extended: Win-2019 Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
diff --git a/.buildkite/auditbeat/scripts/package-step.sh b/.buildkite/auditbeat/scripts/package-step.sh
index 02124058992..6c113f4e3ca 100755
--- a/.buildkite/auditbeat/scripts/package-step.sh
+++ b/.buildkite/auditbeat/scripts/package-step.sh
@@ -13,7 +13,7 @@ changeset="^auditbeat/
 ^\.buildkite/auditbeat/"
 
 if are_files_changed "$changeset"; then
-  cat <<-EOF
+  cat <<-YAML
     steps:
       - label: ":ubuntu: Packaging Linux X86"
         key: "package-linux-x86"
@@ -42,5 +42,8 @@ if are_files_changed "$changeset"; then
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
           instanceType: "t4g.large"
-EOF
+YAML
+else
+  buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning'
+  exit 0
 fi
diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh
index 5e6e5f7cbf0..1365aaace4a 100644
--- a/.buildkite/env-scripts/linux-env.sh
+++ b/.buildkite/env-scripts/linux-env.sh
@@ -9,8 +9,6 @@ DEBIAN_FRONTEND="noninteractive"
 sudo mkdir -p /etc/needrestart
 echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null
 
-echo "--- PLATFORM TYPE $PLATFORM_TYPE"
-
 if [[ $PLATFORM_TYPE == "Linux" ]]; then
   # Remove this code once beats specific agent is set up
   if grep -q 'Ubuntu' /etc/*release; then
diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh
index 7aef69cff38..6a5c36bcd04 100644
--- a/.buildkite/env-scripts/util.sh
+++ b/.buildkite/env-scripts/util.sh
@@ -85,7 +85,6 @@ are_files_changed() {
   if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
     return 0;
   else
-    echo "WARN! No files changed in $changeset"
     return 1;
   fi
 }
diff --git a/.buildkite/env-scripts/win-env.sh b/.buildkite/env-scripts/win-env.sh
index ccf5479b46e..931051d550a 100644
--- a/.buildkite/env-scripts/win-env.sh
+++ b/.buildkite/env-scripts/win-env.sh
@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
 
-echo "--- PLATFORM TYPE: ${PLATFORM_TYPE}"
 if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
   echo "--- Installing Python on Win"
   choco install mingw -y
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
index eda9fb93a66..f86ad7da11d 100644
--- a/.buildkite/filebeat/filebeat-pipeline.yml
+++ b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -19,7 +19,7 @@ steps:
           - ".buildkite/filebeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
-              context: "Filebeat: Unit Tests"
+              context: "Filebeat: linux/Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -33,7 +33,7 @@ steps:
           - ".buildkite/filebeat/scripts/integration-gotests.sh"
         notify:
           - github_commit_status:
-              context: "Filebeat: Integration Tests"
+              context: "Filebeat: Go Integration Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -60,7 +60,7 @@ steps:
         command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
         notify:
           - github_commit_status:
-              context: "Filebeat: Unit Tests"
+              context: "Filebeat: windows/Unit Tests"
         agents:
           provider: "gcp"
           image: "{{matrix.image}}"
diff --git a/.buildkite/filebeat/scripts/package-step.sh b/.buildkite/filebeat/scripts/package-step.sh
index 985125433ce..c2a1bf41a01 100755
--- a/.buildkite/filebeat/scripts/package-step.sh
+++ b/.buildkite/filebeat/scripts/package-step.sh
@@ -13,7 +13,7 @@ changeset="^filebeat/
 ^\.buildkite/filebeat/"
 
 if are_files_changed "$changeset"; then
-  cat <<-EOF
+  cat <<-YAML
     steps:
       - label: ":ubuntu: Packaging Linux X86"
         key: "package-linux-x86"
@@ -42,5 +42,8 @@ if are_files_changed "$changeset"; then
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
           instanceType: "t4g.large"
-EOF
+YAML
+else
+  buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning'
+  exit 0
 fi
diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
index 34321b61161..93cbfc32008 100644
--- a/.buildkite/heartbeat/heartbeat-pipeline.yml
+++ b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -1,5 +1,142 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
+env:
+  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  IMAGE_WIN_2016: "family/core-windows-2016"
+  IMAGE_WIN_2019: "family/core-windows-2019"
+  IMAGE_WIN_2022: "family/core-windows-2022"
+  IMAGE_RHEL9: "family/core-rhel-9"
+  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+  - group: "Heartbeat Mandatory Testing"
+    key: "mandatory-tests"
+    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
+
+    steps:
+      - label: ":linux: Unit Tests / {{matrix.image}}"
+        command:
+          - ".buildkite/heartbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: linux/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "{{matrix.image}}"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_UBUNTU_X86_64}"
+              - "${IMAGE_RHEL9}"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+      - label: ":windows: Unit Tests / {{matrix.image}}"
+        command:
+          - ".buildkite/heartbeat/scripts/unit-tests-win.ps1"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: windows/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "{{matrix.image}}"
+          machine_type: "n2-standard-8"
+          disk_type: "pd-ssd"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_WIN_2016}"
+              - "${IMAGE_WIN_2022}"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+      - label: ":ubuntu: Go Integration Tests"
+        command:
+          - ".buildkite/heartbeat/scripts/integration-gotests.sh"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Go Integration Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+      - label: ":ubuntu: Python Integration Tests"
+        command:
+          - ".buildkite/heartbeat/scripts/integration-pytests.sh"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Python Integration Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+  - group: "Extended Testing"
+    key: "extended-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for extended support"
+
+    steps:
+      - label: ":linux: ARM64 Unit Tests"
+        key: "arm-extended"
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        command:
+          - ".buildkite/heartbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat/Extended: Unit Tests ARM"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "t4g.large"
+        artifact_paths: "heartbeat/build/*.xml"
+
+      - label: ":mac: MacOS Unit Tests"
+        key: "macos-extended"
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        command:
+          - ".buildkite/heartbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat/Extended: MacOS Unit Tests"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
+        artifact_paths: "heartbeat/build/*.xml"
+
+  - group: "Windows Extended Testing"
+    key: "extended-tests-win"
+    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
+
+    steps:
+      - label: ":windows: Win 2019 Unit Tests"
+        key: "win-extended-2019"
+        command: ".buildkite/heartbeat/scripts/unit-tests-win.ps1"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat/Extended: Win-2019 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2019}"
+          machine_type: "n2-standard-8"
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+  - group: "Packaging"
+    key: "packaging"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on:
+      - "mandatory-tests"
+
+    steps:
+      - label: Package pipeline
+        commands: ".buildkite/heartbeat/scripts/package-step.sh | buildkite-agent pipeline upload"
diff --git a/.buildkite/heartbeat/scripts/integration-gotests.sh b/.buildkite/heartbeat/scripts/integration-gotests.sh
new file mode 100755
index 00000000000..8eab0e8b5d8
--- /dev/null
+++ b/.buildkite/heartbeat/scripts/integration-gotests.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Remove when custom image is set up
+source .buildkite/env-scripts/linux-env.sh
+
+echo "--- Executing Integration Tests"
+# Remove when custom image is set up
+sudo chmod -R go-w heartbeat/
+
+cd heartbeat
+# Remove when custom image is set up
+umask 0022
+mage goIntegTest
diff --git a/.buildkite/heartbeat/scripts/integration-pytests.sh b/.buildkite/heartbeat/scripts/integration-pytests.sh
new file mode 100755
index 00000000000..729df5ae6f6
--- /dev/null
+++ b/.buildkite/heartbeat/scripts/integration-pytests.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Remove when custom image is set up
+source .buildkite/env-scripts/linux-env.sh
+
+echo "--- Executing Integration Tests"
+# Remove when custom image is set up
+sudo chmod -R go-w heartbeat/
+
+cd heartbeat
+# Remove when custom image is set up
+umask 0022
+mage pythonIntegTest
diff --git a/.buildkite/heartbeat/scripts/package-step.sh b/.buildkite/heartbeat/scripts/package-step.sh
new file mode 100755
index 00000000000..05ef69b131a
--- /dev/null
+++ b/.buildkite/heartbeat/scripts/package-step.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/util.sh
+
+changeset="^heartbeat/
+^go.mod
+^pytest.ini
+^dev-tools/
+^libbeat/
+^testing/
+^\.buildkite/heartbeat/"
+
+if are_files_changed "$changeset"; then
+  cat <<-YAML
+    steps:
+      - label: ":ubuntu: Packaging Linux X86"
+        key: "package-linux-x86"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+        command:
+          - ".buildkite/heartbeat/scripts/package.sh"
+        notify:
+          - github_commit_status:
+              context: "heartbeat/Packaging: Linux X86"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+
+      - label: ":linux: Packaging Linux ARM"
+        key: "package-linux-arm"
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "docker"
+        command:
+          - ".buildkite/heartbeat/scripts/package.sh"
+        notify:
+          - github_commit_status:
+              context: "heartbeat/Packaging: ARM"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "t4g.large"
+YAML
+else
+  buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning'
+  exit 0
+fi
diff --git a/.buildkite/heartbeat/scripts/package.sh b/.buildkite/heartbeat/scripts/package.sh
new file mode 100755
index 00000000000..7f51a6b5ca1
--- /dev/null
+++ b/.buildkite/heartbeat/scripts/package.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/linux-env.sh
+
+echo "--- Docker Version: $(docker --version)"
+
+echo "--- Start Packaging"
+cd heartbeat
+umask 0022
+mage package
+
diff --git a/.buildkite/heartbeat/scripts/unit-tests-win.ps1 b/.buildkite/heartbeat/scripts/unit-tests-win.ps1
new file mode 100644
index 00000000000..17282813e13
--- /dev/null
+++ b/.buildkite/heartbeat/scripts/unit-tests-win.ps1
@@ -0,0 +1,51 @@
+$ErrorActionPreference = "Stop" # set -e
+$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually
+
+# Forcing to checkout again all the files with a correct autocrlf.
+# Doing this here because we cannot set git clone options before.
+function fixCRLF() {
+    Write-Host "--- Fixing CRLF in git checkout --"
+    git config core.autocrlf false
+    git rm --quiet --cached -r .
+    git reset --quiet --hard
+}
+
+function withGolang() {
+    Write-Host "--- Install golang $GoVersion --"
+    choco install golang -y --version $GoVersion
+
+    $choco = Convert-Path "$((Get-Command choco).Path)\..\.."
+    Import-Module "$choco\helpers\chocolateyProfile.psm1"
+    refreshenv
+    go version
+    go env
+}
+
+function installGoDependencies() {
+    $installPackages = @(
+        "github.com/magefile/mage"
+        "github.com/elastic/go-licenser"
+        "golang.org/x/tools/cmd/goimports"
+        "github.com/jstemmer/go-junit-report"
+        "github.com/tebeka/go2xunit"
+    )
+    foreach ($pkg in $installPackages) {
+        go install "$pkg"
+    }
+}
+
+fixCRLF
+
+$ErrorActionPreference = "Continue" # set +e
+
+Set-Location -Path heartbeat
+New-Item -ItemType Directory -Force -Path "build"
+withGolang
+installGoDependencies
+
+mage build unitTest
+
+$EXITCODE=$LASTEXITCODE
+$ErrorActionPreference = "Stop"
+
+Exit $EXITCODE
diff --git a/.buildkite/heartbeat/scripts/unit-tests.sh b/.buildkite/heartbeat/scripts/unit-tests.sh
new file mode 100755
index 00000000000..4b746da2d57
--- /dev/null
+++ b/.buildkite/heartbeat/scripts/unit-tests.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Remove when custom image is set up
+source .buildkite/env-scripts/linux-env.sh
+
+echo "--- Running Unit Tests"
+# Remove when custom image is set up
+sudo chmod -R go-w heartbeat/
+
+cd heartbeat
+# Remove when custom image is set up
+umask 0022
+mage build unitTest
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index d4b33be7690..cf66e4edf6e 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then
   source .buildkite/env-scripts/env.sh
   source .buildkite/env-scripts/util.sh
   source .buildkite/env-scripts/win-env.sh
@@ -12,7 +12,8 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
   fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
   source .buildkite/scripts/setenv.sh
   if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then
     echo "Skipped pre-command when running the Upload pipeline"
diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml
index 1fb185b59d1..cfaf7818360 100644
--- a/.buildkite/libbeat/pipeline.libbeat.yml
+++ b/.buildkite/libbeat/pipeline.libbeat.yml
@@ -4,7 +4,7 @@ env:
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
-  GCP_HI_PERF_MASHINE_TYPE: "c2d-highcpu-16"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
   AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
   BEATS_PROJECT_NAME: "libbeat"
diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml
index c42f17d2a36..d882edaaa0a 100644
--- a/.buildkite/metricbeat/pipeline.yml
+++ b/.buildkite/metricbeat/pipeline.yml
@@ -11,7 +11,7 @@ env:
   GO_AGENT_IMAGE: "golang:${GO_VERSION}"
   BEATS_PROJECT_NAME: "metricbeat"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
-  GCP_HI_PERF_MASHINE_TYPE: "c2d-highcpu-16"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
   AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
 
diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml
index 34321b61161..0dbb87fc1d1 100644
--- a/.buildkite/packetbeat/pipeline.packetbeat.yml
+++ b/.buildkite/packetbeat/pipeline.packetbeat.yml
@@ -1,5 +1,58 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+env:
+  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  IMAGE_RHEL9_X86_64: "family/core-rhel-9"
+  IMAGE_WIN_10: "family/general-windows-10"
+  IMAGE_WIN_11: "family/general-windows-11"
+  IMAGE_WIN_2016: "family/core-windows-2016"
+  IMAGE_WIN_2019: "family/core-windows-2019"
+  IMAGE_WIN_2022: "family/core-windows-2022"
+  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  BEATS_PROJECT_NAME: "packetbeat"
 
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+
+  - input: "Input Parameters"
+    key: "input-run-all-stages"
+    fields:
+    - select: "Packetbeat - runPacketbeat"
+      key: "runPacketbeat"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+    - select: "Packetbeat - runPacketbeatArmTest"
+      key: "runPacketbeatArmTest"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+    - select: "Packetbeat - runPacketbeatMacOsTests"
+      key: "runPacketbeatMacOsTests"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+    if: "build.source == 'ui'"
+
+  - wait: ~
+    if: "build.source == 'ui'"
+    allow_dependency_failure: false
+
+  - label: ":linux: Load dynamic packetbeat pipeline"
+    key: "packetbeat-pipeline"
+    command: ".buildkite/scripts/generate_packetbeat_pipeline.sh"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json
index 66c508e252c..5d1310f101e 100644
--- a/.buildkite/pull-requests.json
+++ b/.buildkite/pull-requests.json
@@ -73,8 +73,8 @@
             "set_commit_status": true,
             "build_on_commit": true,
             "build_on_comment": true,
-            "trigger_comment_regex": "^/test heartbeat$",
-            "always_trigger_comment_regex": "^/test heartbeat$",
+            "trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/package heartbeat$",
+            "always_trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/package heartbeat$",
             "skip_ci_labels": [  ],
             "skip_target_branches": [ ],
             "skip_ci_on_only_changed": [ ],
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index e3dd2ec4ac4..f51097091f4 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -8,19 +8,27 @@ platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]')
 arch_type="$(uname -m)"
 GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""}
 ONLY_DOCS=${ONLY_DOCS:-"true"}
-runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
-runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})"
-runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
-runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})"
+
+[ -z "${runLibbeat+x}" ] && runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
+[ -z "${runMetricbeat+x}" ] && runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})"
+[ -z "${runPacketbeat+x}" ] && runPacketbeat="$(buildkite-agent meta-data get runPacketbeat --default ${runPacketbeat:-"false"})"
+[ -z "${runLibBeatArmTest+x}" ] && runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
+[ -z "${runPacketbeatArmTest+x}" ] && runPacketbeatArmTest="$(buildkite-agent meta-data get runPacketbeatArmTest --default ${runPacketbeatArmTest:-"false"})"
+[ -z "${runMetricbeatMacOsTests+x}" ] && runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})"
+[ -z "${runPacketbeatMacOsTests+x}" ] && runPacketbeatMacOsTests="$(buildkite-agent meta-data get runPacketbeatMacOsTests --default ${runPacketbeatMacOsTests:-"false"})"
 
 metricbeat_changeset=(
   "^metricbeat/.*"
-  "^go.mod"
-  "^pytest.ini"
-  "^dev-tools/.*"
+  )
+
+libbeat_changeset=(
   "^libbeat/.*"
-  "^testing/.*"
   )
+
+packetbeat_changeset=(
+  "^packetbeat/.*"
+  )
+
 oss_changeset=(
   "^go.mod"
   "^pytest.ini"
@@ -28,16 +36,20 @@ oss_changeset=(
   "^libbeat/.*"
   "^testing/.*"
 )
+
 ci_changeset=(
   "^.buildkite/.*"
 )
+
 go_mod_changeset=(
   "^go.mod"
   )
+
 docs_changeset=(
   ".*\\.(asciidoc|md)"
   "deploy/kubernetes/.*-kubernetes\\.yaml"
   )
+
 packaging_changeset=(
   "^dev-tools/packaging/.*"
   ".go-version"
@@ -77,7 +89,7 @@ check_platform_architeture() {
       go_arch_type="arm64"
       ;;
     *)
-    echo "The current platform/OS type is unsupported yet"
+    echo "The current platform or OS type is unsupported yet"
     ;;
   esac
 }
@@ -109,11 +121,32 @@ with_go() {
   export PATH="${go_path}:${PATH}"
 }
 
+checkLinuxType() {
+  if [ "${platform_type}" == "Linux" ]; then
+    if grep -q 'ubuntu' /etc/os-release; then
+      echo "ubuntu"
+    elif grep -q 'rhel' /etc/os-release; then
+      echo "rhel"
+    else
+      echo "Unsupported Linux"
+    fi
+  else
+      echo "This is not a Linux"
+  fi
+}
+
 with_python() {
+  local linuxType="$(checkLinuxType)"
+  echo "${linuxType}"
   if [ "${platform_type}" == "Linux" ]; then
-    #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions)
-    sudo apt-get update
-    sudo apt-get install -y python3-pip python3-venv
+    if [ "${linuxType}" = "ubuntu" ]; then
+      sudo apt-get update
+      sudo apt-get install -y python3-pip python3-venv
+    elif [ "${linuxType}" = "rhel" ]; then
+      sudo dnf update -y
+      sudo dnf install -y python3 python3-pip
+      pip3 install virtualenv
+    fi
   elif [ "${platform_type}" == "Darwin" ]; then
     brew update
     pip3 install virtualenv
@@ -122,15 +155,30 @@ with_python() {
 }
 
 with_dependencies() {
+  local linuxType="$(checkLinuxType)"
+  echo "${linuxType}"
   if [ "${platform_type}" == "Linux" ]; then
-    #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions)
-    sudo apt-get update
-    sudo apt-get install -y libsystemd-dev libpcap-dev
+    if [ "${linuxType}" = "ubuntu" ]; then
+      sudo apt-get update
+      sudo apt-get install -y libsystemd-dev libpcap-dev
+    elif [ "${linuxType}" = "rhel" ]; then
+      # sudo dnf update -y
+      sudo dnf install -y systemd-devel
+      wget https://mirror.stream.centos.org/9-stream/CRB/${arch_type}/os/Packages/libpcap-devel-1.10.0-4.el9.${arch_type}.rpm     #TODO: move this step to our own image
+      sudo dnf install -y libpcap-devel-1.10.0-4.el9.${arch_type}.rpm     #TODO: move this step to our own image
+    fi
   elif [ "${platform_type}" == "Darwin" ]; then
     pip3 install libpcap
   fi
 }
 
+config_git() {
+  if [ -z "$(git config --get user.email)" ]; then
+    git config --global user.email "beatsmachine@users.noreply.github.com"
+    git config --global user.name "beatsmachine"
+  fi
+}
+
 retry() {
   local retries=$1
   shift
@@ -153,7 +201,6 @@ retry() {
 are_paths_changed() {
   local patterns=("${@}")
   local changelist=()
-
   for pattern in "${patterns[@]}"; do
     changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern"))
     if [ "${#changed_files[@]}" -gt 0 ]; then
@@ -183,13 +230,17 @@ are_changed_only_paths() {
 }
 
 are_conditions_met_mandatory_tests() {
-  if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
+  if are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
     if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
-      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then
+      if are_paths_changed "${metricbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then
         return 0
       fi
     elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
-      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then
+      if are_paths_changed "${libbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then
+        return 0
+      fi
+    elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
+      if are_paths_changed "${packetbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat" || "${GITHUB_PR_LABELS}" =~ Packetbeat || "${runPacketbeat}" == "true" ]]; then
         return 0
       fi
     fi
@@ -197,27 +248,47 @@ are_conditions_met_mandatory_tests() {
   return 1
 }
 
-are_conditions_met_libbeat_arm_tests() {
+are_conditions_met_arm_tests() {
   if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
     if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
       if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${runLibBeatArmTest}" == "true" ]]; then
         return 0
       fi
+    elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
+      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${runPacketbeatArmTest}" == "true" ]]; then
+        return 0
+      fi
     fi
   fi
   return 1
 }
 
-are_conditions_met_metricbeat_macos_tests() {
-  if [[ "${runMetricbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
-    return 0
+are_conditions_met_macos_tests() {
+  if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
+      if [[ "${runMetricbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
+        return 0
+      fi
+    elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
+      if [[ "${runPacketbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then
+        return 0
+      fi
+    fi
   fi
   return 1
 }
 
 are_conditions_met_packaging() {
-  if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || [[ "${BUILDKITE_TAG}" == "" ]] || [[ "${BUILDKITE_PULL_REQUEST}" != "" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103
-    return 0
+  if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
+      if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103
+        return 0
+      fi
+    elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
+      if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then
+        return 0
+      fi
+    fi
   fi
   return 1
 }
diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh
index 6da1bef711d..0674d0b186b 100755
--- a/.buildkite/scripts/generate_libbeat_pipeline.sh
+++ b/.buildkite/scripts/generate_libbeat_pipeline.sh
@@ -30,7 +30,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MASHINE_TYPE}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
 
       - label: ":python: Python Integration Tests"
@@ -39,7 +39,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MASHINE_TYPE}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
 
       - label: ":negative_squared_cross_mark: Cross compile"
@@ -48,7 +48,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MASHINE_TYPE}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths: " ${BEATS_PROJECT_NAME}/build/*.xml"
 
       - label: ":testengine: Stress Tests"
@@ -61,10 +61,13 @@ steps:
         artifact_paths: "${BEATS_PROJECT_NAME}/libbeat-stress-test.xml"
 
 YAML
+else
+  echo "The conditions don't match to requirements for generating pipeline steps."
+  exit 0
 fi
 
 echo "Check and add the Extended Tests into the pipeline"
-if are_conditions_met_libbeat_arm_tests; then
+if are_conditions_met_arm_tests; then
   cat >> $pipelineName <<- YAML
 
   - group: "Extended Tests"
diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh
index 0ea19734c4f..850ef14f9bd 100755
--- a/.buildkite/scripts/generate_metricbeat_pipeline.sh
+++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh
@@ -30,7 +30,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MASHINE_TYPE}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
 
       - label: ":python: Python Integration Tests"
@@ -39,7 +39,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MASHINE_TYPE}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
 
       - label: ":negative_squared_cross_mark: Cross compile"
@@ -107,10 +107,13 @@ steps:
           disk_type: "pd-ssd"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
 YAML
+else
+  echo "The conditions don't match to requirements for generating pipeline steps."
+  exit 0
 fi
 
 echo "Check and add the Extended Tests into the pipeline"
-if are_conditions_met_metricbeat_macos_tests; then
+if are_conditions_met_macos_tests; then
   cat >> $pipelineName <<- YAML
 
   - group: "Extended Tests"
@@ -128,7 +131,7 @@ YAML
 fi
 
 echo "Check and add the Packaging into the pipeline"
-if are_conditions_met_mandatory_tests && are_conditions_met_packaging; then
+if are_conditions_met_packaging; then
   cat >> $pipelineName <<- YAML
 
   - wait: ~
@@ -145,7 +148,7 @@ if are_conditions_met_mandatory_tests && are_conditions_met_packaging; then
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MASHINE_TYPE}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
 
diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh
new file mode 100755
index 00000000000..89ea7a33e20
--- /dev/null
+++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh
@@ -0,0 +1,173 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+pipelineName="pipeline.packetbeat-dynamic.yml"
+
+echo "Add the mandatory and extended tests without additional conditions into the pipeline"
+if are_conditions_met_mandatory_tests; then
+  cat > $pipelineName <<- YAML
+
+steps:
+
+  - group: "Mandatory Tests"
+    key: "mandatory-tests"
+    steps:
+      - label: ":linux: Ubuntu Unit Tests"
+        key: "mandatory-linux-unit-test"
+        command: ".buildkite/scripts/unit_tests.sh"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":rhel: RHEL-9 Unit Tests"
+        key: "mandatory-rhel9-unit-test"
+        command: ".buildkite/scripts/unit_tests.sh"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_RHEL9_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+
+      - label: ":windows: Windows Unit Tests - {{matrix.image}}"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
+        key: "mandatory-win-unit-tests"
+        agents:
+          provider: "gcp"
+          image: "{{matrix.image}}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_WIN_2016}"
+              - "${IMAGE_WIN_2022}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+  - group: "Extended Windowds Tests"
+    key: "extended-win-tests"
+    steps:
+      - label: ":windows: Win 2019 Unit Tests"
+        key: "extended-win-2019-unit-tests"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2019}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows: Windows 10 Unit Tests"
+        key: "extended-win-10-unit-tests"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_10}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows: Windows 11 Unit Tests"
+        key: "extended-win-11-unit-tests"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_11}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+YAML
+else
+  echo "The conditions don't match to requirements for generating pipeline steps."
+  exit 0
+fi
+
+if are_conditions_met_arm_tests && are_conditions_met_macos_tests; then
+  cat >> $pipelineName <<- YAML
+
+  - group: "Extended Tests"
+    key: "extended-tests"
+    steps:
+
+YAML
+fi
+
+if  are_conditions_met_macos_tests; then
+  cat >> $pipelineName <<- YAML
+
+      - label: ":mac: MacOS Unit Tests"
+        key: "extended-macos-unit-tests"
+        command: ".buildkite/scripts/unit_tests.sh"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+YAML
+fi
+
+if  are_conditions_met_arm_tests; then
+  cat >> $pipelineName <<- YAML
+      - label: ":linux: ARM Ubuntu Unit Tests"
+        key: "extended-arm64-unit-test"
+        command: ".buildkite/scripts/unit_tests.sh"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+YAML
+fi
+
+
+if are_conditions_met_packaging; then
+  cat >> $pipelineName <<- YAML
+
+  - wait: ~
+    depends_on:
+      - step: "mandatory-tests"
+        allow_failure: false
+
+  - group: "Packaging"    # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589
+    key: "packaging"
+    steps:
+      - label: ":linux: Packaging Linux"
+        key: "packaging-linux"
+        command: ".buildkite/scripts/packaging.sh"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+
+      - label: ":linux: Packaging ARM"
+        key: "packaging-arm"
+        command: ".buildkite/scripts/packaging.sh"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "docker"
+
+YAML
+fi
+
+echo "--- Printing dynamic steps"     #TODO: remove if the pipeline is public
+cat $pipelineName
+
+echo "--- Loading dynamic steps"
+buildkite-agent pipeline upload $pipelineName
diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh
old mode 100644
new mode 100755
index 1a1e3a29f54..80e70ae96c5
--- a/.buildkite/scripts/install_tools.sh
+++ b/.buildkite/scripts/install_tools.sh
@@ -9,10 +9,14 @@ echo "--- Env preparation"
 # Temporary solution to fix the issues with "sudo apt get...." https://elastic.slack.com/archives/C0522G6FBNE/p1706003603442859?thread_ts=1706003209.424539&cid=C0522G6FBNE
 # It could be removed when we use our own image for the BK agent.
 if [ "${platform_type}" == "Linux" ]; then
-  DEBIAN_FRONTEND="noninteractive"
-  #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions)
-  sudo mkdir -p /etc/needrestart
-  echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null
+  if [ "${platform_type}" == "Linux" ]; then
+    if [ $(checkLinuxType) = "ubuntu" ]; then
+      DEBIAN_FRONTEND="noninteractive"
+      #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions)
+      sudo mkdir -p /etc/needrestart
+      echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null
+    fi
+  fi
 fi
 
 add_bin_path
diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh
index 901ba9891c2..dbf23c3e5ef 100755
--- a/.buildkite/scripts/setenv.sh
+++ b/.buildkite/scripts/setenv.sh
@@ -8,8 +8,42 @@ DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0"
 SETUP_WIN_PYTHON_VERSION="3.11.0"
 GO_VERSION=$(cat .go-version)
 
+
 export SETUP_GVM_VERSION
 export DOCKER_COMPOSE_VERSION
 export DOCKER_COMPOSE_VERSION_AARCH64
 export SETUP_WIN_PYTHON_VERSION
 export GO_VERSION
+
+exportVars() {
+  local platform_type="$(uname)"
+  local arch_type="$(uname -m)"
+  if [ "${arch_type}" == "x86_64" ]; then
+    case "${platform_type}" in
+      Linux|Darwin)
+        export GOX_FLAGS="-arch amd64"
+        export testResults="**/build/TEST*.xml"
+        export artifacts="**/build/TEST*.out"
+        ;;
+      MINGW*)
+        export GOX_FLAGS="-arch 386"
+        export testResults="**\\build\\TEST*.xml"
+        export artifacts="**\\build\\TEST*.out"
+        ;;
+    esac
+  elif [[ "${arch_type}" == "aarch64" || "${arch_type}" == "arm64" ]]; then
+    export GOX_FLAGS="-arch arm"
+    export testResults="**/build/TEST*.xml"
+    export artifacts="**/build/TEST*.out"
+  else
+    echo "Unsupported OS"
+  fi
+}
+
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
+  exportVars
+  export RACE_DETECTOR="true"
+  export TEST_COVERAGE="true"
+  export DOCKER_PULL="0"
+fi
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 0df9a9a5877..ea38e969a3d 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -18,7 +18,7 @@ CHANGELOG*
 /go.sum @elastic/beats-tech-leads
 /NOTICE.txt @elastic/beats-tech-leads
 
-/.ci/ @elastic/elastic-agent-data-plane
+/.ci/ @elastic/elastic-agent-data-plane @elastic/ingest-eng-prod
 /.github/ @elastic/elastic-agent-data-plane
 /auditbeat/ @elastic/sec-linux-platform
 /deploy/ @elastic/elastic-agent-data-plane
@@ -221,3 +221,9 @@ CHANGELOG*
 /x-pack/osquerybeat/ @elastic/sec-deployment-and-devices
 /x-pack/packetbeat/ @elastic/sec-linux-platform
 /x-pack/winlogbeat/ @elastic/sec-windows-platform
+
+# Ownership of CI or related files by the Ingest Eng Prod team
+/.buildkite @elastic/ingest-eng-prod
+/catalog-info.yml @elastic/ingest-eng-prod
+/libbeat/scripts @elastic/ingest-eng-prod
+/metricbeat/tests @elastic/ingest-eng-prod
diff --git a/.go-version b/.go-version
index c262b1f0dfd..8819d012cee 100644
--- a/.go-version
+++ b/.go-version
@@ -1 +1 @@
-1.21.6
+1.21.7
diff --git a/.golangci.yml b/.golangci.yml
index 03a01e24c4f..79b77eab0d1 100755
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -114,7 +114,7 @@ linters-settings:
 
   gosimple:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.6"
+    go: "1.21.7"
 
   nakedret:
     # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
@@ -132,19 +132,19 @@ linters-settings:
 
   staticcheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.6"
+    go: "1.21.7"
     checks: ["all"]
 
   stylecheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.6"
+    go: "1.21.7"
     # Disabled:
     # ST1005: error strings should not be capitalized
     checks: ["all", "-ST1005"]
 
   unused:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.6"
+    go: "1.21.7"
 
   gosec:
     excludes:
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index cb5aa3b6354..8b8e799f8af 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -74,6 +74,7 @@ If you are not using the Elasticsearch output, set `queue.mem.flush.timeout: 1s`
 
 *Heartbeat*
 - Decrease the ES default timeout to 10 for the load monitor state requests.
+- Windows MSI installers now store configuration in C:\Program Files instead of C:\ProgramData. https://github.com/elastic/elastic-stack-installers/pull/209
 
 *Osquerybeat*
 
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 8281f7b79ec..9e27962b7e5 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -10,7 +10,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Affecting all Beats*
 
-- Upgrade to Go 1.21.6. Removes support for Windows 8.1. See https://tip.golang.org/doc/go1.21#windows. {pull}37615[37615]
+- Upgrade to Go 1.21.7. Removes support for Windows 8.1. See https://tip.golang.org/doc/go1.21#windows. {pull}37913[37913]
 - add_cloud_metadata processor: `huawei` provider is now treated as `openstack`. Huawei cloud runs on OpenStack
 platform, and when viewed from a metadata API standpoint, it is impossible to differentiate it from OpenStack. If you
 know that your deployments run on Huawei Cloud exclusively, and you wish to have `cloud.provider` value as `huawei`,
@@ -89,6 +89,7 @@ fields added to events containing the Beats version. {pull}37553[37553]
 - Fix m365_defender cursor value and query building. {pull}37116[37116]
 - Fix TCP/UDP metric queue length parsing base. {pull}37714[37714]
 - Update github.com/lestrrat-go/jwx dependency. {pull}37799[37799]
+- [threatintel] MISP pagination fixes {pull}37898[37898]
 
 *Heartbeat*
 
diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile
index 59eb4860b40..df038d2edf8 100644
--- a/auditbeat/Dockerfile
+++ b/auditbeat/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.21.6
+FROM golang:1.21.7
 
 RUN \
     apt-get update \
diff --git a/dev-tools/kubernetes/filebeat/Dockerfile.debug b/dev-tools/kubernetes/filebeat/Dockerfile.debug
index 842da44f3ab..e8dfaf392ab 100644
--- a/dev-tools/kubernetes/filebeat/Dockerfile.debug
+++ b/dev-tools/kubernetes/filebeat/Dockerfile.debug
@@ -1,4 +1,4 @@
-FROM golang:1.21.6 as builder
+FROM golang:1.21.7 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 
diff --git a/dev-tools/kubernetes/heartbeat/Dockerfile.debug b/dev-tools/kubernetes/heartbeat/Dockerfile.debug
index fd9970a5b08..473ce7484c8 100644
--- a/dev-tools/kubernetes/heartbeat/Dockerfile.debug
+++ b/dev-tools/kubernetes/heartbeat/Dockerfile.debug
@@ -1,4 +1,4 @@
-FROM golang:1.21.6 as builder
+FROM golang:1.21.7 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 
diff --git a/dev-tools/kubernetes/metricbeat/Dockerfile.debug b/dev-tools/kubernetes/metricbeat/Dockerfile.debug
index 00df9d9be1d..8adf8a45901 100644
--- a/dev-tools/kubernetes/metricbeat/Dockerfile.debug
+++ b/dev-tools/kubernetes/metricbeat/Dockerfile.debug
@@ -1,4 +1,4 @@
-FROM golang:1.21.6 as builder
+FROM golang:1.21.7 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 
diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile
index 335bf29256e..eb52ad4d130 100644
--- a/heartbeat/Dockerfile
+++ b/heartbeat/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.21.6
+FROM golang:1.21.7
 
 RUN \
     apt-get update \
diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc
index 13456f9cd18..1cae4fe1ad8 100644
--- a/libbeat/docs/version.asciidoc
+++ b/libbeat/docs/version.asciidoc
@@ -1,6 +1,6 @@
 :stack-version: 8.13.0
 :doc-branch: main
-:go-version: 1.21.6
+:go-version: 1.21.7
 :release-state: unreleased
 :python: 3.7
 :docker: 1.12
diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile
index baf372f1859..31f13aeea2c 100644
--- a/metricbeat/Dockerfile
+++ b/metricbeat/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.21.6
+FROM golang:1.21.7
 
 RUN \
     apt update \
diff --git a/metricbeat/module/http/_meta/Dockerfile b/metricbeat/module/http/_meta/Dockerfile
index 55df10b1294..a46a3dbb3e2 100644
--- a/metricbeat/module/http/_meta/Dockerfile
+++ b/metricbeat/module/http/_meta/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.21.6
+FROM golang:1.21.7
 
 COPY test/main.go main.go
 
diff --git a/metricbeat/module/nats/_meta/Dockerfile b/metricbeat/module/nats/_meta/Dockerfile
index ef6e428ae87..4df0b81a912 100644
--- a/metricbeat/module/nats/_meta/Dockerfile
+++ b/metricbeat/module/nats/_meta/Dockerfile
@@ -2,7 +2,7 @@ ARG NATS_VERSION=2.0.4
 FROM nats:$NATS_VERSION
 
 # build stage
-FROM golang:1.21.6 AS build-env
+FROM golang:1.21.7 AS build-env
 RUN apt-get install git mercurial gcc
 RUN git clone https://github.com/nats-io/nats.go.git /nats-go
 RUN cd /nats-go/examples/nats-bench && git checkout tags/v1.10.0 && go build .
diff --git a/metricbeat/module/vsphere/_meta/Dockerfile b/metricbeat/module/vsphere/_meta/Dockerfile
index 1f0881c14ec..3db3cccbab2 100644
--- a/metricbeat/module/vsphere/_meta/Dockerfile
+++ b/metricbeat/module/vsphere/_meta/Dockerfile
@@ -1,5 +1,5 @@
 ARG VSPHERE_GOLANG_VERSION
-FROM golang:1.21.6
+FROM golang:1.21.7
 
 RUN apt-get install curl git
 RUN go install github.com/vmware/govmomi/vcsim@v0.30.4
diff --git a/packetbeat/Dockerfile b/packetbeat/Dockerfile
index 6e5c1d0bab4..17075d9da65 100644
--- a/packetbeat/Dockerfile
+++ b/packetbeat/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.21.6
+FROM golang:1.21.7
 
 RUN \
     apt-get update \
diff --git a/x-pack/filebeat/module/threatintel/misp/config/config.yml b/x-pack/filebeat/module/threatintel/misp/config/config.yml
index 3bd5aac30ec..9ad66efcf54 100644
--- a/x-pack/filebeat/module/threatintel/misp/config/config.yml
+++ b/x-pack/filebeat/module/threatintel/misp/config/config.yml
@@ -32,8 +32,20 @@ request.transforms:
     value: json
 - set:
     target: body.timestamp
-    value: '[[.cursor.timestamp]]'
-    default: '[[ formatDate (now (parseDuration "-{{ .first_interval }}")) "UnixDate" ]]'
+    value: >-
+      [[- if index .cursor "timestamp" -]]
+        [[- .cursor.timestamp -]]
+      [[- else -]]
+        [[- .last_response.url.params.Get "timestamp" -]]
+      [[- end -]]
+    default: '[[ (now (parseDuration "-{{ .first_interval }}")).Unix ]]'
+- set:
+    target: body.order
+    value: timestamp
+- set:
+    # Ignored by MISP, set as a workaround to make it available in response.pagination.
+    target: url.params.timestamp
+    value: '[[.body.timestamp]]'
 
 response.split:
   target: body.response
@@ -51,8 +63,15 @@ response.request_body_on_pagination: true
 response.pagination:
 - set:
     target: body.page
-    value: '[[if (ne (len .last_response.body.response) 0)]][[add .last_response.page 1]][[end]]'
+    # Add 2 because the httpjson page counter is zero-based while the MISP page parameter starts at 1.
+    value: '[[if (ne (len .last_response.body.response) 0)]][[add .last_response.page 2]][[end]]'
     fail_on_template_error: true
+- set:
+    target: body.timestamp
+    value: '[[.last_response.url.params.Get "timestamp"]]'
+- set:
+    target: url.params.timestamp
+    value: '[[.last_response.url.params.Get "timestamp"]]'
 cursor:
   timestamp:
     value: '[[.last_event.Event.timestamp]]'
diff --git a/x-pack/functionbeat/Dockerfile b/x-pack/functionbeat/Dockerfile
index aec1914698c..b2c85869921 100644
--- a/x-pack/functionbeat/Dockerfile
+++ b/x-pack/functionbeat/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.21.6
+FROM golang:1.21.7
 
 RUN \
     apt-get update \
diff --git a/x-pack/metricbeat/module/stan/_meta/Dockerfile b/x-pack/metricbeat/module/stan/_meta/Dockerfile
index b4da8bf79a3..ffce883a60c 100644
--- a/x-pack/metricbeat/module/stan/_meta/Dockerfile
+++ b/x-pack/metricbeat/module/stan/_meta/Dockerfile
@@ -2,7 +2,7 @@ ARG STAN_VERSION=0.15.1
 FROM nats-streaming:$STAN_VERSION
 
 # build stage
-FROM golang:1.21.6 AS build-env
+FROM golang:1.21.7 AS build-env
 RUN apt-get install git mercurial gcc
 RUN git clone https://github.com/nats-io/stan.go.git /stan-go
 RUN cd /stan-go/examples/stan-bench && git checkout tags/v0.5.2 && go build .