Merge branch 'main' into 10463-create-agentless-agent

.buildkite/scripts/generate_notice.py

@@ -31,6 +31,7 @@
     {"name": "github.com/spdx/tools-golang", "licenceFile": "LICENSE.code", "licenceType": "Apache-2.0"},
     {"name": "github.com/aquasecurity/trivy-policies", "licenceType": "MIT"},
     {"name": "github.com/csaf-poc/csaf_distribution/v3", "licenceType": "Apache-2.0"},
+    {"name": "github.com/xi2/xz", "licenceType": "Unlicense"},
 ]
 
 # Additional third-party, non-source code dependencies, to add to the CSV output.

.ci/updatecli/updatecli.d/update-beats.yml

@@ -16,7 +16,7 @@ scms:
 
 actions:
   default:
-    title: '[updatecli] Update to elastic/beats@{{ source "beats" }}'
+    title: '[updatecli] {{ requiredEnv "GIT_BRANCH" }} - Update to elastic/beats@{{ source "beats" }}'
     kind: github/pullrequest
     scmid: default
     spec:

.ci/updatecli/updatecli.d/update-golang.yml

@@ -16,16 +16,16 @@ scms:
 
 actions:
   cloudbeat:
-    title: '[updatecli] Update Golang version to {{ source "latestGoVersion" }}'
+    title: '[updatecli] {{ requiredEnv "GIT_BRANCH" }} - Update Golang version to {{ source "latestGoVersion" }}'
     kind: github/pullrequest
     scmid: default
     sourceid: latestGoVersion
     spec:
       automerge: false
       labels:
         - automation
+        - backport-skip
         - dependency
-        - backport-v8.x
         - go
       description: |-
         ### What

.ci/updatecli/updatecli.d/update-hermit.yml

@@ -16,14 +16,14 @@ scms:
 
 actions:
   default:
-    title: '[updatecli] Update hermit and pre-commit dependencies'
+    title: '[updatecli] {{ requiredEnv "GIT_BRANCH" }} - Update hermit and pre-commit dependencies'
     kind: github/pullrequest
     scmid: default
     spec:
       automerge: false
       labels:
         - automation
-        - backport-v8.x
+        - backport-skip
         - dependency
       description: |-
         ### What

.ci/updatecli/updatecli.d/update-mods.yml

@@ -16,14 +16,14 @@ scms:
 
 actions:
   default:
-    title: '[updatecli] Update golang mod dependencies'
+    title: '[updatecli] {{ requiredEnv "GIT_BRANCH" }} - Update golang mod dependencies'
     kind: github/pullrequest
     scmid: default
     spec:
       automerge: false
       labels:
         - automation
-        - backport-v8.x
+        - backport-skip
         - dependency
         - go
       description: |-

.github/dependabot.yml

@@ -2,9 +2,12 @@
 
 version: 2
 updates:
-  # Maintain dependencies for go.mod
+  # Maintain dependencies for go.mod main branch
   - package-ecosystem: "gomod"
     directory: "/"
+    commit-message:
+      include: scope
+      prefix: "main"
     groups:
       azure:
         patterns:
@@ -26,24 +29,83 @@ updates:
         patterns:
           - "github.com/aquasecurity/*"
     labels:
-      - "backport-v8.x"
       - "dependency"
+      - "backport-skip"
       - "go"
-    open-pull-requests-limit: 10
+    open-pull-requests-limit: 2
     # elastic/beats is using v7 in their go.mod, leading to automatic updates not working. Manual update is needed with
     # go get -u github.com/elastic/beats/v7@<commit sha>
     ignore:
       - dependency-name: "github.com/elastic/beats/v7"
     schedule:
       interval: "monthly"
 
+  # Maintain dependencies for go.mod 8.x branch
+  - package-ecosystem: "gomod"
+    target-branch: "8.x"
+    directory: "/"
+    commit-message:
+      include: scope
+      prefix: "8.x"
+    groups:
+      azure:
+        patterns:
+          - "github.com/Azure/*"
+      aws:
+        patterns:
+          - "github.com/aws/*"
+      google:
+        patterns:
+          - "cloud.google.com/*"
+          - "google.golang.org/api"
+      elastic:
+        patterns:
+          - "github.com/elastic/*"
+      k8s:
+        patterns:
+          - "k8s.io/*"
+      trivy:
+        patterns:
+          - "github.com/aquasecurity/*"
+    labels:
+      - "dependency"
+      - "backport-skip"
+      - "go"
+    open-pull-requests-limit: 2
+    # elastic/beats is using v7 in their go.mod, leading to automatic updates not working. Manual update is needed with
+    # go get -u github.com/elastic/beats/v7@<commit sha>
+    ignore:
+      - dependency-name: "github.com/elastic/beats/v7"
+    schedule:
+      interval: "monthly"
+
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: ".github/workflows"
+    schedule:
+      interval: "monthly"
+    labels:
+      - "dependency"
+      - "github_actions"
+    groups:
+      artifact:
+        patterns:
+        - "actions/download-artifact"
+        - "actions/upload-artifact"
+    ignore:
+        # Skip versions 4.x upload-artifact and download-artifact due to this issue: https://github.com/actions/upload-artifact/issues/478
+      - dependency-name: "actions/download-artifact"
+        versions: [">=4.0.0"]
+      - dependency-name: "actions/upload-artifact"
+        versions: [">=4.0.0"]
+
   # Maintain dependencies for GitHub Actions
   - package-ecosystem: "github-actions"
     directory: ".github/workflows"
+    target-branch: "8.x"
     schedule:
       interval: "monthly"
     labels:
-      - "backport-v8.x"
       - "dependency"
       - "github_actions"
     groups:

.github/workflows/ci-pull_request.yml

@@ -94,7 +94,7 @@ jobs:
         run: |
           go install gotest.tools/gotestsum
           GOOS=linux TEST_DIRECTORY=./... gotestsum --format pkgname -- -race -coverpkg=./... -coverprofile=cover.out.tmp
-          cat cover.out.tmp | grep -v "mock_.*.go" > cover.out # remove mock files from coverage report
+          cat cover.out.tmp | grep -v "mock_.*.go" | grep -v "elastic/cloudbeat/deploy" > cover.out # remove mock files and deploy dir
 
       - name: Upload coverage artifact
         uses: actions/upload-artifact@v4

.github/workflows/ci.yml

@@ -19,6 +19,29 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  check-asset-inventory-supported-version:
+    name: Check Asset Inventory supported version
+    runs-on: ubuntu-22.04
+    outputs:
+      asset_inventory_supported: ${{ steps.check-asset-inventory-supported-version.outputs.asset_inventory_supported }}
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Check Asset Inventory supported version
+        id: check-asset-inventory-supported-version
+        env:
+          STACK_VERSION: ${{ env.ELK_VERSION }}
+        run: |
+          MIN_VERSION="8.16.0"
+          if [[ "$(echo -e "$MIN_VERSION\n$STACK_VERSION" | sort -V | head -n 1)" == "$MIN_VERSION" ]]; then
+            echo "Stack version meets the requirement: $STACK_VERSION >= $MIN_VERSION."
+            echo "asset_inventory_supported=true" >> $GITHUB_ENV
+          else
+            echo "Stack version is below the requirement: $STACK_VERSION < $MIN_VERSION."
+            echo "asset_inventory_supported=false" >> $GITHUB_ENV
+          fi
+
   init-hermit:
     name: Init Hermit Tools
     runs-on: ubuntu-22.04
@@ -55,9 +78,10 @@ jobs:
           azure-client-secret: ${{ fromJSON(secrets.AZURE_CREDENTIALS).clientSecret }}
 
   ci-azure-asset-inventory:
-    needs: [ init-hermit ]
+    needs: [ init-hermit, check-asset-inventory-supported-version ]
     name: Azure Asset Inventory CI
     runs-on: ubuntu-22.04
+    if: needs.check-asset-inventory-supported-version.outputs.asset_inventory_supported == 'true'
     timeout-minutes: 60
     permissions:
       contents: "read"
@@ -101,9 +125,10 @@ jobs:
           aws-account-type: single-account
 
   ci-aws-asset-inventory:
-    needs: [ init-hermit ]
+    needs: [ init-hermit, check-asset-inventory-supported-version ]
     name: AWS Asset Inventory CI
     runs-on: ubuntu-22.04
+    if: needs.check-asset-inventory-supported-version.outputs.asset_inventory_supported == 'true'
     timeout-minutes: 60
     permissions:
       contents: "read"
@@ -147,9 +172,10 @@ jobs:
           gcp-account-type: 'single-account'
 
   ci-gcp-asset-inventory:
-    needs: [init-hermit]
+    needs: [ init-hermit, check-asset-inventory-supported-version ]
     name: GCP Asset Inventory CI
     runs-on: ubuntu-22.04
+    if: needs.check-asset-inventory-supported-version.outputs.asset_inventory_supported == 'true'
     timeout-minutes: 60
     permissions:
       contents: "read"
@@ -329,7 +355,7 @@ jobs:
 
       - name: Publish allure report
         if: ${{ always() }}
-        uses: andrcuns/allure-publish-action@v2.7.1
+        uses: andrcuns/allure-publish-action@v2.8.0
         env:
           GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           AWS_REGION: ${{ env.AWS_REGION }}

.github/workflows/eks-ci.yml

@@ -241,7 +241,7 @@ jobs:
 
       - name: Publish allure report
         if: always()
-        uses: andrcuns/allure-publish-action@v2.7.1
+        uses: andrcuns/allure-publish-action@v2.8.0
         env:
           GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           AWS_REGION: eu-west-1

.github/workflows/publish-cloudformation.yml

@@ -6,6 +6,7 @@ on:
       - main
       - "[0-9]+.[0-9]+"
     paths:
+      - deploy/asset-inventory-cloudformation/*.yml
       - deploy/cloudformation/*.yml
       - scripts/publish_cft.sh
       - .github/workflows/publish-cloudformation.yml

.github/workflows/test-environment.yml

@@ -118,7 +118,6 @@ env:
   WORKING_DIR: deploy/test-environments
   INTEGRATIONS_SETUP_DIR: tests/integrations_setup
   AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments"
-  GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments,owner=${{ github.actor }}"
   GCP_ZONE: "us-central1-a"
   AZURE_DEFAULT_TAGS: "division=engineering org=security team=cloud-security-posture project=test-environments owner=${{ github.actor }}"
   TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }}
@@ -403,7 +402,14 @@ jobs:
       - name: Deploy CSPM GCP agent
         id: cspm-gcp-agent
         working-directory: deploy/deployment-manager
+        env:
+          ACTOR: ${{ github.actor }}
         run: |
+          # GCP labeling rules:
+          # Only hyphens (-), underscores (_), lowercase characters, and numbers are allowed. International characters are allowed.
+          # Convert github.actor to lowercase, replace disallowed characters
+          GCP_LABEL=$(echo "$ACTOR" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9_-]/_/g')
+          GCP_DEFAULT_TAGS="division=engineering,org=security,team=cloud-security-posture,project=test-environments,owner=$GCP_LABEL"
           . ./set_env.sh && ./deploy.sh && gcloud compute instances update "${DEPLOYMENT_NAME}" --update-labels "${GCP_DEFAULT_TAGS}" --zone="${GCP_ZONE}"
 
       - name: Install CSPM Azure integration
@@ -419,15 +425,29 @@ jobs:
           AZURE_TAGS: ${{ env.AZURE_DEFAULT_TAGS }}
         run: ./install_agent_az_cli.sh
 
+      - name: Check Asset Inventory supported version
+        id: asset-inventory-version-check
+        run: |
+          MIN_VERSION="8.16.0"
+          if [[ "$(echo -e "$MIN_VERSION\n$STACK_VERSION" | sort -V | head -n 1)" == "$MIN_VERSION" ]]; then
+            echo "Stack version meets the requirement: $STACK_VERSION >= $MIN_VERSION."
+            echo "asset_inventory_supported=true" >> $GITHUB_ENV
+          else
+            echo "Stack version is below the requirement: $STACK_VERSION < $MIN_VERSION."
+            echo "asset_inventory_supported=false" >> $GITHUB_ENV
+          fi
+
       - name: Install Azure Asset Inventory integration
         id: azure-asset-inventory-integration
         working-directory: ${{ env.INTEGRATIONS_SETUP_DIR }}
+        if: env.asset_inventory_supported == 'true'
         run: |
           poetry run python ./install_azure_asset_inventory_integration.py
 
       - name: Deploy Azure Asset Inventory agent
         id: azure-asset-inventory-agent
         working-directory: deploy/azure
+        if: env.asset_inventory_supported == 'true'
         env:
           AZURE_TAGS: ${{ env.AZURE_DEFAULT_TAGS }}
           DEPLOYMENT_NAME: "${{ env.DEPLOYMENT_NAME }}-inventory"
@@ -485,10 +505,12 @@ jobs:
       - name: Install AWS Asset Inventory integration
         id: aws-asset-inventory
         working-directory: ${{ env.INTEGRATIONS_SETUP_DIR }}
+        if: env.asset_inventory_supported == 'true'
         run: |
           poetry run python ./install_aws_asset_inventory_integration.py
 
       - name: Deploy AWS Asset Inventory agent
+        if: env.asset_inventory_supported == 'true'
         run: |
           scriptname="aws-asset-inventory-linux.sh"
           src="../../$INTEGRATIONS_SETUP_DIR/$scriptname"
@@ -499,13 +521,16 @@ jobs:
         if: always()
         env:
           S3_BUCKET: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
+          ASSET_INVENTORY_SUPPORTED: "${{ env.asset_inventory_supported }}"
         working-directory: ${{ env.INTEGRATIONS_SETUP_DIR }}
         run: |
           aws s3 cp "./kspm_unmanaged.yaml" "$S3_BUCKET/kspm_unmanaged.yaml"
           aws s3 cp "./kspm_d4c.yaml" "$S3_BUCKET/kspm_d4c.yaml"
           aws s3 cp "./kspm_eks.yaml" "$S3_BUCKET/kspm_eks.yaml"
           aws s3 cp "./cspm-linux.sh" "$S3_BUCKET/cspm-linux.sh"
-          aws s3 cp "./aws-asset-inventory-linux.sh" "$S3_BUCKET/aws-asset-inventory-linux.sh"
+          if [[ "${ASSET_INVENTORY_SUPPORTED}" == "true" ]]; then
+            aws s3 cp "./aws-asset-inventory-linux.sh" "$S3_BUCKET/aws-asset-inventory-linux.sh"
+          fi
           aws s3 cp "./state_data.json" "$S3_BUCKET/state_data.json"
 
       - name: Install Agentless integrations

.github/workflows/updatecli.yml

@@ -43,22 +43,23 @@ env:
 
 jobs:
   updatecli:
-    name: Update ${{ matrix.pipeline-name }} dependencies
+    name: "[${{ matrix.git-branch }}] Update ${{ matrix.pipeline-name }} dependencies"
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         pipeline-name: [ beats, golang, hermit, mods ]
+        git-branch: [ main, 8.x ]
     steps:
       - uses: actions/checkout@v4
       - name: Init Hermit
         run: ./bin/hermit env -r >> $GITHUB_ENV
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@92a13b95c2cd9f1c6742c965509203c6a5635ed7 # v0.76.1
+        uses: updatecli/updatecli-action@704a64517239e0993c5e3bf6749a063b8f950d9f # v0.76.1
       - name: Run Updatecli in Apply mode
         run: updatecli apply --config .ci/updatecli/updatecli.d/update-${{ matrix.pipeline-name }}.yml --values .ci/updatecli/values.yml
         env:
-          GIT_BRANCH: main
+          GIT_BRANCH: ${{ matrix.git-branch }}
           GIT_USER: "Cloud Security Machine"
           GITHUB_TOKEN: ${{ secrets.CLOUDSEC_MACHINE_TOKEN }}
 
@@ -84,7 +85,7 @@ jobs:
 
           echo "GIT_BRANCH=$branchName" >> $GITHUB_ENV
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@92a13b95c2cd9f1c6742c965509203c6a5635ed7 # v0.76.1
+        uses: updatecli/updatecli-action@704a64517239e0993c5e3bf6749a063b8f950d9f # v0.76.1
       - name: Run Updatecli in Apply mode
         run: updatecli apply --config .ci/updatecli/updatecli.d/update-${{ matrix.pipeline-name }}.yml --values .ci/updatecli/values.yml
         env: