add cdr action

.github/actions/cdr/action.yml

@@ -0,0 +1,299 @@
+name: 'CDR Integrations Installation'
+description: 'Deploy CDR Integrations to Elastic Cloud'
+inputs:
+  deployment-name:
+    description: |
+      Name with letters, numbers, hyphens; start with a letter. Max 20 chars. e.g., 'my-env-123'
+    required: true
+    type: string
+  aws-region:
+    description: "AWS region"
+    default: "eu-west-1"
+    required: false
+    type: string
+  azure-location:
+    description: "Azure location"
+    default: "East US"
+    required: false
+    type: string
+  gcp-project-id:
+    description: "GCP project ID"
+    default: "default"
+    required: false
+    type: string
+  gcp-service-account-json:
+    description: "GCP Service Account JSON"
+    default: "default"
+    required: false
+    type: string
+  deploy-az-vm:
+    description: "Deploy Azure VM resources"
+    default: true
+    required: false
+    type: boolean
+  deploy-gcp-vm:
+    description: "Deploy GCP VM resources"
+    default: true
+    required: false
+    type: boolean
+  deploy-aws-ec2:
+    description: "Deploy AWS EC2 resources"
+    default: true
+    required: false
+    type: boolean
+  deploy-aws-asset-inventory:
+    description: "Deploy AWS Asset Inventory EC2 resources"
+    default: true
+    required: false
+    type: boolean
+  aws-cloudtrail-s3-bucket:
+    description: "AWS Cloudtrail S3 bucket"
+    default: "default"
+    required: false
+    type: string
+  azure-eventhub-connection-string:
+    description: "Azure EventHub connection string"
+    default: "default"
+    required: false
+    type: string
+  azure-storage-account-key:
+    description: "Azure Storage Account key"
+    default: "default"
+    required: false
+    type: string
+  es-user:
+    description: "Elasticsearch user"
+    default: "elastic"
+    required: false
+    type: string
+  es-password:
+    description: "Elasticsearch password"
+    default: "changeme"
+    required: false
+    type: string
+  elk-stack-version:
+    description: "ELK Stack version"
+    default: "8.16.0"
+    required: false
+    type: string
+  kibana-url:
+    description: "Kibana URL"
+    default: "default"
+    required: false
+    type: string
+  azure-tags:
+    description: "Azure default tags"
+    default: "Key=division,Value=engineering"
+    required: false
+    type: string
+  tag-division:
+    description: "Optional division resource tag"
+    default: "engineering"
+    required: false
+    type: string
+  tag-org:
+    description: "Optional org resource tag"
+    default: "security"
+    required: false
+    type: string
+  tag-team:
+    description: "Optional team resource tag"
+    default: "cloud-security-posture"
+    required: false
+    type: string
+  tag-project:
+    description: "Optional project resource tag"
+    default: "test-environments"
+    required: false
+    type: string
+  tag-owner:
+    description: "Optional owner tag"
+    default: "cloudbeat"
+    required: false
+    type: string
+
+runs:
+  using: composite
+  steps:
+    - name: Deploy CDR Infrastructure
+      id: deploy-cdr-infra
+      env:
+        TF_VAR_deployment_name: ${{ inputs.deployment-name }}
+        TF_VAR_region: ${{ inputs.aws-region }}
+        TF_VAR_location: ${{ inputs.azure-location }}
+        TF_VAR_gcp_project_id: ${{ inputs.gcp-project-id }}
+        TF_VAR_gcp_service_account_json: ${{ inputs.gcp-service-account-json }}
+        TF_VAR_deploy_az_vm: ${{ inputs.deploy-az-vm }}
+        TF_VAR_deploy_gcp_vm: ${{ inputs.deploy-gcp-vm }}
+        TF_VAR_deploy_aws_ec2: ${{ inputs.deploy-aws-ec2 }}
+        TF_VAR_deploy_aws_asset_inventory: ${{ inputs.deploy-aws-asset-inventory }}
+        TF_VAR_division: ${{ inputs.tag-division }}
+        TF_VAR_org: ${{ inputs.tag-org }}
+        TF_VAR_team: ${{ inputs.tag-team }}
+        TF_VAR_project: ${{ inputs.tag-project }}
+        TF_VAR_owner: ${{ inputs.tag-owner }}
+      shell: bash
+      working-directory: "deploy/test-environments/cdr"
+      run: |
+        terraform init
+        terraform validate
+        terraform apply -auto-approve
+
+    - name: Get CDR Outputs
+      id: generate-data
+      if: success()
+      shell: bash
+      working-directory: "deploy/test-environments/cdr"
+      run: |
+        ec2_cloudtrail_public_ip=$(terraform output -raw ec2_cloudtrail_public_ip)
+        echo "::add-mask::$ec2_cloudtrail_public_ip"
+        echo "CLOUDTRAIL_PUBLIC_IP=$ec2_cloudtrail_public_ip" >>"$GITHUB_ENV"
+
+        ec2_cloudtrail_key=$(terraform output -raw ec2_cloudtrail_key)
+        echo "::add-mask::$ec2_cloudtrail_key"
+        echo "CLOUDTRAIL_KEY=$ec2_cloudtrail_key" >>"$GITHUB_ENV"
+
+        az_vm_activity_logs_public_ip=$(terraform output -raw az_vm_activity_logs_public_ip)
+        echo "::add-mask::$az_vm_activity_logs_public_ip"
+        echo "ACTIVITY_LOGS_PUBLIC_IP=$az_vm_activity_logs_public_ip" >>"$GITHUB_ENV"
+
+        az_vm_activity_logs_key=$(terraform output -raw az_vm_activity_logs_key)
+        echo "::add-mask::$az_vm_activity_logs_key"
+        echo "ACTIVITY_LOGS_KEY=$az_vm_activity_logs_key" >>"$GITHUB_ENV"
+
+        gcp_audit_logs_public_ip=$(terraform output -raw gcp_audit_logs_public_ip)
+        echo "::add-mask::$gcp_audit_logs_public_ip"
+        echo "AUDIT_LOGS_PUBLIC_IP=$gcp_audit_logs_public_ip" >>"$GITHUB_ENV"
+
+        gcp_audit_logs_key=$(terraform output -raw gcp_audit_logs_key)
+        echo "::add-mask::$gcp_audit_logs_key"
+        echo "AUDIT_LOGS_KEY=$gcp_audit_logs_key" >>"$GITHUB_ENV"
+
+        ec2_asset_inv_key=$(terraform output -raw ec2_asset_inventory_key)
+        echo "::add-mask::$ec2_asset_inv_key"
+        echo "EC2_ASSET_INV_KEY=$ec2_asset_inv_key" >>"$GITHUB_ENV"
+
+        asset_inv_public_ip=$(terraform output -raw ec2_asset_inventory_public_ip)
+        echo "::add-mask::$asset_inv_public_ip"
+        echo "ASSET_INV_PUBLIC_IP=$asset_inv_public_ip" >>"$GITHUB_ENV"
+
+    - name: Install AWS Cloudtrail integration
+      id: cloudtrail-integration
+      if: steps.deploy-cdr-infra.outcome  == 'success'
+      working-directory: tests/integrations_setup
+      env:
+        CLOUDTRAIL_S3: ${{ inputs.aws-cloudtrail-s3-bucket }}
+        ES_USER: ${{ inputs.es-user }}
+        ES_PASSWORD: ${{ inputs.es-password }}
+        KIBANA_URL: ${{ inputs.kibana-url }}
+      run: |
+        poetry run python ./install_cloudtrail_integration.py
+
+    - name: Deploy AWS Cloudtrail agent
+      if: steps.deploy-cdr-infra.outcome  == 'success' && steps.cloudtrail-integration.outcome == 'success'
+      working-directory: deploy/test-environments/cdr
+      run: |
+        scriptname="cloudtrail-linux.sh"
+        src="../../../tests/integrations_setup/$scriptname"
+        cmd="chmod +x $scriptname && ./$scriptname"
+        ../remote_setup.sh -k "$CLOUDTRAIL_KEY" -s "$src" -h "$CLOUDTRAIL_PUBLIC_IP" -d "~/$scriptname" -c "$cmd"
+
+    - name: Install Azure Activity Logs integration
+      id: az-activity-logs-integration
+      if: steps.deploy-cdr-infra.outcome  == 'success'
+      working-directory: tests/integrations_setup
+      env:
+        EVENTHUB: "activity-logs"
+        CONNECTION_STRING: ${{ inputs.azure-eventhub-connection-string }}
+        STORAGE_ACCOUNT: "testenvsactivitylogs"
+        STORAGE_ACCOUNT_KEY: ${{ inputs.azure-storage-account-key }}
+        ES_USER: ${{ inputs.es-user }}
+        ES_PASSWORD: ${{ inputs.es-password }}
+        KIBANA_URL: ${{ inputs.kibana-url }}
+      run: |
+        poetry run python ./install_az_activity_logs_integration.py
+
+    - name: Deploy Azure Activity Logs agent
+      if: steps.deploy-cdr-infra.outcome  == 'success' && steps.az-activity-logs-integration.outcome == 'success'
+      working-directory: deploy/test-environments/cdr
+      run: |
+        scriptname="az_activity_logs.sh"
+        src="../../../tests/integrations_setup/$scriptname"
+        cmd="chmod +x $scriptname && ./$scriptname"
+        ../remote_setup.sh -k "$ACTIVITY_LOGS_KEY" -s "$src" -h "$ACTIVITY_LOGS_PUBLIC_IP" -d "~/$scriptname" -c "$cmd"
+
+    - name: Install GCP Audit Logs integration
+      id: gcp-audit-logs-integration
+      if: steps.deploy-cdr-infra.outcome  == 'success'
+      working-directory: tests/integrations_setup
+      env:
+        GCP_TOPIC_NAME: "test-envs-topic"
+        GCP_SUBSCRIPTION_NAME: "test-envs-topic-sub-id"
+        ES_USER: ${{ inputs.es-user }}
+        ES_PASSWORD: ${{ inputs.es-password }}
+        KIBANA_URL: ${{ inputs.kibana-url }}
+      run: |
+        poetry run python ./install_gcp_audit_logs_integration.py
+
+    - name: Deploy GCP Audit Logs agent
+      if: steps.deploy-cdr-infra.outcome  == 'success' && steps.gcp-audit-logs-integration.outcome == 'success'
+      working-directory: deploy/test-environments/cdr
+      run: |
+        scriptname="gcp_audit_logs.sh"
+        src="../../../tests/integrations_setup/$scriptname"
+        cmd="chmod +x $scriptname && ./$scriptname"
+        ../remote_setup.sh -k "$AUDIT_LOGS_KEY" -s "$src" -h "$AUDIT_LOGS_PUBLIC_IP" -d "~/$scriptname" -c "$cmd"
+
+    - name: Check Asset Inventory supported version
+      id: asset-inventory-version-check
+      env:
+        STACK_VERSION: ${{ inputs.elk-stack-version }}
+      run: |
+        MIN_VERSION="8.16.0"
+        if [[ "$(echo -e "$MIN_VERSION\n$STACK_VERSION" | sort -V | head -n 1)" == "$MIN_VERSION" ]]; then
+          echo "Stack version meets the requirement: $STACK_VERSION >= $MIN_VERSION."
+          echo "asset_inventory_supported=true" >> $GITHUB_OUTPUT
+        else
+          echo "Stack version is below the requirement: $STACK_VERSION < $MIN_VERSION."
+          echo "asset_inventory_supported=false" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Install Azure Asset Inventory integration
+      id: azure-asset-inventory-integration
+      working-directory: tests/integrations_setup
+      if: steps.asset-inventory-version-check.outputs.asset_inventory_supported == 'true'
+      env:
+        ES_USER: ${{ inputs.es-user }}
+        ES_PASSWORD: ${{ inputs.es-password }}
+        KIBANA_URL: ${{ inputs.kibana-url }}
+      run: |
+        poetry run python ./install_azure_asset_inventory_integration.py
+
+    - name: Deploy Azure Asset Inventory agent
+      id: azure-asset-inventory-agent
+      working-directory: deploy/azure
+      if: steps.asset-inventory-version-check.outputs.asset_inventory_supported == 'true'
+      env:
+        AZURE_TAGS: ${{ inputs.azure-tags }}
+        DEPLOYMENT_NAME: "${{ inputs.deployment-name }}-inventory"
+      run: ./install_agent_az_cli.sh
+
+    - name: Install AWS Asset Inventory integration
+      id: aws-asset-inventory
+      if: steps.asset-inventory-version-check.outputs.asset_inventory_supported == 'true'
+      working-directory: tests/integrations_setup
+      env:
+        ES_USER: ${{ inputs.es-user }}
+        ES_PASSWORD: ${{ inputs.es-password }}
+        KIBANA_URL: ${{ inputs.kibana-url }}
+      run: |
+        poetry run python ./install_aws_asset_inventory_integration.py
+
+    - name: Deploy AWS Asset Inventory agent
+      if: steps.asset-inventory-version-check.outputs.asset_inventory_supported == 'true'
+      working-directory: deploy/test-environments/cdr
+      run: |
+        scriptname="aws-asset-inventory-linux.sh"
+        src="../../../tests/integrations_setup/$scriptname"
+        cmd="chmod +x $scriptname && ./$scriptname"
+        ../remote_setup.sh -k "$EC2_ASSET_INV_KEY" -s "$src" -h "$ASSET_INV_PUBLIC_IP" -d "~/$scriptname" -c "$cmd"

.github/workflows/test-environment.yml

@@ -267,7 +267,8 @@ jobs:
         run: |
           echo "TF_VAR_gcp_project_id=$GCP_PROJECT" >> $GITHUB_ENV
           echo "TF_STATE_FOLDER=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_ENV
-
+      # TODO: REMOVE THIS SECTION
+      #========================================================
       # - name: Provision Infrastructure
       #   id: provision-terraform
       #   if: success()
@@ -278,6 +279,7 @@ jobs:
       #     TF_VAR_owner: ${{ github.actor }}
       #   run: |
       #     ./manage_infrastructure.sh "$INFRA_TYPE" "apply"
+      #========================================================
 
       - name: Deploy ELK Cloud Stack
         id: elk-stack
@@ -292,10 +294,13 @@ jobs:
           tag-project: ${{ github.actor }}
           tag-owner: ${{ github.actor }}   
 
+      # TODO: REMOVE THIS SECTION
+      #========================================================
       # - name: Set Environment Output
       #   id: env-output
       #   run: |
       #     ./manage_infrastructure.sh "$INFRA_TYPE" "output"
+      #========================================================
 
       - name: Upload tf state
         id: upload-state
@@ -327,6 +332,40 @@ jobs:
           echo "$summary" >> $GITHUB_STEP_SUMMARY
           echo "$summary" # Print the summary to the workflow log
 
+      - name: Deploy CDR Integrations
+        id: cdr-integrations
+        if: env.INFRA_TYPE != 'cis'
+        uses: ./.github/actions/cdr
+        with:
+          deployment-name: ${{ env.DEPLOYMENT_NAME }}
+          aws-region: ${{ env.AWS_REGION }}
+          azure-location: "East US"
+          gcp-project-id: ${{ env.GCP_PROJECT }}
+          gcp-service-account-json: ${{ secrets.GCP_AGENT_CREDENTIALS }}
+          deploy-az-vm: true
+          deploy-gcp-vm: true
+          deploy-aws-ec2: true
+          deploy-aws-asset-inventory: true
+          aws-cloudtrail-s3-bucket: ${{ secrets.CLOUDTRAIL_S3 }}
+          azure-eventhub-connection-string: ${{ secrets.AZURE_EVENTHUB_CONNECTION_STRING }}
+          azure-storage-account-key: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }}
+          es-user: ${{ steps.elk-stack.outputs.es-user }}
+          es-password: ${{ steps.elk-stack.outputs.es-password }}
+          kibana-url: ${{ steps.elk-stack.outputs.kibana-url }}
+          elk-stack-version: ${{ env.STACK_VERSION }}
+          azure-tags: ${{ env.AZURE_DEFAULT_TAGS }}
+          tag-project: ${{ github.actor }}
+          tag-owner: ${{ github.actor }}
+
+      - name: Upload tf state
+        id: env.INFRA_TYPE != 'cis'
+        if: always()
+        env:
+          S3_BUCKET: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
+        run: |
+          ./manage_infrastructure.sh "cdr" "upload-state"
+
+      # TODO: REMOVE THIS SECTION    
       # - name: Install AWS Cloudtrail integration
       #   id: cloudtrail-integration
       #   if: env.INFRA_TYPE != 'cis'
@@ -384,6 +423,7 @@ jobs:
       #     src="../../../$INTEGRATIONS_SETUP_DIR/$scriptname"
       #     cmd="chmod +x $scriptname && ./$scriptname"
       #     ../remote_setup.sh -k "$AUDIT_LOGS_KEY" -s "$src" -h "$AUDIT_LOGS_PUBLIC_IP" -d "~/$scriptname" -c "$cmd"
+      #========================================================
 
       # - name: Install CNVM integration
       #   id: cnvm
@@ -520,6 +560,8 @@ jobs:
       #     cmd="chmod +x $scriptname && ./$scriptname"
       #     ../remote_setup.sh -k "$EC2_CSPM_KEY" -s "$src" -h "$CSPM_PUBLIC_IP" -d "~/$scriptname" -c "$cmd"
 
+      # TODO: REMOVE THIS SECTION
+      #========================================================
       # - name: Install AWS Asset Inventory integration
       #   id: aws-asset-inventory
       #   if: env.INFRA_TYPE != 'cis'
@@ -536,6 +578,7 @@ jobs:
       #     src="../../../$INTEGRATIONS_SETUP_DIR/$scriptname"
       #     cmd="chmod +x $scriptname && ./$scriptname"
       #     ../remote_setup.sh -k "$EC2_ASSET_INV_KEY" -s "$src" -h "$ASSET_INV_PUBLIC_IP" -d "~/$scriptname" -c "$cmd"
+      #========================================================
 
       # - name: Upload Integrations data
       #   if: always()