diff --git a/.github/actions/cdr/action.yml b/.github/actions/cdr/action.yml new file mode 100644 index 0000000000..6817b1d0ac --- /dev/null +++ b/.github/actions/cdr/action.yml @@ -0,0 +1,299 @@ +name: 'CDR Integrations Installation' +description: 'Deploy CDR Integrations to Elastic Cloud' +inputs: + deployment-name: + description: | + Name with letters, numbers, hyphens; start with a letter. Max 20 chars. e.g., 'my-env-123' + required: true + type: string + aws-region: + description: "AWS region" + default: "eu-west-1" + required: false + type: string + azure-location: + description: "Azure location" + default: "East US" + required: false + type: string + gcp-project-id: + description: "GCP project ID" + default: "default" + required: false + type: string + gcp-service-account-json: + description: "GCP Service Account JSON" + default: "default" + required: false + type: string + deploy-az-vm: + description: "Deploy Azure VM resources" + default: true + required: false + type: boolean + deploy-gcp-vm: + description: "Deploy GCP VM resources" + default: true + required: false + type: boolean + deploy-aws-ec2: + description: "Deploy AWS EC2 resources" + default: true + required: false + type: boolean + deploy-aws-asset-inventory: + description: "Deploy AWS Asset Inventory EC2 resources" + default: true + required: false + type: boolean + aws-cloudtrail-s3-bucket: + description: "AWS Cloudtrail S3 bucket" + default: "default" + required: false + type: string + azure-eventhub-connection-string: + description: "Azure EventHub connection string" + default: "default" + required: false + type: string + azure-storage-account-key: + description: "Azure Storage Account key" + default: "default" + required: false + type: string + es-user: + description: "Elasticsearch user" + default: "elastic" + required: false + type: string + es-password: + description: "Elasticsearch password" + default: "changeme" + required: false + type: string + elk-stack-version: + description: "ELK Stack version" + default: "8.16.0" + required: false + type: string + kibana-url: + description: "Kibana URL" + default: "default" + required: false + type: string + azure-tags: + description: "Azure default tags" + default: "Key=division,Value=engineering" + required: false + type: string + tag-division: + description: "Optional division resource tag" + default: "engineering" + required: false + type: string + tag-org: + description: "Optional org resource tag" + default: "security" + required: false + type: string + tag-team: + description: "Optional team resource tag" + default: "cloud-security-posture" + required: false + type: string + tag-project: + description: "Optional project resource tag" + default: "test-environments" + required: false + type: string + tag-owner: + description: "Optional owner tag" + default: "cloudbeat" + required: false + type: string + +runs: + using: composite + steps: + - name: Deploy CDR Infrastructure + id: deploy-cdr-infra + env: + TF_VAR_deployment_name: ${{ inputs.deployment-name }} + TF_VAR_region: ${{ inputs.aws-region }} + TF_VAR_location: ${{ inputs.azure-location }} + TF_VAR_gcp_project_id: ${{ inputs.gcp-project-id }} + TF_VAR_gcp_service_account_json: ${{ inputs.gcp-service-account-json }} + TF_VAR_deploy_az_vm: ${{ inputs.deploy-az-vm }} + TF_VAR_deploy_gcp_vm: ${{ inputs.deploy-gcp-vm }} + TF_VAR_deploy_aws_ec2: ${{ inputs.deploy-aws-ec2 }} + TF_VAR_deploy_aws_asset_inventory: ${{ inputs.deploy-aws-asset-inventory }} + TF_VAR_division: ${{ inputs.tag-division }} + TF_VAR_org: ${{ inputs.tag-org }} + TF_VAR_team: ${{ inputs.tag-team }} + TF_VAR_project: ${{ inputs.tag-project }} + TF_VAR_owner: ${{ inputs.tag-owner }} + shell: bash + working-directory: "deploy/test-environments/cdr" + run: | + terraform init + terraform validate + terraform apply -auto-approve + + - name: Get CDR Outputs + id: generate-data + if: success() + shell: bash + working-directory: "deploy/test-environments/cdr" + run: | + ec2_cloudtrail_public_ip=$(terraform output -raw ec2_cloudtrail_public_ip) + echo "::add-mask::$ec2_cloudtrail_public_ip" + echo "CLOUDTRAIL_PUBLIC_IP=$ec2_cloudtrail_public_ip" >>"$GITHUB_ENV" + + ec2_cloudtrail_key=$(terraform output -raw ec2_cloudtrail_key) + echo "::add-mask::$ec2_cloudtrail_key" + echo "CLOUDTRAIL_KEY=$ec2_cloudtrail_key" >>"$GITHUB_ENV" + + az_vm_activity_logs_public_ip=$(terraform output -raw az_vm_activity_logs_public_ip) + echo "::add-mask::$az_vm_activity_logs_public_ip" + echo "ACTIVITY_LOGS_PUBLIC_IP=$az_vm_activity_logs_public_ip" >>"$GITHUB_ENV" + + az_vm_activity_logs_key=$(terraform output -raw az_vm_activity_logs_key) + echo "::add-mask::$az_vm_activity_logs_key" + echo "ACTIVITY_LOGS_KEY=$az_vm_activity_logs_key" >>"$GITHUB_ENV" + + gcp_audit_logs_public_ip=$(terraform output -raw gcp_audit_logs_public_ip) + echo "::add-mask::$gcp_audit_logs_public_ip" + echo "AUDIT_LOGS_PUBLIC_IP=$gcp_audit_logs_public_ip" >>"$GITHUB_ENV" + + gcp_audit_logs_key=$(terraform output -raw gcp_audit_logs_key) + echo "::add-mask::$gcp_audit_logs_key" + echo "AUDIT_LOGS_KEY=$gcp_audit_logs_key" >>"$GITHUB_ENV" + + ec2_asset_inv_key=$(terraform output -raw ec2_asset_inventory_key) + echo "::add-mask::$ec2_asset_inv_key" + echo "EC2_ASSET_INV_KEY=$ec2_asset_inv_key" >>"$GITHUB_ENV" + + asset_inv_public_ip=$(terraform output -raw ec2_asset_inventory_public_ip) + echo "::add-mask::$asset_inv_public_ip" + echo "ASSET_INV_PUBLIC_IP=$asset_inv_public_ip" >>"$GITHUB_ENV" + + - name: Install AWS Cloudtrail integration + id: cloudtrail-integration + if: steps.deploy-cdr-infra.outcome == 'success' + working-directory: tests/integrations_setup + env: + CLOUDTRAIL_S3: ${{ inputs.aws-cloudtrail-s3-bucket }} + ES_USER: ${{ inputs.es-user }} + ES_PASSWORD: ${{ inputs.es-password }} + KIBANA_URL: ${{ inputs.kibana-url }} + run: | + poetry run python ./install_cloudtrail_integration.py + + - name: Deploy AWS Cloudtrail agent + if: steps.deploy-cdr-infra.outcome == 'success' && steps.cloudtrail-integration.outcome == 'success' + working-directory: deploy/test-environments/cdr + run: | + scriptname="cloudtrail-linux.sh" + src="../../../tests/integrations_setup/$scriptname" + cmd="chmod +x $scriptname && ./$scriptname" + ../remote_setup.sh -k "$CLOUDTRAIL_KEY" -s "$src" -h "$CLOUDTRAIL_PUBLIC_IP" -d "~/$scriptname" -c "$cmd" + + - name: Install Azure Activity Logs integration + id: az-activity-logs-integration + if: steps.deploy-cdr-infra.outcome == 'success' + working-directory: tests/integrations_setup + env: + EVENTHUB: "activity-logs" + CONNECTION_STRING: ${{ inputs.azure-eventhub-connection-string }} + STORAGE_ACCOUNT: "testenvsactivitylogs" + STORAGE_ACCOUNT_KEY: ${{ inputs.azure-storage-account-key }} + ES_USER: ${{ inputs.es-user }} + ES_PASSWORD: ${{ inputs.es-password }} + KIBANA_URL: ${{ inputs.kibana-url }} + run: | + poetry run python ./install_az_activity_logs_integration.py + + - name: Deploy Azure Activity Logs agent + if: steps.deploy-cdr-infra.outcome == 'success' && steps.az-activity-logs-integration.outcome == 'success' + working-directory: deploy/test-environments/cdr + run: | + scriptname="az_activity_logs.sh" + src="../../../tests/integrations_setup/$scriptname" + cmd="chmod +x $scriptname && ./$scriptname" + ../remote_setup.sh -k "$ACTIVITY_LOGS_KEY" -s "$src" -h "$ACTIVITY_LOGS_PUBLIC_IP" -d "~/$scriptname" -c "$cmd" + + - name: Install GCP Audit Logs integration + id: gcp-audit-logs-integration + if: steps.deploy-cdr-infra.outcome == 'success' + working-directory: tests/integrations_setup + env: + GCP_TOPIC_NAME: "test-envs-topic" + GCP_SUBSCRIPTION_NAME: "test-envs-topic-sub-id" + ES_USER: ${{ inputs.es-user }} + ES_PASSWORD: ${{ inputs.es-password }} + KIBANA_URL: ${{ inputs.kibana-url }} + run: | + poetry run python ./install_gcp_audit_logs_integration.py + + - name: Deploy GCP Audit Logs agent + if: steps.deploy-cdr-infra.outcome == 'success' && steps.gcp-audit-logs-integration.outcome == 'success' + working-directory: deploy/test-environments/cdr + run: | + scriptname="gcp_audit_logs.sh" + src="../../../tests/integrations_setup/$scriptname" + cmd="chmod +x $scriptname && ./$scriptname" + ../remote_setup.sh -k "$AUDIT_LOGS_KEY" -s "$src" -h "$AUDIT_LOGS_PUBLIC_IP" -d "~/$scriptname" -c "$cmd" + + - name: Check Asset Inventory supported version + id: asset-inventory-version-check + env: + STACK_VERSION: ${{ inputs.elk-stack-version }} + run: | + MIN_VERSION="8.16.0" + if [[ "$(echo -e "$MIN_VERSION\n$STACK_VERSION" | sort -V | head -n 1)" == "$MIN_VERSION" ]]; then + echo "Stack version meets the requirement: $STACK_VERSION >= $MIN_VERSION." + echo "asset_inventory_supported=true" >> $GITHUB_OUTPUT + else + echo "Stack version is below the requirement: $STACK_VERSION < $MIN_VERSION." + echo "asset_inventory_supported=false" >> $GITHUB_OUTPUT + fi + + - name: Install Azure Asset Inventory integration + id: azure-asset-inventory-integration + working-directory: tests/integrations_setup + if: steps.asset-inventory-version-check.outputs.asset_inventory_supported == 'true' + env: + ES_USER: ${{ inputs.es-user }} + ES_PASSWORD: ${{ inputs.es-password }} + KIBANA_URL: ${{ inputs.kibana-url }} + run: | + poetry run python ./install_azure_asset_inventory_integration.py + + - name: Deploy Azure Asset Inventory agent + id: azure-asset-inventory-agent + working-directory: deploy/azure + if: steps.asset-inventory-version-check.outputs.asset_inventory_supported == 'true' + env: + AZURE_TAGS: ${{ inputs.azure-tags }} + DEPLOYMENT_NAME: "${{ inputs.deployment-name }}-inventory" + run: ./install_agent_az_cli.sh + + - name: Install AWS Asset Inventory integration + id: aws-asset-inventory + if: steps.asset-inventory-version-check.outputs.asset_inventory_supported == 'true' + working-directory: tests/integrations_setup + env: + ES_USER: ${{ inputs.es-user }} + ES_PASSWORD: ${{ inputs.es-password }} + KIBANA_URL: ${{ inputs.kibana-url }} + run: | + poetry run python ./install_aws_asset_inventory_integration.py + + - name: Deploy AWS Asset Inventory agent + if: steps.asset-inventory-version-check.outputs.asset_inventory_supported == 'true' + working-directory: deploy/test-environments/cdr + run: | + scriptname="aws-asset-inventory-linux.sh" + src="../../../tests/integrations_setup/$scriptname" + cmd="chmod +x $scriptname && ./$scriptname" + ../remote_setup.sh -k "$EC2_ASSET_INV_KEY" -s "$src" -h "$ASSET_INV_PUBLIC_IP" -d "~/$scriptname" -c "$cmd" diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 4c77a2cb3f..029f7824e6 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -267,7 +267,8 @@ jobs: run: | echo "TF_VAR_gcp_project_id=$GCP_PROJECT" >> $GITHUB_ENV echo "TF_STATE_FOLDER=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_ENV - + # TODO: REMOVE THIS SECTION + #======================================================== # - name: Provision Infrastructure # id: provision-terraform # if: success() @@ -278,6 +279,7 @@ jobs: # TF_VAR_owner: ${{ github.actor }} # run: | # ./manage_infrastructure.sh "$INFRA_TYPE" "apply" + #======================================================== - name: Deploy ELK Cloud Stack id: elk-stack @@ -292,10 +294,13 @@ jobs: tag-project: ${{ github.actor }} tag-owner: ${{ github.actor }} + # TODO: REMOVE THIS SECTION + #======================================================== # - name: Set Environment Output # id: env-output # run: | # ./manage_infrastructure.sh "$INFRA_TYPE" "output" + #======================================================== - name: Upload tf state id: upload-state @@ -327,6 +332,40 @@ jobs: echo "$summary" >> $GITHUB_STEP_SUMMARY echo "$summary" # Print the summary to the workflow log + - name: Deploy CDR Integrations + id: cdr-integrations + if: env.INFRA_TYPE != 'cis' + uses: ./.github/actions/cdr + with: + deployment-name: ${{ env.DEPLOYMENT_NAME }} + aws-region: ${{ env.AWS_REGION }} + azure-location: "East US" + gcp-project-id: ${{ env.GCP_PROJECT }} + gcp-service-account-json: ${{ secrets.GCP_AGENT_CREDENTIALS }} + deploy-az-vm: true + deploy-gcp-vm: true + deploy-aws-ec2: true + deploy-aws-asset-inventory: true + aws-cloudtrail-s3-bucket: ${{ secrets.CLOUDTRAIL_S3 }} + azure-eventhub-connection-string: ${{ secrets.AZURE_EVENTHUB_CONNECTION_STRING }} + azure-storage-account-key: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }} + es-user: ${{ steps.elk-stack.outputs.es-user }} + es-password: ${{ steps.elk-stack.outputs.es-password }} + kibana-url: ${{ steps.elk-stack.outputs.kibana-url }} + elk-stack-version: ${{ env.STACK_VERSION }} + azure-tags: ${{ env.AZURE_DEFAULT_TAGS }} + tag-project: ${{ github.actor }} + tag-owner: ${{ github.actor }} + + - name: Upload tf state + id: env.INFRA_TYPE != 'cis' + if: always() + env: + S3_BUCKET: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}" + run: | + ./manage_infrastructure.sh "cdr" "upload-state" + + # TODO: REMOVE THIS SECTION # - name: Install AWS Cloudtrail integration # id: cloudtrail-integration # if: env.INFRA_TYPE != 'cis' @@ -384,6 +423,7 @@ jobs: # src="../../../$INTEGRATIONS_SETUP_DIR/$scriptname" # cmd="chmod +x $scriptname && ./$scriptname" # ../remote_setup.sh -k "$AUDIT_LOGS_KEY" -s "$src" -h "$AUDIT_LOGS_PUBLIC_IP" -d "~/$scriptname" -c "$cmd" + #======================================================== # - name: Install CNVM integration # id: cnvm @@ -520,6 +560,8 @@ jobs: # cmd="chmod +x $scriptname && ./$scriptname" # ../remote_setup.sh -k "$EC2_CSPM_KEY" -s "$src" -h "$CSPM_PUBLIC_IP" -d "~/$scriptname" -c "$cmd" + # TODO: REMOVE THIS SECTION + #======================================================== # - name: Install AWS Asset Inventory integration # id: aws-asset-inventory # if: env.INFRA_TYPE != 'cis' @@ -536,6 +578,7 @@ jobs: # src="../../../$INTEGRATIONS_SETUP_DIR/$scriptname" # cmd="chmod +x $scriptname && ./$scriptname" # ../remote_setup.sh -k "$EC2_ASSET_INV_KEY" -s "$src" -h "$ASSET_INV_PUBLIC_IP" -d "~/$scriptname" -c "$cmd" + #======================================================== # - name: Upload Integrations data # if: always()