From cfa85d02364a8937cf6bc80296167d1df8bd6f9c Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 12:09:24 -0700 Subject: [PATCH 01/77] updating only CI files --- .github/workflows/detection-testing.yml | 347 ------------------ .github/workflows/unit-testing.yml | 101 +++++ .../7zip_commandline_to_smb_share_path.yml | 2 +- 3 files changed, 102 insertions(+), 348 deletions(-) delete mode 100644 .github/workflows/detection-testing.yml create mode 100644 .github/workflows/unit-testing.yml diff --git a/.github/workflows/detection-testing.yml b/.github/workflows/detection-testing.yml deleted file mode 100644 index df6254ce12..0000000000 --- a/.github/workflows/detection-testing.yml +++ /dev/null @@ -1,347 +0,0 @@ -# name: detection-testing -# on: -# push: -# pull_request: -# types: [opened, reopened] -# schedule: -# - cron: "44 4 * * *" -# jobs: - -# validate-tag-if-present: -# runs-on: ubuntu-latest - -# steps: -# - name: TAGGED, Validate that the tag is in the correct format - -# run: | -# echo "The GITHUB_REF: $GITHUB_REF" -# #First check to see if the release is a tag -# if [[ $GITHUB_REF =~ refs/tags/* ]]; then -# #Yes, this is a tag, so we need to test to make sure that the tag -# #is in the correct format (like v1.10.20) -# if [[ $GITHUB_REF =~ refs/tags/v[0-9]+.[0-9]+.[0-9]+ ]]; then -# echo "PASS: Tagged release with good format" -# exit 0 -# else -# echo "FAIL: Tagged release with bad format" -# exit 1 -# fi -# else -# echo "PASS: Not a tagged release" -# exit 0 -# fi - -# quit-for-dependabot: -# runs-on: ubuntu-latest -# if: github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]' -# steps: -# - name: "Placeholder" -# run: | -# echo "yes it ran" - -# docker-detection-testing-setup: -# runs-on: ubuntu-latest -# if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job -# needs: [validate-tag-if-present, quit-for-dependabot] -# steps: -# - name: Get branch and PR required for detection testing main.py -# id: vars -# run: | -# echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" - -# - name: Checkout Repo -# uses: actions/checkout@v2 -# #with: -# # ref: develop - - - -# - uses: actions/setup-python@v2 -# with: -# python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy -# architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified -# cache: 'pip' - -# - name: Install Python Dependencies -# run: | -# python -m venv .venv -# source .venv/bin/activate -# python -m pip install wheel -# python -m pip install -r requirements.txt - -# - name: Run the CI -# run: | -# source .venv/bin/activate -# cd bin/docker_detection_tester -# echo "github.event.issue.pull_request : [${{ github.event.issue.pull_request }}]" -# echo "github.event.pull_request.number : [${{ github.event.pull_request.number }}]" -# echo "steps.vars.outputs.branch : [${{ steps.vars.outputs.branch }}]" -# echo "github.event.pull_request.head.ref : [${{ github.event.pull_request.head.ref }}]" -# echo "github.event_name : [${{ github.event_name }}]" - - -# if [[ ${{ github.event_name }} == schedule ]]; then -# # Note that scheduled actions ONLY run on the default branch, so it won't run on all other branches! -# echo "Running a nightly test on all detections OR a commit was made directly to develop" -# python detection_testing_execution.py run --branch develop --mode all --mock --config_file test_config_github_actions.json -# elif [[ ! -z "${{ github.event.pull_request.head.ref }}" && ! -z "${{ github.event.pull_request.number }}" ]]; then -# echo "Pull request from source branch [${{ github.event.pull_request.head.ref }}] for PR number [${{ github.event.issue.number }}]" -# python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }} --mode changes --mock --config_file test_config_github_actions.json -# else -# echo "Push from branch [${{ steps.vars.outputs.branch }}]" -# python detection_testing_execution.py run --branch ${{ steps.vars.outputs.branch }} --mode changes --mock --config_file test_config_github_actions.json -# fi - -# mv *-test-run.json replicate_test.json -# - name: Upload Test Results Files -# uses: actions/upload-artifact@v2 -# with: -# name: testing-results-config -# path: | -# bin/docker_detection_tester/prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz -# bin/docker_detection_tester/prior_config/config_tests_0.json -# bin/docker_detection_tester/prior_config/config_tests_1.json -# bin/docker_detection_tester/prior_config/config_tests_2.json -# bin/docker_detection_tester/prior_config/config_tests_3.json -# bin/docker_detection_tester/prior_config/config_tests_4.json -# bin/docker_detection_tester/prior_config/config_tests_5.json -# bin/docker_detection_tester/prior_config/config_tests_6.json -# bin/docker_detection_tester/prior_config/config_tests_7.json -# bin/docker_detection_tester/prior_config/config_tests_8.json -# bin/docker_detection_tester/prior_config/config_tests_9.json - -# - name: Upload File to Enable Replication of the Test at a Different Time or Place -# uses: actions/upload-artifact@v2 -# with: -# name: replicate_test -# path: | -# bin/docker_detection_tester/replicate_test.json - -# docker-detection-testing-execution: -# runs-on: ubuntu-latest -# if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job -# needs: [docker-detection-testing-setup] -# strategy: -# matrix: -# manifest_filename: ["config_tests_0.json", -# "config_tests_1.json", -# "config_tests_2.json", -# "config_tests_3.json", -# "config_tests_4.json", -# "config_tests_5.json", -# "config_tests_6.json", -# "config_tests_7.json", -# "config_tests_8.json", -# "config_tests_9.json"] -# steps: -# - name: Get branch and PR required for detection testing main.py -# id: vars -# run: | -# echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" - -# - name: Checkout Repo -# uses: actions/checkout@v2 -# #with: -# # ref: develop - -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: testing-results-config -# path: bin/docker_detection_tester/prior_config - - -# - uses: actions/setup-python@v2 -# with: -# python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy -# architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified -# cache: 'pip' - -# - name: Install Python Dependencies -# run: | -# python -m venv .venv -# source .venv/bin/activate -# python -m pip install wheel -# python -m pip install -r requirements.txt - -# - name: Run the CI -# run: | -# source .venv/bin/activate -# cd bin/docker_detection_tester - - -# python detection_testing_execution.py run -c prior_config/${{ matrix.manifest_filename}} - - -# - name: Upload Test Results Files -# uses: actions/upload-artifact@v2 -# with: -# name: ${{ matrix.manifest_filename}}.results -# path: | -# bin/docker_detection_tester/test_results/success.csv -# bin/docker_detection_tester/test_results/error.csv -# bin/docker_detection_tester/test_results/failure.csv -# bin/docker_detection_tester/test_results/combined.csv -# bin/docker_detection_tester/test_results/success.json -# bin/docker_detection_tester/test_results/error.json -# bin/docker_detection_tester/test_results/failure.json -# bin/docker_detection_tester/test_results/combined.json - -# bin/docker_detection_tester/test_results/summary.json - -# docker-detection-testing-execution-merge-results: -# runs-on: ubuntu-latest -# if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job -# needs: [docker-detection-testing-setup, docker-detection-testing-execution] - -# steps: -# - name: Get branch and PR required for detection testing main.py -# id: vars -# run: | -# echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" - -# - name: Checkout Repo -# uses: actions/checkout@v2 -# #with: -# # ref: develop - -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_0.json.results -# path: bin/docker_detection_tester/results_0 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_1.json.results -# path: bin/docker_detection_tester/results_1 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_2.json.results -# path: bin/docker_detection_tester/results_2 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_3.json.results -# path: bin/docker_detection_tester/results_3 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_4.json.results -# path: bin/docker_detection_tester/results_4 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_5.json.results -# path: bin/docker_detection_tester/results_5 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_6.json.results -# path: bin/docker_detection_tester/results_6 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_7.json.results -# path: bin/docker_detection_tester/results_7 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_8.json.results -# path: bin/docker_detection_tester/results_8 -# - name: Download artifacts -# uses: actions/download-artifact@v2 -# with: -# name: config_tests_9.json.results -# path: bin/docker_detection_tester/results_9 - -# - uses: actions/setup-python@v2 -# with: -# python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy -# architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified -# cache: 'pip' - -# - name: Install Python Dependencies -# run: | -# python -m venv .venv -# source .venv/bin/activate -# python -m pip install wheel -# python -m pip install -r requirements.txt - -# - name: Merge Detections into single File -# run: | -# source .venv/bin/activate -# cd bin/docker_detection_tester -# python summarize_json.py --files results_*/summary.json --output_filename summary_test_results.json - - -# - name: Upload Summary Test Results JSON -# uses: actions/upload-artifact@v2 -# if: always() -# with: -# name: SummaryTestResults -# path: | -# bin/docker_detection_tester/summary_test_results.json - -# - name: Upload Failures Manifest on Failure -# uses: actions/upload-artifact@v2 -# if: failure() -# with: -# name: DetectionFailureManifest -# path: | -# bin/docker_detection_tester/detection_failure_manifest.json - - -# #Always clean these up, they make the output messy -# - name: Clean up intermediate Files -# uses: geekyeggo/delete-artifact@v1 -# if: always() -# with: -# name: | -# config_tests_0.json.results -# config_tests_1.json.results -# config_tests_2.json.results -# config_tests_3.json.results -# config_tests_4.json.results -# config_tests_5.json.results -# config_tests_6.json.results -# config_tests_7.json.results -# config_tests_8.json.results -# config_tests_9.json.results - -# - name: Log in to S3 for Artifact Uploads -# if: ${{ github.event_name == 'schedule' }} -# uses: aws-actions/configure-aws-credentials@v1 -# with: -# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - -# aws-region: us-west-2 - -# - name: Upload S3 Badge and Summary Artifacts for Nightly Scheduled Run -# if: ${{ github.event_name == 'schedule' }} -# run: | -# cd bin/docker_detection_tester -# python generate_detection_coverage_badge.py --input_summary_file summary_test_results.json --output_badge_file detection_coverage.svg --badge_string "Pass Rate" - - -# #Upload artifact (summary test results) -# aws s3 cp summary_test_results.json s3://security-content/reporting/summary_test_results.json - -# #Since these reside in a public bucket, no need to explicitly mark as public -# # make the file public since it is not by default -# #aws s3api put-object-acl --bucket security-content --key reporting/summary_test_results.json --acl public-read - - -# #Upload artifact (test results coverage badge) -# aws s3 cp detection_coverage.svg s3://security-content/reporting/detection_coverage.svg - -# #Since these reside in a public bucket, no need to explicitly mark as public -# # make the file public since it is not by default -# #aws s3api put-object-acl --bucket security-content --key reporting/detection_coverage.svg --acl public-read - - - - - diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml new file mode 100644 index 0000000000..7b49501cb2 --- /dev/null +++ b/.github/workflows/unit-testing.yml @@ -0,0 +1,101 @@ +name: unit-testing +on: + push: + pull_request: + types: [opened, reopened] +jobs: + contentctl-unit-testing-setup: + runs-on: ubuntu-latest + if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job + # needs: [validate-tag-if-present, quit-for-dependabot] + steps: + - name: Check out the repository code + uses: actions/checkout@v3 + with: + ref: develop + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy + architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified + + - name: Install System Packages + run: | + sudo apt update -qq + sudo apt install jq -qq + + - name: Install Python Dependencies + run: | + python3.11 -m venv .venv + source .venv/bin/activate + pip install contentctl + git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git + + - name: print variable names + run: | + source .venv/bin/activate + echo "github.event.issue.pull_request : [${{ github.event.issue.pull_request }}]" + echo "github.event.pull_request.number : [${{ github.event.pull_request.number }}]" + echo "steps.vars.outputs.branch : [${{ steps.vars.outputs.branch }}]" + echo "github.event.pull_request.head.ref : [${{ github.event.pull_request.head.ref }}]" + echo "github.event_name : [${{ github.event_name }}]" + - name: Extract Branch Name + run: | + echo "Branch name: ${GITHUB_REF#refs/heads/}" + env: + GITHUB_REF: ${{ github.ref }} + + - name: Run ContentCTL test for changes against develop + run: | + pwd + ls + git status + git pull + source .venv/bin/activate + git checkout ${GITHUB_REF#refs/heads/} + # cat summary.yml + contentctl test --post-test-behavior never_pause mode:changes --mode.target-branch develop + mkdir artifacts + cp test_results/summary.yml artifacts/ + cat test_results/summary.yml + + - name: store_artifacts + uses: actions/upload-artifact@v3 + with: + name: content-latest + path: | + artifacts/summary.yml + - name: Install dependencies + run: sudo apt-get install -y jq + + - name: Extract total_fail value + run: | + total_fail=$(yq e '.summary.total_fail' summary.yml) + + - name: Print the failures + run: | + # Debug: Print the content of summary.yml + cat test_results/summary.yml + # Extract total_fail value and debug print it + total_fail=$(yq e '.summary.total_fail' test_results/summary.yml) + echo "Extracted total_fail: [$total_fail]" + # Check if total_fail is a valid integer and greater than one + if [[ "$total_fail" =~ ^[0-9]+$ ]] && [ "$total_fail" -gt 1 ]; then + echo "CI Failure: There are failed tests." + echo -e "Name | Status | Test Type" + echo -e "---- | ------ | ---------" + # Loop through each item in tested_detections and print required fields with color + yq e '.tested_detections[] | .name as $name | .tests[].status as $status | .tests[].test_type as $test_type | "\($name) | \($status) | \($test_type)"' test_results/summary.yml | while read line; do + name=$(echo $line | cut -d '|' -f 1) + status=$(echo $line | cut -d '|' -f 2 | xargs) + test_type=$(echo $line | cut -d '|' -f 3) + if [ "$status" == "pass" ]; then + echo -e "${name} | \033[32m${status}\033[0m | ${test_type}" + else + echo -e "${name} | \033[31m${status}\033[0m | ${test_type}" + fi + done + exit 1 # Fail the CI job + else + echo "CI Success: No failed tests." + fi diff --git a/detections/endpoint/7zip_commandline_to_smb_share_path.yml b/detections/endpoint/7zip_commandline_to_smb_share_path.yml index 76c046406f..876bb44f2f 100644 --- a/detections/endpoint/7zip_commandline_to_smb_share_path.yml +++ b/detections/endpoint/7zip_commandline_to_smb_share_path.yml @@ -22,7 +22,7 @@ search: '| tstats `security_content_summariesonly` count min(_time) as firstTime Processes.parent_process_name Processes.parent_process Processes.process_name Processes.process Processes.parent_process_id Processes.process_id Processes.dest Processes.user | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` - | `7zip_commandline_to_smb_share_path_filter`' + | `7zip_commandline_to_smb_share_path_filter' how_to_implement: The detection is based on data that originates from Endpoint Detection and Response (EDR) agents. These agents are designed to provide security-related telemetry from the endpoints where the agent is installed. To implement this search, From 9827e0b1629b235aa7c412a10f4384730a785dbb Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 12:14:37 -0700 Subject: [PATCH 02/77] fail the search --- detections/endpoint/7zip_commandline_to_smb_share_path.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/detections/endpoint/7zip_commandline_to_smb_share_path.yml b/detections/endpoint/7zip_commandline_to_smb_share_path.yml index 876bb44f2f..70d0e88420 100644 --- a/detections/endpoint/7zip_commandline_to_smb_share_path.yml +++ b/detections/endpoint/7zip_commandline_to_smb_share_path.yml @@ -15,14 +15,14 @@ description: The following analytic detects the execution of 7z or 7za processes data_source: - Sysmon EventID 1 search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) - as lastTime from datamodel=Endpoint.Processes where (Processes.process_name ="7z.exe" + as lastTime from datamodel=Endpoint.Processes where Processes.process_name ="7z.exe" OR Processes.process_name = "7za.exe" OR Processes.original_file_name = "7z.exe" OR Processes.original_file_name = "7za.exe") AND (Processes.process="*\\C$\\*" OR Processes.process="*\\Admin$\\*" OR Processes.process="*\\IPC$\\*") by Processes.original_file_name Processes.parent_process_name Processes.parent_process Processes.process_name Processes.process Processes.parent_process_id Processes.process_id Processes.dest Processes.user | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` - | `7zip_commandline_to_smb_share_path_filter' + | `7zip_commandline_to_smb_share_path_filter`' how_to_implement: The detection is based on data that originates from Endpoint Detection and Response (EDR) agents. These agents are designed to provide security-related telemetry from the endpoints where the agent is installed. To implement this search, From 6ea505678ae60807264800c60ed697b2ba45d84c Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 12:28:49 -0700 Subject: [PATCH 03/77] continue on error --- .github/workflows/unit-testing.yml | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 7b49501cb2..d7caa26559 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -31,25 +31,8 @@ jobs: pip install contentctl git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git - - name: print variable names - run: | - source .venv/bin/activate - echo "github.event.issue.pull_request : [${{ github.event.issue.pull_request }}]" - echo "github.event.pull_request.number : [${{ github.event.pull_request.number }}]" - echo "steps.vars.outputs.branch : [${{ steps.vars.outputs.branch }}]" - echo "github.event.pull_request.head.ref : [${{ github.event.pull_request.head.ref }}]" - echo "github.event_name : [${{ github.event_name }}]" - - name: Extract Branch Name - run: | - echo "Branch name: ${GITHUB_REF#refs/heads/}" - env: - GITHUB_REF: ${{ github.ref }} - - name: Run ContentCTL test for changes against develop run: | - pwd - ls - git status git pull source .venv/bin/activate git checkout ${GITHUB_REF#refs/heads/} @@ -58,21 +41,23 @@ jobs: mkdir artifacts cp test_results/summary.yml artifacts/ cat test_results/summary.yml - + continue-on-error: true + - name: store_artifacts uses: actions/upload-artifact@v3 with: name: content-latest path: | artifacts/summary.yml - - name: Install dependencies + + - name: Install JQ run: sudo apt-get install -y jq - name: Extract total_fail value run: | total_fail=$(yq e '.summary.total_fail' summary.yml) - - name: Print the failures + - name: Formatted Final Report run: | # Debug: Print the content of summary.yml cat test_results/summary.yml From 81de107f1a5748323bd3a22f72559b6c81e943c3 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 12:29:16 -0700 Subject: [PATCH 04/77] name update --- .github/workflows/unit-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index d7caa26559..33a88a075f 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -4,7 +4,7 @@ on: pull_request: types: [opened, reopened] jobs: - contentctl-unit-testing-setup: + contentctl-unit-testing: runs-on: ubuntu-latest if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job # needs: [validate-tag-if-present, quit-for-dependabot] From 87cddf5e0884c62eb7cc8fc9a8e7a729972eec3c Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 12:51:06 -0700 Subject: [PATCH 05/77] updating output --- .github/workflows/unit-testing.yml | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 33a88a075f..c287133d16 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -53,34 +53,35 @@ jobs: - name: Install JQ run: sudo apt-get install -y jq - - name: Extract total_fail value - run: | - total_fail=$(yq e '.summary.total_fail' summary.yml) + - name: Print entire test_results/summary.xml + run: cat test_results/summary.yml - name: Formatted Final Report run: | - # Debug: Print the content of summary.yml - cat test_results/summary.yml # Extract total_fail value and debug print it total_fail=$(yq e '.summary.total_fail' test_results/summary.yml) echo "Extracted total_fail: [$total_fail]" + # Check if total_fail is a valid integer and greater than one if [[ "$total_fail" =~ ^[0-9]+$ ]] && [ "$total_fail" -gt 1 ]; then echo "CI Failure: There are failed tests." echo -e "Name | Status | Test Type" echo -e "---- | ------ | ---------" - # Loop through each item in tested_detections and print required fields with color - yq e '.tested_detections[] | .name as $name | .tests[].status as $status | .tests[].test_type as $test_type | "\($name) | \($status) | \($test_type)"' test_results/summary.yml | while read line; do - name=$(echo $line | cut -d '|' -f 1) - status=$(echo $line | cut -d '|' -f 2 | xargs) - test_type=$(echo $line | cut -d '|' -f 3) - if [ "$status" == "pass" ]; then - echo -e "${name} | \033[32m${status}\033[0m | ${test_type}" + + # Loop through each item in tested_detections and print required fields with color only for unit testing + yq e '.tested_detections[] | .name as $name | .tests[] | select(.test_type == "unit") | "\($name) | \(.success) | \(.test_type)"' test_results/summary.yml | while IFS='|' read -r name status test_type; do + name=$(echo $name | xargs) # Trim whitespace + status=$(echo $status | xargs) # Trim whitespace + test_type=$(echo $test_type | xargs) # Trim whitespace + + if [ "$status" == "true" ]; then + echo -e "${name} | \033[32mPASS\033[0m | ${test_type}" else - echo -e "${name} | \033[31m${status}\033[0m | ${test_type}" + echo -e "${name} | \033[31mFAIL\033[0m | ${test_type}" fi done + exit 1 # Fail the CI job else echo "CI Success: No failed tests." - fi + fi \ No newline at end of file From a19f3f14848e8e05add35087d9d6b8deb54d6cb6 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 13:08:47 -0700 Subject: [PATCH 06/77] testing printouts --- .github/workflows/format_test_summary.sh | 27 ++ .github/workflows/unit-testing.yml | 36 +-- summary.yml | 370 +++++++++++++++++++++++ 3 files changed, 404 insertions(+), 29 deletions(-) create mode 100755 .github/workflows/format_test_summary.sh create mode 100644 summary.yml diff --git a/.github/workflows/format_test_summary.sh b/.github/workflows/format_test_summary.sh new file mode 100755 index 0000000000..6031cccf40 --- /dev/null +++ b/.github/workflows/format_test_summary.sh @@ -0,0 +1,27 @@ +# Extract total_fail value and debug print it +total_fail=$(yq e '.summary.total_fail' test_results/summary.yml) +echo "Extracted total_fail: [$total_fail]" + +# Check if total_fail is a valid integer and greater than one +if [[ "$total_fail" =~ ^[0-9]+$ ]] && [ "$total_fail" -gt 1 ]; then + echo "CI Failure: There are failed tests." + echo -e "Name | Status | Test Type" + echo -e "---- | ------ | ---------" + + # Loop through each item in tested_detections and print required fields with color only for unit testing + yq e '.tested_detections[] | .name as $name | .tests[] | select(.test_type == "unit") | "\($name) | \(.success) | \(.test_type)"' test_results/summary.yml | while IFS='|' read -r name status test_type; do + name=$(echo $name | xargs) # Trim whitespace + status=$(echo $status | xargs) # Trim whitespace + test_type=$(echo $test_type | xargs) # Trim whitespace + + if [ "$status" == "true" ]; then + echo -e "${name} | \033[32mPASS\033[0m | ${test_type}" + else + echo -e "${name} | \033[31mFAIL\033[0m | ${test_type}" + fi + done + + exit 1 # Fail the CI job +else + echo "CI Success: No failed tests." +fi \ No newline at end of file diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index c287133d16..721204fd20 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -37,10 +37,13 @@ jobs: source .venv/bin/activate git checkout ${GITHUB_REF#refs/heads/} # cat summary.yml - contentctl test --post-test-behavior never_pause mode:changes --mode.target-branch develop + # contentctl test --post-test-behavior never_pause mode:changes --mode.target-branch develop mkdir artifacts + mkdir test_results + cp summary.yml test_results/ cp test_results/summary.yml artifacts/ cat test_results/summary.yml + continue-on-error: true - name: store_artifacts @@ -53,35 +56,10 @@ jobs: - name: Install JQ run: sudo apt-get install -y jq - - name: Print entire test_results/summary.xml + - name: Print entire test_results/summary.yml run: cat test_results/summary.yml - name: Formatted Final Report run: | - # Extract total_fail value and debug print it - total_fail=$(yq e '.summary.total_fail' test_results/summary.yml) - echo "Extracted total_fail: [$total_fail]" - - # Check if total_fail is a valid integer and greater than one - if [[ "$total_fail" =~ ^[0-9]+$ ]] && [ "$total_fail" -gt 1 ]; then - echo "CI Failure: There are failed tests." - echo -e "Name | Status | Test Type" - echo -e "---- | ------ | ---------" - - # Loop through each item in tested_detections and print required fields with color only for unit testing - yq e '.tested_detections[] | .name as $name | .tests[] | select(.test_type == "unit") | "\($name) | \(.success) | \(.test_type)"' test_results/summary.yml | while IFS='|' read -r name status test_type; do - name=$(echo $name | xargs) # Trim whitespace - status=$(echo $status | xargs) # Trim whitespace - test_type=$(echo $test_type | xargs) # Trim whitespace - - if [ "$status" == "true" ]; then - echo -e "${name} | \033[32mPASS\033[0m | ${test_type}" - else - echo -e "${name} | \033[31mFAIL\033[0m | ${test_type}" - fi - done - - exit 1 # Fail the CI job - else - echo "CI Success: No failed tests." - fi \ No newline at end of file + chmod +x .github/workflows/format_test_summary.sh + ./format_test_summary.sh \ No newline at end of file diff --git a/summary.yml b/summary.yml new file mode 100644 index 0000000000..cc437d9783 --- /dev/null +++ b/summary.yml @@ -0,0 +1,370 @@ +summary: + success: false + total_detections: 11 + total_pass: 8 + total_fail: 3 + total_skipped: 0 + total_untested: 0 + total_experimental_or_deprecated: 0 + success_rate: 72.7% +tested_detections: +- name: Windows AD Replication Request Initiated from Unsanctioned Location + search: '`wineventlog_security` EventCode=4662 ObjectType IN ("%{19195a5b-6da0-11d0-afd3-00c04fd930c9}", + "domainDNS") AND Properties IN ("*Replicating Directory Changes All*", "*{1131f6ad-9c07-11d1-f79f-00c04fc2dcd2}*", + "*{9923a32a-3607-11d2-b9be-0000f87a36b2}*","*{1131f6ac-9c07-11d1-f79f-00c04fc2dcd2}*") + AND AccessMask="0x100" AND (SubjectUserSid="NT AUT*" OR SubjectUserSid="S-1-5-18" + OR SubjectDomainName="Window Manager" OR SubjectUserName="*$") | stats min(_time) + as attack_time, count by SubjectDomainName, SubjectUserName, Computer, Logon_ID, + ObjectName, ObjectServer, ObjectType, OperationType, status | rename SubjectDomainName + as Target_Domain, SubjectUserName as user, Logon_ID as TargetLogonId | appendpipe + [| map search="search `wineventlog_security` EventCode=4624 TargetLogonId=$TargetLogonId$"] + | table attack_time, AuthenticationPackageName, LogonProcessName, LogonType, TargetUserSid, + Target_Domain, user, Computer, TargetLogonId, status, src_ip, src_category, ObjectName, + ObjectServer, ObjectType, OperationType | stats min(attack_time) as _time, values(TargetUserSid) + as TargetUserSid, values(Target_Domain) as Target_Domain, values(user) as user, + values(Computer) as Computer, values(status) as status, values(src_category) as + src_category, values(src_ip) as src_ip by TargetLogonId | search NOT src_category="domain_controller" + | `windows_ad_replication_request_initiated_from_unsanctioned_location_filter`' + success: false + tests: + - name: True Positive Test + test_type: unit + success: false + message: TEST ERROR + exception: The observable field(s) {'src_ip'} are missing in the detection results + status: error + duration: 6.49 + wait_duration: null + resultCount: '1' + runDuration: '1.320' + - name: True Positive Test + test_type: integration + success: false + message: 'TEST FAILED (PREEMPTIVE): associated unit test failed or encountered + an error' + exception: The observable field(s) {'src_ip'} are missing in the detection results + status: error + duration: 3.08 + wait_duration: null + resultCount: null + runDuration: null +- name: Windows Admon Default Group Policy Object Modified + search: ' `admon` admonEventType=Update objectCategory="CN=Group-Policy-Container,CN=Schema,CN=Configuration,DC=*" + (displayName="Default Domain Policy" OR displayName="Default Domain Controllers + Policy") | appendpipe [ | map search="search `wineventlog_security` EventCode=5136 AttributeSyntaxOID=2.5.5.12 + AttributeValue=$displayName$" | rename AttributeValue as displayName] | stats + min(_time) as _time values(displayName) as gp_name, values(gPCFileSysPath) as + gPCFileSysPath, values(src_user) as src_user, values(dcName) as dcName, values(dest_category) + as dest_category, values(src_user_category) as src_user_category by displayName + | `windows_admon_default_group_policy_object_modified_filter`' + success: false + tests: + - name: True Positive Test + test_type: unit + success: false + message: TEST ERROR + exception: The observable field(s) {'src_user'} are missing in the detection results + status: error + duration: 7.63 + wait_duration: null + resultCount: '1' + runDuration: '2.158' + - name: True Positive Test + test_type: integration + success: false + message: 'TEST FAILED (PREEMPTIVE): associated unit test failed or encountered + an error' + exception: The observable field(s) {'src_user'} are missing in the detection results + status: error + duration: 3.17 + wait_duration: null + resultCount: null + runDuration: null +- name: Windows Admon Group Policy Object Created + search: ' `admon` admonEventType=Update objectCategory="CN=Group-Policy-Container,CN=Schema,CN=Configuration,DC=*" + versionNumber=0 displayName!="New Group Policy Object" | appendpipe [ | map search="search + `wineventlog_security` EventCode=5136 AttributeSyntaxOID=2.5.5.12 AttributeValue=$displayName$" + | rename AttributeValue as displayName] | stats min(_time) as _time values(displayName) + as gp_name, values(gPCFileSysPath) as gPCFileSysPath, values(src_user) as src_user, + values(dcName) as dcName, values(dest_category) as dest_category, values(src_user_category) + as src_user_category by displayName | `windows_admon_group_policy_object_created_filter`' + success: false + tests: + - name: True Positive Test + test_type: unit + success: false + message: TEST ERROR + exception: The observable field(s) {'src_user'} are missing in the detection results + status: error + duration: 11.45 + wait_duration: null + resultCount: '1' + runDuration: '1.270' + - name: True Positive Test + test_type: integration + success: false + message: 'TEST FAILED (PREEMPTIVE): associated unit test failed or encountered + an error' + exception: The observable field(s) {'src_user'} are missing in the detection results + status: error + duration: 3.26 + wait_duration: null + resultCount: null + runDuration: null +- name: Azure AD Admin Consent Bypassed by Service Principal + search: '`azure_monitor_aad` (operationName="Add app role assignment to service + principal" OR operationName="Add member to role*") src_user_type=servicePrincipal | + rename properties.* as * | eval roleId = mvindex(''targetResources{}.modifiedProperties{}.newValue'', + 0) | eval roleValue = mvindex(''targetResources{}.modifiedProperties{}.newValue'', + 1) | eval roleDescription = mvindex(''targetResources{}.modifiedProperties{}.newValue'', + 2) | eval user_id = mvindex(''targetResources{}.id'', 0), user=coalesce(user,mvindex(''targetResources{}.displayName'', + 0)) | rename initiatedBy.app.displayName as src_user | stats count earliest(_time) + as firstTime latest(_time) as lastTime by src_user user user_id roleId roleValue + roleDescription | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | + `azure_ad_admin_consent_bypassed_by_service_principal_filter`' + success: true + tests: + - name: True Positive Test + test_type: unit + success: true + message: TEST PASSED + exception: null + status: pass + duration: 5.84 + wait_duration: null + resultCount: '1' + runDuration: '0.654' + - name: True Positive Test + test_type: integration + success: true + message: 'TEST SKIPPED: Skipping all integration tests' + exception: null + status: skip + duration: 0 + wait_duration: null + resultCount: null + runDuration: null +- name: Azure AD Global Administrator Role Assigned + search: '`azure_monitor_aad` operationName="Add member to role" properties.targetResources{}.modifiedProperties{}.newValue="\"Global + Administrator\"" | rename properties.* as *, initiatedBy.user.userPrincipalName + as userPrincipalName, targetResources{}.displayName as displayName | eval initiatedBy + = coalesce(userPrincipalName,src_user) | eval user = coalesce(user,mvfilter(displayName!="null")) + | stats count min(_time) as firstTime max(_time) as lastTime values(user) as user + by initiatedBy, result, operationName | `security_content_ctime(firstTime)` | + `security_content_ctime(lastTime)` | `azure_ad_global_administrator_role_assigned_filter`' + success: true + tests: + - name: True Positive Test + test_type: unit + success: true + message: TEST PASSED + exception: null + status: pass + duration: 5.97 + wait_duration: null + resultCount: '1' + runDuration: '0.666' + - name: True Positive Test + test_type: integration + success: true + message: 'TEST SKIPPED: Skipping all integration tests' + exception: null + status: skip + duration: 0 + wait_duration: null + resultCount: null + runDuration: null +- name: Azure AD Privileged Role Assigned + search: ' `azure_monitor_aad` "operationName"="Add member to role" | rename properties.* + as *, initiatedBy.user.userPrincipalName as userPrincipalName, targetResources{}.displayName + as displayName | eval initiatedBy = coalesce(userPrincipalName,src_user) | eval + user = coalesce(user,mvfilter(displayName!="null")) | rename targetResources{}.modifiedProperties{}.newValue as + roles | eval role=mvindex(roles,1) | stats count min(_time) as firstTime max(_time) + as lastTime values(user) as user by initiatedBy, result, operationName, role | + lookup privileged_azure_ad_roles azureadrole AS role OUTPUT isprvilegedadrole + description | search isprvilegedadrole = True | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` | `azure_ad_privileged_role_assigned_filter`' + success: true + tests: + - name: True Positive Test + test_type: unit + success: true + message: TEST PASSED + exception: null + status: pass + duration: 5.94 + wait_duration: null + resultCount: '1' + runDuration: '0.648' + - name: True Positive Test + test_type: integration + success: true + message: 'TEST SKIPPED: Skipping all integration tests' + exception: null + status: skip + duration: 0 + wait_duration: null + resultCount: null + runDuration: null +- name: Azure AD Privileged Role Assigned to Service Principal + search: ' `azure_monitor_aad` operationName="Add member to role" | rename properties.* + as * | search "targetResources{}.type"=ServicePrincipal | rename initiatedBy.user.userPrincipalName + as initiatedBy | rename targetResources{}.modifiedProperties{}.newValue as roles + | eval role=mvindex(roles,1) | rename targetResources{}.displayName as apps | + eval displayName=mvindex(apps,0) | stats count min(_time) as firstTime max(_time) + as lastTime values(displayName) as displayName by initiatedBy, result, operationName, + role | lookup privileged_azure_ad_roles azureadrole AS role OUTPUT isprvilegedadrole + description | search isprvilegedadrole = True | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` | `azure_ad_privileged_role_assigned_to_service_principal_filter`' + success: true + tests: + - name: True Positive Test + test_type: unit + success: true + message: TEST PASSED + exception: null + status: pass + duration: 9.3 + wait_duration: null + resultCount: '1' + runDuration: '3.134' + - name: True Positive Test + test_type: integration + success: true + message: 'TEST SKIPPED: Skipping all integration tests' + exception: null + status: skip + duration: 0 + wait_duration: null + resultCount: null + runDuration: null +- name: Azure AD Service Principal New Client Credentials + search: ' `azure_monitor_aad` category=AuditLogs operationName="Update application*Certificates + and secrets management*" | rename properties.* as * | rename targetResources{}.* + as * | rename modifiedProperties{}.* as * | eval src_user=coalesce(user,identity), + newValue=mvfilter(newValue!="\"KeyDescription\"") | stats count min(_time) as + firstTime max(_time) as lastTime values(displayName) as displayName values(src_ip) + as src_ip values(eval(mvfilter(oldValue!="null"))) as oldValue by src_user, object, + newValue | spath input=oldValue output=oldValues path={} | spath input=newValue + output=newValues path={} | mvexpand newValues | where NOT newValues IN (oldValues) + | fields - newValue, oldValue, oldValues | rename newValues as newValue | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` | `azure_ad_service_principal_new_client_credentials_filter`' + success: true + tests: + - name: True Positive Test + test_type: unit + success: true + message: TEST PASSED + exception: null + status: pass + duration: 5.94 + wait_duration: null + resultCount: '1' + runDuration: '0.782' + - name: True Positive Test + test_type: integration + success: true + message: 'TEST SKIPPED: Skipping all integration tests' + exception: null + status: skip + duration: 0 + wait_duration: null + resultCount: null + runDuration: null +- name: Detect New Local Admin account + search: '`wineventlog_security` (EventCode=4720) OR (EventCode=4732 Group_Name=Administrators) + | stats dc(EventCode) as evCount min(_time) as _time range(_time) as duration + values(src_user) as src_user values(src_user_category) as src_user_category values(dest_category) + as dest_category by user dest | where evCount=2 | fields - evCount, duration | + `detect_new_local_admin_account_filter`' + success: true + tests: + - name: True Positive Test + test_type: unit + success: true + message: TEST PASSED + exception: null + status: pass + duration: 14.33 + wait_duration: null + resultCount: '1' + runDuration: '0.696' + - name: True Positive Test + test_type: integration + success: true + message: 'TEST SKIPPED: Skipping all integration tests' + exception: null + status: skip + duration: 0 + wait_duration: null + resultCount: null + runDuration: null +- name: Kerberos Pre-Authentication Flag Disabled in UserAccountControl + search: '`wineventlog_security` EventCode=4738 (UserAccountControl="%%2096" OR MSADChangedAttributes="*Don''t + Require Preauth'' - Enabled*") | eval MSADChangedAttributes="''Don''t Require + Preauth'' - Enabled" | table _time, source, EventCode, src_user, src_user_category, + user, user_category, MSADChangedAttributes | `kerberos_pre_authentication_flag_disabled_in_useraccountcontrol_filter`' + success: true + tests: + - name: True Positive Test + test_type: unit + success: true + message: TEST PASSED + exception: null + status: pass + duration: 6.08 + wait_duration: null + resultCount: '1' + runDuration: '0.630' + - name: True Positive Test + test_type: integration + success: true + message: 'TEST SKIPPED: Skipping all integration tests' + exception: null + status: skip + duration: 0 + wait_duration: null + resultCount: null + runDuration: null +- name: Windows AD Replication Request Initiated by User Account + search: '`wineventlog_security` EventCode=4662 ObjectType IN ("%{19195a5b-6da0-11d0-afd3-00c04fd930c9}", + "domainDNS") AND Properties IN ("*Replicating Directory Changes All*", "*{1131f6ad-9c07-11d1-f79f-00c04fc2dcd2}*", + "*{9923a32a-3607-11d2-b9be-0000f87a36b2}*","*{1131f6ac-9c07-11d1-f79f-00c04fc2dcd2}*") + AND AccessMask="0x100" AND NOT (SubjectUserSid="NT AUT*" OR SubjectUserSid="S-1-5-18" + OR SubjectDomainName="Window Manager" OR SubjectUserName="*$") | stats min(_time) + as _time, count by SubjectDomainName, SubjectUserName, Computer, Logon_ID, ObjectName, + ObjectServer, ObjectType, OperationType, status | rename SubjectDomainName as + Target_Domain, SubjectUserName as user, Logon_ID as TargetLogonId, _time as attack_time + | appendpipe [| map search="search `wineventlog_security` EventCode=4624 TargetLogonId=$TargetLogonId$"] + | table attack_time, AuthenticationPackageName, LogonProcessName, LogonType, TargetUserSid, + Target_Domain, user, Computer, TargetLogonId, status, src_ip, src_category, ObjectName, + ObjectServer, ObjectType, OperationType | stats min(attack_time) as _time values(TargetUserSid) + as TargetUserSid, values(Target_Domain) as Target_Domain, values(user) as user, + values(Computer) as Computer, values(status) as status, values(src_category) as + src_category, values(src_ip) as src_ip by TargetLogonId | `windows_ad_replication_request_initiated_by_user_account_filter`' + success: true + tests: + - name: True Positive Test + test_type: unit + success: true + message: TEST PASSED + exception: null + status: pass + duration: 11.26 + wait_duration: null + resultCount: '1' + runDuration: '1.289' + - name: True Positive Test + test_type: integration + success: true + message: 'TEST SKIPPED: Skipping all integration tests' + exception: null + status: skip + duration: 0 + wait_duration: null + resultCount: null + runDuration: null +untested_detections: [] +percent_complete: UKNOWN +deprecated_detections: [] +experimental_detections: [] From aa75d4fa5e7332181807560da35ab0979049b9c0 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 13:12:34 -0700 Subject: [PATCH 07/77] updating scirpt --- .github/workflows/unit-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 721204fd20..de482057e5 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -62,4 +62,4 @@ jobs: - name: Formatted Final Report run: | chmod +x .github/workflows/format_test_summary.sh - ./format_test_summary.sh \ No newline at end of file + ./.github/workflows/format_test_summary.sh \ No newline at end of file From 78ef3b827fcd2c8ffb4a79586255e72dbfe6f17f Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 13:25:00 -0700 Subject: [PATCH 08/77] enable testing --- .github/workflows/unit-testing.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index de482057e5..9ed50214e1 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -36,11 +36,10 @@ jobs: git pull source .venv/bin/activate git checkout ${GITHUB_REF#refs/heads/} - # cat summary.yml - # contentctl test --post-test-behavior never_pause mode:changes --mode.target-branch develop + contentctl test --post-test-behavior never_pause mode:changes --mode.target-branch develop mkdir artifacts - mkdir test_results - cp summary.yml test_results/ + # mkdir test_results + # cp summary.yml test_results/ cp test_results/summary.yml artifacts/ cat test_results/summary.yml From f8bd88110a08a00f12785106728042ffe06fb8df Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 13:50:42 -0700 Subject: [PATCH 09/77] diable tqdm and remove venv --- .github/workflows/unit-testing.yml | 18 ++++++++---------- .github/workflows/validate-and-build.yml | 10 ++++------ 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 9ed50214e1..0d8542dbb9 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -2,7 +2,7 @@ name: unit-testing on: push: pull_request: - types: [opened, reopened] + types: [opened, synchronize, reopened] jobs: contentctl-unit-testing: runs-on: ubuntu-latest @@ -26,20 +26,21 @@ jobs: - name: Install Python Dependencies run: | - python3.11 -m venv .venv - source .venv/bin/activate + # python3.11 -m venv .venv + # source .venv/bin/activate pip install contentctl git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git - name: Run ContentCTL test for changes against develop run: | git pull - source .venv/bin/activate + # source .venv/bin/activate git checkout ${GITHUB_REF#refs/heads/} - contentctl test --post-test-behavior never_pause mode:changes --mode.target-branch develop + echo "The target branch for this PR is ${{ github.base_ref }}" + contentctl test --disable-tqdm --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} mkdir artifacts - # mkdir test_results - # cp summary.yml test_results/ + mkdir test_results + cp summary.yml test_results/ cp test_results/summary.yml artifacts/ cat test_results/summary.yml @@ -52,9 +53,6 @@ jobs: path: | artifacts/summary.yml - - name: Install JQ - run: sudo apt-get install -y jq - - name: Print entire test_results/summary.yml run: cat test_results/summary.yml diff --git a/.github/workflows/validate-and-build.yml b/.github/workflows/validate-and-build.yml index 1e4ddc884f..6a0de77ed7 100644 --- a/.github/workflows/validate-and-build.yml +++ b/.github/workflows/validate-and-build.yml @@ -4,7 +4,6 @@ on: pull_request: types: [opened, reopened] jobs: - validate-and-build: #Note that the CircleCI job used a Container. The way to do this with Github Actions #is to first start up a Virtual Machine, then we can by following: @@ -23,23 +22,22 @@ jobs: run: | sudo apt update -qq sudo apt install jq -qq - - name: Install Python Dependencies and ContentCTL and Atomic Red Team run: | - python3.11 -m venv .venv - source .venv/bin/activate + # python3.11 -m venv .venv + # source .venv/bin/activate pip install contentctl git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git - name: content_ctl validate run: | - source .venv/bin/activate + # source .venv/bin/activate contentctl validate - name: contentctl generate run: | - source .venv/bin/activate + # source .venv/bin/activate contentctl build --enrichments mkdir artifacts mv dist/DA-ESS-ContentUpdate-latest.tar.gz artifacts/ From a75a6bdd63e0c827b1df67e300902b379b17fe22 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 13:54:52 -0700 Subject: [PATCH 10/77] remove pull --- .github/workflows/unit-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 0d8542dbb9..a498e6741b 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -33,7 +33,7 @@ jobs: - name: Run ContentCTL test for changes against develop run: | - git pull + # git pull # source .venv/bin/activate git checkout ${GITHUB_REF#refs/heads/} echo "The target branch for this PR is ${{ github.base_ref }}" From d996a8186fe401231f673a649a775a96b9cf1bf9 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 14:11:28 -0700 Subject: [PATCH 11/77] udpating git pull --- .github/workflows/unit-testing.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index a498e6741b..28ba22244f 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -33,17 +33,15 @@ jobs: - name: Run ContentCTL test for changes against develop run: | - # git pull + git pull > /dev/null 2>&1 # source .venv/bin/activate git checkout ${GITHUB_REF#refs/heads/} echo "The target branch for this PR is ${{ github.base_ref }}" contentctl test --disable-tqdm --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} mkdir artifacts mkdir test_results - cp summary.yml test_results/ + # cp summary.yml test_results/ cp test_results/summary.yml artifacts/ - cat test_results/summary.yml - continue-on-error: true - name: store_artifacts From a42b0c7036f98c707e18d9325f55595e478dcfcb Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 15:10:48 -0700 Subject: [PATCH 12/77] updating formatting --- .github/workflows/format_test_summary.sh | 9 +++++---- .github/workflows/unit-testing.yml | 14 ++++++-------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/format_test_summary.sh b/.github/workflows/format_test_summary.sh index 6031cccf40..730a84855b 100755 --- a/.github/workflows/format_test_summary.sh +++ b/.github/workflows/format_test_summary.sh @@ -5,8 +5,9 @@ echo "Extracted total_fail: [$total_fail]" # Check if total_fail is a valid integer and greater than one if [[ "$total_fail" =~ ^[0-9]+$ ]] && [ "$total_fail" -gt 1 ]; then echo "CI Failure: There are failed tests." - echo -e "Name | Status | Test Type" - echo -e "---- | ------ | ---------" + # Adjust the column widths here + printf "%-80s | %-6s | %-10s\n" "Name" "Status" "Test Type" + printf "%-80s | %-6s | %-10s\n" "----" "------" "---------" # Loop through each item in tested_detections and print required fields with color only for unit testing yq e '.tested_detections[] | .name as $name | .tests[] | select(.test_type == "unit") | "\($name) | \(.success) | \(.test_type)"' test_results/summary.yml | while IFS='|' read -r name status test_type; do @@ -15,9 +16,9 @@ if [[ "$total_fail" =~ ^[0-9]+$ ]] && [ "$total_fail" -gt 1 ]; then test_type=$(echo $test_type | xargs) # Trim whitespace if [ "$status" == "true" ]; then - echo -e "${name} | \033[32mPASS\033[0m | ${test_type}" + printf "%-80s | \033[32m%-6s\033[0m | %-10s\n" "$name" "PASS" "$test_type" else - echo -e "${name} | \033[31mFAIL\033[0m | ${test_type}" + printf "%-80s | \033[31m%-6s\033[0m | %-10s\n" "$name" "FAIL" "$test_type" fi done diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 28ba22244f..bf5f1286cc 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -26,8 +26,6 @@ jobs: - name: Install Python Dependencies run: | - # python3.11 -m venv .venv - # source .venv/bin/activate pip install contentctl git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git @@ -37,19 +35,19 @@ jobs: # source .venv/bin/activate git checkout ${GITHUB_REF#refs/heads/} echo "The target branch for this PR is ${{ github.base_ref }}" - contentctl test --disable-tqdm --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} - mkdir artifacts + contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} mkdir test_results # cp summary.yml test_results/ - cp test_results/summary.yml artifacts/ + cp test_results/summary.yml artifacts/ continue-on-error: true - name: store_artifacts uses: actions/upload-artifact@v3 with: - name: content-latest + name: test_summary_results path: | - artifacts/summary.yml + test_results/summary.yml + dist/DA-ESS-ContentUpdate-latest.tar.gz - name: Print entire test_results/summary.yml run: cat test_results/summary.yml @@ -57,4 +55,4 @@ jobs: - name: Formatted Final Report run: | chmod +x .github/workflows/format_test_summary.sh - ./.github/workflows/format_test_summary.sh \ No newline at end of file + ./.github/workflows/format_test_summary.sh >> $GITHUB_STEP_SUMMARY \ No newline at end of file From 1f17367916acdd66c8830a7ab14e59a6fa5eb06b Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 15:16:47 -0700 Subject: [PATCH 13/77] testing no integration --- .github/workflows/unit-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index bf5f1286cc..d005e5af9f 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -2,7 +2,7 @@ name: unit-testing on: push: pull_request: - types: [opened, synchronize, reopened] + types: [opened, reopened] jobs: contentctl-unit-testing: runs-on: ubuntu-latest From c16658d8d807a275a0e88f5c9d1427be02055904 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 15:23:00 -0700 Subject: [PATCH 14/77] git pull --- .github/workflows/unit-testing.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index d005e5af9f..2acec04fe8 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -31,8 +31,7 @@ jobs: - name: Run ContentCTL test for changes against develop run: | - git pull > /dev/null 2>&1 - # source .venv/bin/activate + git pull git checkout ${GITHUB_REF#refs/heads/} echo "The target branch for this PR is ${{ github.base_ref }}" contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} @@ -48,9 +47,11 @@ jobs: path: | test_results/summary.yml dist/DA-ESS-ContentUpdate-latest.tar.gz + continue-on-error: true - name: Print entire test_results/summary.yml run: cat test_results/summary.yml + - name: Formatted Final Report run: | From e82f484cb4f47ff0d599241bed715d577647cf98 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 15:27:50 -0700 Subject: [PATCH 15/77] remove no enable --- .github/workflows/unit-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 2acec04fe8..e85690bd5d 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -34,7 +34,7 @@ jobs: git pull git checkout ${GITHUB_REF#refs/heads/} echo "The target branch for this PR is ${{ github.base_ref }}" - contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} + contentctl test --disable-tqdm never_pause mode:changes --mode.target-branch ${{ github.base_ref }} mkdir test_results # cp summary.yml test_results/ cp test_results/summary.yml artifacts/ From cf825c76056aee5f492b8017ef1d0ddf41faa52b Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 15:32:30 -0700 Subject: [PATCH 16/77] updating ctl command --- .github/workflows/unit-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index e85690bd5d..c61d219321 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -34,7 +34,7 @@ jobs: git pull git checkout ${GITHUB_REF#refs/heads/} echo "The target branch for this PR is ${{ github.base_ref }}" - contentctl test --disable-tqdm never_pause mode:changes --mode.target-branch ${{ github.base_ref }} + contentctl test --disable-tqdm --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} mkdir test_results # cp summary.yml test_results/ cp test_results/summary.yml artifacts/ From e4ff59ded5cccf435b8e1811e4ce94c7f11d408d Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 15:39:11 -0700 Subject: [PATCH 17/77] updating syncronize --- .github/workflows/unit-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index c61d219321..b5552ec17f 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -2,7 +2,7 @@ name: unit-testing on: push: pull_request: - types: [opened, reopened] + types: [opened, synchronize, reopened] jobs: contentctl-unit-testing: runs-on: ubuntu-latest From de8e09485dff1373418bb52da8f924e25e81b807 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 15:47:05 -0700 Subject: [PATCH 18/77] print branch names --- .github/workflows/unit-testing.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index b5552ec17f..dd88eed08d 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -31,12 +31,14 @@ jobs: - name: Run ContentCTL test for changes against develop run: | - git pull - git checkout ${GITHUB_REF#refs/heads/} + echo "Current Branch (Head Ref): ${{ github.head_ref }}" + echo "Target Branch (Base Ref): ${{ github.base_ref }}" + # git pull + git checkout ${GITHUB_REF#refs/base/} echo "The target branch for this PR is ${{ github.base_ref }}" - contentctl test --disable-tqdm --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} + # contentctl test --disable-tqdm --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} mkdir test_results - # cp summary.yml test_results/ + cp summary.yml test_results/ cp test_results/summary.yml artifacts/ continue-on-error: true From 8d8f5abfacbc3d5fedc348d91aa8c8da39600f26 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 15:49:13 -0700 Subject: [PATCH 19/77] print branch names --- .github/workflows/unit-testing.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index dd88eed08d..8b81bb2b6b 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -2,7 +2,7 @@ name: unit-testing on: push: pull_request: - types: [opened, synchronize, reopened] + types: [opened, reopened] jobs: contentctl-unit-testing: runs-on: ubuntu-latest @@ -33,8 +33,8 @@ jobs: run: | echo "Current Branch (Head Ref): ${{ github.head_ref }}" echo "Target Branch (Base Ref): ${{ github.base_ref }}" - # git pull - git checkout ${GITHUB_REF#refs/base/} + git pull + git checkout ${GITHUB_REF#refs/heads/} echo "The target branch for this PR is ${{ github.base_ref }}" # contentctl test --disable-tqdm --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} mkdir test_results From a02f44f863c1f02457e646856cbeeb53ab3a1b64 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 15:58:10 -0700 Subject: [PATCH 20/77] no luck with branch name --- .github/workflows/unit-testing.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 8b81bb2b6b..e8400d0ba8 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -2,7 +2,7 @@ name: unit-testing on: push: pull_request: - types: [opened, reopened] + types: [opened, reopened, synchronize] jobs: contentctl-unit-testing: runs-on: ubuntu-latest @@ -11,8 +11,8 @@ jobs: steps: - name: Check out the repository code uses: actions/checkout@v3 - with: - ref: develop + # with: + # ref: develop - uses: actions/setup-python@v4 with: @@ -33,11 +33,13 @@ jobs: run: | echo "Current Branch (Head Ref): ${{ github.head_ref }}" echo "Target Branch (Base Ref): ${{ github.base_ref }}" + echo "reff - ${GITHUB_REF#refs/heads/}" git pull git checkout ${GITHUB_REF#refs/heads/} echo "The target branch for this PR is ${{ github.base_ref }}" # contentctl test --disable-tqdm --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} mkdir test_results + mkdir artifacts cp summary.yml test_results/ cp test_results/summary.yml artifacts/ continue-on-error: true From 82630b260ba76d513d9f6e97e18827849dc1acf9 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 16:08:14 -0700 Subject: [PATCH 21/77] testing with sync --- .github/workflows/unit-testing.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index e8400d0ba8..65fb8455cb 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -1,6 +1,5 @@ name: unit-testing on: - push: pull_request: types: [opened, reopened, synchronize] jobs: @@ -37,7 +36,7 @@ jobs: git pull git checkout ${GITHUB_REF#refs/heads/} echo "The target branch for this PR is ${{ github.base_ref }}" - # contentctl test --disable-tqdm --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} + contentctl test --disable-tqdm --post-test-behavior never_pause mode:changes target-branch ${{ github.base_ref }} mkdir test_results mkdir artifacts cp summary.yml test_results/ From df6eddf81fe8fecfbb0be1bed0ba8ed9fbecc930 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 16:11:15 -0700 Subject: [PATCH 22/77] ref develop --- .github/workflows/unit-testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 65fb8455cb..b28be5bc24 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -10,8 +10,8 @@ jobs: steps: - name: Check out the repository code uses: actions/checkout@v3 - # with: - # ref: develop + with: + ref: develop - uses: actions/setup-python@v4 with: From 2092c5455d25c024976fedba0663a7ce09a27020 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 16:15:44 -0700 Subject: [PATCH 23/77] silent output --- .github/workflows/unit-testing.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index b28be5bc24..5da07aa83b 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -32,9 +32,8 @@ jobs: run: | echo "Current Branch (Head Ref): ${{ github.head_ref }}" echo "Target Branch (Base Ref): ${{ github.base_ref }}" - echo "reff - ${GITHUB_REF#refs/heads/}" - git pull - git checkout ${GITHUB_REF#refs/heads/} + git pull > /dev/null 2>&1 + git checkout ${{ github.head_ref }} echo "The target branch for this PR is ${{ github.base_ref }}" contentctl test --disable-tqdm --post-test-behavior never_pause mode:changes target-branch ${{ github.base_ref }} mkdir test_results From 513c6b2f2117ab020e5013754eb27a93c07a54d6 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 16:19:21 -0700 Subject: [PATCH 24/77] update ctl comand --- .github/workflows/unit-testing.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 5da07aa83b..9c43cd8c4c 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -18,15 +18,12 @@ jobs: python-version: '3.11' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified - - name: Install System Packages + - name: Install System Packages and contentctl run: | sudo apt update -qq sudo apt install jq -qq - - - name: Install Python Dependencies - run: | pip install contentctl - git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git + - name: Run ContentCTL test for changes against develop run: | @@ -35,7 +32,7 @@ jobs: git pull > /dev/null 2>&1 git checkout ${{ github.head_ref }} echo "The target branch for this PR is ${{ github.base_ref }}" - contentctl test --disable-tqdm --post-test-behavior never_pause mode:changes target-branch ${{ github.base_ref }} + contentctl test --disable-tqdm --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} mkdir test_results mkdir artifacts cp summary.yml test_results/ From 689e003c26da09a5fd6ccdd2e9467485a7f2dfd9 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 16:40:47 -0700 Subject: [PATCH 25/77] updating validate --- .github/workflows/validate-and-build.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/validate-and-build.yml b/.github/workflows/validate-and-build.yml index 6a0de77ed7..b405b86093 100644 --- a/.github/workflows/validate-and-build.yml +++ b/.github/workflows/validate-and-build.yml @@ -25,25 +25,21 @@ jobs: - name: Install Python Dependencies and ContentCTL and Atomic Red Team run: | - # python3.11 -m venv .venv - # source .venv/bin/activate pip install contentctl git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git - - name: content_ctl validate + - name: Running validate run: | - # source .venv/bin/activate contentctl validate - - name: contentctl generate + - name: Running build with enrichments run: | - # source .venv/bin/activate contentctl build --enrichments mkdir artifacts mv dist/DA-ESS-ContentUpdate-latest.tar.gz artifacts/ - name: store_artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: content-latest path: | From 0d0ba2704b9d7337870294755c6aeb8a3d23905a Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 16:48:26 -0700 Subject: [PATCH 26/77] updating scirpt --- .github/workflows/format_test_summary.sh | 4 ++-- .github/workflows/unit-testing.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/format_test_summary.sh b/.github/workflows/format_test_summary.sh index 730a84855b..3492ad014b 100755 --- a/.github/workflows/format_test_summary.sh +++ b/.github/workflows/format_test_summary.sh @@ -2,8 +2,8 @@ total_fail=$(yq e '.summary.total_fail' test_results/summary.yml) echo "Extracted total_fail: [$total_fail]" -# Check if total_fail is a valid integer and greater than one -if [[ "$total_fail" =~ ^[0-9]+$ ]] && [ "$total_fail" -gt 1 ]; then +# Check if total_fail is a valid integer and greater than or equal to one +if [[ "$total_fail" =~ ^[0-9]+$ ]] && [ "$total_fail" -ge 1 ]; then echo "CI Failure: There are failed tests." # Adjust the column widths here printf "%-80s | %-6s | %-10s\n" "Name" "Status" "Test Type" diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 9c43cd8c4c..0567d01ff1 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -32,7 +32,7 @@ jobs: git pull > /dev/null 2>&1 git checkout ${{ github.head_ref }} echo "The target branch for this PR is ${{ github.base_ref }}" - contentctl test --disable-tqdm --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} + contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} mkdir test_results mkdir artifacts cp summary.yml test_results/ From 255f7191188fa5f00025039ae964dbe76e969a2c Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 16:49:52 -0700 Subject: [PATCH 27/77] remove aritfacts dir --- .github/workflows/unit-testing.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 0567d01ff1..731129a34e 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -32,11 +32,8 @@ jobs: git pull > /dev/null 2>&1 git checkout ${{ github.head_ref }} echo "The target branch for this PR is ${{ github.base_ref }}" - contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} - mkdir test_results - mkdir artifacts - cp summary.yml test_results/ - cp test_results/summary.yml artifacts/ + contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} + # cp summary.yml test_results/ continue-on-error: true - name: store_artifacts From bc28aa25e199640ff288be76203376ed14ab0a75 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 17:02:18 -0700 Subject: [PATCH 28/77] udpating formatting --- .github/workflows/format_test_summary.sh | 4 ++-- .github/workflows/unit-testing.yml | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/format_test_summary.sh b/.github/workflows/format_test_summary.sh index 3492ad014b..56ebfb5fb9 100755 --- a/.github/workflows/format_test_summary.sh +++ b/.github/workflows/format_test_summary.sh @@ -16,9 +16,9 @@ if [[ "$total_fail" =~ ^[0-9]+$ ]] && [ "$total_fail" -ge 1 ]; then test_type=$(echo $test_type | xargs) # Trim whitespace if [ "$status" == "true" ]; then - printf "%-80s | \033[32m%-6s\033[0m | %-10s\n" "$name" "PASS" "$test_type" + printf "%-80s | %-6s | %-10s\n" "$name" "PASS" "$test_type" else - printf "%-80s | \033[31m%-6s\033[0m | %-10s\n" "$name" "FAIL" "$test_type" + printf "%-80s | %-6s | %-10s\n" "$name" "FAIL" "$test_type" fi done diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 731129a34e..96519f2c73 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -27,13 +27,13 @@ jobs: - name: Run ContentCTL test for changes against develop run: | - echo "Current Branch (Head Ref): ${{ github.head_ref }}" - echo "Target Branch (Base Ref): ${{ github.base_ref }}" - git pull > /dev/null 2>&1 - git checkout ${{ github.head_ref }} - echo "The target branch for this PR is ${{ github.base_ref }}" - contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} - # cp summary.yml test_results/ + # echo "Current Branch (Head Ref): ${{ github.head_ref }}" + # echo "Target Branch (Base Ref): ${{ github.base_ref }}" + # git pull > /dev/null 2>&1 + # git checkout ${{ github.head_ref }} + # echo "The target branch for this PR is ${{ github.base_ref }}" + # contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} + cp summary.yml test_results/ continue-on-error: true - name: store_artifacts From 1224b2e6322e3d2dd5b7858a5d6b5316099783ae Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 17:05:36 -0700 Subject: [PATCH 29/77] run full test --- .github/workflows/unit-testing.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 96519f2c73..4219a794c6 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -27,13 +27,12 @@ jobs: - name: Run ContentCTL test for changes against develop run: | - # echo "Current Branch (Head Ref): ${{ github.head_ref }}" - # echo "Target Branch (Base Ref): ${{ github.base_ref }}" - # git pull > /dev/null 2>&1 - # git checkout ${{ github.head_ref }} - # echo "The target branch for this PR is ${{ github.base_ref }}" - # contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} - cp summary.yml test_results/ + echo "Current Branch (Head Ref): ${{ github.head_ref }}" + echo "Target Branch (Base Ref): ${{ github.base_ref }}" + git pull > /dev/null 2>&1 + git checkout ${{ github.head_ref }} + echo "The target branch for this PR is ${{ github.base_ref }}" + contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} continue-on-error: true - name: store_artifacts @@ -47,8 +46,8 @@ jobs: - name: Print entire test_results/summary.yml run: cat test_results/summary.yml + continue-on-error: true - - name: Formatted Final Report run: | chmod +x .github/workflows/format_test_summary.sh From fdf9d9263914758652b25942ca2c2fc2f5fb5ed9 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 17:06:04 -0700 Subject: [PATCH 30/77] testing one passed detections --- detections/endpoint/access_lsass_memory_for_dump_creation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detections/endpoint/access_lsass_memory_for_dump_creation.yml b/detections/endpoint/access_lsass_memory_for_dump_creation.yml index 04d4d021a0..5e5a80f73a 100644 --- a/detections/endpoint/access_lsass_memory_for_dump_creation.yml +++ b/detections/endpoint/access_lsass_memory_for_dump_creation.yml @@ -1,6 +1,6 @@ name: Access LSASS Memory for Dump Creation id: fb4c31b0-13e8-4155-8aa5-24de4b8d6717 -version: 3 +version: 4 date: '2024-05-13' author: Patrick Bareiss, Splunk status: production From 3da355651104ec2f9855b139cc2b1bb82d9c38b7 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 17:46:44 -0700 Subject: [PATCH 31/77] testing one more time --- .github/workflows/unit-testing.yml | 4 ++-- .github/workflows/validate-and-build.yml | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 4219a794c6..0a8ec17481 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -3,7 +3,7 @@ on: pull_request: types: [opened, reopened, synchronize] jobs: - contentctl-unit-testing: + unit-testing: runs-on: ubuntu-latest if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job # needs: [validate-tag-if-present, quit-for-dependabot] @@ -36,7 +36,7 @@ jobs: continue-on-error: true - name: store_artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test_summary_results path: | diff --git a/.github/workflows/validate-and-build.yml b/.github/workflows/validate-and-build.yml index b405b86093..2ec11cab6a 100644 --- a/.github/workflows/validate-and-build.yml +++ b/.github/workflows/validate-and-build.yml @@ -1,8 +1,7 @@ name: validate-and-build on: - push: pull_request: - types: [opened, reopened] + types: [opened, reopened, synchronize] jobs: validate-and-build: #Note that the CircleCI job used a Container. The way to do this with Github Actions From a9acaa60eec0a1722756ef7a3e86e610b46f6745 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 18:26:27 -0700 Subject: [PATCH 32/77] testing this detection --- ...supply_chain_attack_network_indicators.yml | 2 +- .../7zip_commandline_to_smb_share_path.yml | 2 +- summary.yml | 370 ------------------ 3 files changed, 2 insertions(+), 372 deletions(-) delete mode 100644 summary.yml diff --git a/detections/endpoint/3cx_supply_chain_attack_network_indicators.yml b/detections/endpoint/3cx_supply_chain_attack_network_indicators.yml index e14aa34e3d..ac914d186e 100644 --- a/detections/endpoint/3cx_supply_chain_attack_network_indicators.yml +++ b/detections/endpoint/3cx_supply_chain_attack_network_indicators.yml @@ -1,6 +1,6 @@ name: 3CX Supply Chain Attack Network Indicators id: 791b727c-deec-4fbe-a732-756131b3c5a1 -version: 2 +version: 3 date: "2024-05-21" author: Michael Haag, Splunk type: TTP diff --git a/detections/endpoint/7zip_commandline_to_smb_share_path.yml b/detections/endpoint/7zip_commandline_to_smb_share_path.yml index 70d0e88420..76c046406f 100644 --- a/detections/endpoint/7zip_commandline_to_smb_share_path.yml +++ b/detections/endpoint/7zip_commandline_to_smb_share_path.yml @@ -15,7 +15,7 @@ description: The following analytic detects the execution of 7z or 7za processes data_source: - Sysmon EventID 1 search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) - as lastTime from datamodel=Endpoint.Processes where Processes.process_name ="7z.exe" + as lastTime from datamodel=Endpoint.Processes where (Processes.process_name ="7z.exe" OR Processes.process_name = "7za.exe" OR Processes.original_file_name = "7z.exe" OR Processes.original_file_name = "7za.exe") AND (Processes.process="*\\C$\\*" OR Processes.process="*\\Admin$\\*" OR Processes.process="*\\IPC$\\*") by Processes.original_file_name diff --git a/summary.yml b/summary.yml deleted file mode 100644 index cc437d9783..0000000000 --- a/summary.yml +++ /dev/null @@ -1,370 +0,0 @@ -summary: - success: false - total_detections: 11 - total_pass: 8 - total_fail: 3 - total_skipped: 0 - total_untested: 0 - total_experimental_or_deprecated: 0 - success_rate: 72.7% -tested_detections: -- name: Windows AD Replication Request Initiated from Unsanctioned Location - search: '`wineventlog_security` EventCode=4662 ObjectType IN ("%{19195a5b-6da0-11d0-afd3-00c04fd930c9}", - "domainDNS") AND Properties IN ("*Replicating Directory Changes All*", "*{1131f6ad-9c07-11d1-f79f-00c04fc2dcd2}*", - "*{9923a32a-3607-11d2-b9be-0000f87a36b2}*","*{1131f6ac-9c07-11d1-f79f-00c04fc2dcd2}*") - AND AccessMask="0x100" AND (SubjectUserSid="NT AUT*" OR SubjectUserSid="S-1-5-18" - OR SubjectDomainName="Window Manager" OR SubjectUserName="*$") | stats min(_time) - as attack_time, count by SubjectDomainName, SubjectUserName, Computer, Logon_ID, - ObjectName, ObjectServer, ObjectType, OperationType, status | rename SubjectDomainName - as Target_Domain, SubjectUserName as user, Logon_ID as TargetLogonId | appendpipe - [| map search="search `wineventlog_security` EventCode=4624 TargetLogonId=$TargetLogonId$"] - | table attack_time, AuthenticationPackageName, LogonProcessName, LogonType, TargetUserSid, - Target_Domain, user, Computer, TargetLogonId, status, src_ip, src_category, ObjectName, - ObjectServer, ObjectType, OperationType | stats min(attack_time) as _time, values(TargetUserSid) - as TargetUserSid, values(Target_Domain) as Target_Domain, values(user) as user, - values(Computer) as Computer, values(status) as status, values(src_category) as - src_category, values(src_ip) as src_ip by TargetLogonId | search NOT src_category="domain_controller" - | `windows_ad_replication_request_initiated_from_unsanctioned_location_filter`' - success: false - tests: - - name: True Positive Test - test_type: unit - success: false - message: TEST ERROR - exception: The observable field(s) {'src_ip'} are missing in the detection results - status: error - duration: 6.49 - wait_duration: null - resultCount: '1' - runDuration: '1.320' - - name: True Positive Test - test_type: integration - success: false - message: 'TEST FAILED (PREEMPTIVE): associated unit test failed or encountered - an error' - exception: The observable field(s) {'src_ip'} are missing in the detection results - status: error - duration: 3.08 - wait_duration: null - resultCount: null - runDuration: null -- name: Windows Admon Default Group Policy Object Modified - search: ' `admon` admonEventType=Update objectCategory="CN=Group-Policy-Container,CN=Schema,CN=Configuration,DC=*" - (displayName="Default Domain Policy" OR displayName="Default Domain Controllers - Policy") | appendpipe [ | map search="search `wineventlog_security` EventCode=5136 AttributeSyntaxOID=2.5.5.12 - AttributeValue=$displayName$" | rename AttributeValue as displayName] | stats - min(_time) as _time values(displayName) as gp_name, values(gPCFileSysPath) as - gPCFileSysPath, values(src_user) as src_user, values(dcName) as dcName, values(dest_category) - as dest_category, values(src_user_category) as src_user_category by displayName - | `windows_admon_default_group_policy_object_modified_filter`' - success: false - tests: - - name: True Positive Test - test_type: unit - success: false - message: TEST ERROR - exception: The observable field(s) {'src_user'} are missing in the detection results - status: error - duration: 7.63 - wait_duration: null - resultCount: '1' - runDuration: '2.158' - - name: True Positive Test - test_type: integration - success: false - message: 'TEST FAILED (PREEMPTIVE): associated unit test failed or encountered - an error' - exception: The observable field(s) {'src_user'} are missing in the detection results - status: error - duration: 3.17 - wait_duration: null - resultCount: null - runDuration: null -- name: Windows Admon Group Policy Object Created - search: ' `admon` admonEventType=Update objectCategory="CN=Group-Policy-Container,CN=Schema,CN=Configuration,DC=*" - versionNumber=0 displayName!="New Group Policy Object" | appendpipe [ | map search="search - `wineventlog_security` EventCode=5136 AttributeSyntaxOID=2.5.5.12 AttributeValue=$displayName$" - | rename AttributeValue as displayName] | stats min(_time) as _time values(displayName) - as gp_name, values(gPCFileSysPath) as gPCFileSysPath, values(src_user) as src_user, - values(dcName) as dcName, values(dest_category) as dest_category, values(src_user_category) - as src_user_category by displayName | `windows_admon_group_policy_object_created_filter`' - success: false - tests: - - name: True Positive Test - test_type: unit - success: false - message: TEST ERROR - exception: The observable field(s) {'src_user'} are missing in the detection results - status: error - duration: 11.45 - wait_duration: null - resultCount: '1' - runDuration: '1.270' - - name: True Positive Test - test_type: integration - success: false - message: 'TEST FAILED (PREEMPTIVE): associated unit test failed or encountered - an error' - exception: The observable field(s) {'src_user'} are missing in the detection results - status: error - duration: 3.26 - wait_duration: null - resultCount: null - runDuration: null -- name: Azure AD Admin Consent Bypassed by Service Principal - search: '`azure_monitor_aad` (operationName="Add app role assignment to service - principal" OR operationName="Add member to role*") src_user_type=servicePrincipal | - rename properties.* as * | eval roleId = mvindex(''targetResources{}.modifiedProperties{}.newValue'', - 0) | eval roleValue = mvindex(''targetResources{}.modifiedProperties{}.newValue'', - 1) | eval roleDescription = mvindex(''targetResources{}.modifiedProperties{}.newValue'', - 2) | eval user_id = mvindex(''targetResources{}.id'', 0), user=coalesce(user,mvindex(''targetResources{}.displayName'', - 0)) | rename initiatedBy.app.displayName as src_user | stats count earliest(_time) - as firstTime latest(_time) as lastTime by src_user user user_id roleId roleValue - roleDescription | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | - `azure_ad_admin_consent_bypassed_by_service_principal_filter`' - success: true - tests: - - name: True Positive Test - test_type: unit - success: true - message: TEST PASSED - exception: null - status: pass - duration: 5.84 - wait_duration: null - resultCount: '1' - runDuration: '0.654' - - name: True Positive Test - test_type: integration - success: true - message: 'TEST SKIPPED: Skipping all integration tests' - exception: null - status: skip - duration: 0 - wait_duration: null - resultCount: null - runDuration: null -- name: Azure AD Global Administrator Role Assigned - search: '`azure_monitor_aad` operationName="Add member to role" properties.targetResources{}.modifiedProperties{}.newValue="\"Global - Administrator\"" | rename properties.* as *, initiatedBy.user.userPrincipalName - as userPrincipalName, targetResources{}.displayName as displayName | eval initiatedBy - = coalesce(userPrincipalName,src_user) | eval user = coalesce(user,mvfilter(displayName!="null")) - | stats count min(_time) as firstTime max(_time) as lastTime values(user) as user - by initiatedBy, result, operationName | `security_content_ctime(firstTime)` | - `security_content_ctime(lastTime)` | `azure_ad_global_administrator_role_assigned_filter`' - success: true - tests: - - name: True Positive Test - test_type: unit - success: true - message: TEST PASSED - exception: null - status: pass - duration: 5.97 - wait_duration: null - resultCount: '1' - runDuration: '0.666' - - name: True Positive Test - test_type: integration - success: true - message: 'TEST SKIPPED: Skipping all integration tests' - exception: null - status: skip - duration: 0 - wait_duration: null - resultCount: null - runDuration: null -- name: Azure AD Privileged Role Assigned - search: ' `azure_monitor_aad` "operationName"="Add member to role" | rename properties.* - as *, initiatedBy.user.userPrincipalName as userPrincipalName, targetResources{}.displayName - as displayName | eval initiatedBy = coalesce(userPrincipalName,src_user) | eval - user = coalesce(user,mvfilter(displayName!="null")) | rename targetResources{}.modifiedProperties{}.newValue as - roles | eval role=mvindex(roles,1) | stats count min(_time) as firstTime max(_time) - as lastTime values(user) as user by initiatedBy, result, operationName, role | - lookup privileged_azure_ad_roles azureadrole AS role OUTPUT isprvilegedadrole - description | search isprvilegedadrole = True | `security_content_ctime(firstTime)` - | `security_content_ctime(lastTime)` | `azure_ad_privileged_role_assigned_filter`' - success: true - tests: - - name: True Positive Test - test_type: unit - success: true - message: TEST PASSED - exception: null - status: pass - duration: 5.94 - wait_duration: null - resultCount: '1' - runDuration: '0.648' - - name: True Positive Test - test_type: integration - success: true - message: 'TEST SKIPPED: Skipping all integration tests' - exception: null - status: skip - duration: 0 - wait_duration: null - resultCount: null - runDuration: null -- name: Azure AD Privileged Role Assigned to Service Principal - search: ' `azure_monitor_aad` operationName="Add member to role" | rename properties.* - as * | search "targetResources{}.type"=ServicePrincipal | rename initiatedBy.user.userPrincipalName - as initiatedBy | rename targetResources{}.modifiedProperties{}.newValue as roles - | eval role=mvindex(roles,1) | rename targetResources{}.displayName as apps | - eval displayName=mvindex(apps,0) | stats count min(_time) as firstTime max(_time) - as lastTime values(displayName) as displayName by initiatedBy, result, operationName, - role | lookup privileged_azure_ad_roles azureadrole AS role OUTPUT isprvilegedadrole - description | search isprvilegedadrole = True | `security_content_ctime(firstTime)` - | `security_content_ctime(lastTime)` | `azure_ad_privileged_role_assigned_to_service_principal_filter`' - success: true - tests: - - name: True Positive Test - test_type: unit - success: true - message: TEST PASSED - exception: null - status: pass - duration: 9.3 - wait_duration: null - resultCount: '1' - runDuration: '3.134' - - name: True Positive Test - test_type: integration - success: true - message: 'TEST SKIPPED: Skipping all integration tests' - exception: null - status: skip - duration: 0 - wait_duration: null - resultCount: null - runDuration: null -- name: Azure AD Service Principal New Client Credentials - search: ' `azure_monitor_aad` category=AuditLogs operationName="Update application*Certificates - and secrets management*" | rename properties.* as * | rename targetResources{}.* - as * | rename modifiedProperties{}.* as * | eval src_user=coalesce(user,identity), - newValue=mvfilter(newValue!="\"KeyDescription\"") | stats count min(_time) as - firstTime max(_time) as lastTime values(displayName) as displayName values(src_ip) - as src_ip values(eval(mvfilter(oldValue!="null"))) as oldValue by src_user, object, - newValue | spath input=oldValue output=oldValues path={} | spath input=newValue - output=newValues path={} | mvexpand newValues | where NOT newValues IN (oldValues) - | fields - newValue, oldValue, oldValues | rename newValues as newValue | `security_content_ctime(firstTime)` - | `security_content_ctime(lastTime)` | `azure_ad_service_principal_new_client_credentials_filter`' - success: true - tests: - - name: True Positive Test - test_type: unit - success: true - message: TEST PASSED - exception: null - status: pass - duration: 5.94 - wait_duration: null - resultCount: '1' - runDuration: '0.782' - - name: True Positive Test - test_type: integration - success: true - message: 'TEST SKIPPED: Skipping all integration tests' - exception: null - status: skip - duration: 0 - wait_duration: null - resultCount: null - runDuration: null -- name: Detect New Local Admin account - search: '`wineventlog_security` (EventCode=4720) OR (EventCode=4732 Group_Name=Administrators) - | stats dc(EventCode) as evCount min(_time) as _time range(_time) as duration - values(src_user) as src_user values(src_user_category) as src_user_category values(dest_category) - as dest_category by user dest | where evCount=2 | fields - evCount, duration | - `detect_new_local_admin_account_filter`' - success: true - tests: - - name: True Positive Test - test_type: unit - success: true - message: TEST PASSED - exception: null - status: pass - duration: 14.33 - wait_duration: null - resultCount: '1' - runDuration: '0.696' - - name: True Positive Test - test_type: integration - success: true - message: 'TEST SKIPPED: Skipping all integration tests' - exception: null - status: skip - duration: 0 - wait_duration: null - resultCount: null - runDuration: null -- name: Kerberos Pre-Authentication Flag Disabled in UserAccountControl - search: '`wineventlog_security` EventCode=4738 (UserAccountControl="%%2096" OR MSADChangedAttributes="*Don''t - Require Preauth'' - Enabled*") | eval MSADChangedAttributes="''Don''t Require - Preauth'' - Enabled" | table _time, source, EventCode, src_user, src_user_category, - user, user_category, MSADChangedAttributes | `kerberos_pre_authentication_flag_disabled_in_useraccountcontrol_filter`' - success: true - tests: - - name: True Positive Test - test_type: unit - success: true - message: TEST PASSED - exception: null - status: pass - duration: 6.08 - wait_duration: null - resultCount: '1' - runDuration: '0.630' - - name: True Positive Test - test_type: integration - success: true - message: 'TEST SKIPPED: Skipping all integration tests' - exception: null - status: skip - duration: 0 - wait_duration: null - resultCount: null - runDuration: null -- name: Windows AD Replication Request Initiated by User Account - search: '`wineventlog_security` EventCode=4662 ObjectType IN ("%{19195a5b-6da0-11d0-afd3-00c04fd930c9}", - "domainDNS") AND Properties IN ("*Replicating Directory Changes All*", "*{1131f6ad-9c07-11d1-f79f-00c04fc2dcd2}*", - "*{9923a32a-3607-11d2-b9be-0000f87a36b2}*","*{1131f6ac-9c07-11d1-f79f-00c04fc2dcd2}*") - AND AccessMask="0x100" AND NOT (SubjectUserSid="NT AUT*" OR SubjectUserSid="S-1-5-18" - OR SubjectDomainName="Window Manager" OR SubjectUserName="*$") | stats min(_time) - as _time, count by SubjectDomainName, SubjectUserName, Computer, Logon_ID, ObjectName, - ObjectServer, ObjectType, OperationType, status | rename SubjectDomainName as - Target_Domain, SubjectUserName as user, Logon_ID as TargetLogonId, _time as attack_time - | appendpipe [| map search="search `wineventlog_security` EventCode=4624 TargetLogonId=$TargetLogonId$"] - | table attack_time, AuthenticationPackageName, LogonProcessName, LogonType, TargetUserSid, - Target_Domain, user, Computer, TargetLogonId, status, src_ip, src_category, ObjectName, - ObjectServer, ObjectType, OperationType | stats min(attack_time) as _time values(TargetUserSid) - as TargetUserSid, values(Target_Domain) as Target_Domain, values(user) as user, - values(Computer) as Computer, values(status) as status, values(src_category) as - src_category, values(src_ip) as src_ip by TargetLogonId | `windows_ad_replication_request_initiated_by_user_account_filter`' - success: true - tests: - - name: True Positive Test - test_type: unit - success: true - message: TEST PASSED - exception: null - status: pass - duration: 11.26 - wait_duration: null - resultCount: '1' - runDuration: '1.289' - - name: True Positive Test - test_type: integration - success: true - message: 'TEST SKIPPED: Skipping all integration tests' - exception: null - status: skip - duration: 0 - wait_duration: null - resultCount: null - runDuration: null -untested_detections: [] -percent_complete: UKNOWN -deprecated_detections: [] -experimental_detections: [] From 936ede8cb5651f0d4973f8f5195e5d2a3c51a299 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 18:47:26 -0700 Subject: [PATCH 33/77] CI trigger From 62c75b3417a20bab2c18a2d55d6e44087001be5e Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 19:09:59 -0700 Subject: [PATCH 34/77] fixing warning and testing a detection --- .github/workflows/unit-testing.yml | 2 ++ .github/workflows/validate-and-build.yml | 3 +++ ...bnormally_high_number_of_cloud_infrastructure_api_calls.yml | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 0a8ec17481..d3b59a8979 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -12,11 +12,13 @@ jobs: uses: actions/checkout@v3 with: ref: develop + node-version: '20' - uses: actions/setup-python@v4 with: python-version: '3.11' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified + node-version: '20' - name: Install System Packages and contentctl run: | diff --git a/.github/workflows/validate-and-build.yml b/.github/workflows/validate-and-build.yml index 2ec11cab6a..bea48cc794 100644 --- a/.github/workflows/validate-and-build.yml +++ b/.github/workflows/validate-and-build.yml @@ -11,11 +11,14 @@ jobs: steps: - name: Check out the repository code uses: actions/checkout@v3 + with: + node-version: '20' - uses: actions/setup-python@v4 with: python-version: '3.11' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified + node-version: '20' - name: Install System Packages run: | diff --git a/detections/cloud/abnormally_high_number_of_cloud_infrastructure_api_calls.yml b/detections/cloud/abnormally_high_number_of_cloud_infrastructure_api_calls.yml index 343122859f..7d42fbf457 100644 --- a/detections/cloud/abnormally_high_number_of_cloud_infrastructure_api_calls.yml +++ b/detections/cloud/abnormally_high_number_of_cloud_infrastructure_api_calls.yml @@ -3,7 +3,7 @@ id: 0840ddf1-8c89-46ff-b730-c8d6722478c0 version: 2 date: '2024-05-12' author: David Dorsey, Splunk -status: experimental +status: production type: Anomaly description: The following analytic detects a spike in the number of API calls made to your cloud infrastructure by a user. It leverages cloud infrastructure logs and From 5825cd777637f85c33817df4b2355d3e78a3edd9 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Mon, 8 Jul 2024 19:13:16 -0700 Subject: [PATCH 35/77] testing another detection --- .../application/crushftp_server_side_template_injection.yml | 2 +- ...abnormally_high_number_of_cloud_infrastructure_api_calls.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/detections/application/crushftp_server_side_template_injection.yml b/detections/application/crushftp_server_side_template_injection.yml index 71c70e6b6c..6b593489eb 100644 --- a/detections/application/crushftp_server_side_template_injection.yml +++ b/detections/application/crushftp_server_side_template_injection.yml @@ -5,7 +5,7 @@ date: '2024-05-16' author: Michael Haag, Splunk data_source: [] type: TTP -status: production +status: experimental description: This analytic is designed to identify attempts to exploit a server-side template injection vulnerability in CrushFTP, designated as CVE-2024-4040. This severe vulnerability enables unauthenticated remote attackers to access and read files beyond the VFS Sandbox, circumvent authentication protocols, and execute arbitrary commands on the affected server. The issue impacts all versions of CrushFTP up to 10.7.1 and 11.1.0 on all supported platforms. It is highly recommended to apply patches immediately to prevent unauthorized access to the system and avoid potential data compromises. The search specifically looks for patterns in the raw log data that match the exploitation attempts, including READ or WRITE actions, and extracts relevant information such as the protocol, session ID, user, IP address, HTTP method, and the URI queried. It then evaluates these logs to confirm traces of exploitation based on the presence of specific keywords and the originating IP address, counting and sorting these events for further analysis. search: '`crushftp` | rex field=_raw "\[(?HTTPS|HTTP):(?[^\:]+):(?[^\:]+):(?\d+\.\d+\.\d+\.\d+)\] (?READ|WROTE): \*(?[A-Z]+) (?[^\s]+) HTTP/[^\*]+\*" diff --git a/detections/cloud/abnormally_high_number_of_cloud_infrastructure_api_calls.yml b/detections/cloud/abnormally_high_number_of_cloud_infrastructure_api_calls.yml index 7d42fbf457..343122859f 100644 --- a/detections/cloud/abnormally_high_number_of_cloud_infrastructure_api_calls.yml +++ b/detections/cloud/abnormally_high_number_of_cloud_infrastructure_api_calls.yml @@ -3,7 +3,7 @@ id: 0840ddf1-8c89-46ff-b730-c8d6722478c0 version: 2 date: '2024-05-12' author: David Dorsey, Splunk -status: production +status: experimental type: Anomaly description: The following analytic detects a spike in the number of API calls made to your cloud infrastructure by a user. It leverages cloud infrastructure logs and From 561ab4fa4642fd0204d7397b7cba446d42435b06 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 07:18:16 -0700 Subject: [PATCH 36/77] testing another detection --- .github/workflows/unit-testing.yml | 3 +-- .../application/crushftp_server_side_template_injection.yml | 2 +- .../endpoint/3cx_supply_chain_attack_network_indicators.yml | 2 +- detections/endpoint/account_discovery_with_net_app.yml | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index d3b59a8979..ef3cfbf30e 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -9,7 +9,7 @@ jobs: # needs: [validate-tag-if-present, quit-for-dependabot] steps: - name: Check out the repository code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: develop node-version: '20' @@ -18,7 +18,6 @@ jobs: with: python-version: '3.11' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified - node-version: '20' - name: Install System Packages and contentctl run: | diff --git a/detections/application/crushftp_server_side_template_injection.yml b/detections/application/crushftp_server_side_template_injection.yml index 6b593489eb..71c70e6b6c 100644 --- a/detections/application/crushftp_server_side_template_injection.yml +++ b/detections/application/crushftp_server_side_template_injection.yml @@ -5,7 +5,7 @@ date: '2024-05-16' author: Michael Haag, Splunk data_source: [] type: TTP -status: experimental +status: production description: This analytic is designed to identify attempts to exploit a server-side template injection vulnerability in CrushFTP, designated as CVE-2024-4040. This severe vulnerability enables unauthenticated remote attackers to access and read files beyond the VFS Sandbox, circumvent authentication protocols, and execute arbitrary commands on the affected server. The issue impacts all versions of CrushFTP up to 10.7.1 and 11.1.0 on all supported platforms. It is highly recommended to apply patches immediately to prevent unauthorized access to the system and avoid potential data compromises. The search specifically looks for patterns in the raw log data that match the exploitation attempts, including READ or WRITE actions, and extracts relevant information such as the protocol, session ID, user, IP address, HTTP method, and the URI queried. It then evaluates these logs to confirm traces of exploitation based on the presence of specific keywords and the originating IP address, counting and sorting these events for further analysis. search: '`crushftp` | rex field=_raw "\[(?HTTPS|HTTP):(?[^\:]+):(?[^\:]+):(?\d+\.\d+\.\d+\.\d+)\] (?READ|WROTE): \*(?[A-Z]+) (?[^\s]+) HTTP/[^\*]+\*" diff --git a/detections/endpoint/3cx_supply_chain_attack_network_indicators.yml b/detections/endpoint/3cx_supply_chain_attack_network_indicators.yml index ac914d186e..e14aa34e3d 100644 --- a/detections/endpoint/3cx_supply_chain_attack_network_indicators.yml +++ b/detections/endpoint/3cx_supply_chain_attack_network_indicators.yml @@ -1,6 +1,6 @@ name: 3CX Supply Chain Attack Network Indicators id: 791b727c-deec-4fbe-a732-756131b3c5a1 -version: 3 +version: 2 date: "2024-05-21" author: Michael Haag, Splunk type: TTP diff --git a/detections/endpoint/account_discovery_with_net_app.yml b/detections/endpoint/account_discovery_with_net_app.yml index 2f4a8fd313..ce9e97269f 100644 --- a/detections/endpoint/account_discovery_with_net_app.yml +++ b/detections/endpoint/account_discovery_with_net_app.yml @@ -1,6 +1,6 @@ name: Account Discovery With Net App id: 339805ce-ac30-11eb-b87d-acde48001122 -version: 5 +version: 6 date: '2024-05-22' author: Teoderick Contreras, Splunk, TheLawsOfChaos, Github Community status: production From cf25e65c64684c89102b6c510607c85ed63aee8e Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 07:41:08 -0700 Subject: [PATCH 37/77] testing with python --- .github/workflows/format_test_results.py | 42 +++++++++++++++++++ .github/workflows/unit-testing.yml | 5 ++- ...vanti_connect_secure_bookmark_endpoint.yml | 2 +- ...adobe_coldfusion_access_control_bypass.yml | 2 +- 4 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/format_test_results.py diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py new file mode 100644 index 0000000000..55ee78bd26 --- /dev/null +++ b/.github/workflows/format_test_results.py @@ -0,0 +1,42 @@ +""" +A simple script formatting test_results/summary.yml to display on github actions +""" + +import yaml +import re + +def main(): + # Load the YAML file + with open('test_results/summary.yml', 'r') as file: + data = yaml.safe_load(file) + + # Extract total_fail value and debug print it + total_fail = data['summary']['total_fail'] + print(f"Extracted total_fail: [{total_fail}]") + + # Print all unit test details first + print("Unit Test Details:") + print(f"{'Name':<80} | {'Status':<6} | {'Test Type':<10} | {'Exception':<50}") + print(f"{'----':<80} | {'------':<6} | {'---------':<10} | {'---------':<50}") + for detection in data['tested_detections']: + for test in detection['tests']: + if test['test_type'].strip() == "unit": # Check if the test type is "unit" + name = detection['name'].strip() + status = 'PASS' if test['success'] else 'FAIL' + test_type = test['test_type'].strip() + exception = test.get('exception', 'N/A') # Get exception if exists, else 'N/A' + if status == 'FAIL': + print(f"{name:<80} | {status:<6} | {test_type:<10} | {exception:<50}") + else: + print(f"{name:<80} | {status:<6} | {test_type:<10} | {'-':<50}") + + # Check if total_fail is a valid integer and greater than or equal to one + if re.match(r'^[0-9]+$', str(total_fail)) and int(total_fail) >= 1: + print("CI Failure: There are failed tests.") + exit(1) # Fail the CI job + else: + print("CI Success: No failed tests.\n\n") + + print("Download the job artifacts of this run and view complete summary in test_results/summary.yml for troubleshooting failures .\n") +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index ef3cfbf30e..ccabc836e2 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -51,5 +51,6 @@ jobs: - name: Formatted Final Report run: | - chmod +x .github/workflows/format_test_summary.sh - ./.github/workflows/format_test_summary.sh >> $GITHUB_STEP_SUMMARY \ No newline at end of file + pip install pyyaml + pip install re + python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/detections/web/access_to_vulnerable_ivanti_connect_secure_bookmark_endpoint.yml b/detections/web/access_to_vulnerable_ivanti_connect_secure_bookmark_endpoint.yml index 2ee10a8f52..e9e20c80d9 100644 --- a/detections/web/access_to_vulnerable_ivanti_connect_secure_bookmark_endpoint.yml +++ b/detections/web/access_to_vulnerable_ivanti_connect_secure_bookmark_endpoint.yml @@ -1,6 +1,6 @@ name: Access to Vulnerable Ivanti Connect Secure Bookmark Endpoint id: 15838756-f425-43fa-9d88-a7f88063e81a -version: 2 +version: 3 date: '2024-05-14' author: Michael Haag, Splunk status: production diff --git a/detections/web/adobe_coldfusion_access_control_bypass.yml b/detections/web/adobe_coldfusion_access_control_bypass.yml index 5828ebffb2..f344db1e74 100644 --- a/detections/web/adobe_coldfusion_access_control_bypass.yml +++ b/detections/web/adobe_coldfusion_access_control_bypass.yml @@ -15,7 +15,7 @@ description: The following analytic detects potential exploitation attempts agai unauthorized access to ColdFusion administration endpoints. If confirmed malicious, this could result in data theft, brute force attacks, or further exploitation of other vulnerabilities, posing a serious security risk to the environment. -search: '| tstats count min(_time) as firstTime max(_time) as lastTime from datamodel=Web +search: 'x| tstats count min(_time) as firstTime max(_time) as lastTime from datamodel=Web where Web.url IN ("//restplay*", "//CFIDE/restplay*", "//CFIDE/administrator*", "//CFIDE/adminapi*", "//CFIDE/main*", "//CFIDE/componentutils*", "//CFIDE/wizards*", "//CFIDE/servermanager*","/restplay*", "/CFIDE/restplay*", "/CFIDE/administrator*", From ff20804c6442a54e2bbfebcd2a0a4e03ec1dca2d Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 07:56:41 -0700 Subject: [PATCH 38/77] format script update --- .github/workflows/unit-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index ccabc836e2..09f2f13215 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -51,6 +51,6 @@ jobs: - name: Formatted Final Report run: | + python -m pip install --upgrade pip pip install pyyaml - pip install re python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY \ No newline at end of file From 9f6ea7073b11ae5b302217da476d9077e984036e Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 08:17:50 -0700 Subject: [PATCH 39/77] testing experiemental --- .../endpoint/3cx_supply_chain_attack_network_indicators.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detections/endpoint/3cx_supply_chain_attack_network_indicators.yml b/detections/endpoint/3cx_supply_chain_attack_network_indicators.yml index e14aa34e3d..d775d00161 100644 --- a/detections/endpoint/3cx_supply_chain_attack_network_indicators.yml +++ b/detections/endpoint/3cx_supply_chain_attack_network_indicators.yml @@ -15,7 +15,7 @@ description: The following analytic identifies DNS queries to domains associated malicious, this activity could allow attackers to establish a foothold in the network, exfiltrate sensitive data, or further propagate malware, leading to extensive damage and data breaches. -search: '| tstats `security_content_summariesonly` values(DNS.answer) as IPs min(_time) +search: 'x| tstats `security_content_summariesonly` values(DNS.answer) as IPs min(_time) as firstTime from datamodel=Network_Resolution by DNS.src, DNS.query | `drop_dm_object_name(DNS)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | lookup 3cx_ioc_domains domain as query OUTPUT Description isIOC | search isIOC=true | `3cx_supply_chain_attack_network_indicators_filter`' From 2d099c2ce78e0272e472817e93674c903ea94343 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 08:23:54 -0700 Subject: [PATCH 40/77] fixing path --- .github/workflows/format_test_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index 55ee78bd26..9e788fae6c 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -7,7 +7,7 @@ def main(): # Load the YAML file - with open('test_results/summary.yml', 'r') as file: + with open('/home/runner/work/security_content/security_content/test_results/summary.yml', 'r') as file: data = yaml.safe_load(file) # Extract total_fail value and debug print it From 5ec9d4c7bb8877a414ddfc39bee8d24953f6c277 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 08:54:01 -0700 Subject: [PATCH 41/77] warnings --- .github/workflows/format_test_results.py | 23 ++++++++++++++++++----- .github/workflows/unit-testing.yml | 1 - 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index 9e788fae6c..5563da92f7 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -4,18 +4,28 @@ import yaml import re +import os def main(): + + # Define the path to the YAML file + yaml_file_path = 'test_results/summary.yml' + + # Check if the YAML file exists + if not os.path.exists(yaml_file_path): + print(f"Error: The file {yaml_file_path} does not exist.") + exit(1) # Exit with an error code + # Load the YAML file - with open('/home/runner/work/security_content/security_content/test_results/summary.yml', 'r') as file: + with open(yaml_file_path, 'r') as file: data = yaml.safe_load(file) # Extract total_fail value and debug print it total_fail = data['summary']['total_fail'] - print(f"Extracted total_fail: [{total_fail}]") + print(f"Extracted total_fail: [{total_fail}]\n") # Print all unit test details first - print("Unit Test Details:") + print("Unit Test Details:\n") print(f"{'Name':<80} | {'Status':<6} | {'Test Type':<10} | {'Exception':<50}") print(f"{'----':<80} | {'------':<6} | {'---------':<10} | {'---------':<50}") for detection in data['tested_detections']: @@ -32,11 +42,14 @@ def main(): # Check if total_fail is a valid integer and greater than or equal to one if re.match(r'^[0-9]+$', str(total_fail)) and int(total_fail) >= 1: - print("CI Failure: There are failed tests.") + # Print the message in bold + print("\033[1m\nCI Failure: There are failed tests.\n\033[0m") + print("\033[1mDownload the job artifacts of this run and view complete summary in test_results/summary.yml for troubleshooting failures.\n\033[0m") exit(1) # Fail the CI job else: print("CI Success: No failed tests.\n\n") - print("Download the job artifacts of this run and view complete summary in test_results/summary.yml for troubleshooting failures .\n") + + if __name__ == "__main__": main() \ No newline at end of file diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 09f2f13215..3033961b99 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -12,7 +12,6 @@ jobs: uses: actions/checkout@v4 with: ref: develop - node-version: '20' - uses: actions/setup-python@v4 with: From f225d2692a7165c5594fe99843ce72593f2aaf44 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 09:06:09 -0700 Subject: [PATCH 42/77] update file path --- .github/workflows/format_test_results.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index 5563da92f7..dc7f4838d5 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -9,7 +9,8 @@ def main(): # Define the path to the YAML file - yaml_file_path = 'test_results/summary.yml' + #yaml_file_path = 'test_results/summary.yml' + yaml_file_path = '/home/runner/work/security_content/security_content/test_results/summary.yml' # Check if the YAML file exists if not os.path.exists(yaml_file_path): From 34f09dcd908c42009a9863b6620cbbc4ff892d43 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 09:06:40 -0700 Subject: [PATCH 43/77] remove file --- .github/workflows/format_test_summary.sh | 28 ------------------------ 1 file changed, 28 deletions(-) delete mode 100755 .github/workflows/format_test_summary.sh diff --git a/.github/workflows/format_test_summary.sh b/.github/workflows/format_test_summary.sh deleted file mode 100755 index 56ebfb5fb9..0000000000 --- a/.github/workflows/format_test_summary.sh +++ /dev/null @@ -1,28 +0,0 @@ -# Extract total_fail value and debug print it -total_fail=$(yq e '.summary.total_fail' test_results/summary.yml) -echo "Extracted total_fail: [$total_fail]" - -# Check if total_fail is a valid integer and greater than or equal to one -if [[ "$total_fail" =~ ^[0-9]+$ ]] && [ "$total_fail" -ge 1 ]; then - echo "CI Failure: There are failed tests." - # Adjust the column widths here - printf "%-80s | %-6s | %-10s\n" "Name" "Status" "Test Type" - printf "%-80s | %-6s | %-10s\n" "----" "------" "---------" - - # Loop through each item in tested_detections and print required fields with color only for unit testing - yq e '.tested_detections[] | .name as $name | .tests[] | select(.test_type == "unit") | "\($name) | \(.success) | \(.test_type)"' test_results/summary.yml | while IFS='|' read -r name status test_type; do - name=$(echo $name | xargs) # Trim whitespace - status=$(echo $status | xargs) # Trim whitespace - test_type=$(echo $test_type | xargs) # Trim whitespace - - if [ "$status" == "true" ]; then - printf "%-80s | %-6s | %-10s\n" "$name" "PASS" "$test_type" - else - printf "%-80s | %-6s | %-10s\n" "$name" "FAIL" "$test_type" - fi - done - - exit 1 # Fail the CI job -else - echo "CI Success: No failed tests." -fi \ No newline at end of file From 256574259bd446e0f55d8ea06549324ca108b82b Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 09:18:55 -0700 Subject: [PATCH 44/77] test stder --- .github/workflows/format_test_results.py | 9 +++------ .github/workflows/unit-testing.yml | 3 +-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index dc7f4838d5..0f5a9e21fa 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -23,6 +23,7 @@ def main(): # Extract total_fail value and debug print it total_fail = data['summary']['total_fail'] + print("\033[1mDownload the job artifacts of this run and view complete summary in test_results/summary.yml for troubleshooting failures.\n\033[0m") print(f"Extracted total_fail: [{total_fail}]\n") # Print all unit test details first @@ -44,13 +45,9 @@ def main(): # Check if total_fail is a valid integer and greater than or equal to one if re.match(r'^[0-9]+$', str(total_fail)) and int(total_fail) >= 1: # Print the message in bold - print("\033[1m\nCI Failure: There are failed tests.\n\033[0m") - print("\033[1mDownload the job artifacts of this run and view complete summary in test_results/summary.yml for troubleshooting failures.\n\033[0m") - exit(1) # Fail the CI job + print("CI Failure: There are failed tests.\n") else: print("CI Success: No failed tests.\n\n") - - - + if __name__ == "__main__": main() \ No newline at end of file diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 3033961b99..5a15d8a326 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -51,5 +51,4 @@ jobs: - name: Formatted Final Report run: | python -m pip install --upgrade pip - pip install pyyaml - python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY \ No newline at end of file + python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY 2>&1 \ No newline at end of file From d26ede11d0f52a911fd8b0bbbf6c690a6619010d Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 09:28:39 -0700 Subject: [PATCH 45/77] re adding summary --- .github/workflows/format_test_results.py | 2 + .github/workflows/unit-testing.yml | 25 +- summary.yml | 370 +++++++++++++++++++++++ 3 files changed, 389 insertions(+), 8 deletions(-) create mode 100644 summary.yml diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index 0f5a9e21fa..150f804461 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -46,8 +46,10 @@ def main(): if re.match(r'^[0-9]+$', str(total_fail)) and int(total_fail) >= 1: # Print the message in bold print("CI Failure: There are failed tests.\n") + sys.exit(0) else: print("CI Success: No failed tests.\n\n") + sys.exit(1) if __name__ == "__main__": main() \ No newline at end of file diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 5a15d8a326..00e1509e87 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -27,12 +27,13 @@ jobs: - name: Run ContentCTL test for changes against develop run: | - echo "Current Branch (Head Ref): ${{ github.head_ref }}" - echo "Target Branch (Base Ref): ${{ github.base_ref }}" - git pull > /dev/null 2>&1 - git checkout ${{ github.head_ref }} - echo "The target branch for this PR is ${{ github.base_ref }}" - contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} + # echo "Current Branch (Head Ref): ${{ github.head_ref }}" + # echo "Target Branch (Base Ref): ${{ github.base_ref }}" + # git pull > /dev/null 2>&1 + # git checkout ${{ github.head_ref }} + # echo "The target branch for this PR is ${{ github.base_ref }}" + # contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} + cp summary.yml test_results/ continue-on-error: true - name: store_artifacts @@ -41,7 +42,7 @@ jobs: name: test_summary_results path: | test_results/summary.yml - dist/DA-ESS-ContentUpdate-latest.tar.gz + # dist/DA-ESS-ContentUpdate-latest.tar.gz continue-on-error: true - name: Print entire test_results/summary.yml @@ -51,4 +52,12 @@ jobs: - name: Formatted Final Report run: | python -m pip install --upgrade pip - python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY 2>&1 \ No newline at end of file + python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY 2>&1 + + - name: Check script result + if: ${{ success() }} + run: echo "Script indicated failed tests, failing the CI job." && exit 1 + + - name: Success handling + if: ${{ failure() }} + run: echo "Unit Testing passed for all detections, CI job is successful." \ No newline at end of file diff --git a/summary.yml b/summary.yml new file mode 100644 index 0000000000..cc437d9783 --- /dev/null +++ b/summary.yml @@ -0,0 +1,370 @@ +summary: + success: false + total_detections: 11 + total_pass: 8 + total_fail: 3 + total_skipped: 0 + total_untested: 0 + total_experimental_or_deprecated: 0 + success_rate: 72.7% +tested_detections: +- name: Windows AD Replication Request Initiated from Unsanctioned Location + search: '`wineventlog_security` EventCode=4662 ObjectType IN ("%{19195a5b-6da0-11d0-afd3-00c04fd930c9}", + "domainDNS") AND Properties IN ("*Replicating Directory Changes All*", "*{1131f6ad-9c07-11d1-f79f-00c04fc2dcd2}*", + "*{9923a32a-3607-11d2-b9be-0000f87a36b2}*","*{1131f6ac-9c07-11d1-f79f-00c04fc2dcd2}*") + AND AccessMask="0x100" AND (SubjectUserSid="NT AUT*" OR SubjectUserSid="S-1-5-18" + OR SubjectDomainName="Window Manager" OR SubjectUserName="*$") | stats min(_time) + as attack_time, count by SubjectDomainName, SubjectUserName, Computer, Logon_ID, + ObjectName, ObjectServer, ObjectType, OperationType, status | rename SubjectDomainName + as Target_Domain, SubjectUserName as user, Logon_ID as TargetLogonId | appendpipe + [| map search="search `wineventlog_security` EventCode=4624 TargetLogonId=$TargetLogonId$"] + | table attack_time, AuthenticationPackageName, LogonProcessName, LogonType, TargetUserSid, + Target_Domain, user, Computer, TargetLogonId, status, src_ip, src_category, ObjectName, + ObjectServer, ObjectType, OperationType | stats min(attack_time) as _time, values(TargetUserSid) + as TargetUserSid, values(Target_Domain) as Target_Domain, values(user) as user, + values(Computer) as Computer, values(status) as status, values(src_category) as + src_category, values(src_ip) as src_ip by TargetLogonId | search NOT src_category="domain_controller" + | `windows_ad_replication_request_initiated_from_unsanctioned_location_filter`' + success: false + tests: + - name: True Positive Test + test_type: unit + success: false + message: TEST ERROR + exception: The observable field(s) {'src_ip'} are missing in the detection results + status: error + duration: 6.49 + wait_duration: null + resultCount: '1' + runDuration: '1.320' + - name: True Positive Test + test_type: integration + success: false + message: 'TEST FAILED (PREEMPTIVE): associated unit test failed or encountered + an error' + exception: The observable field(s) {'src_ip'} are missing in the detection results + status: error + duration: 3.08 + wait_duration: null + resultCount: null + runDuration: null +- name: Windows Admon Default Group Policy Object Modified + search: ' `admon` admonEventType=Update objectCategory="CN=Group-Policy-Container,CN=Schema,CN=Configuration,DC=*" + (displayName="Default Domain Policy" OR displayName="Default Domain Controllers + Policy") | appendpipe [ | map search="search `wineventlog_security` EventCode=5136 AttributeSyntaxOID=2.5.5.12 + AttributeValue=$displayName$" | rename AttributeValue as displayName] | stats + min(_time) as _time values(displayName) as gp_name, values(gPCFileSysPath) as + gPCFileSysPath, values(src_user) as src_user, values(dcName) as dcName, values(dest_category) + as dest_category, values(src_user_category) as src_user_category by displayName + | `windows_admon_default_group_policy_object_modified_filter`' + success: false + tests: + - name: True Positive Test + test_type: unit + success: false + message: TEST ERROR + exception: The observable field(s) {'src_user'} are missing in the detection results + status: error + duration: 7.63 + wait_duration: null + resultCount: '1' + runDuration: '2.158' + - name: True Positive Test + test_type: integration + success: false + message: 'TEST FAILED (PREEMPTIVE): associated unit test failed or encountered + an error' + exception: The observable field(s) {'src_user'} are missing in the detection results + status: error + duration: 3.17 + wait_duration: null + resultCount: null + runDuration: null +- name: Windows Admon Group Policy Object Created + search: ' `admon` admonEventType=Update objectCategory="CN=Group-Policy-Container,CN=Schema,CN=Configuration,DC=*" + versionNumber=0 displayName!="New Group Policy Object" | appendpipe [ | map search="search + `wineventlog_security` EventCode=5136 AttributeSyntaxOID=2.5.5.12 AttributeValue=$displayName$" + | rename AttributeValue as displayName] | stats min(_time) as _time values(displayName) + as gp_name, values(gPCFileSysPath) as gPCFileSysPath, values(src_user) as src_user, + values(dcName) as dcName, values(dest_category) as dest_category, values(src_user_category) + as src_user_category by displayName | `windows_admon_group_policy_object_created_filter`' + success: false + tests: + - name: True Positive Test + test_type: unit + success: false + message: TEST ERROR + exception: The observable field(s) {'src_user'} are missing in the detection results + status: error + duration: 11.45 + wait_duration: null + resultCount: '1' + runDuration: '1.270' + - name: True Positive Test + test_type: integration + success: false + message: 'TEST FAILED (PREEMPTIVE): associated unit test failed or encountered + an error' + exception: The observable field(s) {'src_user'} are missing in the detection results + status: error + duration: 3.26 + wait_duration: null + resultCount: null + runDuration: null +- name: Azure AD Admin Consent Bypassed by Service Principal + search: '`azure_monitor_aad` (operationName="Add app role assignment to service + principal" OR operationName="Add member to role*") src_user_type=servicePrincipal | + rename properties.* as * | eval roleId = mvindex(''targetResources{}.modifiedProperties{}.newValue'', + 0) | eval roleValue = mvindex(''targetResources{}.modifiedProperties{}.newValue'', + 1) | eval roleDescription = mvindex(''targetResources{}.modifiedProperties{}.newValue'', + 2) | eval user_id = mvindex(''targetResources{}.id'', 0), user=coalesce(user,mvindex(''targetResources{}.displayName'', + 0)) | rename initiatedBy.app.displayName as src_user | stats count earliest(_time) + as firstTime latest(_time) as lastTime by src_user user user_id roleId roleValue + roleDescription | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | + `azure_ad_admin_consent_bypassed_by_service_principal_filter`' + success: true + tests: + - name: True Positive Test + test_type: unit + success: true + message: TEST PASSED + exception: null + status: pass + duration: 5.84 + wait_duration: null + resultCount: '1' + runDuration: '0.654' + - name: True Positive Test + test_type: integration + success: true + message: 'TEST SKIPPED: Skipping all integration tests' + exception: null + status: skip + duration: 0 + wait_duration: null + resultCount: null + runDuration: null +- name: Azure AD Global Administrator Role Assigned + search: '`azure_monitor_aad` operationName="Add member to role" properties.targetResources{}.modifiedProperties{}.newValue="\"Global + Administrator\"" | rename properties.* as *, initiatedBy.user.userPrincipalName + as userPrincipalName, targetResources{}.displayName as displayName | eval initiatedBy + = coalesce(userPrincipalName,src_user) | eval user = coalesce(user,mvfilter(displayName!="null")) + | stats count min(_time) as firstTime max(_time) as lastTime values(user) as user + by initiatedBy, result, operationName | `security_content_ctime(firstTime)` | + `security_content_ctime(lastTime)` | `azure_ad_global_administrator_role_assigned_filter`' + success: true + tests: + - name: True Positive Test + test_type: unit + success: true + message: TEST PASSED + exception: null + status: pass + duration: 5.97 + wait_duration: null + resultCount: '1' + runDuration: '0.666' + - name: True Positive Test + test_type: integration + success: true + message: 'TEST SKIPPED: Skipping all integration tests' + exception: null + status: skip + duration: 0 + wait_duration: null + resultCount: null + runDuration: null +- name: Azure AD Privileged Role Assigned + search: ' `azure_monitor_aad` "operationName"="Add member to role" | rename properties.* + as *, initiatedBy.user.userPrincipalName as userPrincipalName, targetResources{}.displayName + as displayName | eval initiatedBy = coalesce(userPrincipalName,src_user) | eval + user = coalesce(user,mvfilter(displayName!="null")) | rename targetResources{}.modifiedProperties{}.newValue as + roles | eval role=mvindex(roles,1) | stats count min(_time) as firstTime max(_time) + as lastTime values(user) as user by initiatedBy, result, operationName, role | + lookup privileged_azure_ad_roles azureadrole AS role OUTPUT isprvilegedadrole + description | search isprvilegedadrole = True | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` | `azure_ad_privileged_role_assigned_filter`' + success: true + tests: + - name: True Positive Test + test_type: unit + success: true + message: TEST PASSED + exception: null + status: pass + duration: 5.94 + wait_duration: null + resultCount: '1' + runDuration: '0.648' + - name: True Positive Test + test_type: integration + success: true + message: 'TEST SKIPPED: Skipping all integration tests' + exception: null + status: skip + duration: 0 + wait_duration: null + resultCount: null + runDuration: null +- name: Azure AD Privileged Role Assigned to Service Principal + search: ' `azure_monitor_aad` operationName="Add member to role" | rename properties.* + as * | search "targetResources{}.type"=ServicePrincipal | rename initiatedBy.user.userPrincipalName + as initiatedBy | rename targetResources{}.modifiedProperties{}.newValue as roles + | eval role=mvindex(roles,1) | rename targetResources{}.displayName as apps | + eval displayName=mvindex(apps,0) | stats count min(_time) as firstTime max(_time) + as lastTime values(displayName) as displayName by initiatedBy, result, operationName, + role | lookup privileged_azure_ad_roles azureadrole AS role OUTPUT isprvilegedadrole + description | search isprvilegedadrole = True | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` | `azure_ad_privileged_role_assigned_to_service_principal_filter`' + success: true + tests: + - name: True Positive Test + test_type: unit + success: true + message: TEST PASSED + exception: null + status: pass + duration: 9.3 + wait_duration: null + resultCount: '1' + runDuration: '3.134' + - name: True Positive Test + test_type: integration + success: true + message: 'TEST SKIPPED: Skipping all integration tests' + exception: null + status: skip + duration: 0 + wait_duration: null + resultCount: null + runDuration: null +- name: Azure AD Service Principal New Client Credentials + search: ' `azure_monitor_aad` category=AuditLogs operationName="Update application*Certificates + and secrets management*" | rename properties.* as * | rename targetResources{}.* + as * | rename modifiedProperties{}.* as * | eval src_user=coalesce(user,identity), + newValue=mvfilter(newValue!="\"KeyDescription\"") | stats count min(_time) as + firstTime max(_time) as lastTime values(displayName) as displayName values(src_ip) + as src_ip values(eval(mvfilter(oldValue!="null"))) as oldValue by src_user, object, + newValue | spath input=oldValue output=oldValues path={} | spath input=newValue + output=newValues path={} | mvexpand newValues | where NOT newValues IN (oldValues) + | fields - newValue, oldValue, oldValues | rename newValues as newValue | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` | `azure_ad_service_principal_new_client_credentials_filter`' + success: true + tests: + - name: True Positive Test + test_type: unit + success: true + message: TEST PASSED + exception: null + status: pass + duration: 5.94 + wait_duration: null + resultCount: '1' + runDuration: '0.782' + - name: True Positive Test + test_type: integration + success: true + message: 'TEST SKIPPED: Skipping all integration tests' + exception: null + status: skip + duration: 0 + wait_duration: null + resultCount: null + runDuration: null +- name: Detect New Local Admin account + search: '`wineventlog_security` (EventCode=4720) OR (EventCode=4732 Group_Name=Administrators) + | stats dc(EventCode) as evCount min(_time) as _time range(_time) as duration + values(src_user) as src_user values(src_user_category) as src_user_category values(dest_category) + as dest_category by user dest | where evCount=2 | fields - evCount, duration | + `detect_new_local_admin_account_filter`' + success: true + tests: + - name: True Positive Test + test_type: unit + success: true + message: TEST PASSED + exception: null + status: pass + duration: 14.33 + wait_duration: null + resultCount: '1' + runDuration: '0.696' + - name: True Positive Test + test_type: integration + success: true + message: 'TEST SKIPPED: Skipping all integration tests' + exception: null + status: skip + duration: 0 + wait_duration: null + resultCount: null + runDuration: null +- name: Kerberos Pre-Authentication Flag Disabled in UserAccountControl + search: '`wineventlog_security` EventCode=4738 (UserAccountControl="%%2096" OR MSADChangedAttributes="*Don''t + Require Preauth'' - Enabled*") | eval MSADChangedAttributes="''Don''t Require + Preauth'' - Enabled" | table _time, source, EventCode, src_user, src_user_category, + user, user_category, MSADChangedAttributes | `kerberos_pre_authentication_flag_disabled_in_useraccountcontrol_filter`' + success: true + tests: + - name: True Positive Test + test_type: unit + success: true + message: TEST PASSED + exception: null + status: pass + duration: 6.08 + wait_duration: null + resultCount: '1' + runDuration: '0.630' + - name: True Positive Test + test_type: integration + success: true + message: 'TEST SKIPPED: Skipping all integration tests' + exception: null + status: skip + duration: 0 + wait_duration: null + resultCount: null + runDuration: null +- name: Windows AD Replication Request Initiated by User Account + search: '`wineventlog_security` EventCode=4662 ObjectType IN ("%{19195a5b-6da0-11d0-afd3-00c04fd930c9}", + "domainDNS") AND Properties IN ("*Replicating Directory Changes All*", "*{1131f6ad-9c07-11d1-f79f-00c04fc2dcd2}*", + "*{9923a32a-3607-11d2-b9be-0000f87a36b2}*","*{1131f6ac-9c07-11d1-f79f-00c04fc2dcd2}*") + AND AccessMask="0x100" AND NOT (SubjectUserSid="NT AUT*" OR SubjectUserSid="S-1-5-18" + OR SubjectDomainName="Window Manager" OR SubjectUserName="*$") | stats min(_time) + as _time, count by SubjectDomainName, SubjectUserName, Computer, Logon_ID, ObjectName, + ObjectServer, ObjectType, OperationType, status | rename SubjectDomainName as + Target_Domain, SubjectUserName as user, Logon_ID as TargetLogonId, _time as attack_time + | appendpipe [| map search="search `wineventlog_security` EventCode=4624 TargetLogonId=$TargetLogonId$"] + | table attack_time, AuthenticationPackageName, LogonProcessName, LogonType, TargetUserSid, + Target_Domain, user, Computer, TargetLogonId, status, src_ip, src_category, ObjectName, + ObjectServer, ObjectType, OperationType | stats min(attack_time) as _time values(TargetUserSid) + as TargetUserSid, values(Target_Domain) as Target_Domain, values(user) as user, + values(Computer) as Computer, values(status) as status, values(src_category) as + src_category, values(src_ip) as src_ip by TargetLogonId | `windows_ad_replication_request_initiated_by_user_account_filter`' + success: true + tests: + - name: True Positive Test + test_type: unit + success: true + message: TEST PASSED + exception: null + status: pass + duration: 11.26 + wait_duration: null + resultCount: '1' + runDuration: '1.289' + - name: True Positive Test + test_type: integration + success: true + message: 'TEST SKIPPED: Skipping all integration tests' + exception: null + status: skip + duration: 0 + wait_duration: null + resultCount: null + runDuration: null +untested_detections: [] +percent_complete: UKNOWN +deprecated_detections: [] +experimental_detections: [] From afbe04f9e0f8917997f12bb2b91388d00e4f871b Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 09:33:07 -0700 Subject: [PATCH 46/77] cehck python path --- .github/workflows/unit-testing.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 00e1509e87..d0d271bcdb 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -22,6 +22,7 @@ jobs: run: | sudo apt update -qq sudo apt install jq -qq + python -m pip install --upgrade pip pip install contentctl @@ -51,7 +52,8 @@ jobs: - name: Formatted Final Report run: | - python -m pip install --upgrade pip + pwd + ls -la .github/workflows python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY 2>&1 - name: Check script result From a88a7a89e932bdcaf638b0ba1f0ba5eb6618c07c Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 09:35:45 -0700 Subject: [PATCH 47/77] wrong comments for testing --- .github/workflows/unit-testing.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index d0d271bcdb..3f8bd1559f 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -28,11 +28,11 @@ jobs: - name: Run ContentCTL test for changes against develop run: | - # echo "Current Branch (Head Ref): ${{ github.head_ref }}" - # echo "Target Branch (Base Ref): ${{ github.base_ref }}" - # git pull > /dev/null 2>&1 - # git checkout ${{ github.head_ref }} - # echo "The target branch for this PR is ${{ github.base_ref }}" + echo "Current Branch (Head Ref): ${{ github.head_ref }}" + echo "Target Branch (Base Ref): ${{ github.base_ref }}" + git pull > /dev/null 2>&1 + git checkout ${{ github.head_ref }} + echo "The target branch for this PR is ${{ github.base_ref }}" # contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} cp summary.yml test_results/ continue-on-error: true From a22076ca899dce5b6707a93f6f63b981741015f9 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 09:41:02 -0700 Subject: [PATCH 48/77] create sample summary --- .github/workflows/unit-testing.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 3f8bd1559f..2073620c00 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -33,7 +33,8 @@ jobs: git pull > /dev/null 2>&1 git checkout ${{ github.head_ref }} echo "The target branch for this PR is ${{ github.base_ref }}" - # contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} + # contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} + mkdir test_results/ cp summary.yml test_results/ continue-on-error: true From 9648a2321f194fdba0d54cc41cb98bc19eccaeda Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 09:44:58 -0700 Subject: [PATCH 49/77] import sys --- .github/workflows/format_test_results.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index 150f804461..78d4bd11a6 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -5,6 +5,7 @@ import yaml import re import os +import sys def main(): From f641cb916f9ec6356b7f472ae78bef799d5b8c1c Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 09:51:57 -0700 Subject: [PATCH 50/77] summary has failed --- .github/workflows/format_test_results.py | 2 +- .github/workflows/unit-testing.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index 78d4bd11a6..0c4f0fc363 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -24,7 +24,7 @@ def main(): # Extract total_fail value and debug print it total_fail = data['summary']['total_fail'] - print("\033[1mDownload the job artifacts of this run and view complete summary in test_results/summary.yml for troubleshooting failures.\n\033[0m") + print("**Download the job artifacts of this run and view complete summary in test_results/summary.yml for troubleshooting failures.**\n") print(f"Extracted total_fail: [{total_fail}]\n") # Print all unit test details first diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 2073620c00..8c1d86f63d 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -57,9 +57,9 @@ jobs: ls -la .github/workflows python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY 2>&1 - - name: Check script result + - name: Check Test Summary Results if: ${{ success() }} - run: echo "Script indicated failed tests, failing the CI job." && exit 1 + run: echo "Test Summary failed tests, failing the CI job. See details in the unit-testing job summary UI" && exit 1 - name: Success handling if: ${{ failure() }} From 125417dc829f28a5ee41db1bef151ad4d59b1252 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 09:59:05 -0700 Subject: [PATCH 51/77] update with 0 failures --- .github/workflows/format_test_results.py | 2 +- summary.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index 0c4f0fc363..3514aa76ca 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -10,7 +10,7 @@ def main(): # Define the path to the YAML file - #yaml_file_path = 'test_results/summary.yml' + # yaml_file_path = 'summary.yml' yaml_file_path = '/home/runner/work/security_content/security_content/test_results/summary.yml' # Check if the YAML file exists diff --git a/summary.yml b/summary.yml index cc437d9783..ad5e338d4b 100644 --- a/summary.yml +++ b/summary.yml @@ -2,7 +2,7 @@ summary: success: false total_detections: 11 total_pass: 8 - total_fail: 3 + total_fail: 0 total_skipped: 0 total_untested: 0 total_experimental_or_deprecated: 0 From d84aac3196999a006beb1a400d57534181a29dea Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 10:30:11 -0700 Subject: [PATCH 52/77] updating test handlers --- .github/workflows/format_test_results.py | 10 +++++----- .github/workflows/unit-testing.yml | 18 ++++++++---------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index 3514aa76ca..753a3ad3e8 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -44,13 +44,13 @@ def main(): print(f"{name:<80} | {status:<6} | {test_type:<10} | {'-':<50}") # Check if total_fail is a valid integer and greater than or equal to one - if re.match(r'^[0-9]+$', str(total_fail)) and int(total_fail) >= 1: + if int(total_fail) >=1: # Print the message in bold - print("CI Failure: There are failed tests.\n") - sys.exit(0) - else: - print("CI Success: No failed tests.\n\n") + print("**CI Failure: There are failed tests.**\n") sys.exit(1) + else: + print("**CI Success: No failed tests.**\n\n") + sys.exit(0) if __name__ == "__main__": main() \ No newline at end of file diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 8c1d86f63d..750aad364b 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -52,15 +52,13 @@ jobs: continue-on-error: true - name: Formatted Final Report - run: | - pwd - ls -la .github/workflows - python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY 2>&1 + run: | + python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY - - name: Check Test Summary Results - if: ${{ success() }} - run: echo "Test Summary failed tests, failing the CI job. See details in the unit-testing job summary UI" && exit 1 + # - name: Check Test Summary Results + # if: ${{ success() }} + # run: echo "Test Summary failed tests, failing the CI job. See details in the unit-testing job summary UI" && exit 1 - - name: Success handling - if: ${{ failure() }} - run: echo "Unit Testing passed for all detections, CI job is successful." \ No newline at end of file + # - name: Success handling + # if: ${{ failure() }} + # run: echo "Unit Testing passed for all detections, CI job is successful." \ No newline at end of file From 9e9d16df856e3aa84ecbb0332286dd6bcd94edb4 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 10:33:19 -0700 Subject: [PATCH 53/77] udpating with failures for script to fail --- summary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/summary.yml b/summary.yml index ad5e338d4b..cc437d9783 100644 --- a/summary.yml +++ b/summary.yml @@ -2,7 +2,7 @@ summary: success: false total_detections: 11 total_pass: 8 - total_fail: 0 + total_fail: 3 total_skipped: 0 total_untested: 0 total_experimental_or_deprecated: 0 From 78944aaaa64494a70a7eb71daf52f75b4312db85 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 10:40:39 -0700 Subject: [PATCH 54/77] updating formatting --- .github/workflows/unit-testing.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 750aad364b..46ec4d1f2b 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -51,9 +51,11 @@ jobs: run: cat test_results/summary.yml continue-on-error: true - - name: Formatted Final Report - run: | + - name: Check the Test Result Summary + run: | + echo "Check the Test Result Summary for pass/fail" python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY + echo "The Unit testing is completed. See details in the unit-testing job summary UI " # - name: Check Test Summary Results # if: ${{ success() }} From 6fee68703463475731a9e977e87c697b924b5e87 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 10:50:30 -0700 Subject: [PATCH 55/77] 0 failures --- summary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/summary.yml b/summary.yml index cc437d9783..ad5e338d4b 100644 --- a/summary.yml +++ b/summary.yml @@ -2,7 +2,7 @@ summary: success: false total_detections: 11 total_pass: 8 - total_fail: 3 + total_fail: 0 total_skipped: 0 total_untested: 0 total_experimental_or_deprecated: 0 From 65b0cd505d17c41cf54de27b40c03d40190d4659 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 10:59:51 -0700 Subject: [PATCH 56/77] one more test before review for passing CI --- .github/workflows/unit-testing.yml | 2 +- .github/workflows/validate-and-build.yml | 5 +---- .../endpoint/access_lsass_memory_for_dump_creation.yml | 2 +- detections/endpoint/account_discovery_with_net_app.yml | 2 +- ...to_vulnerable_ivanti_connect_secure_bookmark_endpoint.yml | 2 +- detections/web/adobe_coldfusion_access_control_bypass.yml | 2 +- 6 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 46ec4d1f2b..3b143f5652 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -13,7 +13,7 @@ jobs: with: ref: develop - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.11' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified diff --git a/.github/workflows/validate-and-build.yml b/.github/workflows/validate-and-build.yml index bea48cc794..95b35acd10 100644 --- a/.github/workflows/validate-and-build.yml +++ b/.github/workflows/validate-and-build.yml @@ -11,14 +11,11 @@ jobs: steps: - name: Check out the repository code uses: actions/checkout@v3 - with: - node-version: '20' - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.11' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified - node-version: '20' - name: Install System Packages run: | diff --git a/detections/endpoint/access_lsass_memory_for_dump_creation.yml b/detections/endpoint/access_lsass_memory_for_dump_creation.yml index 5e5a80f73a..04d4d021a0 100644 --- a/detections/endpoint/access_lsass_memory_for_dump_creation.yml +++ b/detections/endpoint/access_lsass_memory_for_dump_creation.yml @@ -1,6 +1,6 @@ name: Access LSASS Memory for Dump Creation id: fb4c31b0-13e8-4155-8aa5-24de4b8d6717 -version: 4 +version: 3 date: '2024-05-13' author: Patrick Bareiss, Splunk status: production diff --git a/detections/endpoint/account_discovery_with_net_app.yml b/detections/endpoint/account_discovery_with_net_app.yml index ce9e97269f..2f4a8fd313 100644 --- a/detections/endpoint/account_discovery_with_net_app.yml +++ b/detections/endpoint/account_discovery_with_net_app.yml @@ -1,6 +1,6 @@ name: Account Discovery With Net App id: 339805ce-ac30-11eb-b87d-acde48001122 -version: 6 +version: 5 date: '2024-05-22' author: Teoderick Contreras, Splunk, TheLawsOfChaos, Github Community status: production diff --git a/detections/web/access_to_vulnerable_ivanti_connect_secure_bookmark_endpoint.yml b/detections/web/access_to_vulnerable_ivanti_connect_secure_bookmark_endpoint.yml index e9e20c80d9..2ee10a8f52 100644 --- a/detections/web/access_to_vulnerable_ivanti_connect_secure_bookmark_endpoint.yml +++ b/detections/web/access_to_vulnerable_ivanti_connect_secure_bookmark_endpoint.yml @@ -1,6 +1,6 @@ name: Access to Vulnerable Ivanti Connect Secure Bookmark Endpoint id: 15838756-f425-43fa-9d88-a7f88063e81a -version: 3 +version: 2 date: '2024-05-14' author: Michael Haag, Splunk status: production diff --git a/detections/web/adobe_coldfusion_access_control_bypass.yml b/detections/web/adobe_coldfusion_access_control_bypass.yml index f344db1e74..5828ebffb2 100644 --- a/detections/web/adobe_coldfusion_access_control_bypass.yml +++ b/detections/web/adobe_coldfusion_access_control_bypass.yml @@ -15,7 +15,7 @@ description: The following analytic detects potential exploitation attempts agai unauthorized access to ColdFusion administration endpoints. If confirmed malicious, this could result in data theft, brute force attacks, or further exploitation of other vulnerabilities, posing a serious security risk to the environment. -search: 'x| tstats count min(_time) as firstTime max(_time) as lastTime from datamodel=Web +search: '| tstats count min(_time) as firstTime max(_time) as lastTime from datamodel=Web where Web.url IN ("//restplay*", "//CFIDE/restplay*", "//CFIDE/administrator*", "//CFIDE/adminapi*", "//CFIDE/main*", "//CFIDE/componentutils*", "//CFIDE/wizards*", "//CFIDE/servermanager*","/restplay*", "/CFIDE/restplay*", "/CFIDE/administrator*", From 434f7aeb1916ea2fd189a07c8040cc1f2e86e064 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 11:08:34 -0700 Subject: [PATCH 57/77] final updates - real test --- .github/workflows/unit-testing.yml | 15 +- summary.yml | 370 ----------------------------- 2 files changed, 4 insertions(+), 381 deletions(-) delete mode 100644 summary.yml diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 3b143f5652..cc1f7e9fc1 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -44,23 +44,16 @@ jobs: name: test_summary_results path: | test_results/summary.yml - # dist/DA-ESS-ContentUpdate-latest.tar.gz + dist/DA-ESS-ContentUpdate-latest.tar.gz continue-on-error: true - name: Print entire test_results/summary.yml run: cat test_results/summary.yml continue-on-error: true - - name: Check the Test Result Summary + - name: Check the test_results/summary.yml for pass/fail. run: | - echo "Check the Test Result Summary for pass/fail" + echo "This job will fail if there are failures in unit-testing" python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY echo "The Unit testing is completed. See details in the unit-testing job summary UI " - - # - name: Check Test Summary Results - # if: ${{ success() }} - # run: echo "Test Summary failed tests, failing the CI job. See details in the unit-testing job summary UI" && exit 1 - - # - name: Success handling - # if: ${{ failure() }} - # run: echo "Unit Testing passed for all detections, CI job is successful." \ No newline at end of file + \ No newline at end of file diff --git a/summary.yml b/summary.yml deleted file mode 100644 index ad5e338d4b..0000000000 --- a/summary.yml +++ /dev/null @@ -1,370 +0,0 @@ -summary: - success: false - total_detections: 11 - total_pass: 8 - total_fail: 0 - total_skipped: 0 - total_untested: 0 - total_experimental_or_deprecated: 0 - success_rate: 72.7% -tested_detections: -- name: Windows AD Replication Request Initiated from Unsanctioned Location - search: '`wineventlog_security` EventCode=4662 ObjectType IN ("%{19195a5b-6da0-11d0-afd3-00c04fd930c9}", - "domainDNS") AND Properties IN ("*Replicating Directory Changes All*", "*{1131f6ad-9c07-11d1-f79f-00c04fc2dcd2}*", - "*{9923a32a-3607-11d2-b9be-0000f87a36b2}*","*{1131f6ac-9c07-11d1-f79f-00c04fc2dcd2}*") - AND AccessMask="0x100" AND (SubjectUserSid="NT AUT*" OR SubjectUserSid="S-1-5-18" - OR SubjectDomainName="Window Manager" OR SubjectUserName="*$") | stats min(_time) - as attack_time, count by SubjectDomainName, SubjectUserName, Computer, Logon_ID, - ObjectName, ObjectServer, ObjectType, OperationType, status | rename SubjectDomainName - as Target_Domain, SubjectUserName as user, Logon_ID as TargetLogonId | appendpipe - [| map search="search `wineventlog_security` EventCode=4624 TargetLogonId=$TargetLogonId$"] - | table attack_time, AuthenticationPackageName, LogonProcessName, LogonType, TargetUserSid, - Target_Domain, user, Computer, TargetLogonId, status, src_ip, src_category, ObjectName, - ObjectServer, ObjectType, OperationType | stats min(attack_time) as _time, values(TargetUserSid) - as TargetUserSid, values(Target_Domain) as Target_Domain, values(user) as user, - values(Computer) as Computer, values(status) as status, values(src_category) as - src_category, values(src_ip) as src_ip by TargetLogonId | search NOT src_category="domain_controller" - | `windows_ad_replication_request_initiated_from_unsanctioned_location_filter`' - success: false - tests: - - name: True Positive Test - test_type: unit - success: false - message: TEST ERROR - exception: The observable field(s) {'src_ip'} are missing in the detection results - status: error - duration: 6.49 - wait_duration: null - resultCount: '1' - runDuration: '1.320' - - name: True Positive Test - test_type: integration - success: false - message: 'TEST FAILED (PREEMPTIVE): associated unit test failed or encountered - an error' - exception: The observable field(s) {'src_ip'} are missing in the detection results - status: error - duration: 3.08 - wait_duration: null - resultCount: null - runDuration: null -- name: Windows Admon Default Group Policy Object Modified - search: ' `admon` admonEventType=Update objectCategory="CN=Group-Policy-Container,CN=Schema,CN=Configuration,DC=*" - (displayName="Default Domain Policy" OR displayName="Default Domain Controllers - Policy") | appendpipe [ | map search="search `wineventlog_security` EventCode=5136 AttributeSyntaxOID=2.5.5.12 - AttributeValue=$displayName$" | rename AttributeValue as displayName] | stats - min(_time) as _time values(displayName) as gp_name, values(gPCFileSysPath) as - gPCFileSysPath, values(src_user) as src_user, values(dcName) as dcName, values(dest_category) - as dest_category, values(src_user_category) as src_user_category by displayName - | `windows_admon_default_group_policy_object_modified_filter`' - success: false - tests: - - name: True Positive Test - test_type: unit - success: false - message: TEST ERROR - exception: The observable field(s) {'src_user'} are missing in the detection results - status: error - duration: 7.63 - wait_duration: null - resultCount: '1' - runDuration: '2.158' - - name: True Positive Test - test_type: integration - success: false - message: 'TEST FAILED (PREEMPTIVE): associated unit test failed or encountered - an error' - exception: The observable field(s) {'src_user'} are missing in the detection results - status: error - duration: 3.17 - wait_duration: null - resultCount: null - runDuration: null -- name: Windows Admon Group Policy Object Created - search: ' `admon` admonEventType=Update objectCategory="CN=Group-Policy-Container,CN=Schema,CN=Configuration,DC=*" - versionNumber=0 displayName!="New Group Policy Object" | appendpipe [ | map search="search - `wineventlog_security` EventCode=5136 AttributeSyntaxOID=2.5.5.12 AttributeValue=$displayName$" - | rename AttributeValue as displayName] | stats min(_time) as _time values(displayName) - as gp_name, values(gPCFileSysPath) as gPCFileSysPath, values(src_user) as src_user, - values(dcName) as dcName, values(dest_category) as dest_category, values(src_user_category) - as src_user_category by displayName | `windows_admon_group_policy_object_created_filter`' - success: false - tests: - - name: True Positive Test - test_type: unit - success: false - message: TEST ERROR - exception: The observable field(s) {'src_user'} are missing in the detection results - status: error - duration: 11.45 - wait_duration: null - resultCount: '1' - runDuration: '1.270' - - name: True Positive Test - test_type: integration - success: false - message: 'TEST FAILED (PREEMPTIVE): associated unit test failed or encountered - an error' - exception: The observable field(s) {'src_user'} are missing in the detection results - status: error - duration: 3.26 - wait_duration: null - resultCount: null - runDuration: null -- name: Azure AD Admin Consent Bypassed by Service Principal - search: '`azure_monitor_aad` (operationName="Add app role assignment to service - principal" OR operationName="Add member to role*") src_user_type=servicePrincipal | - rename properties.* as * | eval roleId = mvindex(''targetResources{}.modifiedProperties{}.newValue'', - 0) | eval roleValue = mvindex(''targetResources{}.modifiedProperties{}.newValue'', - 1) | eval roleDescription = mvindex(''targetResources{}.modifiedProperties{}.newValue'', - 2) | eval user_id = mvindex(''targetResources{}.id'', 0), user=coalesce(user,mvindex(''targetResources{}.displayName'', - 0)) | rename initiatedBy.app.displayName as src_user | stats count earliest(_time) - as firstTime latest(_time) as lastTime by src_user user user_id roleId roleValue - roleDescription | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | - `azure_ad_admin_consent_bypassed_by_service_principal_filter`' - success: true - tests: - - name: True Positive Test - test_type: unit - success: true - message: TEST PASSED - exception: null - status: pass - duration: 5.84 - wait_duration: null - resultCount: '1' - runDuration: '0.654' - - name: True Positive Test - test_type: integration - success: true - message: 'TEST SKIPPED: Skipping all integration tests' - exception: null - status: skip - duration: 0 - wait_duration: null - resultCount: null - runDuration: null -- name: Azure AD Global Administrator Role Assigned - search: '`azure_monitor_aad` operationName="Add member to role" properties.targetResources{}.modifiedProperties{}.newValue="\"Global - Administrator\"" | rename properties.* as *, initiatedBy.user.userPrincipalName - as userPrincipalName, targetResources{}.displayName as displayName | eval initiatedBy - = coalesce(userPrincipalName,src_user) | eval user = coalesce(user,mvfilter(displayName!="null")) - | stats count min(_time) as firstTime max(_time) as lastTime values(user) as user - by initiatedBy, result, operationName | `security_content_ctime(firstTime)` | - `security_content_ctime(lastTime)` | `azure_ad_global_administrator_role_assigned_filter`' - success: true - tests: - - name: True Positive Test - test_type: unit - success: true - message: TEST PASSED - exception: null - status: pass - duration: 5.97 - wait_duration: null - resultCount: '1' - runDuration: '0.666' - - name: True Positive Test - test_type: integration - success: true - message: 'TEST SKIPPED: Skipping all integration tests' - exception: null - status: skip - duration: 0 - wait_duration: null - resultCount: null - runDuration: null -- name: Azure AD Privileged Role Assigned - search: ' `azure_monitor_aad` "operationName"="Add member to role" | rename properties.* - as *, initiatedBy.user.userPrincipalName as userPrincipalName, targetResources{}.displayName - as displayName | eval initiatedBy = coalesce(userPrincipalName,src_user) | eval - user = coalesce(user,mvfilter(displayName!="null")) | rename targetResources{}.modifiedProperties{}.newValue as - roles | eval role=mvindex(roles,1) | stats count min(_time) as firstTime max(_time) - as lastTime values(user) as user by initiatedBy, result, operationName, role | - lookup privileged_azure_ad_roles azureadrole AS role OUTPUT isprvilegedadrole - description | search isprvilegedadrole = True | `security_content_ctime(firstTime)` - | `security_content_ctime(lastTime)` | `azure_ad_privileged_role_assigned_filter`' - success: true - tests: - - name: True Positive Test - test_type: unit - success: true - message: TEST PASSED - exception: null - status: pass - duration: 5.94 - wait_duration: null - resultCount: '1' - runDuration: '0.648' - - name: True Positive Test - test_type: integration - success: true - message: 'TEST SKIPPED: Skipping all integration tests' - exception: null - status: skip - duration: 0 - wait_duration: null - resultCount: null - runDuration: null -- name: Azure AD Privileged Role Assigned to Service Principal - search: ' `azure_monitor_aad` operationName="Add member to role" | rename properties.* - as * | search "targetResources{}.type"=ServicePrincipal | rename initiatedBy.user.userPrincipalName - as initiatedBy | rename targetResources{}.modifiedProperties{}.newValue as roles - | eval role=mvindex(roles,1) | rename targetResources{}.displayName as apps | - eval displayName=mvindex(apps,0) | stats count min(_time) as firstTime max(_time) - as lastTime values(displayName) as displayName by initiatedBy, result, operationName, - role | lookup privileged_azure_ad_roles azureadrole AS role OUTPUT isprvilegedadrole - description | search isprvilegedadrole = True | `security_content_ctime(firstTime)` - | `security_content_ctime(lastTime)` | `azure_ad_privileged_role_assigned_to_service_principal_filter`' - success: true - tests: - - name: True Positive Test - test_type: unit - success: true - message: TEST PASSED - exception: null - status: pass - duration: 9.3 - wait_duration: null - resultCount: '1' - runDuration: '3.134' - - name: True Positive Test - test_type: integration - success: true - message: 'TEST SKIPPED: Skipping all integration tests' - exception: null - status: skip - duration: 0 - wait_duration: null - resultCount: null - runDuration: null -- name: Azure AD Service Principal New Client Credentials - search: ' `azure_monitor_aad` category=AuditLogs operationName="Update application*Certificates - and secrets management*" | rename properties.* as * | rename targetResources{}.* - as * | rename modifiedProperties{}.* as * | eval src_user=coalesce(user,identity), - newValue=mvfilter(newValue!="\"KeyDescription\"") | stats count min(_time) as - firstTime max(_time) as lastTime values(displayName) as displayName values(src_ip) - as src_ip values(eval(mvfilter(oldValue!="null"))) as oldValue by src_user, object, - newValue | spath input=oldValue output=oldValues path={} | spath input=newValue - output=newValues path={} | mvexpand newValues | where NOT newValues IN (oldValues) - | fields - newValue, oldValue, oldValues | rename newValues as newValue | `security_content_ctime(firstTime)` - | `security_content_ctime(lastTime)` | `azure_ad_service_principal_new_client_credentials_filter`' - success: true - tests: - - name: True Positive Test - test_type: unit - success: true - message: TEST PASSED - exception: null - status: pass - duration: 5.94 - wait_duration: null - resultCount: '1' - runDuration: '0.782' - - name: True Positive Test - test_type: integration - success: true - message: 'TEST SKIPPED: Skipping all integration tests' - exception: null - status: skip - duration: 0 - wait_duration: null - resultCount: null - runDuration: null -- name: Detect New Local Admin account - search: '`wineventlog_security` (EventCode=4720) OR (EventCode=4732 Group_Name=Administrators) - | stats dc(EventCode) as evCount min(_time) as _time range(_time) as duration - values(src_user) as src_user values(src_user_category) as src_user_category values(dest_category) - as dest_category by user dest | where evCount=2 | fields - evCount, duration | - `detect_new_local_admin_account_filter`' - success: true - tests: - - name: True Positive Test - test_type: unit - success: true - message: TEST PASSED - exception: null - status: pass - duration: 14.33 - wait_duration: null - resultCount: '1' - runDuration: '0.696' - - name: True Positive Test - test_type: integration - success: true - message: 'TEST SKIPPED: Skipping all integration tests' - exception: null - status: skip - duration: 0 - wait_duration: null - resultCount: null - runDuration: null -- name: Kerberos Pre-Authentication Flag Disabled in UserAccountControl - search: '`wineventlog_security` EventCode=4738 (UserAccountControl="%%2096" OR MSADChangedAttributes="*Don''t - Require Preauth'' - Enabled*") | eval MSADChangedAttributes="''Don''t Require - Preauth'' - Enabled" | table _time, source, EventCode, src_user, src_user_category, - user, user_category, MSADChangedAttributes | `kerberos_pre_authentication_flag_disabled_in_useraccountcontrol_filter`' - success: true - tests: - - name: True Positive Test - test_type: unit - success: true - message: TEST PASSED - exception: null - status: pass - duration: 6.08 - wait_duration: null - resultCount: '1' - runDuration: '0.630' - - name: True Positive Test - test_type: integration - success: true - message: 'TEST SKIPPED: Skipping all integration tests' - exception: null - status: skip - duration: 0 - wait_duration: null - resultCount: null - runDuration: null -- name: Windows AD Replication Request Initiated by User Account - search: '`wineventlog_security` EventCode=4662 ObjectType IN ("%{19195a5b-6da0-11d0-afd3-00c04fd930c9}", - "domainDNS") AND Properties IN ("*Replicating Directory Changes All*", "*{1131f6ad-9c07-11d1-f79f-00c04fc2dcd2}*", - "*{9923a32a-3607-11d2-b9be-0000f87a36b2}*","*{1131f6ac-9c07-11d1-f79f-00c04fc2dcd2}*") - AND AccessMask="0x100" AND NOT (SubjectUserSid="NT AUT*" OR SubjectUserSid="S-1-5-18" - OR SubjectDomainName="Window Manager" OR SubjectUserName="*$") | stats min(_time) - as _time, count by SubjectDomainName, SubjectUserName, Computer, Logon_ID, ObjectName, - ObjectServer, ObjectType, OperationType, status | rename SubjectDomainName as - Target_Domain, SubjectUserName as user, Logon_ID as TargetLogonId, _time as attack_time - | appendpipe [| map search="search `wineventlog_security` EventCode=4624 TargetLogonId=$TargetLogonId$"] - | table attack_time, AuthenticationPackageName, LogonProcessName, LogonType, TargetUserSid, - Target_Domain, user, Computer, TargetLogonId, status, src_ip, src_category, ObjectName, - ObjectServer, ObjectType, OperationType | stats min(attack_time) as _time values(TargetUserSid) - as TargetUserSid, values(Target_Domain) as Target_Domain, values(user) as user, - values(Computer) as Computer, values(status) as status, values(src_category) as - src_category, values(src_ip) as src_ip by TargetLogonId | `windows_ad_replication_request_initiated_by_user_account_filter`' - success: true - tests: - - name: True Positive Test - test_type: unit - success: true - message: TEST PASSED - exception: null - status: pass - duration: 11.26 - wait_duration: null - resultCount: '1' - runDuration: '1.289' - - name: True Positive Test - test_type: integration - success: true - message: 'TEST SKIPPED: Skipping all integration tests' - exception: null - status: skip - duration: 0 - wait_duration: null - resultCount: null - runDuration: null -untested_detections: [] -percent_complete: UKNOWN -deprecated_detections: [] -experimental_detections: [] From 9174d5125d540e0ea3379332fd71cd460da3e5ad Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 11:09:19 -0700 Subject: [PATCH 58/77] final updates - real test2 --- detections/endpoint/7zip_commandline_to_smb_share_path.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/detections/endpoint/7zip_commandline_to_smb_share_path.yml b/detections/endpoint/7zip_commandline_to_smb_share_path.yml index 76c046406f..496dee8b71 100644 --- a/detections/endpoint/7zip_commandline_to_smb_share_path.yml +++ b/detections/endpoint/7zip_commandline_to_smb_share_path.yml @@ -1,6 +1,6 @@ name: 7zip CommandLine To SMB Share Path id: 01d29b48-ff6f-11eb-b81e-acde48001123 -version: 2 +version: 3 date: '2024-05-17' author: Teoderick Contreras, Splunk status: production @@ -14,7 +14,7 @@ description: The following analytic detects the execution of 7z or 7za processes sensitive information and potentially aiding further attacks. data_source: - Sysmon EventID 1 -search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) +search: 'x| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where (Processes.process_name ="7z.exe" OR Processes.process_name = "7za.exe" OR Processes.original_file_name = "7z.exe" OR Processes.original_file_name = "7za.exe") AND (Processes.process="*\\C$\\*" From c7eacfa3e261f4a2ae3db7b7d8772a4d38e255f1 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 11:12:37 -0700 Subject: [PATCH 59/77] run test --- .github/workflows/unit-testing.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index cc1f7e9fc1..f18dcb3a1c 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -33,9 +33,8 @@ jobs: git pull > /dev/null 2>&1 git checkout ${{ github.head_ref }} echo "The target branch for this PR is ${{ github.base_ref }}" - # contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} - mkdir test_results/ - cp summary.yml test_results/ + contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} + echo "contentctl test - COMPLETED" continue-on-error: true - name: store_artifacts From c7264218b5d8004b6e514f82cf166699f095afc4 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 11:20:48 -0700 Subject: [PATCH 60/77] coloring --- .github/workflows/format_test_results.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index 753a3ad3e8..46728dfe34 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -29,8 +29,8 @@ def main(): # Print all unit test details first print("Unit Test Details:\n") - print(f"{'Name':<80} | {'Status':<6} | {'Test Type':<10} | {'Exception':<50}") - print(f"{'----':<80} | {'------':<6} | {'---------':<10} | {'---------':<50}") + print(f"{'Name':<80} | {'Status':<6} | 🔴 {'Test Type':<10} | {'Exception':<50}") + print(f"{'----':<80} | {'------':<6} | 🟢 {'---------':<10} | {'---------':<50}") for detection in data['tested_detections']: for test in detection['tests']: if test['test_type'].strip() == "unit": # Check if the test type is "unit" @@ -46,10 +46,10 @@ def main(): # Check if total_fail is a valid integer and greater than or equal to one if int(total_fail) >=1: # Print the message in bold - print("**CI Failure: There are failed tests.**\n") + print("🔴 ** CI Failure: There are failed tests.**\n\n") sys.exit(1) else: - print("**CI Success: No failed tests.**\n\n") + print("🟢 ** CI Success: No failed tests.**\n\n") sys.exit(0) if __name__ == "__main__": From 9072925758d26847894f207f40b3be24ecdadfe2 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 11:25:01 -0700 Subject: [PATCH 61/77] pass-fail-colors --- .github/workflows/format_test_results.py | 13 +++++++------ .../crushftp_server_side_template_injection.yml | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index 46728dfe34..7a45ff7969 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -24,13 +24,14 @@ def main(): # Extract total_fail value and debug print it total_fail = data['summary']['total_fail'] - print("**Download the job artifacts of this run and view complete summary in test_results/summary.yml for troubleshooting failures.**\n") + print("**Download the job artifacts of this run and view complete summary in test_results/summary.yml for troubleshooting failures.**\n") + print("** 📝 Experimental or manual_test detections are not tested 📝 **\n") print(f"Extracted total_fail: [{total_fail}]\n") # Print all unit test details first - print("Unit Test Details:\n") - print(f"{'Name':<80} | {'Status':<6} | 🔴 {'Test Type':<10} | {'Exception':<50}") - print(f"{'----':<80} | {'------':<6} | 🟢 {'---------':<10} | {'---------':<50}") + print("**Unit Test Details:**\n") + print(f"{'Name':<80} | {'Status':<6} | {'Test Type':<10} | {'Exception':<50}") + print(f"{'----':<80} | {'------':<6} | {'---------':<10} | {'---------':<50}") for detection in data['tested_detections']: for test in detection['tests']: if test['test_type'].strip() == "unit": # Check if the test type is "unit" @@ -39,9 +40,9 @@ def main(): test_type = test['test_type'].strip() exception = test.get('exception', 'N/A') # Get exception if exists, else 'N/A' if status == 'FAIL': - print(f"{name:<80} | {status:<6} | {test_type:<10} | {exception:<50}") + print(f"{name:<80} | 🔴 {status:<6} | {test_type:<10} | {exception:<50}") else: - print(f"{name:<80} | {status:<6} | {test_type:<10} | {'-':<50}") + print(f"{name:<80} | 🟢 {status:<6} | {test_type:<10} | {'-':<50}") # Check if total_fail is a valid integer and greater than or equal to one if int(total_fail) >=1: diff --git a/detections/application/crushftp_server_side_template_injection.yml b/detections/application/crushftp_server_side_template_injection.yml index 71c70e6b6c..6974723969 100644 --- a/detections/application/crushftp_server_side_template_injection.yml +++ b/detections/application/crushftp_server_side_template_injection.yml @@ -1,6 +1,6 @@ name: CrushFTP Server Side Template Injection id: ccf6b7a3-bd39-4bc9-a949-143a8d640dbc -version: 1 +version: 3 date: '2024-05-16' author: Michael Haag, Splunk data_source: [] From 8e577b35decef30b4f202637e25dde7fe3066296 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 11:39:56 -0700 Subject: [PATCH 62/77] udpates to footer --- .github/workflows/format_test_results.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index 7a45ff7969..1dba8c23a9 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -25,7 +25,7 @@ def main(): # Extract total_fail value and debug print it total_fail = data['summary']['total_fail'] print("**Download the job artifacts of this run and view complete summary in test_results/summary.yml for troubleshooting failures.**\n") - print("** 📝 Experimental or manual_test detections are not tested 📝 **\n") + print(" 📝 **Experimental or manual_test detections are not tested** 📝 **\n") print(f"Extracted total_fail: [{total_fail}]\n") # Print all unit test details first @@ -43,14 +43,16 @@ def main(): print(f"{name:<80} | 🔴 {status:<6} | {test_type:<10} | {exception:<50}") else: print(f"{name:<80} | 🟢 {status:<6} | {test_type:<10} | {'-':<50}") + # Print table footer + print(f"{'----':<80} | {'------':<6} | {'---------':<10} | {'---------':<50}") # Check if total_fail is a valid integer and greater than or equal to one if int(total_fail) >=1: # Print the message in bold - print("🔴 ** CI Failure: There are failed tests.**\n\n") + print("🔴 - **CI Failure: There are failed tests.**\n\n") sys.exit(1) else: - print("🟢 ** CI Success: No failed tests.**\n\n") + print("🟢 - **CI Success: No failed tests.**\n\n") sys.exit(0) if __name__ == "__main__": From d18dcaaf2f53d57418b7bc925db606909c64c721 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 11:47:21 -0700 Subject: [PATCH 63/77] final final --- .github/workflows/format_test_results.py | 2 +- .../application/crushftp_server_side_template_injection.yml | 2 +- .../endpoint/3cx_supply_chain_attack_network_indicators.yml | 2 +- detections/endpoint/7zip_commandline_to_smb_share_path.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index 1dba8c23a9..b11e9081c2 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -40,7 +40,7 @@ def main(): test_type = test['test_type'].strip() exception = test.get('exception', 'N/A') # Get exception if exists, else 'N/A' if status == 'FAIL': - print(f"{name:<80} | 🔴 {status:<6} | {test_type:<10} | {exception:<50}") + print(f"{name:<80} | 🔴 {status:<6} | {test_type:<10} | {exception:<50}") else: print(f"{name:<80} | 🟢 {status:<6} | {test_type:<10} | {'-':<50}") # Print table footer diff --git a/detections/application/crushftp_server_side_template_injection.yml b/detections/application/crushftp_server_side_template_injection.yml index 6974723969..71c70e6b6c 100644 --- a/detections/application/crushftp_server_side_template_injection.yml +++ b/detections/application/crushftp_server_side_template_injection.yml @@ -1,6 +1,6 @@ name: CrushFTP Server Side Template Injection id: ccf6b7a3-bd39-4bc9-a949-143a8d640dbc -version: 3 +version: 1 date: '2024-05-16' author: Michael Haag, Splunk data_source: [] diff --git a/detections/endpoint/3cx_supply_chain_attack_network_indicators.yml b/detections/endpoint/3cx_supply_chain_attack_network_indicators.yml index d775d00161..e14aa34e3d 100644 --- a/detections/endpoint/3cx_supply_chain_attack_network_indicators.yml +++ b/detections/endpoint/3cx_supply_chain_attack_network_indicators.yml @@ -15,7 +15,7 @@ description: The following analytic identifies DNS queries to domains associated malicious, this activity could allow attackers to establish a foothold in the network, exfiltrate sensitive data, or further propagate malware, leading to extensive damage and data breaches. -search: 'x| tstats `security_content_summariesonly` values(DNS.answer) as IPs min(_time) +search: '| tstats `security_content_summariesonly` values(DNS.answer) as IPs min(_time) as firstTime from datamodel=Network_Resolution by DNS.src, DNS.query | `drop_dm_object_name(DNS)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | lookup 3cx_ioc_domains domain as query OUTPUT Description isIOC | search isIOC=true | `3cx_supply_chain_attack_network_indicators_filter`' diff --git a/detections/endpoint/7zip_commandline_to_smb_share_path.yml b/detections/endpoint/7zip_commandline_to_smb_share_path.yml index 496dee8b71..76c046406f 100644 --- a/detections/endpoint/7zip_commandline_to_smb_share_path.yml +++ b/detections/endpoint/7zip_commandline_to_smb_share_path.yml @@ -1,6 +1,6 @@ name: 7zip CommandLine To SMB Share Path id: 01d29b48-ff6f-11eb-b81e-acde48001123 -version: 3 +version: 2 date: '2024-05-17' author: Teoderick Contreras, Splunk status: production @@ -14,7 +14,7 @@ description: The following analytic detects the execution of 7z or 7za processes sensitive information and potentially aiding further attacks. data_source: - Sysmon EventID 1 -search: 'x| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where (Processes.process_name ="7z.exe" OR Processes.process_name = "7za.exe" OR Processes.original_file_name = "7z.exe" OR Processes.original_file_name = "7za.exe") AND (Processes.process="*\\C$\\*" From acca14faad5f3719d1c09cffa3a29a339a64b501 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 11:51:42 -0700 Subject: [PATCH 64/77] adding overall status --- .github/workflows/format_test_results.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index b11e9081c2..f49fd11538 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -47,6 +47,10 @@ def main(): print(f"{'----':<80} | {'------':<6} | {'---------':<10} | {'---------':<50}") # Check if total_fail is a valid integer and greater than or equal to one + print("\n") # Print a newline for separation + print("**Overall Status**") + print("-------------------------------") + # Continue with additional prints or other logic if int(total_fail) >=1: # Print the message in bold print("🔴 - **CI Failure: There are failed tests.**\n\n") From 18fb21f78e5b465dd9985d21328814a60a13af09 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 12:19:28 -0700 Subject: [PATCH 65/77] if conditions --- .github/workflows/unit-testing.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index f18dcb3a1c..7c244e0663 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -5,6 +5,8 @@ on: jobs: unit-testing: runs-on: ubuntu-latest + outputs: + has_detections: ${{ steps.check_detections.outputs.has_detections }} if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job # needs: [validate-tag-if-present, quit-for-dependabot] steps: @@ -33,11 +35,25 @@ jobs: git pull > /dev/null 2>&1 git checkout ${{ github.head_ref }} echo "The target branch for this PR is ${{ github.base_ref }}" - contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} + # contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} + output = $(echo "With Detection Testing Mode 'Changes', there were [0] detections found to test.") echo "contentctl test - COMPLETED" continue-on-error: true + + - name: Check detections + id: check_detections + run: | + detections="${{ steps.contentctl_run.outputs.detections }}" + if [[ "$detections" == "0" ]]; then + echo "::set-output name=has_detections::false" + echo "No detections to test." + else + echo "::set-output name=has_detections::true" + echo "Proceeding with $detections detections to test." + fi - name: store_artifacts + if: ${{ steps.check_detections.outputs.has_detections == 'true' }} uses: actions/upload-artifact@v4 with: name: test_summary_results @@ -47,10 +63,12 @@ jobs: continue-on-error: true - name: Print entire test_results/summary.yml + if: ${{ steps.check_detections.outputs.has_detections == 'true' }} run: cat test_results/summary.yml continue-on-error: true - name: Check the test_results/summary.yml for pass/fail. + if: ${{ steps.check_detections.outputs.has_detections == 'true' }} run: | echo "This job will fail if there are failures in unit-testing" python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY From c9ae4161d467fa19e9d01950eac7c488806269a6 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 12:21:09 -0700 Subject: [PATCH 66/77] varaibles --- .github/workflows/unit-testing.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 7c244e0663..ea47b2855a 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -37,6 +37,8 @@ jobs: echo "The target branch for this PR is ${{ github.base_ref }}" # contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} output = $(echo "With Detection Testing Mode 'Changes', there were [0] detections found to test.") + detections=$(echo "$output" | grep -oP 'there were \[\K[0-9]+(?=\] detections found to test)') + echo "::set-output name=detections::$detections" echo "contentctl test - COMPLETED" continue-on-error: true From 1f6ac618f955726c2f400bdde3f01fd0c015693d Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 12:21:44 -0700 Subject: [PATCH 67/77] updating bash variables --- .github/workflows/unit-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index ea47b2855a..4e3c7f9727 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -36,7 +36,7 @@ jobs: git checkout ${{ github.head_ref }} echo "The target branch for this PR is ${{ github.base_ref }}" # contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} - output = $(echo "With Detection Testing Mode 'Changes', there were [0] detections found to test.") + output=$(echo "With Detection Testing Mode 'Changes', there were [0] detections found to test.") detections=$(echo "$output" | grep -oP 'there were \[\K[0-9]+(?=\] detections found to test)') echo "::set-output name=detections::$detections" echo "contentctl test - COMPLETED" From 19fc2a6ea37c94904de535733eff92f67ebfe9c8 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 12:28:54 -0700 Subject: [PATCH 68/77] revert --- .github/workflows/unit-testing.yml | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 4e3c7f9727..541591b022 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -35,27 +35,12 @@ jobs: git pull > /dev/null 2>&1 git checkout ${{ github.head_ref }} echo "The target branch for this PR is ${{ github.base_ref }}" - # contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} - output=$(echo "With Detection Testing Mode 'Changes', there were [0] detections found to test.") - detections=$(echo "$output" | grep -oP 'there were \[\K[0-9]+(?=\] detections found to test)') - echo "::set-output name=detections::$detections" + contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} echo "contentctl test - COMPLETED" continue-on-error: true - - name: Check detections - id: check_detections - run: | - detections="${{ steps.contentctl_run.outputs.detections }}" - if [[ "$detections" == "0" ]]; then - echo "::set-output name=has_detections::false" - echo "No detections to test." - else - echo "::set-output name=has_detections::true" - echo "Proceeding with $detections detections to test." - fi - name: store_artifacts - if: ${{ steps.check_detections.outputs.has_detections == 'true' }} uses: actions/upload-artifact@v4 with: name: test_summary_results @@ -65,12 +50,10 @@ jobs: continue-on-error: true - name: Print entire test_results/summary.yml - if: ${{ steps.check_detections.outputs.has_detections == 'true' }} run: cat test_results/summary.yml continue-on-error: true - - name: Check the test_results/summary.yml for pass/fail. - if: ${{ steps.check_detections.outputs.has_detections == 'true' }} + - name: Check the test_results/summary.yml for pass/fail. run: | echo "This job will fail if there are failures in unit-testing" python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY From b1e19f8c1a666abd18b6b24da25d200b702e1750 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 13:00:11 -0700 Subject: [PATCH 69/77] testing formatting --- .github/workflows/format_test_results.py | 6 +++++- .../application/crushftp_server_side_template_injection.yml | 2 +- detections/endpoint/7zip_commandline_to_smb_share_path.yml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index f49fd11538..8ef274b219 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -24,6 +24,7 @@ def main(): # Extract total_fail value and debug print it total_fail = data['summary']['total_fail'] + total_detections = data['summary']['total_detections'] print("**Download the job artifacts of this run and view complete summary in test_results/summary.yml for troubleshooting failures.**\n") print(" 📝 **Experimental or manual_test detections are not tested** 📝 **\n") print(f"Extracted total_fail: [{total_fail}]\n") @@ -55,9 +56,12 @@ def main(): # Print the message in bold print("🔴 - **CI Failure: There are failed tests.**\n\n") sys.exit(1) - else: + if int(total_fail) < 1: print("🟢 - **CI Success: No failed tests.**\n\n") sys.exit(0) + if int(total_detections) < 1: + print("🟢 - **CI Success: No detections to test**\n\n") + sys.exit(0) if __name__ == "__main__": main() \ No newline at end of file diff --git a/detections/application/crushftp_server_side_template_injection.yml b/detections/application/crushftp_server_side_template_injection.yml index 71c70e6b6c..9a9ac0ebe0 100644 --- a/detections/application/crushftp_server_side_template_injection.yml +++ b/detections/application/crushftp_server_side_template_injection.yml @@ -1,6 +1,6 @@ name: CrushFTP Server Side Template Injection id: ccf6b7a3-bd39-4bc9-a949-143a8d640dbc -version: 1 +version: 2 date: '2024-05-16' author: Michael Haag, Splunk data_source: [] diff --git a/detections/endpoint/7zip_commandline_to_smb_share_path.yml b/detections/endpoint/7zip_commandline_to_smb_share_path.yml index 76c046406f..67e9c3a747 100644 --- a/detections/endpoint/7zip_commandline_to_smb_share_path.yml +++ b/detections/endpoint/7zip_commandline_to_smb_share_path.yml @@ -1,6 +1,6 @@ name: 7zip CommandLine To SMB Share Path id: 01d29b48-ff6f-11eb-b81e-acde48001123 -version: 2 +version: 3 date: '2024-05-17' author: Teoderick Contreras, Splunk status: production From 3d3482316b4fbb5aac757cc2e43c8feaa9dbdf4e Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 13:24:31 -0700 Subject: [PATCH 70/77] updating print --- .github/workflows/format_test_results.py | 2 +- .github/workflows/unit-testing.yml | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index 8ef274b219..aeb804516e 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -45,7 +45,7 @@ def main(): else: print(f"{name:<80} | 🟢 {status:<6} | {test_type:<10} | {'-':<50}") # Print table footer - print(f"{'----':<80} | {'------':<6} | {'---------':<10} | {'---------':<50}") + # print(f"{'----':<80} | {'------':<6} | {'---------':<10} | {'---------':<50}") # Check if total_fail is a valid integer and greater than or equal to one print("\n") # Print a newline for separation diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 541591b022..a6f0f00fb7 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -27,8 +27,8 @@ jobs: python -m pip install --upgrade pip pip install contentctl - - - name: Run ContentCTL test for changes against develop + # Running contentctl test with a few arguments, before running the command make sure you checkout into the current branch of the pull request. This step only performs unit testing on all the changes against the target-branch. In most cases this target branch will be develop + - name: Run ContentCTL test for changes against target branch run: | echo "Current Branch (Head Ref): ${{ github.head_ref }}" echo "Target Branch (Base Ref): ${{ github.base_ref }}" @@ -39,7 +39,7 @@ jobs: echo "contentctl test - COMPLETED" continue-on-error: true - + # Store test_results/summary.yml and dist/DA-ESS-ContentUpdate-latest.tar.gz to job artifact-test_summary_results.zip - name: store_artifacts uses: actions/upload-artifact@v4 with: @@ -49,10 +49,12 @@ jobs: dist/DA-ESS-ContentUpdate-latest.tar.gz continue-on-error: true + # Print entire result summary so that the users can view it in the Github Actions logs - name: Print entire test_results/summary.yml run: cat test_results/summary.yml continue-on-error: true - + + # Run a simple custom script created to pretty print results in a markdown friendly format in Github Actions Summary - name: Check the test_results/summary.yml for pass/fail. run: | echo "This job will fail if there are failures in unit-testing" From cdec009e6a49a656ffe2624a36864b116368d1e2 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 14:05:44 -0700 Subject: [PATCH 71/77] CI trigger From 8d7a909f85f82937ce3dc0d267f5587e1685e7ef Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 14:09:38 -0700 Subject: [PATCH 72/77] test no changes in detections --- .../application/crushftp_server_side_template_injection.yml | 2 +- detections/endpoint/7zip_commandline_to_smb_share_path.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/detections/application/crushftp_server_side_template_injection.yml b/detections/application/crushftp_server_side_template_injection.yml index 9a9ac0ebe0..71c70e6b6c 100644 --- a/detections/application/crushftp_server_side_template_injection.yml +++ b/detections/application/crushftp_server_side_template_injection.yml @@ -1,6 +1,6 @@ name: CrushFTP Server Side Template Injection id: ccf6b7a3-bd39-4bc9-a949-143a8d640dbc -version: 2 +version: 1 date: '2024-05-16' author: Michael Haag, Splunk data_source: [] diff --git a/detections/endpoint/7zip_commandline_to_smb_share_path.yml b/detections/endpoint/7zip_commandline_to_smb_share_path.yml index 67e9c3a747..76c046406f 100644 --- a/detections/endpoint/7zip_commandline_to_smb_share_path.yml +++ b/detections/endpoint/7zip_commandline_to_smb_share_path.yml @@ -1,6 +1,6 @@ name: 7zip CommandLine To SMB Share Path id: 01d29b48-ff6f-11eb-b81e-acde48001123 -version: 3 +version: 2 date: '2024-05-17' author: Teoderick Contreras, Splunk status: production From 847e67aaaf1f00935864b63328ef29f5a8bdc23f Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 14:28:15 -0700 Subject: [PATCH 73/77] udpating job names --- .github/workflows/{validate-and-build.yml => build.yml} | 0 .github/workflows/format_test_results.py | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{validate-and-build.yml => build.yml} (100%) diff --git a/.github/workflows/validate-and-build.yml b/.github/workflows/build.yml similarity index 100% rename from .github/workflows/validate-and-build.yml rename to .github/workflows/build.yml diff --git a/.github/workflows/format_test_results.py b/.github/workflows/format_test_results.py index aeb804516e..d142af3321 100644 --- a/.github/workflows/format_test_results.py +++ b/.github/workflows/format_test_results.py @@ -30,7 +30,7 @@ def main(): print(f"Extracted total_fail: [{total_fail}]\n") # Print all unit test details first - print("**Unit Test Details:**\n") + print(" 🏗️⚒️ **Unit Test Details:**\n") print(f"{'Name':<80} | {'Status':<6} | {'Test Type':<10} | {'Exception':<50}") print(f"{'----':<80} | {'------':<6} | {'---------':<10} | {'---------':<50}") for detection in data['tested_detections']: @@ -60,7 +60,7 @@ def main(): print("🟢 - **CI Success: No failed tests.**\n\n") sys.exit(0) if int(total_detections) < 1: - print("🟢 - **CI Success: No detections to test**\n\n") + print("🔵 - **CI Success: No detections to test**\n\n") sys.exit(0) if __name__ == "__main__": From e16828647b5edc3cf93429e4bff61d94261a4f82 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 14:32:29 -0700 Subject: [PATCH 74/77] updating to build --- .github/workflows/build.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 95b35acd10..deb5286370 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,16 +1,16 @@ -name: validate-and-build +name: build on: pull_request: types: [opened, reopened, synchronize] jobs: - validate-and-build: + build: #Note that the CircleCI job used a Container. The way to do this with Github Actions #is to first start up a Virtual Machine, then we can by following: # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idcontainer runs-on: ubuntu-latest steps: - name: Check out the repository code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: @@ -27,10 +27,6 @@ jobs: pip install contentctl git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git - - name: Running validate - run: | - contentctl validate - - name: Running build with enrichments run: | contentctl build --enrichments From a16995743f34f8f57e0074072330f8bca328527d Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 14:40:04 -0700 Subject: [PATCH 75/77] testing images --- .github/workflows/unit-testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index a6f0f00fb7..d684ddbf48 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -60,4 +60,5 @@ jobs: echo "This job will fail if there are failures in unit-testing" python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY echo "The Unit testing is completed. See details in the unit-testing job summary UI " + echo 'Image' >> "$GITHUB_STEP_SUMMARY" \ No newline at end of file From da14348a16784a56ba9e9bd431b6c233e5f0be5e Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 14:54:25 -0700 Subject: [PATCH 76/77] final update --- .github/workflows/unit-testing.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index d684ddbf48..a6f0f00fb7 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -60,5 +60,4 @@ jobs: echo "This job will fail if there are failures in unit-testing" python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY echo "The Unit testing is completed. See details in the unit-testing job summary UI " - echo 'Image' >> "$GITHUB_STEP_SUMMARY" \ No newline at end of file From b36c64f8ed6ef8215e823d9cdcfa4312478a1572 Mon Sep 17 00:00:00 2001 From: Bhavin Patel Date: Tue, 9 Jul 2024 17:02:32 -0700 Subject: [PATCH 77/77] remove jq --- .github/workflows/build.yml | 6 +----- .github/workflows/unit-testing.yml | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index deb5286370..9d15eb041c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,11 +17,7 @@ jobs: python-version: '3.11' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified - - name: Install System Packages - run: | - sudo apt update -qq - sudo apt install jq -qq - + - name: Install Python Dependencies and ContentCTL and Atomic Red Team run: | pip install contentctl diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index a6f0f00fb7..970339f562 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -5,8 +5,6 @@ on: jobs: unit-testing: runs-on: ubuntu-latest - outputs: - has_detections: ${{ steps.check_detections.outputs.has_detections }} if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job # needs: [validate-tag-if-present, quit-for-dependabot] steps: @@ -20,10 +18,8 @@ jobs: python-version: '3.11' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified - - name: Install System Packages and contentctl + - name: Install Python Dependencies and ContentCTL run: | - sudo apt update -qq - sudo apt install jq -qq python -m pip install --upgrade pip pip install contentctl