From c6c1b1f548a0886db6bf991c73b8f4464e0112e4 Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Fri, 25 Oct 2024 15:09:44 +0200 Subject: [PATCH 1/4] Add workflow to start Prow tests when relevant Use CTF to identify changes in OCP rules, identify the profiles selecting these rules, and start Prow tests. --- .github/workflows/ocp-test-profiles.yaml | 126 +++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 .github/workflows/ocp-test-profiles.yaml diff --git a/.github/workflows/ocp-test-profiles.yaml b/.github/workflows/ocp-test-profiles.yaml new file mode 100644 index 00000000000..069fc705086 --- /dev/null +++ b/.github/workflows/ocp-test-profiles.yaml @@ -0,0 +1,126 @@ +name: Trigger OCP Tests When Relevant +on: + pull_request: + branches: [ master, 'stabilization*' ] +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.run_id }} + cancel-in-progress: true +jobs: + check-and-trigger-ocp-prow-tests: + name: Identify rules changed in PR and test them in OCP Prow + runs-on: ubuntu-latest + container: + image: fedora:latest + steps: + - name: Install Deps + run: dnf install -y cmake make openscap-utils python3-pyyaml python3-jinja2 git python3-deepdiff python3-requests jq python3-pip nodejs + - name: Install deps python + run: pip install gitpython xmldiff + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + with: + fetch-depth: 0 + - name: Checkout (CTF) + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + with: + repository: ComplianceAsCode/content-test-filtering + path: ctf + # https://github.com/actions/checkout/issues/766 + - name: Set git safe directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Find forking point + env: + BASE_BRANCH: ${{ github.base_ref }} + run: echo "FORK_POINT=$(git merge-base origin/$BASE_BRANCH ${{ github.event.pull_request.head.sha }})" >> $GITHUB_OUTPUT + id: fork_point + - name: Detect content changes in the PR + run: python3 ./ctf/content_test_filtering.py pr --base ${{ steps.fork_point.outputs.FORK_POINT }} --remote_repo ${{ github.server_url }}/${{ github.repository }} --verbose --rule --output json ${{ github.event.pull_request.number }} > ctf-output.json + - name: Test if there are no content changes + run: echo "CTF_OUTPUT_SIZE=$(stat --printf="%s" ctf-output.json)" >> $GITHUB_OUTPUT + id: ctf + - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 + if: ${{ steps.ctf.outputs.CTF_OUTPUT_SIZE != '0' }} + with: + name: ctf-output + path: ctf-output.json + - name: Print changes to content detected if any + if: ${{ steps.ctf.outputs.CTF_OUTPUT_SIZE != '0' }} + run: cat ctf-output.json + - name: Get product attribute + if: ${{ steps.ctf.outputs.CTF_OUTPUT_SIZE != '0' }} + id: product + uses: notiz-dev/github-action-json-property@a5a9c668b16513c737c3e1f8956772c99c73f6e8 # v0.2.0 + with: + path: 'ctf-output.json' + prop_path: 'product' + + - name: Build product OCP and RHCOS content + if: ${{ steps.ctf.outputs.CTF_OUTPUT_SIZE != '0' && (contains(steps.product.outputs.prop, 'ocp4') || contains(steps.product.outputs.prop, 'rhcos4')) }} + run: ./build_product -d ocp4 rhcos4 + + - name: Process list of rules into a list of product-profiles to test + if: ${{ steps.ctf.outputs.CTF_OUTPUT_SIZE != '0' && (contains(steps.product.outputs.prop, 'ocp4') || contains(steps.product.outputs.prop, 'rhcos4')) }} + id: profiles_to_test + run: | + OCP_VERSIONS=(4.17 4.16) + RULES=$(cat ctf-output.json | jq -r '.rules[]') + + # Let's grab one profile for each changed rule + PROFILES=() + ALL_PROFILES=() + + # Let's consistently grab a random profile for each rule, in order to do that we use the + # PR number as the seed + RANDOM=${{ github.event.pull_request.number }} + for rule in $RULES; do + readarray -t TEMP <<< $(grep -lr -e "- ${rule}\$" build/*/profiles | sort) + ALL_PROFILES+=(${TEMP[@]}) + PROFILES+=(${TEMP[$(($RANDOM%(${#TEMP[@]})))]}) + done + + # Sort and ensure that the profiles are unique + readarray -t UNIQUE_PROFILES <<< $(echo ${PROFILES[@]} | tr ' ' '\n' | sort -u | tr '\n' ' ') + readarray -t ALL_UNIQUE_PROFILES <<< $(echo ${ALL_PROFILES[@]} | tr ' ' '\n' | sort -u | tr '\n' ' ') + + # Craft a command to trigger tests + COMMAND=$(for profile in ${UNIQUE_PROFILES[@]}; do + for OCP_V in "${OCP_VERSIONS[@]}"; do + echo ${profile} | sed 's/build\/\(.*\)\/profiles\/\(.*\)\.profile/\/test '"${OCP_V}"'-e2e-aws-\1-\2/' + done + done) + + # COMMAND is a multiline string, so we need to set it this way + { + echo 'TEST_PROFILES_COMMAND<> $GITHUB_OUTPUT + + # Format all identified profiles for display + ALL_PROFILES_FORMATTED=$(for profile in ${ALL_UNIQUE_PROFILES[@]}; do + echo ${profile} | sed 's/build\/\(.*\)\/profiles\/\(.*\)\.profile/- `-e2e-aws-\1-\2`/' + done) + { + echo 'ALL_PROFILES_COMMENT<> $GITHUB_OUTPUT + - uses: thollander/actions-comment-pull-request@e2c37e53a7d2227b61585343765f73a9ca57eda9 # v2 + if: ${{ steps.profiles_to_test.outputs.TEST_PROFILES_COMMAND != '' }} + with: + message: | + :robot: Trigger prow tests based on changed rules + + ${{ steps.profiles_to_test.outputs.TEST_PROFILES_COMMAND }} + + Note: if a test is not started it could be that a CI Job is not configure for that particular profile or product. + +
+ Click here to see all the relevant profiles + + ${{ steps.profiles_to_test.outputs.ALL_PROFILES_COMMENT}} + +
+ comment-tag: kubernetes_start_prow_tests + pr-number: ${{ github.event.pull_request.number }} + mode: recreate From 0cf1bbf959054f87174a1810a27d8753ba87fbf6 Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Mon, 28 Oct 2024 10:42:49 +0100 Subject: [PATCH 2/4] Filter out profiles that are not tested by Prow Not all profiles are configured in Prow. This uses a hardcoded list of profiles to filter out the "untestable" ones. --- .github/workflows/ocp-test-profiles.yaml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ocp-test-profiles.yaml b/.github/workflows/ocp-test-profiles.yaml index 069fc705086..fdf864d7d0f 100644 --- a/.github/workflows/ocp-test-profiles.yaml +++ b/.github/workflows/ocp-test-profiles.yaml @@ -74,8 +74,25 @@ jobs: RANDOM=${{ github.event.pull_request.number }} for rule in $RULES; do readarray -t TEMP <<< $(grep -lr -e "- ${rule}\$" build/*/profiles | sort) - ALL_PROFILES+=(${TEMP[@]}) - PROFILES+=(${TEMP[$(($RANDOM%(${#TEMP[@]})))]}) + + # Let's ilter out profiles for which we don't have a CI job configured + # Here is an example of how to quicly update this variable in the future + # TESTED_PROFILES=$(grep -r PROFILE= ./ComplianceAsCode-content-master__4.16.yaml | sort -u | sed 's/.*export PROFILE=\(.*\)/\1/') + # echo -n TESTED_PROFILES=\(${TESTED_PROFILES[@]}\) + # Copy and paste the profiles here + TESTED_PROFILES=(bsi bsi-node cis cis-node e8 high high-node moderate moderate-node pci-dss pci-dss-4-0 pci-dss-node pci-dss-node-4-0 stig stig-node) + + ELIGIBLE_PROFILES=() + for index in "${!TEMP[@]}"; do + for tp in ${TESTED_PROFILES[@]}; do + if [[ ${TEMP[$index]} =~ build\/.*\/profiles\/${tp}\.profile ]]; then + ELIGIBLE_PROFILES+=(${TEMP[$index]}); + fi + done + done + + ALL_PROFILES+=(${ELIGIBLE_PROFILES[@]}) + PROFILES+=(${ELIGIBLE_PROFILES[$(($RANDOM%(${#ELIGIBLE_PROFILES[@]})))]}) done # Sort and ensure that the profiles are unique From c163ed8e52940c8d86ef31e1619084bab42ce840 Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Tue, 29 Oct 2024 22:09:41 +0100 Subject: [PATCH 3/4] Just test on latest OCP for now The version-less jobs are expected to be running on latest OCP. --- .github/workflows/ocp-test-profiles.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ocp-test-profiles.yaml b/.github/workflows/ocp-test-profiles.yaml index fdf864d7d0f..f4ea0fa10df 100644 --- a/.github/workflows/ocp-test-profiles.yaml +++ b/.github/workflows/ocp-test-profiles.yaml @@ -62,7 +62,6 @@ jobs: if: ${{ steps.ctf.outputs.CTF_OUTPUT_SIZE != '0' && (contains(steps.product.outputs.prop, 'ocp4') || contains(steps.product.outputs.prop, 'rhcos4')) }} id: profiles_to_test run: | - OCP_VERSIONS=(4.17 4.16) RULES=$(cat ctf-output.json | jq -r '.rules[]') # Let's grab one profile for each changed rule @@ -101,9 +100,7 @@ jobs: # Craft a command to trigger tests COMMAND=$(for profile in ${UNIQUE_PROFILES[@]}; do - for OCP_V in "${OCP_VERSIONS[@]}"; do - echo ${profile} | sed 's/build\/\(.*\)\/profiles\/\(.*\)\.profile/\/test '"${OCP_V}"'-e2e-aws-\1-\2/' - done + echo ${profile} | sed 's/build\/\(.*\)\/profiles\/\(.*\)\.profile/\/test e2e-aws-\1-\2/' done) # COMMAND is a multiline string, so we need to set it this way From 2999443223c5787b173087aeb67c89a6b70a27cd Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Tue, 29 Oct 2024 22:13:21 +0100 Subject: [PATCH 4/4] Directly source test config for testable profiles Let's directly parse the source of CI Job configuration to know what profiles we can trigger jobs for. Also, move it out of the rule iteration loop. We just need to define it once. --- .github/workflows/ocp-test-profiles.yaml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ocp-test-profiles.yaml b/.github/workflows/ocp-test-profiles.yaml index f4ea0fa10df..df5f81dc950 100644 --- a/.github/workflows/ocp-test-profiles.yaml +++ b/.github/workflows/ocp-test-profiles.yaml @@ -62,6 +62,11 @@ jobs: if: ${{ steps.ctf.outputs.CTF_OUTPUT_SIZE != '0' && (contains(steps.product.outputs.prop, 'ocp4') || contains(steps.product.outputs.prop, 'rhcos4')) }} id: profiles_to_test run: | + # Let's grab the profiles for which we have a CI job configured + PROW_CONFIG=https://raw.githubusercontent.com/openshift/release/refs/heads/master/ci-operator/config/ComplianceAsCode/content/ComplianceAsCode-content-master.yaml + curl -o prow_config.yaml ${PROW_CONFIG} + readarray -t TESTED_PROFILES <<< $(grep -r PROFILE= ./prow_config.yaml | sort -u | sed 's/.*export PROFILE=\(.*\)/\1/') + RULES=$(cat ctf-output.json | jq -r '.rules[]') # Let's grab one profile for each changed rule @@ -74,13 +79,6 @@ jobs: for rule in $RULES; do readarray -t TEMP <<< $(grep -lr -e "- ${rule}\$" build/*/profiles | sort) - # Let's ilter out profiles for which we don't have a CI job configured - # Here is an example of how to quicly update this variable in the future - # TESTED_PROFILES=$(grep -r PROFILE= ./ComplianceAsCode-content-master__4.16.yaml | sort -u | sed 's/.*export PROFILE=\(.*\)/\1/') - # echo -n TESTED_PROFILES=\(${TESTED_PROFILES[@]}\) - # Copy and paste the profiles here - TESTED_PROFILES=(bsi bsi-node cis cis-node e8 high high-node moderate moderate-node pci-dss pci-dss-4-0 pci-dss-node pci-dss-node-4-0 stig stig-node) - ELIGIBLE_PROFILES=() for index in "${!TEMP[@]}"; do for tp in ${TESTED_PROFILES[@]}; do