ci: twister: split publishing results out of main workflow

.github/workflows/twister-prep.yaml

@@ -0,0 +1,150 @@
+name: Prep
+
+on:
+  workflow_call:
+    outputs:
+      subset:
+        description: subset
+        value: ${{ jobs.prep_push.outputs.subset }} || ${{ jobs.prep_pr.outputs.subset }}
+      size:
+        description: size
+        value: ${{ jobs.prep_push.outputs.size }} || ${{ jobs.prep_pr.outputs.size }}
+      fullrun:
+        description: fullrun
+        value: ${{ jobs.prep_push.outputs.fullrun }} || ${{ jobs.prep_pr.outputs.size }}
+
+jobs:
+  prep_pr:
+    if: github.repository_owner == 'zephyrproject-rtos' && github.event_name == 'pull_request_target'
+    runs-on:
+      group: zephyr-runner-v2-linux-x64-4xlarge
+    container:
+      image: ghcr.io/zephyrproject-rtos/ci-repo-cache:v0.27.4.20241026
+      options: '--entrypoint /bin/bash'
+    outputs:
+      subset: ${{ steps.output-services.outputs.subset }}
+      size: ${{ steps.output-services.outputs.size }}
+      fullrun: ${{ steps.output-services.outputs.fullrun }}
+    env:
+      MATRIX_SIZE: 10
+      PUSH_MATRIX_SIZE: 20
+      DAILY_MATRIX_SIZE: 80
+      BSIM_OUT_PATH: /opt/bsim/
+      BSIM_COMPONENTS_PATH: /opt/bsim/components
+      TESTS_PER_BUILDER: 700
+      COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
+      BASE_REF: ${{ github.base_ref }}
+    steps:
+      - name: Apply container owner mismatch workaround
+        run: |
+          # FIXME: The owner UID of the GITHUB_WORKSPACE directory may not
+          #        match the container user UID because of the way GitHub
+          #        Actions runner is implemented. Remove this workaround when
+          #        GitHub comes up with a fundamental fix for this problem.
+          git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Print cloud service information
+        run: |
+          echo "ZEPHYR_RUNNER_CLOUD_PROVIDER = ${ZEPHYR_RUNNER_CLOUD_PROVIDER}"
+          echo "ZEPHYR_RUNNER_CLOUD_NODE = ${ZEPHYR_RUNNER_CLOUD_NODE}"
+          echo "ZEPHYR_RUNNER_CLOUD_POD = ${ZEPHYR_RUNNER_CLOUD_POD}"
+
+      - name: Clone cached Zephyr repository
+        if: github.event_name == 'pull_request_target'
+        continue-on-error: true
+        run: |
+          git clone --shared /repo-cache/zephyrproject/zephyr .
+          git remote set-url origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}
+
+      - name: Checkout
+        if: github.event_name == 'pull_request_target'
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Environment Setup
+        if: github.event_name == 'pull_request_target'
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "Zephyr Bot"
+          rm -fr ".git/rebase-apply"
+          git rebase origin/${BASE_REF}
+          git clean -f -d
+          git log  --pretty=oneline | head -n 10
+          west init -l . || true
+          west config manifest.group-filter -- +ci,+optional
+          west config --global update.narrow true
+          west update --path-cache /repo-cache/zephyrproject 2>&1 1> west.update.log || west update --path-cache /repo-cache/zephyrproject 2>&1 1> west.update.log || ( rm -rf ../modules ../bootloader ../tools && west update --path-cache /repo-cache/zephyrproject)
+          west forall -c 'git reset --hard HEAD'
+
+          echo "ZEPHYR_SDK_INSTALL_DIR=/opt/toolchains/zephyr-sdk-$( cat SDK_VERSION )" >> $GITHUB_ENV
+
+      - name: Generate Test Plan with Twister
+        if: github.event_name == 'pull_request_target'
+        id: test-plan
+        run: |
+          export ZEPHYR_BASE=${PWD}
+          export ZEPHYR_TOOLCHAIN_VARIANT=zephyr
+          python3 ./scripts/ci/test_plan.py -c origin/${BASE_REF}.. --pull-request -t $TESTS_PER_BUILDER
+          if [ -s .testplan ]; then
+            cat .testplan >> $GITHUB_ENV
+          else
+            echo "TWISTER_NODES=${MATRIX_SIZE}" >> $GITHUB_ENV
+          fi
+          rm -f testplan.json .testplan
+
+      - name: Determine matrix size
+        id: output-services
+        run: |
+          if [ -n "${TWISTER_NODES}" ]; then
+            subset="[$(seq -s',' 1 ${TWISTER_NODES})]"
+          else
+            subset="[$(seq -s',' 1 ${MATRIX_SIZE})]"
+          fi
+          size=${TWISTER_NODES}
+
+          echo "subset=${subset}" >> $GITHUB_OUTPUT
+          echo "size=${size}" >> $GITHUB_OUTPUT
+          echo "fullrun=${TWISTER_FULL}" >> $GITHUB_OUTPUT
+
+  prep_push:
+    if: github.repository_owner == 'zephyrproject-rtos' && (github.event_name == 'push' || github.event_name == 'schedule')
+    runs-on: ubuntu-22.04
+    outputs:
+      subset: ${{ steps.output-services.outputs.subset }}
+      size: ${{ steps.output-services.outputs.size }}
+      fullrun: ${{ steps.output-services.outputs.fullrun }}
+    env:
+      MATRIX_SIZE: 10
+      PUSH_MATRIX_SIZE: 20
+      DAILY_MATRIX_SIZE: 80
+      BSIM_OUT_PATH: /opt/bsim/
+      BSIM_COMPONENTS_PATH: /opt/bsim/components
+      TESTS_PER_BUILDER: 700
+      COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
+      BASE_REF: ${{ github.base_ref }}
+    steps:
+      - name: Print cloud service information
+        run: |
+          echo "ZEPHYR_RUNNER_CLOUD_PROVIDER = ${ZEPHYR_RUNNER_CLOUD_PROVIDER}"
+          echo "ZEPHYR_RUNNER_CLOUD_NODE = ${ZEPHYR_RUNNER_CLOUD_NODE}"
+          echo "ZEPHYR_RUNNER_CLOUD_POD = ${ZEPHYR_RUNNER_CLOUD_POD}"
+
+      - name: Determine matrix size
+        id: output-services
+        run: |
+          if [ "${{github.event_name}}" = "push" ]; then
+            subset="[$(seq -s',' 1 ${PUSH_MATRIX_SIZE})]"
+            size=${MATRIX_SIZE}
+          elif [ "${{github.event_name}}" = "schedule" -a "${{github.repository}}" = "zephyrproject-rtos/zephyr" ]; then
+            subset="[$(seq -s',' 1 ${DAILY_MATRIX_SIZE})]"
+            size=${DAILY_MATRIX_SIZE}
+          else
+            size=0
+          fi
+
+          echo "subset=${subset}" >> $GITHUB_OUTPUT
+          echo "size=${size}" >> $GITHUB_OUTPUT
+          echo "fullrun=${TWISTER_FULL}" >> $GITHUB_OUTPUT

.github/workflows/twister-publish.yaml

@@ -0,0 +1,47 @@
+name: Publish Twister Test Results
+
+on:
+  workflow_run:
+    workflows: ["Run tests with twister"]
+    branches:
+      - main
+      - v*
+    types:
+      - completed
+
+jobs:
+  upload-to-elasticsearch:
+    if: github.repository == 'zephyrproject-rtos/zephyr'
+    env:
+      ELASTICSEARCH_KEY: ${{ secrets.ELASTICSEARCH_KEY }}
+      ELASTICSEARCH_SERVER: "https://elasticsearch.zephyrproject.io:443"
+    runs-on: ubuntu-22.04
+    steps:
+      # Needed for elasticearch and upload script
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Download Artifacts
+        uses: dawidd6/action-download-artifact@v6
+        with:
+          path: artifacts
+          workflow: twister.yml
+          run_id: ${{ github.event.workflow_run.id }}
+
+      - name: Upload to elasticsearch
+        run: |
+          pip3 install elasticsearch
+          # set run date on upload to get consistent and unified data across the matrix.
+          run_date=`date --iso-8601=minutes`
+          if [ "${{github.event_name}}" = "push" ]; then
+          python3 ./scripts/ci/upload_test_results_es.py -r ${run_date} \
+          --run-attempt ${{github.run_attempt}} \
+          --index zephyr-main-ci-push-1 artifacts/*/*/twister.json
+          elif [ "${{github.event_name}}" = "schedule" ]; then
+          python3 ./scripts/ci/upload_test_results_es.py -r ${run_date} \
+          --run-attempt ${{github.run_attempt}} \
+          --index zephyr-main-ci-weekly-1 artifacts/*/*/twister.json
+          fi

.github/workflows/twister.yaml

@@ -21,108 +21,7 @@ concurrency:
 
 jobs:
   twister-build-prep:
-    if: github.repository_owner == 'zephyrproject-rtos'
-    runs-on:
-      group: zephyr-runner-v2-linux-x64-4xlarge
-    container:
-      image: ghcr.io/zephyrproject-rtos/ci-repo-cache:v0.27.4.20241026
-      options: '--entrypoint /bin/bash'
-    outputs:
-      subset: ${{ steps.output-services.outputs.subset }}
-      size: ${{ steps.output-services.outputs.size }}
-      fullrun: ${{ steps.output-services.outputs.fullrun }}
-    env:
-      MATRIX_SIZE: 10
-      PUSH_MATRIX_SIZE: 20
-      DAILY_MATRIX_SIZE: 80
-      BSIM_OUT_PATH: /opt/bsim/
-      BSIM_COMPONENTS_PATH: /opt/bsim/components
-      TESTS_PER_BUILDER: 700
-      COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
-      BASE_REF: ${{ github.base_ref }}
-    steps:
-      - name: Apply container owner mismatch workaround
-        run: |
-          # FIXME: The owner UID of the GITHUB_WORKSPACE directory may not
-          #        match the container user UID because of the way GitHub
-          #        Actions runner is implemented. Remove this workaround when
-          #        GitHub comes up with a fundamental fix for this problem.
-          git config --global --add safe.directory ${GITHUB_WORKSPACE}
-
-      - name: Print cloud service information
-        run: |
-          echo "ZEPHYR_RUNNER_CLOUD_PROVIDER = ${ZEPHYR_RUNNER_CLOUD_PROVIDER}"
-          echo "ZEPHYR_RUNNER_CLOUD_NODE = ${ZEPHYR_RUNNER_CLOUD_NODE}"
-          echo "ZEPHYR_RUNNER_CLOUD_POD = ${ZEPHYR_RUNNER_CLOUD_POD}"
-
-      - name: Clone cached Zephyr repository
-        if: github.event_name == 'pull_request_target'
-        continue-on-error: true
-        run: |
-          git clone --shared /repo-cache/zephyrproject/zephyr .
-          git remote set-url origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}
-
-      - name: Checkout
-        if: github.event_name == 'pull_request_target'
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
-          persist-credentials: false
-
-      - name: Environment Setup
-        if: github.event_name == 'pull_request_target'
-        run: |
-          git config --global user.email "[email protected]"
-          git config --global user.name "Zephyr Bot"
-          rm -fr ".git/rebase-apply"
-          git rebase origin/${BASE_REF}
-          git clean -f -d
-          git log  --pretty=oneline | head -n 10
-          west init -l . || true
-          west config manifest.group-filter -- +ci,+optional
-          west config --global update.narrow true
-          west update --path-cache /repo-cache/zephyrproject 2>&1 1> west.update.log || west update --path-cache /repo-cache/zephyrproject 2>&1 1> west.update.log || ( rm -rf ../modules ../bootloader ../tools && west update --path-cache /repo-cache/zephyrproject)
-          west forall -c 'git reset --hard HEAD'
-
-          echo "ZEPHYR_SDK_INSTALL_DIR=/opt/toolchains/zephyr-sdk-$( cat SDK_VERSION )" >> $GITHUB_ENV
-
-      - name: Generate Test Plan with Twister
-        if: github.event_name == 'pull_request_target'
-        id: test-plan
-        run: |
-          export ZEPHYR_BASE=${PWD}
-          export ZEPHYR_TOOLCHAIN_VARIANT=zephyr
-          python3 ./scripts/ci/test_plan.py -c origin/${BASE_REF}.. --pull-request -t $TESTS_PER_BUILDER
-          if [ -s .testplan ]; then
-            cat .testplan >> $GITHUB_ENV
-          else
-            echo "TWISTER_NODES=${MATRIX_SIZE}" >> $GITHUB_ENV
-          fi
-          rm -f testplan.json .testplan
-
-      - name: Determine matrix size
-        id: output-services
-        run: |
-          if [ "${{github.event_name}}" = "pull_request_target" ]; then
-            if [ -n "${TWISTER_NODES}" ]; then
-              subset="[$(seq -s',' 1 ${TWISTER_NODES})]"
-            else
-              subset="[$(seq -s',' 1 ${MATRIX_SIZE})]"
-            fi
-            size=${TWISTER_NODES}
-          elif [ "${{github.event_name}}" = "push" ]; then
-            subset="[$(seq -s',' 1 ${PUSH_MATRIX_SIZE})]"
-            size=${MATRIX_SIZE}
-          elif [ "${{github.event_name}}" = "schedule" -a "${{github.repository}}" = "zephyrproject-rtos/zephyr" ]; then
-            subset="[$(seq -s',' 1 ${DAILY_MATRIX_SIZE})]"
-            size=${DAILY_MATRIX_SIZE}
-          else
-            size=0
-          fi
-          echo "subset=${subset}" >> $GITHUB_OUTPUT
-          echo "size=${size}" >> $GITHUB_OUTPUT
-          echo "fullrun=${TWISTER_FULL}" >> $GITHUB_OUTPUT
+    uses: ./.github/workflows/twister-prep.yaml
 
   twister-build:
     runs-on:
@@ -230,6 +129,7 @@ jobs:
 
       - if: github.event_name == 'push'
         name: Run Tests with Twister (Push)
+        id: run_twister
         run: |
           export ZEPHYR_BASE=${PWD}
           export ZEPHYR_TOOLCHAIN_VARIANT=zephyr
@@ -243,6 +143,7 @@ jobs:
 
       - if: github.event_name == 'pull_request_target'
         name: Run Tests with Twister (Pull Request)
+        id: run_twister_pr
         run: |
           rm -f testplan.json
           export ZEPHYR_BASE=${PWD}
@@ -258,6 +159,7 @@ jobs:
 
       - if: github.event_name == 'schedule'
         name: Run Tests with Twister (Daily)
+        id: run_twister_sched
         run: |
           export ZEPHYR_BASE=${PWD}
           export ZEPHYR_TOOLCHAIN_VARIANT=zephyr
@@ -306,42 +208,18 @@ jobs:
 
   twister-test-results:
     name: "Publish Unit Tests Results"
-    env:
-      ELASTICSEARCH_KEY: ${{ secrets.ELASTICSEARCH_KEY }}
-      ELASTICSEARCH_SERVER: "https://elasticsearch.zephyrproject.io:443"
-    needs: twister-build
+    needs:
+      - twister-build
     runs-on: ubuntu-22.04
     # the build-and-test job might be skipped, we don't need to run this job then
     if: success() || failure()
 
     steps:
-      # Needed for elasticearch and upload script
-      - if: github.event_name == 'push' || github.event_name == 'schedule'
-        name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
       - name: Download Artifacts
         uses: actions/download-artifact@v4
         with:
           path: artifacts
 
-      - if: github.event_name == 'push' || github.event_name == 'schedule'
-        name: Upload to elasticsearch
-        run: |
-          pip3 install elasticsearch
-          # set run date on upload to get consistent and unified data across the matrix.
-          run_date=`date --iso-8601=minutes`
-          if [ "${{github.event_name}}" = "push" ]; then
-            python3 ./scripts/ci/upload_test_results_es.py -r ${run_date} \
-              --index zephyr-main-ci-push-1 artifacts/*/*/twister.json
-          elif [ "${{github.event_name}}" = "schedule" ]; then
-            python3 ./scripts/ci/upload_test_results_es.py -r ${run_date} \
-              --index zephyr-main-ci-weekly-1 artifacts/*/*/twister.json
-          fi
-
       - name: Merge Test Results
         run: |
           pip3 install junitparser junit2html