Download test artifacts (#4127)

.github/actions/on_device_tests/action.yaml

@@ -4,23 +4,22 @@ description: Runs on-device tests.
 runs:
   using: "composite"
   steps:
-    - name: Install requirements
+    - name: Install Requirements
       run: |
         pip3 install grpcio==1.38.0 grpcio-tools==1.38.0
       shell: bash
     - name: Generate gRPC files
       run: |
         python -m grpc_tools.protoc -Itools/ --python_out=tools/ --grpc_python_out=tools/ tools/on_device_tests_gateway.proto
       shell: bash
-    - name: Set up Cloud SDK
+    - name: Set Up Cloud SDK
       uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1
     - name: Set env vars
-      env:
-        WORKFLOW: ${{ github.workflow }}
       run: |
         echo "PROJECT_NAME=$(gcloud config get-value project)" >> $GITHUB_ENV
-        echo "GITHUB_RUN_NUMBER=${GITHUB_RUN_NUMBER}" >> $GITHUB_ENV
-        echo "WORKFLOW=${WORKFLOW}" >> $GITHUB_ENV
+
+        # Test results and logs
+        echo "GCS_RESULTS_PATH=gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }}_${{ matrix.shard }}" >> $GITHUB_ENV
 
         # Boot loader env
         if [ "${COBALT_EVERGREEN_LOADER}" != "null" ]; then
@@ -43,8 +42,9 @@ runs:
           echo "USE_SHARDING=1" >> $GITHUB_ENV
         fi
       shell: bash
-    - name: run ${{ env.SHARD_NAME }} tests on ${{ matrix.platform }} platform
+    - name: Run ${{ env.SHARD_NAME }} Tests on ${{ matrix.platform }} Platform
       env:
+        GCS_PATH: gs://${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}_${{ matrix.config }}
         GITHUB_SHA: ${{ github.sha }}
         GITHUB_TOKEN: ${{ github.token }}
         GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
@@ -57,13 +57,17 @@ runs:
         GITHUB_PR_HEAD_USER_ID: ${{ github.event.pull_request.head.user.id }}
         GITHUB_COMMIT_AUTHOR_USERNAME: ${{ github.event.commits[0].author.username }}
         GITHUB_COMMIT_AUTHOR_EMAIL: ${{ github.event.commits[0].author.email }}
+        GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
+        GITHUB_RUN_NUMBER: ${{ github.run_number }}
+        GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        GITHUB_WORKFLOW: ${{ github.workflow }}
       run: |
         set -uxe
         python3 tools/on_device_tests_gateway_client.py \
           --token ${GITHUB_TOKEN} \
           --change_id "${GITHUB_PR_HEAD_SHA:-$GITHUB_SHA}" \
           trigger \
-          --test_type ${{ env.TEST_TYPE }} \
+          --test_type ${TEST_TYPE} \
           --platform ${{ matrix.target_platform }} \
           --config ${{ matrix.config }} \
           --tag cobalt_github_${GITHUB_EVENT_NAME} \
@@ -74,10 +78,11 @@ runs:
           ${DIMENSION:+"--dimension" "$DIMENSION"} \
           ${USE_SHARDING:+"--unittest_shard_index" "${{ matrix.shard }}"} \
           ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \
-          --archive_path gs://${PROJECT_NAME}-test-artifacts/${WORKFLOW}/${GITHUB_RUN_NUMBER}/${{ matrix.platform }}_${{ matrix.config }}/artifacts.tar \
+          --archive_path "${GCS_PATH}/artifacts.tar" \
+          --gcs_result_path "${GCS_RESULTS_PATH}" \
           --label github \
           --label ${GITHUB_EVENT_NAME} \
-          --label ${WORKFLOW} \
+          --label ${GITHUB_WORKFLOW} \
           --label actor-${GITHUB_ACTOR} \
           --label actor_id-${GITHUB_ACTOR_ID} \
           --label triggering_actor-${GITHUB_TRIGGERING_ACTOR} \
@@ -86,3 +91,115 @@ runs:
           --label author-${GITHUB_PR_HEAD_USER_LOGIN:-$GITHUB_COMMIT_AUTHOR_USERNAME} \
           --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL}
       shell: bash
+    - name: Download ${{ matrix.platform }} Test Results
+      if: always() && env.TEST_TYPE == 'unit_test'
+      run: |
+        # Don't break on error (-e), some commands are expected to fail.
+        set -ux
+
+        COBALT_LOGS_DIR="${GITHUB_WORKSPACE}/cobalt_logs"
+        UNIT_TEST_RESULT_PATH="${GITHUB_WORKSPACE}/unit-test-results"
+        COBALT_XMLS_FILENAME="cobalt_xmls.zip"
+
+        # Forward environment variables for uploading artifacts in later steps.
+        echo "UNIT_TEST_RESULT_PATH=${UNIT_TEST_RESULT_PATH}" >> $GITHUB_ENV
+        echo "COBALT_LOGS_DIR=${COBALT_LOGS_DIR}" >> $GITHUB_ENV
+
+        mkdir -p "${GITHUB_WORKSPACE}/test_results"
+        cd "${GITHUB_WORKSPACE}/test_results"
+
+        i=0
+        # Try downloading the results for 6x 10 seconds before giving up.
+        while [ $i -lt 6 ]; do
+          # The results are uploaded after the test has completed.
+          sleep 10
+
+          # The log files are named by the device lab test driver.
+          COBALT_ERROR_LOG_FILENAME="webDriverTestLog.ERROR"
+          COBALT_INFO_LOG_FILENAME="webDriverTestLog.INFO"
+
+          # This command will fail until the results have been uploaded.
+          gsutil cp "${GCS_RESULTS_PATH}/${COBALT_ERROR_LOG_FILENAME}" .
+          gsutil cp "${GCS_RESULTS_PATH}/${COBALT_INFO_LOG_FILENAME}" .
+          gsutil cp "${GCS_RESULTS_PATH}/${COBALT_XMLS_FILENAME}" .
+
+          # Break if all files were downloaded.
+          if [[ -f "${COBALT_XMLS_FILENAME}" && -f "${COBALT_ERROR_LOG_FILENAME}" && -f "${COBALT_INFO_LOG_FILENAME}" ]]; then
+            break
+          fi
+
+          i=$(( ${i} + 1 ))
+        done
+
+        # Rename log files for archiving to not expose legacy weirdness.
+        mkdir -p "${COBALT_LOGS_DIR}/${{ matrix.platform }}/"
+        cp "${COBALT_ERROR_LOG_FILENAME}" "${COBALT_LOGS_DIR}/${{ matrix.platform }}/stderr_${{ matrix.shard }}.log"
+        cp "${COBALT_INFO_LOG_FILENAME}" "${COBALT_LOGS_DIR}/${{ matrix.platform }}/stdout_${{ matrix.shard }}.log"
+
+        # Prepare unit test results for DataDog upload.
+        RESULT_PATH=${UNIT_TEST_RESULT_PATH}/${{ matrix.platform }}/${{ matrix.shard }}/
+        mkdir -p ${RESULT_PATH}
+
+        # Set tags for test differentiation.
+        tags="os.platform:${{ matrix.platform }}"
+        echo $tags > ${UNIT_TEST_RESULT_PATH}/${{ matrix.platform }}/TAGS
+
+        unzip ${COBALT_XMLS_FILENAME} -d ${RESULT_PATH}
+      shell: bash
+    - name: Archive Unit Test Logs
+      uses: actions/upload-artifact@v3
+      if: always() && env.TEST_TYPE == 'unit_test'
+      with:
+        name: Device logs
+        path: ${{ env.COBALT_LOGS_DIR }}/
+    - name: Archive Unit Test Results
+      uses: actions/upload-artifact@v3
+      if: always() && env.TEST_TYPE == 'unit_test'
+      with:
+        name: unit-test-results
+        path: ${{ env.UNIT_TEST_RESULT_PATH }}/
+    - name: Get Datadog CLI
+      if: always() && env.TEST_TYPE == 'unit_test' && github.event == 'push'
+      shell: bash
+      env:
+        DD_VERSION: 'v2.18.0'
+        DD_SHA256SUM: 'adbe9b3a41faaf0b1d9702ba256cf8fa9e474c0cc8216f25e5b489c53d6f0a70  datadog-ci'
+      run: |
+        set -e
+        download_url="https://github.com/DataDog/datadog-ci/releases/download/${DD_VERSION}/datadog-ci_linux-x64"
+        curl -L --fail $download_url --output datadog-ci
+        echo ${DD_SHA256SUM} | sha256sum --check
+        chmod +x datadog-ci
+    - name: Upload to DataDog
+      if: always() && env.TEST_TYPE == 'unit_test' && github.event == 'push'
+      env:
+        DATADOG_API_KEY: ${{ secrets.DD_API_KEY }}
+        DATADOG_SITE: us5.datadoghq.com
+        DD_ENV: ci
+        DD_SERVICE: ${{ github.event.repository.name }}
+        # Need to populate git info via env vars as we don't have the repo to look at.
+        DD_GIT_REPOSITORY_URL: ${{ github.event.repository.git_url }}
+        DD_GIT_COMMIT_SHA: ${{ github.event.workflow_run.head_sha }}
+        DD_GIT_BRANCH: ${{ github.event.workflow_run.head_branch }}
+        DD_GIT_COMMIT_MESSAGE: ${{ github.event.workflow_run.head_commit.message }}
+        DD_GIT_COMMIT_AUTHOR_NAME: ${{ github.event.workflow_run.head_commit.author.name }}
+        DD_GIT_COMMIT_AUTHOR_EMAIL: ${{ github.event.workflow_run.head_commit.author.email }}
+        DD_GIT_COMMIT_AUTHOR_DATE: ${{ github.event.workflow_run.head_commit.timestamp }}
+        DD_GIT_COMMIT_COMMITTER_NAME: ${{ github.event.workflow_run.head_commit.committer.name }}
+        DD_GIT_COMMIT_COMMITTER_EMAIL: ${{ github.event.workflow_run.head_commit.committer.email }}
+        DD_GIT_COMMIT_COMMITTER_DATE:  ${{ github.event.workflow_run.head_commit.timestamp }}
+      run: |
+        # Loop over each platform, extract the tags and upload xml results.
+        echo "Uploading $dir test report"
+        export DD_TAGS=`cat ${dir}TAGS`
+        ./datadog-ci junit upload unit-test-results/*.xml
+      shell: bash
+    - name: Print device logs
+      if: always()
+      run: |
+        if ls ${COBALT_LOGS_DIR}/**/*.log 1> /dev/null 2>&1; then
+          cat ${COBALT_LOGS_DIR}/**/*.log
+        else
+          echo "No device logs found"
+        fi
+      shell: bash

.github/actions/on_host_test/action.yaml

@@ -84,16 +84,17 @@ runs:
             xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -s ${{matrix.shard}} -r ${loader_args} --xml_output_dir=${TEST_RESULTS_DIR}
           fi
         fi
-    - name: Process unit test results
-      if: failure()
+    - name: Populate TAGS for unit test report
+      if: always() && steps.run-tests.outputs.test_type == 'unit_tests'
       shell: bash
       run: |
-        set -x
-        echo "Saving unit test report to ${TEST_REPORT_FILE}"
-        python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_report_parser.py ${TEST_RESULTS_DIR} > ${TEST_REPORT_FILE}
-    - name: Upload unit test report
+        # Set tags for test differentiation.
+        tags="os.platform:${{ matrix.platform }}"
+        tags="${tags},os.family:${{ inputs.os }}"
+        echo $tags > ${TEST_RESULTS_DIR}/${{ matrix.platform }}/TAGS
+    - name: Archive unit test results
       uses: actions/upload-artifact@v3
-      if: failure()
+      if: always() && steps.run-tests.outputs.test_type == 'unit_tests'
       with:
         name: unit-test-reports
         path: ${{env.TEST_REPORT_FILE}}

tools/on_device_tests_gateway.proto

@@ -52,6 +52,10 @@ message OnDeviceTestsCommand {
   string unittest_shard_index = 17;
   string test_attempts = 18;
   string retry_level = 19;
+  string start_timeout = 20;
+  string test_timeout = 21;
+  string builder_url = 22;
+  string gcs_result_path = 23;
 }
 
 // Working directory and command line arguments to be passed to the gateway.

tools/on_device_tests_gateway_client.py

@@ -77,6 +77,7 @@ def run_trigger_command(self, workdir: str, args: argparse.Namespace):
             config=args.config,
             tag=args.tag,
             labels=args.label,
+            gcs_result_path=args.gcs_result_path,
             builder_name=args.builder_name,
             change_id=args.change_id,
             build_number=args.build_number,
@@ -189,6 +190,12 @@ def main():
       type=str,
       help='Name of the builder that built the artifacts, '
       'if any. Saved with performance test results')
+  trigger_parser.add_argument(
+      '--builder_url', type=str, help='Url to the run, if any.')
+  trigger_parser.add_argument(
+      '--gcs_result_path',
+      type=str,
+      help='GCS url where test result files should be uploaded.')
   trigger_parser.add_argument(
       '-n',
       '--build_number',