Merge pull request #21 from buildkite/feature/send-ci-info-to-test-an…

…alytics Detect CI environment and send associated data along with the test suite
buildkite · Jul 12, 2024 · d878ce0 · d878ce0
2 parents dfd4711 + b581775
commit d878ce0
Show file tree

Hide file tree

Showing 30 changed files with 1,293 additions and 425 deletions.
diff --git a/.github/workflows/build-and-test-sdk.yml b/.github/workflows/build-and-test-sdk.yml
@@ -14,6 +14,15 @@ jobs:
     env:
       BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
       BUILDKITE_ANALYTICS_DEBUG_ENABLED: ${{ secrets.BUILDKITE_ANALYTICS_DEBUG_ENABLED }}
+      GITHUB_ACTION: ${{ github.action }}
+      GITHUB_RUN_ID: ${{ github.run_id }}
+      GITHUB_RUN_NUMBER: ${{ github.run_number }}
+      GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+      GITHUB_REPOSITORY: ${{ github.repository }}
+      GITHUB_REF_NAME: ${{ github.ref_name }}
+      GITHUB_SHA: ${{ github.sha }}
+      GITHUB_WORKFLOW: ${{ github.workflow }}
+      GITHUB_ACTOR: ${{ github.actor }}
 
     steps:
       - name: Git Checkout

diff --git a/.github/workflows/test-example-project.yml b/.github/workflows/test-example-project.yml
@@ -6,21 +6,30 @@ on:
   pull_request:
     branches: [main]
 
+env:
+  BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
+  BUILDKITE_ANALYTICS_DEBUG_ENABLED: ${{ secrets.BUILDKITE_ANALYTICS_DEBUG_ENABLED }}
+  GITHUB_ACTION: ${{ github.action }}
+  GITHUB_RUN_ID: ${{ github.run_id }}
+  GITHUB_RUN_NUMBER: ${{ github.run_number }}
+  GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+  GITHUB_REPOSITORY: ${{ github.repository }}
+  GITHUB_REF_NAME: ${{ github.ref_name }}
+  GITHUB_SHA: ${{ github.sha }}
+  GITHUB_WORKFLOW: ${{ github.workflow }}
+  GITHUB_ACTOR: ${{ github.actor }}
+
 jobs:
   example-project-unit-tests:
     name: Run Example Project Unit Tests
     runs-on: ubuntu-latest
 
-    env:
-      BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
-      BUILDKITE_ANALYTICS_DEBUG_ENABLED: ${{ secrets.BUILDKITE_ANALYTICS_DEBUG_ENABLED }}
-
     steps:
       - name: Git Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up JDK 17
-        uses: actions/setup-java@v3.3.0
+        uses: actions/setup-java@v4
         with:
           distribution: zulu
           java-version: 17
@@ -33,28 +42,39 @@ jobs:
 
   example-project-instrumented-tests:
     name: Run Example Project Instrumented Tests
-    runs-on: macos-latest
-
-    env:
-      BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
-      BUILDKITE_ANALYTICS_DEBUG_ENABLED: ${{ secrets.BUILDKITE_ANALYTICS_DEBUG_ENABLED }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        api-level: [28]
+    timeout-minutes: 90
 
     steps:
+      - name: "Enable KVM group perms"
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+          ls /dev/kvm
+
       - name: Git Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up JDK 17
-        uses: actions/setup-java@v3.3.0
+        uses: actions/setup-java@v4
         with:
           distribution: zulu
           java-version: 17
 
-      - name: Debug Build
-        run: ./gradlew :example:buildDebug
-
-      - name: Example Project Instrumented Tests
-        uses: reactivecircus/android-emulator-runner@v2
+      - name: "Run instrumented tests"
+        uses: reactivecircus/[email protected]
         with:
-          api-level: 27
+          api-level: ${{ matrix.api-level }}
+          force-avd-creation: false
+          emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -timezone Australia/Melbourne -camera-back none
+          disable-animations: true
+          disable-spellchecker: true
+          profile: Nexus 6
           arch: x86_64
+          ram-size: 4096M
+          heap-size: 512M
           script: "support/scripts/example-instrumented-tests"
diff --git a/CI_CONFIGURATION.md b/CI_CONFIGURATION.md
@@ -0,0 +1,92 @@
+# CI Environment Variables Setup
+
+This document provides instructions for setting up environment variables for different CI platforms to enrich test reports.
+
+## Accessing Environment Variables in CI Configuration
+
+To enrich your test reports with valuable CI information such as commit messages, branch names, and build numbers, you need to pass environment variables in your CI pipeline.
+For example, here is how you can set up the environment variables for GitHub Actions:
+
+### GitHub Actions
+
+In your GitHub Actions workflow configuration, add the following environment variables:
+
+```yaml
+env:
+  BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
+  GITHUB_ACTION: ${{ github.action }}
+  GITHUB_RUN_ID: ${{ github.run_id }}
+  GITHUB_RUN_NUMBER: ${{ github.run_number }}
+  GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+  GITHUB_REPOSITORY: ${{ github.repository }}
+  GITHUB_REF_NAME: ${{ github.ref_name }}
+  GITHUB_SHA: ${{ github.sha }}
+  GITHUB_WORKFLOW: ${{ github.workflow }}
+  GITHUB_ACTOR: ${{ github.actor }}
+```
+
+## Additional Setup for Instrumented Tests
+
+While the above setup is sufficient for unit tests collector, instrumented tests collector require additional configuration. Below are examples for different CI platforms supported by the test collectors.
+
+In your build.gradle.kts file, add the following to pass the required environment variables for instrumented tests:
+
+### Buildkite
+
+```
+android {
+    ...
+    defaultConfig {
+        ...
+        testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_TOKEN"] = System.getenv("BUILDKITE_ANALYTICS_TOKEN")
+        testInstrumentationRunnerArguments["BUILDKITE_BUILD_ID"] = System.getenv("BUILDKITE_BUILD_ID") ?: ""
+        testInstrumentationRunnerArguments["BUILDKITE_BUILD_URL"] = System.getenv("BUILDKITE_BUILD_URL") ?: ""
+        testInstrumentationRunnerArguments["BUILDKITE_BRANCH"] = System.getenv("BUILDKITE_BRANCH") ?: ""
+        testInstrumentationRunnerArguments["BUILDKITE_COMMIT"] = System.getenv("BUILDKITE_COMMIT") ?: ""
+        testInstrumentationRunnerArguments["BUILDKITE_BUILD_NUMBER"] = System.getenv("BUILDKITE_BUILD_NUMBER") ?: ""
+        testInstrumentationRunnerArguments["BUILDKITE_JOB_ID"] = System.getenv("BUILDKITE_JOB_ID") ?: ""
+        testInstrumentationRunnerArguments["BUILDKITE_MESSAGE"] = System.getenv("BUILDKITE_MESSAGE") ?: ""
+    }
+}
+```
+
+### GitHub Actions
+
+```
+android {
+    ...
+    defaultConfig {
+        ...
+        testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_TOKEN"] = System.getenv("BUILDKITE_ANALYTICS_TOKEN")
+        testInstrumentationRunnerArguments["GITHUB_ACTION"] = System.getenv("GITHUB_ACTION") ?: ""
+        testInstrumentationRunnerArguments["GITHUB_RUN_ID"] = System.getenv("GITHUB_RUN_ID") ?: ""
+        testInstrumentationRunnerArguments["GITHUB_RUN_NUMBER"] = System.getenv("GITHUB_RUN_NUMBER") ?: ""
+        testInstrumentationRunnerArguments["GITHUB_RUN_ATTEMPT"] = System.getenv("GITHUB_RUN_ATTEMPT") ?: ""
+        testInstrumentationRunnerArguments["GITHUB_REPOSITORY"] = System.getenv("GITHUB_REPOSITORY") ?: ""
+        testInstrumentationRunnerArguments["GITHUB_REF_NAME"] = System.getenv("GITHUB_REF_NAME") ?: ""
+        testInstrumentationRunnerArguments["GITHUB_SHA"] = System.getenv("GITHUB_SHA") ?: ""
+        testInstrumentationRunnerArguments["GITHUB_WORKFLOW"] = System.getenv("GITHUB_WORKFLOW") ?: ""
+        testInstrumentationRunnerArguments["GITHUB_ACTOR"] = System.getenv("GITHUB_ACTOR") ?: ""
+    }
+}
+```
+
+### CircleCI
+
+```
+android {
+    ...
+    defaultConfig {
+        ...
+        testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_TOKEN"] = System.getenv("BUILDKITE_ANALYTICS_TOKEN")
+        testInstrumentationRunnerArguments["CIRCLE_BUILD_NUM"] = System.getenv("CIRCLE_BUILD_NUM") ?: ""
+        testInstrumentationRunnerArguments["CIRCLE_WORKFLOW_ID"] = System.getenv("CIRCLE_WORKFLOW_ID") ?: ""
+        testInstrumentationRunnerArguments["CIRCLE_BUILD_URL"] = System.getenv("CIRCLE_BUILD_URL") ?: ""
+        testInstrumentationRunnerArguments["CIRCLE_BRANCH"] = System.getenv("CIRCLE_BRANCH") ?: ""
+        testInstrumentationRunnerArguments["CIRCLE_SHA1"] = System.getenv("CIRCLE_SHA1") ?: ""
+    }
+}
+```
+
+By following these steps, you ensure that your CI environment variables are passed correctly to your Android instrumentation and unit tests. 
+For a complete setup example, check out the [example](example) project in the repository.
diff --git a/README.md b/README.md
@@ -40,43 +40,29 @@ Add the following dependency:
 androidTestImplementation("com.buildkite.test-collector-android:instrumented-test-collector:0.2.0")
 ```
 
+Again, in your app-level build.gradle.kts file, instruct Gradle to use your test collector and pass analytics token argument:
+
 ```
 android {
     ...
     
     defaultConfig {
         ...
         
-        buildConfigField(
-            "String", 
-            "BUILDKITE_ANALYTICS_TOKEN", 
-            "\"${System.getenv("BUILDKITE_ANALYTICS_TOKEN")}\""
-        )
+        testInstrumentationRunnerArguments["listener"] = "com.buildkite.test.collector.android.InstrumentedTestCollector"
+        testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_TOKEN"] = System.getenv("BUILDKITE_ANALYTICS_TOKEN")
     }
 }    
 ```
 
-Sync gradle, and rebuild the project to ensure the `BuildConfig` is generated.
-
-Create the following class in your `androidTest` directory,
-i.e. `src/androidTest/java/com/myapp/MyTestCollector.kt`
+Note: This test collector uploads test data via the device under test. Make sure your Android device/emulator has network access.
 
-```
-class MyTestCollector : InstrumentedTestCollector(
-    apiToken = BuildConfig.BUILDKITE_ANALYTICS_TOKEN
-)
-```
+### Step 5 (Optional) - Passing CI Environment Variables
 
-Again, in your app-level build.gradle.kts file, instruct Gradle to use your test collector:
-
-```
-testInstrumentationRunnerArguments += mapOf(
-    "listener" to "com.mycompany.myapp.MyTestCollector" // Make sure to use the correct package name here
-)
-```
+The only required environment variable is the analytics token, but if you're using one of the supported CI platforms, 
+you can pass extra information to the test-collector to enrich the reports. These include commit messages, branch names, build numbers, etc.
 
-Note: This test collector uploads test data via the device under test. Make sure your Android
-device/emulator has network access.
+For detailed instructions on setting up environment variables for different CI platforms, see the [CI Environment Variables Setup](CI_CONFIGURATION.md) document.
 
 ## 🔍 Debugging
 

diff --git a/collector/instrumented-test-collector/build.gradle.kts b/collector/instrumented-test-collector/build.gradle.kts
@@ -35,4 +35,5 @@ android {
 dependencies {
     implementation(projects.collector.testDataUploader)
     implementation(libs.testing.junit)
+    implementation(libs.androidx.monitor)
 }