From b89a8dd245786e25d0e01f7b38e173504fa8ab17 Mon Sep 17 00:00:00 2001 From: Bhumil Soni Date: Mon, 3 Jun 2024 12:32:09 +1000 Subject: [PATCH 01/11] Update test collector version reference in libs to 0.3.0-SNAPSHOT for testing --- gradle/libs.versions.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 8f68b33..9b9ff07 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -8,13 +8,12 @@ androidxCore = "1.12.0" androidxLifecycle = "2.7.0" androidxTestExt = "1.1.5" detekt = "1.23.1" -instrumentedTestCollector = "0.2.0-SNAPSHOT" junit = "4.13.2" kotlin = "1.8.22" okhttp = "5.0.0-alpha.12" retrofit = "2.9.0" mavenPublish = "0.27.0" -buildkiteTestCollector = "0.2.0-SNAPSHOT" +buildkiteTestCollector = "0.3.0-SNAPSHOT" [libraries] androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "androidxCore" } From f001e71e3e504a8b56ed430d43737de9d9ad8939 Mon Sep 17 00:00:00 2001 From: Bhumil Soni Date: Tue, 4 Jun 2024 19:38:31 +1000 Subject: [PATCH 02/11] Simplify environment variable handling for InstrumentedTestCollector - Updated setup to use Gradle configuration to pass environment variables directly to instrumentation arguments instead of using BuildConfig. - Simplified the instrumentation test collector as InstrumentedTestCollector is no longer abstract and now handles environment variables internally, thus removing the need to create a custom test collector class. --- README.md | 31 +++--------------- .../build.gradle.kts | 1 + .../android/InstrumentedTestCollector.kt | 21 ++++++------ .../android/util/ConfigureTestUploader.kt | 25 +++++++++++++++ .../collector/android/TestDataUploader.kt | 4 +-- ...alues.kt => BuildkiteEnvironmentValues.kt} | 2 +- .../environment/UploaderConfiguration.kt | 32 ------------------- .../android/UnitTestCollectorPlugin.kt | 2 +- .../android/util/ConfigureTestUploader.kt | 18 +++++++++++ example/build.gradle.kts | 18 +++-------- .../android/example/ExampleTestCollector.kt | 12 ------- gradle/libs.versions.toml | 2 ++ 12 files changed, 69 insertions(+), 99 deletions(-) create mode 100644 collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt rename collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/{EnvironmentValues.kt => BuildkiteEnvironmentValues.kt} (86%) delete mode 100644 collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/UploaderConfiguration.kt create mode 100644 collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt delete mode 100644 example/src/androidTest/kotlin/com/buildkite/test/collector/android/example/ExampleTestCollector.kt diff --git a/README.md b/README.md index 0ea22bb..aeafd4b 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ Add the following dependency: androidTestImplementation("com.buildkite.test-collector-android:instrumented-test-collector:0.2.0") ``` +Again, in your app-level build.gradle.kts file, instruct Gradle to use your test collector and pass analytics token argument: + ``` android { ... @@ -47,36 +49,13 @@ android { defaultConfig { ... - buildConfigField( - "String", - "BUILDKITE_ANALYTICS_TOKEN", - "\"${System.getenv("BUILDKITE_ANALYTICS_TOKEN")}\"" - ) + testInstrumentationRunnerArguments["listener"] = "com.buildkite.test.collector.android.InstrumentedTestCollector" + testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_TOKEN"] = System.getenv("BUILDKITE_ANALYTICS_TOKEN") } } ``` -Sync gradle, and rebuild the project to ensure the `BuildConfig` is generated. - -Create the following class in your `androidTest` directory, -i.e. `src/androidTest/java/com/myapp/MyTestCollector.kt` - -``` -class MyTestCollector : InstrumentedTestCollector( - apiToken = BuildConfig.BUILDKITE_ANALYTICS_TOKEN -) -``` - -Again, in your app-level build.gradle.kts file, instruct Gradle to use your test collector: - -``` -testInstrumentationRunnerArguments += mapOf( - "listener" to "com.mycompany.myapp.MyTestCollector" // Make sure to use the correct package name here -) -``` - -Note: This test collector uploads test data via the device under test. Make sure your Android -device/emulator has network access. +Note: This test collector uploads test data via the device under test. Make sure your Android device/emulator has network access. ## 🔍 Debugging diff --git a/collector/instrumented-test-collector/build.gradle.kts b/collector/instrumented-test-collector/build.gradle.kts index 02a39c4..3184b8c 100644 --- a/collector/instrumented-test-collector/build.gradle.kts +++ b/collector/instrumented-test-collector/build.gradle.kts @@ -35,4 +35,5 @@ android { dependencies { implementation(projects.collector.testDataUploader) implementation(libs.testing.junit) + implementation(libs.androidx.monitor) } diff --git a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt index 1494edc..38bd423 100644 --- a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt +++ b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt @@ -1,11 +1,12 @@ package com.buildkite.test.collector.android +import androidx.test.platform.app.InstrumentationRegistry import com.buildkite.test.collector.android.model.TestDetails import com.buildkite.test.collector.android.model.TestFailureExpanded import com.buildkite.test.collector.android.model.TestHistory import com.buildkite.test.collector.android.model.TestOutcome import com.buildkite.test.collector.android.tracer.TestObserver -import com.buildkite.test.collector.android.tracer.environment.configureInstrumentedTestUploader +import com.buildkite.test.collector.android.util.configureInstrumentedTestUploader import org.junit.runner.Description import org.junit.runner.notification.Failure import org.junit.runner.notification.RunListener @@ -14,20 +15,16 @@ import org.junit.runner.notification.RunListener * Serves as an abstract foundation for creating instrumented test collectors that interface with Buildkite's Test Analytics service. * This class extends JUnit's [RunListener] to capture real-time events during the execution of instrumented tests, enabling precise monitoring * and reporting of test outcomes. It automatically gathers detailed test results and uploads them directly to the analytics portal at the conclusion of test suite. - * - * @param apiToken The API token for the test suite, necessary for authenticating requests with Test Analytics. - * @param isDebugEnabled When true, enables logging to assist with debugging. */ -abstract class InstrumentedTestCollector( - apiToken: String, - isDebugEnabled: Boolean = false -) : RunListener() { +class InstrumentedTestCollector : RunListener() { private val testObserver = TestObserver() - private val testUploader = configureInstrumentedTestUploader( - apiToken = apiToken, - isDebugEnabled = isDebugEnabled - ) private val testCollection: MutableList = mutableListOf() + private val testUploader: TestDataUploader + + init { + val arguments = InstrumentationRegistry.getArguments() + testUploader = configureInstrumentedTestUploader(instrumentationArguments = arguments) + } override fun testSuiteStarted(testDescription: Description) { /* Nothing to do before the test suite has started */ diff --git a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt new file mode 100644 index 0000000..cb8d9c9 --- /dev/null +++ b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt @@ -0,0 +1,25 @@ +package com.buildkite.test.collector.android.util + +import android.os.Bundle +import com.buildkite.test.collector.android.TestDataUploader +import com.buildkite.test.collector.android.tracer.environment.BuildkiteEnvironmentValues + +/** + * Creates a [TestDataUploader] instance for instrumented tests. + */ +fun configureInstrumentedTestUploader(instrumentationArguments: Bundle): TestDataUploader { + return TestDataUploader( + testSuiteApiToken = instrumentationArguments.getStringEnvironmentValue( + BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_TOKEN + ), + isDebugEnabled = instrumentationArguments.getBooleanEnvironmentValue( + BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_DEBUG_ENABLED + ) + ) +} + +private fun Bundle.getStringEnvironmentValue(key: String): String? = + getString(key)?.takeIf { value -> value.isNotBlank() && value != "null" } + +private fun Bundle.getBooleanEnvironmentValue(key: String): Boolean = + getString(key)?.toBoolean() ?: false diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/TestDataUploader.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/TestDataUploader.kt index fbf12eb..c332c16 100644 --- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/TestDataUploader.kt +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/TestDataUploader.kt @@ -45,7 +45,7 @@ class TestDataUploader( private fun uploadTestData(testData: TestData) { if (testSuiteApiToken == null) { logger.info { - "Test Suite API token is missing. Please ensure the 'BUILDKITE_ANALYTICS_TOKEN' environment variable is set correctly to upload test data." + "Incorrect or missing Test Suite API token. Please ensure the 'BUILDKITE_ANALYTICS_TOKEN' environment variable is set correctly to upload test data." } return } @@ -67,7 +67,7 @@ class TestDataUploader( private fun logApiResponse(testUploadResponse: Response) { if (testUploadResponse.isSuccessful) { - logger.debug { "Test analytics data successfully uploaded. URL: ${testUploadResponse.body()?.runUrl}" } + logger.info { "Test analytics data successfully uploaded. URL: ${testUploadResponse.body()?.runUrl}" } } else { logger.error { "Error uploading test analytics data. HTTP error code: ${testUploadResponse.code()}. Ensure the test suite API token is correct and properly configured." diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/EnvironmentValues.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/BuildkiteEnvironmentValues.kt similarity index 86% rename from collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/EnvironmentValues.kt rename to collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/BuildkiteEnvironmentValues.kt index 6b59389..c155336 100644 --- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/EnvironmentValues.kt +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/BuildkiteEnvironmentValues.kt @@ -1,6 +1,6 @@ package com.buildkite.test.collector.android.tracer.environment -internal object EnvironmentValues { +object BuildkiteEnvironmentValues { const val BUILDKITE_ANALYTICS_TOKEN = "BUILDKITE_ANALYTICS_TOKEN" const val BUILDKITE_ANALYTICS_DEBUG_ENABLED = "BUILDKITE_ANALYTICS_DEBUG_ENABLED" } diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/UploaderConfiguration.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/UploaderConfiguration.kt deleted file mode 100644 index efa382e..0000000 --- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/UploaderConfiguration.kt +++ /dev/null @@ -1,32 +0,0 @@ -@file:Suppress("SameParameterValue") - -package com.buildkite.test.collector.android.tracer.environment - -import com.buildkite.test.collector.android.TestDataUploader - -/** - * Creates a [TestDataUploader] instance for unit tests, fetching configuration from environment variables. - */ -fun configureUnitTestUploader() = TestDataUploader( - testSuiteApiToken = getStringEnvironmentValue(name = EnvironmentValues.BUILDKITE_ANALYTICS_TOKEN), - isDebugEnabled = getBooleanEnvironmentValue(name = EnvironmentValues.BUILDKITE_ANALYTICS_DEBUG_ENABLED) -) - -/** - * Creates a [TestDataUploader] instance for instrumented tests with the given [apiToken] and [isDebugEnabled] flag, filtering out 'null' tokens. - */ -fun configureInstrumentedTestUploader( - apiToken: String, - isDebugEnabled: Boolean -) = TestDataUploader( - testSuiteApiToken = apiToken.takeIf { token -> token != "null" }, - isDebugEnabled = isDebugEnabled -) - -private fun getStringEnvironmentValue(name: String): String? { - return System.getenv(name) -} - -private fun getBooleanEnvironmentValue(name: String): Boolean { - return System.getenv(name)?.let { value -> value.toBoolean() } ?: false -} diff --git a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt index cab4a55..80b90d4 100644 --- a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt +++ b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt @@ -4,7 +4,7 @@ import com.buildkite.test.collector.android.model.TestDetails import com.buildkite.test.collector.android.model.TestFailureExpanded import com.buildkite.test.collector.android.model.TestHistory import com.buildkite.test.collector.android.tracer.TestObserver -import com.buildkite.test.collector.android.tracer.environment.configureUnitTestUploader +import com.buildkite.test.collector.android.util.configureUnitTestUploader import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.tasks.testing.Test diff --git a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt new file mode 100644 index 0000000..bd9ec6d --- /dev/null +++ b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt @@ -0,0 +1,18 @@ +package com.buildkite.test.collector.android.util + +import com.buildkite.test.collector.android.TestDataUploader +import com.buildkite.test.collector.android.tracer.environment.BuildkiteEnvironmentValues + +/** + * Configures a [TestDataUploader] instance for unit tests. + */ +fun configureUnitTestUploader() = TestDataUploader( + testSuiteApiToken = getStringEnvironmentValue(key = BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_TOKEN), + isDebugEnabled = getBooleanEnvironmentValue(key = BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_DEBUG_ENABLED) +) + +private fun getStringEnvironmentValue(key: String): String? = + System.getenv(key)?.takeIf { value -> value.isNotBlank() && value != "null" } + +private fun getBooleanEnvironmentValue(key: String): Boolean = + System.getenv(key)?.toBoolean() ?: false diff --git a/example/build.gradle.kts b/example/build.gradle.kts index a700057..c654170 100644 --- a/example/build.gradle.kts +++ b/example/build.gradle.kts @@ -16,23 +16,15 @@ android { testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" // Specifies `ExampleTestCollector` as the instrumented test listener for collecting test analytics. - testInstrumentationRunnerArguments += mapOf("listener" to "com.buildkite.test.collector.android.example.ExampleTestCollector") + testInstrumentationRunnerArguments["listener"] = "com.buildkite.test.collector.android.InstrumentedTestCollector" + + // Passes environment variables as instrumentation arguments + testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_TOKEN"] = System.getenv("BUILDKITE_ANALYTICS_TOKEN") ?: "" + testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_DEBUG_ENABLED"] = System.getenv("BUILDKITE_ANALYTICS_DEBUG_ENABLED") ?: "false" vectorDrawables { useSupportLibrary = true } - - // Fetches local/CI environment variables for Buildkite test collector setup - buildConfigField( - "String", - "BUILDKITE_ANALYTICS_TOKEN", - "\"${System.getenv("BUILDKITE_ANALYTICS_TOKEN")}\"" - ) - buildConfigField( - "boolean", - "BUILDKITE_ANALYTICS_DEBUG_ENABLED", - System.getenv("BUILDKITE_ANALYTICS_DEBUG_ENABLED") ?: "false" - ) } buildTypes { diff --git a/example/src/androidTest/kotlin/com/buildkite/test/collector/android/example/ExampleTestCollector.kt b/example/src/androidTest/kotlin/com/buildkite/test/collector/android/example/ExampleTestCollector.kt deleted file mode 100644 index 5ecbf45..0000000 --- a/example/src/androidTest/kotlin/com/buildkite/test/collector/android/example/ExampleTestCollector.kt +++ /dev/null @@ -1,12 +0,0 @@ -package com.buildkite.test.collector.android.example - -import com.buildkite.test.collector.android.InstrumentedTestCollector - -/** - * Configures `ExampleTestCollector` with Buildkite test collector configurations. - * Values are derived from local/CI environment variables and set in [BuildConfig] at build time. - */ -class ExampleTestCollector : InstrumentedTestCollector( - apiToken = BuildConfig.BUILDKITE_ANALYTICS_TOKEN, - isDebugEnabled = BuildConfig.BUILDKITE_ANALYTICS_DEBUG_ENABLED -) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 9b9ff07..3706e00 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -14,6 +14,7 @@ okhttp = "5.0.0-alpha.12" retrofit = "2.9.0" mavenPublish = "0.27.0" buildkiteTestCollector = "0.3.0-SNAPSHOT" +monitor = "1.6.1" [libraries] androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "androidxCore" } @@ -34,6 +35,7 @@ testing-junit = { group = "junit", name = "junit", version.ref = "junit" } # build-logic module dependencies detekt-gradlePlugin = { group = "io.gitlab.arturbosch.detekt", name = "detekt-gradle-plugin", version.ref = "detekt" } +androidx-monitor = { group = "androidx.test", name = "monitor", version.ref = "monitor" } [plugins] android-application = { id = "com.android.application", version.ref = "androidGradlePlugin" } From 4c8d8134cb610035613a479d597207753356b43e Mon Sep 17 00:00:00 2001 From: Bhumil Soni Date: Wed, 5 Jun 2024 18:29:45 +1000 Subject: [PATCH 03/11] Introduce Interfaces for Test Data Handling and Observing Added TestDataUploader and TestObserver interfaces to allow custom implementations and also improve flexibility and testability. --- .../android/InstrumentedTestCollector.kt | 18 ++-- .../android/util/ConfigureTestUploader.kt | 3 +- .../android/BuildKiteTestDataUploader.kt | 83 +++++++++++++++++++ .../collector/android/TestDataUploader.kt | 73 ++-------------- .../android/tracer/BuildkiteTestObserver.kt | 57 +++++++++++++ .../collector/android/tracer/TestObserver.kt | 77 +++++++++-------- .../android/UnitTestCollectorPlugin.kt | 6 +- .../android/util/ConfigureTestUploader.kt | 3 +- 8 files changed, 198 insertions(+), 122 deletions(-) create mode 100644 collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/BuildKiteTestDataUploader.kt create mode 100644 collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/BuildkiteTestObserver.kt diff --git a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt index 38bd423..35297b1 100644 --- a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt +++ b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt @@ -5,7 +5,7 @@ import com.buildkite.test.collector.android.model.TestDetails import com.buildkite.test.collector.android.model.TestFailureExpanded import com.buildkite.test.collector.android.model.TestHistory import com.buildkite.test.collector.android.model.TestOutcome -import com.buildkite.test.collector.android.tracer.TestObserver +import com.buildkite.test.collector.android.tracer.BuildkiteTestObserver import com.buildkite.test.collector.android.util.configureInstrumentedTestUploader import org.junit.runner.Description import org.junit.runner.notification.Failure @@ -17,14 +17,9 @@ import org.junit.runner.notification.RunListener * and reporting of test outcomes. It automatically gathers detailed test results and uploads them directly to the analytics portal at the conclusion of test suite. */ class InstrumentedTestCollector : RunListener() { - private val testObserver = TestObserver() + private val testObserver = BuildkiteTestObserver() private val testCollection: MutableList = mutableListOf() - private val testUploader: TestDataUploader - - init { - val arguments = InstrumentationRegistry.getArguments() - testUploader = configureInstrumentedTestUploader(instrumentationArguments = arguments) - } + private val testUploader: TestDataUploader by lazy { configureTestUploader() } override fun testSuiteStarted(testDescription: Description) { /* Nothing to do before the test suite has started */ @@ -32,7 +27,7 @@ class InstrumentedTestCollector : RunListener() { override fun testSuiteFinished(description: Description) { if (isFinalTestSuiteCall(testDescription = description)) { - testUploader.configureUploadData(testCollection = testCollection) + testUploader.uploadTestData(testCollection = testCollection) } } @@ -97,4 +92,9 @@ class InstrumentedTestCollector : RunListener() { */ private fun isFinalTestSuiteCall(testDescription: Description) = (testDescription.displayName.isNullOrEmpty() || testDescription.displayName == "null") && (testDescription.className.isNullOrEmpty() || testDescription.className == "null") + + private fun configureTestUploader(): TestDataUploader { + val arguments = InstrumentationRegistry.getArguments() + return configureInstrumentedTestUploader(instrumentationArguments = arguments) + } } diff --git a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt index cb8d9c9..ffacfbc 100644 --- a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt +++ b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt @@ -1,6 +1,7 @@ package com.buildkite.test.collector.android.util import android.os.Bundle +import com.buildkite.test.collector.android.BuildKiteTestDataUploader import com.buildkite.test.collector.android.TestDataUploader import com.buildkite.test.collector.android.tracer.environment.BuildkiteEnvironmentValues @@ -8,7 +9,7 @@ import com.buildkite.test.collector.android.tracer.environment.BuildkiteEnvironm * Creates a [TestDataUploader] instance for instrumented tests. */ fun configureInstrumentedTestUploader(instrumentationArguments: Bundle): TestDataUploader { - return TestDataUploader( + return BuildKiteTestDataUploader( testSuiteApiToken = instrumentationArguments.getStringEnvironmentValue( BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_TOKEN ), diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/BuildKiteTestDataUploader.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/BuildKiteTestDataUploader.kt new file mode 100644 index 0000000..8fd9957 --- /dev/null +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/BuildKiteTestDataUploader.kt @@ -0,0 +1,83 @@ +package com.buildkite.test.collector.android + +import com.buildkite.test.collector.android.model.RunEnvironment +import com.buildkite.test.collector.android.model.TestData +import com.buildkite.test.collector.android.model.TestDetails +import com.buildkite.test.collector.android.model.TestUploadResponse +import com.buildkite.test.collector.android.network.TestAnalyticsRetrofit +import com.buildkite.test.collector.android.network.api.TestUploaderApi +import com.buildkite.test.collector.android.util.CollectorUtils +import com.buildkite.test.collector.android.util.Logger +import retrofit2.Response + +/** + * Manages the upload of test data to the Buildkite Test Analytics Suite associated with the provided API token. + * + * @property testSuiteApiToken The API token for the test suite, necessary for authenticating requests with Test Analytics. + * Test data will not be uploaded if this is null. + * @property isDebugEnabled When true, enables logging to assist with debugging. + */ +class BuildKiteTestDataUploader( + private val testSuiteApiToken: String?, + private val isDebugEnabled: Boolean +) : TestDataUploader { + private val logger = + Logger(minLevel = if (isDebugEnabled) Logger.LogLevel.DEBUG else Logger.LogLevel.INFO) + + /** + * Uploads test data to the Buildkite Test Analytics Suite. + * The number of test data uploaded in a single request is constrained by [CollectorUtils.Uploader.TEST_DATA_UPLOAD_LIMIT]. + * + * @param testCollection A list of [TestDetails] representing all the tests within the suite. + */ + override fun uploadTestData(testCollection: List) { + val runEnvironment = RunEnvironment().getEnvironmentValues() + + val testData = TestData( + format = "json", + runEnvironment = runEnvironment, + data = testCollection.take(CollectorUtils.Uploader.TEST_DATA_UPLOAD_LIMIT) + ) + + uploadTestData(testData = testData) + } + + /** + * Performs the actual upload of the provided test data to the Buildkite Test Analytics Suite. + * + * @param testData The test data to be uploaded. + */ + private fun uploadTestData(testData: TestData) { + if (testSuiteApiToken == null) { + logger.info { + "Incorrect or missing Test Suite API token. Please ensure the 'BUILDKITE_ANALYTICS_TOKEN' environment variable is set correctly to upload test data." + } + return + } + + try { + logger.debug { "Uploading test analytics data." } + + val testUploaderService = + TestAnalyticsRetrofit.getRetrofitInstance(testSuiteApiToken = testSuiteApiToken) + .create(TestUploaderApi::class.java) + val uploadTestDataApiCall = testUploaderService.uploadTestData(testData = testData) + val testUploadResponse = uploadTestDataApiCall.execute() + + logApiResponse(testUploadResponse = testUploadResponse) + } catch (e: Exception) { + logger.error { "Error uploading test analytics data: ${e.message}." } + } + } + + private fun logApiResponse(testUploadResponse: Response) { + if (testUploadResponse.isSuccessful) { + logger.info { "Test analytics data successfully uploaded. URL: ${testUploadResponse.body()?.runUrl}" } + } else { + logger.error { + "Error uploading test analytics data. HTTP error code: ${testUploadResponse.code()}. Ensure the test suite API token is correct and properly configured." + } + logger.debug { "Failed response details: ${testUploadResponse.errorBody()?.string()}" } + } + } +} diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/TestDataUploader.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/TestDataUploader.kt index c332c16..74b1046 100644 --- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/TestDataUploader.kt +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/TestDataUploader.kt @@ -1,78 +1,15 @@ package com.buildkite.test.collector.android -import com.buildkite.test.collector.android.model.RunEnvironment -import com.buildkite.test.collector.android.model.TestData import com.buildkite.test.collector.android.model.TestDetails -import com.buildkite.test.collector.android.model.TestUploadResponse -import com.buildkite.test.collector.android.network.TestAnalyticsRetrofit -import com.buildkite.test.collector.android.network.api.TestUploaderApi -import com.buildkite.test.collector.android.util.CollectorUtils.Uploader -import com.buildkite.test.collector.android.util.Logger -import retrofit2.Response /** - * Manages the upload of test data to the Buildkite Test Analytics Suite associated with the provided API token. - * - * @property testSuiteApiToken The API token for the test suite, necessary for authenticating requests with Test Analytics. - * Test data will not be uploaded if this is null. - * @property isDebugEnabled When true, enables logging to assist with debugging. + * Interface for uploading test data to a test analytics service. */ -class TestDataUploader( - private val testSuiteApiToken: String?, - private val isDebugEnabled: Boolean -) { - private val logger = - Logger(minLevel = if (isDebugEnabled) Logger.LogLevel.DEBUG else Logger.LogLevel.INFO) - +interface TestDataUploader { /** - * Configures and uploads test data. - * The number of test data uploaded in a single request is constrained by [Uploader.TEST_DATA_UPLOAD_LIMIT]. + * Uploads test data to the test analytics service. * * @param testCollection A list of [TestDetails] representing all the tests within the suite. - * */ - fun configureUploadData(testCollection: List) { - val runEnvironment = RunEnvironment().getEnvironmentValues() - - val testData = TestData( - format = "json", - runEnvironment = runEnvironment, - data = testCollection.take(Uploader.TEST_DATA_UPLOAD_LIMIT) - ) - - uploadTestData(testData = testData) - } - - private fun uploadTestData(testData: TestData) { - if (testSuiteApiToken == null) { - logger.info { - "Incorrect or missing Test Suite API token. Please ensure the 'BUILDKITE_ANALYTICS_TOKEN' environment variable is set correctly to upload test data." - } - return - } - - try { - logger.debug { "Uploading test analytics data." } - - val testUploaderService = - TestAnalyticsRetrofit.getRetrofitInstance(testSuiteApiToken = testSuiteApiToken) - .create(TestUploaderApi::class.java) - val uploadTestDataApiCall = testUploaderService.uploadTestData(testData = testData) - val testUploadResponse = uploadTestDataApiCall.execute() - - logApiResponse(testUploadResponse = testUploadResponse) - } catch (e: Exception) { - logger.error { "Error uploading test analytics data: ${e.message}." } - } - } - - private fun logApiResponse(testUploadResponse: Response) { - if (testUploadResponse.isSuccessful) { - logger.info { "Test analytics data successfully uploaded. URL: ${testUploadResponse.body()?.runUrl}" } - } else { - logger.error { - "Error uploading test analytics data. HTTP error code: ${testUploadResponse.code()}. Ensure the test suite API token is correct and properly configured." - } - logger.debug { "Failed response details: ${testUploadResponse.errorBody()?.string()}" } - } - } + */ + fun uploadTestData(testCollection: List) } diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/BuildkiteTestObserver.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/BuildkiteTestObserver.kt new file mode 100644 index 0000000..9ec4633 --- /dev/null +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/BuildkiteTestObserver.kt @@ -0,0 +1,57 @@ +package com.buildkite.test.collector.android.tracer + +import com.buildkite.test.collector.android.model.TestFailureExpanded +import com.buildkite.test.collector.android.model.TestOutcome +import kotlin.math.max + +/** + * Default implementation of [TestObserver] that observes and records details for an individual test, + * including timing, outcome, and failure details. + */ +class BuildkiteTestObserver : TestObserver { + override var startTime: Long = 0 + private set + override var endTime: Long = 0 + private set + override var outcome: TestOutcome = TestOutcome.Unknown + private set + override var failureReason: String? = null + private set + override var failureDetails: List? = null + private set + + override fun startTest() { + startTime = System.nanoTime() + } + + override fun endTest() { + endTime = System.nanoTime() + } + + override fun getDuration(): Double = max(0.0, (endTime - startTime) / 1_000_000_000.0) + + override fun recordSuccess() { + outcome = TestOutcome.Passed + } + + override fun recordFailure( + reason: String, + details: List + ) { + outcome = TestOutcome.Failed + failureReason = reason + failureDetails = details + } + + override fun recordSkipped() { + outcome = TestOutcome.Skipped + } + + override fun reset() { + startTime = 0 + endTime = 0 + outcome = TestOutcome.Unknown + failureReason = null + failureDetails = null + } +} diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/TestObserver.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/TestObserver.kt index b2165b2..bea741a 100644 --- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/TestObserver.kt +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/TestObserver.kt @@ -2,76 +2,73 @@ package com.buildkite.test.collector.android.tracer import com.buildkite.test.collector.android.model.TestFailureExpanded import com.buildkite.test.collector.android.model.TestOutcome -import kotlin.math.max /** * Observes and records details for an individual test, including timing, outcome, and failure details. + * The start and end times are recorded in nanoseconds. */ -class TestObserver { - var startTime: Long = 0 - private set - var endTime: Long = 0 - private set - var outcome: TestOutcome = TestOutcome.Unknown - private set - var failureReason: String? = null - private set - var failureDetails: List? = null - private set +interface TestObserver { + /** + * The start time of the test in nanoseconds. + */ + val startTime: Long + + /** + * The end time of the test in nanoseconds. + */ + val endTime: Long + + /** + * The outcome of the test. + */ + val outcome: TestOutcome + + /** + * The reason for test failure, if any. + */ + val failureReason: String? + + /** + * Detailed information about test failures. + */ + val failureDetails: List? /** * Records the start time of a test in nanoseconds. */ - fun startTest() { - startTime = System.nanoTime() - } + fun startTest() /** * Records the end time of a test in nanoseconds. */ - fun endTest() { - endTime = System.nanoTime() - } + fun endTest() /** * Returns the duration of the test in seconds, ensuring a non-negative result. + * The duration is calculated from the recorded start and end times. */ - fun getDuration(): Double = max(0.0, (endTime - startTime) / 1_000_000_000.0) + fun getDuration(): Double /** * Marks the test as successfully passed. */ - fun recordSuccess() { - outcome = TestOutcome.Passed - } + fun recordSuccess() /** * Records a test failure with a reason and detailed explanation. + * + * @param reason The reason for the test failure. + * @param details Detailed information about the failure. */ - fun recordFailure( - reason: String, - details: List = emptyList() - ) { - outcome = TestOutcome.Failed - failureReason = reason - failureDetails = details - } + fun recordFailure(reason: String, details: List = emptyList()) /** * Marks the test as skipped. */ - fun recordSkipped() { - outcome = TestOutcome.Skipped - } + fun recordSkipped() /** * Resets all test data to initial state. */ - fun reset() { - startTime = 0 - endTime = 0 - outcome = TestOutcome.Unknown - failureReason = null - failureDetails = null - } + fun reset() } diff --git a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt index 80b90d4..059ae1a 100644 --- a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt +++ b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt @@ -3,7 +3,7 @@ package com.buildkite.test.collector.android import com.buildkite.test.collector.android.model.TestDetails import com.buildkite.test.collector.android.model.TestFailureExpanded import com.buildkite.test.collector.android.model.TestHistory -import com.buildkite.test.collector.android.tracer.TestObserver +import com.buildkite.test.collector.android.tracer.BuildkiteTestObserver import com.buildkite.test.collector.android.util.configureUnitTestUploader import org.gradle.api.Plugin import org.gradle.api.Project @@ -24,7 +24,7 @@ class UnitTestCollectorPlugin : Plugin { test.addTestListener(object : TestListener { private val testUploader = configureUnitTestUploader() - private val testObserver = TestObserver() + private val testObserver = BuildkiteTestObserver() private val testCollection: MutableList = mutableListOf() override fun beforeSuite(suite: TestDescriptor) { @@ -33,7 +33,7 @@ class UnitTestCollectorPlugin : Plugin { override fun afterSuite(suite: TestDescriptor, result: TestResult) { if (isFinalTestSuiteCall(testDescription = suite)) { - testUploader.configureUploadData(testCollection = testCollection) + testUploader.uploadTestData(testCollection = testCollection) } } diff --git a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt index bd9ec6d..92f606e 100644 --- a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt +++ b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt @@ -1,12 +1,13 @@ package com.buildkite.test.collector.android.util +import com.buildkite.test.collector.android.BuildKiteTestDataUploader import com.buildkite.test.collector.android.TestDataUploader import com.buildkite.test.collector.android.tracer.environment.BuildkiteEnvironmentValues /** * Configures a [TestDataUploader] instance for unit tests. */ -fun configureUnitTestUploader() = TestDataUploader( +fun configureUnitTestUploader() = BuildKiteTestDataUploader( testSuiteApiToken = getStringEnvironmentValue(key = BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_TOKEN), isDebugEnabled = getBooleanEnvironmentValue(key = BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_DEBUG_ENABLED) ) From 24203c4a84b0678edb14b03029bf444a62c3431b Mon Sep 17 00:00:00 2001 From: Bhumil Soni Date: Wed, 5 Jun 2024 18:29:51 +1000 Subject: [PATCH 04/11] Extract TestListener implementation from UnitTestCollectorPlugin Extracted TestListener implementation to a new class UnitTestListener from UnitTestCollectorPlugin for better separation of concerns. --- .../android/UnitTestCollectorPlugin.kt | 108 ++---------------- .../collector/android/UnitTestListener.kt | 107 +++++++++++++++++ .../android/util/ConfigureTestUploader.kt | 19 --- .../android/util/UnitTestCollectorUtils.kt | 25 ++++ example/build.gradle.kts | 2 +- 5 files changed, 142 insertions(+), 119 deletions(-) create mode 100644 collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestListener.kt delete mode 100644 collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt create mode 100644 collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/UnitTestCollectorUtils.kt diff --git a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt index 059ae1a..a0d8de4 100644 --- a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt +++ b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt @@ -1,115 +1,25 @@ package com.buildkite.test.collector.android -import com.buildkite.test.collector.android.model.TestDetails -import com.buildkite.test.collector.android.model.TestFailureExpanded -import com.buildkite.test.collector.android.model.TestHistory import com.buildkite.test.collector.android.tracer.BuildkiteTestObserver -import com.buildkite.test.collector.android.util.configureUnitTestUploader +import com.buildkite.test.collector.android.util.UnitTestCollectorUtils import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.tasks.testing.Test -import org.gradle.api.tasks.testing.TestDescriptor -import org.gradle.api.tasks.testing.TestListener -import org.gradle.api.tasks.testing.TestResult /** * A Gradle plugin that automates the collection and uploading of unit test results to Buildkite's Test Analytics service. - * It attaches a listener [Test] task within Gradle projects, capturing real-time test events and outcomes. + * It attaches a [UnitTestListener] to the [Test] tasks within Gradle projects, capturing real-time test events and outcomes. * This enables detailed monitoring and systematic reporting of test results, which are uploaded directly to the analytics portal - * at the conclusion of test suite. + * at the conclusion of the test suite. */ class UnitTestCollectorPlugin : Plugin { override fun apply(project: Project) { - project.tasks.withType(Test::class.java) { test -> - - test.addTestListener(object : TestListener { - private val testUploader = configureUnitTestUploader() - private val testObserver = BuildkiteTestObserver() - private val testCollection: MutableList = mutableListOf() - - override fun beforeSuite(suite: TestDescriptor) { - /* Nothing to do before the test suite has started */ - } - - override fun afterSuite(suite: TestDescriptor, result: TestResult) { - if (isFinalTestSuiteCall(testDescription = suite)) { - testUploader.uploadTestData(testCollection = testCollection) - } - } - - override fun beforeTest(testDescriptor: TestDescriptor) { - testObserver.startTest() - } - - override fun afterTest(testDescriptor: TestDescriptor, result: TestResult) { - testObserver.endTest() - - when (result.resultType) { - TestResult.ResultType.SUCCESS -> { - testObserver.recordSuccess() - } - - TestResult.ResultType.FAILURE -> { - val failureReason = result.exception.toString() - val failureDetails = result.exceptions.map { exception -> - TestFailureExpanded( - expanded = listOf("${exception.message}:${exception.cause}"), - backtrace = exception.stackTraceToString().split("\n") - .map { it.trim() } - ) - } - testObserver.recordFailure( - reason = failureReason, - details = failureDetails - ) - } - - TestResult.ResultType.SKIPPED -> { - testObserver.recordSkipped() - } - - null -> { - // Handle the case where [TestResult] is unexpectedly null - testObserver.recordFailure( - reason = "TestResult type was unexpectedly null, indicating an error in the test framework." - ) - } - } - - addTestDetailsToCollection(test = testDescriptor) - } - - private fun addTestDetailsToCollection(test: TestDescriptor) { - val testHistory = TestHistory( - startAt = testObserver.startTime, - endAt = testObserver.endTime, - duration = testObserver.getDuration() - ) - - val testDetails = TestDetails( - scope = test.className, - name = test.displayName, - location = test.className, - fileName = null, - result = testObserver.outcome, - failureReason = testObserver.failureReason, - failureExpanded = testObserver.failureDetails, - history = testHistory - ) - - testCollection.add(testDetails) - testObserver.reset() - } - - /** - * Determines if the provided test suite descriptor indicates the final call of the test suite, - * which is true when both className and parent are null. - * - * @param testDescription The test description. [TestDescriptor] can be atomic (a single test) or compound (containing children tests). - */ - private fun isFinalTestSuiteCall(testDescription: TestDescriptor) = - testDescription.className == null && testDescription.parent == null - }) + project.tasks.withType(Test::class.java).configureEach { test -> + val testListener = UnitTestListener( + testUploader = UnitTestCollectorUtils.configureTestUploader(), + testObserver = BuildkiteTestObserver() + ) + test.addTestListener(testListener) } } } diff --git a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestListener.kt b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestListener.kt new file mode 100644 index 0000000..4e99666 --- /dev/null +++ b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestListener.kt @@ -0,0 +1,107 @@ +package com.buildkite.test.collector.android + +import com.buildkite.test.collector.android.model.TestDetails +import com.buildkite.test.collector.android.model.TestFailureExpanded +import com.buildkite.test.collector.android.model.TestHistory +import com.buildkite.test.collector.android.tracer.TestObserver +import org.gradle.api.tasks.testing.Test +import org.gradle.api.tasks.testing.TestDescriptor +import org.gradle.api.tasks.testing.TestListener +import org.gradle.api.tasks.testing.TestResult + +/** + * A listener for Gradle's [Test] tasks that collects and uploads test results. + * This listener observes the test execution, records details about each test, and uploads the results at the end of the test suite. + * + * @property testUploader The uploader used to send test data to the analytics service. + * @property testObserver The observer used to track and record test details. + */ +internal class UnitTestListener( + private val testUploader: TestDataUploader, + private val testObserver: TestObserver +) : TestListener { + private val testCollection: MutableList = mutableListOf() + + override fun beforeSuite(suite: TestDescriptor) { + /* Nothing to do before the test suite has started */ + } + + override fun afterSuite(suite: TestDescriptor, result: TestResult) { + if (isFinalTestSuiteCall(testDescription = suite)) { + testUploader.uploadTestData(testCollection = testCollection) + } + } + + override fun beforeTest(testDescriptor: TestDescriptor) { + testObserver.startTest() + } + + override fun afterTest(testDescriptor: TestDescriptor, result: TestResult) { + testObserver.endTest() + + when (result.resultType) { + TestResult.ResultType.SUCCESS -> { + testObserver.recordSuccess() + } + + TestResult.ResultType.FAILURE -> { + val failureReason = result.exception.toString() + val failureDetails = result.exceptions.map { exception -> + TestFailureExpanded( + expanded = listOf("${exception.message}:${exception.cause}"), + backtrace = exception.stackTraceToString().split("\n") + .map { it.trim() } + ) + } + testObserver.recordFailure( + reason = failureReason, + details = failureDetails + ) + } + + TestResult.ResultType.SKIPPED -> { + testObserver.recordSkipped() + } + + null -> { + // Handle the case where TestResult is unexpectedly null */ + testObserver.recordFailure( + reason = "TestResult type was unexpectedly null, indicating an error in the test framework." + ) + } + } + + addTestDetailsToCollection(test = testDescriptor) + } + + private fun addTestDetailsToCollection(test: TestDescriptor) { + val testHistory = TestHistory( + startAt = testObserver.startTime, + endAt = testObserver.endTime, + duration = testObserver.getDuration() + ) + + val testDetails = TestDetails( + scope = test.className, + name = test.displayName, + location = test.className, + fileName = null, + result = testObserver.outcome, + failureReason = testObserver.failureReason, + failureExpanded = testObserver.failureDetails, + history = testHistory + ) + + testCollection.add(testDetails) + testObserver.reset() + } + + /** + * Determines if the provided test suite descriptor indicates the final call of the test suite, + * which is true when both className and parent are null. + * + * @param testDescription The test description. [TestDescriptor] can be atomic (a single test) or compound (containing children tests). + */ + private fun isFinalTestSuiteCall(testDescription: TestDescriptor) = + testDescription.className == null && testDescription.parent == null +} diff --git a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt deleted file mode 100644 index 92f606e..0000000 --- a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt +++ /dev/null @@ -1,19 +0,0 @@ -package com.buildkite.test.collector.android.util - -import com.buildkite.test.collector.android.BuildKiteTestDataUploader -import com.buildkite.test.collector.android.TestDataUploader -import com.buildkite.test.collector.android.tracer.environment.BuildkiteEnvironmentValues - -/** - * Configures a [TestDataUploader] instance for unit tests. - */ -fun configureUnitTestUploader() = BuildKiteTestDataUploader( - testSuiteApiToken = getStringEnvironmentValue(key = BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_TOKEN), - isDebugEnabled = getBooleanEnvironmentValue(key = BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_DEBUG_ENABLED) -) - -private fun getStringEnvironmentValue(key: String): String? = - System.getenv(key)?.takeIf { value -> value.isNotBlank() && value != "null" } - -private fun getBooleanEnvironmentValue(key: String): Boolean = - System.getenv(key)?.toBoolean() ?: false diff --git a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/UnitTestCollectorUtils.kt b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/UnitTestCollectorUtils.kt new file mode 100644 index 0000000..f3391d7 --- /dev/null +++ b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/UnitTestCollectorUtils.kt @@ -0,0 +1,25 @@ +package com.buildkite.test.collector.android.util + +import com.buildkite.test.collector.android.BuildKiteTestDataUploader +import com.buildkite.test.collector.android.TestDataUploader +import com.buildkite.test.collector.android.tracer.environment.BuildkiteEnvironmentValues + +internal object UnitTestCollectorUtils { + /** + * Configures a [BuildKiteTestDataUploader] instance for unit tests. + */ + fun configureTestUploader(): TestDataUploader { + return BuildKiteTestDataUploader( + testSuiteApiToken = getStringEnvironmentValue(key = BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_TOKEN), + isDebugEnabled = getBooleanEnvironmentValue( + key = BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_DEBUG_ENABLED + ) + ) + } + + private fun getStringEnvironmentValue(key: String): String? = + System.getenv(key)?.takeIf { value -> value.isNotBlank() && value != "null" } + + private fun getBooleanEnvironmentValue(key: String): Boolean = + System.getenv(key)?.toBoolean() ?: false +} diff --git a/example/build.gradle.kts b/example/build.gradle.kts index c654170..11a715a 100644 --- a/example/build.gradle.kts +++ b/example/build.gradle.kts @@ -15,7 +15,7 @@ android { testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" - // Specifies `ExampleTestCollector` as the instrumented test listener for collecting test analytics. + // Specifies `InstrumentedTestCollector` as the instrumented test listener for collecting test analytics. testInstrumentationRunnerArguments["listener"] = "com.buildkite.test.collector.android.InstrumentedTestCollector" // Passes environment variables as instrumentation arguments From d5610512fcf3ec0cda64f791c8231955d8418e52 Mon Sep 17 00:00:00 2001 From: Bhumil Soni Date: Wed, 5 Jun 2024 18:42:33 +1000 Subject: [PATCH 05/11] Extract RunListener implementation from InstrumentedTestCollector Extracted RunListener implementation to a new class InstrumentedTestListener from InstrumentedTestCollector for better separation of concerns and testability. --- .../android/InstrumentedTestCollector.kt | 95 +++--------------- .../android/InstrumentedTestListener.kt | 97 +++++++++++++++++++ .../android/util/ConfigureTestUploader.kt | 26 ----- .../util/InstrumentedTestCollectorUtils.kt | 29 ++++++ 4 files changed, 142 insertions(+), 105 deletions(-) create mode 100644 collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestListener.kt delete mode 100644 collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt create mode 100644 collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/InstrumentedTestCollectorUtils.kt diff --git a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt index 35297b1..edfed67 100644 --- a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt +++ b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt @@ -1,100 +1,37 @@ package com.buildkite.test.collector.android import androidx.test.platform.app.InstrumentationRegistry -import com.buildkite.test.collector.android.model.TestDetails -import com.buildkite.test.collector.android.model.TestFailureExpanded -import com.buildkite.test.collector.android.model.TestHistory -import com.buildkite.test.collector.android.model.TestOutcome import com.buildkite.test.collector.android.tracer.BuildkiteTestObserver -import com.buildkite.test.collector.android.util.configureInstrumentedTestUploader +import com.buildkite.test.collector.android.util.InstrumentedTestCollectorUtils import org.junit.runner.Description import org.junit.runner.notification.Failure import org.junit.runner.notification.RunListener /** - * Serves as an abstract foundation for creating instrumented test collectors that interface with Buildkite's Test Analytics service. - * This class extends JUnit's [RunListener] to capture real-time events during the execution of instrumented tests, enabling precise monitoring - * and reporting of test outcomes. It automatically gathers detailed test results and uploads them directly to the analytics portal at the conclusion of test suite. + * Collects and uploads instrumented test results to Buildkite's Test Analytics service. + * This class extends JUnit's [RunListener] to capture real-time events during the execution of instrumented tests, + * enabling precise monitoring and reporting of test outcomes. It delegates actual event handling to [InstrumentedTestListener]. */ class InstrumentedTestCollector : RunListener() { - private val testObserver = BuildkiteTestObserver() - private val testCollection: MutableList = mutableListOf() - private val testUploader: TestDataUploader by lazy { configureTestUploader() } + private val listener: InstrumentedTestListener by lazy { configureTestListener() } - override fun testSuiteStarted(testDescription: Description) { - /* Nothing to do before the test suite has started */ - } - - override fun testSuiteFinished(description: Description) { - if (isFinalTestSuiteCall(testDescription = description)) { - testUploader.uploadTestData(testCollection = testCollection) - } - } - - override fun testStarted(testDescription: Description) { - testObserver.startTest() - } - - override fun testFinished(testDescription: Description) { - testObserver.endTest() - - if (testObserver.outcome != TestOutcome.Failed) { - testObserver.recordSuccess() - } - - addTestDetailsToCollection(test = testDescription) - } - - override fun testFailure(failureDetails: Failure) { - val failureReason = failureDetails.exception.toString() - val details = listOf( - TestFailureExpanded( - expanded = failureDetails.trimmedTrace.split("\n").map { it.trim() }, - backtrace = failureDetails.trace.split("\n").map { it.trim() }, - ) + private fun configureTestListener(): InstrumentedTestListener { + val arguments = InstrumentationRegistry.getArguments() + return InstrumentedTestListener( + testObserver = BuildkiteTestObserver(), + testUploader = InstrumentedTestCollectorUtils.configureTestUploader(arguments = arguments) ) - testObserver.recordFailure(reason = failureReason, details = details) } - override fun testIgnored(testDescription: Description) { - testObserver.recordSkipped() + override fun testSuiteStarted(description: Description) { listener.testSuiteStarted(description) } - addTestDetailsToCollection(test = testDescription) - } + override fun testSuiteFinished(description: Description) { listener.testSuiteFinished(description) } - private fun addTestDetailsToCollection(test: Description) { - val testHistory = TestHistory( - startAt = testObserver.startTime, - endAt = testObserver.endTime, - duration = testObserver.getDuration() - ) + override fun testStarted(description: Description) { listener.testStarted(description) } - val testDetails = TestDetails( - scope = test.testClass.name, - name = test.methodName, - location = test.className, - fileName = null, - result = testObserver.outcome, - failureReason = testObserver.failureReason, - failureExpanded = testObserver.failureDetails, - history = testHistory - ) + override fun testFinished(description: Description) { listener.testFinished(description) } - testCollection.add(testDetails) - testObserver.reset() - } - - /** - * Determines if the provided test suite descriptor indicates the final call of the test suite, - * which is true when both displayName and className are null. - * - * @param testDescription The test description. [Description] can be atomic (a single test) or compound (containing children tests). - */ - private fun isFinalTestSuiteCall(testDescription: Description) = - (testDescription.displayName.isNullOrEmpty() || testDescription.displayName == "null") && (testDescription.className.isNullOrEmpty() || testDescription.className == "null") + override fun testFailure(failure: Failure) { listener.testFailure(failure) } - private fun configureTestUploader(): TestDataUploader { - val arguments = InstrumentationRegistry.getArguments() - return configureInstrumentedTestUploader(instrumentationArguments = arguments) - } + override fun testIgnored(description: Description) { listener.testIgnored(description) } } diff --git a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestListener.kt b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestListener.kt new file mode 100644 index 0000000..bb87cdc --- /dev/null +++ b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestListener.kt @@ -0,0 +1,97 @@ +package com.buildkite.test.collector.android + +import com.buildkite.test.collector.android.model.TestDetails +import com.buildkite.test.collector.android.model.TestFailureExpanded +import com.buildkite.test.collector.android.model.TestHistory +import com.buildkite.test.collector.android.model.TestOutcome +import com.buildkite.test.collector.android.tracer.TestObserver +import org.junit.runner.Description +import org.junit.runner.notification.Failure +import org.junit.runner.notification.RunListener + +/** + * Listens to instrumented test events and collects test details. + * This class extends JUnit's [RunListener] to capture real-time events during the execution of tests. + * It automatically gathers detailed test results and uploads them directly to the analytics portal at the conclusion of the test suite. + * + * @property testUploader The uploader used to send test data to the analytics service. + * @property testObserver The observer used to track and record test details. + */ +internal class InstrumentedTestListener( + private val testUploader: TestDataUploader, + private val testObserver: TestObserver +) : RunListener() { + private val testCollection: MutableList = mutableListOf() + + override fun testSuiteStarted(testDescription: Description) { + /* Nothing to do before the test suite has started */ + } + + override fun testSuiteFinished(description: Description) { + if (isFinalTestSuiteCall(testDescription = description)) { + testUploader.uploadTestData(testCollection = testCollection) + } + } + + override fun testStarted(testDescription: Description) { + testObserver.startTest() + } + + override fun testFinished(testDescription: Description) { + testObserver.endTest() + + if (testObserver.outcome != TestOutcome.Failed) { + testObserver.recordSuccess() + } + + addTestDetailsToCollection(test = testDescription) + } + + override fun testFailure(failureDetails: Failure) { + val failureReason = failureDetails.exception.toString() + val details = listOf( + TestFailureExpanded( + expanded = failureDetails.trimmedTrace.split("\n").map { it.trim() }, + backtrace = failureDetails.trace.split("\n").map { it.trim() }, + ) + ) + testObserver.recordFailure(reason = failureReason, details = details) + } + + override fun testIgnored(testDescription: Description) { + testObserver.recordSkipped() + + addTestDetailsToCollection(test = testDescription) + } + + private fun addTestDetailsToCollection(test: Description) { + val testHistory = TestHistory( + startAt = testObserver.startTime, + endAt = testObserver.endTime, + duration = testObserver.getDuration() + ) + + val testDetails = TestDetails( + scope = test.testClass.name, + name = test.methodName, + location = test.className, + fileName = null, + result = testObserver.outcome, + failureReason = testObserver.failureReason, + failureExpanded = testObserver.failureDetails, + history = testHistory + ) + + testCollection.add(testDetails) + testObserver.reset() + } + + /** + * Determines if the provided test suite descriptor indicates the final call of the test suite, + * which is true when both displayName and className are null. + * + * @param testDescription The test description. [Description] can be atomic (a single test) or compound (containing children tests). + */ + private fun isFinalTestSuiteCall(testDescription: Description) = + (testDescription.displayName.isNullOrEmpty() || testDescription.displayName == "null") && (testDescription.className.isNullOrEmpty() || testDescription.className == "null") +} diff --git a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt deleted file mode 100644 index ffacfbc..0000000 --- a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/ConfigureTestUploader.kt +++ /dev/null @@ -1,26 +0,0 @@ -package com.buildkite.test.collector.android.util - -import android.os.Bundle -import com.buildkite.test.collector.android.BuildKiteTestDataUploader -import com.buildkite.test.collector.android.TestDataUploader -import com.buildkite.test.collector.android.tracer.environment.BuildkiteEnvironmentValues - -/** - * Creates a [TestDataUploader] instance for instrumented tests. - */ -fun configureInstrumentedTestUploader(instrumentationArguments: Bundle): TestDataUploader { - return BuildKiteTestDataUploader( - testSuiteApiToken = instrumentationArguments.getStringEnvironmentValue( - BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_TOKEN - ), - isDebugEnabled = instrumentationArguments.getBooleanEnvironmentValue( - BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_DEBUG_ENABLED - ) - ) -} - -private fun Bundle.getStringEnvironmentValue(key: String): String? = - getString(key)?.takeIf { value -> value.isNotBlank() && value != "null" } - -private fun Bundle.getBooleanEnvironmentValue(key: String): Boolean = - getString(key)?.toBoolean() ?: false diff --git a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/InstrumentedTestCollectorUtils.kt b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/InstrumentedTestCollectorUtils.kt new file mode 100644 index 0000000..1c53b72 --- /dev/null +++ b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/InstrumentedTestCollectorUtils.kt @@ -0,0 +1,29 @@ +package com.buildkite.test.collector.android.util + +import android.os.Bundle +import com.buildkite.test.collector.android.BuildKiteTestDataUploader +import com.buildkite.test.collector.android.TestDataUploader +import com.buildkite.test.collector.android.tracer.environment.BuildkiteEnvironmentValues + +internal object InstrumentedTestCollectorUtils { + + /** + * Creates a [TestDataUploader] instance for instrumented tests. + */ + fun configureTestUploader(arguments: Bundle): TestDataUploader { + return BuildKiteTestDataUploader( + testSuiteApiToken = arguments.getStringEnvironmentValue( + BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_TOKEN + ), + isDebugEnabled = arguments.getBooleanEnvironmentValue( + BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_DEBUG_ENABLED + ) + ) + } + + private fun Bundle.getStringEnvironmentValue(key: String): String? = + getString(key)?.takeIf { value -> value.isNotBlank() && value != "null" } + + private fun Bundle.getBooleanEnvironmentValue(key: String): Boolean = + getString(key)?.toBoolean() ?: false +} From 4282d3e56358fa846f94a7b4fd37eef37914ab85 Mon Sep 17 00:00:00 2001 From: Bhumil Soni Date: Thu, 6 Jun 2024 11:13:11 +1000 Subject: [PATCH 06/11] Add support for uploading environment variables from various CI system The commit adds support for uploading environment variables from various CI system - Buildkite, CircleCI and GitHubActions via accessing variables each system provides. If no CI system is detected, it sends default environment data. --- .../android/InstrumentedTestCollector.kt | 32 +- .../InstrumentedTestEnvironmentProvider.kt | 13 + .../util/InstrumentedTestCollectorUtils.kt | 29 -- .../android/BuildKiteTestDataUploader.kt | 27 +- .../collector/android/model/RunEnvironment.kt | 30 +- .../BaseTestEnvironmentProvider.kt | 100 +++++ .../environment/BuildkiteEnvironmentValues.kt | 6 - .../environment/TestEnvironmentProvider.kt | 27 ++ .../environment/TestEnvironmentValue.kt | 36 ++ .../test/collector/android/util/StringExt.kt | 4 + .../BaseTestEnvironmentProviderTest.kt | 409 ++++++++++++++++++ .../android/UnitTestCollectorPlugin.kt | 5 +- .../UnitTestEnvironmentProvider.kt | 10 + .../android/util/UnitTestCollectorUtils.kt | 25 -- example/build.gradle.kts | 14 +- 15 files changed, 652 insertions(+), 115 deletions(-) create mode 100644 collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/environment/InstrumentedTestEnvironmentProvider.kt delete mode 100644 collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/InstrumentedTestCollectorUtils.kt create mode 100644 collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/BaseTestEnvironmentProvider.kt delete mode 100644 collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/BuildkiteEnvironmentValues.kt create mode 100644 collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/TestEnvironmentProvider.kt create mode 100644 collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/TestEnvironmentValue.kt create mode 100644 collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/StringExt.kt create mode 100644 collector/test-data-uploader/src/test/kotlin/com/buildkite/test/collector/android/tracer/environment/BaseTestEnvironmentProviderTest.kt create mode 100644 collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/environment/UnitTestEnvironmentProvider.kt delete mode 100644 collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/UnitTestCollectorUtils.kt diff --git a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt index edfed67..9753f37 100644 --- a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt +++ b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt @@ -1,8 +1,8 @@ package com.buildkite.test.collector.android import androidx.test.platform.app.InstrumentationRegistry +import com.buildkite.test.collector.android.environment.InstrumentedTestEnvironmentProvider import com.buildkite.test.collector.android.tracer.BuildkiteTestObserver -import com.buildkite.test.collector.android.util.InstrumentedTestCollectorUtils import org.junit.runner.Description import org.junit.runner.notification.Failure import org.junit.runner.notification.RunListener @@ -17,21 +17,35 @@ class InstrumentedTestCollector : RunListener() { private fun configureTestListener(): InstrumentedTestListener { val arguments = InstrumentationRegistry.getArguments() + val environmentProvider = InstrumentedTestEnvironmentProvider(arguments = arguments) + return InstrumentedTestListener( - testObserver = BuildkiteTestObserver(), - testUploader = InstrumentedTestCollectorUtils.configureTestUploader(arguments = arguments) + testUploader = BuildKiteTestDataUploader(testEnvironmentProvider = environmentProvider), + testObserver = BuildkiteTestObserver() ) } - override fun testSuiteStarted(description: Description) { listener.testSuiteStarted(description) } + override fun testSuiteStarted(description: Description) { + listener.testSuiteStarted(description) + } - override fun testSuiteFinished(description: Description) { listener.testSuiteFinished(description) } + override fun testSuiteFinished(description: Description) { + listener.testSuiteFinished(description) + } - override fun testStarted(description: Description) { listener.testStarted(description) } + override fun testStarted(description: Description) { + listener.testStarted(description) + } - override fun testFinished(description: Description) { listener.testFinished(description) } + override fun testFinished(description: Description) { + listener.testFinished(description) + } - override fun testFailure(failure: Failure) { listener.testFailure(failure) } + override fun testFailure(failure: Failure) { + listener.testFailure(failure) + } - override fun testIgnored(description: Description) { listener.testIgnored(description) } + override fun testIgnored(description: Description) { + listener.testIgnored(description) + } } diff --git a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/environment/InstrumentedTestEnvironmentProvider.kt b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/environment/InstrumentedTestEnvironmentProvider.kt new file mode 100644 index 0000000..f4d3a62 --- /dev/null +++ b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/environment/InstrumentedTestEnvironmentProvider.kt @@ -0,0 +1,13 @@ +package com.buildkite.test.collector.android.environment + +import android.os.Bundle +import com.buildkite.test.collector.android.tracer.environment.BaseTestEnvironmentProvider + +/** + * Provides environment values for instrumented tests by fetching from Android [Bundle] arguments. + */ +internal class InstrumentedTestEnvironmentProvider( + private val arguments: Bundle +) : BaseTestEnvironmentProvider() { + override fun getEnvironmentValue(key: String): String? = arguments.getString(key) +} diff --git a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/InstrumentedTestCollectorUtils.kt b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/InstrumentedTestCollectorUtils.kt deleted file mode 100644 index 1c53b72..0000000 --- a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/InstrumentedTestCollectorUtils.kt +++ /dev/null @@ -1,29 +0,0 @@ -package com.buildkite.test.collector.android.util - -import android.os.Bundle -import com.buildkite.test.collector.android.BuildKiteTestDataUploader -import com.buildkite.test.collector.android.TestDataUploader -import com.buildkite.test.collector.android.tracer.environment.BuildkiteEnvironmentValues - -internal object InstrumentedTestCollectorUtils { - - /** - * Creates a [TestDataUploader] instance for instrumented tests. - */ - fun configureTestUploader(arguments: Bundle): TestDataUploader { - return BuildKiteTestDataUploader( - testSuiteApiToken = arguments.getStringEnvironmentValue( - BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_TOKEN - ), - isDebugEnabled = arguments.getBooleanEnvironmentValue( - BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_DEBUG_ENABLED - ) - ) - } - - private fun Bundle.getStringEnvironmentValue(key: String): String? = - getString(key)?.takeIf { value -> value.isNotBlank() && value != "null" } - - private fun Bundle.getBooleanEnvironmentValue(key: String): Boolean = - getString(key)?.toBoolean() ?: false -} diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/BuildKiteTestDataUploader.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/BuildKiteTestDataUploader.kt index 8fd9957..ca461b1 100644 --- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/BuildKiteTestDataUploader.kt +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/BuildKiteTestDataUploader.kt @@ -1,28 +1,33 @@ package com.buildkite.test.collector.android -import com.buildkite.test.collector.android.model.RunEnvironment import com.buildkite.test.collector.android.model.TestData import com.buildkite.test.collector.android.model.TestDetails import com.buildkite.test.collector.android.model.TestUploadResponse import com.buildkite.test.collector.android.network.TestAnalyticsRetrofit import com.buildkite.test.collector.android.network.api.TestUploaderApi +import com.buildkite.test.collector.android.tracer.environment.TestEnvironmentProvider import com.buildkite.test.collector.android.util.CollectorUtils import com.buildkite.test.collector.android.util.Logger import retrofit2.Response /** - * Manages the upload of test data to the Buildkite Test Analytics Suite associated with the provided API token. + * Manages the upload of test data to the Buildkite Test Analytics Suite using the provided environment values. * - * @property testSuiteApiToken The API token for the test suite, necessary for authenticating requests with Test Analytics. - * Test data will not be uploaded if this is null. - * @property isDebugEnabled When true, enables logging to assist with debugging. + * This class fetches the necessary environment configuration from a [TestEnvironmentProvider]. + * + * @property testEnvironmentProvider Provides the environment configuration needed for uploading test data. */ class BuildKiteTestDataUploader( - private val testSuiteApiToken: String?, - private val isDebugEnabled: Boolean + private val testEnvironmentProvider: TestEnvironmentProvider ) : TestDataUploader { + private val testSuiteApiToken = testEnvironmentProvider.testSuiteApiToken + private val logger = - Logger(minLevel = if (isDebugEnabled) Logger.LogLevel.DEBUG else Logger.LogLevel.INFO) + Logger(minLevel = if (testEnvironmentProvider.isDebugEnabled) Logger.LogLevel.DEBUG else Logger.LogLevel.INFO) + + init { + logger.debug { "BuildKiteTestDataUploader: Test RunEnvironment is: ${testEnvironmentProvider.getRunEnvironment()}" } + } /** * Uploads test data to the Buildkite Test Analytics Suite. @@ -31,11 +36,9 @@ class BuildKiteTestDataUploader( * @param testCollection A list of [TestDetails] representing all the tests within the suite. */ override fun uploadTestData(testCollection: List) { - val runEnvironment = RunEnvironment().getEnvironmentValues() - val testData = TestData( format = "json", - runEnvironment = runEnvironment, + runEnvironment = testEnvironmentProvider.getRunEnvironment(), data = testCollection.take(CollectorUtils.Uploader.TEST_DATA_UPLOAD_LIMIT) ) @@ -77,7 +80,7 @@ class BuildKiteTestDataUploader( logger.error { "Error uploading test analytics data. HTTP error code: ${testUploadResponse.code()}. Ensure the test suite API token is correct and properly configured." } - logger.debug { "Failed response details: ${testUploadResponse.errorBody()?.string()}" } + logger.error { "Failed response details: ${testUploadResponse.errorBody()?.string()}" } } } } diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/RunEnvironment.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/RunEnvironment.kt index becc81d..89abfa5 100644 --- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/RunEnvironment.kt +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/RunEnvironment.kt @@ -18,7 +18,7 @@ import com.google.gson.annotations.SerializedName * @property version The current version of the collector. * @property collector The current name of the collector. */ -internal data class RunEnvironment( +data class RunEnvironment( @SerializedName("CI") val ci: String? = null, @SerializedName("key") val key: String = generateUUIDString(), @SerializedName("url") val url: String? = null, @@ -30,34 +30,6 @@ internal data class RunEnvironment( @SerializedName("version") val version: String = VERSION_NAME, @SerializedName("collector") val collector: String = COLLECTOR_NAME ) { - fun getEnvironmentValues(): RunEnvironment { - val buildKiteRunEnvironment: RunEnvironment? = null - val gitHubActionsRunEnvironment: RunEnvironment? = null - val circleCiRunEnvironment: RunEnvironment? = null - val genericCiRunEnvironment: RunEnvironment? = null - - val localRunEnvironment = RunEnvironment( - ci = ci, - key = key, - url = url, - branch = branch, - commitSha = commitSha, - number = number, - jobId = jobId, - message = message, - version = version, - collector = collector - ) - - val ciRunEnvironment: RunEnvironment? = buildKiteRunEnvironment - ?: gitHubActionsRunEnvironment - ?: circleCiRunEnvironment - ?: genericCiRunEnvironment - - return ciRunEnvironment - ?: localRunEnvironment - } - companion object { // When bumping version, update VERSION_NAME to match new version // Used for uploading correct library version diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/BaseTestEnvironmentProvider.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/BaseTestEnvironmentProvider.kt new file mode 100644 index 0000000..610ddfa --- /dev/null +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/BaseTestEnvironmentProvider.kt @@ -0,0 +1,100 @@ +package com.buildkite.test.collector.android.tracer.environment + +import com.buildkite.test.collector.android.model.RunEnvironment +import com.buildkite.test.collector.android.util.takeIfValid + +/** + * Base implementation of [TestEnvironmentProvider], providing methods to fetch environment values from various sources + * such as system properties or Android Bundle arguments. + */ +abstract class BaseTestEnvironmentProvider : TestEnvironmentProvider { + protected abstract fun getEnvironmentValue(key: String): String? + + private fun safeEnvValue(key: String): String? = + getEnvironmentValue(key).takeIfValid() + + final override val testSuiteApiToken: String? + get() = safeEnvValue(key = TestEnvironmentValue.BUILDKITE_ANALYTICS_TOKEN) + + final override val isDebugEnabled: Boolean + get() = safeEnvValue(key = TestEnvironmentValue.BUILDKITE_ANALYTICS_DEBUG_ENABLED).toBoolean() + + final override fun getRunEnvironment(defaultKey: String): RunEnvironment = + getGitHubActionsEnvironment() + ?: getBuildkiteEnvironment() + ?: getCircleCiEnvironment() + ?: getLocalEnvironment(runKey = defaultKey) + + private fun getBuildkiteEnvironment(): RunEnvironment? { + val buildId = safeEnvValue(key = TestEnvironmentValue.BUILDKITE_BUILD_ID) + ?: return null + + return RunEnvironment( + ci = "buildkite", + key = buildId, + url = safeEnvValue(key = TestEnvironmentValue.BUILDKITE_BUILD_URL), + branch = safeEnvValue(key = TestEnvironmentValue.BUILDKITE_BRANCH), + commitSha = safeEnvValue(key = TestEnvironmentValue.BUILDKITE_COMMIT), + number = safeEnvValue(key = TestEnvironmentValue.BUILDKITE_BUILD_NUMBER), + jobId = safeEnvValue(key = TestEnvironmentValue.BUILDKITE_JOB_ID), + message = safeEnvValue(key = TestEnvironmentValue.BUILDKITE_MESSAGE) + ) + } + + private fun getCircleCiEnvironment(): RunEnvironment? { + val buildNumber = safeEnvValue(key = TestEnvironmentValue.CIRCLE_BUILD_NUM) + val workflowId = + buildNumber?.let { safeEnvValue(key = TestEnvironmentValue.CIRCLE_WORKFLOW_ID) } + + if (buildNumber == null || workflowId == null) return null + + return RunEnvironment( + ci = "circleci", + key = "$workflowId-$buildNumber", + url = safeEnvValue(key = TestEnvironmentValue.CIRCLE_BUILD_URL), + branch = safeEnvValue(key = TestEnvironmentValue.CIRCLE_BRANCH), + commitSha = safeEnvValue(key = TestEnvironmentValue.CIRCLE_SHA1), + number = buildNumber, + message = "Build #$buildNumber on branch ${safeEnvValue(key = TestEnvironmentValue.CIRCLE_BRANCH) ?: "[Unknown branch]"}" + ) + } + + private fun getGitHubActionsEnvironment(): RunEnvironment? { + val action = safeEnvValue(key = TestEnvironmentValue.GITHUB_ACTION) + val runNumber = + action?.let { safeEnvValue(key = TestEnvironmentValue.GITHUB_RUN_NUMBER) } + val runAttempt = + runNumber?.let { safeEnvValue(key = TestEnvironmentValue.GITHUB_RUN_ATTEMPT) } + + if (action == null || runNumber == null || runAttempt == null) return null + + val repository = safeEnvValue(key = TestEnvironmentValue.GITHUB_REPOSITORY) + val runId = safeEnvValue(key = TestEnvironmentValue.GITHUB_RUN_ID) + val url = repository?.let { "https://github.com/$repository/actions/runs/$runId" } + val workflowName = safeEnvValue(key = TestEnvironmentValue.GITHUB_WORKFLOW) + val workflowStarter = safeEnvValue(key = TestEnvironmentValue.GITHUB_ACTOR) + val message = buildString { + append("Run #$runNumber attempt #$runAttempt") + if (workflowName != null && workflowStarter != null) { + append(" of $workflowName, started by $workflowStarter") + } + } + + return RunEnvironment( + ci = "github_actions", + key = "$action-$runNumber-$runAttempt", + url = url, + branch = safeEnvValue(key = TestEnvironmentValue.GITHUB_REF_NAME), + commitSha = safeEnvValue(key = TestEnvironmentValue.GITHUB_SHA), + number = runNumber, + message = message + ) + } + + private fun getLocalEnvironment(runKey: String): RunEnvironment { + return RunEnvironment( + ci = null, + key = runKey + ) + } +} diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/BuildkiteEnvironmentValues.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/BuildkiteEnvironmentValues.kt deleted file mode 100644 index c155336..0000000 --- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/BuildkiteEnvironmentValues.kt +++ /dev/null @@ -1,6 +0,0 @@ -package com.buildkite.test.collector.android.tracer.environment - -object BuildkiteEnvironmentValues { - const val BUILDKITE_ANALYTICS_TOKEN = "BUILDKITE_ANALYTICS_TOKEN" - const val BUILDKITE_ANALYTICS_DEBUG_ENABLED = "BUILDKITE_ANALYTICS_DEBUG_ENABLED" -} diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/TestEnvironmentProvider.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/TestEnvironmentProvider.kt new file mode 100644 index 0000000..d418328 --- /dev/null +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/TestEnvironmentProvider.kt @@ -0,0 +1,27 @@ +package com.buildkite.test.collector.android.tracer.environment + +import com.buildkite.test.collector.android.model.RunEnvironment +import com.buildkite.test.collector.android.util.CollectorUtils.generateUUIDString + +/** + * Provides methods to access environment values for test analytics, including CI environment details and local configuration settings. + */ +interface TestEnvironmentProvider { + /** + * The API token for the test suite, used for authenticating requests. + */ + val testSuiteApiToken: String? + + /** + * Indicates whether debug logging is enabled. + */ + val isDebugEnabled: Boolean + + /** + * Retrieves the runtime environment details, such as CI system information or local development defaults. + * + * @param defaultKey A default key to use if no CI environment is detected. + * @return A [RunEnvironment] instance containing the relevant environment details. + */ + fun getRunEnvironment(defaultKey: String = generateUUIDString()): RunEnvironment +} diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/TestEnvironmentValue.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/TestEnvironmentValue.kt new file mode 100644 index 0000000..1259ad8 --- /dev/null +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/TestEnvironmentValue.kt @@ -0,0 +1,36 @@ +package com.buildkite.test.collector.android.tracer.environment + +/** + * Contains environment variable keys for configuring the Buildkite test analytics collector. + * + * These constants are used to retrieve environment values from local machines or Continuous Integration (CI) environments + * to configure the Buildkite test analytics collector for uploading test data. + */ +object TestEnvironmentValue { + const val BUILDKITE_ANALYTICS_TOKEN = "BUILDKITE_ANALYTICS_TOKEN" + const val BUILDKITE_ANALYTICS_DEBUG_ENABLED = "BUILDKITE_ANALYTICS_DEBUG_ENABLED" + + const val BUILDKITE_BRANCH = "BUILDKITE_BRANCH" + const val BUILDKITE_BUILD_ID = "BUILDKITE_BUILD_ID" + const val BUILDKITE_BUILD_NUMBER = "BUILDKITE_BUILD_NUMBER" + const val BUILDKITE_BUILD_URL = "BUILDKITE_BUILD_URL" + const val BUILDKITE_COMMIT = "BUILDKITE_COMMIT" + const val BUILDKITE_JOB_ID = "BUILDKITE_JOB_ID" + const val BUILDKITE_MESSAGE = "BUILDKITE_MESSAGE" + + const val CIRCLE_BRANCH = "CIRCLE_BRANCH" + const val CIRCLE_BUILD_NUM = "CIRCLE_BUILD_NUM" + const val CIRCLE_BUILD_URL = "CIRCLE_BUILD_URL" + const val CIRCLE_SHA1 = "CIRCLE_SHA1" + const val CIRCLE_WORKFLOW_ID = "CIRCLE_WORKFLOW_ID" + + const val GITHUB_ACTION = "GITHUB_ACTION" + const val GITHUB_RUN_ID = "GITHUB_RUN_ID" + const val GITHUB_RUN_NUMBER = "GITHUB_RUN_NUMBER" + const val GITHUB_RUN_ATTEMPT = "GITHUB_RUN_ATTEMPT" + const val GITHUB_REF_NAME = "GITHUB_REF_NAME" + const val GITHUB_REPOSITORY = "GITHUB_REPOSITORY" + const val GITHUB_SHA = "GITHUB_SHA" + const val GITHUB_WORKFLOW = "GITHUB_WORKFLOW" + const val GITHUB_ACTOR = "GITHUB_ACTOR" +} diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/StringExt.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/StringExt.kt new file mode 100644 index 0000000..730f146 --- /dev/null +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/StringExt.kt @@ -0,0 +1,4 @@ +package com.buildkite.test.collector.android.util + +internal fun String?.takeIfValid() = + takeIf { value -> !value.isNullOrBlank() && value != "null" } diff --git a/collector/test-data-uploader/src/test/kotlin/com/buildkite/test/collector/android/tracer/environment/BaseTestEnvironmentProviderTest.kt b/collector/test-data-uploader/src/test/kotlin/com/buildkite/test/collector/android/tracer/environment/BaseTestEnvironmentProviderTest.kt new file mode 100644 index 0000000..a0469dd --- /dev/null +++ b/collector/test-data-uploader/src/test/kotlin/com/buildkite/test/collector/android/tracer/environment/BaseTestEnvironmentProviderTest.kt @@ -0,0 +1,409 @@ +package com.buildkite.test.collector.android.tracer.environment + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +// Mock implementation of BaseTestEnvironmentProvider for testing purposes +class MockTestEnvironmentProvider( + private val environment: Map +) : BaseTestEnvironmentProvider() { + override fun getEnvironmentValue(key: String): String? = environment[key] +} + +class BaseTestEnvironmentProviderTest { + + @Test + fun testNoEnvironmentValues() { + val noEnvironmentValues = emptyMap() + val testProvider = MockTestEnvironmentProvider(environment = noEnvironmentValues) + + assertNull(testProvider.testSuiteApiToken) + assertFalse(testProvider.isDebugEnabled) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertNull(environment.ci) + assertEquals(testDefaultKey, environment.key) + assertNull(environment.url) + assertNull(environment.branch) + assertNull(environment.commitSha) + assertNull(environment.number) + assertNull(environment.jobId) + assertNull(environment.message) + } + + @Test + fun testLocalEnvironmentWithValues() { + val localEnvironmentWithValues = mapOf( + TestEnvironmentValue.BUILDKITE_ANALYTICS_TOKEN to "test-token", + TestEnvironmentValue.BUILDKITE_ANALYTICS_DEBUG_ENABLED to "true" + ) + val testProvider = MockTestEnvironmentProvider(environment = localEnvironmentWithValues) + + assertEquals("test-token", testProvider.testSuiteApiToken) + assertTrue(testProvider.isDebugEnabled) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertNull(environment.ci) + assertEquals(testDefaultKey, environment.key) + assertNull(environment.url) + assertNull(environment.branch) + assertNull(environment.commitSha) + assertNull(environment.number) + assertNull(environment.jobId) + assertNull(environment.message) + } + + @Test + fun testLocalEnvironmentWithInvalidValues() { + val invalidEnvironmentValues = mapOf( + TestEnvironmentValue.BUILDKITE_ANALYTICS_TOKEN to "null", + TestEnvironmentValue.BUILDKITE_ANALYTICS_DEBUG_ENABLED to "invalid-boolean-value" + ) + val testProvider = MockTestEnvironmentProvider(environment = invalidEnvironmentValues) + + assertNull(testProvider.testSuiteApiToken) + assertFalse(testProvider.isDebugEnabled) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertNull(environment.ci) + assertEquals(testDefaultKey, environment.key) + assertNull(environment.url) + assertNull(environment.branch) + assertNull(environment.commitSha) + assertNull(environment.number) + assertNull(environment.jobId) + assertNull(environment.message) + } + + @Test + fun testBuildkiteCIEnvironmentWithAllValues() { + val buildkiteEnvironmentWithAllValues = mapOf( + TestEnvironmentValue.BUILDKITE_BUILD_ID to "build-id", + TestEnvironmentValue.BUILDKITE_BUILD_URL to "http://buildkite.com/build", + TestEnvironmentValue.BUILDKITE_BRANCH to "main", + TestEnvironmentValue.BUILDKITE_COMMIT to "commit-sha", + TestEnvironmentValue.BUILDKITE_BUILD_NUMBER to "42", + TestEnvironmentValue.BUILDKITE_JOB_ID to "job-id", + TestEnvironmentValue.BUILDKITE_MESSAGE to "Build message" + ) + val testProvider = + MockTestEnvironmentProvider(environment = buildkiteEnvironmentWithAllValues) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertEquals("buildkite", environment.ci) + assertEquals("build-id", environment.key) + assertEquals("http://buildkite.com/build", environment.url) + assertEquals("main", environment.branch) + assertEquals("commit-sha", environment.commitSha) + assertEquals("42", environment.number) + assertEquals("job-id", environment.jobId) + assertEquals("Build message", environment.message) + } + + @Test + fun testBuildkiteCIEnvironmentWithMissingValues() { + val buildkiteEnvironmentWithMissingValues = mapOf( + TestEnvironmentValue.BUILDKITE_BUILD_ID to "build-id" + ) + val testProvider = + MockTestEnvironmentProvider(environment = buildkiteEnvironmentWithMissingValues) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertEquals("buildkite", environment.ci) + assertEquals("build-id", environment.key) + assertNull(environment.url) + assertNull(environment.branch) + assertNull(environment.commitSha) + assertNull(environment.number) + assertNull(environment.jobId) + assertNull(environment.message) + } + + @Test + fun testBuildkiteCIEnvironmentWithInvalidValues() { + val buildkiteEnvironmentWithInvalidValues = mapOf( + TestEnvironmentValue.BUILDKITE_BUILD_ID to "null", + TestEnvironmentValue.BUILDKITE_BUILD_URL to " ", + TestEnvironmentValue.BUILDKITE_BRANCH to "null", + TestEnvironmentValue.BUILDKITE_COMMIT to " ", + TestEnvironmentValue.BUILDKITE_BUILD_NUMBER to "null", + TestEnvironmentValue.BUILDKITE_JOB_ID to "null", + TestEnvironmentValue.BUILDKITE_MESSAGE to "null" + ) + val testProvider = + MockTestEnvironmentProvider(environment = buildkiteEnvironmentWithInvalidValues) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertNull(environment.ci) + assertEquals(testDefaultKey, environment.key) + assertNull(environment.url) + assertNull(environment.branch) + assertNull(environment.commitSha) + assertNull(environment.number) + assertNull(environment.jobId) + assertNull(environment.message) + } + + @Test + fun testCircleCiEnvironmentWithAllValues() { + val circleCiEnvironmentWithAllValues = mapOf( + TestEnvironmentValue.CIRCLE_BUILD_NUM to "123", + TestEnvironmentValue.CIRCLE_WORKFLOW_ID to "workflow-id", + TestEnvironmentValue.CIRCLE_BUILD_URL to "http://circleci.com/build", + TestEnvironmentValue.CIRCLE_BRANCH to "main", + TestEnvironmentValue.CIRCLE_SHA1 to "commit-sha" + ) + val testProvider = MockTestEnvironmentProvider(environment = circleCiEnvironmentWithAllValues) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertEquals("circleci", environment.ci) + assertEquals("workflow-id-123", environment.key) + assertEquals("http://circleci.com/build", environment.url) + assertEquals("main", environment.branch) + assertEquals("commit-sha", environment.commitSha) + assertEquals("123", environment.number) + assertEquals("Build #123 on branch main", environment.message) + } + + @Test + fun testCircleCiEnvironmentWithMissingBuildNumberKey() { + val circleCiEnvironmentWithMissingBuildNumber = mapOf( + TestEnvironmentValue.CIRCLE_WORKFLOW_ID to "workflow-id", + TestEnvironmentValue.CIRCLE_BUILD_URL to "http://circleci.com/build", + TestEnvironmentValue.CIRCLE_BRANCH to "main", + TestEnvironmentValue.CIRCLE_SHA1 to "commit-sha" + ) + val testProvider = MockTestEnvironmentProvider(environment = circleCiEnvironmentWithMissingBuildNumber) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertNull(environment.ci) + assertEquals(testDefaultKey, environment.key) + assertNull(environment.url) + assertNull(environment.branch) + assertNull(environment.commitSha) + assertNull(environment.number) + assertNull(environment.message) + } + + @Test + fun testCircleCiEnvironmentWithMissingWorkflowIdKey() { + val circleCiEnvironmentWithMissingWorkflowId = mapOf( + TestEnvironmentValue.CIRCLE_BUILD_NUM to "123", + TestEnvironmentValue.CIRCLE_BUILD_URL to "http://circleci.com/build", + TestEnvironmentValue.CIRCLE_BRANCH to "main", + TestEnvironmentValue.CIRCLE_SHA1 to "commit-sha" + ) + val testProvider = MockTestEnvironmentProvider(environment = circleCiEnvironmentWithMissingWorkflowId) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertNull(environment.ci) + assertEquals(testDefaultKey, environment.key) + assertNull(environment.url) + assertNull(environment.branch) + assertNull(environment.commitSha) + assertNull(environment.number) + assertNull(environment.message) + } + + @Test + fun testCircleCiEnvironmentWithMissingValues() { + val circleCiEnvironmentWithMissingValues = mapOf( + TestEnvironmentValue.CIRCLE_BUILD_NUM to "123", + TestEnvironmentValue.CIRCLE_WORKFLOW_ID to "workflow-id" + ) + val testProvider = MockTestEnvironmentProvider(environment = circleCiEnvironmentWithMissingValues) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertEquals("circleci", environment.ci) + assertEquals("workflow-id-123", environment.key) + assertNull(environment.url) + assertNull(environment.branch) + assertNull(environment.commitSha) + assertEquals("123", environment.number) + assertEquals("Build #123 on branch [Unknown branch]", environment.message) + } + + @Test + fun testCircleCiEnvironmentWithInvalidValues() { + val circleCiEnvironmentWithInvalidValues = mapOf( + TestEnvironmentValue.CIRCLE_BUILD_NUM to "null", + TestEnvironmentValue.CIRCLE_WORKFLOW_ID to " ", + TestEnvironmentValue.CIRCLE_BUILD_URL to "null", + TestEnvironmentValue.CIRCLE_BRANCH to " ", + TestEnvironmentValue.CIRCLE_SHA1 to "null" + ) + val testProvider = MockTestEnvironmentProvider(environment = circleCiEnvironmentWithInvalidValues) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertNull(environment.ci) + assertEquals(testDefaultKey, environment.key) + assertNull(environment.url) + assertNull(environment.branch) + assertNull(environment.commitSha) + assertNull(environment.number) + assertNull(environment.message) + } + + @Test + fun testGitHubActionsEnvironmentWithAllValues() { + val gitHubActionsEnvironmentWithAllValues = mapOf( + TestEnvironmentValue.GITHUB_ACTION to "action", + TestEnvironmentValue.GITHUB_RUN_NUMBER to "1", + TestEnvironmentValue.GITHUB_RUN_ATTEMPT to "1", + TestEnvironmentValue.GITHUB_REPOSITORY to "repository", + TestEnvironmentValue.GITHUB_RUN_ID to "run-id", + TestEnvironmentValue.GITHUB_WORKFLOW to "workflow", + TestEnvironmentValue.GITHUB_ACTOR to "actor", + TestEnvironmentValue.GITHUB_REF_NAME to "branch", + TestEnvironmentValue.GITHUB_SHA to "commit-sha" + ) + val testProvider = MockTestEnvironmentProvider(environment = gitHubActionsEnvironmentWithAllValues) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertEquals("github_actions", environment.ci) + assertEquals("action-1-1", environment.key) + assertEquals("https://github.com/repository/actions/runs/run-id", environment.url) + assertEquals("branch", environment.branch) + assertEquals("commit-sha", environment.commitSha) + assertEquals("1", environment.number) + assertEquals("Run #1 attempt #1 of workflow, started by actor", environment.message) + } + + @Test + fun testGitHubActionsEnvironmentWithMissingActionKey() { + val gitHubActionsEnvironmentWithMissingAction = mapOf( + TestEnvironmentValue.GITHUB_RUN_NUMBER to "1", + TestEnvironmentValue.GITHUB_RUN_ATTEMPT to "1", + TestEnvironmentValue.GITHUB_REPOSITORY to "repository", + TestEnvironmentValue.GITHUB_RUN_ID to "run-id", + TestEnvironmentValue.GITHUB_WORKFLOW to "workflow", + TestEnvironmentValue.GITHUB_ACTOR to "actor", + TestEnvironmentValue.GITHUB_REF_NAME to "branch", + TestEnvironmentValue.GITHUB_SHA to "commit-sha" + ) + val testProvider = MockTestEnvironmentProvider(environment = gitHubActionsEnvironmentWithMissingAction) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertNull(environment.ci) + assertEquals(testDefaultKey, environment.key) + assertNull(environment.url) + assertNull(environment.branch) + assertNull(environment.commitSha) + assertNull(environment.number) + assertNull(environment.message) + } + + @Test + fun testGitHubActionsEnvironmentWithMissingRunNumberKey() { + val gitHubActionsEnvironmentWithMissingRunNumber = mapOf( + TestEnvironmentValue.GITHUB_ACTION to "action", + TestEnvironmentValue.GITHUB_RUN_ATTEMPT to "1", + TestEnvironmentValue.GITHUB_REPOSITORY to "repository", + TestEnvironmentValue.GITHUB_RUN_ID to "run-id", + TestEnvironmentValue.GITHUB_WORKFLOW to "workflow", + TestEnvironmentValue.GITHUB_ACTOR to "actor", + TestEnvironmentValue.GITHUB_REF_NAME to "branch", + TestEnvironmentValue.GITHUB_SHA to "commit-sha" + ) + val testProvider = MockTestEnvironmentProvider(environment = gitHubActionsEnvironmentWithMissingRunNumber) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertNull(environment.ci) + assertEquals(testDefaultKey, environment.key) + assertNull(environment.url) + assertNull(environment.branch) + assertNull(environment.commitSha) + assertNull(environment.number) + assertNull(environment.message) + } + + @Test + fun testGitHubActionsEnvironmentWithMissingRunAttemptKey() { + val gitHubActionsEnvironmentWithMissingRunAttempt = mapOf( + TestEnvironmentValue.GITHUB_ACTION to "action", + TestEnvironmentValue.GITHUB_RUN_NUMBER to "1", + TestEnvironmentValue.GITHUB_REPOSITORY to "repository", + TestEnvironmentValue.GITHUB_RUN_ID to "run-id", + TestEnvironmentValue.GITHUB_WORKFLOW to "workflow", + TestEnvironmentValue.GITHUB_ACTOR to "actor", + TestEnvironmentValue.GITHUB_REF_NAME to "branch", + TestEnvironmentValue.GITHUB_SHA to "commit-sha" + ) + val testProvider = MockTestEnvironmentProvider(environment = gitHubActionsEnvironmentWithMissingRunAttempt) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertNull(environment.ci) + assertEquals(testDefaultKey, environment.key) + assertNull(environment.url) + assertNull(environment.branch) + assertNull(environment.commitSha) + assertNull(environment.number) + assertNull(environment.message) + } + + @Test + fun testGitHubActionsEnvironmentWithMissingValues() { + val gitHubActionsEnvironmentWithMissingValues = mapOf( + TestEnvironmentValue.GITHUB_ACTION to "action", + TestEnvironmentValue.GITHUB_RUN_NUMBER to "1", + TestEnvironmentValue.GITHUB_RUN_ATTEMPT to "1" + ) + val testProvider = MockTestEnvironmentProvider(environment = gitHubActionsEnvironmentWithMissingValues) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertEquals("github_actions", environment.ci) + assertEquals("action-1-1", environment.key) + assertNull(environment.url) + assertNull(environment.branch) + assertNull(environment.commitSha) + assertEquals("1", environment.number) + assertEquals("Run #1 attempt #1", environment.message) + } + + @Test + fun testGitHubActionsEnvironmentWithInvalidValues() { + val gitHubActionsEnvironmentWithInvalidValues = mapOf( + TestEnvironmentValue.GITHUB_ACTION to "null", + TestEnvironmentValue.GITHUB_RUN_NUMBER to " ", + TestEnvironmentValue.GITHUB_RUN_ATTEMPT to "null", + TestEnvironmentValue.GITHUB_REPOSITORY to "null", + TestEnvironmentValue.GITHUB_RUN_ID to "null", + TestEnvironmentValue.GITHUB_WORKFLOW to "null", + TestEnvironmentValue.GITHUB_ACTOR to "null", + TestEnvironmentValue.GITHUB_REF_NAME to " ", + TestEnvironmentValue.GITHUB_SHA to "null" + ) + val testProvider = MockTestEnvironmentProvider(environment = gitHubActionsEnvironmentWithInvalidValues) + + val environment = testProvider.getRunEnvironment(testDefaultKey) + + assertNull(environment.ci) + assertEquals(testDefaultKey, environment.key) + assertNull(environment.url) + assertNull(environment.branch) + assertNull(environment.commitSha) + assertNull(environment.number) + assertNull(environment.message) + } +} + +private const val testDefaultKey = "test-default-run-key" diff --git a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt index a0d8de4..e6ce124 100644 --- a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt +++ b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt @@ -1,7 +1,7 @@ package com.buildkite.test.collector.android +import com.buildkite.test.collector.android.environment.UnitTestEnvironmentProvider import com.buildkite.test.collector.android.tracer.BuildkiteTestObserver -import com.buildkite.test.collector.android.util.UnitTestCollectorUtils import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.tasks.testing.Test @@ -16,9 +16,10 @@ class UnitTestCollectorPlugin : Plugin { override fun apply(project: Project) { project.tasks.withType(Test::class.java).configureEach { test -> val testListener = UnitTestListener( - testUploader = UnitTestCollectorUtils.configureTestUploader(), + testUploader = BuildKiteTestDataUploader(testEnvironmentProvider = UnitTestEnvironmentProvider()), testObserver = BuildkiteTestObserver() ) + test.addTestListener(testListener) } } diff --git a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/environment/UnitTestEnvironmentProvider.kt b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/environment/UnitTestEnvironmentProvider.kt new file mode 100644 index 0000000..9828da2 --- /dev/null +++ b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/environment/UnitTestEnvironmentProvider.kt @@ -0,0 +1,10 @@ +package com.buildkite.test.collector.android.environment + +import com.buildkite.test.collector.android.tracer.environment.BaseTestEnvironmentProvider + +/** + * Provides environment values for unit tests by fetching from system properties. + */ +internal class UnitTestEnvironmentProvider : BaseTestEnvironmentProvider() { + override fun getEnvironmentValue(key: String): String? = System.getenv(key) +} diff --git a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/UnitTestCollectorUtils.kt b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/UnitTestCollectorUtils.kt deleted file mode 100644 index f3391d7..0000000 --- a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/util/UnitTestCollectorUtils.kt +++ /dev/null @@ -1,25 +0,0 @@ -package com.buildkite.test.collector.android.util - -import com.buildkite.test.collector.android.BuildKiteTestDataUploader -import com.buildkite.test.collector.android.TestDataUploader -import com.buildkite.test.collector.android.tracer.environment.BuildkiteEnvironmentValues - -internal object UnitTestCollectorUtils { - /** - * Configures a [BuildKiteTestDataUploader] instance for unit tests. - */ - fun configureTestUploader(): TestDataUploader { - return BuildKiteTestDataUploader( - testSuiteApiToken = getStringEnvironmentValue(key = BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_TOKEN), - isDebugEnabled = getBooleanEnvironmentValue( - key = BuildkiteEnvironmentValues.BUILDKITE_ANALYTICS_DEBUG_ENABLED - ) - ) - } - - private fun getStringEnvironmentValue(key: String): String? = - System.getenv(key)?.takeIf { value -> value.isNotBlank() && value != "null" } - - private fun getBooleanEnvironmentValue(key: String): Boolean = - System.getenv(key)?.toBoolean() ?: false -} diff --git a/example/build.gradle.kts b/example/build.gradle.kts index 11a715a..61c872f 100644 --- a/example/build.gradle.kts +++ b/example/build.gradle.kts @@ -1,3 +1,6 @@ +import com.buildkite.test.collector.android.tracer.environment.TestEnvironmentValue.BUILDKITE_ANALYTICS_DEBUG_ENABLED +import com.buildkite.test.collector.android.tracer.environment.TestEnvironmentValue.BUILDKITE_ANALYTICS_TOKEN + plugins { id("com.android.application") id("org.jetbrains.kotlin.android") @@ -16,11 +19,14 @@ android { testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" // Specifies `InstrumentedTestCollector` as the instrumented test listener for collecting test analytics. - testInstrumentationRunnerArguments["listener"] = "com.buildkite.test.collector.android.InstrumentedTestCollector" + testInstrumentationRunnerArguments["listener"] = + "com.buildkite.test.collector.android.InstrumentedTestCollector" // Passes environment variables as instrumentation arguments - testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_TOKEN"] = System.getenv("BUILDKITE_ANALYTICS_TOKEN") ?: "" - testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_DEBUG_ENABLED"] = System.getenv("BUILDKITE_ANALYTICS_DEBUG_ENABLED") ?: "false" + testInstrumentationRunnerArguments[BUILDKITE_ANALYTICS_TOKEN] = + getEnv(BUILDKITE_ANALYTICS_TOKEN) + testInstrumentationRunnerArguments[BUILDKITE_ANALYTICS_DEBUG_ENABLED] = + getEnv(BUILDKITE_ANALYTICS_DEBUG_ENABLED) vectorDrawables { useSupportLibrary = true @@ -74,3 +80,5 @@ dependencies { androidTestImplementation(libs.androidx.test.ext) androidTestImplementation(libs.androidx.compose.ui.test) } + +fun getEnv(key: String): String = System.getenv(key) ?: "" From 9c807a32cdeaf5473c984f6feaec8c2b5a8b052a Mon Sep 17 00:00:00 2001 From: Bhumil Soni Date: Fri, 7 Jun 2024 13:32:57 +1000 Subject: [PATCH 07/11] Ensure API token is present before instantiating test data collectors Moved validation to check for the presence of the 'BUILDKITE_ANALYTICS_TOKEN' environment variable before creating instances of `UnitTestListener` and `InstrumentedTestListener`. This change ensures that no test data is collected or uploaded without a valid API token unnecessarily. --- .../android/InstrumentedTestCollector.kt | 38 +++++++--- .../android/BuildKiteTestDataUploader.kt | 69 ++++++++----------- .../api/DefaultTestUploaderApiFactory.kt | 13 ++++ .../network/api/TestUploaderApiFactory.kt | 14 ++++ .../android/util/{ => logger}/Logger.kt | 5 +- .../android/util/logger/LoggerFactory.kt | 17 +++++ .../android/UnitTestCollectorPlugin.kt | 22 +++++- 7 files changed, 125 insertions(+), 53 deletions(-) create mode 100644 collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/api/DefaultTestUploaderApiFactory.kt create mode 100644 collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/api/TestUploaderApiFactory.kt rename collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/{ => logger}/Logger.kt (89%) create mode 100644 collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/logger/LoggerFactory.kt diff --git a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt index 9753f37..dfa3437 100644 --- a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt +++ b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt @@ -3,6 +3,7 @@ package com.buildkite.test.collector.android import androidx.test.platform.app.InstrumentationRegistry import com.buildkite.test.collector.android.environment.InstrumentedTestEnvironmentProvider import com.buildkite.test.collector.android.tracer.BuildkiteTestObserver +import com.buildkite.test.collector.android.util.logger.LoggerFactory import org.junit.runner.Description import org.junit.runner.notification.Failure import org.junit.runner.notification.RunListener @@ -13,39 +14,56 @@ import org.junit.runner.notification.RunListener * enabling precise monitoring and reporting of test outcomes. It delegates actual event handling to [InstrumentedTestListener]. */ class InstrumentedTestCollector : RunListener() { - private val listener: InstrumentedTestListener by lazy { configureTestListener() } + private val listener: InstrumentedTestListener? by lazy { configureTestListener() } - private fun configureTestListener(): InstrumentedTestListener { + private fun configureTestListener(): InstrumentedTestListener? { val arguments = InstrumentationRegistry.getArguments() - val environmentProvider = InstrumentedTestEnvironmentProvider(arguments = arguments) + val testEnvironmentProvider = InstrumentedTestEnvironmentProvider(arguments = arguments) + val testSuiteApiToken = testEnvironmentProvider.testSuiteApiToken + val logger = LoggerFactory.create( + isDebugEnabled = testEnvironmentProvider.isDebugEnabled, + tag = "Buildkite-InstrumentedTestCollector" + ) + + if (testSuiteApiToken == null) { + logger.error { + "Missing or invalid 'BUILDKITE_ANALYTICS_TOKEN'. " + + "Instrumented test analytics data will not be collected and uploaded. Please set the environment variable to enable analytics." + } + return null + } return InstrumentedTestListener( - testUploader = BuildKiteTestDataUploader(testEnvironmentProvider = environmentProvider), + testUploader = BuildKiteTestDataUploader( + testSuiteApiToken = testSuiteApiToken, + runEnvironment = testEnvironmentProvider.getRunEnvironment(), + logger = logger + ), testObserver = BuildkiteTestObserver() ) } override fun testSuiteStarted(description: Description) { - listener.testSuiteStarted(description) + listener?.testSuiteStarted(description) } override fun testSuiteFinished(description: Description) { - listener.testSuiteFinished(description) + listener?.testSuiteFinished(description) } override fun testStarted(description: Description) { - listener.testStarted(description) + listener?.testStarted(description) } override fun testFinished(description: Description) { - listener.testFinished(description) + listener?.testFinished(description) } override fun testFailure(failure: Failure) { - listener.testFailure(failure) + listener?.testFailure(failure) } override fun testIgnored(description: Description) { - listener.testIgnored(description) + listener?.testIgnored(description) } } diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/BuildKiteTestDataUploader.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/BuildKiteTestDataUploader.kt index ca461b1..e58336d 100644 --- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/BuildKiteTestDataUploader.kt +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/BuildKiteTestDataUploader.kt @@ -1,32 +1,31 @@ package com.buildkite.test.collector.android +import com.buildkite.test.collector.android.model.RunEnvironment import com.buildkite.test.collector.android.model.TestData import com.buildkite.test.collector.android.model.TestDetails import com.buildkite.test.collector.android.model.TestUploadResponse -import com.buildkite.test.collector.android.network.TestAnalyticsRetrofit -import com.buildkite.test.collector.android.network.api.TestUploaderApi -import com.buildkite.test.collector.android.tracer.environment.TestEnvironmentProvider +import com.buildkite.test.collector.android.network.api.DefaultTestUploaderApiFactory +import com.buildkite.test.collector.android.network.api.TestUploaderApiFactory import com.buildkite.test.collector.android.util.CollectorUtils -import com.buildkite.test.collector.android.util.Logger +import com.buildkite.test.collector.android.util.logger.Logger import retrofit2.Response /** - * Manages the upload of test data to the Buildkite Test Analytics Suite using the provided environment values. + * Manages the upload of test data to the Buildkite Test Analytics Suite. * - * This class fetches the necessary environment configuration from a [TestEnvironmentProvider]. - * - * @property testEnvironmentProvider Provides the environment configuration needed for uploading test data. + * @property testSuiteApiToken The API token for authentication. + * @property runEnvironment The test run environment configuration. + * @property logger The logger for logging messages. */ class BuildKiteTestDataUploader( - private val testEnvironmentProvider: TestEnvironmentProvider + private val testSuiteApiToken: String, + private val runEnvironment: RunEnvironment, + private val logger: Logger = Logger() ) : TestDataUploader { - private val testSuiteApiToken = testEnvironmentProvider.testSuiteApiToken - - private val logger = - Logger(minLevel = if (testEnvironmentProvider.isDebugEnabled) Logger.LogLevel.DEBUG else Logger.LogLevel.INFO) + private val testUploaderApiFactory: TestUploaderApiFactory = DefaultTestUploaderApiFactory() init { - logger.debug { "BuildKiteTestDataUploader: Test RunEnvironment is: ${testEnvironmentProvider.getRunEnvironment()}" } + logger.debug { "TestDataUploader initialized with test analytics API token." } } /** @@ -36,40 +35,30 @@ class BuildKiteTestDataUploader( * @param testCollection A list of [TestDetails] representing all the tests within the suite. */ override fun uploadTestData(testCollection: List) { - val testData = TestData( + val testData = prepareTestData(testCollection) + uploadTestData(testData) + } + + private fun prepareTestData(testCollection: List): TestData { + return TestData( format = "json", - runEnvironment = testEnvironmentProvider.getRunEnvironment(), + runEnvironment = runEnvironment, data = testCollection.take(CollectorUtils.Uploader.TEST_DATA_UPLOAD_LIMIT) ) - - uploadTestData(testData = testData) } - /** - * Performs the actual upload of the provided test data to the Buildkite Test Analytics Suite. - * - * @param testData The test data to be uploaded. - */ private fun uploadTestData(testData: TestData) { - if (testSuiteApiToken == null) { - logger.info { - "Incorrect or missing Test Suite API token. Please ensure the 'BUILDKITE_ANALYTICS_TOKEN' environment variable is set correctly to upload test data." - } - return - } - - try { - logger.debug { "Uploading test analytics data." } + logger.debug { "Uploading test analytics data." } - val testUploaderService = - TestAnalyticsRetrofit.getRetrofitInstance(testSuiteApiToken = testSuiteApiToken) - .create(TestUploaderApi::class.java) - val uploadTestDataApiCall = testUploaderService.uploadTestData(testData = testData) - val testUploadResponse = uploadTestDataApiCall.execute() + runCatching { + val testUploaderApi = testUploaderApiFactory.create(testSuiteApiToken) + val uploadTestDataApiCall = testUploaderApi.uploadTestData(testData = testData) + uploadTestDataApiCall.execute() + }.onSuccess { testUploadResponse -> logApiResponse(testUploadResponse = testUploadResponse) - } catch (e: Exception) { - logger.error { "Error uploading test analytics data: ${e.message}." } + }.onFailure { throwable -> + logger.error { "Error uploading test analytics data: ${throwable.message}." } } } @@ -80,7 +69,7 @@ class BuildKiteTestDataUploader( logger.error { "Error uploading test analytics data. HTTP error code: ${testUploadResponse.code()}. Ensure the test suite API token is correct and properly configured." } - logger.error { "Failed response details: ${testUploadResponse.errorBody()?.string()}" } + logger.error { "Error response details: ${testUploadResponse.errorBody()?.string()}" } } } } diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/api/DefaultTestUploaderApiFactory.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/api/DefaultTestUploaderApiFactory.kt new file mode 100644 index 0000000..2ca2307 --- /dev/null +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/api/DefaultTestUploaderApiFactory.kt @@ -0,0 +1,13 @@ +package com.buildkite.test.collector.android.network.api + +import com.buildkite.test.collector.android.network.TestAnalyticsRetrofit + +/** + * Default implementation of [TestUploaderApiFactory] that creates instance using Retrofit. + */ +internal class DefaultTestUploaderApiFactory : TestUploaderApiFactory { + override fun create(testSuiteApiToken: String): TestUploaderApi { + return TestAnalyticsRetrofit.getRetrofitInstance(testSuiteApiToken = testSuiteApiToken) + .create(TestUploaderApi::class.java) + } +} diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/api/TestUploaderApiFactory.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/api/TestUploaderApiFactory.kt new file mode 100644 index 0000000..e7f3e29 --- /dev/null +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/api/TestUploaderApiFactory.kt @@ -0,0 +1,14 @@ +package com.buildkite.test.collector.android.network.api + +/** + * Factory interface for creating instances of [TestUploaderApi]. + */ +internal interface TestUploaderApiFactory { + /** + * Creates an instance of [TestUploaderApi] using the provided test suite API token. + * + * @param testSuiteApiToken The API token to authenticate the requests. + * @return An instance of [TestUploaderApi]. + */ + fun create(testSuiteApiToken: String): TestUploaderApi +} diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/Logger.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/logger/Logger.kt similarity index 89% rename from collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/Logger.kt rename to collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/logger/Logger.kt index b39bbb6..95e9cff 100644 --- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/Logger.kt +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/logger/Logger.kt @@ -1,4 +1,4 @@ -package com.buildkite.test.collector.android.util +package com.buildkite.test.collector.android.util.logger /** * Provides logging functionality with configurable log level sensitivity. @@ -6,6 +6,7 @@ package com.buildkite.test.collector.android.util * @property minLevel The minimum log level that will be logged. */ class Logger( + private val tag: String = "BuildkiteLogger", private val minLevel: LogLevel = LogLevel.INFO ) { /** @@ -33,7 +34,7 @@ class Logger( private fun log(level: LogLevel, message: () -> String) { if (level >= minLevel) { val output = if (level == LogLevel.ERROR) System.err else System.out - output.println("\nBuildkiteTestCollector-${level.name}: ${message()}") + output.println("\n$tag-${level.name}: ${message()}") } } diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/logger/LoggerFactory.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/logger/LoggerFactory.kt new file mode 100644 index 0000000..de19cf2 --- /dev/null +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/logger/LoggerFactory.kt @@ -0,0 +1,17 @@ +package com.buildkite.test.collector.android.util.logger + +object LoggerFactory { + /** + * Creates a Logger instance with the appropriate log level and tag. + * + * @param isDebugEnabled Indicates if debug logging should be enabled. + * @param tag The tag to be used in log messages. + * @return A new [Logger] instance configured with the appropriate log level and tag. + */ + fun create(isDebugEnabled: Boolean, tag: String): Logger { + return Logger( + minLevel = if (isDebugEnabled) Logger.LogLevel.DEBUG else Logger.LogLevel.INFO, + tag = tag + ) + } +} diff --git a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt index e6ce124..5ccf238 100644 --- a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt +++ b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt @@ -2,6 +2,7 @@ package com.buildkite.test.collector.android import com.buildkite.test.collector.android.environment.UnitTestEnvironmentProvider import com.buildkite.test.collector.android.tracer.BuildkiteTestObserver +import com.buildkite.test.collector.android.util.logger.LoggerFactory import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.tasks.testing.Test @@ -15,8 +16,27 @@ import org.gradle.api.tasks.testing.Test class UnitTestCollectorPlugin : Plugin { override fun apply(project: Project) { project.tasks.withType(Test::class.java).configureEach { test -> + val testEnvironmentProvider = UnitTestEnvironmentProvider() + val testSuiteApiToken = testEnvironmentProvider.testSuiteApiToken + val logger = LoggerFactory.create( + isDebugEnabled = testEnvironmentProvider.isDebugEnabled, + tag = "Buildkite-UnitTestCollector" + ) + + if (testSuiteApiToken == null) { + logger.error { + "Missing or invalid 'BUILDKITE_ANALYTICS_TOKEN'. " + + "Unit test analytics data will not be collected and uploaded. Please set the environment variable to enable analytics." + } + return@configureEach + } + val testListener = UnitTestListener( - testUploader = BuildKiteTestDataUploader(testEnvironmentProvider = UnitTestEnvironmentProvider()), + testUploader = BuildKiteTestDataUploader( + testSuiteApiToken = testSuiteApiToken, + runEnvironment = testEnvironmentProvider.getRunEnvironment(), + logger = logger + ), testObserver = BuildkiteTestObserver() ) From a07bbc4452fb6407f7a6310016f6ca250892f9c0 Mon Sep 17 00:00:00 2001 From: Bhumil Soni Date: Fri, 7 Jun 2024 13:39:13 +1000 Subject: [PATCH 08/11] Access GitHub Actions environment variables in CI scripts --- .github/workflows/build-and-test-sdk.yml | 9 ++++++ .github/workflows/test-example-project.yml | 23 ++++++++------ example/build.gradle.kts | 35 +++++++++++++++++----- 3 files changed, 51 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-and-test-sdk.yml b/.github/workflows/build-and-test-sdk.yml index 7dd8055..bab4fa3 100644 --- a/.github/workflows/build-and-test-sdk.yml +++ b/.github/workflows/build-and-test-sdk.yml @@ -14,6 +14,15 @@ jobs: env: BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }} BUILDKITE_ANALYTICS_DEBUG_ENABLED: ${{ secrets.BUILDKITE_ANALYTICS_DEBUG_ENABLED }} + GITHUB_ACTION: ${{ github.action }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SHA: ${{ github.sha }} + GITHUB_WORKFLOW: ${{ github.workflow }} + GITHUB_ACTOR: ${{ github.actor }} steps: - name: Git Checkout diff --git a/.github/workflows/test-example-project.yml b/.github/workflows/test-example-project.yml index 5cd9d13..c3f919d 100644 --- a/.github/workflows/test-example-project.yml +++ b/.github/workflows/test-example-project.yml @@ -6,15 +6,24 @@ on: pull_request: branches: [main] +env: + BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }} + BUILDKITE_ANALYTICS_DEBUG_ENABLED: ${{ secrets.BUILDKITE_ANALYTICS_DEBUG_ENABLED }} + GITHUB_ACTION: ${{ github.action }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SHA: ${{ github.sha }} + GITHUB_WORKFLOW: ${{ github.workflow }} + GITHUB_ACTOR: ${{ github.actor }} + jobs: example-project-unit-tests: name: Run Example Project Unit Tests runs-on: ubuntu-latest - - env: - BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }} - BUILDKITE_ANALYTICS_DEBUG_ENABLED: ${{ secrets.BUILDKITE_ANALYTICS_DEBUG_ENABLED }} - + steps: - name: Git Checkout uses: actions/checkout@v3 @@ -35,10 +44,6 @@ jobs: name: Run Example Project Instrumented Tests runs-on: macos-latest - env: - BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }} - BUILDKITE_ANALYTICS_DEBUG_ENABLED: ${{ secrets.BUILDKITE_ANALYTICS_DEBUG_ENABLED }} - steps: - name: Git Checkout uses: actions/checkout@v3 diff --git a/example/build.gradle.kts b/example/build.gradle.kts index 61c872f..25d1eaa 100644 --- a/example/build.gradle.kts +++ b/example/build.gradle.kts @@ -1,5 +1,4 @@ -import com.buildkite.test.collector.android.tracer.environment.TestEnvironmentValue.BUILDKITE_ANALYTICS_DEBUG_ENABLED -import com.buildkite.test.collector.android.tracer.environment.TestEnvironmentValue.BUILDKITE_ANALYTICS_TOKEN +import com.android.build.api.dsl.ApplicationDefaultConfig plugins { id("com.android.application") @@ -22,11 +21,7 @@ android { testInstrumentationRunnerArguments["listener"] = "com.buildkite.test.collector.android.InstrumentedTestCollector" - // Passes environment variables as instrumentation arguments - testInstrumentationRunnerArguments[BUILDKITE_ANALYTICS_TOKEN] = - getEnv(BUILDKITE_ANALYTICS_TOKEN) - testInstrumentationRunnerArguments[BUILDKITE_ANALYTICS_DEBUG_ENABLED] = - getEnv(BUILDKITE_ANALYTICS_DEBUG_ENABLED) + setupTestCollectorEnvironment() vectorDrawables { useSupportLibrary = true @@ -81,4 +76,30 @@ dependencies { androidTestImplementation(libs.androidx.compose.ui.test) } +fun ApplicationDefaultConfig.setupTestCollectorEnvironment() { + // Passes environment variables as instrumentation arguments + testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_TOKEN"] = + getEnv("BUILDKITE_ANALYTICS_TOKEN") + testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_DEBUG_ENABLED"] = + getEnv("BUILDKITE_ANALYTICS_DEBUG_ENABLED") + testInstrumentationRunnerArguments["GITHUB_ACTION"] = + getEnv("GITHUB_ACTION") + testInstrumentationRunnerArguments["GITHUB_RUN_ID"] = + getEnv("GITHUB_RUN_ID") + testInstrumentationRunnerArguments["GITHUB_RUN_NUMBER"] = + getEnv("GITHUB_RUN_NUMBER") + testInstrumentationRunnerArguments["GITHUB_RUN_ATTEMPT"] = + getEnv("GITHUB_RUN_ATTEMPT") + testInstrumentationRunnerArguments["GITHUB_REF_NAME"] = + getEnv("GITHUB_REF_NAME") + testInstrumentationRunnerArguments["GITHUB_REPOSITORY"] = + getEnv("GITHUB_REPOSITORY") + testInstrumentationRunnerArguments["GITHUB_SHA"] = + getEnv("GITHUB_SHA") + testInstrumentationRunnerArguments["GITHUB_WORKFLOW"] = + getEnv("GITHUB_WORKFLOW") + testInstrumentationRunnerArguments["GITHUB_ACTOR"] = + getEnv("GITHUB_ACTOR") +} + fun getEnv(key: String): String = System.getenv(key) ?: "" From ec0838d6c456e5c8c52f76a0ae5f63963b552ed5 Mon Sep 17 00:00:00 2001 From: Bhumil Soni Date: Fri, 7 Jun 2024 14:30:29 +1000 Subject: [PATCH 09/11] Update test-example-project workflow --- .github/workflows/test-example-project.yml | 39 +++++++++++++++------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-example-project.yml b/.github/workflows/test-example-project.yml index c3f919d..5c33c4d 100644 --- a/.github/workflows/test-example-project.yml +++ b/.github/workflows/test-example-project.yml @@ -23,13 +23,13 @@ jobs: example-project-unit-tests: name: Run Example Project Unit Tests runs-on: ubuntu-latest - + steps: - name: Git Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up JDK 17 - uses: actions/setup-java@v3.3.0 + uses: actions/setup-java@v4 with: distribution: zulu java-version: 17 @@ -42,24 +42,39 @@ jobs: example-project-instrumented-tests: name: Run Example Project Instrumented Tests - runs-on: macos-latest + runs-on: ubuntu-latest + strategy: + matrix: + api-level: [28] + timeout-minutes: 90 steps: + - name: "Enable KVM group perms" + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + ls /dev/kvm + - name: Git Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up JDK 17 - uses: actions/setup-java@v3.3.0 + uses: actions/setup-java@v4 with: distribution: zulu java-version: 17 - - name: Debug Build - run: ./gradlew :example:buildDebug - - - name: Example Project Instrumented Tests - uses: reactivecircus/android-emulator-runner@v2 + - name: "Run instrumented tests" + uses: reactivecircus/android-emulator-runner@v2.27.0 with: - api-level: 27 + api-level: ${{ matrix.api-level }} + force-avd-creation: false + emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -timezone Australia/Melbourne -camera-back none + disable-animations: true + disable-spellchecker: true + profile: Nexus 6 arch: x86_64 + ram-size: 4096M + heap-size: 512M script: "support/scripts/example-instrumented-tests" From f337bc04d6d6b9523061fcdb270f588fa4fab89d Mon Sep 17 00:00:00 2001 From: Bhumil Soni Date: Fri, 7 Jun 2024 15:45:00 +1000 Subject: [PATCH 10/11] Document setting up environment variables --- CI_CONFIGURATION.md | 92 +++++++++++++++++++++++++++++++++++++++++++++ README.md | 7 ++++ 2 files changed, 99 insertions(+) create mode 100644 CI_CONFIGURATION.md diff --git a/CI_CONFIGURATION.md b/CI_CONFIGURATION.md new file mode 100644 index 0000000..cb74b89 --- /dev/null +++ b/CI_CONFIGURATION.md @@ -0,0 +1,92 @@ +# CI Environment Variables Setup + +This document provides instructions for setting up environment variables for different CI platforms to enrich test reports. + +## Accessing Environment Variables in CI Configuration + +To enrich your test reports with valuable CI information such as commit messages, branch names, and build numbers, you need to pass environment variables in your CI pipeline. +For example, here is how you can set up the environment variables for GitHub Actions: + +### GitHub Actions + +In your GitHub Actions workflow configuration, add the following environment variables: + +```yaml +env: + BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }} + GITHUB_ACTION: ${{ github.action }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SHA: ${{ github.sha }} + GITHUB_WORKFLOW: ${{ github.workflow }} + GITHUB_ACTOR: ${{ github.actor }} +``` + +## Additional Setup for Instrumented Tests + +While the above setup is sufficient for unit tests collector, instrumented tests collector require additional configuration. Below are examples for different CI platforms supported by the test collectors. + +In your build.gradle.kts file, add the following to pass the required environment variables for instrumented tests: + +### Buildkite + +``` +android { + ... + defaultConfig { + ... + testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_TOKEN"] = System.getenv("BUILDKITE_ANALYTICS_TOKEN") + testInstrumentationRunnerArguments["BUILDKITE_BUILD_ID"] = System.getenv("BUILDKITE_BUILD_ID") ?: "" + testInstrumentationRunnerArguments["BUILDKITE_BUILD_URL"] = System.getenv("BUILDKITE_BUILD_URL") ?: "" + testInstrumentationRunnerArguments["BUILDKITE_BRANCH"] = System.getenv("BUILDKITE_BRANCH") ?: "" + testInstrumentationRunnerArguments["BUILDKITE_COMMIT"] = System.getenv("BUILDKITE_COMMIT") ?: "" + testInstrumentationRunnerArguments["BUILDKITE_BUILD_NUMBER"] = System.getenv("BUILDKITE_BUILD_NUMBER") ?: "" + testInstrumentationRunnerArguments["BUILDKITE_JOB_ID"] = System.getenv("BUILDKITE_JOB_ID") ?: "" + testInstrumentationRunnerArguments["BUILDKITE_MESSAGE"] = System.getenv("BUILDKITE_MESSAGE") ?: "" + } +} +``` + +### GitHub Actions + +``` +android { + ... + defaultConfig { + ... + testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_TOKEN"] = System.getenv("BUILDKITE_ANALYTICS_TOKEN") + testInstrumentationRunnerArguments["GITHUB_ACTION"] = System.getenv("GITHUB_ACTION") ?: "" + testInstrumentationRunnerArguments["GITHUB_RUN_ID"] = System.getenv("GITHUB_RUN_ID") ?: "" + testInstrumentationRunnerArguments["GITHUB_RUN_NUMBER"] = System.getenv("GITHUB_RUN_NUMBER") ?: "" + testInstrumentationRunnerArguments["GITHUB_RUN_ATTEMPT"] = System.getenv("GITHUB_RUN_ATTEMPT") ?: "" + testInstrumentationRunnerArguments["GITHUB_REPOSITORY"] = System.getenv("GITHUB_REPOSITORY") ?: "" + testInstrumentationRunnerArguments["GITHUB_REF_NAME"] = System.getenv("GITHUB_REF_NAME") ?: "" + testInstrumentationRunnerArguments["GITHUB_SHA"] = System.getenv("GITHUB_SHA") ?: "" + testInstrumentationRunnerArguments["GITHUB_WORKFLOW"] = System.getenv("GITHUB_WORKFLOW") ?: "" + testInstrumentationRunnerArguments["GITHUB_ACTOR"] = System.getenv("GITHUB_ACTOR") ?: "" + } +} +``` + +### CircleCI + +``` +android { + ... + defaultConfig { + ... + testInstrumentationRunnerArguments["BUILDKITE_ANALYTICS_TOKEN"] = System.getenv("BUILDKITE_ANALYTICS_TOKEN") + testInstrumentationRunnerArguments["CIRCLE_BUILD_NUM"] = System.getenv("CIRCLE_BUILD_NUM") ?: "" + testInstrumentationRunnerArguments["CIRCLE_WORKFLOW_ID"] = System.getenv("CIRCLE_WORKFLOW_ID") ?: "" + testInstrumentationRunnerArguments["CIRCLE_BUILD_URL"] = System.getenv("CIRCLE_BUILD_URL") ?: "" + testInstrumentationRunnerArguments["CIRCLE_BRANCH"] = System.getenv("CIRCLE_BRANCH") ?: "" + testInstrumentationRunnerArguments["CIRCLE_SHA1"] = System.getenv("CIRCLE_SHA1") ?: "" + } +} +``` + +By following these steps, you ensure that your CI environment variables are passed correctly to your Android instrumentation and unit tests. +For a complete setup example, check out the [example](example) project in the repository. \ No newline at end of file diff --git a/README.md b/README.md index aeafd4b..f1d34be 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,13 @@ android { Note: This test collector uploads test data via the device under test. Make sure your Android device/emulator has network access. +### Step 5 (Optional) - Passing CI Environment Variables + +The only required environment variable is the analytics token, but if you're using one of the supported CI platforms, +you can pass extra information to the test-collector to enrich the reports. These include commit messages, branch names, build numbers, etc. + +For detailed instructions on setting up environment variables for different CI platforms, see the [CI Environment Variables Setup](CI_CONFIGURATION.md) document. + ## 🔍 Debugging To enable debugging output, create and set `BUILDKITE_ANALYTICS_DEBUG_ENABLED` environment variable to `true` on your test environment (CI server or local machine). From b581775b2cbe93f73c1ad93f4f23a5632b7abcc3 Mon Sep 17 00:00:00 2001 From: Bhumil Soni Date: Fri, 7 Jun 2024 16:02:57 +1000 Subject: [PATCH 11/11] Add logs related to CI detection --- .../test/collector/android/BuildKiteTestDataUploader.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/BuildKiteTestDataUploader.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/BuildKiteTestDataUploader.kt index e58336d..e4ac95f 100644 --- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/BuildKiteTestDataUploader.kt +++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/BuildKiteTestDataUploader.kt @@ -26,6 +26,11 @@ class BuildKiteTestDataUploader( init { logger.debug { "TestDataUploader initialized with test analytics API token." } + if (runEnvironment.ci != null) { + logger.debug { "CI system detected: ${runEnvironment.ci}" } + } else { + logger.debug { "No CI system detected." } + } } /**