diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml
index 2f86a17..6c0b919 100644
--- a/.idea/codeStyles/Project.xml
+++ b/.idea/codeStyles/Project.xml
@@ -6,8 +6,6 @@
-
-
diff --git a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt
index 2266dfb..1494edc 100644
--- a/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt
+++ b/collector/instrumented-test-collector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt
@@ -1,74 +1,80 @@
package com.buildkite.test.collector.android
-import com.buildkite.test.collector.android.models.FailureExpanded
-import com.buildkite.test.collector.android.models.Span
-import com.buildkite.test.collector.android.models.TestDetails
-import com.buildkite.test.collector.android.models.TraceResult
+import com.buildkite.test.collector.android.model.TestDetails
+import com.buildkite.test.collector.android.model.TestFailureExpanded
+import com.buildkite.test.collector.android.model.TestHistory
+import com.buildkite.test.collector.android.model.TestOutcome
import com.buildkite.test.collector.android.tracer.TestObserver
import com.buildkite.test.collector.android.tracer.environment.configureInstrumentedTestUploader
import org.junit.runner.Description
import org.junit.runner.notification.Failure
import org.junit.runner.notification.RunListener
+/**
+ * Serves as an abstract foundation for creating instrumented test collectors that interface with Buildkite's Test Analytics service.
+ * This class extends JUnit's [RunListener] to capture real-time events during the execution of instrumented tests, enabling precise monitoring
+ * and reporting of test outcomes. It automatically gathers detailed test results and uploads them directly to the analytics portal at the conclusion of test suite.
+ *
+ * @param apiToken The API token for the test suite, necessary for authenticating requests with Test Analytics.
+ * @param isDebugEnabled When true, enables logging to assist with debugging.
+ */
abstract class InstrumentedTestCollector(
apiToken: String,
isDebugEnabled: Boolean = false
) : RunListener() {
private val testObserver = TestObserver()
- private val testUploader = configureInstrumentedTestUploader(apiToken, isDebugEnabled)
+ private val testUploader = configureInstrumentedTestUploader(
+ apiToken = apiToken,
+ isDebugEnabled = isDebugEnabled
+ )
+ private val testCollection: MutableList = mutableListOf()
- override fun testSuiteStarted(testDescription: Description?) { /* Nothing to do */ }
+ override fun testSuiteStarted(testDescription: Description) {
+ /* Nothing to do before the test suite has started */
+ }
- override fun testSuiteFinished(description: Description?) {
- description?.let { testSuite ->
- if ((testSuite.displayName.isNullOrEmpty() || testSuite.displayName == "null") && (testSuite.className.isNullOrEmpty() || testSuite.className == "null")) {
- testUploader.configureUploadData(testCollection = testObserver.collection)
- }
+ override fun testSuiteFinished(description: Description) {
+ if (isFinalTestSuiteCall(testDescription = description)) {
+ testUploader.configureUploadData(testCollection = testCollection)
}
}
- override fun testStarted(testDescription: Description?) {
- testObserver.recordStartTime()
+ override fun testStarted(testDescription: Description) {
+ testObserver.startTest()
}
- override fun testFinished(testDescription: Description?) {
- testObserver.recordEndTime()
+ override fun testFinished(testDescription: Description) {
+ testObserver.endTest()
- if (testObserver.result != TraceResult.Failed) {
- testObserver.result = TraceResult.Passed
+ if (testObserver.outcome != TestOutcome.Failed) {
+ testObserver.recordSuccess()
}
- testDescription?.let { test ->
- addTestDetailsToCollection(test = test)
- }
+ addTestDetailsToCollection(test = testDescription)
}
- override fun testFailure(failureDetails: Failure?) {
- failureDetails?.let { failure ->
- testObserver.result = TraceResult.Failed
- testObserver.failureReason = failure.exception.toString()
- testObserver.failureExpanded = listOf(
- FailureExpanded(
- expanded = failure.trimmedTrace.split("\n").map { it.trim() },
- backtrace = failure.trace.split("\n").map { it.trim() },
- )
+ override fun testFailure(failureDetails: Failure) {
+ val failureReason = failureDetails.exception.toString()
+ val details = listOf(
+ TestFailureExpanded(
+ expanded = failureDetails.trimmedTrace.split("\n").map { it.trim() },
+ backtrace = failureDetails.trace.split("\n").map { it.trim() },
)
- }
+ )
+ testObserver.recordFailure(reason = failureReason, details = details)
}
- override fun testIgnored(testDescription: Description?) {
- testObserver.result = TraceResult.Skipped
+ override fun testIgnored(testDescription: Description) {
+ testObserver.recordSkipped()
- testDescription?.let { test ->
- addTestDetailsToCollection(test = test)
- }
+ addTestDetailsToCollection(test = testDescription)
}
private fun addTestDetailsToCollection(test: Description) {
- val testSpan = Span(
+ val testHistory = TestHistory(
startAt = testObserver.startTime,
endAt = testObserver.endTime,
- duration = testObserver.calculateSpanDuration(),
+ duration = testObserver.getDuration()
)
val testDetails = TestDetails(
@@ -76,13 +82,22 @@ abstract class InstrumentedTestCollector(
name = test.methodName,
location = test.className,
fileName = null,
- result = testObserver.result,
+ result = testObserver.outcome,
failureReason = testObserver.failureReason,
- failureExpanded = testObserver.failureExpanded,
- history = testSpan
+ failureExpanded = testObserver.failureDetails,
+ history = testHistory
)
- testObserver.collection.add(testDetails)
- testObserver.resetTestData()
+ testCollection.add(testDetails)
+ testObserver.reset()
}
+
+ /**
+ * Determines if the provided test suite descriptor indicates the final call of the test suite,
+ * which is true when both displayName and className are null.
+ *
+ * @param testDescription The test description. [Description] can be atomic (a single test) or compound (containing children tests).
+ */
+ private fun isFinalTestSuiteCall(testDescription: Description) =
+ (testDescription.displayName.isNullOrEmpty() || testDescription.displayName == "null") && (testDescription.className.isNullOrEmpty() || testDescription.className == "null")
}
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/TestDataUploader.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/TestDataUploader.kt
index 2d0bc68..fbf12eb 100644
--- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/TestDataUploader.kt
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/TestDataUploader.kt
@@ -1,18 +1,35 @@
package com.buildkite.test.collector.android
-import com.buildkite.test.collector.android.models.RunEnvironment
-import com.buildkite.test.collector.android.models.TestData
-import com.buildkite.test.collector.android.models.TestDetails
-import com.buildkite.test.collector.android.models.TestResponse
-import com.buildkite.test.collector.android.network.RetrofitInstance
+import com.buildkite.test.collector.android.model.RunEnvironment
+import com.buildkite.test.collector.android.model.TestData
+import com.buildkite.test.collector.android.model.TestDetails
+import com.buildkite.test.collector.android.model.TestUploadResponse
+import com.buildkite.test.collector.android.network.TestAnalyticsRetrofit
import com.buildkite.test.collector.android.network.api.TestUploaderApi
import com.buildkite.test.collector.android.util.CollectorUtils.Uploader
+import com.buildkite.test.collector.android.util.Logger
import retrofit2.Response
+/**
+ * Manages the upload of test data to the Buildkite Test Analytics Suite associated with the provided API token.
+ *
+ * @property testSuiteApiToken The API token for the test suite, necessary for authenticating requests with Test Analytics.
+ * Test data will not be uploaded if this is null.
+ * @property isDebugEnabled When true, enables logging to assist with debugging.
+ */
class TestDataUploader(
private val testSuiteApiToken: String?,
private val isDebugEnabled: Boolean
) {
+ private val logger =
+ Logger(minLevel = if (isDebugEnabled) Logger.LogLevel.DEBUG else Logger.LogLevel.INFO)
+
+ /**
+ * Configures and uploads test data.
+ * The number of test data uploaded in a single request is constrained by [Uploader.TEST_DATA_UPLOAD_LIMIT].
+ *
+ * @param testCollection A list of [TestDetails] representing all the tests within the suite.
+ * */
fun configureUploadData(testCollection: List) {
val runEnvironment = RunEnvironment().getEnvironmentValues()
@@ -27,31 +44,35 @@ class TestDataUploader(
private fun uploadTestData(testData: TestData) {
if (testSuiteApiToken == null) {
- println(
- "Buildkite test suite API token is missing. " +
- "Please set up your API token environment variable to upload the analytics data. Follow [README] for further information."
- )
- } else {
- val testUploaderService = RetrofitInstance.getRetrofitInstance(testSuiteApiToken = testSuiteApiToken)
- .create(TestUploaderApi::class.java)
- val uploadTestDataApiCall = testUploaderService.uploadTestData(testData = testData)
+ logger.info {
+ "Test Suite API token is missing. Please ensure the 'BUILDKITE_ANALYTICS_TOKEN' environment variable is set correctly to upload test data."
+ }
+ return
+ }
- val executeApiCall = uploadTestDataApiCall.execute()
+ try {
+ logger.debug { "Uploading test analytics data." }
- if (isDebugEnabled) {
- logApiResponse(executeApiCall)
- }
+ val testUploaderService =
+ TestAnalyticsRetrofit.getRetrofitInstance(testSuiteApiToken = testSuiteApiToken)
+ .create(TestUploaderApi::class.java)
+ val uploadTestDataApiCall = testUploaderService.uploadTestData(testData = testData)
+ val testUploadResponse = uploadTestDataApiCall.execute()
+
+ logApiResponse(testUploadResponse = testUploadResponse)
+ } catch (e: Exception) {
+ logger.error { "Error uploading test analytics data: ${e.message}." }
}
}
- private fun logApiResponse(executeApiCall: Response) {
- when (val apiResponseCode = executeApiCall.raw().code) {
- 202 -> println(
- "\nTest analytics data successfully uploaded to the BuildKite Test Suite. - ${executeApiCall.body()?.runUrl}"
- )
- else -> println(
- "\nError uploading test analytics data to the BuildKite Test Suite. Error code: $apiResponseCode! Ensure that the test suite API Token is correct."
- )
+ private fun logApiResponse(testUploadResponse: Response) {
+ if (testUploadResponse.isSuccessful) {
+ logger.debug { "Test analytics data successfully uploaded. URL: ${testUploadResponse.body()?.runUrl}" }
+ } else {
+ logger.error {
+ "Error uploading test analytics data. HTTP error code: ${testUploadResponse.code()}. Ensure the test suite API token is correct and properly configured."
+ }
+ logger.debug { "Failed response details: ${testUploadResponse.errorBody()?.string()}" }
}
}
}
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/RunEnvironment.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/RunEnvironment.kt
similarity index 69%
rename from collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/RunEnvironment.kt
rename to collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/RunEnvironment.kt
index d901b90..becc81d 100644
--- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/RunEnvironment.kt
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/RunEnvironment.kt
@@ -1,8 +1,23 @@
-package com.buildkite.test.collector.android.models
+package com.buildkite.test.collector.android.model
import com.buildkite.test.collector.android.util.CollectorUtils.generateUUIDString
import com.google.gson.annotations.SerializedName
+/**
+ * Represents information about the environment in which the test run is performed. Suitable for identifying test runs across CI and local environments.
+ *
+ * @property key A unique identifier for the test run.
+ * It's the only required field, especially for local tests where CI-specific fields aren't relevant.
+ * @property ci The continuous integration platform name.
+ * @property url The URL associated with the test run.
+ * @property branch The branch or reference for this build.
+ * @property commitSha The commit hash for the head of the branch.
+ * @property number The build number.
+ * @property jobId The job UUID.
+ * @property message The commit message for the head of the branch.
+ * @property version The current version of the collector.
+ * @property collector The current name of the collector.
+ */
internal data class RunEnvironment(
@SerializedName("CI") val ci: String? = null,
@SerializedName("key") val key: String = generateUUIDString(),
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestData.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestData.kt
new file mode 100644
index 0000000..e2fc3c8
--- /dev/null
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestData.kt
@@ -0,0 +1,17 @@
+package com.buildkite.test.collector.android.model
+
+import com.google.gson.annotations.SerializedName
+
+/**
+ * Represents the payload for Buildkite test analytics API.
+ *
+ * @property format Specifies the format for the upload data.
+ * @property runEnvironment Context of the test execution environment.
+ * Test results with matching run environments will be grouped together by the analytics API.
+ * @property data List of [TestDetails] providing individual test outcomes and related information.
+ */
+internal data class TestData(
+ @SerializedName("format") val format: String = "json",
+ @SerializedName("run_env") val runEnvironment: RunEnvironment,
+ @SerializedName("data") val data: List
+)
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestDetails.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestDetails.kt
new file mode 100644
index 0000000..49d5f21
--- /dev/null
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestDetails.kt
@@ -0,0 +1,30 @@
+package com.buildkite.test.collector.android.model
+
+import com.buildkite.test.collector.android.util.CollectorUtils.generateUUIDString
+import com.google.gson.annotations.SerializedName
+
+/**
+ * Represents a single test run with comprehensive details.
+ *
+ * @property id A unique identifier for this test result, defaulting to a generated UUID string.
+ * Duplicate IDs are ignored by the Test Analytics database.
+ * @property scope Specifies a group or topic for the test.
+ * @property name The name or description of the test.
+ * @property location The file and line number where the test originates.
+ * @property fileName The file where the test is defined.
+ * @property result The outcome of the test, specified as a [TestOutcome].
+ * @property failureReason A short description of the failure.
+ * @property failureExpanded Detailed failure information as a list of [TestFailureExpanded] objects.
+ * @property history The span information capturing the duration and execution details of the test.
+ */
+data class TestDetails(
+ @SerializedName("id") val id: String = generateUUIDString(),
+ @SerializedName("scope") val scope: String?,
+ @SerializedName("name") val name: String,
+ @SerializedName("location") val location: String?,
+ @SerializedName("file_name") val fileName: String?,
+ @SerializedName("result") val result: TestOutcome?,
+ @SerializedName("failure_reason") val failureReason: String?,
+ @SerializedName("failure_expanded") val failureExpanded: List? = null,
+ @SerializedName("history") val history: TestHistory
+)
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestFailureExpanded.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestFailureExpanded.kt
new file mode 100644
index 0000000..6b1b648
--- /dev/null
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestFailureExpanded.kt
@@ -0,0 +1,14 @@
+package com.buildkite.test.collector.android.model
+
+import com.google.gson.annotations.SerializedName
+
+/**
+ * Represents additional details about a test failure.
+ *
+ * @property backtrace A list of strings, each representing a frame in the call stack at the failure point.
+ * @property expanded A list of strings detailing the failure, including error messages and any supplementary context.
+ */
+data class TestFailureExpanded(
+ @SerializedName("backtrace") val backtrace: List = emptyList(),
+ @SerializedName("expanded") val expanded: List = emptyList()
+)
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestHistory.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestHistory.kt
new file mode 100644
index 0000000..51a2e46
--- /dev/null
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestHistory.kt
@@ -0,0 +1,18 @@
+package com.buildkite.test.collector.android.model
+
+import com.google.gson.annotations.SerializedName
+
+/**
+ * Represents the overall duration and phases of an individual test.
+ *
+ * @property startAt The start timestamp of the test.
+ * @property endAt The end timestamp of the test.
+ * @property duration The total duration of the test.
+ * @property children A collection of [TestSpan] objects detailing specific segments or operations within the test.
+ */
+data class TestHistory(
+ @SerializedName("start_at") val startAt: Long,
+ @SerializedName("end_at") val endAt: Long?,
+ @SerializedName("duration") val duration: Double,
+ @SerializedName("children") val children: List? = null,
+)
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/TraceResult.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestOutcome.kt
similarity index 64%
rename from collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/TraceResult.kt
rename to collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestOutcome.kt
index 4eec366..30c15b8 100644
--- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/TraceResult.kt
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestOutcome.kt
@@ -1,8 +1,11 @@
-package com.buildkite.test.collector.android.models
+package com.buildkite.test.collector.android.model
import com.google.gson.annotations.SerializedName
-enum class TraceResult {
+/**
+ * Represents the outcome of the test.
+ * */
+enum class TestOutcome {
@SerializedName("passed")
Passed,
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestSpan.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestSpan.kt
new file mode 100644
index 0000000..301a725
--- /dev/null
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestSpan.kt
@@ -0,0 +1,22 @@
+package com.buildkite.test.collector.android.model
+
+import com.google.gson.annotations.SerializedName
+
+/**
+ * Represents a specific operation or activity segment within a test, detailing its timing and additional information.
+ *
+ * This class captures the finer resolution of a test's execution duration, such as the time taken by an individual database query.
+ *
+ * @property section The category of this span, indicating the type of operation (e.g., SQL query, HTTP request). See [TestSpanSection].
+ * @property startAt The start timestamp of the test span.
+ * @property endAt The end timestamp of the test span.
+ * @property duration The total duration of the test span.
+ * @property detail [TestSpanDetail] object providing additional information about the span, required for certain section types like `http` or `sql`.
+ */
+data class TestSpan(
+ @SerializedName("section") val section: TestSpanSection = TestSpanSection.Top,
+ @SerializedName("start_at") val startAt: Long?,
+ @SerializedName("end_at") val endAt: Long?,
+ @SerializedName("duration") val duration: Double,
+ @SerializedName("detail") val detail: TestSpanDetail? = null
+)
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestSpanDetail.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestSpanDetail.kt
new file mode 100644
index 0000000..b79b5db
--- /dev/null
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestSpanDetail.kt
@@ -0,0 +1,20 @@
+package com.buildkite.test.collector.android.model
+
+import com.google.gson.annotations.SerializedName
+
+/**
+ * Represents additional information relevant to a specific [TestSpan].
+ *
+ * Depending on the [TestSpanSection], different information is provided.
+ *
+ * @property method The HTTP request method used (e.g., GET, POST). Required when [TestSpanSection] is [TestSpanSection.Http].
+ * @property url The URL targeted by the HTTP request. Required when [TestSpanSection] is [TestSpanSection.Http].
+ * @property library The library or tool used to make the HTTP request. Required when [TestSpanSection] is [TestSpanSection.Http].
+ * @property query The SQL query executed. Required when [TestSpanSection] is [TestSpanSection.Sql].
+ */
+data class TestSpanDetail(
+ @SerializedName("method") val method: String? = null,
+ @SerializedName("url") val url: String? = null,
+ @SerializedName("lib") val library: String? = null,
+ @SerializedName("query") val query: String? = null,
+)
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/SpanSection.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestSpanSection.kt
similarity index 61%
rename from collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/SpanSection.kt
rename to collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestSpanSection.kt
index 2602dbe..147e6f9 100644
--- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/SpanSection.kt
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestSpanSection.kt
@@ -1,8 +1,11 @@
-package com.buildkite.test.collector.android.models
+package com.buildkite.test.collector.android.model
import com.google.gson.annotations.SerializedName
-enum class SpanSection {
+/**
+ * Represents [TestSpan] category, classifying the type of operation or activity within a test run.
+ */
+enum class TestSpanSection {
@SerializedName(value = "top")
Top,
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestUploadResponse.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestUploadResponse.kt
new file mode 100644
index 0000000..aaad85c
--- /dev/null
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestUploadResponse.kt
@@ -0,0 +1,22 @@
+package com.buildkite.test.collector.android.model
+
+import com.google.gson.annotations.SerializedName
+
+/**
+ * Represents the response received from the analytics API after uploading test results.
+ *
+ * @property id Test response identifier.
+ * @property runId Identifier for the test run associated with the upload.
+ * @property queued The number of test results queued for processing.
+ * @property skipped The number of test results uploaded but not processed.
+ * @property errors List of error messages, if any, that occurred during the upload.
+ * @property runUrl The URL to access the details of the test run.
+ */
+internal data class TestUploadResponse(
+ @SerializedName("id") val id: String,
+ @SerializedName("run_id") val runId: String,
+ @SerializedName("queued") val queued: Int,
+ @SerializedName("skipped") val skipped: Int,
+ @SerializedName("errors") val errors: List,
+ @SerializedName("run_url") val runUrl: String,
+)
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/FailureExpanded.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/FailureExpanded.kt
deleted file mode 100644
index 6e2d790..0000000
--- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/FailureExpanded.kt
+++ /dev/null
@@ -1,8 +0,0 @@
-package com.buildkite.test.collector.android.models
-
-import com.google.gson.annotations.SerializedName
-
-data class FailureExpanded(
- @SerializedName("backtrace") val backtrace: List? = emptyList(),
- @SerializedName("expanded") val expanded: List? = emptyList()
-)
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/Span.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/Span.kt
deleted file mode 100644
index bd40651..0000000
--- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/Span.kt
+++ /dev/null
@@ -1,12 +0,0 @@
-package com.buildkite.test.collector.android.models
-
-import com.google.gson.annotations.SerializedName
-
-data class Span(
- @SerializedName("section") val section: SpanSection = SpanSection.Top,
- @SerializedName("start_at") val startAt: Long?,
- @SerializedName("end_at") val endAt: Long?,
- @SerializedName("duration") val duration: Double,
- @SerializedName("detail") val detail: Map? = emptyMap(),
- @SerializedName("children") val children: List = emptyList(),
-)
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/TestData.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/TestData.kt
deleted file mode 100644
index ed8e00e..0000000
--- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/TestData.kt
+++ /dev/null
@@ -1,9 +0,0 @@
-package com.buildkite.test.collector.android.models
-
-import com.google.gson.annotations.SerializedName
-
-internal data class TestData(
- @SerializedName("format") val format: String = "json",
- @SerializedName("run_env") val runEnvironment: RunEnvironment,
- @SerializedName("data") val data: List
-)
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/TestDetails.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/TestDetails.kt
deleted file mode 100644
index 7688757..0000000
--- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/TestDetails.kt
+++ /dev/null
@@ -1,16 +0,0 @@
-package com.buildkite.test.collector.android.models
-
-import com.buildkite.test.collector.android.util.CollectorUtils.generateUUIDString
-import com.google.gson.annotations.SerializedName
-
-data class TestDetails(
- @SerializedName("id") val id: String = generateUUIDString(),
- @SerializedName("scope") val scope: String?,
- @SerializedName("name") val name: String,
- @SerializedName("location") val location: String?,
- @SerializedName("file_name") val fileName: String?,
- @SerializedName("result") val result: TraceResult,
- @SerializedName("failure_reason") val failureReason: String?,
- @SerializedName("failure_expanded") val failureExpanded: List = emptyList(),
- @SerializedName("history") val history: Span
-)
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/TestResponse.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/TestResponse.kt
deleted file mode 100644
index 3627739..0000000
--- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/models/TestResponse.kt
+++ /dev/null
@@ -1,12 +0,0 @@
-package com.buildkite.test.collector.android.models
-
-import com.google.gson.annotations.SerializedName
-
-internal data class TestResponse(
- @SerializedName("id") val id: String,
- @SerializedName("run_id") val runId: String,
- @SerializedName("queued") val queued: Int,
- @SerializedName("skipped") val skipped: Int,
- @SerializedName("errors") val errors: List,
- @SerializedName("run_url") val runUrl: String,
-)
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/RetrofitInstance.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/TestAnalyticsRetrofit.kt
similarity index 69%
rename from collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/RetrofitInstance.kt
rename to collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/TestAnalyticsRetrofit.kt
index 7839315..f6923ec 100644
--- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/RetrofitInstance.kt
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/TestAnalyticsRetrofit.kt
@@ -6,7 +6,14 @@ import okhttp3.OkHttpClient
import retrofit2.Retrofit
import retrofit2.converter.gson.GsonConverterFactory
-internal object RetrofitInstance {
+internal object TestAnalyticsRetrofit {
+
+ /**
+ * Creates a Retrofit instance specifically configured for communicating with the Test Analytics API.
+ * Adds an [Interceptor] to append an authorization header containing the provided API token to all outgoing requests.
+ *
+ * @param testSuiteApiToken The test suite's API token to authenticate with Test Analytics.
+ */
fun getRetrofitInstance(testSuiteApiToken: String): Retrofit {
val client = OkHttpClient.Builder()
client.addInterceptor(
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/api/TestUploaderApi.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/api/TestUploaderApi.kt
index 1b52164..f9f70ad 100644
--- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/api/TestUploaderApi.kt
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/network/api/TestUploaderApi.kt
@@ -1,7 +1,7 @@
package com.buildkite.test.collector.android.network.api
-import com.buildkite.test.collector.android.models.TestData
-import com.buildkite.test.collector.android.models.TestResponse
+import com.buildkite.test.collector.android.model.TestData
+import com.buildkite.test.collector.android.model.TestUploadResponse
import com.buildkite.test.collector.android.util.CollectorUtils.Network.TEST_UPLOADER_ENDPOINT
import retrofit2.Call
import retrofit2.http.Body
@@ -14,5 +14,5 @@ internal interface TestUploaderApi {
@Headers(
"Content-Type:application/json"
)
- fun uploadTestData(@Body testData: TestData): Call
+ fun uploadTestData(@Body testData: TestData): Call
}
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/TestObserver.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/TestObserver.kt
index a3187be..b2165b2 100644
--- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/TestObserver.kt
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/TestObserver.kt
@@ -1,34 +1,77 @@
package com.buildkite.test.collector.android.tracer
-import com.buildkite.test.collector.android.models.FailureExpanded
-import com.buildkite.test.collector.android.models.TestDetails
-import com.buildkite.test.collector.android.models.TraceResult
-
-data class TestObserver(
- var startTime: Long = 0,
- var endTime: Long = 0,
- var result: TraceResult = TraceResult.Unknown,
- var failureReason: String = "",
- var failureExpanded: List = emptyList(),
- val collection: MutableList = mutableListOf()
-) {
- fun recordStartTime() {
+import com.buildkite.test.collector.android.model.TestFailureExpanded
+import com.buildkite.test.collector.android.model.TestOutcome
+import kotlin.math.max
+
+/**
+ * Observes and records details for an individual test, including timing, outcome, and failure details.
+ */
+class TestObserver {
+ var startTime: Long = 0
+ private set
+ var endTime: Long = 0
+ private set
+ var outcome: TestOutcome = TestOutcome.Unknown
+ private set
+ var failureReason: String? = null
+ private set
+ var failureDetails: List? = null
+ private set
+
+ /**
+ * Records the start time of a test in nanoseconds.
+ */
+ fun startTest() {
startTime = System.nanoTime()
}
- fun recordEndTime() {
+ /**
+ * Records the end time of a test in nanoseconds.
+ */
+ fun endTest() {
endTime = System.nanoTime()
}
- fun calculateSpanDuration(): Double {
- return endTime.minus(startTime).toDouble() / 1000000000
+ /**
+ * Returns the duration of the test in seconds, ensuring a non-negative result.
+ */
+ fun getDuration(): Double = max(0.0, (endTime - startTime) / 1_000_000_000.0)
+
+ /**
+ * Marks the test as successfully passed.
+ */
+ fun recordSuccess() {
+ outcome = TestOutcome.Passed
+ }
+
+ /**
+ * Records a test failure with a reason and detailed explanation.
+ */
+ fun recordFailure(
+ reason: String,
+ details: List = emptyList()
+ ) {
+ outcome = TestOutcome.Failed
+ failureReason = reason
+ failureDetails = details
+ }
+
+ /**
+ * Marks the test as skipped.
+ */
+ fun recordSkipped() {
+ outcome = TestOutcome.Skipped
}
- fun resetTestData() {
+ /**
+ * Resets all test data to initial state.
+ */
+ fun reset() {
startTime = 0
endTime = 0
- result = TraceResult.Unknown
- failureReason = ""
- failureExpanded = emptyList()
+ outcome = TestOutcome.Unknown
+ failureReason = null
+ failureDetails = null
}
}
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/UploaderConfiguration.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/UploaderConfiguration.kt
index be75e97..efa382e 100644
--- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/UploaderConfiguration.kt
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/tracer/environment/UploaderConfiguration.kt
@@ -4,11 +4,17 @@ package com.buildkite.test.collector.android.tracer.environment
import com.buildkite.test.collector.android.TestDataUploader
+/**
+ * Creates a [TestDataUploader] instance for unit tests, fetching configuration from environment variables.
+ */
fun configureUnitTestUploader() = TestDataUploader(
testSuiteApiToken = getStringEnvironmentValue(name = EnvironmentValues.BUILDKITE_ANALYTICS_TOKEN),
isDebugEnabled = getBooleanEnvironmentValue(name = EnvironmentValues.BUILDKITE_ANALYTICS_DEBUG_ENABLED)
)
+/**
+ * Creates a [TestDataUploader] instance for instrumented tests with the given [apiToken] and [isDebugEnabled] flag, filtering out 'null' tokens.
+ */
fun configureInstrumentedTestUploader(
apiToken: String,
isDebugEnabled: Boolean
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/CollectorUtils.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/CollectorUtils.kt
index 0f85801..0be9ef4 100644
--- a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/CollectorUtils.kt
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/CollectorUtils.kt
@@ -5,6 +5,10 @@ import java.util.UUID
internal object CollectorUtils {
object Uploader {
+ /**
+ * The maximum number of test results that can be uploaded in a single request.
+ * This limit ensures the upload request does not exceed the size constraints set by the Test Analytics API.
+ */
const val TEST_DATA_UPLOAD_LIMIT = 5000
}
diff --git a/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/Logger.kt b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/Logger.kt
new file mode 100644
index 0000000..b39bbb6
--- /dev/null
+++ b/collector/test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/util/Logger.kt
@@ -0,0 +1,48 @@
+package com.buildkite.test.collector.android.util
+
+/**
+ * Provides logging functionality with configurable log level sensitivity.
+ *
+ * @property minLevel The minimum log level that will be logged.
+ */
+class Logger(
+ private val minLevel: LogLevel = LogLevel.INFO
+) {
+ /**
+ * Logs a message at [LogLevel.DEBUG].
+ * - Messages are logged only if [LogLevel.DEBUG] is greater than or equal to [minLevel].
+ */
+ fun debug(message: () -> String) = log(LogLevel.DEBUG, message)
+
+ /**
+ * Logs a message at [LogLevel.INFO].
+ * - Messages are logged only if [LogLevel.INFO] is greater than or equal to [minLevel].
+ */
+ fun info(message: () -> String) = log(LogLevel.INFO, message)
+
+ /**
+ * Logs a message at [LogLevel.ERROR].
+ * - Messages are logged only if [LogLevel.ERROR] is greater than or equal to [minLevel].
+ */
+ fun error(message: () -> String) = log(LogLevel.ERROR, message)
+
+ /**
+ * Conditionally logs messages based on the [minLevel] set.
+ * - Logs to standard output for [LogLevel.DEBUG] and [LogLevel.INFO], and to standard error for [LogLevel.ERROR].
+ */
+ private fun log(level: LogLevel, message: () -> String) {
+ if (level >= minLevel) {
+ val output = if (level == LogLevel.ERROR) System.err else System.out
+ output.println("\nBuildkiteTestCollector-${level.name}: ${message()}")
+ }
+ }
+
+ /**
+ * Defines the log levels, ordered from least to most severe.
+ */
+ enum class LogLevel {
+ DEBUG,
+ INFO,
+ ERROR
+ }
+}
diff --git a/collector/test-data-uploader/src/test/kotlin/com/buildkite/test/collector/android/models/RunEnvironmentTest.kt b/collector/test-data-uploader/src/test/kotlin/com/buildkite/test/collector/android/models/RunEnvironmentTest.kt
index 94731a4..98cc7f8 100644
--- a/collector/test-data-uploader/src/test/kotlin/com/buildkite/test/collector/android/models/RunEnvironmentTest.kt
+++ b/collector/test-data-uploader/src/test/kotlin/com/buildkite/test/collector/android/models/RunEnvironmentTest.kt
@@ -1,5 +1,6 @@
package com.buildkite.test.collector.android.models
+import com.buildkite.test.collector.android.model.RunEnvironment
import com.google.gson.Gson
import junit.framework.TestCase.assertEquals
import org.junit.Test
diff --git a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt
index 6f4d804..cab4a55 100644
--- a/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt
+++ b/collector/unit-test-collector/src/main/kotlin/com/buildkite/test/collector/android/UnitTestCollectorPlugin.kt
@@ -1,9 +1,8 @@
package com.buildkite.test.collector.android
-import com.buildkite.test.collector.android.models.FailureExpanded
-import com.buildkite.test.collector.android.models.Span
-import com.buildkite.test.collector.android.models.TestDetails
-import com.buildkite.test.collector.android.models.TraceResult
+import com.buildkite.test.collector.android.model.TestDetails
+import com.buildkite.test.collector.android.model.TestFailureExpanded
+import com.buildkite.test.collector.android.model.TestHistory
import com.buildkite.test.collector.android.tracer.TestObserver
import com.buildkite.test.collector.android.tracer.environment.configureUnitTestUploader
import org.gradle.api.Plugin
@@ -13,66 +12,78 @@ import org.gradle.api.tasks.testing.TestDescriptor
import org.gradle.api.tasks.testing.TestListener
import org.gradle.api.tasks.testing.TestResult
+/**
+ * A Gradle plugin that automates the collection and uploading of unit test results to Buildkite's Test Analytics service.
+ * It attaches a listener [Test] task within Gradle projects, capturing real-time test events and outcomes.
+ * This enables detailed monitoring and systematic reporting of test results, which are uploaded directly to the analytics portal
+ * at the conclusion of test suite.
+ */
class UnitTestCollectorPlugin : Plugin {
override fun apply(project: Project) {
project.tasks.withType(Test::class.java) { test ->
test.addTestListener(object : TestListener {
- private val testObserver = TestObserver()
private val testUploader = configureUnitTestUploader()
+ private val testObserver = TestObserver()
+ private val testCollection: MutableList = mutableListOf()
- override fun beforeSuite(suite: TestDescriptor?) { /* Nothing to do */ }
+ override fun beforeSuite(suite: TestDescriptor) {
+ /* Nothing to do before the test suite has started */
+ }
- override fun afterSuite(suite: TestDescriptor?, result: TestResult?) {
- suite?.let { testSuite ->
- if (testSuite.className == null && testSuite.parent == null) {
- testUploader.configureUploadData(testCollection = testObserver.collection)
- }
+ override fun afterSuite(suite: TestDescriptor, result: TestResult) {
+ if (isFinalTestSuiteCall(testDescription = suite)) {
+ testUploader.configureUploadData(testCollection = testCollection)
}
}
- override fun beforeTest(testDescriptor: TestDescriptor?) {
- testObserver.recordStartTime()
+ override fun beforeTest(testDescriptor: TestDescriptor) {
+ testObserver.startTest()
}
- override fun afterTest(testDescriptor: TestDescriptor?, result: TestResult?) {
- testObserver.recordEndTime()
+ override fun afterTest(testDescriptor: TestDescriptor, result: TestResult) {
+ testObserver.endTest()
- when (result?.resultType) {
+ when (result.resultType) {
TestResult.ResultType.SUCCESS -> {
- testObserver.result = TraceResult.Passed
+ testObserver.recordSuccess()
}
+
TestResult.ResultType.FAILURE -> {
- testObserver.result = TraceResult.Failed
- testObserver.failureReason = result.exception.toString()
- testObserver.failureExpanded = result.exceptions.map { exception ->
- FailureExpanded(
+ val failureReason = result.exception.toString()
+ val failureDetails = result.exceptions.map { exception ->
+ TestFailureExpanded(
expanded = listOf("${exception.message}:${exception.cause}"),
backtrace = exception.stackTraceToString().split("\n")
.map { it.trim() }
)
}
+ testObserver.recordFailure(
+ reason = failureReason,
+ details = failureDetails
+ )
}
+
TestResult.ResultType.SKIPPED -> {
- testObserver.endTime = 0
- testObserver.startTime = 0
- testObserver.result = TraceResult.Skipped
+ testObserver.recordSkipped()
}
+
null -> {
- testObserver.result = TraceResult.Passed
+ // Handle the case where [TestResult] is unexpectedly null
+ testObserver.recordFailure(
+ reason = "TestResult type was unexpectedly null, indicating an error in the test framework."
+ )
}
}
- testDescriptor?.let { test ->
- addTestDetailsToCollection(test = test)
- }
+ addTestDetailsToCollection(test = testDescriptor)
}
private fun addTestDetailsToCollection(test: TestDescriptor) {
- val testSpan = Span(
+ val testHistory = TestHistory(
startAt = testObserver.startTime,
endAt = testObserver.endTime,
- duration = testObserver.calculateSpanDuration(),
+ duration = testObserver.getDuration()
)
val testDetails = TestDetails(
@@ -80,15 +91,24 @@ class UnitTestCollectorPlugin : Plugin {
name = test.displayName,
location = test.className,
fileName = null,
- result = testObserver.result,
+ result = testObserver.outcome,
failureReason = testObserver.failureReason,
- failureExpanded = testObserver.failureExpanded,
- history = testSpan
+ failureExpanded = testObserver.failureDetails,
+ history = testHistory
)
- testObserver.collection.add(testDetails)
- testObserver.resetTestData()
+ testCollection.add(testDetails)
+ testObserver.reset()
}
+
+ /**
+ * Determines if the provided test suite descriptor indicates the final call of the test suite,
+ * which is true when both className and parent are null.
+ *
+ * @param testDescription The test description. [TestDescriptor] can be atomic (a single test) or compound (containing children tests).
+ */
+ private fun isFinalTestSuiteCall(testDescription: TestDescriptor) =
+ testDescription.className == null && testDescription.parent == null
})
}
}
diff --git a/example/build.gradle.kts b/example/build.gradle.kts
index a48fc94..a700057 100644
--- a/example/build.gradle.kts
+++ b/example/build.gradle.kts
@@ -14,12 +14,15 @@ android {
versionName = "1.0"
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
+
+ // Specifies `ExampleTestCollector` as the instrumented test listener for collecting test analytics.
testInstrumentationRunnerArguments += mapOf("listener" to "com.buildkite.test.collector.android.example.ExampleTestCollector")
vectorDrawables {
useSupportLibrary = true
}
+ // Fetches local/CI environment variables for Buildkite test collector setup
buildConfigField(
"String",
"BUILDKITE_ANALYTICS_TOKEN",
diff --git a/example/src/androidTest/kotlin/com/buildkite/test/collector/android/example/ExampleTestCollector.kt b/example/src/androidTest/kotlin/com/buildkite/test/collector/android/example/ExampleTestCollector.kt
index f3561ea..5ecbf45 100644
--- a/example/src/androidTest/kotlin/com/buildkite/test/collector/android/example/ExampleTestCollector.kt
+++ b/example/src/androidTest/kotlin/com/buildkite/test/collector/android/example/ExampleTestCollector.kt
@@ -2,6 +2,10 @@ package com.buildkite.test.collector.android.example
import com.buildkite.test.collector.android.InstrumentedTestCollector
+/**
+ * Configures `ExampleTestCollector` with Buildkite test collector configurations.
+ * Values are derived from local/CI environment variables and set in [BuildConfig] at build time.
+ */
class ExampleTestCollector : InstrumentedTestCollector(
apiToken = BuildConfig.BUILDKITE_ANALYTICS_TOKEN,
isDebugEnabled = BuildConfig.BUILDKITE_ANALYTICS_DEBUG_ENABLED