-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #20 from buildkite/tech/refine-collector-api-and-d…
…ocumentation Document collector implementation and Enhance API models
- Loading branch information
Showing
28 changed files
with
467 additions
and
190 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
99 changes: 57 additions & 42 deletions
99
...llector/src/main/kotlin/com/buildkite/test/collector/android/InstrumentedTestCollector.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,88 +1,103 @@ | ||
| package com.buildkite.test.collector.android | ||
|
|
||
| import com.buildkite.test.collector.android.models.FailureExpanded | ||
| import com.buildkite.test.collector.android.models.Span | ||
| import com.buildkite.test.collector.android.models.TestDetails | ||
| import com.buildkite.test.collector.android.models.TraceResult | ||
| import com.buildkite.test.collector.android.model.TestDetails | ||
| import com.buildkite.test.collector.android.model.TestFailureExpanded | ||
| import com.buildkite.test.collector.android.model.TestHistory | ||
| import com.buildkite.test.collector.android.model.TestOutcome | ||
| import com.buildkite.test.collector.android.tracer.TestObserver | ||
| import com.buildkite.test.collector.android.tracer.environment.configureInstrumentedTestUploader | ||
| import org.junit.runner.Description | ||
| import org.junit.runner.notification.Failure | ||
| import org.junit.runner.notification.RunListener | ||
|
|
||
| /** | ||
| * Serves as an abstract foundation for creating instrumented test collectors that interface with Buildkite's Test Analytics service. | ||
| * This class extends JUnit's [RunListener] to capture real-time events during the execution of instrumented tests, enabling precise monitoring | ||
| * and reporting of test outcomes. It automatically gathers detailed test results and uploads them directly to the analytics portal at the conclusion of test suite. | ||
| * | ||
| * @param apiToken The API token for the test suite, necessary for authenticating requests with Test Analytics. | ||
| * @param isDebugEnabled When true, enables logging to assist with debugging. | ||
| */ | ||
| abstract class InstrumentedTestCollector( | ||
| apiToken: String, | ||
| isDebugEnabled: Boolean = false | ||
| ) : RunListener() { | ||
| private val testObserver = TestObserver() | ||
| private val testUploader = configureInstrumentedTestUploader(apiToken, isDebugEnabled) | ||
| private val testUploader = configureInstrumentedTestUploader( | ||
| apiToken = apiToken, | ||
| isDebugEnabled = isDebugEnabled | ||
| ) | ||
| private val testCollection: MutableList<TestDetails> = mutableListOf() | ||
|
|
||
| override fun testSuiteStarted(testDescription: Description?) { /* Nothing to do */ } | ||
| override fun testSuiteStarted(testDescription: Description) { | ||
| /* Nothing to do before the test suite has started */ | ||
| } | ||
|
|
||
| override fun testSuiteFinished(description: Description?) { | ||
| description?.let { testSuite -> | ||
| if ((testSuite.displayName.isNullOrEmpty() || testSuite.displayName == "null") && (testSuite.className.isNullOrEmpty() || testSuite.className == "null")) { | ||
| testUploader.configureUploadData(testCollection = testObserver.collection) | ||
| } | ||
| override fun testSuiteFinished(description: Description) { | ||
| if (isFinalTestSuiteCall(testDescription = description)) { | ||
| testUploader.configureUploadData(testCollection = testCollection) | ||
| } | ||
| } | ||
|
|
||
| override fun testStarted(testDescription: Description?) { | ||
| testObserver.recordStartTime() | ||
| override fun testStarted(testDescription: Description) { | ||
| testObserver.startTest() | ||
| } | ||
|
|
||
| override fun testFinished(testDescription: Description?) { | ||
| testObserver.recordEndTime() | ||
| override fun testFinished(testDescription: Description) { | ||
| testObserver.endTest() | ||
|
|
||
| if (testObserver.result != TraceResult.Failed) { | ||
| testObserver.result = TraceResult.Passed | ||
| if (testObserver.outcome != TestOutcome.Failed) { | ||
| testObserver.recordSuccess() | ||
| } | ||
|
|
||
| testDescription?.let { test -> | ||
| addTestDetailsToCollection(test = test) | ||
| } | ||
| addTestDetailsToCollection(test = testDescription) | ||
| } | ||
|
|
||
| override fun testFailure(failureDetails: Failure?) { | ||
| failureDetails?.let { failure -> | ||
| testObserver.result = TraceResult.Failed | ||
| testObserver.failureReason = failure.exception.toString() | ||
| testObserver.failureExpanded = listOf( | ||
| FailureExpanded( | ||
| expanded = failure.trimmedTrace.split("\n").map { it.trim() }, | ||
| backtrace = failure.trace.split("\n").map { it.trim() }, | ||
| ) | ||
| override fun testFailure(failureDetails: Failure) { | ||
| val failureReason = failureDetails.exception.toString() | ||
| val details = listOf( | ||
| TestFailureExpanded( | ||
| expanded = failureDetails.trimmedTrace.split("\n").map { it.trim() }, | ||
| backtrace = failureDetails.trace.split("\n").map { it.trim() }, | ||
| ) | ||
| } | ||
| ) | ||
| testObserver.recordFailure(reason = failureReason, details = details) | ||
| } | ||
|
|
||
| override fun testIgnored(testDescription: Description?) { | ||
| testObserver.result = TraceResult.Skipped | ||
| override fun testIgnored(testDescription: Description) { | ||
| testObserver.recordSkipped() | ||
|
|
||
| testDescription?.let { test -> | ||
| addTestDetailsToCollection(test = test) | ||
| } | ||
| addTestDetailsToCollection(test = testDescription) | ||
| } | ||
|
|
||
| private fun addTestDetailsToCollection(test: Description) { | ||
| val testSpan = Span( | ||
| val testHistory = TestHistory( | ||
| startAt = testObserver.startTime, | ||
| endAt = testObserver.endTime, | ||
| duration = testObserver.calculateSpanDuration(), | ||
| duration = testObserver.getDuration() | ||
| ) | ||
|
|
||
| val testDetails = TestDetails( | ||
| scope = test.testClass.name, | ||
| name = test.methodName, | ||
| location = test.className, | ||
| fileName = null, | ||
| result = testObserver.result, | ||
| result = testObserver.outcome, | ||
| failureReason = testObserver.failureReason, | ||
| failureExpanded = testObserver.failureExpanded, | ||
| history = testSpan | ||
| failureExpanded = testObserver.failureDetails, | ||
| history = testHistory | ||
| ) | ||
|
|
||
| testObserver.collection.add(testDetails) | ||
| testObserver.resetTestData() | ||
| testCollection.add(testDetails) | ||
| testObserver.reset() | ||
| } | ||
|
|
||
| /** | ||
| * Determines if the provided test suite descriptor indicates the final call of the test suite, | ||
| * which is true when both displayName and className are null. | ||
| * | ||
| * @param testDescription The test description. [Description] can be atomic (a single test) or compound (containing children tests). | ||
| */ | ||
| private fun isFinalTestSuiteCall(testDescription: Description) = | ||
| (testDescription.displayName.isNullOrEmpty() || testDescription.displayName == "null") && (testDescription.className.isNullOrEmpty() || testDescription.className == "null") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
17 changes: 16 additions & 1 deletion
17
...ollector/android/models/RunEnvironment.kt → ...collector/android/model/RunEnvironment.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
17 changes: 17 additions & 0 deletions
17
...test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestData.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| package com.buildkite.test.collector.android.model | ||
|
|
||
| import com.google.gson.annotations.SerializedName | ||
|
|
||
| /** | ||
| * Represents the payload for Buildkite test analytics API. | ||
| * | ||
| * @property format Specifies the format for the upload data. | ||
| * @property runEnvironment Context of the test execution environment. | ||
| * Test results with matching run environments will be grouped together by the analytics API. | ||
| * @property data List of [TestDetails] providing individual test outcomes and related information. | ||
| */ | ||
| internal data class TestData( | ||
| @SerializedName("format") val format: String = "json", | ||
| @SerializedName("run_env") val runEnvironment: RunEnvironment, | ||
| @SerializedName("data") val data: List<TestDetails> | ||
| ) |
30 changes: 30 additions & 0 deletions
30
...t-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestDetails.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| package com.buildkite.test.collector.android.model | ||
|
|
||
| import com.buildkite.test.collector.android.util.CollectorUtils.generateUUIDString | ||
| import com.google.gson.annotations.SerializedName | ||
|
|
||
| /** | ||
| * Represents a single test run with comprehensive details. | ||
| * | ||
| * @property id A unique identifier for this test result, defaulting to a generated UUID string. | ||
| * Duplicate IDs are ignored by the Test Analytics database. | ||
| * @property scope Specifies a group or topic for the test. | ||
| * @property name The name or description of the test. | ||
| * @property location The file and line number where the test originates. | ||
| * @property fileName The file where the test is defined. | ||
| * @property result The outcome of the test, specified as a [TestOutcome]. | ||
| * @property failureReason A short description of the failure. | ||
| * @property failureExpanded Detailed failure information as a list of [TestFailureExpanded] objects. | ||
| * @property history The span information capturing the duration and execution details of the test. | ||
| */ | ||
| data class TestDetails( | ||
| @SerializedName("id") val id: String = generateUUIDString(), | ||
| @SerializedName("scope") val scope: String?, | ||
| @SerializedName("name") val name: String, | ||
| @SerializedName("location") val location: String?, | ||
| @SerializedName("file_name") val fileName: String?, | ||
| @SerializedName("result") val result: TestOutcome?, | ||
| @SerializedName("failure_reason") val failureReason: String?, | ||
| @SerializedName("failure_expanded") val failureExpanded: List<TestFailureExpanded>? = null, | ||
| @SerializedName("history") val history: TestHistory | ||
| ) |
14 changes: 14 additions & 0 deletions
14
...ploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestFailureExpanded.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| package com.buildkite.test.collector.android.model | ||
|
|
||
| import com.google.gson.annotations.SerializedName | ||
|
|
||
| /** | ||
| * Represents additional details about a test failure. | ||
| * | ||
| * @property backtrace A list of strings, each representing a frame in the call stack at the failure point. | ||
| * @property expanded A list of strings detailing the failure, including error messages and any supplementary context. | ||
| */ | ||
| data class TestFailureExpanded( | ||
| @SerializedName("backtrace") val backtrace: List<String> = emptyList(), | ||
| @SerializedName("expanded") val expanded: List<String> = emptyList() | ||
| ) |
18 changes: 18 additions & 0 deletions
18
...t-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestHistory.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| package com.buildkite.test.collector.android.model | ||
|
|
||
| import com.google.gson.annotations.SerializedName | ||
|
|
||
| /** | ||
| * Represents the overall duration and phases of an individual test. | ||
| * | ||
| * @property startAt The start timestamp of the test. | ||
| * @property endAt The end timestamp of the test. | ||
| * @property duration The total duration of the test. | ||
| * @property children A collection of [TestSpan] objects detailing specific segments or operations within the test. | ||
| */ | ||
| data class TestHistory( | ||
| @SerializedName("start_at") val startAt: Long, | ||
| @SerializedName("end_at") val endAt: Long?, | ||
| @SerializedName("duration") val duration: Double, | ||
| @SerializedName("children") val children: List<TestSpan>? = null, | ||
| ) |
7 changes: 5 additions & 2 deletions
7
...t/collector/android/models/TraceResult.kt → ...st/collector/android/model/TestOutcome.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
22 changes: 22 additions & 0 deletions
22
...test-data-uploader/src/main/kotlin/com/buildkite/test/collector/android/model/TestSpan.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| package com.buildkite.test.collector.android.model | ||
|
|
||
| import com.google.gson.annotations.SerializedName | ||
|
|
||
| /** | ||
| * Represents a specific operation or activity segment within a test, detailing its timing and additional information. | ||
| * | ||
| * This class captures the finer resolution of a test's execution duration, such as the time taken by an individual database query. | ||
| * | ||
| * @property section The category of this span, indicating the type of operation (e.g., SQL query, HTTP request). See [TestSpanSection]. | ||
| * @property startAt The start timestamp of the test span. | ||
| * @property endAt The end timestamp of the test span. | ||
| * @property duration The total duration of the test span. | ||
| * @property detail [TestSpanDetail] object providing additional information about the span, required for certain section types like `http` or `sql`. | ||
| */ | ||
| data class TestSpan( | ||
| @SerializedName("section") val section: TestSpanSection = TestSpanSection.Top, | ||
| @SerializedName("start_at") val startAt: Long?, | ||
| @SerializedName("end_at") val endAt: Long?, | ||
| @SerializedName("duration") val duration: Double, | ||
| @SerializedName("detail") val detail: TestSpanDetail? = null | ||
| ) |
Oops, something went wrong.