Remove old persistentMetricsJobId places

venigreat · Mar 18, 2021 · 1a6fa19 · 1a6fa19
1 parent 386fdc0
commit 1a6fa19
Show file tree

Hide file tree

Showing 32 changed files with 98 additions and 111 deletions.
diff --git a/Sources/BalancingBucketQueue/Models/JobQueue.swift b/Sources/BalancingBucketQueue/Models/JobQueue.swift
@@ -9,22 +9,19 @@ public struct JobQueue: DefinesExecutionOrder {
     public let job: Job
     public let jobGroup: JobGroup
     public let resultsCollector: ResultsCollector
-    public let persistentMetricsJobId: String
 
     public init(
         analyticsConfiguration: AnalyticsConfiguration,
         bucketQueue: BucketQueue,
         job: Job,
         jobGroup: JobGroup,
-        resultsCollector: ResultsCollector,
-        persistentMetricsJobId: String
+        resultsCollector: ResultsCollector
     ) {
         self.analyticsConfiguration = analyticsConfiguration
         self.bucketQueue = bucketQueue
         self.job = job
         self.jobGroup = jobGroup
         self.resultsCollector = resultsCollector
-        self.persistentMetricsJobId = persistentMetricsJobId
     }
 
     public func executionOrder(relativeTo other: JobQueue) -> ExecutionOrder {

diff --git a/Sources/BalancingBucketQueue/New/MultipleQueuesEnqueueableBucketReceptor.swift b/Sources/BalancingBucketQueue/New/MultipleQueuesEnqueueableBucketReceptor.swift
@@ -38,8 +38,7 @@ public final class MultipleQueuesEnqueueableBucketReceptor: EnqueueableBucketRec
                             jobGroupId: prioritizedJob.jobGroupId,
                             jobGroupPriority: prioritizedJob.jobGroupPriority
                         ),
-                        resultsCollector: ResultsCollector(),
-                        persistentMetricsJobId: prioritizedJob.persistentMetricsJobId
+                        resultsCollector: ResultsCollector()
                     )
                 )
                 multipleQueuesContainer.removeFromDeleted(jobId: prioritizedJob.jobId)

diff --git a/Sources/BalancingBucketQueue/New/MultipleQueuesJobManipulator.swift b/Sources/BalancingBucketQueue/New/MultipleQueuesJobManipulator.swift
@@ -39,16 +39,18 @@ public final class MultipleQueuesJobManipulator: JobManipulator {
             for deletedJobQueue in jobQueuesToDelete {
                 multipleQueuesContainer.untrack(jobGroup: deletedJobQueue.jobGroup)
 
-                try specificMetricRecorderProvider.specificMetricRecorder(
-                    analyticsConfiguration: deletedJobQueue.analyticsConfiguration
-                ).capture(
-                    JobProcessingDurationMetric(
-                        queueHost: LocalHostDeterminer.currentHostAddress,
-                        version: emceeVersion,
-                        persistentMetricsJobId: deletedJobQueue.persistentMetricsJobId,
-                        duration: dateProvider.currentDate().timeIntervalSince(deletedJobQueue.job.creationTime)
+                if let persistentMetricsJobId = deletedJobQueue.analyticsConfiguration.persistentMetricsJobId {
+                    try specificMetricRecorderProvider.specificMetricRecorder(
+                        analyticsConfiguration: deletedJobQueue.analyticsConfiguration
+                    ).capture(
+                        JobProcessingDurationMetric(
+                            queueHost: LocalHostDeterminer.currentHostAddress,
+                            version: emceeVersion,
+                            persistentMetricsJobId: persistentMetricsJobId,
+                            duration: dateProvider.currentDate().timeIntervalSince(deletedJobQueue.job.creationTime)
+                        )
                     )
-                )
+                }
             }
         }
     }

diff --git a/Sources/DistWorker/DistWorker.swift b/Sources/DistWorker/DistWorker.swift
@@ -50,12 +50,11 @@ public final class DistWorker: SchedulerDataSource, SchedulerDelegate {
 
     public init(
         di: DI,
-        logger: ContextualLogger,
         version: Version,
         workerId: WorkerId
-    ) {
+    ) throws {
         self.di = di
-        self.logger = logger.forType(Self.self)
+        self.logger = try di.get(ContextualLogger.self).forType(Self.self)
         self.httpRestServer = HTTPRESTServer(
             automaticTerminationController: StayAliveTerminationController(),
             logger: logger,

diff --git a/Sources/EmceeLib/Commands/DistWorkCommand.swift b/Sources/EmceeLib/Commands/DistWorkCommand.swift
@@ -105,11 +105,13 @@ public final class DistWorkCommand: Command {
             for: WorkerCapabilitiesProvider.self
         )
 
-        return DistWorker(
+        let updatedLogger = try di.get(ContextualLogger.self)
+            .withMetadata(key: .workerId, value: workerId.value)
+            .withMetadata(key: .emceeVersion, value: version.value)
+        di.set(updatedLogger)
+
+        return try DistWorker(
             di: di,
-            logger: try di.get(ContextualLogger.self)
-                .withMetadata(key: .workerId, value: workerId.value)
-                .withMetadata(key: .emceeVersion, value: version.value),
             version: version,
             workerId: workerId
         )

diff --git a/Sources/EmceeLib/Utils/DetailedActivityLoggableProcessControllerProvider.swift b/Sources/EmceeLib/Utils/DetailedActivityLoggableProcessControllerProvider.swift
@@ -7,15 +7,18 @@ import ProcessController
 
 public final class DetailedActivityLoggableProcessControllerProvider: ProcessControllerProvider {
     private let processControllerProvider: LoggableProcessControllerProvider
-    private let logger: ContextualLogger
+
+    private let di: DI
 
     public init(
         di: DI
     ) throws {
-        logger = try di.get(ContextualLogger.self).forType(Self.self)
+        self.di = di
 
         processControllerProvider = LoggableProcessControllerProvider(
-            pathProvider: { [logger] processName -> (stdout: AbsolutePath, stderr: AbsolutePath) in
+            pathProvider: { [di] processName -> (stdout: AbsolutePath, stderr: AbsolutePath) in
+                let logger = try di.get(ContextualLogger.self).forType(Self.self)
+
                 let paths = try di.get(LoggingSetup.self).childProcessLogsContainerProvider().paths(subprocessName: processName)
                 logger.debug(
                     "Subprocess output will be stored: stdout: \(paths.stdout) stderr: \(paths.stderr)",
@@ -31,6 +34,7 @@ public final class DetailedActivityLoggableProcessControllerProvider: ProcessCon
     }
 
     public func createProcessController(subprocess: Subprocess) throws -> ProcessController {
+        let logger = try di.get(ContextualLogger.self).forType(Self.self)
         let processController = try processControllerProvider.createProcessController(subprocess: subprocess)
 
         processController.onStart { [logger] sender, _ in

diff --git a/Sources/EmceeLib/Utils/JobPreparer/JobPreparer.swift b/Sources/EmceeLib/Utils/JobPreparer/JobPreparer.swift
@@ -40,7 +40,6 @@ public final class JobPreparer {
                     duration: duration,
                     emceeVersion: emceeVersion,
                     analyticsConfiguration: testArgFile.prioritizedJob.analyticsConfiguration,
-                    persistentMetricsJobId: testArgFile.prioritizedJob.persistentMetricsJobId,
                     queueHost: queueServerAddress.host,
                     successful: error == nil
                 )
@@ -57,14 +56,13 @@ public final class JobPreparer {
             remoteCache: try di.get(RuntimeDumpRemoteCacheProvider.self).remoteCache(config: remoteCacheConfig),
             testArgFileEntries: testArgFile.entries,
             testDiscoveryQuerier: try di.get(),
-            analyticsConfiguration: testArgFile.prioritizedJob.analyticsConfiguration,
-            persistentMetricsJobId: testArgFile.prioritizedJob.persistentMetricsJobId
+            analyticsConfiguration: testArgFile.prioritizedJob.analyticsConfiguration
         )
 
         let logger = try di.get(ContextualLogger.self)
             .forType(Self.self)
-            .withMetadata(key: .persistentMetricsJobId, value: testArgFile.prioritizedJob.persistentMetricsJobId)
-        
+            .withMetadata(key: .persistentMetricsJobId, value: testArgFile.prioritizedJob.analyticsConfiguration.persistentMetricsJobId)
+
         _ = try testEntriesValidator.validatedTestEntries { testArgFileEntry, validatedTestEntry in
             let testEntryConfigurationGenerator = TestEntryConfigurationGenerator(
                 analyticsConfiguration: testArgFile.prioritizedJob.analyticsConfiguration,
@@ -96,20 +94,21 @@ public final class JobPreparer {
         duration: TimeInterval,
         emceeVersion: Version,
         analyticsConfiguration: AnalyticsConfiguration,
-        persistentMetricsJobId: String,
         queueHost: String,
         successful: Bool
     ) throws {
-        try di.get(SpecificMetricRecorderProvider.self).specificMetricRecorder(
-            analyticsConfiguration: analyticsConfiguration
-        ).capture(
-            JobPreparationDurationMetric(
-                queueHost: queueHost,
-                version: emceeVersion,
-                persistentMetricsJobId: persistentMetricsJobId,
-                successful: successful,
-                duration: duration
+        if let persistentMetricsJobId = analyticsConfiguration.persistentMetricsJobId {
+            try di.get(SpecificMetricRecorderProvider.self).specificMetricRecorder(
+                analyticsConfiguration: analyticsConfiguration
+            ).capture(
+                JobPreparationDurationMetric(
+                    queueHost: queueHost,
+                    version: emceeVersion,
+                    persistentMetricsJobId: persistentMetricsJobId,
+                    successful: successful,
+                    duration: duration
+                )
             )
-        )
+        }
     }
 }
diff --git a/Sources/EmceeLib/Utils/PipelinedTestDiscoverer.swift b/Sources/EmceeLib/Utils/PipelinedTestDiscoverer.swift
@@ -45,7 +45,7 @@ public final class PipelinedTestDiscoverer {
         remoteCacheConfig: RuntimeDumpRemoteCacheConfig?
     ) throws -> [[DiscoveredTestEntry]] {
         let logger = self.logger
-            .withMetadata(key: .persistentMetricsJobId, value: testArgFile.prioritizedJob.persistentMetricsJobId)
+            .withMetadata(key: .persistentMetricsJobId, value: testArgFile.prioritizedJob.analyticsConfiguration.persistentMetricsJobId)
 
         let discoveredTests = AtomicValue<[[DiscoveredTestEntry]]>(
             Array(repeating: [], count: testArgFile.entries.count)
@@ -90,7 +90,6 @@ public final class PipelinedTestDiscoverer {
                         testTimeoutConfiguration: testTimeoutConfigurationForRuntimeDump,
                         testsToValidate: testArgFileEntry.testsToRun,
                         xcTestBundleLocation: testArgFileEntry.buildArtifacts.xcTestBundle.location,
-                        persistentMetricsJobId: testArgFile.prioritizedJob.persistentMetricsJobId,
                         remoteCache: runtimeDumpRemoteCacheProvider.remoteCache(config: remoteCacheConfig)
                     )
 

diff --git a/Sources/EmceeLib/Utils/TestEntriesValidator/TestEntriesValidator.swift b/Sources/EmceeLib/Utils/TestEntriesValidator/TestEntriesValidator.swift
@@ -13,21 +13,18 @@ public final class TestEntriesValidator {
     private let testArgFileEntries: [TestArgFileEntry]
     private let testDiscoveryQuerier: TestDiscoveryQuerier
     private let analyticsConfiguration: AnalyticsConfiguration
-    private let persistentMetricsJobId: String
     private let transformer = TestToRunIntoTestEntryTransformer()
 
     public init(
         remoteCache: RuntimeDumpRemoteCache,
         testArgFileEntries: [TestArgFileEntry],
         testDiscoveryQuerier: TestDiscoveryQuerier,
-        analyticsConfiguration: AnalyticsConfiguration,
-        persistentMetricsJobId: String
+        analyticsConfiguration: AnalyticsConfiguration
     ) {
         self.remoteCache = remoteCache
         self.testArgFileEntries = testArgFileEntries
         self.testDiscoveryQuerier = testDiscoveryQuerier
         self.analyticsConfiguration = analyticsConfiguration
-        self.persistentMetricsJobId = persistentMetricsJobId
     }
 
     public func validatedTestEntries(
@@ -65,7 +62,6 @@ public final class TestEntriesValidator {
             testTimeoutConfiguration: testTimeoutConfigurationForRuntimeDump,
             testsToValidate: testArgFileEntry.testsToRun,
             xcTestBundleLocation: testArgFileEntry.buildArtifacts.xcTestBundle.location,
-            persistentMetricsJobId: persistentMetricsJobId,
             remoteCache: remoteCache
         )
 

diff --git a/Sources/EmceeLogging/ContextualLogger.swift b/Sources/EmceeLogging/ContextualLogger.swift
@@ -141,4 +141,15 @@ public extension ContextualLogger {
         addedMetadata.merge(keyValues) { _, new -> String in new }
         return ContextualLogger(logger: logger, addedMetadata: addedMetadata)
     }
+
+    func withMetadata(key: ContextKeys, value: String?) -> ContextualLogger {
+        withMetadata(key: key.rawValue, value: value)
+    }
+
+    func withMetadata(key: String, value: String?) -> ContextualLogger {
+        if let value = value {
+            return withMetadata(key: key, value: value)
+        }
+        return self
+    }
 }
diff --git a/Sources/EmceeLogging/KibanaHandler.swift b/Sources/EmceeLogging/KibanaHandler.swift
@@ -1,3 +1,4 @@
+import Extensions
 import Foundation
 import Kibana
 import Logging
@@ -29,7 +30,7 @@ public final class KibanaLoggerHandler: LoggerHandler {
         line: UInt
     ) {
         var kibanaPayload = [
-            "fileLine": "\(file):\(line)",
+            "fileLine": "\(file.lastPathComponent):\(line)",
         ]
 
         for keyValue in metadata ?? [:] {

diff --git a/Sources/QueueModels/PrioritizedJob.swift b/Sources/QueueModels/PrioritizedJob.swift
@@ -7,25 +7,22 @@ public struct PrioritizedJob: Hashable, Codable, CustomStringConvertible {
     public let jobGroupPriority: Priority
     public let jobId: JobId
     public let jobPriority: Priority
-    public let persistentMetricsJobId: String
 
     public init(
         analyticsConfiguration: AnalyticsConfiguration,
         jobGroupId: JobGroupId,
         jobGroupPriority: Priority,
         jobId: JobId,
-        jobPriority: Priority,
-        persistentMetricsJobId: String
+        jobPriority: Priority
     ) {
         self.analyticsConfiguration = analyticsConfiguration
         self.jobGroupId = jobGroupId
         self.jobGroupPriority = jobGroupPriority
         self.jobId = jobId
         self.jobPriority = jobPriority
-        self.persistentMetricsJobId = persistentMetricsJobId
     }
 
     public var description: String {
-        return "<\(type(of: self)) \(jobGroupId) \(jobGroupPriority) \(jobId) \(jobPriority) \(persistentMetricsJobId)>"
+        return "<\(type(of: self)) \(jobGroupId) \(jobGroupPriority) \(jobId) \(jobPriority) \(analyticsConfiguration)>"
     }
 }
diff --git a/Sources/TestArgFile/TestArgFile.swift b/Sources/TestArgFile/TestArgFile.swift
@@ -47,15 +47,12 @@ public struct TestArgFile: Codable, Equatable {
                 jobPriority
             let analyticsConfiguration = try container.decodeIfPresent(AnalyticsConfiguration.self, forKey: .analyticsConfiguration) ??
                 TestArgFileDefaultValues.analyticsConfiguration
-            let persistentMetricsJobId = try container.decodeIfPresent(String.self, forKey: .persistentMetricsJobId) ??
-                TestArgFileDefaultValues.persistentMetricsJobId
             return PrioritizedJob(
                 analyticsConfiguration: analyticsConfiguration,
                 jobGroupId: jobGroupId,
                 jobGroupPriority: jobGroupPriority,
                 jobId: jobId,
-                jobPriority: jobPriority,
-                persistentMetricsJobId: persistentMetricsJobId
+                jobPriority: jobPriority
             )
         }()
 

diff --git a/Sources/TestArgFile/TestArgFileDefaultValues.swift b/Sources/TestArgFile/TestArgFileDefaultValues.swift
@@ -48,6 +48,5 @@ public enum TestArgFileDefaultValues {
         testRunnerMaximumSilenceDuration: 60
     )
     public static let workerCapabilityRequirements: Set<WorkerCapabilityRequirement> = []
-    public static let persistentMetricsJobId: String = "UnidentifiedJob"
     public static let analyticsConfiguration = AnalyticsConfiguration()
 }
diff --git a/Sources/TestDiscovery/Models/TestDiscoveryConfiguration.swift b/Sources/TestDiscovery/Models/TestDiscoveryConfiguration.swift
@@ -20,7 +20,6 @@ public struct TestDiscoveryConfiguration {
     public let testTimeoutConfiguration: TestTimeoutConfiguration
     public let testsToValidate: [TestToRun]
     public let xcTestBundleLocation: TestBundleLocation
-    public let persistentMetricsJobId: String
     public let remoteCache: RuntimeDumpRemoteCache
 
     public init(
@@ -36,7 +35,6 @@ public struct TestDiscoveryConfiguration {
         testTimeoutConfiguration: TestTimeoutConfiguration,
         testsToValidate: [TestToRun],
         xcTestBundleLocation: TestBundleLocation,
-        persistentMetricsJobId: String,
         remoteCache: RuntimeDumpRemoteCache
     ) {
         self.analyticsConfiguration = analyticsConfiguration
@@ -51,7 +49,6 @@ public struct TestDiscoveryConfiguration {
         self.testTimeoutConfiguration = testTimeoutConfiguration
         self.testsToValidate = testsToValidate
         self.xcTestBundleLocation = xcTestBundleLocation
-        self.persistentMetricsJobId = persistentMetricsJobId
         self.remoteCache = remoteCache
     }
 }
diff --git a/Sources/TestDiscovery/SpecificTestDiscoverers/RuntimeDumpTestDiscoverer.swift b/Sources/TestDiscovery/SpecificTestDiscoverers/RuntimeDumpTestDiscoverer.swift
@@ -99,7 +99,7 @@ final class RuntimeDumpTestDiscoverer: SpecificTestDiscoverer {
             developerDirLocator: developerDirLocator,
             fileSystem: fileSystem,
             logger: logger,
-            persistentMetricsJobId: configuration.persistentMetricsJobId,
+            persistentMetricsJobId: configuration.analyticsConfiguration.persistentMetricsJobId,
             pluginEventBusProvider: pluginEventBusProvider,
             resourceLocationResolver: resourceLocationResolver,
             specificMetricRecorder: specificMetricRecorder,

diff --git a/Sources/TestDiscovery/TestDiscoveryQuerierImpl.swift b/Sources/TestDiscovery/TestDiscoveryQuerierImpl.swift
@@ -133,7 +133,7 @@ public final class TestDiscoveryQuerierImpl: TestDiscoveryQuerier {
             },
             result: { error, duration in
                 reportDiscoveryDuration(
-                    persistentMetricsJobId: configuration.persistentMetricsJobId,
+                    persistentMetricsJobId: configuration.analyticsConfiguration.persistentMetricsJobId,
                     duration: duration,
                     isSuccessful: error == nil,
                     specificMetricRecorder: specificMetricRecorder
@@ -197,20 +197,22 @@ public final class TestDiscoveryQuerierImpl: TestDiscoveryQuerier {
     }
 
     private func reportDiscoveryDuration(
-        persistentMetricsJobId: String,
+        persistentMetricsJobId: String?,
         duration: TimeInterval,
         isSuccessful: Bool,
         specificMetricRecorder: SpecificMetricRecorder
     ) {
-        specificMetricRecorder.capture(
-            TestDiscoveryDurationMetric(
-                host: LocalHostDeterminer.currentHostAddress,
-                version: version,
-                persistentMetricsJobId: persistentMetricsJobId,
-                isSuccessful: isSuccessful,
-                duration: duration
+        if let persistentMetricsJobId = persistentMetricsJobId {
+            specificMetricRecorder.capture(
+                TestDiscoveryDurationMetric(
+                    host: LocalHostDeterminer.currentHostAddress,
+                    version: version,
+                    persistentMetricsJobId: persistentMetricsJobId,
+                    isSuccessful: isSuccessful,
+                    duration: duration
+                )
             )
-        )
+        }
     }
 
     private func createSpecificTestDiscoverer(