Benchmarks. Change output format and modes

benchmarks/multiplatform/benchmarks/build.gradle.kts

@@ -1,5 +1,6 @@
 import org.jetbrains.kotlin.gradle.ExperimentalWasmDsl
 import org.jetbrains.kotlin.gradle.targets.js.webpack.KotlinWebpack
+import kotlin.text.replace
 
 plugins {
     kotlin("multiplatform")
@@ -55,6 +56,8 @@ kotlin {
                 implementation(compose.material)
                 implementation(compose.runtime)
                 implementation(compose.components.resources)
+                implementation("org.jetbrains.kotlinx:kotlinx-io-core:0.6.0")
+                implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.6.2")
             }
         }
 
@@ -74,21 +77,33 @@ compose.desktop {
 }
 
 val runArguments: String? by project
+val composeVersion: String? = project.properties["compose.version"] as? String
+val kotlinVersion: String? = project.properties["kotlin.version"] as? String
+var appArgs = runArguments
+    ?.split(" ")
+    .orEmpty().let {
+       it + listOf("versionInfo=\"$composeVersion (Kotlin $kotlinVersion)\"")
+    }
+    .map {
+        it.replace(" ", "%20")
+    }
+
+println("runArguments: $appArgs")
 
 // Handle runArguments property
 gradle.taskGraph.whenReady {
     tasks.named<JavaExec>("run") {
-        args(runArguments?.split(" ") ?: listOf<String>())
+        args(appArgs)
     }
     tasks.forEach { t ->
         if ((t is Exec) && t.name.startsWith("runReleaseExecutableMacos")) {
-            t.args(runArguments?.split(" ") ?: listOf<String>())
+            t.args(appArgs)
         }
     }
     tasks.named<KotlinWebpack>("wasmJsBrowserProductionRun") {
-        val args = runArguments?.split(" ")
-            ?.mapIndexed { index, arg -> "arg$index=$arg" }
-            ?.joinToString("&") ?: ""
+        val args = appArgs
+            .mapIndexed { index, arg -> "arg$index=$arg" }
+            .joinToString("&")
 
         devServerProperty = devServerProperty.get().copy(
             open = "http://localhost:8080?$args"

benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Args.kt

@@ -1,14 +1,16 @@
 enum class Mode {
-    CPU,
-    FRAMES,
-    FRAMES_GPU
+    SIMPLE,
+    VSYNC_EMULATION,
 }
 
 object Args {
     private val modes = mutableSetOf<Mode>()
 
     private val benchmarks = mutableMapOf<String, Int>()
 
+    var versionInfo: String? = null
+        private set
+
     private fun argToSet(arg: String): Set<String> = arg.substring(arg.indexOf('=') + 1)
         .split(",").filter{!it.isEmpty()}.map{it.uppercase()}.toSet()
 
@@ -32,13 +34,17 @@ object Args {
     fun parseArgs(args: Array<String>) {
         for (arg in args) {
             if (arg.startsWith("modes=", ignoreCase = true)) {
-                modes.addAll(argToSet(arg).map { Mode.valueOf(it) })
+                modes.addAll(argToSet(arg.decodeArg()).map { Mode.valueOf(it) })
             } else if (arg.startsWith("benchmarks=", ignoreCase = true)) {
-                benchmarks += argToMap(arg)
+                benchmarks += argToMap(arg.decodeArg())
+            } else if (arg.startsWith("versionInfo=", ignoreCase = true)) {
+                versionInfo = arg.substringAfter("=").decodeArg()
             }
         }
     }
 
+    private fun String.decodeArg() = replace("%20", " ")
+
     fun isModeEnabled(mode: Mode): Boolean = modes.isEmpty() || modes.contains(mode)
 
     fun isBenchmarkEnabled(benchmark: String): Boolean = benchmarks.isEmpty() || benchmarks.contains(benchmark.uppercase())

benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/Benchmarks.kt

@@ -4,6 +4,16 @@ import benchmarks.complexlazylist.components.MainUiNoImageUseModel
 import benchmarks.example1.Example1
 import benchmarks.lazygrid.LazyGrid
 import benchmarks.visualeffects.NYContent
+import kotlinx.datetime.Clock
+import kotlinx.datetime.LocalDateTime
+import kotlinx.datetime.TimeZone
+import kotlinx.datetime.format.FormatStringsInDatetimeFormats
+import kotlinx.datetime.format.byUnicodePattern
+import kotlinx.datetime.toLocalDateTime
+import kotlinx.io.buffered
+import kotlinx.io.files.Path
+import kotlinx.io.files.SystemFileSystem
+import kotlinx.io.readByteArray
 import kotlin.math.roundToInt
 import kotlin.time.Duration
 
@@ -29,6 +39,38 @@ data class BenchmarkFrame(
         }
 }
 
+data class BenchmarkConditions(
+    val frameCount: Int,
+    val warmupCount: Int
+) {
+    fun prettyPrint() {
+        println("$frameCount frames (warmup $warmupCount)")
+    }
+
+    fun putFormattedValuesTo(map: MutableMap<String, String>) {
+        map.put("Frames/warmup", "$frameCount/$warmupCount")
+    }
+}
+
+data class FrameInfo(
+    val cpuTime: Duration,
+    val gpuTime: Duration,
+) {
+    val totalTime = cpuTime + gpuTime
+
+    fun prettyPrint() {
+        println("CPU average frame time: $cpuTime")
+        println("GPU average frame time: $gpuTime")
+        println("TOTAL average frame time: $totalTime")
+    }
+
+    fun putFormattedValuesTo(map: MutableMap<String, String>) {
+        map.put("CPU avg frame (ms)", cpuTime.formatAsMilliseconds())
+        map.put("GPU avg frame (ms)", gpuTime.formatAsMilliseconds())
+        map.put("TOTAL avg frame (ms)", totalTime.formatAsMilliseconds())
+    }
+}
+
 data class BenchmarkPercentileAverage(
     val percentile: Double,
     val average: Duration
@@ -48,46 +90,84 @@ data class MissedFrames(
             """.trimIndent()
         )
     }
+
+    fun putFormattedValuesTo(description: String, map: MutableMap<String, String>) {
+        map.put("Missed frames ($description)", "$ratio")
+    }
 }
 
 data class BenchmarkStats(
     val frameBudget: Duration,
-    val frameCount: Int,
-    val renderTime: Duration,
+    val conditions: BenchmarkConditions,
+    val averageFrameInfo: FrameInfo?,
     val percentileCPUAverage: List<BenchmarkPercentileAverage>,
     val percentileGPUAverage: List<BenchmarkPercentileAverage>,
     val noBufferingMissedFrames: MissedFrames,
     val doubleBufferingMissedFrames: MissedFrames
 ) {
     fun prettyPrint() {
-        if (Args.isModeEnabled(Mode.CPU)) {
-            println("$frameCount frames CPU render time: $renderTime")
+        val versionInfo = Args.versionInfo
+        if (versionInfo != null) {
+            println("Version: $versionInfo")
+        }
+        conditions.prettyPrint()
+        println()
+        if (Args.isModeEnabled(Mode.SIMPLE)) {
+            val frameInfo = requireNotNull(averageFrameInfo) { "frameInfo shouldn't be null with Mode.SIMPLE" }
+            frameInfo.prettyPrint()
             println()
         }
-        if (Args.isModeEnabled(Mode.FRAMES)) {
+        if (Args.isModeEnabled(Mode.VSYNC_EMULATION)) {
             percentileCPUAverage.prettyPrint(BenchmarkFrameTimeKind.CPU)
             println()
-            if (Args.isModeEnabled(Mode.FRAMES_GPU)) {
-                percentileGPUAverage.prettyPrint(BenchmarkFrameTimeKind.GPU)
-                println()
-            }
+            percentileGPUAverage.prettyPrint(BenchmarkFrameTimeKind.GPU)
+            println()
             noBufferingMissedFrames.prettyPrint("no buffering")
-            if (Args.isModeEnabled(Mode.FRAMES_GPU)) {
-                doubleBufferingMissedFrames.prettyPrint("double buffering")
-            }
+            doubleBufferingMissedFrames.prettyPrint("double buffering")
+        }
+    }
+
+    fun putFormattedValuesTo(map: MutableMap<String, String>) {
+        val versionInfo = Args.versionInfo
+        if (versionInfo != null) {
+            map.put("Version", versionInfo)
+        }
+        conditions.putFormattedValuesTo(map)
+        if (Args.isModeEnabled(Mode.SIMPLE)) {
+            val frameInfo = requireNotNull(averageFrameInfo) { "frameInfo shouldn't be null with Mode.SIMPLE" }
+            frameInfo.putFormattedValuesTo(map)
+        }
+        if (Args.isModeEnabled(Mode.VSYNC_EMULATION)) {
+            percentileCPUAverage.putFormattedValuesTo(BenchmarkFrameTimeKind.CPU, map)
+            percentileGPUAverage.putFormattedValuesTo(BenchmarkFrameTimeKind.GPU, map)
+            noBufferingMissedFrames.putFormattedValuesTo("no buffering", map)
+            doubleBufferingMissedFrames.putFormattedValuesTo("double buffering", map)
         }
     }
 
     private fun List<BenchmarkPercentileAverage>.prettyPrint(kind: BenchmarkFrameTimeKind) {
         forEach {
-            println("Worst p${(it.percentile * 100).roundToInt()} ${kind.toPrettyPrintString()} average: ${it.average}")
+            println("Worst p${(it.percentile * 100).roundToInt()} ${kind.toPrettyPrintString()} (ms): ${it.average}")
+        }
+    }
+
+    private fun List<BenchmarkPercentileAverage>.putFormattedValuesTo(
+        kind: BenchmarkFrameTimeKind,
+        map: MutableMap<String, String>
+    ) {
+        forEach {
+            map.put(
+                "Worst p${(it.percentile * 100).roundToInt()} ${kind.toPrettyPrintString()} (ms)",
+                it.average.formatAsMilliseconds()
+            )
         }
     }
 }
 
 class BenchmarkResult(
     private val frameBudget: Duration,
-    private val renderTime: Duration,
+    private val conditions: BenchmarkConditions,
+    private val averageFrameInfo: FrameInfo,
     private val frames: List<BenchmarkFrame>,
 ) {
     private fun percentileAverageFrameTime(percentile: Double, kind: BenchmarkFrameTimeKind): Duration {
@@ -113,8 +193,8 @@ class BenchmarkResult(
 
         return BenchmarkStats(
             frameBudget,
-            frames.size,
-            renderTime,
+            conditions,
+            averageFrameInfo,
             listOf(0.01, 0.02, 0.05, 0.1, 0.25, 0.5).map { percentile ->
                 val average = percentileAverageFrameTime(percentile, BenchmarkFrameTimeKind.CPU)
 
@@ -137,6 +217,8 @@ class BenchmarkResult(
 
 }
 
+private fun Duration.formatAsMilliseconds(): String = (inWholeMicroseconds / 1000.0).toString()
+
 suspend fun runBenchmark(
     name: String,
     width: Int,
@@ -148,9 +230,10 @@ suspend fun runBenchmark(
     content: @Composable () -> Unit
 ) {
     if (Args.isBenchmarkEnabled(name)) {
-        println("$name:")
+        println("# $name")
         val stats = measureComposable(warmupCount, Args.getBenchmarkProblemSize(name, frameCount), width, height, targetFps, graphicsContext, content).generateStats()
         stats.prettyPrint()
+        saveBenchmarkStatsOnDisk(name, stats)
     }
 }
 
@@ -168,4 +251,4 @@ suspend fun runBenchmarks(
     runBenchmark("VisualEffects", width, height, targetFps, 1000, graphicsContext) { NYContent(width, height) }
     runBenchmark("LazyList", width, height, targetFps, 1000, graphicsContext) { MainUiNoImageUseModel()}
     runBenchmark("Example1", width, height, targetFps, 1000, graphicsContext) { Example1() }
-}
+ }

benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/BenchmarksSave.kt

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2020-2025 JetBrains s.r.o. and respective authors and developers.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
+ */
+
+import kotlinx.datetime.Clock
+import kotlinx.datetime.LocalDateTime
+import kotlinx.datetime.TimeZone
+import kotlinx.datetime.format.FormatStringsInDatetimeFormats
+import kotlinx.datetime.format.byUnicodePattern
+import kotlinx.datetime.toLocalDateTime
+import kotlinx.io.IOException
+import kotlinx.io.RawSink
+import kotlinx.io.RawSource
+import kotlinx.io.buffered
+import kotlinx.io.files.Path
+import kotlinx.io.files.SystemFileSystem
+import kotlinx.io.readByteArray
+
+fun saveBenchmarkStatsOnDisk(name: String, stats: BenchmarkStats) {
+    try {
+        val path = Path("build/benchmarks/$name.csv")
+
+        val keyToValue = mutableMapOf<String, String>()
+        keyToValue.put("Date", currentFormattedDate)
+        stats.putFormattedValuesTo(keyToValue)
+
+        var text = if (SystemFileSystem.exists(path)) {
+            SystemFileSystem.source(path).readText()
+        } else {
+            keyToValue.keys.joinToString(",") + "\n"
+        }
+
+        fun escapeForCSV(value: String) = value.replace(",", ";")
+        text += keyToValue.values.joinToString(",", transform = ::escapeForCSV) + "\n"
+
+        SystemFileSystem.createDirectories(path.parent!!)
+        SystemFileSystem.sink(path).writeText(text)
+        println("Results saved to ${SystemFileSystem.resolve(path)}")
+        println()
+    } catch (_: IOException) {
+        // IOException "Read-only file system" is thrown on iOS without writing permissions
+    } catch (_: UnsupportedOperationException) {
+        // UnsupportedOperationException is thrown in browser
+    }
+}
+
+private fun RawSource.readText() = use {
+    it.buffered().readByteArray().decodeToString()
+}
+
+private fun RawSink.writeText(text: String) = use {
+    it.buffered().apply {
+        write(text.encodeToByteArray())
+        flush()
+    }
+}
+
+@OptIn(FormatStringsInDatetimeFormats::class)
+private val currentFormattedDate: String get() {
+    val currentTime = Clock.System.now().toLocalDateTime(TimeZone.currentSystemDefault())
+    return LocalDateTime.Format {
+        byUnicodePattern("dd-MM-yyyy HH:mm:ss")
+    }.format(currentTime)
+}

benchmarks/multiplatform/benchmarks/src/commonMain/kotlin/MeasureComposable.kt

@@ -62,26 +62,26 @@ suspend fun measureComposable(
 
         runGC()
 
-        var renderTime = Duration.ZERO
-        var gpuTime = Duration.ZERO
-        if (Args.isModeEnabled(Mode.CPU)) {
-            renderTime = measureTime {
+        var cpuTotalTime = Duration.ZERO
+        var gpuTotalTime = Duration.ZERO
+        if (Args.isModeEnabled(Mode.SIMPLE)) {
+            cpuTotalTime = measureTime {
                 repeat(frameCount) {
                     scene.render(canvas, it * nanosPerFrame)
                     surface.flushAndSubmit(false)
-                    gpuTime += measureTime {
+                    gpuTotalTime += measureTime {
                         graphicsContext?.awaitGPUCompletion()
                     }
                 }
             }
-            renderTime -= gpuTime
+            cpuTotalTime -= gpuTotalTime
         }
 
         val frames = MutableList(frameCount) {
             BenchmarkFrame(Duration.INFINITE, Duration.INFINITE)
         }
 
-        if (Args.isModeEnabled(Mode.FRAMES)) {
+        if (Args.isModeEnabled(Mode.VSYNC_EMULATION)) {
 
             var nextVSync = Duration.ZERO
             var missedFrames = 0;
@@ -119,7 +119,8 @@ suspend fun measureComposable(
 
         return BenchmarkResult(
             nanosPerFrame.nanoseconds,
-            renderTime,
+            BenchmarkConditions(frameCount, warmupCount),
+            FrameInfo(cpuTotalTime / frameCount, gpuTotalTime / frameCount),
             frames
         )
     } finally {