Rli/cs perf test

buildSrc/src/main/kotlin/software/aws/toolkits/gradle/intellij/IdeVersions.kt

@@ -126,6 +126,7 @@ object IdeVersions {
                 ),
                 marketplacePlugins = listOf(
                     "org.toml.lang:242.20224.155",
+                    "PythonCore:242.20224.300",
                     "Pythonid:242.20224.300",
                     "org.jetbrains.plugins.go:242.20224.300",
                 )

plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererCodeFileScanTest.kt

@@ -6,7 +6,6 @@ package software.aws.toolkits.jetbrains.services.codewhisperer.codescan
 import com.intellij.openapi.fileEditor.FileEditor
 import com.intellij.openapi.fileEditor.FileEditorManager
 import com.intellij.psi.PsiFile
-import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.test.runTest
 import org.apache.commons.codec.digest.DigestUtils
 import org.assertj.core.api.Assertions.assertThat
@@ -39,7 +38,6 @@ import software.aws.toolkits.jetbrains.utils.isInstanceOfSatisfying
 import software.aws.toolkits.jetbrains.utils.rules.PythonCodeInsightTestFixtureRule
 import software.aws.toolkits.telemetry.CodewhispererLanguage
 import java.io.FileInputStream
-import java.lang.management.ManagementFactory
 import java.util.Base64
 import java.util.UUID
 import java.util.zip.ZipFile
@@ -126,106 +124,6 @@ class CodeWhispererCodeFileScanTest : CodeWhispererCodeScanTestBase(PythonCodeIn
         }
     }
 
-    @Test
-    fun `test run() - measure CPU and memory usage with payload of 200KB`() {
-        // Create a 200KB file
-        val content = "a".repeat(200 * 1024)
-        val psiFile = projectRule.fixture.addFileToProject("test.txt", content)
-
-        val sessionConfig = spy(
-            CodeScanSessionConfig.create(
-                psiFile.virtualFile,
-                project,
-                CodeWhispererConstants.CodeAnalysisScope.FILE
-            )
-        )
-        setupResponse(psiFile.virtualFile.toNioPath().relativeTo(sessionConfig.projectRoot.toNioPath()))
-        val sessionContext = CodeScanSessionContext(project, sessionConfig, CodeWhispererConstants.CodeAnalysisScope.FILE)
-        val session = spy(CodeWhispererCodeScanSession(sessionContext))
-        doNothing().whenever(session).uploadArtifactToS3(any(), any(), any(), any(), isNull(), any())
-
-        // Set up CPU and Memory monitoring
-        val runtime = Runtime.getRuntime()
-        val bean = ManagementFactory.getThreadMXBean()
-        val startCpuTime = bean.getCurrentThreadCpuTime()
-        val startMemoryUsage = runtime.totalMemory() - runtime.freeMemory()
-        val startSystemTime = System.nanoTime()
-
-        // Run the code scan
-        runBlocking {
-            session.run()
-        }
-
-        // Calculate CPU and memory usage
-        val endCpuTime = bean.getCurrentThreadCpuTime()
-        val endMemoryUsage = runtime.totalMemory() - runtime.freeMemory()
-        val endSystemTime = System.nanoTime()
-
-        val cpuTimeUsedNanos = endCpuTime - startCpuTime
-        val cpuTimeUsedSeconds = cpuTimeUsedNanos / 1_000_000_000.0
-        val elapsedTimeSeconds = (endSystemTime - startSystemTime) / 1_000_000_000.0
-
-        val memoryUsed = endMemoryUsage - startMemoryUsage
-        val memoryUsedInMB = memoryUsed / (1024.0 * 1024.0) // Converting into MB
-
-        // Calculate CPU usage in percentage
-        val cpuUsagePercentage = (cpuTimeUsedSeconds / elapsedTimeSeconds) * 100
-
-        assertThat(cpuTimeUsedSeconds).isLessThan(5.0)
-        assertThat(cpuUsagePercentage).isLessThan(30.0)
-        assertThat(memoryUsedInMB).isLessThan(200.0) // Memory used should be less than 200MB
-    }
-
-    @Test
-    fun `test run() - measure CPU and memory usage with payload of 150KB`() {
-        // Create a 150KB file
-        val codeContentForPayload = "a".repeat(150 * 1024)
-        val psiFile = projectRule.fixture.addFileToProject("test.txt", codeContentForPayload)
-
-        val sessionConfig = spy(
-            CodeScanSessionConfig.create(
-                psiFile.virtualFile,
-                project,
-                CodeWhispererConstants.CodeAnalysisScope.FILE
-            )
-        )
-        setupResponse(psiFile.virtualFile.toNioPath().relativeTo(sessionConfig.projectRoot.toNioPath()))
-        val sessionContext = CodeScanSessionContext(project, sessionConfig, CodeWhispererConstants.CodeAnalysisScope.FILE)
-        val session = spy(CodeWhispererCodeScanSession(sessionContext))
-        doNothing().whenever(session).uploadArtifactToS3(any(), any(), any(), any(), isNull(), any())
-
-        // Set up CPU and Memory monitoring
-        val runtime = Runtime.getRuntime()
-        val bean = ManagementFactory.getThreadMXBean()
-        val startCpuTime = bean.getCurrentThreadCpuTime()
-        val startMemoryUsage = runtime.totalMemory() - runtime.freeMemory()
-        val startSystemTime = System.nanoTime()
-
-        // Run the code scan
-        runBlocking {
-            session.run()
-        }
-
-        // Calculate CPU and memory usage
-        val endCpuTime = bean.getCurrentThreadCpuTime()
-        val endMemoryUsage = runtime.totalMemory() - runtime.freeMemory()
-        val endSystemTime = System.nanoTime()
-
-        val cpuTimeUsedNanos = endCpuTime - startCpuTime
-        val cpuTimeUsedSeconds = cpuTimeUsedNanos / 1_000_000_000.0
-        val elapsedTimeSeconds = (endSystemTime - startSystemTime) / 1_000_000_000.0
-
-        val memoryUsed = endMemoryUsage - startMemoryUsage
-        val memoryUsedInMB = memoryUsed / (1024.0 * 1024.0) // Converting into MB
-
-        // Calculate CPU usage in percentage
-        val cpuUsagePercentage = (cpuTimeUsedSeconds / elapsedTimeSeconds) * 100
-
-        assertThat(cpuTimeUsedSeconds).isLessThan(5.0)
-        assertThat(cpuUsagePercentage).isLessThan(30.0)
-        assertThat(memoryUsedInMB).isLessThan(200.0) // Memory used should be less than 200MB
-    }
-
     @Test
     fun `test createUploadUrlAndUpload()`() {
         val file = pyPsiFile.virtualFile.toNioPath().toFile()

plugins/amazonq/codewhisperer/jetbrains-community/tst/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererCodeScanTestBase.kt → plugins/amazonq/codewhisperer/jetbrains-community/tstFixtures/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererCodeScanTestBase.kt

@@ -1,4 +1,4 @@
-// Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 package software.aws.toolkits.jetbrains.services.codewhisperer.codescan
@@ -10,7 +10,6 @@ import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.wm.RegisterToolWindowTask
 import com.intellij.openapi.wm.ToolWindowManager
-import com.intellij.testFramework.ApplicationRule
 import com.intellij.testFramework.DisposableRule
 import com.intellij.testFramework.replaceService
 import com.intellij.util.io.systemIndependentPath
@@ -22,7 +21,6 @@ import org.junit.jupiter.api.assertThrows
 import org.mockito.kotlin.any
 import org.mockito.kotlin.doNothing
 import org.mockito.kotlin.doReturn
-import org.mockito.kotlin.isNull
 import org.mockito.kotlin.mock
 import org.mockito.kotlin.spy
 import org.mockito.kotlin.stub
@@ -46,15 +44,7 @@ import software.aws.toolkits.telemetry.CodewhispererLanguage
 import java.nio.file.Path
 import kotlin.test.assertNotNull
 
-open class CodeWhispererCodeScanTestBase(projectRule: CodeInsightTestFixtureRule) {
-    @Rule
-    @JvmField
-    val applicationRule = ApplicationRule()
-
-    @Rule
-    @JvmField
-    val projectRule: CodeInsightTestFixtureRule = projectRule
-
+open class CodeWhispererCodeScanTestBase(@Rule @JvmField val projectRule: CodeInsightTestFixtureRule) {
     @Rule
     @JvmField
     val disposableRule = DisposableRule()
@@ -67,35 +57,35 @@ open class CodeWhispererCodeScanTestBase(projectRule: CodeInsightTestFixtureRule
     @JvmField
     val wireMock = WireMockRule(WireMockConfiguration.wireMockConfig().dynamicPort())
 
-    protected lateinit var mockClient: CodeWhispererClientAdaptor
-
-    internal lateinit var s3endpoint: String
+    protected val project
+        get() = projectRule.project
 
-    internal lateinit var fakeCreateUploadUrlResponse: CreateUploadUrlResponse
-    internal lateinit var fakeCreateCodeScanResponse: CreateCodeScanResponse
-    internal lateinit var fakeCreateCodeScanResponseFailed: CreateCodeScanResponse
-    internal lateinit var fakeCreateCodeScanResponsePending: CreateCodeScanResponse
-    internal lateinit var fakeListCodeScanFindingsResponse: ListCodeScanFindingsResponse
-    internal lateinit var fakeListCodeScanFindingsResponseE2E: ListCodeScanFindingsResponse
-    internal lateinit var fakeListCodeScanFindingsOutOfBoundsIndexResponse: ListCodeScanFindingsResponse
-    internal lateinit var fakeGetCodeScanResponse: GetCodeScanResponse
-    internal lateinit var fakeGetCodeScanResponsePending: GetCodeScanResponse
-    internal lateinit var fakeGetCodeScanResponseFailed: GetCodeScanResponse
+    protected lateinit var mockClient: CodeWhispererClientAdaptor
+    protected lateinit var s3endpoint: String
+    protected lateinit var fakeCreateUploadUrlResponse: CreateUploadUrlResponse
+    protected lateinit var fakeCreateCodeScanResponse: CreateCodeScanResponse
+    protected lateinit var fakeCreateCodeScanResponseFailed: CreateCodeScanResponse
+    protected lateinit var fakeCreateCodeScanResponsePending: CreateCodeScanResponse
+    protected lateinit var fakeListCodeScanFindingsResponse: ListCodeScanFindingsResponse
+    protected lateinit var fakeListCodeScanFindingsResponseE2E: ListCodeScanFindingsResponse
+    protected lateinit var fakeListCodeScanFindingsOutOfBoundsIndexResponse: ListCodeScanFindingsResponse
+    protected lateinit var fakeGetCodeScanResponse: GetCodeScanResponse
+    protected lateinit var fakeGetCodeScanResponsePending: GetCodeScanResponse
+    protected lateinit var fakeGetCodeScanResponseFailed: GetCodeScanResponse
 
     internal val metadata: DefaultAwsResponseMetadata = DefaultAwsResponseMetadata.create(
         mapOf(AwsHeader.AWS_REQUEST_ID to CodeWhispererTestUtil.testRequestId)
     )
 
     internal lateinit var scanManagerSpy: CodeWhispererCodeScanManager
-    internal lateinit var project: Project
 
     @Before
     open fun setup() {
-        project = projectRule.project
         s3endpoint = "http://127.0.0.1:${wireMock.port()}"
 
-        scanManagerSpy = spy(CodeWhispererCodeScanManager.getInstance(project))
+        scanManagerSpy = spy(CodeWhispererCodeScanManager(project))
         doNothing().whenever(scanManagerSpy).addCodeScanUI(any())
+        projectRule.project.replaceService(CodeWhispererCodeScanManager::class.java, scanManagerSpy, disposableRule.disposable)
 
         mockClient = mock<CodeWhispererClientAdaptor>().also {
             project.replaceService(CodeWhispererClientAdaptor::class.java, it, disposableRule.disposable)
@@ -197,7 +187,7 @@ open class CodeWhispererCodeScanTestBase(projectRule: CodeInsightTestFixtureRule
         filePath,
         99999,
         99999,
-        kotlin.collections.listOf(
+        listOf(
             1 to "import numpy as np",
             2 to "               import from module1 import helper"
         )
@@ -334,9 +324,20 @@ open class CodeWhispererCodeScanTestBase(projectRule: CodeInsightTestFixtureRule
         expectedTotalSize: Long,
         expectedTotalIssues: Int
     ) {
-        val codeScanContext = CodeScanSessionContext(project, sessionConfigSpy, CodeWhispererConstants.CodeAnalysisScope.PROJECT)
+        val codeScanContext = CodeScanSessionContext(
+            project,
+            sessionConfigSpy,
+            CodeWhispererConstants.CodeAnalysisScope.PROJECT
+        )
         val sessionMock = spy(CodeWhispererCodeScanSession(codeScanContext))
-        doNothing().`when`(sessionMock).uploadArtifactToS3(any(), any(), any(), any(), isNull(), any())
+        doNothing().whenever(sessionMock).uploadArtifactToS3(
+            any(),
+            any(),
+            any(),
+            any(),
+            org.mockito.kotlin.isNull(),
+            any()
+        )
 
         ToolWindowManager.getInstance(project).registerToolWindow(
             RegisterToolWindowTask(

plugins/amazonq/codewhisperer/jetbrains-ultimate/build.gradle.kts

@@ -1,7 +1,11 @@
 // Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import org.jetbrains.intellij.platform.gradle.Constants.Tasks
+import org.jetbrains.intellij.platform.gradle.TestFrameworkType
+import org.jetbrains.intellij.platform.gradle.tasks.PrepareTestTask
 import software.aws.toolkits.gradle.intellij.IdeFlavor
+import software.aws.toolkits.gradle.withCurrentProfileName
 
 plugins {
     id("toolkit-intellij-subplugin")
@@ -13,6 +17,18 @@ intellijToolkit {
 
 dependencies {
     intellijPlatform {
+        // FIX_WHEN_MIN_IS_242
+        withCurrentProfileName {
+            when (it) {
+                "2023.3", "2024.1" -> {
+                    // not available
+                }
+                else -> {
+                    testFramework(TestFrameworkType.Metrics)
+                }
+            }
+        }
+
         localPlugin(project(":plugin-core"))
     }
 
@@ -24,3 +40,9 @@ dependencies {
     testImplementation(testFixtures(project(":plugin-amazonq:codewhisperer:jetbrains-community")))
     testImplementation(project(path = ":plugin-toolkit:jetbrains-ultimate", configuration = "testArtifacts"))
 }
+
+tasks.test {
+    // custom test tasks can retrieve platformPath directly
+    val platformPath = project.tasks.named<PrepareTestTask>(Tasks.PREPARE_TEST).get().platformPath
+    systemProperty("idea.async.profiler.agent.path", platformPath.resolve("lib/async-profiler/libasyncProfiler.dylib").toString())
+}

plugins/amazonq/codewhisperer/jetbrains-ultimate/tst-242+/software/aws/toolkits/jetbrains/services/codewhisperer/codescan/CodeWhispererCodeFileScanPerfTest.kt

@@ -0,0 +1,73 @@
+// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package software.aws.toolkits.jetbrains.services.codewhisperer.codescan
+
+import com.intellij.tools.ide.metrics.benchmark.PerformanceTestUtil
+import com.intellij.tools.ide.metrics.collector.OpenTelemetryJsonMeterCollector
+import com.intellij.tools.ide.metrics.collector.metrics.MetricsSelectionStrategy
+import kotlinx.coroutines.test.runTest
+import org.junit.Test
+import org.mockito.kotlin.any
+import org.mockito.kotlin.doNothing
+import org.mockito.kotlin.isNull
+import org.mockito.kotlin.spy
+import org.mockito.kotlin.whenever
+import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.sessionconfig.CodeScanSessionConfig
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants
+import software.aws.toolkits.jetbrains.utils.rules.PythonCodeInsightTestFixtureRule
+import kotlin.io.path.relativeTo
+
+class CodeWhispererCodeFileScanPerfTest : CodeWhispererCodeScanTestBase(PythonCodeInsightTestFixtureRule()) {
+    @Test
+    fun `test run() - measure CPU and memory usage with payload of 200KB`() {
+        // Create a 200KB file
+        val content = "a".repeat(200 * 1024)
+        val psiFile = projectRule.fixture.addFileToProject("test.txt", content)
+
+        val sessionConfig = spy(
+            CodeScanSessionConfig.create(
+                psiFile.virtualFile,
+                project,
+                CodeWhispererConstants.CodeAnalysisScope.FILE
+            )
+        )
+        setupResponse(psiFile.virtualFile.toNioPath().relativeTo(sessionConfig.projectRoot.toNioPath()))
+        val sessionContext = CodeScanSessionContext(project, sessionConfig, CodeWhispererConstants.CodeAnalysisScope.FILE)
+        val session = spy(CodeWhispererCodeScanSession(sessionContext))
+        doNothing().whenever(session).uploadArtifactToS3(any(), any(), any(), any(), isNull(), any())
+
+        PerformanceTestUtil.newPerformanceTest("scan") {
+            runTest {
+                session.run()
+            }
+        }.withMetricsCollector(OpenTelemetryJsonMeterCollector(MetricsSelectionStrategy.SUM) { true })
+            .start()
+    }
+
+    @Test
+    fun `test run() - measure CPU and memory usage with payload of 150KB`() {
+        // Create a 150KB file
+        val content = "a".repeat(150 * 1024)
+        val psiFile = projectRule.fixture.addFileToProject("test.txt", content)
+
+        val sessionConfig = spy(
+            CodeScanSessionConfig.create(
+                psiFile.virtualFile,
+                project,
+                CodeWhispererConstants.CodeAnalysisScope.FILE
+            )
+        )
+        setupResponse(psiFile.virtualFile.toNioPath().relativeTo(sessionConfig.projectRoot.toNioPath()))
+        val sessionContext = CodeScanSessionContext(project, sessionConfig, CodeWhispererConstants.CodeAnalysisScope.FILE)
+        val session = spy(CodeWhispererCodeScanSession(sessionContext))
+        doNothing().whenever(session).uploadArtifactToS3(any(), any(), any(), any(), isNull(), any())
+
+        PerformanceTestUtil.newPerformanceTest("scan") {
+            runTest {
+                session.run()
+            }
+        }.withMetricsCollector(OpenTelemetryJsonMeterCollector(MetricsSelectionStrategy.SUM) { true })
+            .start()
+    }
+}