Add Chat modify code telemetry

server/aws-lsp-codewhisperer/package.json

@@ -43,6 +43,7 @@
         "got": "^11.8.5",
         "hpagent": "^1.2.0",
         "js-md5": "^0.8.3",
+        "fastest-levenshtein": "^1.0.16",
         "proxy-http-agent": "^1.0.1",
         "uuid": "^9.0.1",
         "vscode-uri": "^3.0.8"

server/aws-lsp-codewhisperer/src/language-server/chat/chatController.ts

@@ -42,12 +42,13 @@ export class ChatController implements ChatHandlers {
         this.#features = features
         this.#chatSessionManagementService = chatSessionManagementService
         this.#triggerContext = new QChatTriggerContext(features.workspace, features.logging)
-        this.#telemetryController = new ChatTelemetryController(features.credentialsProvider, features.telemetry)
+        this.#telemetryController = new ChatTelemetryController(features)
     }
 
     dispose() {
         this.#chatSessionManagementService.dispose()
         this.#triggerContext.dispose()
+        this.#telemetryController.dispose()
     }
 
     async onChatPrompt(params: ChatParams, token: CancellationToken): Promise<ChatResult | ResponseError<ChatResult>> {
@@ -58,7 +59,7 @@ export class ChatController implements ChatHandlers {
         if (!session) {
             this.#log('Get session error', params.tabId)
             return new ResponseError<ChatResult>(
-                ErrorCodes.InvalidParams,
+                LSPErrorCodes.RequestFailed,
                 'error' in sessionResult ? sessionResult.error : 'Unknown error'
             )
         }

server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.test.ts

@@ -14,7 +14,7 @@ describe('TelemetryController', () => {
 
     beforeEach(() => {
         testFeatures = new TestFeatures()
-        telemetryController = new ChatTelemetryController(testFeatures.credentialsProvider, testFeatures.telemetry)
+        telemetryController = new ChatTelemetryController(testFeatures)
     })
 
     it('able to set and get activeTabId', () => {

server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts

@@ -8,9 +8,15 @@ import {
     InteractWithMessageEvent,
 } from '../../telemetry/types'
 import { Features, KeysMatching } from '../../types'
-import { ChatUIEventName, RelevancyVoteType, isClientTelemetryEvent } from './clientTelemetry'
+import {
+    ChatUIEventName,
+    InsertToCursorPositionParams,
+    RelevancyVoteType,
+    isClientTelemetryEvent,
+} from './clientTelemetry'
 import { UserIntent } from '@amzn/codewhisperer-streaming'
 import { TriggerContext } from '../contexts/triggerContext'
+import { AcceptedSuggestionEntry, CodeDiffTracker } from '../../telemetry/codeDiffTracker'
 
 export const CONVERSATION_ID_METRIC_KEY = 'cwsprChatConversationId'
 
@@ -40,19 +46,27 @@ interface ConversationTriggerInfo {
     lastMessageTrigger?: MessageTrigger
 }
 
+interface AcceptedSuggestionChatEntry extends AcceptedSuggestionEntry {
+    messageId: string
+}
+
 export class ChatTelemetryController {
     #activeTabId?: string
     #tabTelemetryInfoByTabId: { [tabId: string]: ConversationTriggerInfo }
     #currentTriggerByTabId: { [tabId: string]: TriggerType } = {}
     #credentialsProvider: Features['credentialsProvider']
     #telemetry: Features['telemetry']
+    #codeDiffTracker: CodeDiffTracker<AcceptedSuggestionChatEntry>
 
-    constructor(credentialsProvider: Features['credentialsProvider'], telemetry: Features['telemetry']) {
+    constructor(features: Features) {
         this.#tabTelemetryInfoByTabId = {}
         this.#currentTriggerByTabId = {}
-        this.#telemetry = telemetry
-        this.#credentialsProvider = credentialsProvider
+        this.#telemetry = features.telemetry
+        this.#credentialsProvider = features.credentialsProvider
         this.#telemetry.onClientTelemetry(params => this.#handleClientTelemetry(params))
+        this.#codeDiffTracker = new CodeDiffTracker(features.workspace, features.logging, (entry, percentage) =>
+            this.emitModifyCodeMetric(entry, percentage)
+        )
     }
 
     public get activeTabId(): string | undefined {
@@ -87,6 +101,16 @@ export class ChatTelemetryController {
         return this.#tabTelemetryInfoByTabId[tabId]?.lastMessageTrigger
     }
 
+    public emitModifyCodeMetric(entry: AcceptedSuggestionChatEntry, percentage: number) {
+        this.emitConversationMetric({
+            name: ChatTelemetryEventName.ModifyCode,
+            data: {
+                cwsprChatMessageId: entry.messageId,
+                cwsprChatModificationPercentage: percentage ? percentage : 0,
+            },
+        })
+    }
+
     public emitChatMetric<TName extends ChatTelemetryEventName>(
         metric: ChatMetricEvent<TName, ChatTelemetryEventMap[TName]>
     ) {
@@ -200,6 +224,27 @@ export class ChatTelemetryController {
         )
     }
 
+    #enqueueCodeDiffEntry(params: InsertToCursorPositionParams) {
+        const documentUri = params.textDocument?.uri
+        const cursorRangeOrPosition = params.cursorState?.[0]
+
+        if (params.code && documentUri && cursorRangeOrPosition) {
+            const startPosition =
+                'position' in cursorRangeOrPosition ? cursorRangeOrPosition.position : cursorRangeOrPosition.range.start
+            const endPosition =
+                'position' in cursorRangeOrPosition ? cursorRangeOrPosition.position : cursorRangeOrPosition.range.end
+
+            this.#codeDiffTracker.enqueue({
+                messageId: params.messageId,
+                fileUrl: documentUri,
+                time: Date.now(),
+                originalString: params.code,
+                startPosition,
+                endPosition,
+            })
+        }
+    }
+
     #handleClientTelemetry(params: unknown) {
         if (isClientTelemetryEvent(params)) {
             switch (params.name) {
@@ -241,6 +286,10 @@ export class ChatTelemetryController {
                     break
                 case ChatUIEventName.InsertToCursorPosition:
                 case ChatUIEventName.CopyToClipboard:
+                    if (params.name === ChatUIEventName.InsertToCursorPosition) {
+                        this.#enqueueCodeDiffEntry(params)
+                    }
+
                     this.emitConversationMetric({
                         name: ChatTelemetryEventName.InteractWithMessage,
                         data: {
@@ -274,6 +323,10 @@ export class ChatTelemetryController {
             }
         }
     }
+
+    public dispose() {
+        this.#codeDiffTracker.shutdown()
+    }
 }
 
 export function convertToTelemetryUserIntent(userIntent?: UserIntent) {

server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/clientTelemetry.ts

@@ -73,7 +73,10 @@ export type SourceLinkClickParams = ServerInterface.SourceLinkClickParams &
     BaseClientTelemetryParams<ChatUIEventName.SourceLinkClick>
 
 export type InsertToCursorPositionParams = ServerInterface.InsertToCursorPositionParams &
-    BaseClientTelemetryParams<ChatUIEventName.InsertToCursorPosition>
+    BaseClientTelemetryParams<ChatUIEventName.InsertToCursorPosition> & {
+        cursorState?: ServerInterface.CursorState[]
+        textDocument?: ServerInterface.TextDocumentIdentifier
+    }
 
 export type ClientTelemetryEvent =
     | BaseClientTelemetryParams<ChatUIEventName.EnterFocusChat>

server/aws-lsp-codewhisperer/src/language-server/telemetry/codeDiffTracker.test.ts

@@ -0,0 +1,139 @@
+import { TestFeatures } from '@aws/language-server-runtimes/testing'
+import { TextDocument } from 'vscode-languageserver-textdocument'
+import { CodeDiffTracker } from './codeDiffTracker'
+import sinon = require('sinon')
+import assert = require('assert')
+
+describe('codeDiffTracker', () => {
+    let codeDiffTracker: CodeDiffTracker
+    let mockRecordMetric: sinon.SinonStub
+    const flushInterval = 1000
+    const timeElapsedThreshold = 5000
+    const maxQueueSize = 3
+
+    beforeEach(() => {
+        sinon.useFakeTimers({
+            now: 0,
+            shouldAdvanceTime: false,
+        })
+        const testFeatures = new TestFeatures()
+        mockRecordMetric = sinon.stub()
+        codeDiffTracker = new CodeDiffTracker(testFeatures.workspace, testFeatures.logging, mockRecordMetric, {
+            flushInterval,
+            timeElapsedThreshold,
+            maxQueueSize,
+        })
+        testFeatures.openDocument(TextDocument.create('test.cs', 'typescript', 1, 'test'))
+    })
+
+    afterEach(() => {
+        sinon.clock.restore()
+        sinon.restore()
+    })
+
+    it('shutdown should flush the queue', async () => {
+        const mockEntry = {
+            startPosition: {
+                line: 0,
+                character: 0,
+            },
+            endPosition: {
+                line: 20,
+                character: 0,
+            },
+            fileUrl: 'test.cs',
+            // fake timer starts at 0
+            time: -timeElapsedThreshold,
+            originalString: "console.log('test console')",
+        }
+
+        // use negative time so we don't have to move the timer which would cause the interval to run
+        codeDiffTracker.enqueue(mockEntry)
+
+        codeDiffTracker.enqueue({
+            startPosition: {
+                line: 0,
+                character: 0,
+            },
+            endPosition: {
+                line: 20,
+                character: 0,
+            },
+            fileUrl: 'test.cs',
+            // fake timer starts at 0
+            time: -timeElapsedThreshold + 100,
+            originalString: "console.log('test console')",
+        })
+
+        await codeDiffTracker.shutdown()
+        sinon.assert.calledOnce(mockRecordMetric)
+        sinon.assert.calledWith(mockRecordMetric, mockEntry, sinon.match.number)
+    })
+
+    it('queue should be flushed after time elapsed threshold is reached', async () => {
+        const mockEntry = {
+            startPosition: {
+                line: 0,
+                character: 0,
+            },
+            endPosition: {
+                line: 20,
+                character: 0,
+            },
+            fileUrl: 'test.cs',
+            time: 0,
+            originalString: "console.log('test console')",
+        }
+        codeDiffTracker.enqueue(mockEntry)
+
+        const mockEntry2 = {
+            startPosition: {
+                line: 1,
+                character: 1,
+            },
+            endPosition: {
+                line: 20,
+                character: 0,
+            },
+            fileUrl: 'test.cs',
+            time: 1000,
+            originalString: "console.log('test console 2')",
+        }
+        codeDiffTracker.enqueue(mockEntry2)
+
+        await sinon.clock.tickAsync(timeElapsedThreshold)
+
+        sinon.assert.calledOnce(mockRecordMetric)
+        sinon.assert.calledWith(mockRecordMetric, mockEntry, sinon.match.number)
+
+        mockRecordMetric.resetHistory()
+        await sinon.clock.tickAsync(flushInterval)
+
+        sinon.assert.calledOnce(mockRecordMetric)
+        sinon.assert.calledWith(mockRecordMetric, mockEntry2, sinon.match.number)
+    })
+
+    it('queue does not exceed the size', async () => {
+        const mockEntry = {
+            startPosition: {
+                line: 0,
+                character: 0,
+            },
+            endPosition: {
+                line: 20,
+                character: 0,
+            },
+            fileUrl: 'test.cs',
+            time: 0,
+            originalString: "console.log('test console')",
+        }
+
+        for (let i = 0; i < maxQueueSize + 5; i++) {
+            codeDiffTracker.enqueue(mockEntry)
+        }
+
+        await sinon.clock.tickAsync(timeElapsedThreshold)
+
+        assert.strictEqual(mockRecordMetric.callCount, maxQueueSize)
+    })
+})