diff --git a/package-lock.json b/package-lock.json index 378d1840..ed972e85 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11563,7 +11563,6 @@ }, "node_modules/fastest-levenshtein": { "version": "1.0.16", - "dev": true, "license": "MIT", "engines": { "node": ">= 4.9.1" @@ -18936,6 +18935,7 @@ "adm-zip": "^0.5.10", "aws-sdk": "^2.1403.0", "deepmerge": "^4.3.1", + "fastest-levenshtein": "^1.0.16", "got": "^11.8.5", "hpagent": "^1.2.0", "js-md5": "^0.8.3", diff --git a/server/aws-lsp-codewhisperer/package.json b/server/aws-lsp-codewhisperer/package.json index dc93dc6e..62393413 100644 --- a/server/aws-lsp-codewhisperer/package.json +++ b/server/aws-lsp-codewhisperer/package.json @@ -43,6 +43,7 @@ "got": "^11.8.5", "hpagent": "^1.2.0", "js-md5": "^0.8.3", + "fastest-levenshtein": "^1.0.16", "proxy-http-agent": "^1.0.1", "uuid": "^9.0.1", "vscode-uri": "^3.0.8" diff --git a/server/aws-lsp-codewhisperer/src/language-server/chat/chatController.ts b/server/aws-lsp-codewhisperer/src/language-server/chat/chatController.ts index 7b5447f7..77060841 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/chat/chatController.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/chat/chatController.ts @@ -42,12 +42,13 @@ export class ChatController implements ChatHandlers { this.#features = features this.#chatSessionManagementService = chatSessionManagementService this.#triggerContext = new QChatTriggerContext(features.workspace, features.logging) - this.#telemetryController = new ChatTelemetryController(features.credentialsProvider, features.telemetry) + this.#telemetryController = new ChatTelemetryController(features) } dispose() { this.#chatSessionManagementService.dispose() this.#triggerContext.dispose() + this.#telemetryController.dispose() } async onChatPrompt(params: ChatParams, token: CancellationToken): Promise> { @@ -58,7 +59,7 @@ export class ChatController implements ChatHandlers { if (!session) { this.#log('Get session error', params.tabId) return new ResponseError( - ErrorCodes.InvalidParams, + LSPErrorCodes.RequestFailed, 'error' in sessionResult ? sessionResult.error : 'Unknown error' ) } diff --git a/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.test.ts b/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.test.ts index 88c80d9b..57c48638 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.test.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.test.ts @@ -14,7 +14,7 @@ describe('TelemetryController', () => { beforeEach(() => { testFeatures = new TestFeatures() - telemetryController = new ChatTelemetryController(testFeatures.credentialsProvider, testFeatures.telemetry) + telemetryController = new ChatTelemetryController(testFeatures) }) it('able to set and get activeTabId', () => { diff --git a/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts b/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts index 65556fee..745a7e8c 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts @@ -8,9 +8,15 @@ import { InteractWithMessageEvent, } from '../../telemetry/types' import { Features, KeysMatching } from '../../types' -import { ChatUIEventName, RelevancyVoteType, isClientTelemetryEvent } from './clientTelemetry' +import { + ChatUIEventName, + InsertToCursorPositionParams, + RelevancyVoteType, + isClientTelemetryEvent, +} from './clientTelemetry' import { UserIntent } from '@amzn/codewhisperer-streaming' import { TriggerContext } from '../contexts/triggerContext' +import { AcceptedSuggestionEntry, CodeDiffTracker } from '../../telemetry/codeDiffTracker' export const CONVERSATION_ID_METRIC_KEY = 'cwsprChatConversationId' @@ -40,19 +46,27 @@ interface ConversationTriggerInfo { lastMessageTrigger?: MessageTrigger } +interface AcceptedSuggestionChatEntry extends AcceptedSuggestionEntry { + messageId: string +} + export class ChatTelemetryController { #activeTabId?: string #tabTelemetryInfoByTabId: { [tabId: string]: ConversationTriggerInfo } #currentTriggerByTabId: { [tabId: string]: TriggerType } = {} #credentialsProvider: Features['credentialsProvider'] #telemetry: Features['telemetry'] + #codeDiffTracker: CodeDiffTracker - constructor(credentialsProvider: Features['credentialsProvider'], telemetry: Features['telemetry']) { + constructor(features: Features) { this.#tabTelemetryInfoByTabId = {} this.#currentTriggerByTabId = {} - this.#telemetry = telemetry - this.#credentialsProvider = credentialsProvider + this.#telemetry = features.telemetry + this.#credentialsProvider = features.credentialsProvider this.#telemetry.onClientTelemetry(params => this.#handleClientTelemetry(params)) + this.#codeDiffTracker = new CodeDiffTracker(features.workspace, features.logging, (entry, percentage) => + this.emitModifyCodeMetric(entry, percentage) + ) } public get activeTabId(): string | undefined { @@ -87,6 +101,16 @@ export class ChatTelemetryController { return this.#tabTelemetryInfoByTabId[tabId]?.lastMessageTrigger } + public emitModifyCodeMetric(entry: AcceptedSuggestionChatEntry, percentage: number) { + this.emitConversationMetric({ + name: ChatTelemetryEventName.ModifyCode, + data: { + cwsprChatMessageId: entry.messageId, + cwsprChatModificationPercentage: percentage ? percentage : 0, + }, + }) + } + public emitChatMetric( metric: ChatMetricEvent ) { @@ -200,6 +224,27 @@ export class ChatTelemetryController { ) } + #enqueueCodeDiffEntry(params: InsertToCursorPositionParams) { + const documentUri = params.textDocument?.uri + const cursorRangeOrPosition = params.cursorState?.[0] + + if (params.code && documentUri && cursorRangeOrPosition) { + const startPosition = + 'position' in cursorRangeOrPosition ? cursorRangeOrPosition.position : cursorRangeOrPosition.range.start + const endPosition = + 'position' in cursorRangeOrPosition ? cursorRangeOrPosition.position : cursorRangeOrPosition.range.end + + this.#codeDiffTracker.enqueue({ + messageId: params.messageId, + fileUrl: documentUri, + time: Date.now(), + originalString: params.code, + startPosition, + endPosition, + }) + } + } + #handleClientTelemetry(params: unknown) { if (isClientTelemetryEvent(params)) { switch (params.name) { @@ -241,6 +286,10 @@ export class ChatTelemetryController { break case ChatUIEventName.InsertToCursorPosition: case ChatUIEventName.CopyToClipboard: + if (params.name === ChatUIEventName.InsertToCursorPosition) { + this.#enqueueCodeDiffEntry(params) + } + this.emitConversationMetric({ name: ChatTelemetryEventName.InteractWithMessage, data: { @@ -274,6 +323,10 @@ export class ChatTelemetryController { } } } + + public dispose() { + this.#codeDiffTracker.shutdown() + } } export function convertToTelemetryUserIntent(userIntent?: UserIntent) { diff --git a/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/clientTelemetry.ts b/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/clientTelemetry.ts index 9bc12e7a..382cb00c 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/clientTelemetry.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/clientTelemetry.ts @@ -73,7 +73,10 @@ export type SourceLinkClickParams = ServerInterface.SourceLinkClickParams & BaseClientTelemetryParams export type InsertToCursorPositionParams = ServerInterface.InsertToCursorPositionParams & - BaseClientTelemetryParams + BaseClientTelemetryParams & { + cursorState?: ServerInterface.CursorState[] + textDocument?: ServerInterface.TextDocumentIdentifier + } export type ClientTelemetryEvent = | BaseClientTelemetryParams diff --git a/server/aws-lsp-codewhisperer/src/language-server/telemetry/codeDiffTracker.test.ts b/server/aws-lsp-codewhisperer/src/language-server/telemetry/codeDiffTracker.test.ts new file mode 100644 index 00000000..f28ea010 --- /dev/null +++ b/server/aws-lsp-codewhisperer/src/language-server/telemetry/codeDiffTracker.test.ts @@ -0,0 +1,139 @@ +import { TestFeatures } from '@aws/language-server-runtimes/testing' +import { TextDocument } from 'vscode-languageserver-textdocument' +import { CodeDiffTracker } from './codeDiffTracker' +import sinon = require('sinon') +import assert = require('assert') + +describe('codeDiffTracker', () => { + let codeDiffTracker: CodeDiffTracker + let mockRecordMetric: sinon.SinonStub + const flushInterval = 1000 + const timeElapsedThreshold = 5000 + const maxQueueSize = 3 + + beforeEach(() => { + sinon.useFakeTimers({ + now: 0, + shouldAdvanceTime: false, + }) + const testFeatures = new TestFeatures() + mockRecordMetric = sinon.stub() + codeDiffTracker = new CodeDiffTracker(testFeatures.workspace, testFeatures.logging, mockRecordMetric, { + flushInterval, + timeElapsedThreshold, + maxQueueSize, + }) + testFeatures.openDocument(TextDocument.create('test.cs', 'typescript', 1, 'test')) + }) + + afterEach(() => { + sinon.clock.restore() + sinon.restore() + }) + + it('shutdown should flush the queue', async () => { + const mockEntry = { + startPosition: { + line: 0, + character: 0, + }, + endPosition: { + line: 20, + character: 0, + }, + fileUrl: 'test.cs', + // fake timer starts at 0 + time: -timeElapsedThreshold, + originalString: "console.log('test console')", + } + + // use negative time so we don't have to move the timer which would cause the interval to run + codeDiffTracker.enqueue(mockEntry) + + codeDiffTracker.enqueue({ + startPosition: { + line: 0, + character: 0, + }, + endPosition: { + line: 20, + character: 0, + }, + fileUrl: 'test.cs', + // fake timer starts at 0 + time: -timeElapsedThreshold + 100, + originalString: "console.log('test console')", + }) + + await codeDiffTracker.shutdown() + sinon.assert.calledOnce(mockRecordMetric) + sinon.assert.calledWith(mockRecordMetric, mockEntry, sinon.match.number) + }) + + it('queue should be flushed after time elapsed threshold is reached', async () => { + const mockEntry = { + startPosition: { + line: 0, + character: 0, + }, + endPosition: { + line: 20, + character: 0, + }, + fileUrl: 'test.cs', + time: 0, + originalString: "console.log('test console')", + } + codeDiffTracker.enqueue(mockEntry) + + const mockEntry2 = { + startPosition: { + line: 1, + character: 1, + }, + endPosition: { + line: 20, + character: 0, + }, + fileUrl: 'test.cs', + time: 1000, + originalString: "console.log('test console 2')", + } + codeDiffTracker.enqueue(mockEntry2) + + await sinon.clock.tickAsync(timeElapsedThreshold) + + sinon.assert.calledOnce(mockRecordMetric) + sinon.assert.calledWith(mockRecordMetric, mockEntry, sinon.match.number) + + mockRecordMetric.resetHistory() + await sinon.clock.tickAsync(flushInterval) + + sinon.assert.calledOnce(mockRecordMetric) + sinon.assert.calledWith(mockRecordMetric, mockEntry2, sinon.match.number) + }) + + it('queue does not exceed the size', async () => { + const mockEntry = { + startPosition: { + line: 0, + character: 0, + }, + endPosition: { + line: 20, + character: 0, + }, + fileUrl: 'test.cs', + time: 0, + originalString: "console.log('test console')", + } + + for (let i = 0; i < maxQueueSize + 5; i++) { + codeDiffTracker.enqueue(mockEntry) + } + + await sinon.clock.tickAsync(timeElapsedThreshold) + + assert.strictEqual(mockRecordMetric.callCount, maxQueueSize) + }) +}) diff --git a/server/aws-lsp-codewhisperer/src/language-server/telemetry/codeDiffTracker.ts b/server/aws-lsp-codewhisperer/src/language-server/telemetry/codeDiffTracker.ts new file mode 100644 index 00000000..174b51e3 --- /dev/null +++ b/server/aws-lsp-codewhisperer/src/language-server/telemetry/codeDiffTracker.ts @@ -0,0 +1,152 @@ +import { distance } from 'fastest-levenshtein' +import { Position } from '@aws/language-server-runtimes/server-interface' +import { Features } from '../types' +import { getErrorMessage } from '../utils' + +export interface AcceptedSuggestionEntry { + fileUrl: string + time: number + originalString: string + startPosition: Position + endPosition: Position +} + +export interface CodeDiffTrackerOptions { + flushInterval?: number + timeElapsedThreshold?: number + maxQueueSize?: number +} + +/** + * This class calculates the percentage of user modification after a time threshold and emits metric + * The current calculation method is (Levenshtein edit distance / acceptedSuggestion.length). + */ +export class CodeDiffTracker { + /** + * time indication the flush frequency of which the checks are + */ + private static readonly FLUSH_INTERVAL = 1000 * 60 // 1 minute + /** + * time threshold before measuring the modification after accepted into the editor + */ + private static readonly TIME_ELAPSED_THRESHOLD = 1000 * 60 * 5 // 5 minutes + private static readonly DEFAULT_MAX_QUEUE_SIZE = 10000 + + #eventQueue: T[] + #interval?: NodeJS.Timeout + #workspace: Features['workspace'] + #logging: Features['logging'] + #recordMetric: (entry: T, codeModificationPercentage: number) => void + #flushInterval: number + #timeElapsedThreshold: number + #maxQueueSize: number + + /** + * This function calculates the Levenshtein edit distance of currString from original accepted String + * then return a percentage against the length of accepted string (capped by 1) + * @param currString the current string in the same location as the previously accepted suggestion + * @param acceptedString the accepted suggestion that was inserted into the editor + */ + public static checkDiff(currString?: string, acceptedString?: string): number { + if (!currString || !acceptedString) { + return 1 + } + + const diff = distance(currString, acceptedString) + return Math.min(1, diff / acceptedString.length) + } + + constructor( + workspace: Features['workspace'], + logging: Features['logging'], + recordMetric: (entry: T, codeModificationPercentage: number) => void, + options?: CodeDiffTrackerOptions + ) { + this.#eventQueue = [] + this.#workspace = workspace + this.#logging = logging + this.#recordMetric = recordMetric + this.#flushInterval = options?.flushInterval ?? CodeDiffTracker.FLUSH_INTERVAL + this.#timeElapsedThreshold = options?.timeElapsedThreshold ?? CodeDiffTracker.TIME_ELAPSED_THRESHOLD + this.#maxQueueSize = options?.maxQueueSize ?? CodeDiffTracker.DEFAULT_MAX_QUEUE_SIZE + } + + public enqueue(suggestion: T) { + this.#eventQueue.push(suggestion) + + // remove the oldest entries if the queue if full + while (this.#eventQueue.length > this.#maxQueueSize) { + this.#eventQueue.shift() + } + + // ensure there is an active interval + this.#startInterval() + } + + public async shutdown() { + this.#clearInterval() + + try { + await this.flush() + } catch (e) { + this.#logging.log(`Error encountered while performing the final flush: ${e}`) + } + } + + private async flush() { + const newEventQueue: T[] = [] + + // emit the ones that reach the time limit and start a new queue with remaining + for (const suggestion of this.#eventQueue) { + if (Date.now() - suggestion.time >= this.#timeElapsedThreshold) { + await this.#emitTelemetryOnSuggestion(suggestion as T) + } else { + newEventQueue.push(suggestion as T) + } + } + + this.#eventQueue = newEventQueue + + // shutdown the interval when queue is empty + if (this.#eventQueue.length === 0) { + this.#clearInterval() + } + } + + async #emitTelemetryOnSuggestion(suggestion: T) { + try { + const document = suggestion.fileUrl && (await this.#workspace.getTextDocument(suggestion.fileUrl)) + + if (document) { + const currString = document.getText({ + start: suggestion.startPosition, + end: suggestion.endPosition, + }) + const percentage = CodeDiffTracker.checkDiff(currString, suggestion.originalString) + + this.#recordMetric(suggestion, percentage) + } + } catch (e) { + this.#logging.log(`Exception Thrown from CodeDiffTracker: ${e}`) + } + } + + #startInterval() { + if (!this.#interval) { + this.#interval = setInterval(async () => { + try { + await this.flush() + } catch (e) { + this.#logging.log(`flush failed: ${getErrorMessage(e)}`) + } finally { + this.#interval?.refresh() + } + }, this.#flushInterval) + } + } + + #clearInterval() { + clearInterval(this.#interval) + this.#interval = undefined + } +} diff --git a/server/aws-lsp-codewhisperer/src/language-server/telemetry/types.ts b/server/aws-lsp-codewhisperer/src/language-server/telemetry/types.ts index 459d48d9..6b7511d6 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/telemetry/types.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/telemetry/types.ts @@ -107,6 +107,7 @@ export enum ChatTelemetryEventName { AddMessage = 'amazonq_addMessage', RunCommand = 'amazonq_runCommand', MessageResponseError = 'amazonq_messageResponseError', + ModifyCode = 'amazonq_modifyCode', } export interface ChatTelemetryEventMap { @@ -119,6 +120,13 @@ export interface ChatTelemetryEventMap { [ChatTelemetryEventName.AddMessage]: AddMessageEvent [ChatTelemetryEventName.RunCommand]: RunCommandEvent [ChatTelemetryEventName.MessageResponseError]: MessageResponseErrorEvent + [ChatTelemetryEventName.ModifyCode]: ModifyCodeEvent +} + +export type ModifyCodeEvent = { + cwsprChatConversationId: string + cwsprChatMessageId: string + cwsprChatModificationPercentage: number } export type AddMessageEvent = {