diff --git a/biome.jsonc b/biome.jsonc index 55ae36d57976..20f061f468d0 100644 --- a/biome.jsonc +++ b/biome.jsonc @@ -84,7 +84,8 @@ "**/fixtures/**", "!vscode/test/fixtures/mock-server.ts", "vscode/.schema-cache/*.json", - "vscode/e2e/utils/vscody/resources/**/*" + "vscode/e2e/utils/vscody/resources/**/*", + "*.snap.json" ] }, "overrides": [ diff --git a/lib/shared/package.json b/lib/shared/package.json index 4933ec617b75..5f8a64a75848 100644 --- a/lib/shared/package.json +++ b/lib/shared/package.json @@ -44,6 +44,7 @@ "@types/lodash": "^4.14.195", "@types/node-fetch": "^2.6.4", "@types/semver": "^7.5.0", - "@types/vscode": "^1.80.0" + "@types/vscode": "^1.80.0", + "type-fest": "^4.25.0" } } diff --git a/lib/shared/src/index.ts b/lib/shared/src/index.ts index 184536b7cd1a..2770c4568951 100644 --- a/lib/shared/src/index.ts +++ b/lib/shared/src/index.ts @@ -256,6 +256,7 @@ export { } from './telemetry-v2/TelemetryRecorderProvider' export type { TelemetryRecorder } from './telemetry-v2/TelemetryRecorderProvider' export * from './telemetry-v2/singleton' +export { events as telemetryEvents } from './telemetry-v2/events' export { testFileUri } from './test/path-helpers' export * from './tracing' export { diff --git a/lib/shared/src/telemetry-v2/TelemetryRecorderProvider.ts b/lib/shared/src/telemetry-v2/TelemetryRecorderProvider.ts index f24cde700ea4..249125cdf586 100644 --- a/lib/shared/src/telemetry-v2/TelemetryRecorderProvider.ts +++ b/lib/shared/src/telemetry-v2/TelemetryRecorderProvider.ts @@ -88,7 +88,7 @@ export class TelemetryRecorderProvider extends BaseTelemetryRecorderProvider< ], { ...defaultEventRecordingOptions, - bufferTimeMs: 0, // disable buffering for now + bufferTimeMs: 0, // disable buffering for now. If this is enabled make sure that it can be disabled during tests. } ) } diff --git a/lib/shared/src/telemetry-v2/events/atMention.ts b/lib/shared/src/telemetry-v2/events/atMention.ts new file mode 100644 index 000000000000..5cab44e7426a --- /dev/null +++ b/lib/shared/src/telemetry-v2/events/atMention.ts @@ -0,0 +1,38 @@ +import { telemetryRecorder } from '../singleton' +import { type Unmapped, event, fallbackValue, pickDefined } from './internal' + +export const events = [ + event( + 'cody.at-mention/selected', + ({ map, maps, action, feature }) => + ( + source: Unmapped, + provider: Unmapped | undefined | null = undefined, + providerMetadata: { id?: string } | undefined = undefined + ) => { + telemetryRecorder.recordEvent(feature, action, { + metadata: pickDefined({ + source: map.source(source), + provider: + provider !== undefined ? map.provider(provider ?? fallbackValue) : undefined, + }), + privateMetadata: { source, provider, providerMetadata }, + billingMetadata: { + product: 'cody', + category: 'core', + }, + }) + }, + { + source: { chat: 1 }, + provider: { + [fallbackValue]: 0, + file: 1, + symbol: 2, + repo: 3, + remoteRepo: 4, + openctx: 5, + }, + } + ), +] as const diff --git a/lib/shared/src/telemetry-v2/events/index.ts b/lib/shared/src/telemetry-v2/events/index.ts new file mode 100644 index 000000000000..0d8a8fb71a15 --- /dev/null +++ b/lib/shared/src/telemetry-v2/events/index.ts @@ -0,0 +1,12 @@ +import { events as atMentionEvents } from './atMention' + +export const events = objectFromPairs([...atMentionEvents]) + +// Function to create an object from pairs with correct typing +type FromPairs = { + [K in T[number][0]]: Extract[1] +} + +function objectFromPairs(pairs: T): FromPairs { + return Object.fromEntries(pairs) as FromPairs +} diff --git a/lib/shared/src/telemetry-v2/events/internal.ts b/lib/shared/src/telemetry-v2/events/internal.ts new file mode 100644 index 000000000000..70ce43a4e4c5 --- /dev/null +++ b/lib/shared/src/telemetry-v2/events/internal.ts @@ -0,0 +1,105 @@ +import type { LiteralUnion } from 'type-fest' +declare const fallbackVariant: unique symbol +// we disguise the fallbackValue as a tagged string so that it can be exported as a type +export const fallbackValue = '__fallback__' as const +export type MapperInput = { [key: string]: number } & ( + | { [fallbackValue]: number } + | { [fallbackValue]?: never } +) + +export type MapperInputs = Record + +type ObjWithFallback> = T & { + [fallbackValue]?: number +} +type KeyOfOmitFallback = T extends ObjWithFallback ? keyof U : never + +type HasFallback = M extends { [fallbackValue]: infer V } + ? V extends number + ? true + : false + : false +export type MapperFn = HasFallback extends true + ? (v: LiteralUnion, string>) => number + : < + V extends LiteralUnion, string> = LiteralUnion< + KeyOfOmitFallback, + string + >, + >( + v: V + ) => V extends KeyOfOmitFallback ? number : null + +export type MapperFns = { + [K in keyof M]: MapperFn +} +export type Unmapped = Strict extends true + ? KeyOfOmitFallback + : LiteralUnion, string> + +type SplitSignature = string extends S + ? string[] + : S extends '' + ? [] + : S extends `${infer T}${D}${infer U}` + ? [T, ...SplitSignature] + : [S] + +function splitSignature(sig: S): SplitSignature { + return sig.split('/') as SplitSignature +} + +export function event< + Signature extends `${string}/${string}`, + M extends MapperInputs, + Args extends readonly unknown[], +>( + featureAction: Signature, + builder: (ctx: { + maps: M + map: MapperFns + feature: SplitSignature[0] + action: SplitSignature[1] + }) => (...args: Args) => void, + maps: M +) { + const [feature, action] = splitSignature(featureAction) + //TODO: Make type-safe + const map = new Proxy(maps, { + get(target, p) { + const mapping = target[p as keyof M] + return (v: any) => mapping[v] ?? mapping[fallbackValue] ?? null + }, + }) as unknown as MapperFns + const ctx = { + map, + maps, + featureAction, + feature, + action, + } + const out = { + ctx, + record: builder(ctx), + } as const + return [featureAction, out] as const +} + +type PickDefined = { + [K in keyof T]-?: T[K] extends undefined ? never : T[K] extends infer U | undefined ? U : T[K] +} + +/** + * Only omit undefined keys. Null keys are not omitted so that the mappers still + * give type errors in case no default value is provided + */ +export function pickDefined(obj: T): PickDefined { + const result: any = {} + for (const key in obj) { + const value = obj[key] + if (value !== undefined) { + result[key] = value + } + } + return result +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 88891dc4de01..639a9fe08af5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -414,6 +414,9 @@ importers: '@types/vscode': specifier: ^1.80.0 version: 1.80.0 + type-fest: + specifier: ^4.25.0 + version: 4.25.0 vscode: dependencies: @@ -784,6 +787,9 @@ importers: http-proxy-middleware: specifier: ^3.0.0 version: 3.0.0 + immer: + specifier: ^10.1.1 + version: 10.1.1 mocha: specifier: ^10.2.0 version: 10.2.0 @@ -10389,6 +10395,10 @@ packages: resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==} dev: true + /immer@10.1.1: + resolution: {integrity: sha512-s2MPrmjovJcoMaHtx6K11Ra7oD05NT97w1IC5zpMkT6Atjr7H8LjaDd81iIxUYpMKSRRNMJE703M1Fhr/TctHw==} + dev: true + /import-fresh@3.3.0: resolution: {integrity: sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==} engines: {node: '>=6'} diff --git a/recordings/vscode/e2e/telemetry/at-mention/_execute-should-not-fire-pre-maturely_4137879082/recording.har.yaml b/recordings/vscode/e2e/telemetry/at-mention/_execute-should-not-fire-pre-maturely_4137879082/recording.har.yaml new file mode 100644 index 000000000000..48319129200d --- /dev/null +++ b/recordings/vscode/e2e/telemetry/at-mention/_execute-should-not-fire-pre-maturely_4137879082/recording.har.yaml @@ -0,0 +1,929 @@ +log: + _recordingName: "`_execute` should not fire pre-maturely" + creator: + comment: persister:fs + name: Polly.JS + version: 6.0.6 + entries: + - _id: d28b5897886b4e3e674b540a038c3cb4 + _order: 0 + cache: {} + request: + bodySize: 0 + cookies: [] + headers: + - name: connection + value: close + - name: host + value: sourcegraph.com + - name: accept-encoding + value: identity + - name: user-agent + value: node-fetch/1.0 (+https://github.com/bitinn/node-fetch) + - name: accept + value: "*/*" + - name: content-type + value: application/json; charset=utf-8 + - name: authorization + value: token + REDACTED_5313821e901984ba7bbf1999deafb5f6f722c05ed13b6dfa6cffbd6128089e39 + - name: x-mitm-proxy-name + value: sourcegraph.dotcom + - name: x-mitm-auth-token-name + value: sourcegraph.dotcom + headersSize: 586 + httpVersion: HTTP/1.1 + method: GET + queryString: [] + url: https://sourcegraph.com/.api/client-config + response: + bodySize: 224 + content: + mimeType: text/plain; charset=utf-8 + size: 224 + text: | + { + "codyEnabled": true, + "chatEnabled": true, + "autoCompleteEnabled": true, + "customCommandsEnabled": true, + "attributionEnabled": false, + "smartContextWindowEnabled": true, + "modelsAPIEnabled": false + } + cookies: [] + headers: + - name: date + value: Fri, 13 Sep 2024 16:27:55 GMT + - name: content-type + value: text/plain; charset=utf-8 + - name: content-length + value: "224" + - name: connection + value: close + - name: access-control-allow-credentials + value: "true" + - name: access-control-allow-origin + value: "" + - name: cache-control + value: no-cache, max-age=0 + - name: vary + value: Cookie,Accept-Encoding,Authorization,Cookie, Authorization, + X-Requested-With,Cookie + - name: x-content-type-options + value: nosniff + - name: x-frame-options + value: DENY + - name: x-xss-protection + value: 1; mode=block + - name: strict-transport-security + value: max-age=31536000; includeSubDomains; preload + headersSize: 1306 + httpVersion: HTTP/1.1 + redirectURL: "" + status: 200 + statusText: OK + startedDateTime: 2024-09-13T16:27:55.355Z + time: 228 + timings: + blocked: -1 + connect: -1 + dns: -1 + receive: 0 + send: 0 + ssl: -1 + wait: 228 + - _id: 28c346c4c160958f92e8f989a1737e5e + _order: 0 + cache: {} + request: + bodySize: 318 + cookies: [] + headers: + - name: connection + value: close + - name: host + value: sourcegraph.com + - name: accept-encoding + value: identity + - name: user-agent + value: node-fetch/1.0 (+https://github.com/bitinn/node-fetch) + - name: content-length + value: "318" + - name: accept + value: "*/*" + - name: content-type + value: application/json; charset=utf-8 + - name: authorization + value: token + REDACTED_5313821e901984ba7bbf1999deafb5f6f722c05ed13b6dfa6cffbd6128089e39 + - name: x-mitm-proxy-name + value: sourcegraph.dotcom + - name: x-mitm-auth-token-name + value: sourcegraph.dotcom + headersSize: 561 + httpVersion: HTTP/1.1 + method: POST + postData: + mimeType: application/json; charset=utf-8 + params: [] + textJSON: + query: |- + + query CurrentSiteCodyLlmConfiguration { + site { + codyLLMConfiguration { + chatModel + chatModelMaxTokens + fastChatModel + fastChatModelMaxTokens + completionModel + completionModelMaxTokens + } + } + } + variables: {} + queryString: + - name: CurrentSiteCodyLlmConfiguration + value: null + url: https://sourcegraph.com/.api/graphql?CurrentSiteCodyLlmConfiguration + response: + bodySize: 293 + content: + mimeType: application/json + size: 293 + text: "{\"data\":{\"site\":{\"codyLLMConfiguration\":{\"chatModel\":\"anthropic\ + /claude-3-sonnet-20240229\",\"chatModelMaxTokens\":12000,\"fastChat\ + Model\":\"anthropic/claude-3-haiku-20240307\",\"fastChatModelMaxTok\ + ens\":12000,\"completionModel\":\"fireworks/deepseek-coder-v2-lite-\ + base\",\"completionModelMaxTokens\":9000}}}}" + cookies: [] + headers: + - name: date + value: Fri, 13 Sep 2024 16:27:54 GMT + - name: content-type + value: application/json + - name: content-length + value: "293" + - name: connection + value: close + - name: access-control-allow-credentials + value: "true" + - name: access-control-allow-origin + value: "" + - name: cache-control + value: no-cache, max-age=0 + - name: vary + value: Cookie,Accept-Encoding,Authorization,Cookie, Authorization, + X-Requested-With,Cookie + - name: x-content-type-options + value: nosniff + - name: x-frame-options + value: DENY + - name: x-xss-protection + value: 1; mode=block + - name: strict-transport-security + value: max-age=31536000; includeSubDomains; preload + headersSize: 1297 + httpVersion: HTTP/1.1 + redirectURL: "" + status: 200 + statusText: OK + startedDateTime: 2024-09-13T16:27:54.529Z + time: 231 + timings: + blocked: -1 + connect: -1 + dns: -1 + receive: 0 + send: 0 + ssl: -1 + wait: 231 + - _id: a4d641ac3093a4efbffe59352b351757 + _order: 0 + cache: {} + request: + bodySize: 165 + cookies: [] + headers: + - name: connection + value: close + - name: host + value: sourcegraph.com + - name: accept-encoding + value: identity + - name: user-agent + value: node-fetch/1.0 (+https://github.com/bitinn/node-fetch) + - name: content-length + value: "165" + - name: accept + value: "*/*" + - name: content-type + value: application/json; charset=utf-8 + - name: authorization + value: token + REDACTED_5313821e901984ba7bbf1999deafb5f6f722c05ed13b6dfa6cffbd6128089e39 + - name: x-mitm-proxy-name + value: sourcegraph.dotcom + - name: x-mitm-auth-token-name + value: sourcegraph.dotcom + headersSize: 561 + httpVersion: HTTP/1.1 + method: POST + postData: + mimeType: application/json; charset=utf-8 + params: [] + textJSON: + query: |- + + query CurrentSiteCodyLlmConfiguration { + site { + codyLLMConfiguration { + smartContextWindow + } + } + } + variables: {} + queryString: + - name: CurrentSiteCodyLlmConfiguration + value: null + url: https://sourcegraph.com/.api/graphql?CurrentSiteCodyLlmConfiguration + response: + bodySize: 75 + content: + mimeType: application/json + size: 75 + text: "{\"data\":{\"site\":{\"codyLLMConfiguration\":{\"smartContextWindow\":\"\ + enabled\"}}}}" + cookies: [] + headers: + - name: date + value: Fri, 13 Sep 2024 16:27:54 GMT + - name: content-type + value: application/json + - name: content-length + value: "75" + - name: connection + value: close + - name: access-control-allow-credentials + value: "true" + - name: access-control-allow-origin + value: "" + - name: cache-control + value: no-cache, max-age=0 + - name: vary + value: Cookie,Accept-Encoding,Authorization,Cookie, Authorization, + X-Requested-With,Cookie + - name: x-content-type-options + value: nosniff + - name: x-frame-options + value: DENY + - name: x-xss-protection + value: 1; mode=block + - name: strict-transport-security + value: max-age=31536000; includeSubDomains; preload + headersSize: 1296 + httpVersion: HTTP/1.1 + redirectURL: "" + status: 200 + statusText: OK + startedDateTime: 2024-09-13T16:27:54.532Z + time: 234 + timings: + blocked: -1 + connect: -1 + dns: -1 + receive: 0 + send: 0 + ssl: -1 + wait: 234 + - _id: 06568f66a76daea32430b82552305046 + _order: 0 + cache: {} + request: + bodySize: 150 + cookies: [] + headers: + - name: connection + value: close + - name: host + value: sourcegraph.com + - name: accept-encoding + value: identity + - name: user-agent + value: node-fetch/1.0 (+https://github.com/bitinn/node-fetch) + - name: content-length + value: "150" + - name: accept + value: "*/*" + - name: content-type + value: application/json; charset=utf-8 + - name: authorization + value: token + REDACTED_5313821e901984ba7bbf1999deafb5f6f722c05ed13b6dfa6cffbd6128089e39 + - name: x-mitm-proxy-name + value: sourcegraph.dotcom + - name: x-mitm-auth-token-name + value: sourcegraph.dotcom + headersSize: 556 + httpVersion: HTTP/1.1 + method: POST + postData: + mimeType: application/json; charset=utf-8 + params: [] + textJSON: + query: |- + + query CurrentSiteCodyLlmProvider { + site { + codyLLMConfiguration { + provider + } + } + } + variables: {} + queryString: + - name: CurrentSiteCodyLlmProvider + value: null + url: https://sourcegraph.com/.api/graphql?CurrentSiteCodyLlmProvider + response: + bodySize: 69 + content: + mimeType: application/json + size: 69 + text: "{\"data\":{\"site\":{\"codyLLMConfiguration\":{\"provider\":\"sourcegraph\ + \"}}}}" + cookies: [] + headers: + - name: date + value: Fri, 13 Sep 2024 16:27:54 GMT + - name: content-type + value: application/json + - name: content-length + value: "69" + - name: connection + value: close + - name: access-control-allow-credentials + value: "true" + - name: access-control-allow-origin + value: "" + - name: cache-control + value: no-cache, max-age=0 + - name: vary + value: Cookie,Accept-Encoding,Authorization,Cookie, Authorization, + X-Requested-With,Cookie + - name: x-content-type-options + value: nosniff + - name: x-frame-options + value: DENY + - name: x-xss-protection + value: 1; mode=block + - name: strict-transport-security + value: max-age=31536000; includeSubDomains; preload + headersSize: 1296 + httpVersion: HTTP/1.1 + redirectURL: "" + status: 200 + statusText: OK + startedDateTime: 2024-09-13T16:27:54.531Z + time: 231 + timings: + blocked: -1 + connect: -1 + dns: -1 + receive: 0 + send: 0 + ssl: -1 + wait: 231 + - _id: d1e9e9d8ae116eb7e71823b775d35828 + _order: 0 + cache: {} + request: + bodySize: 341 + cookies: [] + headers: + - name: connection + value: close + - name: host + value: sourcegraph.com + - name: accept-encoding + value: identity + - name: user-agent + value: node-fetch/1.0 (+https://github.com/bitinn/node-fetch) + - name: content-length + value: "341" + - name: accept + value: "*/*" + - name: content-type + value: application/json; charset=utf-8 + - name: authorization + value: token + REDACTED_5313821e901984ba7bbf1999deafb5f6f722c05ed13b6dfa6cffbd6128089e39 + - name: x-mitm-proxy-name + value: sourcegraph.dotcom + - name: x-mitm-auth-token-name + value: sourcegraph.dotcom + headersSize: 541 + httpVersion: HTTP/1.1 + method: POST + postData: + mimeType: application/json; charset=utf-8 + params: [] + textJSON: + query: |- + + query CurrentUser { + currentUser { + id + hasVerifiedEmail + displayName + username + avatarURL + primaryEmail { + email + } + organizations { + nodes { + id + name + } + } + } + } + variables: {} + queryString: + - name: CurrentUser + value: null + url: https://sourcegraph.com/.api/graphql?CurrentUser + response: + bodySize: 334 + content: + mimeType: application/json + size: 334 + text: "{\"data\":{\"currentUser\":{\"id\":\"VXNlcjozNDQ1Mjc=\",\"hasVerifiedEma\ + il\":true,\"displayName\":\"SourcegraphBot-9000\",\"username\":\"so\ + urcegraphbot9k-fnwmu\",\"avatarURL\":\"https://lh3.googleuserconten\ + t.com/a/ACg8ocKFaqbYeuBkbj5dFEzx8bXV8a7i3sVbKCNPV7G0uyvk=s96-c\",\"\ + primaryEmail\":{\"email\":\"sourcegraphbot9k@gmail.com\"},\"organiz\ + ations\":{\"nodes\":[]}}}}" + cookies: [] + headers: + - name: date + value: Fri, 13 Sep 2024 16:27:54 GMT + - name: content-type + value: application/json + - name: content-length + value: "334" + - name: connection + value: close + - name: access-control-allow-credentials + value: "true" + - name: access-control-allow-origin + value: "" + - name: cache-control + value: no-cache, max-age=0 + - name: vary + value: Cookie,Accept-Encoding,Authorization,Cookie, Authorization, + X-Requested-With,Cookie + - name: x-content-type-options + value: nosniff + - name: x-frame-options + value: DENY + - name: x-xss-protection + value: 1; mode=block + - name: strict-transport-security + value: max-age=31536000; includeSubDomains; preload + headersSize: 1297 + httpVersion: HTTP/1.1 + redirectURL: "" + status: 200 + statusText: OK + startedDateTime: 2024-09-13T16:27:54.533Z + time: 233 + timings: + blocked: -1 + connect: -1 + dns: -1 + receive: 0 + send: 0 + ssl: -1 + wait: 233 + - _id: 9fdf3418a505d251116e020fda591c55 + _order: 0 + cache: {} + request: + bodySize: 268 + cookies: [] + headers: + - name: connection + value: close + - name: host + value: sourcegraph.com + - name: accept-encoding + value: identity + - name: user-agent + value: node-fetch/1.0 (+https://github.com/bitinn/node-fetch) + - name: content-length + value: "268" + - name: accept + value: "*/*" + - name: content-type + value: application/json; charset=utf-8 + - name: authorization + value: token + REDACTED_5313821e901984ba7bbf1999deafb5f6f722c05ed13b6dfa6cffbd6128089e39 + - name: x-mitm-proxy-name + value: sourcegraph.dotcom + - name: x-mitm-auth-token-name + value: sourcegraph.dotcom + headersSize: 557 + httpVersion: HTTP/1.1 + method: POST + postData: + mimeType: application/json; charset=utf-8 + params: [] + textJSON: + query: |- + + query CurrentUserCodySubscription { + currentUser { + codySubscription { + status + plan + applyProRateLimits + currentPeriodStartAt + currentPeriodEndAt + } + } + } + variables: {} + queryString: + - name: CurrentUserCodySubscription + value: null + url: https://sourcegraph.com/.api/graphql?CurrentUserCodySubscription + response: + bodySize: 194 + content: + mimeType: application/json + size: 194 + text: "{\"data\":{\"currentUser\":{\"codySubscription\":{\"status\":\"ACTIVE\",\ + \"plan\":\"PRO\",\"applyProRateLimits\":true,\"currentPeriodStartAt\ + \":\"2024-08-14T22:11:32Z\",\"currentPeriodEndAt\":\"2024-09-14T22:\ + 11:32Z\"}}}}" + cookies: [] + headers: + - name: date + value: Fri, 13 Sep 2024 16:27:55 GMT + - name: content-type + value: application/json + - name: content-length + value: "194" + - name: connection + value: close + - name: access-control-allow-credentials + value: "true" + - name: access-control-allow-origin + value: "" + - name: cache-control + value: no-cache, max-age=0 + - name: vary + value: Cookie,Accept-Encoding,Authorization,Cookie, Authorization, + X-Requested-With,Cookie + - name: x-content-type-options + value: nosniff + - name: x-frame-options + value: DENY + - name: x-xss-protection + value: 1; mode=block + - name: strict-transport-security + value: max-age=31536000; includeSubDomains; preload + headersSize: 1297 + httpVersion: HTTP/1.1 + redirectURL: "" + status: 200 + statusText: OK + startedDateTime: 2024-09-13T16:27:54.772Z + time: 332 + timings: + blocked: -1 + connect: -1 + dns: -1 + receive: 0 + send: 0 + ssl: -1 + wait: 332 + - _id: 39fe3efa22c727dff9fd6b923a145b81 + _order: 0 + cache: {} + request: + bodySize: 247 + cookies: [] + headers: + - name: connection + value: close + - name: host + value: sourcegraph.com + - name: accept-encoding + value: identity + - name: user-agent + value: node-fetch/1.0 (+https://github.com/bitinn/node-fetch) + - name: content-length + value: "247" + - name: accept + value: "*/*" + - name: content-type + value: application/json; charset=utf-8 + - name: authorization + value: token + REDACTED_5313821e901984ba7bbf1999deafb5f6f722c05ed13b6dfa6cffbd6128089e39 + - name: x-mitm-proxy-name + value: sourcegraph.dotcom + - name: x-mitm-auth-token-name + value: sourcegraph.dotcom + headersSize: 615 + httpVersion: HTTP/1.1 + method: POST + postData: + mimeType: application/json; charset=utf-8 + params: [] + textJSON: + query: | + + query Repositories($names: [String!]!, $first: Int!) { + repositories(names: $names, first: $first) { + nodes { + name + id + } + } + } + variables: + first: 10 + names: + - github.com/sourcegraph/cody + queryString: + - name: Repositories + value: null + url: https://sourcegraph.com/.api/graphql?Repositories + response: + bodySize: 112 + content: + mimeType: application/json + size: 112 + text: "{\"data\":{\"repositories\":{\"nodes\":[{\"name\":\"github.com/sourcegra\ + ph/cody\",\"id\":\"UmVwb3NpdG9yeTo2MTMyNTMyOA==\"}]}}}" + cookies: [] + headers: + - name: date + value: Fri, 13 Sep 2024 16:27:56 GMT + - name: content-type + value: application/json + - name: content-length + value: "112" + - name: connection + value: close + - name: access-control-allow-credentials + value: "true" + - name: access-control-allow-origin + value: "" + - name: cache-control + value: no-cache, max-age=0 + - name: vary + value: Cookie,Accept-Encoding,Authorization,Cookie, Authorization, + X-Requested-With,Cookie + - name: x-content-type-options + value: nosniff + - name: x-frame-options + value: DENY + - name: x-xss-protection + value: 1; mode=block + - name: strict-transport-security + value: max-age=31536000; includeSubDomains; preload + headersSize: 1297 + httpVersion: HTTP/1.1 + redirectURL: "" + status: 200 + statusText: OK + startedDateTime: 2024-09-13T16:27:55.786Z + time: 223 + timings: + blocked: -1 + connect: -1 + dns: -1 + receive: 0 + send: 0 + ssl: -1 + wait: 223 + - _id: a4f06f44ac6627e56ffe62ff40b4aaa9 + _order: 0 + cache: {} + request: + bodySize: 101 + cookies: [] + headers: + - name: connection + value: close + - name: host + value: sourcegraph.com + - name: accept-encoding + value: identity + - name: user-agent + value: node-fetch/1.0 (+https://github.com/bitinn/node-fetch) + - name: content-length + value: "101" + - name: accept + value: "*/*" + - name: content-type + value: application/json; charset=utf-8 + - name: authorization + value: token + REDACTED_5313821e901984ba7bbf1999deafb5f6f722c05ed13b6dfa6cffbd6128089e39 + - name: x-mitm-proxy-name + value: sourcegraph.dotcom + - name: x-mitm-auth-token-name + value: sourcegraph.dotcom + headersSize: 621 + httpVersion: HTTP/1.1 + method: POST + postData: + mimeType: application/json; charset=utf-8 + params: [] + textJSON: + query: |- + + query SiteProductVersion { + site { + productVersion + } + } + variables: {} + queryString: + - name: SiteProductVersion + value: null + url: https://sourcegraph.com/.api/graphql?SiteProductVersion + response: + bodySize: 73 + content: + mimeType: application/json + size: 73 + text: "{\"data\":{\"site\":{\"productVersion\":\"291267_2024-09-13_5.7-c635b228\ + 7985\"}}}" + cookies: [] + headers: + - name: date + value: Fri, 13 Sep 2024 16:27:54 GMT + - name: content-type + value: application/json + - name: content-length + value: "73" + - name: connection + value: close + - name: access-control-allow-credentials + value: "true" + - name: access-control-allow-origin + value: "" + - name: cache-control + value: no-cache, max-age=0 + - name: vary + value: Cookie,Accept-Encoding,Authorization,Cookie, Authorization, + X-Requested-With,Cookie + - name: x-content-type-options + value: nosniff + - name: x-frame-options + value: DENY + - name: x-xss-protection + value: 1; mode=block + - name: strict-transport-security + value: max-age=31536000; includeSubDomains; preload + headersSize: 1296 + httpVersion: HTTP/1.1 + redirectURL: "" + status: 200 + statusText: OK + startedDateTime: 2024-09-13T16:27:54.509Z + time: 244 + timings: + blocked: -1 + connect: -1 + dns: -1 + receive: 0 + send: 0 + ssl: -1 + wait: 244 + - _id: a6632f96ba91fb74e4e450ec8da83f1a + _order: 0 + cache: {} + request: + bodySize: 567 + cookies: [] + headers: + - name: connection + value: close + - name: host + value: sourcegraph.com + - name: accept-encoding + value: identity + - name: user-agent + value: node-fetch/1.0 (+https://github.com/bitinn/node-fetch) + - name: content-length + value: "567" + - name: accept + value: "*/*" + - name: content-type + value: application/json; charset=utf-8 + - name: authorization + value: token + REDACTED_5313821e901984ba7bbf1999deafb5f6f722c05ed13b6dfa6cffbd6128089e39 + - name: x-mitm-proxy-name + value: sourcegraph.dotcom + - name: x-mitm-auth-token-name + value: sourcegraph.dotcom + headersSize: 616 + httpVersion: HTTP/1.1 + method: POST + postData: + mimeType: application/json; charset=utf-8 + params: [] + textJSON: + query: >- + + query ViewerPrompts($query: String!) { + prompts(query: $query, first: 100, viewerIsAffiliated: true, orderBy: PROMPT_NAME_WITH_OWNER) { + nodes { + id + name + nameWithOwner + owner { + namespaceName + } + description + draft + definition { + text + } + url + } + totalCount + pageInfo { + hasNextPage + endCursor + } + } + } + variables: + query: "" + queryString: + - name: ViewerPrompts + value: null + url: https://sourcegraph.com/.api/graphql?ViewerPrompts + response: + bodySize: 98 + content: + mimeType: application/json + size: 98 + text: "{\"data\":{\"prompts\":{\"nodes\":[],\"totalCount\":0,\"pageInfo\":{\"ha\ + sNextPage\":false,\"endCursor\":null}}}}" + cookies: [] + headers: + - name: date + value: Fri, 13 Sep 2024 16:27:58 GMT + - name: content-type + value: application/json + - name: content-length + value: "98" + - name: connection + value: close + - name: access-control-allow-credentials + value: "true" + - name: access-control-allow-origin + value: "" + - name: cache-control + value: no-cache, max-age=0 + - name: vary + value: Cookie,Accept-Encoding,Authorization,Cookie, Authorization, + X-Requested-With,Cookie + - name: x-content-type-options + value: nosniff + - name: x-frame-options + value: DENY + - name: x-xss-protection + value: 1; mode=block + - name: strict-transport-security + value: max-age=31536000; includeSubDomains; preload + headersSize: 1296 + httpVersion: HTTP/1.1 + redirectURL: "" + status: 200 + statusText: OK + startedDateTime: 2024-09-13T16:27:58.258Z + time: 234 + timings: + blocked: -1 + connect: -1 + dns: -1 + receive: 0 + send: 0 + ssl: -1 + wait: 234 + pages: [] + version: "1.2" diff --git a/vscode/e2e/features/enterprise/cody-ignore.test.ts b/vscode/e2e/features/enterprise/cody-ignore.test.ts index fc29a836ba52..edaa3b80fa5b 100644 --- a/vscode/e2e/features/enterprise/cody-ignore.test.ts +++ b/vscode/e2e/features/enterprise/cody-ignore.test.ts @@ -12,7 +12,14 @@ test.use({ }) test.describe('cody ignore', {}, () => { - test('it works', async ({ workspaceDir, page, vscodeUI, mitmProxy, polly }, testInfo) => { + test('it works', async ({ + workspaceDir, + page, + vscodeUI, + mitmProxy, + polly, + validOptions, + }, testInfo) => { const session = uix.vscode.Session.pending({ page, vscodeUI, workspaceDir }) const cody = uix.cody.Extension.with({ page, workspaceDir }) diff --git a/vscode/e2e/telemetry/at-mention.test.ts b/vscode/e2e/telemetry/at-mention.test.ts new file mode 100644 index 000000000000..cbc46f1b2d13 --- /dev/null +++ b/vscode/e2e/telemetry/at-mention.test.ts @@ -0,0 +1,94 @@ +import { fixture as test, uix } from '../utils/vscody' +import { MITM_AUTH_TOKEN_PLACEHOLDER } from '../utils/vscody/constants' +import { expect } from '../utils/vscody/uix' +import { modifySettings } from '../utils/vscody/uix/workspace' + +test.describe('cody.at-mention', () => { + test.use({ + templateWorkspaceDir: 'test/fixtures/legacy-polyglot-template', + }) + test('`/execute` should not fire pre-maturely', async ({ + page, + mitmProxy, + vscodeUI, + polly, + workspaceDir, + telemetryRecorder, + }, testInfo) => { + // Behavior is described here: + // https://linear.app/sourcegraph/issue/CODY-3405/fix-mention-telemetry + + const session = uix.vscode.Session.pending({ page, vscodeUI, workspaceDir }) + const cody = uix.cody.Extension.with({ page, workspaceDir }) + + await test.step('setup', async () => { + await modifySettings( + s => ({ + ...s, + 'cody.accessToken': MITM_AUTH_TOKEN_PLACEHOLDER, + 'cody.serverEndpoint': mitmProxy.sourcegraph.dotcom.endpoint, + }), + { workspaceDir } + ) + await session.start() + await cody.waitUntilReady() + await session.editor.openFile({ + workspaceFile: 'buzz.ts', + selection: { start: { line: 3 }, end: { line: 5 } }, + }) + }) + + await session.runCommand('cody.chat.newEditorPanel') + const [chat] = await uix.cody.WebView.all(session, { atLeast: 1 }) + await chat.waitUntilReady() + + //TODO: make a nice UIX class for this + const chatInput = chat.content.getByRole('textbox', { name: 'Chat message' }) + await expect(chatInput).toBeVisible() + const telemetry = uix.telemetry.TelemetrySnapshot.fromNow({ + telemetryRecorder, + }) + await chatInput.fill('@') + + const atMenu = await chat.content.locator('[data-at-mention-menu]') + await expect(atMenu).toBeVisible() + await atMenu.locator('[data-value="provider:file"]').click() + + await expect(atMenu.locator('[data-value^="[\\"file\\""]').first()).toBeVisible() + // we need to wait for some telemetry events to come in + const selectTelemetry = telemetry.snap() + + expect( + selectTelemetry.filter({ matching: { action: 'executed' } }), + 'Execution events should not have fired' + ).toEqual([]) + const [mentionEvent, fileEvent, ...otherEvents] = selectTelemetry.filter({ + matching: { feature: 'cody.at-mention', action: 'selected' }, + }) + expect(otherEvents).toEqual([]) + await expect(mentionEvent.event).toMatchJSONSnapshot('mentionEvent', { + normalizers: snapshotNormalizers, + }) + await expect(fileEvent.event).toMatchJSONSnapshot('fileEvent', { + normalizers: snapshotNormalizers, + }) + + // we now ensure that the event did fire if we do select a file + await atMenu.locator('[data-value^="[\\"file\\""]').first().click() + await expect(atMenu).not.toBeVisible() + await chatInput.press('Enter') + + //@ts-ignore + const executeTelemetry = telemetry.snap(selectTelemetry) + + // finally we check some global conditions + + telemetry.stop() + }) +}) + +const snapshotNormalizers = [ + uix.snapshot.Normalizers.sortKeysDeep, + uix.snapshot.Normalizers.sortPathBy('parameters.metadata', 'key'), + uix.snapshot.Normalizers.omit('source.clientVersion', 'timestamp'), +] diff --git a/vscode/e2e/telemetry/at-mention.test.ts-snapshots/fileEvent.snap.json b/vscode/e2e/telemetry/at-mention.test.ts-snapshots/fileEvent.snap.json new file mode 100644 index 000000000000..16f176d3e1ec --- /dev/null +++ b/vscode/e2e/telemetry/at-mention.test.ts-snapshots/fileEvent.snap.json @@ -0,0 +1,37 @@ +{ + "action": "selected", + "feature": "cody.at-mention", + "parameters": { + "billingMetadata": { + "category": "core", + "product": "cody" + }, + "metadata": [ + { + "key": "contextSelection", + "value": 10 + }, + { + "key": "provider", + "value": 1 + }, + { + "key": "source", + "value": 1 + }, + { + "key": "tier", + "value": 1 + } + ], + "privateMetadata": { + "provider": "file", + "source": "chat" + }, + "version": 0 + }, + "signature": "cody.at-mention/selected", + "source": { + "client": "VSCode.Cody" + } +} \ No newline at end of file diff --git a/vscode/e2e/telemetry/at-mention.test.ts-snapshots/mentionEvent.snap.json b/vscode/e2e/telemetry/at-mention.test.ts-snapshots/mentionEvent.snap.json new file mode 100644 index 000000000000..57a262d6b4ca --- /dev/null +++ b/vscode/e2e/telemetry/at-mention.test.ts-snapshots/mentionEvent.snap.json @@ -0,0 +1,32 @@ +{ + "action": "selected", + "feature": "cody.at-mention", + "parameters": { + "billingMetadata": { + "category": "core", + "product": "cody" + }, + "metadata": [ + { + "key": "contextSelection", + "value": 10 + }, + { + "key": "source", + "value": 1 + }, + { + "key": "tier", + "value": 1 + } + ], + "privateMetadata": { + "source": "chat" + }, + "version": 0 + }, + "signature": "cody.at-mention/selected", + "source": { + "client": "VSCode.Cody" + } +} \ No newline at end of file diff --git a/vscode/e2e/utils/vscody/fixture/index.ts b/vscode/e2e/utils/vscody/fixture/index.ts index 9e4215a29f30..84b3eaf86966 100644 --- a/vscode/e2e/utils/vscody/fixture/index.ts +++ b/vscode/e2e/utils/vscody/fixture/index.ts @@ -11,6 +11,8 @@ import { kitchensinkFixture } from './kitchensink' import { type MitMProxy, mitmProxyFixture } from './mitmProxy' import { type TestOptions, type WorkerOptions, optionsFixture } from './options' import { pollyFixture } from './polly' +// biome-ignore lint/nursery/noRestrictedImports: false positive +import { type TelemetryRecorder, telemetryFixture } from './telemetry' import { vscodeFixture } from './vscode' export interface WorkerContext { validWorkerOptions: WorkerOptions @@ -27,6 +29,7 @@ export interface TestContext { serverRootDir: Directory validOptions: TestOptions & WorkerOptions polly: Polly + telemetryRecorder: TelemetryRecorder mitmProxy: MitMProxy //sourcegraphMitM: { endpoint: string; target: string } workspaceDir: Directory @@ -36,6 +39,7 @@ export const fixture = mergeTests( optionsFixture, mitmProxyFixture, pollyFixture, + telemetryFixture, vscodeFixture, kitchensinkFixture ) as ReturnType> diff --git a/vscode/e2e/utils/vscody/fixture/polly.ts b/vscode/e2e/utils/vscody/fixture/polly.ts index fda65c1f69bb..a5144932cb28 100644 --- a/vscode/e2e/utils/vscody/fixture/polly.ts +++ b/vscode/e2e/utils/vscody/fixture/polly.ts @@ -167,24 +167,6 @@ export const pollyFixture = _test.extend({ res.status(200).json({ data: { logEvent: null } }) }) - polly.server - .any() - .filter(req => { - return ( - !!getFirstOrValue(req.getHeader(MITM_PROXY_SERVICE_NAME_HEADER))?.startsWith( - 'sourcegraph' - ) && - req.pathname.startsWith('/.api/graphql') && - 'RecordTelemetryEvents' in req.query - ) - }) - .intercept((req, res) => { - //TODO: implement this - res.status(200).json({ - data: { telemetry: { recordEvents: { alwaysNil: null } } }, - }) - }) - polly.server .any() .filter(req => { diff --git a/vscode/e2e/utils/vscody/fixture/telemetry.ts b/vscode/e2e/utils/vscody/fixture/telemetry.ts new file mode 100644 index 000000000000..040999c7d8db --- /dev/null +++ b/vscode/e2e/utils/vscody/fixture/telemetry.ts @@ -0,0 +1,69 @@ +import { test as _test } from '@playwright/test' +import 'node:http' +import 'node:https' +import type { TelemetryEventInput } from '@sourcegraph/telemetry' +import jsonStableStringify from 'fast-json-stable-stringify' +import { ulid } from 'ulidx' +import type { TestContext, WorkerContext } from '.' +import { MITM_PROXY_SERVICE_NAME_HEADER } from '../constants' +import { getFirstOrValue } from './util' + +export interface RecordedTelemetryEvent { + id: string + timestamp: Date + proxyName: string + event: TelemetryEventInput +} +export interface TelemetryRecorder { + readonly all: RecordedTelemetryEvent[] +} + +export const telemetryFixture = _test.extend({ + telemetryRecorder: [ + async ({ validOptions, polly }, use, testInfo) => { + const recorder: TelemetryRecorder = { + all: [], + } + + polly.server + .any() + .filter(req => { + return ( + !!getFirstOrValue(req.getHeader(MITM_PROXY_SERVICE_NAME_HEADER))?.startsWith( + 'sourcegraph' + ) && + req.pathname.startsWith('/.api/graphql') && + 'RecordTelemetryEvents' in req.query + ) + }) + .intercept((req, res) => { + const now = new Date() + const body = req.jsonBody() + res.status(200).json({ + data: { telemetry: { recordEvents: { alwaysNil: null } } }, + }) + const rawEvents = body?.variables?.events as any[] + //todo: allow automatic failure of events + for (const event of rawEvents) { + Object.assign(event, { signature: `${event?.feature}/${event?.action}` }) + recorder.all.push({ + id: ulid(), + event, + timestamp: now, + proxyName: getFirstOrValue(req.getHeader(MITM_PROXY_SERVICE_NAME_HEADER))!, + }) + } + }) + + await use(recorder) + + await testInfo.attach('telemetryEvents.json', { + body: JSON.stringify(JSON.parse(jsonStableStringify(recorder.all)), null, 2), + contentType: 'application/json', + }) + + //todo: add as attachments + }, + { scope: 'test' }, + ], +}) diff --git a/vscode/e2e/utils/vscody/fixture/vscode.ts b/vscode/e2e/utils/vscody/fixture/vscode.ts index e71a634a1722..31d223e631db 100644 --- a/vscode/e2e/utils/vscody/fixture/vscode.ts +++ b/vscode/e2e/utils/vscody/fixture/vscode.ts @@ -62,7 +62,14 @@ export const vscodeFixture = _test.extend({ { scope: 'test' }, ], vscodeUI: [ - async ({ validOptions, debugMode, serverRootDir, mitmProxy, page, polly }, use, testInfo) => { + async ( + { validOptions, debugMode, serverRootDir, mitmProxy, page, polly, telemetryRecorder }, + use, + testInfo + ) => { + if (telemetryRecorder) { + //do nothing, we just import it so it is auto included when using VSCode + } polly.pause() const executableDir = path.resolve(CODY_VSCODE_ROOT_DIR, validOptions.vscodeTmpDir) @@ -192,7 +199,7 @@ export const vscodeFixture = _test.extend({ polly.pause() // Turn of logging browser logging and navigate away from the UI // Otherwise we needlessly add a bunch of noisy error logs - if (!page.isClosed && page.url().startsWith(config.url)) { + if (!page.isClosed() && page.url().startsWith(config.url)) { await page.evaluate(() => { console.log = () => {} console.info = () => {} diff --git a/vscode/e2e/utils/vscody/uix/index.ts b/vscode/e2e/utils/vscody/uix/index.ts index 497718ce68d1..94403203f855 100644 --- a/vscode/e2e/utils/vscody/uix/index.ts +++ b/vscode/e2e/utils/vscody/uix/index.ts @@ -1,7 +1,48 @@ -import type { PlaywrightTestArgs, PlaywrightWorkerArgs } from '@playwright/test' +import { + type PlaywrightTestArgs, + type PlaywrightWorkerArgs, + type TestInfo, + expect as baseExpect, +} from '@playwright/test' +import { stretchTimeout } from '../../helpers' import type { TestContext, WorkerContext } from '../fixture' -export * as vscode from './vscode' export * as cody from './cody' +export * as vscode from './vscode' export * as workspace from './workspace' +export * as snapshot from './snapshot' +// biome-ignore lint/nursery/noRestrictedImports: false positive +export * as telemetry from './telemetry' +import { expect as snapshotExpects } from './snapshot' + +export const expect = baseExpect.extend({ + ...snapshotExpects, +}) export type UIXContextFnContext = TestContext & WorkerContext & PlaywrightTestArgs & PlaywrightWorkerArgs + +/** + * Waits for a fake condition without triggering a timeout. To continue execution call the __continueTest() function from the console. + */ +export async function wait(ctx: Pick & { testInfo: TestInfo }) { + // this works by inserting a fake dom meta dom node into the page and waiting for it to disappear + // we allow the user to do this by evaluating a script that defines a global function that can be called. + await ctx.page.evaluate(() => { + const meta = document.createElement('meta') + meta.id = 'cody-test-wait-for-condition' + meta.content = 'true' + document.head.appendChild(meta) + //@ts-ignore + globalThis.__continueTest = () => { + document.head.removeChild(meta) + } + }) + + await stretchTimeout( + async () => { + await expect(ctx.page.locator('#cody-test-wait-for-condition')).not.toBeAttached({ + timeout: 0, + }) + }, + { max: 60 * 60 * 60 * 1000, testInfo: ctx.testInfo } + ) +} diff --git a/vscode/e2e/utils/vscody/uix/snapshot.ts b/vscode/e2e/utils/vscody/uix/snapshot.ts new file mode 100644 index 000000000000..e1e191fff99a --- /dev/null +++ b/vscode/e2e/utils/vscody/uix/snapshot.ts @@ -0,0 +1,152 @@ +import _fs from 'node:fs' +import fs from 'node:fs/promises' +import path from 'node:path' +import { type ExpectMatcherState, type MatcherReturnType, type TestInfo, test } from '@playwright/test' +import { produce } from 'immer' +import _ from 'lodash' +import type { ArraySlice } from 'type-fest' +export { test } from '@playwright/test' + +export const expect = { + async toMatchJSONSnapshot( + this: ExpectMatcherState, + received: T, + snapshotName: string, + options?: { + normalizers?: ((obj: any) => any)[] + } + ): Promise { + const name = 'toMatchJSONSnapshot' + if (this.isNot) { + throw new Error('not implemented') + } + + const testInfo = test.info() as TestInfo & { _projectInternal: any } + let normalized = received + for (const normalizer of options?.normalizers ?? []) { + normalized = produce(normalized, normalizer) + } + + const serialized = JSON.stringify(normalized, null, 2) + + const snapshotDir = testInfo.snapshotDir + const snapshotPath = path.join(snapshotDir, `${snapshotName}.snap.json`) + + if (testInfo._projectInternal.ignoreSnapshots) + return { + pass: true, + message: () => '', + name, + expected: snapshotName, + } + + const updateSnapshots = testInfo.config.updateSnapshots === 'all' + const exists = _fs.existsSync(snapshotPath) + if (exists) { + const previousJSON = await fs.readFile(snapshotPath, 'utf-8') + const previous = JSON.parse(previousJSON) + const current = JSON.parse(serialized) + if (!_.isEqual(current, previous)) { + return { + pass: false, + message: () => + `Snapshot does not match ${snapshotName}\n\n${this.utils.diff( + previous, + current + )}`, + name, + expected: previous, + actual: current, + } + } + } + + if (updateSnapshots || !exists) { + await fs.writeFile(snapshotPath, serialized) + } + + return { + message: () => 'Snapshot matches', + pass: true, + } + }, +} as const + +export namespace Normalizers { + export const pick = + (...paths: string[]) => + (draft: any) => { + return _.pick(draft, ...paths) + } + + export const omit = + (...paths: string[]) => + (draft: any) => { + return _.omit(draft, ...paths) + } + + export const fixedDates = + (fixedDate = new Date('2000-01-01T00:00:00Z')) => + (draft: any) => { + function recurse(current: any): any { + if (current instanceof Date) { + return new Date(fixedDate) + } + + if (Array.isArray(current)) { + return current.map(recurse) + } + + if (typeof current === 'object' && current !== null) { + for (const key of Object.keys(current)) { + current[key] = recurse(current[key]) + } + } + + return current + } + return recurse(draft) + } + + /** + * Allows you to sort a specified path by an arbitrary key + */ + export const sortPathBy = ( + path: string, + ...args: ArraySlice, 1> + ) => { + return (draft: any) => { + const item = _.get(draft, path) + if (_.isArray(item)) { + const sorted: any[] = _.sortBy(item, ...args) + item.sort((a, b) => sorted.indexOf(a) - sorted.indexOf(b)) + } + return draft + } + } + + export function sortKeysDeep(obj: any) { + return produce(obj, (draft: any) => { + if (typeof draft !== 'object' || draft === null) { + return + } + + if (Array.isArray(draft)) { + for (const [index, item] of draft.entries()) { + draft[index] = sortKeysDeep(item) + } + return + } + + const sortedKeys = Object.keys(draft).sort() + const newObj = {} + + for (const key of sortedKeys) { + //@ts-ignore + newObj[key] = sortKeysDeep(draft[key]) + } + + return newObj + }) + } +} diff --git a/vscode/e2e/utils/vscody/uix/telemetry.ts b/vscode/e2e/utils/vscody/uix/telemetry.ts new file mode 100644 index 000000000000..4b54a7d16db4 --- /dev/null +++ b/vscode/e2e/utils/vscody/uix/telemetry.ts @@ -0,0 +1,107 @@ +import type { telemetryEvents } from '@sourcegraph/cody-shared' +import { isArray } from 'lodash' +import type { LiteralUnion } from 'type-fest' +import type { UIXContextFnContext } from '.' +import type { RecordedTelemetryEvent } from '../fixture/telemetry' + +export type Signatures = keyof typeof telemetryEvents +export type Actions = SplitSignature[1] +export type Features = SplitSignature[0] + +type Options = Pick +type Ctx = { + start?: number + end?: number +} & Options +export class TelemetrySnapshot { + private constructor(private ctx: Ctx) {} + + static fromNow(opts: Options) { + return new TelemetrySnapshot({ ...opts, start: opts.telemetryRecorder.all.length }) + } + + static untilNow(opts: Options) { + return new TelemetrySnapshot({ ...opts, start: 0, end: opts.telemetryRecorder.all.length }) + } + + /** + * Returns a new stopped snapshot but keeps the original one running. If a + * previous snapshot is passed in the new snapshot starts after the last one + * was taken. + */ + snap(previous?: TelemetrySnapshot): TelemetrySnapshot { + return new TelemetrySnapshot({ + ...this.ctx, + start: previous?.ctx.end ?? this.ctx.start, + end: this.ctx.telemetryRecorder.all.length, + }) + } + + /** + * Freezes this telemetry snapshot and returns + */ + stop(): TelemetrySnapshot { + this.ctx.end = this.ctx.end ?? this.ctx.telemetryRecorder.all.length + return this + } + + get events() { + return this.ctx.telemetryRecorder.all.slice(this.ctx.start ?? 0, this.ctx.end ?? undefined) + } + + filter({ + matching, + notMatching, + }: { + valid?: boolean + matching?: MatchFn | PropMatchFnOpts | PropMatchFnOpts[] + notMatching?: MatchFn | PropMatchFnOpts | PropMatchFnOpts[] + }) { + function apply( + input: RecordedTelemetryEvent[], + m: MatchFn | PropMatchFnOpts | PropMatchFnOpts[] | undefined, + shouldMatch = true + ) { + if (m === undefined) { + return input + } + if (typeof m === 'function') { + return input.filter(v => m(v) === shouldMatch) + } + const propMatcher = isArray(m) ? m : [m] + const matcherFn = propMatchFn(...propMatcher) + return input.filter(v => matcherFn(v) === shouldMatch) + } + let filtered = this.events + filtered = apply(filtered, matching) + filtered = apply(filtered, notMatching, false) + return filtered + } +} + +export type MatchFn = (event: RecordedTelemetryEvent) => boolean +export interface PropMatchFnOpts { + feature?: LiteralUnion + action?: LiteralUnion +} +function propMatchFn(...opts: PropMatchFnOpts[]): MatchFn { + return ({ event }) => { + for (const opt of opts) { + const matchesFeature = opt.feature !== undefined ? opt.feature === event.feature : true + const matchesAction = opt.action !== undefined ? opt.action === event.action : true + + if (matchesFeature && matchesAction) { + return true + } + } + return false + } +} + +type SplitSignature = string extends S + ? string[] + : S extends '' + ? [] + : S extends `${infer T}${D}${infer U}` + ? [T, ...SplitSignature] + : [S] diff --git a/vscode/package.json b/vscode/package.json index 2b62b7127679..ea0ece752c8a 100644 --- a/vscode/package.json +++ b/vscode/package.json @@ -1474,6 +1474,7 @@ "fs-extra": "^11.2.0", "fuzzysort": "^2.0.4", "http-proxy-middleware": "^3.0.0", + "immer": "^10.1.1", "mocha": "^10.2.0", "node-fetch": "^2.6.4", "normalize-url": "^5.3.1", diff --git a/vscode/src/chat/context/chatContext.ts b/vscode/src/chat/context/chatContext.ts index 41b532f8d34b..d8dc6db3fa74 100644 --- a/vscode/src/chat/context/chatContext.ts +++ b/vscode/src/chat/context/chatContext.ts @@ -15,7 +15,7 @@ import { mentionProvidersMetadata, openCtx, promiseFactoryToObservable, - telemetryRecorder, + telemetryEvents, } from '@sourcegraph/cody-shared' import { Observable, map } from 'observable-fns' import * as vscode from 'vscode' @@ -43,35 +43,12 @@ export function getMentionMenuData(options: { query: MentionQuery chatModel: ChatModel }): Observable { - const source = 'chat' - - // Use numerical mapping to send source values to metadata, making this data available on all instances. - const atMentionSourceTelemetryMetadataMapping: Record = { - chat: 1, - } as const - const scopedTelemetryRecorder: GetContextItemsTelemetry = { empty: () => { - telemetryRecorder.recordEvent('cody.at-mention', 'executed', { - metadata: { - source: atMentionSourceTelemetryMetadataMapping[source], - }, - privateMetadata: { source }, - billingMetadata: { - product: 'cody', - category: 'core', - }, - }) + telemetryEvents['cody.at-mention/selected'].record('chat') }, withProvider: (provider, providerMetadata) => { - telemetryRecorder.recordEvent(`cody.at-mention.${provider}`, 'executed', { - metadata: { source: atMentionSourceTelemetryMetadataMapping[source] }, - privateMetadata: { source, providerMetadata }, - billingMetadata: { - product: 'cody', - category: 'core', - }, - }) + telemetryEvents['cody.at-mention/selected'].record('chat', provider, providerMetadata) }, }