From d4c164e7b85364cfdc24b4db23858abe2ff7194b Mon Sep 17 00:00:00 2001 From: Joyce Er Date: Fri, 2 Aug 2024 16:35:14 -0700 Subject: [PATCH] feat: chat participation detection api (#224651) * feat: chat participation detection api --- .../api/browser/mainThreadChatAgents2.ts | 20 +++- .../workbench/api/common/extHost.api.impl.ts | 4 + .../workbench/api/common/extHost.protocol.ts | 13 +++ .../api/common/extHostChatAgents2.ts | 99 +++++++++++++------ .../contrib/chat/common/chatAgents.ts | 63 ++++++++++++ .../chat/test/common/voiceChatService.test.ts | 8 +- ...ode.proposed.chatParticipantAdditions.d.ts | 17 ++++ 7 files changed, 190 insertions(+), 34 deletions(-) diff --git a/src/vs/workbench/api/browser/mainThreadChatAgents2.ts b/src/vs/workbench/api/browser/mainThreadChatAgents2.ts index 78173e8502ba5..3a403ad039cee 100644 --- a/src/vs/workbench/api/browser/mainThreadChatAgents2.ts +++ b/src/vs/workbench/api/browser/mainThreadChatAgents2.ts @@ -21,11 +21,11 @@ import { ILanguageFeaturesService } from 'vs/editor/common/services/languageFeat import { ExtensionIdentifier } from 'vs/platform/extensions/common/extensions'; import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation'; import { ILogService } from 'vs/platform/log/common/log'; -import { ExtHostChatAgentsShape2, ExtHostContext, IChatProgressDto, IDynamicChatAgentProps, IExtensionChatAgentMetadata, MainContext, MainThreadChatAgentsShape2 } from 'vs/workbench/api/common/extHost.protocol'; +import { ExtHostChatAgentsShape2, ExtHostContext, IChatParticipantMetadata, IChatProgressDto, IDynamicChatAgentProps, IExtensionChatAgentMetadata, MainContext, MainThreadChatAgentsShape2 } from 'vs/workbench/api/common/extHost.protocol'; import { IChatWidgetService } from 'vs/workbench/contrib/chat/browser/chat'; import { ChatInputPart } from 'vs/workbench/contrib/chat/browser/chatInputPart'; import { AddDynamicVariableAction, IAddDynamicVariableContext } from 'vs/workbench/contrib/chat/browser/contrib/chatDynamicVariables'; -import { ChatAgentLocation, IChatAgentImplementation, IChatAgentService } from 'vs/workbench/contrib/chat/common/chatAgents'; +import { ChatAgentLocation, IChatAgentHistoryEntry, IChatAgentImplementation, IChatAgentRequest, IChatAgentService } from 'vs/workbench/contrib/chat/common/chatAgents'; import { ChatRequestAgentPart } from 'vs/workbench/contrib/chat/common/chatParserTypes'; import { ChatRequestParser } from 'vs/workbench/contrib/chat/common/chatRequestParser'; import { IChatContentReference, IChatFollowup, IChatProgress, IChatService, IChatTask, IChatWarningMessage } from 'vs/workbench/contrib/chat/common/chatService'; @@ -76,6 +76,8 @@ export class MainThreadChatAgents2 extends Disposable implements MainThreadChatA private readonly _agentCompletionProviders = this._register(new DisposableMap()); private readonly _agentIdsToCompletionProviders = this._register(new DisposableMap); + private readonly _chatParticipantDetectionProviders = this._register(new DisposableMap()); + private readonly _pendingProgress = new Map void>(); private readonly _proxy: ExtHostChatAgentsShape2; @@ -298,6 +300,20 @@ export class MainThreadChatAgents2 extends Disposable implements MainThreadChatA this._agentCompletionProviders.deleteAndDispose(handle); this._agentIdsToCompletionProviders.deleteAndDispose(id); } + + $registerChatParticipantDetectionProvider(handle: number): void { + this._chatParticipantDetectionProviders.set(handle, this._chatAgentService.registerChatParticipantDetectionProvider(handle, + { + provideParticipantDetection: async (request: IChatAgentRequest, history: IChatAgentHistoryEntry[], options: { location: ChatAgentLocation; participants: IChatParticipantMetadata[] }, token: CancellationToken) => { + return await this._proxy.$detectChatParticipant(handle, request, { history }, options, token); + } + } + )); + } + + $unregisterChatParticipantDetectionProvider(handle: number): void { + this._chatParticipantDetectionProviders.deleteAndDispose(handle); + } } diff --git a/src/vs/workbench/api/common/extHost.api.impl.ts b/src/vs/workbench/api/common/extHost.api.impl.ts index d3c2ab00b46e5..ae8a6fd90b2c8 100644 --- a/src/vs/workbench/api/common/extHost.api.impl.ts +++ b/src/vs/workbench/api/common/extHost.api.impl.ts @@ -1519,6 +1519,10 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I checkProposedApiEnabled(extension, 'chatParticipantPrivate'); return extHostChatAgents2.createDynamicChatAgent(extension, id, dynamicProps, handler); }, + registerChatParticipantDetectionProvider(provider: vscode.ChatParticipantDetectionProvider) { + checkProposedApiEnabled(extension, 'chatParticipantAdditions'); + return extHostChatAgents2.registerChatParticipantDetectionProvider(provider); + }, }; // namespace: lm diff --git a/src/vs/workbench/api/common/extHost.protocol.ts b/src/vs/workbench/api/common/extHost.protocol.ts index d0bf4de24170e..76f6a658ba65b 100644 --- a/src/vs/workbench/api/common/extHost.protocol.ts +++ b/src/vs/workbench/api/common/extHost.protocol.ts @@ -1244,6 +1244,8 @@ export interface IDynamicChatAgentProps { export interface MainThreadChatAgentsShape2 extends IDisposable { $registerAgent(handle: number, extension: ExtensionIdentifier, id: string, metadata: IExtensionChatAgentMetadata, dynamicProps: IDynamicChatAgentProps | undefined): void; + $registerChatParticipantDetectionProvider(handle: number): void; + $unregisterChatParticipantDetectionProvider(handle: number): void; $registerAgentCompletionsProvider(handle: number, id: string, triggerCharacters: string[]): void; $unregisterAgentCompletionsProvider(handle: number, id: string): void; $updateAgent(handle: number, metadataUpdate: IExtensionChatAgentMetadata): void; @@ -1284,6 +1286,17 @@ export interface ExtHostChatAgentsShape2 { $provideWelcomeMessage(handle: number, location: ChatAgentLocation, token: CancellationToken): Promise<(string | IMarkdownString)[] | undefined>; $provideSampleQuestions(handle: number, location: ChatAgentLocation, token: CancellationToken): Promise; $releaseSession(sessionId: string): void; + $detectChatParticipant(handle: number, request: Dto, context: { history: IChatAgentHistoryEntryDto[] }, options: { participants: IChatParticipantMetadata[]; location: ChatAgentLocation }, token: CancellationToken): Promise; +} +export interface IChatParticipantMetadata { + participant: string; + command?: string; + description?: string; +} + +export interface IChatParticipantDetectionResult { + participant: string; + command?: string; } export type IChatVariableResolverProgressDto = diff --git a/src/vs/workbench/api/common/extHostChatAgents2.ts b/src/vs/workbench/api/common/extHostChatAgents2.ts index 1abb128a5c3c5..3e5225b8dee13 100644 --- a/src/vs/workbench/api/common/extHostChatAgents2.ts +++ b/src/vs/workbench/api/common/extHostChatAgents2.ts @@ -10,7 +10,7 @@ import { toErrorMessage } from 'vs/base/common/errorMessage'; import { Emitter } from 'vs/base/common/event'; import { IMarkdownString } from 'vs/base/common/htmlContent'; import { Iterable } from 'vs/base/common/iterator'; -import { Disposable, DisposableMap, DisposableStore } from 'vs/base/common/lifecycle'; +import { Disposable, DisposableMap, DisposableStore, toDisposable } from 'vs/base/common/lifecycle'; import { revive } from 'vs/base/common/marshalling'; import { StopWatch } from 'vs/base/common/stopwatch'; import { assertType } from 'vs/base/common/types'; @@ -270,6 +270,9 @@ export class ExtHostChatAgents2 extends Disposable implements ExtHostChatAgentsS private readonly _agents = new Map(); private readonly _proxy: MainThreadChatAgentsShape2; + private static _participantDetectionProviderIdPool = 0; + private readonly _participantDetectionProviders = new Map(); + private readonly _sessionDisposables: DisposableMap = this._register(new DisposableMap()); private readonly _completionDisposables: DisposableMap = this._register(new DisposableMap()); @@ -305,44 +308,78 @@ export class ExtHostChatAgents2 extends Disposable implements ExtHostChatAgentsS return agent.apiAgent; } + registerChatParticipantDetectionProvider(provider: vscode.ChatParticipantDetectionProvider): vscode.Disposable { + const handle = ExtHostChatAgents2._participantDetectionProviderIdPool++; + this._participantDetectionProviders.set(handle, provider); + this._proxy.$registerChatParticipantDetectionProvider(handle); + return toDisposable(() => { + this._participantDetectionProviders.delete(handle); + this._proxy.$unregisterChatParticipantDetectionProvider(handle); + }); + } + + async $detectChatParticipant(handle: number, requestDto: Dto, context: { history: IChatAgentHistoryEntryDto[] }, options: { location: ChatAgentLocation; participants?: vscode.ChatParticipantMetadata[] }, token: CancellationToken): Promise { + const { request, location, history } = await this._createRequest(requestDto, context); + + const provider = this._participantDetectionProviders.get(handle); + if (!provider) { + return undefined; + } + + return provider.provideParticipantDetection( + typeConvert.ChatAgentRequest.to(request, location), + { history }, + { participants: options.participants, location: typeConvert.ChatLocation.to(options.location) }, + token + ); + } + + private async _createRequest(requestDto: Dto, context: { history: IChatAgentHistoryEntryDto[] }) { + const request = revive(requestDto); + const convertedHistory = await this.prepareHistoryTurns(request.agentId, context); + + // in-place converting for location-data + let location: vscode.ChatRequestEditorData | vscode.ChatRequestNotebookData | undefined; + if (request.locationData?.type === ChatAgentLocation.Editor) { + // editor data + const document = this._documents.getDocument(request.locationData.document); + location = new extHostTypes.ChatRequestEditorData(document, typeConvert.Selection.to(request.locationData.selection), typeConvert.Range.to(request.locationData.wholeRange)); + + } else if (request.locationData?.type === ChatAgentLocation.Notebook) { + // notebook data + const cell = this._documents.getDocument(request.locationData.sessionInputUri); + location = new extHostTypes.ChatRequestNotebookData(cell); + + } else if (request.locationData?.type === ChatAgentLocation.Terminal) { + // TBD + } + + return { request, location, history: convertedHistory }; + } + async $invokeAgent(handle: number, requestDto: Dto, context: { history: IChatAgentHistoryEntryDto[] }, token: CancellationToken): Promise { const agent = this._agents.get(handle); if (!agent) { throw new Error(`[CHAT](${handle}) CANNOT invoke agent because the agent is not registered`); } - const request = revive(requestDto); + let stream: ChatAgentResponseStream | undefined; - // Init session disposables - let sessionDisposables = this._sessionDisposables.get(request.sessionId); - if (!sessionDisposables) { - sessionDisposables = new DisposableStore(); - this._sessionDisposables.set(request.sessionId, sessionDisposables); - } - - const stream = new ChatAgentResponseStream(agent.extension, request, this._proxy, this._commands.converter, sessionDisposables); try { - const convertedHistory = await this.prepareHistoryTurns(request.agentId, context); - - // in-place converting for location-data - let location2: vscode.ChatRequestEditorData | vscode.ChatRequestNotebookData | undefined; - if (request.locationData?.type === ChatAgentLocation.Editor) { - // editor data - const document = this._documents.getDocument(request.locationData.document); - location2 = new extHostTypes.ChatRequestEditorData(document, typeConvert.Selection.to(request.locationData.selection), typeConvert.Range.to(request.locationData.wholeRange)); - - } else if (request.locationData?.type === ChatAgentLocation.Notebook) { - // notebook data - const cell = this._documents.getDocument(request.locationData.sessionInputUri); - location2 = new extHostTypes.ChatRequestNotebookData(cell); - - } else if (request.locationData?.type === ChatAgentLocation.Terminal) { - // TBD + const { request, location, history } = await this._createRequest(requestDto, context); + + // Init session disposables + let sessionDisposables = this._sessionDisposables.get(request.sessionId); + if (!sessionDisposables) { + sessionDisposables = new DisposableStore(); + this._sessionDisposables.set(request.sessionId, sessionDisposables); } + stream = new ChatAgentResponseStream(agent.extension, request, this._proxy, this._commands.converter, sessionDisposables); + const task = agent.invoke( - typeConvert.ChatAgentRequest.to(request, location2), - { history: convertedHistory }, + typeConvert.ChatAgentRequest.to(request, location), + { history }, stream.apiObject, token ); @@ -354,7 +391,7 @@ export class ExtHostChatAgents2 extends Disposable implements ExtHostChatAgentsS } catch (err) { const msg = `result.metadata MUST be JSON.stringify-able. Got error: ${err.message}`; this._logService.error(`[${agent.extension.identifier.value}] [@${agent.id}] ${msg}`, agent.extension); - return { errorDetails: { message: msg }, timings: stream.timings, nextQuestion: result.nextQuestion }; + return { errorDetails: { message: msg }, timings: stream?.timings, nextQuestion: result.nextQuestion }; } } let errorDetails: IChatResponseErrorDetails | undefined; @@ -368,7 +405,7 @@ export class ExtHostChatAgents2 extends Disposable implements ExtHostChatAgentsS checkProposedApiEnabled(agent.extension, 'chatParticipantPrivate'); } - return { errorDetails, timings: stream.timings, metadata: result?.metadata, nextQuestion: result?.nextQuestion } satisfies IChatAgentResult; + return { errorDetails, timings: stream?.timings, metadata: result?.metadata, nextQuestion: result?.nextQuestion } satisfies IChatAgentResult; }), token); } catch (e) { this._logService.error(e, agent.extension); @@ -380,7 +417,7 @@ export class ExtHostChatAgents2 extends Disposable implements ExtHostChatAgentsS return { errorDetails: { message: toErrorMessage(e), responseIsIncomplete: true } }; } finally { - stream.close(); + stream?.close(); } } diff --git a/src/vs/workbench/contrib/chat/common/chatAgents.ts b/src/vs/workbench/contrib/chat/common/chatAgents.ts index 8992ac1406873..bc387d63bd67e 100644 --- a/src/vs/workbench/contrib/chat/common/chatAgents.ts +++ b/src/vs/workbench/contrib/chat/common/chatAgents.ts @@ -83,6 +83,21 @@ export interface IChatAgentImplementation { provideSampleQuestions?(location: ChatAgentLocation, token: CancellationToken): ProviderResult; } +export interface IChatParticipantDetectionResult { + participant: string; + command?: string; +} + +export interface IChatParticipantMetadata { + participant: string; + command?: string; + description?: string; +} + +export interface IChatParticipantDetectionProvider { + provideParticipantDetection(request: IChatAgentRequest, history: IChatAgentHistoryEntry[], options: { location: ChatAgentLocation; participants: IChatParticipantMetadata[] }, token: CancellationToken): Promise; +} + export type IChatAgent = IChatAgentData & IChatAgentImplementation; export interface IChatAgentCommand extends IRawChatCommandContribution { @@ -173,6 +188,7 @@ export interface IChatAgentService { registerAgentImplementation(id: string, agent: IChatAgentImplementation): IDisposable; registerDynamicAgent(data: IChatAgentData, agentImpl: IChatAgentImplementation): IDisposable; registerAgentCompletionProvider(id: string, provider: (query: string, token: CancellationToken) => Promise): IDisposable; + registerChatParticipantDetectionProvider(handle: number, provider: IChatParticipantDetectionProvider): IDisposable; getAgentCompletionItems(id: string, query: string, token: CancellationToken): Promise; invokeAgent(agent: string, request: IChatAgentRequest, progress: (part: IChatProgress) => void, history: IChatAgentHistoryEntry[], token: CancellationToken): Promise; getFollowups(id: string, request: IChatAgentRequest, result: IChatAgentResult, history: IChatAgentHistoryEntry[], token: CancellationToken): Promise; @@ -384,6 +400,53 @@ export class ChatAgentService implements IChatAgentService { return data.impl.provideFollowups(request, result, history, token); } + + private _chatParticipantDetectionProviders = new Map(); + registerChatParticipantDetectionProvider(handle: number, provider: IChatParticipantDetectionProvider) { + this._chatParticipantDetectionProviders.set(handle, provider); + return toDisposable(() => { + this._chatParticipantDetectionProviders.delete(handle); + }); + } + + async detectAgentOrCommand(request: IChatAgentRequest, history: IChatAgentHistoryEntry[], options: { location: ChatAgentLocation }, token: CancellationToken): Promise<{ agent: IChatAgentData; command?: IChatAgentCommand } | undefined> { + // TODO@joyceerhl should we have a selector to be able to narrow down which provider to use + const provider = Iterable.first(this._chatParticipantDetectionProviders.values()); + if (!provider) { + return; + } + + const participants = this.getAgents().reduce((acc, a) => { + acc.push({ participant: a.id, description: undefined }); + for (const command of a.slashCommands) { + acc.push({ participant: a.id, command: command.name, description: undefined }); + } + return acc; + }, []); + + const result = await provider.provideParticipantDetection(request, history, { ...options, participants }, token); + if (!result) { + return; + } + + const agent = this.getAgent(result.participant); + if (!agent) { + // Couldn't find a participant matching the participant detection result + return; + } + + if (!result.command) { + return { agent }; + } + + const command = agent?.slashCommands.find(c => c.name === result.command); + if (!command) { + // Couldn't find a slash command matching the participant detection result + return; + } + + return { agent, command }; + } } export class MergedChatAgent implements IChatAgent { diff --git a/src/vs/workbench/contrib/chat/test/common/voiceChatService.test.ts b/src/vs/workbench/contrib/chat/test/common/voiceChatService.test.ts index 2ad2f91c6b1f8..a1257cad61680 100644 --- a/src/vs/workbench/contrib/chat/test/common/voiceChatService.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/voiceChatService.test.ts @@ -12,7 +12,7 @@ import { ensureNoDisposablesAreLeakedInTestSuite } from 'vs/base/test/common/uti import { ProviderResult } from 'vs/editor/common/languages'; import { ExtensionIdentifier } from 'vs/platform/extensions/common/extensions'; import { MockContextKeyService } from 'vs/platform/keybinding/test/common/mockKeybindingService'; -import { ChatAgentLocation, IChatAgent, IChatAgentCommand, IChatAgentCompletionItem, IChatAgentData, IChatAgentHistoryEntry, IChatAgentImplementation, IChatAgentMetadata, IChatAgentRequest, IChatAgentResult, IChatAgentService } from 'vs/workbench/contrib/chat/common/chatAgents'; +import { ChatAgentLocation, IChatAgent, IChatAgentCommand, IChatAgentCompletionItem, IChatAgentData, IChatAgentHistoryEntry, IChatAgentImplementation, IChatAgentMetadata, IChatAgentRequest, IChatAgentResult, IChatAgentService, IChatParticipantDetectionProvider } from 'vs/workbench/contrib/chat/common/chatAgents'; import { IChatModel } from 'vs/workbench/contrib/chat/common/chatModel'; import { IChatProgress, IChatFollowup } from 'vs/workbench/contrib/chat/common/chatService'; import { IVoiceChatSessionOptions, IVoiceChatTextEvent, VoiceChatService } from 'vs/workbench/contrib/chat/common/voiceChatService'; @@ -52,6 +52,12 @@ suite('VoiceChat', () => { ]; class TestChatAgentService implements IChatAgentService { + registerChatParticipantDetectionProvider(handle: number, provider: IChatParticipantDetectionProvider): IDisposable { + throw new Error('Method not implemented.'); + } + detectAgentOrCommand(request: IChatAgentRequest, history: IChatAgentHistoryEntry[], options: { location: ChatAgentLocation }, token: CancellationToken): Promise<{ agent: IChatAgentData; command?: IChatAgentCommand } | undefined> { + throw new Error('Method not implemented.'); + } _serviceBrand: undefined; readonly onDidChangeAgents = Event.None; registerAgentImplementation(id: string, agent: IChatAgentImplementation): IDisposable { throw new Error(); } diff --git a/src/vscode-dts/vscode.proposed.chatParticipantAdditions.d.ts b/src/vscode-dts/vscode.proposed.chatParticipantAdditions.d.ts index fce4277f139d3..05db1eb1c9986 100644 --- a/src/vscode-dts/vscode.proposed.chatParticipantAdditions.d.ts +++ b/src/vscode-dts/vscode.proposed.chatParticipantAdditions.d.ts @@ -245,6 +245,23 @@ declare module 'vscode' { * The chat extension should not activate if it doesn't support the current version. */ export const _version: 1 | number; + + export function registerChatParticipantDetectionProvider(participantDetectionProvider: ChatParticipantDetectionProvider): Disposable; + } + + export interface ChatParticipantMetadata { + participant: string; + command?: string; + description?: string; + } + + export interface ChatParticipantDetectionResult { + participant: string; + command?: string; + } + + export interface ChatParticipantDetectionProvider { + provideParticipantDetection(chatRequest: ChatRequest, context: ChatContext, options: { participants?: ChatParticipantMetadata[]; location: ChatLocation }, token: CancellationToken): ProviderResult; } /*