Skip to content

Commit

Permalink
feat: chat participation detection api (#224651)
Browse files Browse the repository at this point in the history 
* feat: chat participation detection api
  • Loading branch information
@joyceerhl
joyceerhl authored Aug 2, 2024
1 parent 161ef6b commit d4c164e
Show file tree
Hide file tree
Showing 7 changed files with 190 additions and 34 deletions.
20 changes: 18 additions & 2 deletions src/vs/workbench/api/browser/mainThreadChatAgents2.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@ import { ILanguageFeaturesService } from 'vs/editor/common/services/languageFeat
import { ExtensionIdentifier } from 'vs/platform/extensions/common/extensions';
import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
import { ILogService } from 'vs/platform/log/common/log';
import { ExtHostChatAgentsShape2, ExtHostContext, IChatProgressDto, IDynamicChatAgentProps, IExtensionChatAgentMetadata, MainContext, MainThreadChatAgentsShape2 } from 'vs/workbench/api/common/extHost.protocol';
import { ExtHostChatAgentsShape2, ExtHostContext, IChatParticipantMetadata, IChatProgressDto, IDynamicChatAgentProps, IExtensionChatAgentMetadata, MainContext, MainThreadChatAgentsShape2 } from 'vs/workbench/api/common/extHost.protocol';
import { IChatWidgetService } from 'vs/workbench/contrib/chat/browser/chat';
import { ChatInputPart } from 'vs/workbench/contrib/chat/browser/chatInputPart';
import { AddDynamicVariableAction, IAddDynamicVariableContext } from 'vs/workbench/contrib/chat/browser/contrib/chatDynamicVariables';
import { ChatAgentLocation, IChatAgentImplementation, IChatAgentService } from 'vs/workbench/contrib/chat/common/chatAgents';
import { ChatAgentLocation, IChatAgentHistoryEntry, IChatAgentImplementation, IChatAgentRequest, IChatAgentService } from 'vs/workbench/contrib/chat/common/chatAgents';
import { ChatRequestAgentPart } from 'vs/workbench/contrib/chat/common/chatParserTypes';
import { ChatRequestParser } from 'vs/workbench/contrib/chat/common/chatRequestParser';
import { IChatContentReference, IChatFollowup, IChatProgress, IChatService, IChatTask, IChatWarningMessage } from 'vs/workbench/contrib/chat/common/chatService';
Expand Down Expand Up @@ -76,6 +76,8 @@ export class MainThreadChatAgents2 extends Disposable implements MainThreadChatA
private readonly _agentCompletionProviders = this._register(new DisposableMap<number, IDisposable>());
private readonly _agentIdsToCompletionProviders = this._register(new DisposableMap<string, IDisposable>);

private readonly _chatParticipantDetectionProviders = this._register(new DisposableMap<number, IDisposable>());

private readonly _pendingProgress = new Map<string, (part: IChatProgress) => void>();
private readonly _proxy: ExtHostChatAgentsShape2;

Expand Down Expand Up @@ -298,6 +300,20 @@ export class MainThreadChatAgents2 extends Disposable implements MainThreadChatA
this._agentCompletionProviders.deleteAndDispose(handle);
this._agentIdsToCompletionProviders.deleteAndDispose(id);
}

$registerChatParticipantDetectionProvider(handle: number): void {
this._chatParticipantDetectionProviders.set(handle, this._chatAgentService.registerChatParticipantDetectionProvider(handle,
{
provideParticipantDetection: async (request: IChatAgentRequest, history: IChatAgentHistoryEntry[], options: { location: ChatAgentLocation; participants: IChatParticipantMetadata[] }, token: CancellationToken) => {
return await this._proxy.$detectChatParticipant(handle, request, { history }, options, token);
}
}
));
}

$unregisterChatParticipantDetectionProvider(handle: number): void {
this._chatParticipantDetectionProviders.deleteAndDispose(handle);
}
}


Expand Down
4 changes: 4 additions & 0 deletions src/vs/workbench/api/common/extHost.api.impl.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1519,6 +1519,10 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
checkProposedApiEnabled(extension, 'chatParticipantPrivate');
return extHostChatAgents2.createDynamicChatAgent(extension, id, dynamicProps, handler);
},
registerChatParticipantDetectionProvider(provider: vscode.ChatParticipantDetectionProvider) {
checkProposedApiEnabled(extension, 'chatParticipantAdditions');
return extHostChatAgents2.registerChatParticipantDetectionProvider(provider);
},
};

// namespace: lm
Expand Down
13 changes: 13 additions & 0 deletions src/vs/workbench/api/common/extHost.protocol.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1244,6 +1244,8 @@ export interface IDynamicChatAgentProps {

export interface MainThreadChatAgentsShape2 extends IDisposable {
$registerAgent(handle: number, extension: ExtensionIdentifier, id: string, metadata: IExtensionChatAgentMetadata, dynamicProps: IDynamicChatAgentProps | undefined): void;
$registerChatParticipantDetectionProvider(handle: number): void;
$unregisterChatParticipantDetectionProvider(handle: number): void;
$registerAgentCompletionsProvider(handle: number, id: string, triggerCharacters: string[]): void;
$unregisterAgentCompletionsProvider(handle: number, id: string): void;
$updateAgent(handle: number, metadataUpdate: IExtensionChatAgentMetadata): void;
Expand Down Expand Up @@ -1284,6 +1286,17 @@ export interface ExtHostChatAgentsShape2 {
$provideWelcomeMessage(handle: number, location: ChatAgentLocation, token: CancellationToken): Promise<(string | IMarkdownString)[] | undefined>;
$provideSampleQuestions(handle: number, location: ChatAgentLocation, token: CancellationToken): Promise<IChatFollowup[] | undefined>;
$releaseSession(sessionId: string): void;
$detectChatParticipant(handle: number, request: Dto<IChatAgentRequest>, context: { history: IChatAgentHistoryEntryDto[] }, options: { participants: IChatParticipantMetadata[]; location: ChatAgentLocation }, token: CancellationToken): Promise<IChatParticipantDetectionResult | null | undefined>;
}
export interface IChatParticipantMetadata {
participant: string;
command?: string;
description?: string;
}

export interface IChatParticipantDetectionResult {
participant: string;
command?: string;
}

export type IChatVariableResolverProgressDto =
Expand Down
99 changes: 68 additions & 31 deletions src/vs/workbench/api/common/extHostChatAgents2.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { toErrorMessage } from 'vs/base/common/errorMessage';
import { Emitter } from 'vs/base/common/event';
import { IMarkdownString } from 'vs/base/common/htmlContent';
import { Iterable } from 'vs/base/common/iterator';
import { Disposable, DisposableMap, DisposableStore } from 'vs/base/common/lifecycle';
import { Disposable, DisposableMap, DisposableStore, toDisposable } from 'vs/base/common/lifecycle';
import { revive } from 'vs/base/common/marshalling';
import { StopWatch } from 'vs/base/common/stopwatch';
import { assertType } from 'vs/base/common/types';
Expand Down Expand Up @@ -270,6 +270,9 @@ export class ExtHostChatAgents2 extends Disposable implements ExtHostChatAgentsS
private readonly _agents = new Map<number, ExtHostChatAgent>();
private readonly _proxy: MainThreadChatAgentsShape2;

private static _participantDetectionProviderIdPool = 0;
private readonly _participantDetectionProviders = new Map<number, vscode.ChatParticipantDetectionProvider>();

private readonly _sessionDisposables: DisposableMap<string, DisposableStore> = this._register(new DisposableMap());
private readonly _completionDisposables: DisposableMap<number, DisposableStore> = this._register(new DisposableMap());

Expand Down Expand Up @@ -305,44 +308,78 @@ export class ExtHostChatAgents2 extends Disposable implements ExtHostChatAgentsS
return agent.apiAgent;
}

registerChatParticipantDetectionProvider(provider: vscode.ChatParticipantDetectionProvider): vscode.Disposable {
const handle = ExtHostChatAgents2._participantDetectionProviderIdPool++;
this._participantDetectionProviders.set(handle, provider);
this._proxy.$registerChatParticipantDetectionProvider(handle);
return toDisposable(() => {
this._participantDetectionProviders.delete(handle);
this._proxy.$unregisterChatParticipantDetectionProvider(handle);
});
}

async $detectChatParticipant(handle: number, requestDto: Dto<IChatAgentRequest>, context: { history: IChatAgentHistoryEntryDto[] }, options: { location: ChatAgentLocation; participants?: vscode.ChatParticipantMetadata[] }, token: CancellationToken): Promise<vscode.ChatParticipantDetectionResult | null | undefined> {
const { request, location, history } = await this._createRequest(requestDto, context);

const provider = this._participantDetectionProviders.get(handle);
if (!provider) {
return undefined;
}

return provider.provideParticipantDetection(
typeConvert.ChatAgentRequest.to(request, location),
{ history },
{ participants: options.participants, location: typeConvert.ChatLocation.to(options.location) },
token
);
}

private async _createRequest(requestDto: Dto<IChatAgentRequest>, context: { history: IChatAgentHistoryEntryDto[] }) {
const request = revive<IChatAgentRequest>(requestDto);
const convertedHistory = await this.prepareHistoryTurns(request.agentId, context);

// in-place converting for location-data
let location: vscode.ChatRequestEditorData | vscode.ChatRequestNotebookData | undefined;
if (request.locationData?.type === ChatAgentLocation.Editor) {
// editor data
const document = this._documents.getDocument(request.locationData.document);
location = new extHostTypes.ChatRequestEditorData(document, typeConvert.Selection.to(request.locationData.selection), typeConvert.Range.to(request.locationData.wholeRange));

} else if (request.locationData?.type === ChatAgentLocation.Notebook) {
// notebook data
const cell = this._documents.getDocument(request.locationData.sessionInputUri);
location = new extHostTypes.ChatRequestNotebookData(cell);

} else if (request.locationData?.type === ChatAgentLocation.Terminal) {
// TBD
}

return { request, location, history: convertedHistory };
}

async $invokeAgent(handle: number, requestDto: Dto<IChatAgentRequest>, context: { history: IChatAgentHistoryEntryDto[] }, token: CancellationToken): Promise<IChatAgentResult | undefined> {
const agent = this._agents.get(handle);
if (!agent) {
throw new Error(`[CHAT](${handle}) CANNOT invoke agent because the agent is not registered`);
}

const request = revive<IChatAgentRequest>(requestDto);
let stream: ChatAgentResponseStream | undefined;

// Init session disposables
let sessionDisposables = this._sessionDisposables.get(request.sessionId);
if (!sessionDisposables) {
sessionDisposables = new DisposableStore();
this._sessionDisposables.set(request.sessionId, sessionDisposables);
}

const stream = new ChatAgentResponseStream(agent.extension, request, this._proxy, this._commands.converter, sessionDisposables);
try {
const convertedHistory = await this.prepareHistoryTurns(request.agentId, context);

// in-place converting for location-data
let location2: vscode.ChatRequestEditorData | vscode.ChatRequestNotebookData | undefined;
if (request.locationData?.type === ChatAgentLocation.Editor) {
// editor data
const document = this._documents.getDocument(request.locationData.document);
location2 = new extHostTypes.ChatRequestEditorData(document, typeConvert.Selection.to(request.locationData.selection), typeConvert.Range.to(request.locationData.wholeRange));

} else if (request.locationData?.type === ChatAgentLocation.Notebook) {
// notebook data
const cell = this._documents.getDocument(request.locationData.sessionInputUri);
location2 = new extHostTypes.ChatRequestNotebookData(cell);

} else if (request.locationData?.type === ChatAgentLocation.Terminal) {
// TBD
const { request, location, history } = await this._createRequest(requestDto, context);

// Init session disposables
let sessionDisposables = this._sessionDisposables.get(request.sessionId);
if (!sessionDisposables) {
sessionDisposables = new DisposableStore();
this._sessionDisposables.set(request.sessionId, sessionDisposables);
}

stream = new ChatAgentResponseStream(agent.extension, request, this._proxy, this._commands.converter, sessionDisposables);

const task = agent.invoke(
typeConvert.ChatAgentRequest.to(request, location2),
{ history: convertedHistory },
typeConvert.ChatAgentRequest.to(request, location),
{ history },
stream.apiObject,
token
);
Expand All @@ -354,7 +391,7 @@ export class ExtHostChatAgents2 extends Disposable implements ExtHostChatAgentsS
} catch (err) {
const msg = `result.metadata MUST be JSON.stringify-able. Got error: ${err.message}`;
this._logService.error(`[${agent.extension.identifier.value}] [@${agent.id}] ${msg}`, agent.extension);
return { errorDetails: { message: msg }, timings: stream.timings, nextQuestion: result.nextQuestion };
return { errorDetails: { message: msg }, timings: stream?.timings, nextQuestion: result.nextQuestion };
}
}
let errorDetails: IChatResponseErrorDetails | undefined;
Expand All @@ -368,7 +405,7 @@ export class ExtHostChatAgents2 extends Disposable implements ExtHostChatAgentsS
checkProposedApiEnabled(agent.extension, 'chatParticipantPrivate');
}

return { errorDetails, timings: stream.timings, metadata: result?.metadata, nextQuestion: result?.nextQuestion } satisfies IChatAgentResult;
return { errorDetails, timings: stream?.timings, metadata: result?.metadata, nextQuestion: result?.nextQuestion } satisfies IChatAgentResult;
}), token);
} catch (e) {
this._logService.error(e, agent.extension);
Expand All @@ -380,7 +417,7 @@ export class ExtHostChatAgents2 extends Disposable implements ExtHostChatAgentsS
return { errorDetails: { message: toErrorMessage(e), responseIsIncomplete: true } };

} finally {
stream.close();
stream?.close();
}
}

Expand Down
63 changes: 63 additions & 0 deletions src/vs/workbench/contrib/chat/common/chatAgents.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,21 @@ export interface IChatAgentImplementation {
provideSampleQuestions?(location: ChatAgentLocation, token: CancellationToken): ProviderResult<IChatFollowup[] | undefined>;
}

export interface IChatParticipantDetectionResult {
participant: string;
command?: string;
}

export interface IChatParticipantMetadata {
participant: string;
command?: string;
description?: string;
}

export interface IChatParticipantDetectionProvider {
provideParticipantDetection(request: IChatAgentRequest, history: IChatAgentHistoryEntry[], options: { location: ChatAgentLocation; participants: IChatParticipantMetadata[] }, token: CancellationToken): Promise<IChatParticipantDetectionResult | null | undefined>;
}

export type IChatAgent = IChatAgentData & IChatAgentImplementation;

export interface IChatAgentCommand extends IRawChatCommandContribution {
Expand Down Expand Up @@ -173,6 +188,7 @@ export interface IChatAgentService {
registerAgentImplementation(id: string, agent: IChatAgentImplementation): IDisposable;
registerDynamicAgent(data: IChatAgentData, agentImpl: IChatAgentImplementation): IDisposable;
registerAgentCompletionProvider(id: string, provider: (query: string, token: CancellationToken) => Promise<IChatAgentCompletionItem[]>): IDisposable;
registerChatParticipantDetectionProvider(handle: number, provider: IChatParticipantDetectionProvider): IDisposable;
getAgentCompletionItems(id: string, query: string, token: CancellationToken): Promise<IChatAgentCompletionItem[]>;
invokeAgent(agent: string, request: IChatAgentRequest, progress: (part: IChatProgress) => void, history: IChatAgentHistoryEntry[], token: CancellationToken): Promise<IChatAgentResult>;
getFollowups(id: string, request: IChatAgentRequest, result: IChatAgentResult, history: IChatAgentHistoryEntry[], token: CancellationToken): Promise<IChatFollowup[]>;
Expand Down Expand Up @@ -384,6 +400,53 @@ export class ChatAgentService implements IChatAgentService {

return data.impl.provideFollowups(request, result, history, token);
}

private _chatParticipantDetectionProviders = new Map<number, IChatParticipantDetectionProvider>();
registerChatParticipantDetectionProvider(handle: number, provider: IChatParticipantDetectionProvider) {
this._chatParticipantDetectionProviders.set(handle, provider);
return toDisposable(() => {
this._chatParticipantDetectionProviders.delete(handle);
});
}

async detectAgentOrCommand(request: IChatAgentRequest, history: IChatAgentHistoryEntry[], options: { location: ChatAgentLocation }, token: CancellationToken): Promise<{ agent: IChatAgentData; command?: IChatAgentCommand } | undefined> {
// TODO@joyceerhl should we have a selector to be able to narrow down which provider to use
const provider = Iterable.first(this._chatParticipantDetectionProviders.values());
if (!provider) {
return;
}

const participants = this.getAgents().reduce<IChatParticipantMetadata[]>((acc, a) => {
acc.push({ participant: a.id, description: undefined });
for (const command of a.slashCommands) {
acc.push({ participant: a.id, command: command.name, description: undefined });
}
return acc;
}, []);

const result = await provider.provideParticipantDetection(request, history, { ...options, participants }, token);
if (!result) {
return;
}

const agent = this.getAgent(result.participant);
if (!agent) {
// Couldn't find a participant matching the participant detection result
return;
}

if (!result.command) {
return { agent };
}

const command = agent?.slashCommands.find(c => c.name === result.command);
if (!command) {
// Couldn't find a slash command matching the participant detection result
return;
}

return { agent, command };
}
}

export class MergedChatAgent implements IChatAgent {
Expand Down
8 changes: 7 additions & 1 deletion src/vs/workbench/contrib/chat/test/common/voiceChatService.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { ensureNoDisposablesAreLeakedInTestSuite } from 'vs/base/test/common/uti
import { ProviderResult } from 'vs/editor/common/languages';
import { ExtensionIdentifier } from 'vs/platform/extensions/common/extensions';
import { MockContextKeyService } from 'vs/platform/keybinding/test/common/mockKeybindingService';
import { ChatAgentLocation, IChatAgent, IChatAgentCommand, IChatAgentCompletionItem, IChatAgentData, IChatAgentHistoryEntry, IChatAgentImplementation, IChatAgentMetadata, IChatAgentRequest, IChatAgentResult, IChatAgentService } from 'vs/workbench/contrib/chat/common/chatAgents';
import { ChatAgentLocation, IChatAgent, IChatAgentCommand, IChatAgentCompletionItem, IChatAgentData, IChatAgentHistoryEntry, IChatAgentImplementation, IChatAgentMetadata, IChatAgentRequest, IChatAgentResult, IChatAgentService, IChatParticipantDetectionProvider } from 'vs/workbench/contrib/chat/common/chatAgents';
import { IChatModel } from 'vs/workbench/contrib/chat/common/chatModel';
import { IChatProgress, IChatFollowup } from 'vs/workbench/contrib/chat/common/chatService';
import { IVoiceChatSessionOptions, IVoiceChatTextEvent, VoiceChatService } from 'vs/workbench/contrib/chat/common/voiceChatService';
Expand Down Expand Up @@ -52,6 +52,12 @@ suite('VoiceChat', () => {
];

class TestChatAgentService implements IChatAgentService {
registerChatParticipantDetectionProvider(handle: number, provider: IChatParticipantDetectionProvider): IDisposable {
throw new Error('Method not implemented.');
}
detectAgentOrCommand(request: IChatAgentRequest, history: IChatAgentHistoryEntry[], options: { location: ChatAgentLocation }, token: CancellationToken): Promise<{ agent: IChatAgentData; command?: IChatAgentCommand } | undefined> {
throw new Error('Method not implemented.');
}
_serviceBrand: undefined;
readonly onDidChangeAgents = Event.None;
registerAgentImplementation(id: string, agent: IChatAgentImplementation): IDisposable { throw new Error(); }
Expand Down
17 changes: 17 additions & 0 deletions src/vscode-dts/vscode.proposed.chatParticipantAdditions.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,23 @@ declare module 'vscode' {
* The chat extension should not activate if it doesn't support the current version.
*/
export const _version: 1 | number;

export function registerChatParticipantDetectionProvider(participantDetectionProvider: ChatParticipantDetectionProvider): Disposable;
}

export interface ChatParticipantMetadata {
participant: string;
command?: string;
description?: string;
}

export interface ChatParticipantDetectionResult {
participant: string;
command?: string;
}

export interface ChatParticipantDetectionProvider {
provideParticipantDetection(chatRequest: ChatRequest, context: ChatContext, options: { participants?: ChatParticipantMetadata[]; location: ChatLocation }, token: CancellationToken): ProviderResult<ChatParticipantDetectionResult>;
}

/*
Expand Down

0 comments on commit d4c164e

Please sign in to comment.