'Remote coding agent' entrypoint in chat widget (#252363)

* initial frame

* simple system working

* wired up history

* restore css

* restore

* comment

* Add and implement ChatSummarizer from proposed. defaultChatParticipant.d.ts (https://github.com/microsoft/vscode-copilot/issues/18919)

* remove demo extension

* tidy

src/vs/platform/extensions/common/extensionsApiProposals.ts

@@ -305,6 +305,9 @@ const _allApiProposals = {
 	quickPickSortByLabel: {
 		proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.quickPickSortByLabel.d.ts',
 	},
+	remoteCodingAgents: {
+		proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.remoteCodingAgents.d.ts',
+	},
 	resolvers: {
 		proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.resolvers.d.ts',
 	},

src/vs/workbench/api/browser/mainThreadChatAgents2.ts

@@ -187,6 +187,9 @@ export class MainThreadChatAgents2 extends Disposable implements MainThreadChatA
 			provideChatTitle: (history, token) => {
 				return this._proxy.$provideChatTitle(handle, history, token);
 			},
+			provideChatSummary: (history, token) => {
+				return this._proxy.$provideChatSummary(handle, history, token);
+			},
 		};
 
 		let disposable: IDisposable;

src/vs/workbench/api/common/extHost.protocol.ts

@@ -1364,6 +1364,7 @@ export interface ExtHostChatAgentsShape2 {
 	$acceptAction(handle: number, result: IChatAgentResult, action: IChatUserActionEvent): void;
 	$invokeCompletionProvider(handle: number, query: string, token: CancellationToken): Promise<IChatAgentCompletionItem[]>;
 	$provideChatTitle(handle: number, context: IChatAgentHistoryEntryDto[], token: CancellationToken): Promise<string | undefined>;
+	$provideChatSummary(handle: number, context: IChatAgentHistoryEntryDto[], token: CancellationToken): Promise<string | undefined>;
 	$releaseSession(sessionId: string): void;
 	$detectChatParticipant(handle: number, request: Dto<IChatAgentRequest>, context: { history: IChatAgentHistoryEntryDto[] }, options: { participants: IChatParticipantMetadata[]; location: ChatAgentLocation }, token: CancellationToken): Promise<IChatParticipantDetectionResult | null | undefined>;
 	$provideRelatedFiles(handle: number, request: Dto<IChatRequestDraft>, token: CancellationToken): Promise<Dto<IChatRelatedFile>[] | undefined>;

src/vs/workbench/api/common/extHostChatAgents2.ts

@@ -757,6 +757,16 @@ export class ExtHostChatAgents2 extends Disposable implements ExtHostChatAgentsS
 		const history = await this.prepareHistoryTurns(agent.extension, agent.id, { history: context });
 		return await agent.provideTitle({ history }, token);
 	}
+
+	async $provideChatSummary(handle: number, context: IChatAgentHistoryEntryDto[], token: CancellationToken): Promise<string | undefined> {
+		const agent = this._agents.get(handle);
+		if (!agent) {
+			return;
+		}
+
+		const history = await this.prepareHistoryTurns(agent.extension, agent.id, { history: context });
+		return await agent.provideSummary({ history }, token);
+	}
 }
 
 class ExtHostParticipantDetector {
@@ -786,6 +796,7 @@ class ExtHostChatAgent {
 	private _agentVariableProvider?: { provider: vscode.ChatParticipantCompletionItemProvider; triggerCharacters: string[] };
 	private _additionalWelcomeMessage?: string | vscode.MarkdownString | undefined;
 	private _titleProvider?: vscode.ChatTitleProvider | undefined;
+	private _summarizer?: vscode.ChatSummarizer | undefined;
 	private _requester: vscode.ChatRequesterInformation | undefined;
 	private _pauseStateEmitter = new Emitter<vscode.ChatParticipantPauseStateEvent>();
 
@@ -841,6 +852,14 @@ class ExtHostChatAgent {
 		return await this._titleProvider.provideChatTitle(context, token) ?? undefined;
 	}
 
+	async provideSummary(context: vscode.ChatContext, token: CancellationToken): Promise<string | undefined> {
+		if (!this._summarizer) {
+			return;
+		}
+
+		return await this._summarizer.provideChatSummary(context, token) ?? undefined;
+	}
+
 	get apiAgent(): vscode.ChatParticipant {
 		let disposed = false;
 		let updateScheduled = false;
@@ -974,6 +993,14 @@ class ExtHostChatAgent {
 				checkProposedApiEnabled(that.extension, 'defaultChatParticipant');
 				return that._titleProvider;
 			},
+			set summarizer(v) {
+				checkProposedApiEnabled(that.extension, 'defaultChatParticipant');
+				that._summarizer = v;
+			},
+			get summarizer() {
+				checkProposedApiEnabled(that.extension, 'defaultChatParticipant');
+				return that._summarizer;
+			},
 			get onDidChangePauseState() {
 				checkProposedApiEnabled(that.extension, 'chatParticipantAdditions');
 				return that._pauseStateEmitter.event;

src/vs/workbench/contrib/chat/browser/actions/chatExecuteActions.ts

@@ -4,6 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { basename } from '../../../../../base/common/resources.js';
+import { CancellationToken } from '../../../../../base/common/cancellation.js';
 import { Codicon } from '../../../../../base/common/codicons.js';
 import { KeyCode, KeyMod } from '../../../../../base/common/keyCodes.js';
 import { ThemeIcon } from '../../../../../base/common/themables.js';
@@ -13,11 +14,13 @@ import { localize, localize2 } from '../../../../../nls.js';
 import { Action2, MenuId, registerAction2 } from '../../../../../platform/actions/common/actions.js';
 import { ICommandService } from '../../../../../platform/commands/common/commands.js';
 import { IConfigurationService } from '../../../../../platform/configuration/common/configuration.js';
-import { ContextKeyExpr } from '../../../../../platform/contextkey/common/contextkey.js';
+import { ContextKeyExpr, IContextKeyService } from '../../../../../platform/contextkey/common/contextkey.js';
 import { IDialogService } from '../../../../../platform/dialogs/common/dialogs.js';
 import { IInstantiationService } from '../../../../../platform/instantiation/common/instantiation.js';
 import { KeybindingWeight } from '../../../../../platform/keybinding/common/keybindingsRegistry.js';
+import { IChatAgentService, IChatAgentHistoryEntry } from '../../common/chatAgents.js';
 import { ChatContextKeys } from '../../common/chatContextKeys.js';
+import { toChatHistoryContent } from '../../common/chatModel.js';
 import { ChatMode2, IChatMode, validateChatMode2 } from '../../common/chatModes.js';
 import { chatVariableLeader } from '../../common/chatParserTypes.js';
 import { IChatService } from '../../common/chatService.js';
@@ -28,6 +31,7 @@ import { IChatWidget, IChatWidgetService } from '../chat.js';
 import { getEditingSessionContext } from '../chatEditing/chatEditingActions.js';
 import { ACTION_ID_NEW_CHAT, CHAT_CATEGORY, handleCurrentEditingSession, handleModeSwitch } from './chatActions.js';
 import { EditorContextKeys } from '../../../../../editor/common/editorContextKeys.js';
+import { IRemoteCodingAgentsService } from '../../../remoteCodingAgents/common/remoteCodingAgentsService.js';
 
 export interface IVoiceChatExecuteActionContext {
 	readonly disableTimeout?: boolean;
@@ -454,6 +458,100 @@ class SubmitWithoutDispatchingAction extends Action2 {
 	}
 }
 
+export class CreateRemoteAgentJobAction extends Action2 {
+	static readonly ID = 'workbench.action.chat.createRemoteAgentJob';
+
+	constructor() {
+		const precondition = ContextKeyExpr.and(
+			ContextKeyExpr.or(ChatContextKeys.inputHasText, ChatContextKeys.hasPromptFile),
+			whenNotInProgressOrPaused,
+			ChatContextKeys.remoteJobCreating.negate(),
+		);
+
+		super({
+			id: CreateRemoteAgentJobAction.ID,
+			title: localize2('actions.chat.createRemoteJob', "Create Remote Job"),
+			icon: Codicon.cloudUpload,
+			precondition,
+			toggled: {
+				condition: ChatContextKeys.remoteJobCreating,
+				icon: Codicon.sync,
+				tooltip: localize('remoteJobCreating', "Remote job is being created"),
+			},
+			menu: {
+				id: MenuId.ChatExecute,
+				group: 'navigation',
+				order: 0,
+				when: ChatContextKeys.hasRemoteCodingAgent
+			}
+		});
+	}
+
+	async run(accessor: ServicesAccessor, ...args: any[]) {
+		const contextKeyService = accessor.get(IContextKeyService);
+		const remoteJobCreatingKey = ChatContextKeys.remoteJobCreating.bindTo(contextKeyService);
+
+		try {
+			remoteJobCreatingKey.set(true);
+
+			const remoteCodingAgent = accessor.get(IRemoteCodingAgentsService);
+			const commandService = accessor.get(ICommandService);
+			const widgetService = accessor.get(IChatWidgetService);
+			const chatAgentService = accessor.get(IChatAgentService);
+
+			const widget = widgetService.lastFocusedWidget;
+			if (!widget) {
+				return;
+			}
+			const session = widget.viewModel?.sessionId;
+			if (!session) {
+				return;
+			}
+
+			const userPrompt = widget.getInput();
+			widget.setInput();
+
+			const chatModel = widget.viewModel?.model;
+			const chatRequests = chatModel.getRequests();
+			const agents = remoteCodingAgent.getRegisteredAgents();
+			const defaultAgent = chatAgentService.getDefaultAgent(ChatAgentLocation.Panel);
+
+			const agent = agents[0]; // TODO: We just pick the first one for testing
+			if (!agent) {
+				return;
+			}
+
+			let summary: string | undefined;
+			if (defaultAgent && chatRequests.length > 0) {
+				const historyEntries: IChatAgentHistoryEntry[] = chatRequests
+					.filter(req => req.response) // Only include completed requests
+					.map(req => ({
+						request: {
+							sessionId: session,
+							requestId: req.id,
+							agentId: req.response?.agent?.id ?? '',
+							message: req.message.text,
+							command: req.response?.slashCommand?.name,
+							variables: req.variableData,
+							location: ChatAgentLocation.Panel,
+							editedFileEvents: req.editedFileEvents,
+						},
+						response: toChatHistoryContent(req.response!.response.value),
+						result: req.response?.result ?? {}
+					}));
+
+				summary = await chatAgentService.getChatSummary(defaultAgent.id, historyEntries, CancellationToken.None);
+			}
+			await commandService.executeCommand(agent.command, {
+				userPrompt,
+				summary: summary || `Chat session with ${chatRequests.length} messages`
+			});
+		} finally {
+			remoteJobCreatingKey.set(false);
+		}
+	}
+}
+
 export class ChatSubmitWithCodebaseAction extends Action2 {
 	static readonly ID = 'workbench.action.chat.submitWithCodebase';
 
@@ -642,6 +740,7 @@ export function registerChatExecuteActions() {
 	registerAction2(CancelAction);
 	registerAction2(SendToNewChatAction);
 	registerAction2(ChatSubmitWithCodebaseAction);
+	registerAction2(CreateRemoteAgentJobAction);
 	registerAction2(ToggleChatModeAction);
 	registerAction2(ToggleRequestPausedAction);
 	registerAction2(SwitchToNextModelAction);

src/vs/workbench/contrib/chat/common/chatAgents.ts

@@ -73,6 +73,7 @@ export interface IChatAgentImplementation {
 	setRequestPaused?(requestId: string, isPaused: boolean): void;
 	provideFollowups?(request: IChatAgentRequest, result: IChatAgentResult, history: IChatAgentHistoryEntry[], token: CancellationToken): Promise<IChatFollowup[]>;
 	provideChatTitle?: (history: IChatAgentHistoryEntry[], token: CancellationToken) => Promise<string | undefined>;
+	provideChatSummary?: (history: IChatAgentHistoryEntry[], token: CancellationToken) => Promise<string | undefined>;
 }
 
 export interface IChatParticipantDetectionResult {
@@ -195,6 +196,7 @@ export interface IChatAgentService {
 	setRequestPaused(agent: string, requestId: string, isPaused: boolean): void;
 	getFollowups(id: string, request: IChatAgentRequest, result: IChatAgentResult, history: IChatAgentHistoryEntry[], token: CancellationToken): Promise<IChatFollowup[]>;
 	getChatTitle(id: string, history: IChatAgentHistoryEntry[], token: CancellationToken): Promise<string | undefined>;
+	getChatSummary(id: string, history: IChatAgentHistoryEntry[], token: CancellationToken): Promise<string | undefined>;
 	getAgent(id: string, includeDisabled?: boolean): IChatAgentData | undefined;
 	getAgentByFullyQualifiedId(id: string): IChatAgentData | undefined;
 	getAgents(): IChatAgentData[];
@@ -502,6 +504,15 @@ export class ChatAgentService extends Disposable implements IChatAgentService {
 		return data.impl.provideChatTitle(history, token);
 	}
 
+	async getChatSummary(id: string, history: IChatAgentHistoryEntry[], token: CancellationToken): Promise<string | undefined> {
+		const data = this._agents.get(id);
+		if (!data?.impl?.provideChatSummary) {
+			return undefined;
+		}
+
+		return data.impl.provideChatSummary(history, token);
+	}
+
 	registerChatParticipantDetectionProvider(handle: number, provider: IChatParticipantDetectionProvider) {
 		this._chatParticipantDetectionProviders.set(handle, provider);
 		return toDisposable(() => {

src/vs/workbench/contrib/chat/common/chatContextKeys.ts

@@ -53,6 +53,9 @@ export namespace ChatContextKeys {
 
 	export const languageModelsAreUserSelectable = new RawContextKey<boolean>('chatModelsAreUserSelectable', false, { type: 'boolean', description: localize('chatModelsAreUserSelectable', "True when the chat model can be selected manually by the user.") });
 
+	export const remoteJobCreating = new RawContextKey<boolean>('chatRemoteJobCreating', false, { type: 'boolean', description: localize('chatRemoteJobCreating', "True when a remote coding agent job is being created.") });
+	export const hasRemoteCodingAgent = new RawContextKey<boolean>('hasRemoteCodingAgent', false, localize('hasRemoteCodingAgent', "Whether any remote coding agent is available"));
+
 	export const Setup = {
 		hidden: new RawContextKey<boolean>('chatSetupHidden', false, true), 		// True when chat setup is explicitly hidden.
 		installed: new RawContextKey<boolean>('chatSetupInstalled', false, true),  	// True when the chat extension is installed and enabled.

src/vs/workbench/contrib/chat/test/common/voiceChatService.test.ts

@@ -81,6 +81,7 @@ suite('VoiceChat', () => {
 		getAgentCompletionItems(id: string, query: string, token: CancellationToken): Promise<IChatAgentCompletionItem[]> { throw new Error('Method not implemented.'); }
 		agentHasDupeName(id: string): boolean { throw new Error('Method not implemented.'); }
 		getChatTitle(id: string, history: IChatAgentHistoryEntry[], token: CancellationToken): Promise<string | undefined> { throw new Error('Method not implemented.'); }
+		getChatSummary(id: string, history: IChatAgentHistoryEntry[], token: CancellationToken): Promise<string | undefined> { throw new Error('Method not implemented.'); }
 		hasToolsAgent: boolean = false;
 		hasChatParticipantDetectionProviders(): boolean {
 			throw new Error('Method not implemented.');

src/vs/workbench/contrib/remoteCodingAgents/browser/remoteCodingAgents.contribution.ts

@@ -0,0 +1,96 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { Disposable } from '../../../../base/common/lifecycle.js';
+import { localize } from '../../../../nls.js';
+import { MenuRegistry } from '../../../../platform/actions/common/actions.js';
+import { ILogService } from '../../../../platform/log/common/log.js';
+import { Registry } from '../../../../platform/registry/common/platform.js';
+import { IWorkbenchContribution, Extensions as WorkbenchExtensions, IWorkbenchContributionsRegistry } from '../../../common/contributions.js';
+import { isProposedApiEnabled } from '../../../services/extensions/common/extensions.js';
+import { ExtensionsRegistry } from '../../../services/extensions/common/extensionsRegistry.js';
+import { LifecyclePhase } from '../../../services/lifecycle/common/lifecycle.js';
+import { IRemoteCodingAgent, IRemoteCodingAgentsService } from '../common/remoteCodingAgentsService.js';
+
+interface IRemoteCodingAgentExtensionPoint {
+	id: string;
+	command: string;
+	displayName: string;
+	description?: string;
+	when?: string;
+}
+
+const extensionPoint = ExtensionsRegistry.registerExtensionPoint<IRemoteCodingAgentExtensionPoint[]>({
+	extensionPoint: 'remoteCodingAgents',
+	jsonSchema: {
+		description: localize('remoteCodingAgentsExtPoint', 'Contributes remote coding agent integrations to the chat widget.'),
+		type: 'array',
+		items: {
+			type: 'object',
+			properties: {
+				id: {
+					description: localize('remoteCodingAgentsExtPoint.id', 'A unique identifier for this item.'),
+					type: 'string',
+				},
+				command: {
+					description: localize('remoteCodingAgentsExtPoint.command', 'Identifier of the command to execute. The command must be declared in the "commands" section.'),
+					type: 'string'
+				},
+				displayName: {
+					description: localize('remoteCodingAgentsExtPoint.displayName', 'A user-friendly name for this item which is used for display in menus.'),
+					type: 'string'
+				},
+				description: {
+					description: localize('remoteCodingAgentsExtPoint.description', 'Description of the remote agent for use in menus and tooltips.'),
+					type: 'string'
+				},
+				when: {
+					description: localize('remoteCodingAgentsExtPoint.when', 'Condition which must be true to show this item.'),
+					type: 'string'
+				},
+			},
+			required: ['command', 'displayName'],
+		}
+	}
+});
+
+export class RemoteCodingAgentsContribution extends Disposable implements IWorkbenchContribution {
+	constructor(
+		@ILogService private readonly logService: ILogService,
+		@IRemoteCodingAgentsService private readonly remoteCodingAgentsService: IRemoteCodingAgentsService
+	) {
+		super();
+		extensionPoint.setHandler(extensions => {
+			for (const ext of extensions) {
+				if (!isProposedApiEnabled(ext.description, 'remoteCodingAgents')) {
+					continue;
+				}
+				if (!Array.isArray(ext.value)) {
+					continue;
+				}
+				for (const contribution of ext.value) {
+					const command = MenuRegistry.getCommand(contribution.command);
+					if (!command) {
+						continue;
+					}
+
+					// TODO: Handle 'when' clause
+
+					const agent: IRemoteCodingAgent = {
+						id: contribution.id,
+						command: contribution.command,
+						displayName: contribution.displayName,
+						description: contribution.description
+					};
+					this.logService.info(`Registering remote coding agent: ${agent.displayName} (${agent.command})`);
+					this.remoteCodingAgentsService.registerAgent(agent);
+				}
+			}
+		});
+	}
+}
+
+const workbenchRegistry = Registry.as<IWorkbenchContributionsRegistry>(WorkbenchExtensions.Workbench);
+workbenchRegistry.registerWorkbenchContribution(RemoteCodingAgentsContribution, LifecyclePhase.Restored);

src/vs/workbench/contrib/remoteCodingAgents/common/remoteCodingAgentsService.ts

@@ -0,0 +1,50 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { IContextKey, IContextKeyService } from '../../../../platform/contextkey/common/contextkey.js';
+import { InstantiationType, registerSingleton } from '../../../../platform/instantiation/common/extensions.js';
+import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
+import { ChatContextKeys } from '../../chat/common/chatContextKeys.js';
+
+export interface IRemoteCodingAgent {
+	id: string;
+	command: string;
+	displayName: string;
+	description?: string;
+}
+
+export interface IRemoteCodingAgentsService {
+	readonly _serviceBrand: undefined;
+	getRegisteredAgents(): IRemoteCodingAgent[];
+	registerAgent(agent: IRemoteCodingAgent): void;
+}
+
+export const IRemoteCodingAgentsService = createDecorator<IRemoteCodingAgentsService>('remoteCodingAgentsService');
+
+export class RemoteCodingAgentsService implements IRemoteCodingAgentsService {
+	readonly _serviceBrand: undefined;
+	private readonly _ctxHasRemoteCodingAgent: IContextKey<boolean>;
+
+	constructor(
+		@IContextKeyService private readonly contextKeyService: IContextKeyService
+	) {
+		this._ctxHasRemoteCodingAgent = ChatContextKeys.hasRemoteCodingAgent.bindTo(this.contextKeyService);
+	}
+
+	private agents: IRemoteCodingAgent[] = [];
+
+	getRegisteredAgents(): IRemoteCodingAgent[] {
+		return this.agents;
+	}
+
+	registerAgent(agent: IRemoteCodingAgent): void {
+		if (!this.agents.includes(agent)) {
+			this.agents.push(agent);
+			this._ctxHasRemoteCodingAgent.set(true);
+		}
+	}
+}
+
+registerSingleton(IRemoteCodingAgentsService, RemoteCodingAgentsService, InstantiationType.Delayed);

src/vs/workbench/workbench.common.main.ts

@@ -374,6 +374,9 @@ import './contrib/userDataProfile/browser/userDataProfile.contribution.js';
 // Continue Edit Session
 import './contrib/editSessions/browser/editSessions.contribution.js';
 
+// Remote Coding Agents
+import './contrib/remoteCodingAgents/browser/remoteCodingAgents.contribution.js';
+
 // Code Actions
 import './contrib/codeActions/browser/codeActions.contribution.js';
 

src/vscode-dts/vscode.proposed.defaultChatParticipant.d.ts

@@ -29,6 +29,10 @@ declare module 'vscode' {
 		provideChatTitle(context: ChatContext, token: CancellationToken): ProviderResult<string>;
 	}
 
+	export interface ChatSummarizer {
+		provideChatSummary(context: ChatContext, token: CancellationToken): ProviderResult<string>;
+	}
+
 	export interface ChatParticipant {
 		/**
 		 * A string that will be added before the listing of chat participants in `/help`.
@@ -47,6 +51,7 @@ declare module 'vscode' {
 
 		additionalWelcomeMessage?: string | MarkdownString;
 		titleProvider?: ChatTitleProvider;
+		summarizer?: ChatSummarizer;
 		requester?: ChatRequesterInformation;
 	}
 }

src/vscode-dts/vscode.proposed.remoteCodingAgents.d.ts

@@ -0,0 +1,8 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+// empty placeholder for coding agent contribution point from core
+
+// @joshspicer