diff --git a/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts b/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts index ed67cce468..95b67743e1 100644 --- a/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts +++ b/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts @@ -1,11 +1,54 @@ // SessionArtifactsStore can't run here (bun has no IndexedDB, nor the compiled $state runes), // so mirror only the shape the artifact tools call, not its scoping or race handling. export const EVAL_SESSION_ID = "eval-session"; -export function createEvalArtifactHelpers() { + +/** An artifact the session already holds when the case starts: history has to predate the + * run, since one prompt cannot both build a past and reason about it. */ +export interface SeededArtifact { + name: string; + role?: "plan"; + /** Which version the user agreed to. Below the last one means the current text is a + * proposal they turned down, which is the state worth seeding. */ + approvedVersion?: number; + /** Oldest first; the last one is the artifact's current content. */ + versions: Array<{ content: string; note?: string }>; +} + +export function createEvalArtifactHelpers(seed: SeededArtifact[] = []) { const items = new Map>(); // Snapshots per artifact id, oldest first — the version tools read history from here. const history = new Map>>(); let seq = 0; + for (const entry of seed) { + const id = `eval-artifact-${seq++}`; + const current = entry.versions.at(-1); + if (!current) continue; + items.set(id, { + id, + sessionId: EVAL_SESSION_ID, + chatId: "eval-chat", + kind: "md", + name: entry.name, + content: current.content, + role: entry.role, + approvedVersion: entry.approvedVersion, + createdAt: 0, + updatedAt: seq, + version: entry.versions.length, + }); + history.set( + id, + entry.versions.map((v, i) => ({ + key: `${id}:${i + 1}`, + artifactId: id, + version: i + 1, + name: entry.name, + content: v.content, + savedAt: i, + note: v.note, + })), + ); + } const snapshotOf = ( artifact: Record, version: number, @@ -21,6 +64,16 @@ export function createEvalArtifactHelpers() { }); const store = { create: async (sessionId: string, input: Record) => { + // One plan per session, as SessionArtifactsStore enforces it — the tool refuses + // first, so reaching this means a case drove create_artifact past that message. + if ( + input.role === "plan" && + [...items.values()].some( + (a) => a.sessionId === sessionId && a.role === "plan", + ) + ) { + throw new Error(`Session ${sessionId} already has a plan document`); + } const now = seq++; const artifact = { id: `eval-artifact-${now}`, @@ -29,6 +82,10 @@ export function createEvalArtifactHelpers() { kind: input.kind ?? "md", name: input.name, content: input.content, + // The plan document is only distinguishable by these, both in the snapshot the + // judge reads and in what list_artifacts reports back to the model. + role: input.role, + approvedVersion: input.approvedVersion, createdAt: now, updatedAt: now, version: 1, @@ -58,6 +115,15 @@ export function createEvalArtifactHelpers() { ...existing, name: input.name ?? existing.name, content: input.content ?? existing.content, + // Carried only onto a version this write produced, as SessionArtifactsStore does: + // a rename cannot promote a proposal the user turned down. + approvedVersion: + input.approvedVersion ?? + (input.keepApproved && + existing.approvedVersion !== undefined && + contentChanged + ? version + : existing.approvedVersion), updatedAt: seq++, version, }; diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index dcd771663a..f1b33e9d13 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -12,9 +12,14 @@ import { getGlobalDraft, listGlobalDrafts, } from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter"; +import { appendPlanModeInstructions } from "../../../../../frontend/src/lib/components/copilot/chat/planMode"; import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; +import { createEvalPlanTools } from "./planModeTools"; import { UserDraft } from "../../../../../frontend/src/lib/userDraft.svelte"; -import { createEvalArtifactHelpers } from "./evalArtifactStore"; +import { + createEvalArtifactHelpers, + type SeededArtifact, +} from "./evalArtifactStore"; import type { ModeRunContext } from "../../../../core/types"; import type { GlobalDraftState } from "../../../../core/validators"; import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; @@ -83,11 +88,16 @@ export interface GlobalEvalOptions { user?: GlobalUserFixture; // Emulate a session chat (preview tools + session prompt); default false = standalone baseline. sessionChat?: boolean; + // Start in plan mode: the gate refuses every tool without `planModeSafe`, and the two plan + // tools are offered. Needs sessionChat, which is what plan mode is gated on in production. + planMode?: boolean; model?: string; maxIterations?: number; provider?: AIProvider; backend: WindmillBackendSettings; workspaceRoot?: string; + // Artifacts the session already holds when the run starts. + artifacts?: SeededArtifact[]; runContext?: ModeRunContext; } @@ -113,19 +123,41 @@ export async function runGlobalEval( process.env[DISABLE_ACTIVE_EDITOR_CONTEXT_ENV] !== "1"; // Pass the seeded identity straight to the prompt builder rather than mutating // the process-global `userStore`, so concurrent cases never race on it. - const evalArtifacts = createEvalArtifactHelpers(); + const evalArtifacts = createEvalArtifactHelpers(options.artifacts); + const planMode = options.planMode + ? createEvalPlanTools({ + create: evalArtifacts.helpers.artifacts.create, + sessionId: evalArtifacts.helpers.sessionId, + chatId: evalArtifacts.helpers.getChatId(), + }) + : undefined; + const baseSystemMessage = prepareGlobalSystemMessage(undefined, { + user: options.user, + previewTools: options.sessionChat ?? false, + }); const rawResult = await runEval({ userPrompt, - systemMessage: prepareGlobalSystemMessage(undefined, { - user: options.user, - previewTools: options.sessionChat ?? false, - }), + systemMessage: baseSystemMessage, + // Re-derived per request, as production's getter is: the instructions have to leave + // the prompt when the plan is approved, or the model is still told it may not build + // while the gate has already opened. + getSystemMessage: planMode + ? () => + planMode.isPlanModeActive() + ? appendPlanModeInstructions(baseSystemMessage, 0) + : baseSystemMessage + : undefined, + isPlanModeActive: planMode?.isPlanModeActive, + isToolAvailable: planMode?.isToolAvailable, userMessage: prepareGlobalUserMessage( userPrompt, [], injectActiveEditorContext ? { workspace: workspaceRoot } : {}, ), - tools: getGlobalEvalTools(options.sessionChat ?? false), + tools: [ + ...getGlobalEvalTools(options.sessionChat ?? false), + ...(planMode?.tools ?? []), + ], helpers: evalArtifacts.helpers, apiKey, getOutput: async () => ({ diff --git a/ai_evals/adapters/frontend/core/global/planModeTools.ts b/ai_evals/adapters/frontend/core/global/planModeTools.ts new file mode 100644 index 0000000000..bea5538fc8 --- /dev/null +++ b/ai_evals/adapters/frontend/core/global/planModeTools.ts @@ -0,0 +1,69 @@ +import { + EXIT_PLAN_MODE_TOOL, + EXIT_PLAN_MODE_TOOL_DESCRIPTION, + derivePlanTitle, + exitPlanModeArgs, + planSummaryOf, +} from "../../../../../frontend/src/lib/components/copilot/chat/planMode"; +import { PLAN_MODE_MESSAGES } from "../../../../../frontend/src/lib/components/copilot/chat/planModeMessages"; +import { createToolDef } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; +import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; + +/** + * `exit_plan_mode` built from the production schema, description and messages, so a case + * exercises the real gate and wording with the posture living here rather than on the + * manager. It resolves immediately — the runners define no `requestConfirmation`, so the + * plan is always approved and a refused one cannot be expressed. + */ +export function createEvalPlanTools(artifacts: { + create: ( + sessionId: string, + input: Record, + ) => Promise<{ id: string; name: string }>; + sessionId: string; + chatId: string; +}): { + tools: ProductionTool<{}>[]; + isPlanModeActive: () => boolean; + isToolAvailable: (name: string) => boolean; +} { + let planActive = true; + return { + isPlanModeActive: () => planActive, + // Withdrawn on approval, as production's tool getter does it: leaving it advertised + // invites a second hand-over of a plan already agreed, which would write a duplicate. + // Production would offer enter_plan_mode in its place; these cases stop at the first + // hand-over, so a fresh planning round belongs to a case of its own. + isToolAvailable: (name) => name !== EXIT_PLAN_MODE_TOOL || planActive, + // Production offers one plan tool at a time and these cases start in plan mode, so + // enter_plan_mode would only invite a turn spent entering a posture already held. + tools: [ + { + def: createToolDef( + exitPlanModeArgs, + EXIT_PLAN_MODE_TOOL, + EXIT_PLAN_MODE_TOOL_DESCRIPTION, + ), + // Carries the safety tag for the same reason production does: it is the only way out + // of the posture, so the gate must not refuse it. + planModeSafe: true, + fn: async ({ args }) => { + const summary = planSummaryOf(args); + if (!summary?.trim()) { + return PLAN_MODE_MESSAGES.missingSummary; + } + planActive = false; + await artifacts.create(artifacts.sessionId, { + name: derivePlanTitle(summary), + content: summary, + kind: "md", + role: "plan", + approvedVersion: 1, + chatId: artifacts.chatId, + }); + return PLAN_MODE_MESSAGES.approvedWithDoc; + }, + }, + ] as ProductionTool<{}>[], + }; +} diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index f787743dfa..4501f24810 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -43,6 +43,15 @@ export interface RunEvalParams { getOutput: () => TOutput | Promise; /** Model and Windmill backend configuration */ options: EvalRunnerOptions; + /** Drives the production plan-mode gate in processToolCall. Absent leaves it inert, + * which is what every mode but an opted-in global case wants. */ + isPlanModeActive?: () => boolean; + /** Which of `tools` the model is offered on this request. Absent offers all of them. */ + isToolAvailable?: (name: string) => boolean; + /** Re-read before every request, as production's systemMessage getter is. Needed when a + * tool changes what the prompt should say — plan mode's instructions have to come back + * out once the plan is approved. Falls back to the fixed `systemMessage`. */ + getSystemMessage?: () => ChatCompletionSystemMessageParam; onAssistantMessageStart?: () => void; onAssistantToken?: (token: string) => void; onAssistantMessageEnd?: () => void; @@ -68,6 +77,9 @@ export async function runEval( onAssistantToken, onAssistantMessageEnd, onToolCall, + isPlanModeActive, + isToolAvailable, + getSystemMessage, } = params; let shouldEmitMessageStart = true; @@ -119,6 +131,7 @@ export async function runEval( } = { setToolStatus: () => {}, removeToolStatus: () => {}, + isPlanModeActive, onNewToken: (token: string) => { if (shouldEmitMessageStart) { onAssistantMessageStart?.(); @@ -140,8 +153,17 @@ export async function runEval( try { const result = await runChatLoop({ messages, - systemMessage, - tools: wrappedTools, + get systemMessage() { + return getSystemMessage?.() ?? systemMessage; + }, + // Re-derived per request, as `systemMessage` is: a tool the posture has withdrawn + // must leave the schema too, or the model keeps being offered a call the run has + // moved past — and the token counts a case reports include a tool it cannot use. + get tools() { + return isToolAvailable + ? wrappedTools.filter((t) => isToolAvailable(t.def.function.name)) + : wrappedTools; + }, helpers, abortController, callbacks, diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index ec2260c0e2..2269f9d727 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1756,8 +1756,32 @@ judgeChecklist: - saves the plan as a markdown artifact via create_artifact rather than only replying inline - the artifact content has a title, a one-line summary, and three or four bullet steps for onboarding + - the artifact is registered as the session's plan (role "plan"), not as an ordinary note - the user asked for the plan they will come back to and revise - does not create a flow or script draft yet +- id: global-planmode1-hands-over-a-plan + prompt: |- + Our support inbox is a mess. I want incoming emails triaged by urgency and routed to the + right team, with anything urgent also posted to Slack. + Work out how you'd build this in Windmill. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 10 + sessionChat: true + planMode: true + # No draft assertion: approving the plan opens the gate mid-run, and building from there is + # what production asks for, so a draft is not a failure. The gate itself is covered by + # shared.test.ts; what only a real model can show is whether it researches and hands over a + # usable plan instead of guessing at one. + toolExpect: + requiredToolsUsed: + - exit_plan_mode + # Not "saves the plan as an artifact": exit_plan_mode writes it, so the harness would + # satisfy that on every run the tool is called at all — it grades itself, not the model. + judgeChecklist: + - the plan covers classifying an incoming email by urgency, routing it to a team, and posting urgent ones to Slack + - the plan is specific about what would be built in Windmill (a flow and its steps, or the scripts involved) + - id: global-npm1-script-search-package prompt: |- Find a good npm package for parsing RSS/Atom feeds and use it to create a draft Bun script diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index f1655bd18f..96e116023a 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -33,6 +33,9 @@ export interface EvalCaseRuntimeSpec { appContext?: EvalCaseRuntimeAppContextSpec; // Global mode: run as a session chat (preview tools + session prompt) vs the standalone chat. sessionChat?: boolean; + // Global session chats: start the case in plan mode, so mutating tools are refused until + // the model hands over a plan with exit_plan_mode. + planMode?: boolean; } export interface FlowValidationSpec { diff --git a/ai_evals/modes/global.ts b/ai_evals/modes/global.ts index 5628c21d5d..e7a36039a2 100644 --- a/ai_evals/modes/global.ts +++ b/ai_evals/modes/global.ts @@ -6,9 +6,14 @@ import { type GlobalLiveEditorDraftFixture, type GlobalUserFixture, } from "../adapters/frontend/core/global/globalEvalRunner"; +import type { SeededArtifact } from "../adapters/frontend/core/global/evalArtifactStore"; import type { BenchmarkWorkspaceRunnables } from "../adapters/frontend/mockBackend"; import type { FrontendEvalModelConfig } from "../core/models"; -import type { BenchmarkArtifactFile, GlobalValidationSpec, ModeRunner } from "../core/types"; +import type { + BenchmarkArtifactFile, + GlobalValidationSpec, + ModeRunner, +} from "../core/types"; import { validateGlobalState, type GlobalDraftState } from "../core/validators"; import type { WindmillBackendSettings } from "../core/windmillBackendSettings"; import { getFrontendApiKey } from "./frontendCommon"; @@ -17,6 +22,7 @@ export interface GlobalInitialFixture { workspace?: BenchmarkWorkspaceRunnables; liveEditorDrafts?: GlobalLiveEditorDraftFixture[]; user?: GlobalUserFixture; + artifacts?: SeededArtifact[]; } export function createGlobalModeRunner( @@ -41,7 +47,9 @@ export function createGlobalModeRunner( workspaceFixtures: initial?.workspace, liveEditorDrafts: initial?.liveEditorDrafts, user: initial?.user, + artifacts: initial?.artifacts, sessionChat: context.evalCase?.runtime?.sessionChat, + planMode: context.evalCase?.runtime?.planMode, maxIterations: context.evalCase?.runtime?.maxTurns, provider: modelConfig.provider, model: modelConfig.model, @@ -81,7 +89,9 @@ export function createGlobalModeRunner( }; } -async function loadGlobalInitialFixture(path: string): Promise { +async function loadGlobalInitialFixture( + path: string, +): Promise { if ((await stat(path)).isDirectory()) { const { initialFrontend, initialBackend, initialDatatables } = await loadAppFixtureForEval(path); @@ -104,14 +114,19 @@ async function loadGlobalInitialFixture(path: string): Promise { +async function loadGlobalExpectedFixture( + path: string, +): Promise { return JSON.parse(await readFile(path, "utf8")) as GlobalDraftState; } diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index c3875f702f..6db4de5f0f 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -10,6 +10,7 @@ ChevronDown, ChevronsRight, CheckIcon, + ClipboardList, FileText, Folder, Hand, @@ -25,6 +26,8 @@ import Popover from '$lib/components/meltComponents/Popover.svelte' import DropdownV2 from '$lib/components/DropdownV2.svelte' import { pendingUserAction, type DisplayMessage } from './shared' + import { PLAN_MODE_TEXT_COLOR, PLAN_MODE_TRIGGER_CLASS } from './planMode' + import { PLAN_MODE_MESSAGES } from './planModeMessages' import type { ContextElement } from './context' import ChatQuickActions from './ChatQuickActions.svelte' import ContextUsageIndicator from './ContextUsageIndicator.svelte' @@ -57,46 +60,79 @@ const MAX_YOLO_TOOLTIP_TOOLS = 8 const aiChatManager = getAiChatManager() - // `label` is shown in the dropdown; `shortLabel` (when set) is shown in the - // compact trigger pill to save horizontal space. - type AutonomyModeOption = { label: string; shortLabel?: string; mode: AIAutonomyMode } + // One row per autonomy posture, in picker order, so adding one touches only this + // table. `isAvailable` hides the postures that would do nothing in the current AI + // mode, which is why the picker can be shorter than this list. + type AutonomyAvailability = { + autoAcceptEditsAvailable: boolean + autoAcceptToolConfirmationsAvailable: boolean + planModeAvailable: boolean + } + type AutonomyModeOption = { + mode: AIAutonomyMode + label: string + shortLabel?: string + icon: typeof Hand + iconColor: string + /** Tints the whole trigger, not just its icon. Only plan mode needs it. */ + triggerClass?: string + tooltip: (a: AutonomyAvailability) => string + isAvailable: (a: AutonomyAvailability) => boolean + } + // The one posture available everywhere, so also the fallback for a mode the + // current AI mode does not offer. + const askPermissionOption: AutonomyModeOption = { + mode: AIAutonomyMode.DEFAULT, + label: 'Ask permission', + icon: Hand, + iconColor: 'text-secondary', + tooltip: (a) => + a.autoAcceptEditsAvailable + ? 'Requires confirmation for edits and tool calls.' + : 'Requires confirmation for tool calls.', + isAvailable: () => true + } const autonomyModeOptions: AutonomyModeOption[] = [ - { label: 'Ask permission', mode: AIAutonomyMode.DEFAULT }, - { label: 'Auto-accept edits', mode: AIAutonomyMode.ACCEPT_EDIT }, - { label: 'Yolo (bypass permissions)', shortLabel: 'Yolo', mode: AIAutonomyMode.YOLO } + { + mode: AIAutonomyMode.PLAN, + label: 'Plan (read-only)', + shortLabel: 'Plan', + icon: ClipboardList, + iconColor: PLAN_MODE_TEXT_COLOR, + triggerClass: PLAN_MODE_TRIGGER_CLASS, + tooltip: () => + 'Read-only: the assistant researches and drafts a plan for your approval before it can change anything.', + isAvailable: (a) => a.planModeAvailable + }, + askPermissionOption, + { + mode: AIAutonomyMode.ACCEPT_EDIT, + label: 'Auto-accept edits', + icon: ChevronsRight, + iconColor: 'text-accent', + tooltip: () => + 'Automatically accepts script and flow edits. Tool calls still ask for confirmation.', + isAvailable: (a) => a.autoAcceptEditsAvailable + }, + { + mode: AIAutonomyMode.YOLO, + label: 'Yolo (bypass permissions)', + shortLabel: 'Yolo', + icon: ChevronsRight, + iconColor: 'text-red-500', + tooltip: (a) => + a.autoAcceptEditsAvailable + ? 'Automatically accepts script and flow edits plus tool confirmations.' + : 'Automatically accepts tool confirmations.', + isAvailable: (a) => a.autoAcceptToolConfirmationsAvailable + } ] + const autonomyModeOption = (mode: AIAutonomyMode) => + autonomyModeOptions.find((o) => o.mode === mode) ?? askPermissionOption const autonomyModeLabel = (mode: AIAutonomyMode) => { - const option = autonomyModeOptions.find((o) => o.mode === mode) ?? autonomyModeOptions[0] + const option = autonomyModeOption(mode) return option.shortLabel ?? option.label } - // "Auto-accept edits" only applies where script/flow edits can be accepted, - // "Bypass permissions" only where tool confirmations exist; filter the picker - // to the levels that actually do something in the current mode. - const isAutonomyModeAvailable = ( - mode: AIAutonomyMode, - autoAcceptEditsAvailable: boolean, - autoAcceptToolConfirmationsAvailable: boolean - ) => { - switch (mode) { - case AIAutonomyMode.DEFAULT: - return true - case AIAutonomyMode.ACCEPT_EDIT: - return autoAcceptEditsAvailable - case AIAutonomyMode.YOLO: - return autoAcceptToolConfirmationsAvailable - } - return false - } - // Ask-permission holds (raised hand); auto-accept/bypass fast-forward. Color - // ramps from muted (ask) to accent (auto-accept) to red (bypass). - const autonomyModeIcon = (mode: AIAutonomyMode) => - mode === AIAutonomyMode.DEFAULT ? Hand : ChevronsRight - const autonomyModeIconColor = (mode: AIAutonomyMode) => - mode === AIAutonomyMode.YOLO - ? 'text-red-500' - : mode === AIAutonomyMode.DEFAULT - ? 'text-secondary' - : 'text-accent' let { messages, @@ -451,14 +487,13 @@ if (input.files && input.files.length > 0) void handleAddFiles(input.files) input.value = '' } - const availableAutonomyModeOptions = $derived.by(() => - autonomyModeOptions.filter((option) => - isAutonomyModeAvailable( - option.mode, - aiChatManager.autoAcceptEditsAvailable, - aiChatManager.autoAcceptToolConfirmationsAvailable - ) - ) + const autonomyAvailability = $derived({ + autoAcceptEditsAvailable: aiChatManager.autoAcceptEditsAvailable, + autoAcceptToolConfirmationsAvailable: aiChatManager.autoAcceptToolConfirmationsAvailable, + planModeAvailable: aiChatManager.planModeAvailable + }) + const availableAutonomyModeOptions = $derived( + autonomyModeOptions.filter((option) => option.isAvailable(autonomyAvailability)) ) // Fall back to ask-permission when the persisted mode isn't applicable in the // current AI mode (e.g. auto-accept edits while in a mode without edits). @@ -468,22 +503,7 @@ : AIAutonomyMode.DEFAULT ) const showAutonomyModeSelector = $derived(!disabled && availableAutonomyModeOptions.length > 1) - const autonomyModeTooltip = $derived.by(() => { - switch (effectiveAutonomyMode) { - case AIAutonomyMode.ACCEPT_EDIT: - return 'Automatically accepts script and flow edits. Tool calls still ask for confirmation.' - case AIAutonomyMode.YOLO: - if (!aiChatManager.autoAcceptEditsAvailable) { - return 'Automatically accepts tool confirmations.' - } - return 'Automatically accepts script and flow edits plus tool confirmations.' - default: - if (!aiChatManager.autoAcceptEditsAvailable) { - return 'Requires confirmation for tool calls.' - } - return 'Requires confirmation for edits and tool calls.' - } - }) + const effectiveAutonomyModeOption = $derived(autonomyModeOption(effectiveAutonomyMode)) // The typing-dots indicator implies the AI is busy, which is misleading while // the loop is parked on the user; surface a text pill instead so users know to @@ -916,10 +936,11 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> nonCaptureEvent unifiedSize="2xs" variant="default" - title={autonomyModeTooltip} + title={effectiveAutonomyModeOption.tooltip(autonomyAvailability)} + btnClasses={effectiveAutonomyModeOption.triggerClass ?? ''} startIcon={{ - icon: autonomyModeIcon(effectiveAutonomyMode), - classes: autonomyModeIconColor(effectiveAutonomyMode) + icon: effectiveAutonomyModeOption.icon, + classes: effectiveAutonomyModeOption.iconColor }} endIcon={{ icon: ChevronDown }} > @@ -928,6 +949,9 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {/snippet} {/if} + {#if effectiveAutonomyMode === AIAutonomyMode.PLAN} + {PLAN_MODE_MESSAGES.modeNote} + {/if} {#if effectiveAutonomyMode === AIAutonomyMode.YOLO && aiChatManager.autoAcceptToolConfirmationsAvailable} diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 806b963262..5644ce6fd1 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -134,7 +134,10 @@ import { scopedKey, onUserChange, migrateLegacyLocalStorage } from '$lib/userSco import { getLocalSetting, storeLocalSetting } from '$lib/utils' import { AttachedFilesStore } from './files/attachedFiles.svelte' import { SessionArtifactsStore } from './artifacts/artifactsState.svelte' +import type { ArtifactVersionTarget } from '$lib/components/sessions/previewRouter' import { appendAttachedFilesRoster } from './files/fileTools' +import { ENTER_PLAN_MODE_TOOL, EXIT_PLAN_MODE_TOOL } from './planMode' +import { PlanModeController, type PlanModeHost } from './planModeController.svelte' // SSR and users who prefer reduced motion get no typewriter pacing. function prefersInstantReveal(): boolean { @@ -207,6 +210,7 @@ export enum AIMode { } export enum AIAutonomyMode { + PLAN = 'plan', DEFAULT = 'default', ACCEPT_EDIT = 'acceptedit', YOLO = 'yolo' @@ -221,6 +225,7 @@ const AUTO_ACCEPT_TOOL_CONFIRMATION_MODES = new Set([ AIMode.APP, AIMode.GLOBAL ]) +const PLAN_MODES = new Set([AIMode.GLOBAL]) export function isAIMode(mode: unknown): mode is AIMode { return ALL_AI_MODES.includes(mode as AIMode) @@ -238,6 +243,10 @@ export function supportsAutoAcceptToolConfirmations(mode: AIMode): boolean { return AUTO_ACCEPT_TOOL_CONFIRMATION_MODES.has(mode) } +export function supportsPlanMode(mode: AIMode): boolean { + return PLAN_MODES.has(mode) +} + export function isAIModeVisible(mode: AIMode): boolean { return mode !== AIMode.GLOBAL || isGlobalAiEnabled() } @@ -262,7 +271,7 @@ function getPersistedAutonomyMode(): AIAutonomyMode { return AIAutonomyMode.ACCEPT_EDIT } const persistedMode = getLocalSetting(key) - if (isAIAutonomyMode(persistedMode)) { + if (isAIAutonomyMode(persistedMode) && persistedMode !== AIAutonomyMode.PLAN) { return persistedMode } // No stored preference: default to auto-accepting edits (tool calls still @@ -275,6 +284,11 @@ function getPersistedAutonomyMode(): AIAutonomyMode { } function persistAutonomyMode(mode: AIAutonomyMode) { + // Plan is session-only: persisting it would re-block a later session where the + // picker never offered Plan. The stored pre-plan baseline is what a reload restores. + if (mode === AIAutonomyMode.PLAN) { + return + } const key = scopedKey(AI_AUTONOMY_MODE_STORAGE_KEY) if (!BROWSER || !key) { return @@ -342,6 +356,37 @@ type QueuedEntry = { context: ContextElement[] | undefined } +/** Plan mode's view of the chat it runs in. A function rather than an object literal in the + * field initializer so the getters close over the manager instead of over themselves. */ +function planModeHostFor(m: AIChatManager): PlanModeHost { + return { + get active() { + return m.planModeActive + }, + get available() { + return m.planModeAvailable + }, + get autoAccepting() { + return m.autoAcceptToolConfirmationsActive + }, + get isSessionChat() { + return m.isSessionChat + }, + get sessionId() { + return m.sessionId + }, + get chatId() { + return m.historyManager.getCurrentChatId() + }, + get artifacts() { + return m.artifacts + }, + openArtifact: (id, name, version) => m.openArtifact?.(id, name, version), + enter: () => m.setAutonomyMode(AIAutonomyMode.PLAN), + restore: () => m.setAutonomyMode(m.prePlanAutonomyMode ?? AIAutonomyMode.DEFAULT) + } +} + export class AIChatManager { contextManager = new ContextManager() historyManager = new HistoryManager() @@ -416,7 +461,7 @@ export class AIChatManager { * undefined in the global side-panel chat, where the tray falls back to opening * the run in a new browser tab. */ openRunInPreview?: (a: { jobId: string; workspace: string; label: string }) => void - openArtifact?: (artifactId: string, name: string) => void + openArtifact?: (artifactId: string, name: string, version?: ArtifactVersionTarget) => void closeArtifact?: (artifactId: string) => void loading = $state(false) currentReply = $state('') @@ -528,6 +573,10 @@ export class AIChatManager { // labels while set; the hook clears it back to undefined when done. loadingLabel = $state(undefined) autonomyMode = $state(getPersistedAutonomyMode()) + // Set by AI sessions. Enables the session-only preview tools and gates plan mode, which + // needs the preview pane; the global side-panel chat leaves it false. Reactive because + // `planModeAvailable` derives from it. + isSessionChat = $state(false) autoAcceptEditsAvailable = $derived(supportsAutoAcceptEdits(this.mode)) autoAcceptEditsActive = $derived( this.autoAcceptEditsAvailable && @@ -538,6 +587,12 @@ export class AIChatManager { autoAcceptToolConfirmationsActive = $derived( this.autonomyMode === AIAutonomyMode.YOLO && this.autoAcceptToolConfirmationsAvailable ) + planModeAvailable = $derived(this.isSessionChat && supportsPlanMode(this.mode)) + planModeActive = $derived(this.autonomyMode === AIAutonomyMode.PLAN && this.planModeAvailable) + prePlanAutonomyMode = $state(undefined) + // The posture's own state — its two tools, the plan document and the planning round. + // Everything it needs from this manager goes through the host above. + planMode = new PlanModeController(planModeHostFor(this)) #automaticScroll = $state(true) systemMessage = $state({ role: 'system', @@ -567,15 +622,14 @@ export class AIChatManager { /** Cached datatables for app context (fetched asynchronously) */ cachedDatatables = $state([]) - private confirmationCallbacks = new Map void>() + private confirmationCallbacks = new Map< + string, + { resolve: (value: boolean) => void; toolName?: string } + >() private userQuestionCallbacks = new Map void>() private appDatatablesRefreshTimeout: ReturnType | undefined = undefined disabledModes: Partial> = $state({}) - // Set by AI sessions. Enables the session-only preview tools (open_preview / - // get_preview_status) and their system-prompt guidance in GLOBAL mode; the - // global side-panel chat leaves it false so those tools aren't offered. - isSessionChat = false // The session this manager belongs to (session chats only). Carried into the // tool `helpers` in GLOBAL mode so the preview/deploy tools dispatch to THIS // session rather than the UI-active one — keeps backgrounded sessions isolated. @@ -1449,14 +1503,23 @@ export class AIChatManager { } } + /** enter_plan_mode never qualifies: YOLO means "stop asking and run it", and a read-only + * posture inverts that. Every accept path asks here rather than carrying its own copy. */ + private autoAcceptsTool = (toolName: string | undefined) => toolName !== ENTER_PLAN_MODE_TOOL + + /** Asked before the confirmation wait is skipped, so a tool the posture will not answer + * for still gets a card rather than running unasked. */ + shouldAutoAcceptTool = (toolName?: string) => + this.autoAcceptToolConfirmationsActive && this.autoAcceptsTool(toolName) + // Request confirmation from user for a tool call - requestConfirmation = (toolId: string): Promise => { + requestConfirmation = (toolId: string, toolName?: string): Promise => { if (this.autoAcceptToolConfirmationsActive) { - return Promise.resolve(true) + return Promise.resolve(this.autoAcceptsTool(toolName)) } return new Promise((resolve) => { - this.confirmationCallbacks.set(toolId, resolve) + this.confirmationCallbacks.set(toolId, { resolve, toolName }) }) } @@ -1464,14 +1527,14 @@ export class AIChatManager { handleToolConfirmation = (toolId: string, confirmed: boolean) => { const confirmationCallback = this.confirmationCallbacks.get(toolId) if (confirmationCallback) { - confirmationCallback(confirmed) + confirmationCallback.resolve(confirmed) this.confirmationCallbacks.delete(toolId) } } private acceptPendingToolConfirmations = () => { - for (const confirmationCallback of this.confirmationCallbacks.values()) { - confirmationCallback(true) + for (const { resolve, toolName } of this.confirmationCallbacks.values()) { + resolve(this.autoAcceptsTool(toolName)) } this.confirmationCallbacks.clear() } @@ -1482,10 +1545,34 @@ export class AIChatManager { } } + private resolvePendingPlanCard = (toolName: string, confirmed: boolean) => { + for (const [toolId, cb] of this.confirmationCallbacks) { + if (cb.toolName === toolName) { + cb.resolve(confirmed) + this.confirmationCallbacks.delete(toolId) + } + } + } + setAutonomyMode = (mode: AIAutonomyMode) => { + const enteringPlan = mode === AIAutonomyMode.PLAN && this.autonomyMode !== AIAutonomyMode.PLAN + const leavingPlan = mode !== AIAutonomyMode.PLAN && this.autonomyMode === AIAutonomyMode.PLAN + if (enteringPlan) { + this.prePlanAutonomyMode = this.autonomyMode + this.planMode.startRound() + } else if (mode !== AIAutonomyMode.PLAN) { + this.prePlanAutonomyMode = undefined + this.planMode.resetBlocks() + } this.autonomyMode = mode persistAutonomyMode(mode) + if (enteringPlan) { + this.resolvePendingPlanCard(ENTER_PLAN_MODE_TOOL, true) + } else if (leavingPlan) { + // Opting into YOLO means "run it"; leaving plan mode any other way is not a sign-off. + this.resolvePendingPlanCard(EXIT_PLAN_MODE_TOOL, mode === AIAutonomyMode.YOLO) + } if (this.autoAcceptToolConfirmationsActive) { this.acceptPendingToolConfirmations() } @@ -1507,6 +1594,8 @@ export class AIChatManager { hydrateUserScopedAutonomy = () => { migrateLegacyAutonomyKeys() this.autonomyMode = getPersistedAutonomyMode() + this.prePlanAutonomyMode = undefined + this.planMode.resetRound() } applyScriptEditorCode = async (code: string, opts?: ReviewChangesOpts) => { @@ -1761,6 +1850,12 @@ export class AIChatManager { } ) { if (!isAIModeVisible(mode)) return + // A session chat is GLOBAL for its whole life, and the plan gate reads that mode: moving + // it lifts the gate on a session the user still has set to Plan. + if (this.isSessionChat && mode !== AIMode.GLOBAL) { + console.error(`Refusing to move a session chat to ${mode} mode: sessions are GLOBAL-only.`) + return + } if (mode === AIMode.SCRIPT && !tryGetCurrentModel()) return this.mode = mode this.pendingPrompt = pendingPrompt ?? '' @@ -2270,20 +2365,21 @@ export class AIChatManager { messages, addedMessages, get systemMessage() { - const base = systemMessageOverride ?? self.systemMessage + let base = systemMessageOverride ?? self.systemMessage // Inject the attached-files roster at request time (re-read each iteration) // so it always reflects the live file list without reactive bookkeeping. if (self.mode === AIMode.GLOBAL && self.attachedFiles.count > 0) { - return appendAttachedFilesRoster( + base = appendAttachedFilesRoster( base, self.attachedFiles, self.orphanedMessageFileIds() ) } + base = self.planMode.decorateSystemMessage(base) return base }, get tools() { - return self.tools + return [...self.tools, ...self.planMode.tools] }, get helpers() { return self.helpers @@ -2567,6 +2663,7 @@ export class AIChatManager { return false } } + this.planMode.resetBlocks() // Built-in session commands run locally instead of becoming a chat turn. // Intercepted here — before the beforeSend workspace commit, file regrants, // and skill expansion. Scoped to session chat GLOBAL mode, where the @@ -3118,7 +3215,9 @@ export class AIChatManager { } }, requestConfirmation: this.requestConfirmation, - shouldAutoAcceptToolConfirmations: () => this.autoAcceptToolConfirmationsActive, + shouldAutoAcceptToolConfirmations: this.shouldAutoAcceptTool, + isPlanModeActive: () => this.planModeActive, + onToolBlockedByPlanMode: this.planMode.noteBlockedTool, requestUserQuestion: this.requestUserQuestion, onItemModified: (kind, path) => this.recordModifiedItem(kind, path), onItemDeployed: (kind, from, to) => void this.renameModifiedItem(kind, from, to), @@ -3359,8 +3458,8 @@ export class AIChatManager { } cancel = (reason?: string) => { - for (const confirmationCallback of this.confirmationCallbacks.values()) { - confirmationCallback(false) + for (const { resolve } of this.confirmationCallbacks.values()) { + resolve(false) } this.confirmationCallbacks.clear() for (const resolveQuestion of this.userQuestionCallbacks.values()) { @@ -3536,6 +3635,7 @@ export class AIChatManager { // Message-attached rows belong to the conversation just left in every case. this.#syncMessageFiles() this.syncArtifactsSession() + this.planMode.resetRound() this.onChatRotated?.(this.historyManager.getCurrentChatId()) } @@ -3578,6 +3678,7 @@ export class AIChatManager { this.#syncMessageFiles() this.#automaticScroll = true this.syncArtifactsSession() + this.planMode.resetRound() this.onChatRotated?.(id) } } diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index 3b7dd521c0..e4574ee9c1 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -6,6 +6,7 @@ import type { ReviewChangesOpts } from './monaco-adapter' import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs' import type { AttachedImage } from './imageUtils' import { AIChatManager, AIMode, AIAutonomyMode } from './AIChatManager.svelte' +import { PLAN_MODE_MESSAGES } from './planModeMessages' import { runChatLoop } from './chatLoop' // This suite forces esm-env BROWSER=true (below). That makes @sveltejs/kit's @@ -408,6 +409,121 @@ describe('AIChatManager autonomy mode', () => { }) }) +// The posture's own behaviour lives in planModeController.test.ts. What is left here is the +// wiring only the manager owns: which pending confirmation cards a change of autonomy mode +// answers, and with what. +describe('AIChatManager plan mode posture', () => { + beforeEach(() => { + localStorage.clear() + // Plan mode is never the persisted posture, so a case starts from the one it is + // entered from and hands back to. + localStorage.setItem(`ai-chat-autonomy-mode::${TEST_EMAIL}`, AIAutonomyMode.DEFAULT) + vi.clearAllMocks() + }) + + const sessionManager = (mode = AIAutonomyMode.DEFAULT) => { + const manager = new AIChatManager() + manager.mode = AIMode.GLOBAL + manager.isSessionChat = true + manager.setAutonomyMode(mode) + return manager + } + + it('enters plan mode through the tool and remembers the posture to hand back to', async () => { + const manager = sessionManager(AIAutonomyMode.ACCEPT_EDIT) + + await manager.planMode.enterTool.fn({ + args: { reason: 'research the change first' }, + workspace: 'test-workspace', + helpers: {}, + toolCallbacks: { setToolStatus: vi.fn(), removeToolStatus: vi.fn() }, + toolId: 'call_enter' + }) + + expect(manager.planModeActive).toBe(true) + expect(manager.prePlanAutonomyMode).toBe(AIAutonomyMode.ACCEPT_EDIT) + }) + + it('refuses to move a session chat out of GLOBAL, so the gate cannot lift under it', () => { + const manager = sessionManager(AIAutonomyMode.ACCEPT_EDIT) + manager.setAutonomyMode(AIAutonomyMode.PLAN) + expect(manager.planModeActive).toBe(true) + // Without a configured model changeMode returns early on SCRIPT, and the case would pass + // against the very guard it is meant to pin. + mocks.getCurrentModel.mockReturnValue({ provider: 'openai', model: 'gpt-4o' }) + mocks.tryGetCurrentModel.mockReturnValue({ provider: 'openai', model: 'gpt-4o' }) + const logged = vi.spyOn(console, 'error').mockImplementation(() => {}) + + manager.changeMode(AIMode.SCRIPT) + + expect(manager.mode).toBe(AIMode.GLOBAL) + expect(manager.planModeActive).toBe(true) + // A switch that silently does nothing gives its caller no way to learn why. + expect(logged).toHaveBeenCalled() + logged.mockRestore() + }) + + it('never auto-accepts an enter_plan_mode card, whichever side of the switch it lands on', async () => { + // Switching to YOLO answers every pending confirmation — except this one. "Run it + // without asking" must not be answered by forcing the user into a read-only posture. + const before = sessionManager() + const enterPending = before.requestConfirmation('call_enter', 'enter_plan_mode') + const writePending = before.requestConfirmation('call_write', 'write_script') + before.setAutonomyMode(AIAutonomyMode.YOLO) + expect(await enterPending).toBe(false) + expect(await writePending).toBe(true) + }) + + it('declines an enter_plan_mode that arrives after the switch to YOLO', async () => { + // The tool set is snapshotted per iteration, so a call can still arrive once the user + // has moved to YOLO. Driven through processToolCall rather than requestConfirmation + // directly: an auto-accepting posture skips the confirmation wait entirely, so asserting + // against the wait would pass on a build that never reaches it. + const { processToolCall } = await import('./shared') + const manager = sessionManager(AIAutonomyMode.YOLO) + + const result = await processToolCall({ + tools: [manager.planMode.enterTool] as any, + toolCall: { + id: 'call_enter', + type: 'function', + function: { name: 'enter_plan_mode', arguments: JSON.stringify({ reason: 'research' }) } + } as any, + helpers: {}, + workspace: 'test-workspace', + toolCallbacks: { + setToolStatus: vi.fn(), + removeToolStatus: vi.fn(), + requestConfirmation: manager.requestConfirmation, + shouldAutoAcceptToolConfirmations: manager.shouldAutoAcceptTool + } as any + }) + + expect(result.content).toBe(PLAN_MODE_MESSAGES.enterDeclined) + expect(manager.autonomyMode).toBe(AIAutonomyMode.YOLO) + expect(manager.planModeActive).toBe(false) + }) + + it('answers a pending plan card the way the picker was moved', async () => { + const entering = sessionManager() + const enterPending = entering.requestConfirmation('call_enter', 'enter_plan_mode') + entering.setAutonomyMode(AIAutonomyMode.PLAN) + expect(await enterPending).toBe(true) + + // Leaving plan mode any other way is not a sign-off on the plan on the card. + const leaving = sessionManager(AIAutonomyMode.PLAN) + const exitPending = leaving.requestConfirmation('call_exit', 'exit_plan_mode') + leaving.setAutonomyMode(AIAutonomyMode.DEFAULT) + expect(await exitPending).toBe(false) + + // Opting into YOLO does mean "run it". + const yolo = sessionManager(AIAutonomyMode.PLAN) + const yoloPending = yolo.requestConfirmation('call_exit', 'exit_plan_mode') + yolo.setAutonomyMode(AIAutonomyMode.YOLO) + expect(await yoloPending).toBe(true) + }) +}) + describe('AIChatManager persisted autonomy default', () => { // Mirrors the private storage keys in AIChatManager.svelte.ts, namespaced by // the logged-in user's email (see userScopedStorage). diff --git a/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte b/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte new file mode 100644 index 0000000000..699b79a292 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte @@ -0,0 +1,55 @@ + + +
+ + +
diff --git a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte index 754db1aac0..34b1adcf68 100644 --- a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte @@ -1,12 +1,39 @@