From caa189868c6ec9ebc2b6311e07308a83fa25ad8d Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:27:38 +0200 Subject: [PATCH] feat(ai-sessions): add plan mode (#10057) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(sessions): let an opener name the artifact version to show A tab already remembers the version a reader pinned, and re-pointing it keeps that pin. Plan mode needs the two intents that leaves out: a plan card scrolled up the transcript wants the version it proposed, and a plan going up for approval wants the current text with no pin at all. `ArtifactVersionTarget` is those two alongside the existing one: a number, `'latest'`, or omitted. Omitted still cannot double as `'latest'` — every artifact tool re-opens the document it just wrote, so taking that as a request to move would yank a reader out of the version they chose on every edit the agent makes. Co-Authored-By: Claude Opus 5 (1M context) * feat(copilot): add the plan-mode gate and tag plan-mode-safe tools Plan mode is a read-only posture, so something has to decide which tools it may still run. `Tool.planModeSafe` is that tag, and processToolCall fails closed on it: untagged means mutating means blocked. Deriving it from `requiresConfirmation` was not an option — unconfirmed mutating tools exist, and a posture that leaks one is not a posture. The gate runs twice per call. Before `validateBeforeConfirmation`, so a validator cannot reach out while planning; and again after the confirmation wait, because plan mode can be entered while a mutating tool's card is already pending, and that approval must not carry it through. Arguments are read one field at a time rather than through a parse of the whole call. `change_note` is optional and cosmetic, and a model that sends it as `null` would otherwise fail the object parse and take the plan down with it — the user being told there was no plan to approve, which is false. Also here, because refusing a call well needs them: a validator may now return the row the user reads and the result the model gets separately, a tool may word its own cancellation, and a tool may start work when its card appears rather than when it is approved. The gate is consulted before any of them. `shouldAutoAcceptToolConfirmations` is asked about the tool by name, because skipping the confirmation wait is itself an answer on the user's behalf and one tool must not be answered for. Deciding that without the name would put the exception out of reach of the only path that needs it. The gate stays inert until a chat supplies `isPlanModeActive`. Co-Authored-By: Claude Opus 5 (1M context) * feat(copilot): give a session one versioned plan document The plan the user agrees to has to survive `/clear`, so it belongs to the session rather than the conversation, and a session holds exactly one. Its id is the session's, so the primary key is the constraint — there is no second row to mint, no index to maintain and no schema change at all. Every write reads the row it is about to replace inside the transaction that replaces it. Read outside, two tabs both see version N, both stamp N+1, and the later write silently drops the earlier one's text and its snapshot; IndexedDB serialises readwrite transactions over a store, so read and write together cannot interleave. Approval takes the same route but patches only the pointer: an approval computed while another tab was revising must not carry this tab's older content back over the newer text. Approval is `approvedVersion`, a pointer at a version, never a flag. Below the current version means the newest text is a proposal the user has not agreed to; absent means nothing here was ever approved. Only exit_plan_mode can leave the pointer behind, since every write outside plan mode carries it forward — an amendment the user's posture already trusts is still the agreed plan. Declining writes nothing at all: the refused proposal stands as the newest version, with the agreed one still in history. Nor can create_artifact confer approval. It asks for no confirmation, so the model writing a plan document is not the user agreeing to one; a plan written there holds the session's slot as a draft until a decision lands on it. That is also why the approved version is exempt from pruning. A plan approved at v1 and then planned against for twenty more rounds would otherwise lose the very version that stands as agreed, and with it the card that opens it, the banner offering it back, and read_artifact at that version. It is excluded from the pruning candidates rather than added on top, so the budget is unchanged and what survives simply stops being contiguous. The write reports whether the database took it. Most callers still degrade like the reads do, but a plan cannot: returning one the database refused would let the user approve and execute against a document that disappears on reload — a refused plan write raises instead. Co-Authored-By: Claude Opus 5 (1M context) * feat(copilot): add plan mode — the posture and its two tools enter_plan_mode asks to hold work; exit_plan_mode hands over a plan and, on approval, gives the posture back to whatever preceded it. Both carry `planModeSafe`, since a posture with no exit is a trap. Only the transition the current posture allows is offered, so there is no tool for leaving a posture the chat is not in. A planning round runs from entering plan mode to the proposal the user decides on. It remembers only the write it made, because nothing it does is undone — and that write is shared between the card's confirmation hook and the tool's `fn`, so the plan is on screen while the user is deciding whether to approve it rather than after. The round is identified by an epoch bumped on *entering*, not by the conversation. A chat rotation mid-approval must still let that approval hand the posture back; a round the user has since left and re-entered must not, or approving the old plan would drop them out of a read-only posture they just chose. Saving a proposal revises the session's plan document and creates one only when there is none — both halves in a single transaction, so a second tab proposing at the same moment revises the row this one wrote rather than racing it. Persistence failures hold the posture. Approval is reported only once both the proposal and the approval pointer are durable, so a plan the database refused cannot unblock mutating tools. The failure is reported from `fn` and no earlier: the write settles while the card is still waiting to be confirmed, and clearing that card from underneath the wait would take away the only control that resolves it. An auto-accepting posture answers for the user through one predicate, asked by every path that answers: the pending-card sweep, the confirmation itself, and the decision to skip the wait at all. enter_plan_mode never qualifies: YOLO means "stop asking and run it", and a call from a tool set snapshotted before the switch must not answer that with a read-only posture — whether its card is already pending or has yet to be registered. Plan mode lives in its own controller with a narrow view of the chat it runs in: it reads that autonomy state and asks for the two changes it can cause, rather than owning any of it. Plan mode is offered only in a session chat, and a session chat is GLOBAL for its whole life. The gate reads that mode, so `changeMode` refuses to move one out of GLOBAL rather than resting the invariant on a picker being hidden. Co-Authored-By: Claude Opus 5 (1M context) * feat(copilot): surface plan mode in the chat and the artifact list Plan mode is the only posture that refuses work, so the composer says so before the user types the request it is about to turn down: the mode pill is tinted whole rather than by its icon, and the empty placeholder carries the constraint in words. Teal, not the house green — green is the transcript's success colour a few rows up, and a mode signal in it would read as "this worked" rather than "this is held". A blocked tool renders as its own lean row naming the tool, not as an error: the call did what plan mode says it should, and "why can't it edit" is answered where it is asked. A plan card names the decision — proposed, approved, or not approved — and never the button, since a Stop and a posture switch resolve it too. Its button opens the version that card proposed, so a card far up the transcript still shows the plan it put forward rather than whatever the document has become since. The artifact list and the preview header both label the plan through one badge helper, so the two cannot disagree about what counts as one: a plan the user never approved keeps the plan icon and takes the neutral badge, leaving the teal to mean exactly one thing. In the viewer, an unapproved revision says so in a bar that cannot be scrolled past, with the version the user did agree to one click away. The autonomy picker became a table with one row per posture, so adding one touches a single place instead of four parallel switch statements. A version of a plan is read against the one the user approved, not against the newest: latest is only where the model happened to stop. So the approved version is never stale — its bar is teal and points forward to the draft rather than warning about it — the version in front of it is the draft, and anything behind it is history that is neither and takes no pill at all. The list opens a plan at the approved version for the same reason, which is what lets its pill say `plan` while an unapproved draft sits at the head. One helper answers all of it, so the list and the preview header cannot drift apart on what counts as the plan. Co-Authored-By: Claude Opus 5 (1M context) * test(ai-evals): exercise plan mode end to end A case a unit test cannot stand in for: it starts in plan mode against the real gate and the real exit_plan_mode, and grades whether the model researches and hands over a usable plan instead of guessing at one. The checklist does not grade what the harness does for the model — exit_plan_mode writes the plan document itself, so "saves the plan as an artifact" would pass on any run where the tool is called at all. The eval store seeds artifacts with history and mirrors the store's own approval rules, so a rename cannot promote a proposal the user turned down. Co-Authored-By: Claude Opus 5 (1M context) * fix(ai-evals): import the plan-mode messages from the module that owns them `PLAN_MODE_MESSAGES` moved to `planModeMessages.ts`; `planMode.ts` imports it without re-exporting. Under vitest, which runs the frontend adapters, the stale import resolved to `undefined` rather than failing to link, so `global-planmode1-hands-over-a-plan` threw on the approval message after the posture had already been dropped and the tool withdrawn. Co-Authored-By: Claude Opus 5 (1M context) * fix(copilot): state plan mode's constraint in neutral text The composer's two-tone placeholder becomes a plain "Read-only" beside the autonomy picker, next to where YOLO puts its own warning, and a blocked call's row drops the mode colour. Teal is left marking what the posture is — the badge, the version bars, the pill — rather than every call it refuses. ContextTextarea goes back to main with the accent: `placeholderAccent` had no other consumer, and the aria-label existed only because the accent blanked the native placeholder. Co-Authored-By: Claude Opus 5 (1M context) * fix(copilot): hold the plan header's verdict until the snapshot lands Opening a plan at the version its reader approved pins a version behind the head, and until that read resolves `shownVersion` is still the head — so the header wore the draft's badge and its orange "not approved" bar over the very case the pin exists to serve, then flipped. The header now says nothing while `restoringPin`, as the body already does. Judging `pinned` instead would print the approved signal over text that is still the draft, trading a true transient signal for a false one. Co-Authored-By: Claude Opus 5 (1M context) * fix(copilot): refuse a hand-over once plan mode has ended A response can carry two exit_plan_mode calls, and the tool list they run against is snapshotted before the first one restores the posture. The second then found the tool with plan mode already over: under YOLO every confirmation is answered for the user, so it wrote its own summary and stamped the user's approval on a plan no card had shown them. Refused in `validateBeforeConfirmation` rather than in `fn`, since `onConfirmationRequested` writes the document too. The maintenance path is untouched — a plan still gets revised outside the posture with update_artifact, which is what the tool's own description already tells the model to use. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- .../frontend/core/global/evalArtifactStore.ts | 68 +++- .../frontend/core/global/globalEvalRunner.ts | 46 ++- .../frontend/core/global/planModeTools.ts | 69 ++++ .../frontend/core/shared/baseEvalRunner.ts | 26 +- ai_evals/cases/global.yaml | 24 ++ ai_evals/core/types.ts | 3 + ai_evals/modes/global.ts | 23 +- .../copilot/chat/AIChatDisplay.svelte | 148 +++++---- .../copilot/chat/AIChatManager.svelte.ts | 139 ++++++-- .../copilot/chat/AIChatManager.test.ts | 116 +++++++ .../chat/ToolConfirmationFooter.svelte | 55 ++++ .../copilot/chat/ToolExecutionDisplay.svelte | 190 +++++++++-- .../components/copilot/chat/api/apiTools.ts | 1 + .../lib/components/copilot/chat/app/core.ts | 6 + .../chat/artifacts/ArtifactViewer.svelte | 87 ++++- .../chat/artifacts/ArtifactsSegment.svelte | 52 +-- .../copilot/chat/artifacts/artifactLimits.ts | 23 ++ .../chat/artifacts/artifactTools.test.ts | 60 +++- .../copilot/chat/artifacts/artifactTools.ts | 79 +++-- .../chat/artifacts/artifactsDB.test.ts | 33 +- .../copilot/chat/artifacts/artifactsDB.ts | 101 +++--- .../chat/artifacts/artifactsState.svelte.ts | 203 +++++++++--- .../chat/artifacts/artifactsState.test.ts | 156 ++++++++- .../components/copilot/chat/datatableTools.ts | 2 + .../lib/components/copilot/chat/docs/core.ts | 10 +- .../components/copilot/chat/ducklakeTools.ts | 1 + .../copilot/chat/files/fileTools.ts | 2 + .../lib/components/copilot/chat/flow/core.ts | 4 + .../copilot/chat/global/apiCatalogTools.ts | 16 +- .../copilot/chat/global/core.test.ts | 20 ++ .../components/copilot/chat/global/core.ts | 30 +- .../components/copilot/chat/navigator/core.ts | 3 + .../components/copilot/chat/pipeline/core.ts | 2 + .../components/copilot/chat/planMode.test.ts | 260 +++++++++++++++ .../lib/components/copilot/chat/planMode.ts | 261 +++++++++++++++ .../copilot/chat/planModeController.svelte.ts | 234 ++++++++++++++ .../copilot/chat/planModeController.test.ts | 299 ++++++++++++++++++ .../copilot/chat/planModeMessages.ts | 46 +++ .../components/copilot/chat/script/core.ts | 7 + .../components/copilot/chat/shared.test.ts | 210 +++++++++++- .../src/lib/components/copilot/chat/shared.ts | 123 +++++-- .../sessions/SessionStatusPopover.svelte | 12 +- .../lib/components/sessions/previewRouter.ts | 10 +- .../sessions/sessionPreviewTabs.svelte.ts | 21 +- .../sessions/sessionPreviewTabs.test.ts | 15 + .../sessions/sessionRuntime.svelte.ts | 4 +- 46 files changed, 2999 insertions(+), 301 deletions(-) create mode 100644 ai_evals/adapters/frontend/core/global/planModeTools.ts create mode 100644 frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte create mode 100644 frontend/src/lib/components/copilot/chat/artifacts/artifactLimits.ts create mode 100644 frontend/src/lib/components/copilot/chat/planMode.test.ts create mode 100644 frontend/src/lib/components/copilot/chat/planMode.ts create mode 100644 frontend/src/lib/components/copilot/chat/planModeController.svelte.ts create mode 100644 frontend/src/lib/components/copilot/chat/planModeController.test.ts create mode 100644 frontend/src/lib/components/copilot/chat/planModeMessages.ts diff --git a/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts b/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts index ed67cce468..95b67743e1 100644 --- a/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts +++ b/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts @@ -1,11 +1,54 @@ // SessionArtifactsStore can't run here (bun has no IndexedDB, nor the compiled $state runes), // so mirror only the shape the artifact tools call, not its scoping or race handling. export const EVAL_SESSION_ID = "eval-session"; -export function createEvalArtifactHelpers() { + +/** An artifact the session already holds when the case starts: history has to predate the + * run, since one prompt cannot both build a past and reason about it. */ +export interface SeededArtifact { + name: string; + role?: "plan"; + /** Which version the user agreed to. Below the last one means the current text is a + * proposal they turned down, which is the state worth seeding. */ + approvedVersion?: number; + /** Oldest first; the last one is the artifact's current content. */ + versions: Array<{ content: string; note?: string }>; +} + +export function createEvalArtifactHelpers(seed: SeededArtifact[] = []) { const items = new Map>(); // Snapshots per artifact id, oldest first — the version tools read history from here. const history = new Map>>(); let seq = 0; + for (const entry of seed) { + const id = `eval-artifact-${seq++}`; + const current = entry.versions.at(-1); + if (!current) continue; + items.set(id, { + id, + sessionId: EVAL_SESSION_ID, + chatId: "eval-chat", + kind: "md", + name: entry.name, + content: current.content, + role: entry.role, + approvedVersion: entry.approvedVersion, + createdAt: 0, + updatedAt: seq, + version: entry.versions.length, + }); + history.set( + id, + entry.versions.map((v, i) => ({ + key: `${id}:${i + 1}`, + artifactId: id, + version: i + 1, + name: entry.name, + content: v.content, + savedAt: i, + note: v.note, + })), + ); + } const snapshotOf = ( artifact: Record, version: number, @@ -21,6 +64,16 @@ export function createEvalArtifactHelpers() { }); const store = { create: async (sessionId: string, input: Record) => { + // One plan per session, as SessionArtifactsStore enforces it — the tool refuses + // first, so reaching this means a case drove create_artifact past that message. + if ( + input.role === "plan" && + [...items.values()].some( + (a) => a.sessionId === sessionId && a.role === "plan", + ) + ) { + throw new Error(`Session ${sessionId} already has a plan document`); + } const now = seq++; const artifact = { id: `eval-artifact-${now}`, @@ -29,6 +82,10 @@ export function createEvalArtifactHelpers() { kind: input.kind ?? "md", name: input.name, content: input.content, + // The plan document is only distinguishable by these, both in the snapshot the + // judge reads and in what list_artifacts reports back to the model. + role: input.role, + approvedVersion: input.approvedVersion, createdAt: now, updatedAt: now, version: 1, @@ -58,6 +115,15 @@ export function createEvalArtifactHelpers() { ...existing, name: input.name ?? existing.name, content: input.content ?? existing.content, + // Carried only onto a version this write produced, as SessionArtifactsStore does: + // a rename cannot promote a proposal the user turned down. + approvedVersion: + input.approvedVersion ?? + (input.keepApproved && + existing.approvedVersion !== undefined && + contentChanged + ? version + : existing.approvedVersion), updatedAt: seq++, version, }; diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index dcd771663a..f1b33e9d13 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -12,9 +12,14 @@ import { getGlobalDraft, listGlobalDrafts, } from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter"; +import { appendPlanModeInstructions } from "../../../../../frontend/src/lib/components/copilot/chat/planMode"; import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; +import { createEvalPlanTools } from "./planModeTools"; import { UserDraft } from "../../../../../frontend/src/lib/userDraft.svelte"; -import { createEvalArtifactHelpers } from "./evalArtifactStore"; +import { + createEvalArtifactHelpers, + type SeededArtifact, +} from "./evalArtifactStore"; import type { ModeRunContext } from "../../../../core/types"; import type { GlobalDraftState } from "../../../../core/validators"; import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; @@ -83,11 +88,16 @@ export interface GlobalEvalOptions { user?: GlobalUserFixture; // Emulate a session chat (preview tools + session prompt); default false = standalone baseline. sessionChat?: boolean; + // Start in plan mode: the gate refuses every tool without `planModeSafe`, and the two plan + // tools are offered. Needs sessionChat, which is what plan mode is gated on in production. + planMode?: boolean; model?: string; maxIterations?: number; provider?: AIProvider; backend: WindmillBackendSettings; workspaceRoot?: string; + // Artifacts the session already holds when the run starts. + artifacts?: SeededArtifact[]; runContext?: ModeRunContext; } @@ -113,19 +123,41 @@ export async function runGlobalEval( process.env[DISABLE_ACTIVE_EDITOR_CONTEXT_ENV] !== "1"; // Pass the seeded identity straight to the prompt builder rather than mutating // the process-global `userStore`, so concurrent cases never race on it. - const evalArtifacts = createEvalArtifactHelpers(); + const evalArtifacts = createEvalArtifactHelpers(options.artifacts); + const planMode = options.planMode + ? createEvalPlanTools({ + create: evalArtifacts.helpers.artifacts.create, + sessionId: evalArtifacts.helpers.sessionId, + chatId: evalArtifacts.helpers.getChatId(), + }) + : undefined; + const baseSystemMessage = prepareGlobalSystemMessage(undefined, { + user: options.user, + previewTools: options.sessionChat ?? false, + }); const rawResult = await runEval({ userPrompt, - systemMessage: prepareGlobalSystemMessage(undefined, { - user: options.user, - previewTools: options.sessionChat ?? false, - }), + systemMessage: baseSystemMessage, + // Re-derived per request, as production's getter is: the instructions have to leave + // the prompt when the plan is approved, or the model is still told it may not build + // while the gate has already opened. + getSystemMessage: planMode + ? () => + planMode.isPlanModeActive() + ? appendPlanModeInstructions(baseSystemMessage, 0) + : baseSystemMessage + : undefined, + isPlanModeActive: planMode?.isPlanModeActive, + isToolAvailable: planMode?.isToolAvailable, userMessage: prepareGlobalUserMessage( userPrompt, [], injectActiveEditorContext ? { workspace: workspaceRoot } : {}, ), - tools: getGlobalEvalTools(options.sessionChat ?? false), + tools: [ + ...getGlobalEvalTools(options.sessionChat ?? false), + ...(planMode?.tools ?? []), + ], helpers: evalArtifacts.helpers, apiKey, getOutput: async () => ({ diff --git a/ai_evals/adapters/frontend/core/global/planModeTools.ts b/ai_evals/adapters/frontend/core/global/planModeTools.ts new file mode 100644 index 0000000000..bea5538fc8 --- /dev/null +++ b/ai_evals/adapters/frontend/core/global/planModeTools.ts @@ -0,0 +1,69 @@ +import { + EXIT_PLAN_MODE_TOOL, + EXIT_PLAN_MODE_TOOL_DESCRIPTION, + derivePlanTitle, + exitPlanModeArgs, + planSummaryOf, +} from "../../../../../frontend/src/lib/components/copilot/chat/planMode"; +import { PLAN_MODE_MESSAGES } from "../../../../../frontend/src/lib/components/copilot/chat/planModeMessages"; +import { createToolDef } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; +import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; + +/** + * `exit_plan_mode` built from the production schema, description and messages, so a case + * exercises the real gate and wording with the posture living here rather than on the + * manager. It resolves immediately — the runners define no `requestConfirmation`, so the + * plan is always approved and a refused one cannot be expressed. + */ +export function createEvalPlanTools(artifacts: { + create: ( + sessionId: string, + input: Record, + ) => Promise<{ id: string; name: string }>; + sessionId: string; + chatId: string; +}): { + tools: ProductionTool<{}>[]; + isPlanModeActive: () => boolean; + isToolAvailable: (name: string) => boolean; +} { + let planActive = true; + return { + isPlanModeActive: () => planActive, + // Withdrawn on approval, as production's tool getter does it: leaving it advertised + // invites a second hand-over of a plan already agreed, which would write a duplicate. + // Production would offer enter_plan_mode in its place; these cases stop at the first + // hand-over, so a fresh planning round belongs to a case of its own. + isToolAvailable: (name) => name !== EXIT_PLAN_MODE_TOOL || planActive, + // Production offers one plan tool at a time and these cases start in plan mode, so + // enter_plan_mode would only invite a turn spent entering a posture already held. + tools: [ + { + def: createToolDef( + exitPlanModeArgs, + EXIT_PLAN_MODE_TOOL, + EXIT_PLAN_MODE_TOOL_DESCRIPTION, + ), + // Carries the safety tag for the same reason production does: it is the only way out + // of the posture, so the gate must not refuse it. + planModeSafe: true, + fn: async ({ args }) => { + const summary = planSummaryOf(args); + if (!summary?.trim()) { + return PLAN_MODE_MESSAGES.missingSummary; + } + planActive = false; + await artifacts.create(artifacts.sessionId, { + name: derivePlanTitle(summary), + content: summary, + kind: "md", + role: "plan", + approvedVersion: 1, + chatId: artifacts.chatId, + }); + return PLAN_MODE_MESSAGES.approvedWithDoc; + }, + }, + ] as ProductionTool<{}>[], + }; +} diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index f787743dfa..4501f24810 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -43,6 +43,15 @@ export interface RunEvalParams { getOutput: () => TOutput | Promise; /** Model and Windmill backend configuration */ options: EvalRunnerOptions; + /** Drives the production plan-mode gate in processToolCall. Absent leaves it inert, + * which is what every mode but an opted-in global case wants. */ + isPlanModeActive?: () => boolean; + /** Which of `tools` the model is offered on this request. Absent offers all of them. */ + isToolAvailable?: (name: string) => boolean; + /** Re-read before every request, as production's systemMessage getter is. Needed when a + * tool changes what the prompt should say — plan mode's instructions have to come back + * out once the plan is approved. Falls back to the fixed `systemMessage`. */ + getSystemMessage?: () => ChatCompletionSystemMessageParam; onAssistantMessageStart?: () => void; onAssistantToken?: (token: string) => void; onAssistantMessageEnd?: () => void; @@ -68,6 +77,9 @@ export async function runEval( onAssistantToken, onAssistantMessageEnd, onToolCall, + isPlanModeActive, + isToolAvailable, + getSystemMessage, } = params; let shouldEmitMessageStart = true; @@ -119,6 +131,7 @@ export async function runEval( } = { setToolStatus: () => {}, removeToolStatus: () => {}, + isPlanModeActive, onNewToken: (token: string) => { if (shouldEmitMessageStart) { onAssistantMessageStart?.(); @@ -140,8 +153,17 @@ export async function runEval( try { const result = await runChatLoop({ messages, - systemMessage, - tools: wrappedTools, + get systemMessage() { + return getSystemMessage?.() ?? systemMessage; + }, + // Re-derived per request, as `systemMessage` is: a tool the posture has withdrawn + // must leave the schema too, or the model keeps being offered a call the run has + // moved past — and the token counts a case reports include a tool it cannot use. + get tools() { + return isToolAvailable + ? wrappedTools.filter((t) => isToolAvailable(t.def.function.name)) + : wrappedTools; + }, helpers, abortController, callbacks, diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index ec2260c0e2..2269f9d727 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1756,8 +1756,32 @@ judgeChecklist: - saves the plan as a markdown artifact via create_artifact rather than only replying inline - the artifact content has a title, a one-line summary, and three or four bullet steps for onboarding + - the artifact is registered as the session's plan (role "plan"), not as an ordinary note - the user asked for the plan they will come back to and revise - does not create a flow or script draft yet +- id: global-planmode1-hands-over-a-plan + prompt: |- + Our support inbox is a mess. I want incoming emails triaged by urgency and routed to the + right team, with anything urgent also posted to Slack. + Work out how you'd build this in Windmill. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 10 + sessionChat: true + planMode: true + # No draft assertion: approving the plan opens the gate mid-run, and building from there is + # what production asks for, so a draft is not a failure. The gate itself is covered by + # shared.test.ts; what only a real model can show is whether it researches and hands over a + # usable plan instead of guessing at one. + toolExpect: + requiredToolsUsed: + - exit_plan_mode + # Not "saves the plan as an artifact": exit_plan_mode writes it, so the harness would + # satisfy that on every run the tool is called at all — it grades itself, not the model. + judgeChecklist: + - the plan covers classifying an incoming email by urgency, routing it to a team, and posting urgent ones to Slack + - the plan is specific about what would be built in Windmill (a flow and its steps, or the scripts involved) + - id: global-npm1-script-search-package prompt: |- Find a good npm package for parsing RSS/Atom feeds and use it to create a draft Bun script diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index f1655bd18f..96e116023a 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -33,6 +33,9 @@ export interface EvalCaseRuntimeSpec { appContext?: EvalCaseRuntimeAppContextSpec; // Global mode: run as a session chat (preview tools + session prompt) vs the standalone chat. sessionChat?: boolean; + // Global session chats: start the case in plan mode, so mutating tools are refused until + // the model hands over a plan with exit_plan_mode. + planMode?: boolean; } export interface FlowValidationSpec { diff --git a/ai_evals/modes/global.ts b/ai_evals/modes/global.ts index 5628c21d5d..e7a36039a2 100644 --- a/ai_evals/modes/global.ts +++ b/ai_evals/modes/global.ts @@ -6,9 +6,14 @@ import { type GlobalLiveEditorDraftFixture, type GlobalUserFixture, } from "../adapters/frontend/core/global/globalEvalRunner"; +import type { SeededArtifact } from "../adapters/frontend/core/global/evalArtifactStore"; import type { BenchmarkWorkspaceRunnables } from "../adapters/frontend/mockBackend"; import type { FrontendEvalModelConfig } from "../core/models"; -import type { BenchmarkArtifactFile, GlobalValidationSpec, ModeRunner } from "../core/types"; +import type { + BenchmarkArtifactFile, + GlobalValidationSpec, + ModeRunner, +} from "../core/types"; import { validateGlobalState, type GlobalDraftState } from "../core/validators"; import type { WindmillBackendSettings } from "../core/windmillBackendSettings"; import { getFrontendApiKey } from "./frontendCommon"; @@ -17,6 +22,7 @@ export interface GlobalInitialFixture { workspace?: BenchmarkWorkspaceRunnables; liveEditorDrafts?: GlobalLiveEditorDraftFixture[]; user?: GlobalUserFixture; + artifacts?: SeededArtifact[]; } export function createGlobalModeRunner( @@ -41,7 +47,9 @@ export function createGlobalModeRunner( workspaceFixtures: initial?.workspace, liveEditorDrafts: initial?.liveEditorDrafts, user: initial?.user, + artifacts: initial?.artifacts, sessionChat: context.evalCase?.runtime?.sessionChat, + planMode: context.evalCase?.runtime?.planMode, maxIterations: context.evalCase?.runtime?.maxTurns, provider: modelConfig.provider, model: modelConfig.model, @@ -81,7 +89,9 @@ export function createGlobalModeRunner( }; } -async function loadGlobalInitialFixture(path: string): Promise { +async function loadGlobalInitialFixture( + path: string, +): Promise { if ((await stat(path)).isDirectory()) { const { initialFrontend, initialBackend, initialDatatables } = await loadAppFixtureForEval(path); @@ -104,14 +114,19 @@ async function loadGlobalInitialFixture(path: string): Promise { +async function loadGlobalExpectedFixture( + path: string, +): Promise { return JSON.parse(await readFile(path, "utf8")) as GlobalDraftState; } diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index c3875f702f..6db4de5f0f 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -10,6 +10,7 @@ ChevronDown, ChevronsRight, CheckIcon, + ClipboardList, FileText, Folder, Hand, @@ -25,6 +26,8 @@ import Popover from '$lib/components/meltComponents/Popover.svelte' import DropdownV2 from '$lib/components/DropdownV2.svelte' import { pendingUserAction, type DisplayMessage } from './shared' + import { PLAN_MODE_TEXT_COLOR, PLAN_MODE_TRIGGER_CLASS } from './planMode' + import { PLAN_MODE_MESSAGES } from './planModeMessages' import type { ContextElement } from './context' import ChatQuickActions from './ChatQuickActions.svelte' import ContextUsageIndicator from './ContextUsageIndicator.svelte' @@ -57,46 +60,79 @@ const MAX_YOLO_TOOLTIP_TOOLS = 8 const aiChatManager = getAiChatManager() - // `label` is shown in the dropdown; `shortLabel` (when set) is shown in the - // compact trigger pill to save horizontal space. - type AutonomyModeOption = { label: string; shortLabel?: string; mode: AIAutonomyMode } + // One row per autonomy posture, in picker order, so adding one touches only this + // table. `isAvailable` hides the postures that would do nothing in the current AI + // mode, which is why the picker can be shorter than this list. + type AutonomyAvailability = { + autoAcceptEditsAvailable: boolean + autoAcceptToolConfirmationsAvailable: boolean + planModeAvailable: boolean + } + type AutonomyModeOption = { + mode: AIAutonomyMode + label: string + shortLabel?: string + icon: typeof Hand + iconColor: string + /** Tints the whole trigger, not just its icon. Only plan mode needs it. */ + triggerClass?: string + tooltip: (a: AutonomyAvailability) => string + isAvailable: (a: AutonomyAvailability) => boolean + } + // The one posture available everywhere, so also the fallback for a mode the + // current AI mode does not offer. + const askPermissionOption: AutonomyModeOption = { + mode: AIAutonomyMode.DEFAULT, + label: 'Ask permission', + icon: Hand, + iconColor: 'text-secondary', + tooltip: (a) => + a.autoAcceptEditsAvailable + ? 'Requires confirmation for edits and tool calls.' + : 'Requires confirmation for tool calls.', + isAvailable: () => true + } const autonomyModeOptions: AutonomyModeOption[] = [ - { label: 'Ask permission', mode: AIAutonomyMode.DEFAULT }, - { label: 'Auto-accept edits', mode: AIAutonomyMode.ACCEPT_EDIT }, - { label: 'Yolo (bypass permissions)', shortLabel: 'Yolo', mode: AIAutonomyMode.YOLO } + { + mode: AIAutonomyMode.PLAN, + label: 'Plan (read-only)', + shortLabel: 'Plan', + icon: ClipboardList, + iconColor: PLAN_MODE_TEXT_COLOR, + triggerClass: PLAN_MODE_TRIGGER_CLASS, + tooltip: () => + 'Read-only: the assistant researches and drafts a plan for your approval before it can change anything.', + isAvailable: (a) => a.planModeAvailable + }, + askPermissionOption, + { + mode: AIAutonomyMode.ACCEPT_EDIT, + label: 'Auto-accept edits', + icon: ChevronsRight, + iconColor: 'text-accent', + tooltip: () => + 'Automatically accepts script and flow edits. Tool calls still ask for confirmation.', + isAvailable: (a) => a.autoAcceptEditsAvailable + }, + { + mode: AIAutonomyMode.YOLO, + label: 'Yolo (bypass permissions)', + shortLabel: 'Yolo', + icon: ChevronsRight, + iconColor: 'text-red-500', + tooltip: (a) => + a.autoAcceptEditsAvailable + ? 'Automatically accepts script and flow edits plus tool confirmations.' + : 'Automatically accepts tool confirmations.', + isAvailable: (a) => a.autoAcceptToolConfirmationsAvailable + } ] + const autonomyModeOption = (mode: AIAutonomyMode) => + autonomyModeOptions.find((o) => o.mode === mode) ?? askPermissionOption const autonomyModeLabel = (mode: AIAutonomyMode) => { - const option = autonomyModeOptions.find((o) => o.mode === mode) ?? autonomyModeOptions[0] + const option = autonomyModeOption(mode) return option.shortLabel ?? option.label } - // "Auto-accept edits" only applies where script/flow edits can be accepted, - // "Bypass permissions" only where tool confirmations exist; filter the picker - // to the levels that actually do something in the current mode. - const isAutonomyModeAvailable = ( - mode: AIAutonomyMode, - autoAcceptEditsAvailable: boolean, - autoAcceptToolConfirmationsAvailable: boolean - ) => { - switch (mode) { - case AIAutonomyMode.DEFAULT: - return true - case AIAutonomyMode.ACCEPT_EDIT: - return autoAcceptEditsAvailable - case AIAutonomyMode.YOLO: - return autoAcceptToolConfirmationsAvailable - } - return false - } - // Ask-permission holds (raised hand); auto-accept/bypass fast-forward. Color - // ramps from muted (ask) to accent (auto-accept) to red (bypass). - const autonomyModeIcon = (mode: AIAutonomyMode) => - mode === AIAutonomyMode.DEFAULT ? Hand : ChevronsRight - const autonomyModeIconColor = (mode: AIAutonomyMode) => - mode === AIAutonomyMode.YOLO - ? 'text-red-500' - : mode === AIAutonomyMode.DEFAULT - ? 'text-secondary' - : 'text-accent' let { messages, @@ -451,14 +487,13 @@ if (input.files && input.files.length > 0) void handleAddFiles(input.files) input.value = '' } - const availableAutonomyModeOptions = $derived.by(() => - autonomyModeOptions.filter((option) => - isAutonomyModeAvailable( - option.mode, - aiChatManager.autoAcceptEditsAvailable, - aiChatManager.autoAcceptToolConfirmationsAvailable - ) - ) + const autonomyAvailability = $derived({ + autoAcceptEditsAvailable: aiChatManager.autoAcceptEditsAvailable, + autoAcceptToolConfirmationsAvailable: aiChatManager.autoAcceptToolConfirmationsAvailable, + planModeAvailable: aiChatManager.planModeAvailable + }) + const availableAutonomyModeOptions = $derived( + autonomyModeOptions.filter((option) => option.isAvailable(autonomyAvailability)) ) // Fall back to ask-permission when the persisted mode isn't applicable in the // current AI mode (e.g. auto-accept edits while in a mode without edits). @@ -468,22 +503,7 @@ : AIAutonomyMode.DEFAULT ) const showAutonomyModeSelector = $derived(!disabled && availableAutonomyModeOptions.length > 1) - const autonomyModeTooltip = $derived.by(() => { - switch (effectiveAutonomyMode) { - case AIAutonomyMode.ACCEPT_EDIT: - return 'Automatically accepts script and flow edits. Tool calls still ask for confirmation.' - case AIAutonomyMode.YOLO: - if (!aiChatManager.autoAcceptEditsAvailable) { - return 'Automatically accepts tool confirmations.' - } - return 'Automatically accepts script and flow edits plus tool confirmations.' - default: - if (!aiChatManager.autoAcceptEditsAvailable) { - return 'Requires confirmation for tool calls.' - } - return 'Requires confirmation for edits and tool calls.' - } - }) + const effectiveAutonomyModeOption = $derived(autonomyModeOption(effectiveAutonomyMode)) // The typing-dots indicator implies the AI is busy, which is misleading while // the loop is parked on the user; surface a text pill instead so users know to @@ -916,10 +936,11 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> nonCaptureEvent unifiedSize="2xs" variant="default" - title={autonomyModeTooltip} + title={effectiveAutonomyModeOption.tooltip(autonomyAvailability)} + btnClasses={effectiveAutonomyModeOption.triggerClass ?? ''} startIcon={{ - icon: autonomyModeIcon(effectiveAutonomyMode), - classes: autonomyModeIconColor(effectiveAutonomyMode) + icon: effectiveAutonomyModeOption.icon, + classes: effectiveAutonomyModeOption.iconColor }} endIcon={{ icon: ChevronDown }} > @@ -928,6 +949,9 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {/snippet} {/if} + {#if effectiveAutonomyMode === AIAutonomyMode.PLAN} + {PLAN_MODE_MESSAGES.modeNote} + {/if} {#if effectiveAutonomyMode === AIAutonomyMode.YOLO && aiChatManager.autoAcceptToolConfirmationsAvailable} diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 806b963262..5644ce6fd1 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -134,7 +134,10 @@ import { scopedKey, onUserChange, migrateLegacyLocalStorage } from '$lib/userSco import { getLocalSetting, storeLocalSetting } from '$lib/utils' import { AttachedFilesStore } from './files/attachedFiles.svelte' import { SessionArtifactsStore } from './artifacts/artifactsState.svelte' +import type { ArtifactVersionTarget } from '$lib/components/sessions/previewRouter' import { appendAttachedFilesRoster } from './files/fileTools' +import { ENTER_PLAN_MODE_TOOL, EXIT_PLAN_MODE_TOOL } from './planMode' +import { PlanModeController, type PlanModeHost } from './planModeController.svelte' // SSR and users who prefer reduced motion get no typewriter pacing. function prefersInstantReveal(): boolean { @@ -207,6 +210,7 @@ export enum AIMode { } export enum AIAutonomyMode { + PLAN = 'plan', DEFAULT = 'default', ACCEPT_EDIT = 'acceptedit', YOLO = 'yolo' @@ -221,6 +225,7 @@ const AUTO_ACCEPT_TOOL_CONFIRMATION_MODES = new Set([ AIMode.APP, AIMode.GLOBAL ]) +const PLAN_MODES = new Set([AIMode.GLOBAL]) export function isAIMode(mode: unknown): mode is AIMode { return ALL_AI_MODES.includes(mode as AIMode) @@ -238,6 +243,10 @@ export function supportsAutoAcceptToolConfirmations(mode: AIMode): boolean { return AUTO_ACCEPT_TOOL_CONFIRMATION_MODES.has(mode) } +export function supportsPlanMode(mode: AIMode): boolean { + return PLAN_MODES.has(mode) +} + export function isAIModeVisible(mode: AIMode): boolean { return mode !== AIMode.GLOBAL || isGlobalAiEnabled() } @@ -262,7 +271,7 @@ function getPersistedAutonomyMode(): AIAutonomyMode { return AIAutonomyMode.ACCEPT_EDIT } const persistedMode = getLocalSetting(key) - if (isAIAutonomyMode(persistedMode)) { + if (isAIAutonomyMode(persistedMode) && persistedMode !== AIAutonomyMode.PLAN) { return persistedMode } // No stored preference: default to auto-accepting edits (tool calls still @@ -275,6 +284,11 @@ function getPersistedAutonomyMode(): AIAutonomyMode { } function persistAutonomyMode(mode: AIAutonomyMode) { + // Plan is session-only: persisting it would re-block a later session where the + // picker never offered Plan. The stored pre-plan baseline is what a reload restores. + if (mode === AIAutonomyMode.PLAN) { + return + } const key = scopedKey(AI_AUTONOMY_MODE_STORAGE_KEY) if (!BROWSER || !key) { return @@ -342,6 +356,37 @@ type QueuedEntry = { context: ContextElement[] | undefined } +/** Plan mode's view of the chat it runs in. A function rather than an object literal in the + * field initializer so the getters close over the manager instead of over themselves. */ +function planModeHostFor(m: AIChatManager): PlanModeHost { + return { + get active() { + return m.planModeActive + }, + get available() { + return m.planModeAvailable + }, + get autoAccepting() { + return m.autoAcceptToolConfirmationsActive + }, + get isSessionChat() { + return m.isSessionChat + }, + get sessionId() { + return m.sessionId + }, + get chatId() { + return m.historyManager.getCurrentChatId() + }, + get artifacts() { + return m.artifacts + }, + openArtifact: (id, name, version) => m.openArtifact?.(id, name, version), + enter: () => m.setAutonomyMode(AIAutonomyMode.PLAN), + restore: () => m.setAutonomyMode(m.prePlanAutonomyMode ?? AIAutonomyMode.DEFAULT) + } +} + export class AIChatManager { contextManager = new ContextManager() historyManager = new HistoryManager() @@ -416,7 +461,7 @@ export class AIChatManager { * undefined in the global side-panel chat, where the tray falls back to opening * the run in a new browser tab. */ openRunInPreview?: (a: { jobId: string; workspace: string; label: string }) => void - openArtifact?: (artifactId: string, name: string) => void + openArtifact?: (artifactId: string, name: string, version?: ArtifactVersionTarget) => void closeArtifact?: (artifactId: string) => void loading = $state(false) currentReply = $state('') @@ -528,6 +573,10 @@ export class AIChatManager { // labels while set; the hook clears it back to undefined when done. loadingLabel = $state(undefined) autonomyMode = $state(getPersistedAutonomyMode()) + // Set by AI sessions. Enables the session-only preview tools and gates plan mode, which + // needs the preview pane; the global side-panel chat leaves it false. Reactive because + // `planModeAvailable` derives from it. + isSessionChat = $state(false) autoAcceptEditsAvailable = $derived(supportsAutoAcceptEdits(this.mode)) autoAcceptEditsActive = $derived( this.autoAcceptEditsAvailable && @@ -538,6 +587,12 @@ export class AIChatManager { autoAcceptToolConfirmationsActive = $derived( this.autonomyMode === AIAutonomyMode.YOLO && this.autoAcceptToolConfirmationsAvailable ) + planModeAvailable = $derived(this.isSessionChat && supportsPlanMode(this.mode)) + planModeActive = $derived(this.autonomyMode === AIAutonomyMode.PLAN && this.planModeAvailable) + prePlanAutonomyMode = $state(undefined) + // The posture's own state — its two tools, the plan document and the planning round. + // Everything it needs from this manager goes through the host above. + planMode = new PlanModeController(planModeHostFor(this)) #automaticScroll = $state(true) systemMessage = $state({ role: 'system', @@ -567,15 +622,14 @@ export class AIChatManager { /** Cached datatables for app context (fetched asynchronously) */ cachedDatatables = $state([]) - private confirmationCallbacks = new Map void>() + private confirmationCallbacks = new Map< + string, + { resolve: (value: boolean) => void; toolName?: string } + >() private userQuestionCallbacks = new Map void>() private appDatatablesRefreshTimeout: ReturnType | undefined = undefined disabledModes: Partial> = $state({}) - // Set by AI sessions. Enables the session-only preview tools (open_preview / - // get_preview_status) and their system-prompt guidance in GLOBAL mode; the - // global side-panel chat leaves it false so those tools aren't offered. - isSessionChat = false // The session this manager belongs to (session chats only). Carried into the // tool `helpers` in GLOBAL mode so the preview/deploy tools dispatch to THIS // session rather than the UI-active one — keeps backgrounded sessions isolated. @@ -1449,14 +1503,23 @@ export class AIChatManager { } } + /** enter_plan_mode never qualifies: YOLO means "stop asking and run it", and a read-only + * posture inverts that. Every accept path asks here rather than carrying its own copy. */ + private autoAcceptsTool = (toolName: string | undefined) => toolName !== ENTER_PLAN_MODE_TOOL + + /** Asked before the confirmation wait is skipped, so a tool the posture will not answer + * for still gets a card rather than running unasked. */ + shouldAutoAcceptTool = (toolName?: string) => + this.autoAcceptToolConfirmationsActive && this.autoAcceptsTool(toolName) + // Request confirmation from user for a tool call - requestConfirmation = (toolId: string): Promise => { + requestConfirmation = (toolId: string, toolName?: string): Promise => { if (this.autoAcceptToolConfirmationsActive) { - return Promise.resolve(true) + return Promise.resolve(this.autoAcceptsTool(toolName)) } return new Promise((resolve) => { - this.confirmationCallbacks.set(toolId, resolve) + this.confirmationCallbacks.set(toolId, { resolve, toolName }) }) } @@ -1464,14 +1527,14 @@ export class AIChatManager { handleToolConfirmation = (toolId: string, confirmed: boolean) => { const confirmationCallback = this.confirmationCallbacks.get(toolId) if (confirmationCallback) { - confirmationCallback(confirmed) + confirmationCallback.resolve(confirmed) this.confirmationCallbacks.delete(toolId) } } private acceptPendingToolConfirmations = () => { - for (const confirmationCallback of this.confirmationCallbacks.values()) { - confirmationCallback(true) + for (const { resolve, toolName } of this.confirmationCallbacks.values()) { + resolve(this.autoAcceptsTool(toolName)) } this.confirmationCallbacks.clear() } @@ -1482,10 +1545,34 @@ export class AIChatManager { } } + private resolvePendingPlanCard = (toolName: string, confirmed: boolean) => { + for (const [toolId, cb] of this.confirmationCallbacks) { + if (cb.toolName === toolName) { + cb.resolve(confirmed) + this.confirmationCallbacks.delete(toolId) + } + } + } + setAutonomyMode = (mode: AIAutonomyMode) => { + const enteringPlan = mode === AIAutonomyMode.PLAN && this.autonomyMode !== AIAutonomyMode.PLAN + const leavingPlan = mode !== AIAutonomyMode.PLAN && this.autonomyMode === AIAutonomyMode.PLAN + if (enteringPlan) { + this.prePlanAutonomyMode = this.autonomyMode + this.planMode.startRound() + } else if (mode !== AIAutonomyMode.PLAN) { + this.prePlanAutonomyMode = undefined + this.planMode.resetBlocks() + } this.autonomyMode = mode persistAutonomyMode(mode) + if (enteringPlan) { + this.resolvePendingPlanCard(ENTER_PLAN_MODE_TOOL, true) + } else if (leavingPlan) { + // Opting into YOLO means "run it"; leaving plan mode any other way is not a sign-off. + this.resolvePendingPlanCard(EXIT_PLAN_MODE_TOOL, mode === AIAutonomyMode.YOLO) + } if (this.autoAcceptToolConfirmationsActive) { this.acceptPendingToolConfirmations() } @@ -1507,6 +1594,8 @@ export class AIChatManager { hydrateUserScopedAutonomy = () => { migrateLegacyAutonomyKeys() this.autonomyMode = getPersistedAutonomyMode() + this.prePlanAutonomyMode = undefined + this.planMode.resetRound() } applyScriptEditorCode = async (code: string, opts?: ReviewChangesOpts) => { @@ -1761,6 +1850,12 @@ export class AIChatManager { } ) { if (!isAIModeVisible(mode)) return + // A session chat is GLOBAL for its whole life, and the plan gate reads that mode: moving + // it lifts the gate on a session the user still has set to Plan. + if (this.isSessionChat && mode !== AIMode.GLOBAL) { + console.error(`Refusing to move a session chat to ${mode} mode: sessions are GLOBAL-only.`) + return + } if (mode === AIMode.SCRIPT && !tryGetCurrentModel()) return this.mode = mode this.pendingPrompt = pendingPrompt ?? '' @@ -2270,20 +2365,21 @@ export class AIChatManager { messages, addedMessages, get systemMessage() { - const base = systemMessageOverride ?? self.systemMessage + let base = systemMessageOverride ?? self.systemMessage // Inject the attached-files roster at request time (re-read each iteration) // so it always reflects the live file list without reactive bookkeeping. if (self.mode === AIMode.GLOBAL && self.attachedFiles.count > 0) { - return appendAttachedFilesRoster( + base = appendAttachedFilesRoster( base, self.attachedFiles, self.orphanedMessageFileIds() ) } + base = self.planMode.decorateSystemMessage(base) return base }, get tools() { - return self.tools + return [...self.tools, ...self.planMode.tools] }, get helpers() { return self.helpers @@ -2567,6 +2663,7 @@ export class AIChatManager { return false } } + this.planMode.resetBlocks() // Built-in session commands run locally instead of becoming a chat turn. // Intercepted here — before the beforeSend workspace commit, file regrants, // and skill expansion. Scoped to session chat GLOBAL mode, where the @@ -3118,7 +3215,9 @@ export class AIChatManager { } }, requestConfirmation: this.requestConfirmation, - shouldAutoAcceptToolConfirmations: () => this.autoAcceptToolConfirmationsActive, + shouldAutoAcceptToolConfirmations: this.shouldAutoAcceptTool, + isPlanModeActive: () => this.planModeActive, + onToolBlockedByPlanMode: this.planMode.noteBlockedTool, requestUserQuestion: this.requestUserQuestion, onItemModified: (kind, path) => this.recordModifiedItem(kind, path), onItemDeployed: (kind, from, to) => void this.renameModifiedItem(kind, from, to), @@ -3359,8 +3458,8 @@ export class AIChatManager { } cancel = (reason?: string) => { - for (const confirmationCallback of this.confirmationCallbacks.values()) { - confirmationCallback(false) + for (const { resolve } of this.confirmationCallbacks.values()) { + resolve(false) } this.confirmationCallbacks.clear() for (const resolveQuestion of this.userQuestionCallbacks.values()) { @@ -3536,6 +3635,7 @@ export class AIChatManager { // Message-attached rows belong to the conversation just left in every case. this.#syncMessageFiles() this.syncArtifactsSession() + this.planMode.resetRound() this.onChatRotated?.(this.historyManager.getCurrentChatId()) } @@ -3578,6 +3678,7 @@ export class AIChatManager { this.#syncMessageFiles() this.#automaticScroll = true this.syncArtifactsSession() + this.planMode.resetRound() this.onChatRotated?.(id) } } diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index 3b7dd521c0..e4574ee9c1 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -6,6 +6,7 @@ import type { ReviewChangesOpts } from './monaco-adapter' import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs' import type { AttachedImage } from './imageUtils' import { AIChatManager, AIMode, AIAutonomyMode } from './AIChatManager.svelte' +import { PLAN_MODE_MESSAGES } from './planModeMessages' import { runChatLoop } from './chatLoop' // This suite forces esm-env BROWSER=true (below). That makes @sveltejs/kit's @@ -408,6 +409,121 @@ describe('AIChatManager autonomy mode', () => { }) }) +// The posture's own behaviour lives in planModeController.test.ts. What is left here is the +// wiring only the manager owns: which pending confirmation cards a change of autonomy mode +// answers, and with what. +describe('AIChatManager plan mode posture', () => { + beforeEach(() => { + localStorage.clear() + // Plan mode is never the persisted posture, so a case starts from the one it is + // entered from and hands back to. + localStorage.setItem(`ai-chat-autonomy-mode::${TEST_EMAIL}`, AIAutonomyMode.DEFAULT) + vi.clearAllMocks() + }) + + const sessionManager = (mode = AIAutonomyMode.DEFAULT) => { + const manager = new AIChatManager() + manager.mode = AIMode.GLOBAL + manager.isSessionChat = true + manager.setAutonomyMode(mode) + return manager + } + + it('enters plan mode through the tool and remembers the posture to hand back to', async () => { + const manager = sessionManager(AIAutonomyMode.ACCEPT_EDIT) + + await manager.planMode.enterTool.fn({ + args: { reason: 'research the change first' }, + workspace: 'test-workspace', + helpers: {}, + toolCallbacks: { setToolStatus: vi.fn(), removeToolStatus: vi.fn() }, + toolId: 'call_enter' + }) + + expect(manager.planModeActive).toBe(true) + expect(manager.prePlanAutonomyMode).toBe(AIAutonomyMode.ACCEPT_EDIT) + }) + + it('refuses to move a session chat out of GLOBAL, so the gate cannot lift under it', () => { + const manager = sessionManager(AIAutonomyMode.ACCEPT_EDIT) + manager.setAutonomyMode(AIAutonomyMode.PLAN) + expect(manager.planModeActive).toBe(true) + // Without a configured model changeMode returns early on SCRIPT, and the case would pass + // against the very guard it is meant to pin. + mocks.getCurrentModel.mockReturnValue({ provider: 'openai', model: 'gpt-4o' }) + mocks.tryGetCurrentModel.mockReturnValue({ provider: 'openai', model: 'gpt-4o' }) + const logged = vi.spyOn(console, 'error').mockImplementation(() => {}) + + manager.changeMode(AIMode.SCRIPT) + + expect(manager.mode).toBe(AIMode.GLOBAL) + expect(manager.planModeActive).toBe(true) + // A switch that silently does nothing gives its caller no way to learn why. + expect(logged).toHaveBeenCalled() + logged.mockRestore() + }) + + it('never auto-accepts an enter_plan_mode card, whichever side of the switch it lands on', async () => { + // Switching to YOLO answers every pending confirmation — except this one. "Run it + // without asking" must not be answered by forcing the user into a read-only posture. + const before = sessionManager() + const enterPending = before.requestConfirmation('call_enter', 'enter_plan_mode') + const writePending = before.requestConfirmation('call_write', 'write_script') + before.setAutonomyMode(AIAutonomyMode.YOLO) + expect(await enterPending).toBe(false) + expect(await writePending).toBe(true) + }) + + it('declines an enter_plan_mode that arrives after the switch to YOLO', async () => { + // The tool set is snapshotted per iteration, so a call can still arrive once the user + // has moved to YOLO. Driven through processToolCall rather than requestConfirmation + // directly: an auto-accepting posture skips the confirmation wait entirely, so asserting + // against the wait would pass on a build that never reaches it. + const { processToolCall } = await import('./shared') + const manager = sessionManager(AIAutonomyMode.YOLO) + + const result = await processToolCall({ + tools: [manager.planMode.enterTool] as any, + toolCall: { + id: 'call_enter', + type: 'function', + function: { name: 'enter_plan_mode', arguments: JSON.stringify({ reason: 'research' }) } + } as any, + helpers: {}, + workspace: 'test-workspace', + toolCallbacks: { + setToolStatus: vi.fn(), + removeToolStatus: vi.fn(), + requestConfirmation: manager.requestConfirmation, + shouldAutoAcceptToolConfirmations: manager.shouldAutoAcceptTool + } as any + }) + + expect(result.content).toBe(PLAN_MODE_MESSAGES.enterDeclined) + expect(manager.autonomyMode).toBe(AIAutonomyMode.YOLO) + expect(manager.planModeActive).toBe(false) + }) + + it('answers a pending plan card the way the picker was moved', async () => { + const entering = sessionManager() + const enterPending = entering.requestConfirmation('call_enter', 'enter_plan_mode') + entering.setAutonomyMode(AIAutonomyMode.PLAN) + expect(await enterPending).toBe(true) + + // Leaving plan mode any other way is not a sign-off on the plan on the card. + const leaving = sessionManager(AIAutonomyMode.PLAN) + const exitPending = leaving.requestConfirmation('call_exit', 'exit_plan_mode') + leaving.setAutonomyMode(AIAutonomyMode.DEFAULT) + expect(await exitPending).toBe(false) + + // Opting into YOLO does mean "run it". + const yolo = sessionManager(AIAutonomyMode.PLAN) + const yoloPending = yolo.requestConfirmation('call_exit', 'exit_plan_mode') + yolo.setAutonomyMode(AIAutonomyMode.YOLO) + expect(await yoloPending).toBe(true) + }) +}) + describe('AIChatManager persisted autonomy default', () => { // Mirrors the private storage keys in AIChatManager.svelte.ts, namespaced by // the logged-in user's email (see userScopedStorage). diff --git a/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte b/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte new file mode 100644 index 0000000000..699b79a292 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte @@ -0,0 +1,55 @@ + + +
+ + +
diff --git a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte index 754db1aac0..34b1adcf68 100644 --- a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte @@ -1,12 +1,39 @@