feat: prepare browser actions before approval, allow yolo in the panel, add the windmill icon

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

browser_extension/README.md

@@ -27,7 +27,7 @@ icon to open the panel.
 - Only the active tab of the panel's window is read or acted on.
 - Reading and screenshots run without asking. Each click, type and navigation waits for the
   chat's own approval card, which lives in Windmill's frame where the page cannot reach it.
-  YOLO auto-approval is unavailable in the panel.
+  The chat's Yolo mode skips the card, and its warning says pages can try to steer it.
 - Approval runs in two steps. When the card opens, the extension pins the target (tab,
   document and element, the element held in the extension's isolated world) and describes
   it for the card. After approval it acts only on that target, and refuses if the tab is no

browser_extension/package.json

@@ -3,7 +3,7 @@
 	"private": true,
 	"version": "0.1.0",
 	"scripts": {
-		"build": "rm -rf dist && tsc && cp static/* dist/"
+		"build": "rm -rf dist && tsc && cp -r static/. dist/"
 	},
 	"devDependencies": {
 		"@types/chrome": "^0.3.4",

browser_extension/src/panel.ts

@@ -210,6 +210,7 @@ function pageAction(
 			el.innerText ||
 			input.placeholder ||
 			(input.type === 'submit' || input.type === 'button' ? input.value : '') ||
+			el.title ||
 			input.name ||
 			''
 		)

browser_extension/static/manifest.json

@@ -5,8 +5,25 @@
 	"description": "Windmill AI sessions in the side panel, with tools that act on the active tab.",
 	"permissions": ["sidePanel", "storage", "scripting", "tabs"],
 	"host_permissions": ["<all_urls>"],
-	"background": { "service_worker": "background.js", "type": "module" },
-	"action": { "default_title": "Open Windmill" },
-	"side_panel": { "default_path": "panel.html" },
-	"options_page": "options.html"
+	"background": {
+		"service_worker": "background.js",
+		"type": "module"
+	},
+	"action": {
+		"default_title": "Open Windmill",
+		"default_icon": {
+			"16": "icons/icon16.png",
+			"32": "icons/icon32.png"
+		}
+	},
+	"side_panel": {
+		"default_path": "panel.html"
+	},
+	"options_page": "options.html",
+	"icons": {
+		"16": "icons/icon16.png",
+		"32": "icons/icon32.png",
+		"48": "icons/icon48.png",
+		"128": "icons/icon128.png"
+	}
 }

frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte

@@ -1277,6 +1277,12 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. -->
 												? 'This can result in edits being applied or tools being called without user confirmation.'
 												: 'This can result in tools being called without user confirmation.'}
 										</p>
+										{#if chatHost.browserTools}
+											<p class="mt-1">
+												That includes clicking, typing and navigating on your page, and a page you
+												visit can try to steer the assistant into doing so.
+											</p>
+										{/if}
 										{#if yoloBypassedTools.length > 0}
 											<p class="mt-2 font-semibold">Bypassed in current mode:</p>
 											<ul class="mt-1 list-disc pl-4 space-y-0.5">

frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts

@@ -737,10 +737,7 @@ export class AIChatManager implements ChatViewHost {
 			(this.autonomyMode === AIAutonomyMode.ACCEPT_EDIT ||
 				this.autonomyMode === AIAutonomyMode.YOLO)
 	)
-	// Never in the side panel: page content in context could steer an unconfirmed workspace tool.
-	autoAcceptToolConfirmationsAvailable = $derived(
-		!this.browserTools && supportsAutoAcceptToolConfirmations(this.mode)
-	)
+	autoAcceptToolConfirmationsAvailable = $derived(supportsAutoAcceptToolConfirmations(this.mode))
 	autoAcceptToolConfirmationsActive = $derived(
 		this.autonomyMode === AIAutonomyMode.YOLO && this.autoAcceptToolConfirmationsAvailable
 	)

frontend/src/lib/components/copilot/chat/chatViewHost.ts

@@ -157,6 +157,8 @@ export interface ChatViewHost {
 	readonly autoAcceptEditsAvailable: boolean
 	readonly autoAcceptToolConfirmationsAvailable: boolean
 	readonly planModeAvailable: boolean
+	/** Hosted in the browser extension's side panel, with tools acting on the user's tab. */
+	readonly browserTools?: boolean
 	attachedFiles: AttachedFilesStore
 	artifacts: SessionArtifactsStore
 	openArtifact?: (artifactId: string, name: string, version?: ArtifactVersionTarget) => void

frontend/src/lib/components/copilot/chat/global/browserTools.ts

@@ -18,8 +18,9 @@ type Action = 'click' | 'type' | 'navigate'
 let parentOrigin: string | undefined
 let connection: Promise<boolean> | undefined
 const pending = new Map<string, { resolve: (v: any) => void; reject: (e: Error) => void }>()
-/** By tool call: the extension's description of the target, made when the approval card opens. */
-const prepared = new Map<string, Promise<{ approvalId: string; label: string }>>()
+/** By the call's own `args` object, the one object the validation, the approval card and the
+ * tool all receive: the target the extension pinned and described before the card opened. */
+const prepared = new WeakMap<object, { approvalId: string; label: string }>()
 
 /** Resolves true once the extension framing this page answers the handshake, accepting only
  * its own window's messages. */
@@ -50,7 +51,6 @@ export function connectBrowserBridge(): Promise<boolean> {
 export function cancelBrowserCalls() {
 	for (const p of pending.values()) p.reject(new Error('Stopped by the user'))
 	pending.clear()
-	prepared.clear()
 	if (parentOrigin) window.parent.postMessage({ type: 'wm-browser:cancel' }, parentOrigin)
 }
 
@@ -91,7 +91,9 @@ function fail(toolCallbacks: ToolCallbacks, toolId: string, content: string, e:
 }
 
 /** Click, type and navigate go through the chat's own approval card. The extension pins and
- * describes the target as the card opens, and after approval runs only what it described. */
+ * describes the target before the card opens, so Run is only offered on a described target,
+ * and after approval it runs only that. A call with nothing prepared is refused, never
+ * prepared again: a preparation made after consent would bind whatever the page holds then. */
 function actionTool(
 	tool: Action,
 	schema: z.ZodObject<any>,
@@ -99,26 +101,25 @@ function actionTool(
 	description: string,
 	label: (args: any) => string
 ): SessionTool<{}> {
-	const prepare = (args: any) => callBrowser('prepare', { tool, args: schema.parse(args) })
 	return {
 		requires: NONE,
 		def: createToolDef(schema, name, description),
 		showDetails: true,
 		requiresConfirmation: true,
-		confirmationMessage: label,
-		onConfirmationRequested: ({ args, toolCallbacks, toolId }) => {
-			const p = prepare(args)
-			prepared.set(toolId, p)
-			p.then(
-				(r) => toolCallbacks.setToolStatus(toolId, { content: r.label }),
-				() => {}
-			)
-		},
-		fn: async ({ args, toolId, toolCallbacks }) => {
-			const p = prepared.get(toolId) ?? prepare(args)
-			prepared.delete(toolId)
+		validateBeforeConfirmation: async ({ args }) => {
 			try {
-				const { approvalId, label: described } = await p
+				prepared.set(args, await callBrowser('prepare', { tool, args: schema.parse(args) }))
+			} catch (e) {
+				return `Failed: ${e instanceof Error ? e.message : String(e)}`
+			}
+		},
+		confirmationMessage: (args) => prepared.get(args)?.label ?? label(args),
+		fn: async ({ args, toolId, toolCallbacks }) => {
+			const entry = prepared.get(args)
+			prepared.delete(args)
+			try {
+				if (!entry) throw new Error('This action was withdrawn. Ask again.')
+				const { approvalId, label: described } = entry
 				const result = String(await callBrowser('act', { approvalId }))
 				toolCallbacks.setToolStatus(toolId, { content: described, result, isLoading: false })
 				return result