From 5cdf984afc16b3bbb3cee1d6f88ac11ab723c135 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 8 Oct 2026 18:13:13 +0200 Subject: [PATCH] feat: approve browser actions in the chat's own card and restyle the extension pages Co-Authored-By: Claude Opus 5.5 --- backend/ee-repo-ref.txt | 2 +- browser_extension/README.md | 10 +- browser_extension/src/options.ts | 24 ++- browser_extension/src/panel.ts | 146 ++++++++---------- browser_extension/static/logo.svg | 30 ++++ browser_extension/static/options.html | 39 +++-- browser_extension/static/panel.html | 38 ++--- browser_extension/static/style.css | 140 +++++++++++++++++ .../copilot/chat/global/browserTools.ts | 144 ++++++++--------- 9 files changed, 375 insertions(+), 198 deletions(-) create mode 100644 browser_extension/static/logo.svg create mode 100644 browser_extension/static/style.css diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index d9cc367108..8269a8502b 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -88d09892319116b701797704493e06cc098c66d3 +3bf2821b6847c1224d3e814ca39aa7c5345cdfe3 diff --git a/browser_extension/README.md b/browser_extension/README.md index 4a2024c857..a8aef0dcf2 100644 --- a/browser_extension/README.md +++ b/browser_extension/README.md @@ -25,9 +25,13 @@ icon to open the panel. ## Guardrails - Only the active tab of the panel's window is read or acted on. -- Reading and screenshots run without asking. Each click, type and navigation waits for - Allow in a bar the extension draws below the frame, which neither the chat nor the page - can answer. +- Reading and screenshots run without asking. Each click, type and navigation waits for the + chat's own approval card, which lives in Windmill's frame where the page cannot reach it. + YOLO auto-approval is unavailable in the panel. +- Approval runs in two steps. When the card opens, the extension pins the target (tab, + document and element, the element held in the extension's isolated world) and describes + it for the card. After approval it acts only on that target, and refuses if the tab is no + longer active, the page navigated or reloaded, or the element changed. - Both sides check the bridge's origins. The panel accepts messages only from its own frame at the configured instance origin. Windmill accepts them only from its parent window, when that parent is a `chrome-extension://` origin (`location.ancestorOrigins`). diff --git a/browser_extension/src/options.ts b/browser_extension/src/options.ts index caabfe413b..de99e2d3d6 100644 --- a/browser_extension/src/options.ts +++ b/browser_extension/src/options.ts @@ -1,6 +1,12 @@ const input = document.getElementById('url') as HTMLInputElement +const save = document.getElementById('save') as HTMLButtonElement const status = document.getElementById('status')! +function show(text: string, kind: 'ok' | 'error' | '' = '') { + status.textContent = text + status.className = `status ${kind}` +} + chrome.storage.sync .get('instanceUrl') .then(({ instanceUrl }) => (input.value = (instanceUrl as string) ?? '')) @@ -12,14 +18,24 @@ document.getElementById('form')!.addEventListener('submit', async (e) => { url = new URL(input.value.trim()) if (url.protocol !== 'https:' && url.protocol !== 'http:') throw new Error() } catch { - status.textContent = 'Enter an http(s) URL, such as https://app.windmill.dev' - return + return show('Enter an http(s) URL, such as https://app.windmill.dev', 'error') } // Keeps a base path (an instance served under /windmill), drops everything after it. const instanceUrl = (url.origin + url.pathname).replace(/\/+$/, '') - await chrome.storage.sync.set({ instanceUrl }) input.value = instanceUrl - status.textContent = 'Saved. Reopen the side panel to use it.' + save.disabled = true + show('Checking…') + try { + const res = await fetch(`${instanceUrl}/api/version`) + const version = res.ok ? (await res.text()).trim() : '' + if (!version) throw new Error() + await chrome.storage.sync.set({ instanceUrl }) + show(`Connected to Windmill ${version}. Open the side panel from the toolbar.`, 'ok') + } catch { + show(`No Windmill instance answered at ${instanceUrl}.`, 'error') + } finally { + save.disabled = false + } }) export {} diff --git a/browser_extension/src/panel.ts b/browser_extension/src/panel.ts index f3622deb64..0d15e1ebf1 100644 --- a/browser_extension/src/panel.ts +++ b/browser_extension/src/panel.ts @@ -1,15 +1,24 @@ // Hosts Windmill's chat-only session route in an iframe and executes the browser tools it // relays (frontend `global/browserTools.ts`) on the active tab of this window. +// +// Click, type and navigate run in two steps around the chat's own approval card: `prepare` +// pins the tab, document and element and describes them for the card, and `act` runs only a +// prepared action, after re-checking that what was described is still there. type Target = { ref?: number; selector?: string } -type Call = - | { tool: 'read'; args: {} } - | { tool: 'screenshot'; args: {} } +type Action = | { tool: 'click'; args: Target } | { tool: 'type'; args: Target & { text: string; submit?: boolean } } | { tool: 'navigate'; args: { url: string } } +type Call = + | { tool: 'read'; args: {} } + | { tool: 'screenshot'; args: {} } + | { tool: 'prepare'; args: Action } + | { tool: 'act'; args: { approvalId: string } } +type Prepared = Action & { tab: chrome.tabs.Tab; documentId?: string; nonce: string } const frame = document.getElementById('frame') as HTMLIFrameElement +const prepared = new Map() async function main() { const { instanceUrl } = (await chrome.storage.sync.get('instanceUrl')) as { instanceUrl?: string } @@ -31,11 +40,10 @@ async function main() { if (msg?.type === 'wm-browser:hello') { reply({ type: 'wm-browser:ready' }) } else if (msg?.type === 'wm-browser:cancel') { - generation++ - declineOpen?.() + prepared.clear() } else if (msg?.type === 'wm-browser:call' && typeof msg.id === 'string') { try { - const result = await run({ tool: msg.tool, args: msg.args ?? {} } as Call, generation) + const result = await run({ tool: msg.tool, args: msg.args ?? {} } as Call) reply({ type: 'wm-browser:result', id: msg.id, ok: true, result }) } catch (err) { const error = err instanceof Error ? err.message : String(err) @@ -45,7 +53,8 @@ async function main() { }) } -async function run(call: Call, gen: number): Promise { +async function run(call: Call): Promise { + if (call.tool === 'act') return act(call.args.approvalId) const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }) if (tab?.id === undefined) throw new Error('No active tab') const tabId = tab.id @@ -54,48 +63,61 @@ async function run(call: Call, gen: number): Promise { return (await exec({ tabId }, readPage, [30_000]))[0] case 'screenshot': return chrome.tabs.captureVisibleTab(tab.windowId, { format: 'jpeg', quality: 80 }) - case 'click': - case 'type': { - // Only the element described for the approval is acted on after Allow. - const nonce = crypto.randomUUID() - const [label, documentId] = await exec({ tabId }, pageAction, ['describe', call.args, nonce]) - const submit = call.tool === 'type' && call.args.submit ? ' and submit' : '' - await confirmAction( - call.tool === 'click' - ? `Click ${label}` - : `Type "${call.args.text}" into ${label}${submit}`, - tab, - gen - ) - await stillApproved(tab, documentId) - const target = { tabId, documentIds: [documentId] } - const [result] = await exec(target, pageAction, [call.tool, call.args, nonce]).catch( - (e: Error) => { - throw /No document with id/.test(e.message) ? new Error(PAGE_CHANGED) : e - } - ) - return result - } - case 'navigate': { - const url = new URL(call.args.url) - if (url.protocol !== 'https:' && url.protocol !== 'http:') { - throw new Error('Only http(s) URLs can be opened') - } - // A page the extension cannot script (a browser page) is pinned by its URL alone. - const documentId = await exec({ tabId }, () => ({ result: null }), []).then( - ([, id]) => id, - () => undefined - ) - await confirmAction(`Open ${url.href}`, tab, gen) - await stillApproved(tab, documentId) - await chrome.tabs.update(tabId, { url: url.href }) - return `Navigating to ${url.href}` - } + case 'prepare': + return prepare(call.args, tab) default: throw new Error(`Unknown browser tool ${(call as { tool: string }).tool}`) } } +async function prepare(action: Action, tab: chrome.tabs.Tab) { + const tabId = tab.id! + const nonce = crypto.randomUUID() + const host = tab.url ? new URL(tab.url).host : 'the active tab' + let label: string + let documentId: string | undefined + if (action.tool === 'navigate') { + const url = new URL(action.args.url) + if (url.protocol !== 'https:' && url.protocol !== 'http:') { + throw new Error('Only http(s) URLs can be opened') + } + // A page the extension cannot script (a browser page) is pinned by its URL alone. + documentId = await exec({ tabId }, () => ({ result: null }), []).then( + ([, id]) => id, + () => undefined + ) + label = `Open ${url.href}` + } else { + const [target, id] = await exec({ tabId }, pageAction, ['describe', action.args, nonce]) + documentId = id + const submit = action.tool === 'type' && action.args.submit ? ' and submit' : '' + label = + action.tool === 'click' + ? `Click ${target} on ${host}` + : `Type "${action.args.text}" into ${target} on ${host}${submit}` + } + const approvalId = crypto.randomUUID() + prepared.set(approvalId, { ...action, tab, documentId, nonce }) + return { approvalId, label } +} + +async function act(approvalId: string): Promise { + const p = prepared.get(approvalId) + prepared.delete(approvalId) + if (!p) throw new Error('This action was withdrawn. Ask again.') + await stillApproved(p.tab, p.documentId) + const tabId = p.tab.id! + if (p.tool === 'navigate') { + await chrome.tabs.update(tabId, { url: new URL(p.args.url).href }) + return `Navigating to ${p.args.url}` + } + const target = { tabId, documentIds: [p.documentId!] } + const [result] = await exec(target, pageAction, [p.tool, p.args, p.nonce]).catch((e: Error) => { + throw /No document with id/.test(e.message) ? new Error(PAGE_CHANGED) : e + }) + return result +} + // Injected functions run in the page and must be self-contained; they report failure as // `{ error }` because a throw inside them reaches us without its message. async function exec( @@ -128,41 +150,6 @@ async function stillApproved(tab: chrome.tabs.Tab, documentId: string | undefine } } -let confirmQueue: Promise = Promise.resolve() -// Bumped when the chat stops its turn: a confirmation asked before that is never shown, and -// the open one is declined. -let generation = 0 -let declineOpen: (() => void) | undefined - -/** Shown outside the Windmill frame, so neither the chat nor the page can answer it. */ -function confirmAction(text: string, tab: chrome.tabs.Tab, gen: number): Promise { - const ask = () => - new Promise((resolve, reject) => { - if (gen !== generation) return reject(new Error('Stopped by the user')) - const box = document.getElementById('confirm')! - document.getElementById('confirm-text')!.textContent = text - document.getElementById('confirm-host')!.textContent = `On ${tab.url ?? 'the active tab'}` - box.hidden = false - const allow = document.getElementById('allow') as HTMLButtonElement - const deny = document.getElementById('deny') as HTMLButtonElement - const done = (ok: boolean) => { - box.hidden = true - allow.onclick = deny.onclick = null - declineOpen = undefined - if (ok) resolve() - // The frontend matches this exact message to count declines. - else reject(new Error('The user declined this action')) - } - allow.onclick = () => done(true) - deny.onclick = () => done(false) - declineOpen = () => done(false) - // Nothing is focused: a keystroke meant for the chat composer must not approve. - }) - const next = confirmQueue.then(ask, ask) - confirmQueue = next.catch(() => {}) - return next -} - function readPage(maxChars: number) { document.querySelectorAll('[data-wm-ref]').forEach((el) => el.removeAttribute('data-wm-ref')) const elements: string[] = [] @@ -222,6 +209,7 @@ function pageAction( el.getAttribute('aria-label') || el.innerText || input.placeholder || + (input.type === 'submit' || input.type === 'button' ? input.value : '') || input.name || '' ) diff --git a/browser_extension/static/logo.svg b/browser_extension/static/logo.svg new file mode 100644 index 0000000000..3b7a0ab2a6 --- /dev/null +++ b/browser_extension/static/logo.svg @@ -0,0 +1,30 @@ + + + + + + + + + + + + + diff --git a/browser_extension/static/options.html b/browser_extension/static/options.html index e3a4c16fd1..a9b32d542d 100644 --- a/browser_extension/static/options.html +++ b/browser_extension/static/options.html @@ -3,29 +3,36 @@ Windmill extension settings + -
- - - -

-
+
+
+ +
+

Windmill

+

AI sessions in your browser's side panel

+
+
+
+ +

The address you open Windmill at, including any base path.

+ +
+ +

+
+
+
diff --git a/browser_extension/static/panel.html b/browser_extension/static/panel.html index 479e6bac53..1d060d7f1d 100644 --- a/browser_extension/static/panel.html +++ b/browser_extension/static/panel.html @@ -3,14 +3,11 @@ Windmill + -