diff --git a/skill-guides/orca-cli/references/browser.md b/skill-guides/orca-cli/references/browser.md index 7c664aa884b..6b68f1ffc8a 100644 --- a/skill-guides/orca-cli/references/browser.md +++ b/skill-guides/orca-cli/references/browser.md @@ -75,6 +75,7 @@ Browser rules: - Use `ORCA set device` or `ORCA viewport --mobile`, not `ORCA exec --command "set device ..."`, so the page also gets touch input and `pointer: coarse`. Each page load (reload, `goto`, link click) resets it, so run it again after navigating. - Prefer `wait --text`, `--url`, `--selector`, or `--load` after async page changes instead of bare timeouts. Avoid `--load networkidle` except on pages known to go quiet: polling, websockets and streaming keep it from ever resolving. - Anything not listed above goes through `ORCA exec --page --command ""`, which takes agent-browser's own command syntax (for example `"get text @e3"` or `"is visible @e3"`) and prints JSON. It has no `help` command. +- `upload` takes the file input itself or the button that opens a file picker, in any frame, including cross-origin iframes; Orca catches the picker and sets the files on its input. Use `ORCA upload`, not `ORCA exec --command "upload ..."`. On server-hosted pages pass the file input's ref. - `fill` sets a field's whole value in one step, at any size. `type` presses real keys into the focused element, so `minlength`/`maxlength`, key handlers, Enter and Tab act as they would for a person; `ORCA focus` the element first. Use `type` when the page reacts to keystrokes. - On server-hosted pages `fill` and `type` still insert text the engine's own way. If either does nothing on a custom input there, try `ORCA focus --page --element @e1 --json` then `ORCA inserttext --page --text "text" --json`. - A client-hosted page renders in the paired desktop's browser engine, so every command against it needs that desktop online and returns `browser_host_unavailable` while it is closed, asleep, or disconnected. Server-hosted pages run with no desktop attached; prefer them for long or unattended automation. @@ -83,7 +84,7 @@ Common recoveries: - `browser_no_tab`: open a tab with `ORCA tab create --url --json`. - `browser_stale_ref`: run `ORCA snapshot --page --json` and retry on the same page with fresh refs. -- `browser_element_not_interactable`: the element can't take that command, and the message says why (not a text field, disabled, read-only, not a ``, opened no file picker). Re-snapshot and pick the right ref; for a custom dropdown, click it and then click the option. - `browser_tab_not_found`: run `ORCA tab list --json` before switching or closing. - `browser_dialog_open` / `browser_dialog_opened`: the page has a JavaScript alert, confirm or prompt open, and the error quotes it. Answer it with `ORCA dialog accept` (`--text` for a prompt; without it a prompt takes its default) or `ORCA dialog dismiss` before any other command on that page; a command that opened it may already have taken effect, so check before repeating it. - `browser_host_unavailable`: the desktop hosting the page is offline. Bring it back, or recreate the page with server placement if the work must outlive the desktop session. diff --git a/src/cli/bundled-skill-guides.ts b/src/cli/bundled-skill-guides.ts index c36b33c265b..6c66a809f41 100644 --- a/src/cli/bundled-skill-guides.ts +++ b/src/cli/bundled-skill-guides.ts @@ -894,6 +894,7 @@ const ORCA_CLI_BROWSER_REFERENCE_MARKDOWN = [ "- Use `ORCA set device` or `ORCA viewport --mobile`, not `ORCA exec --command \"set device ...\"`, so the page also gets touch input and `pointer: coarse`. Each page load (reload, `goto`, link click) resets it, so run it again after navigating.", "- Prefer `wait --text`, `--url`, `--selector`, or `--load` after async page changes instead of bare timeouts. Avoid `--load networkidle` except on pages known to go quiet: polling, websockets and streaming keep it from ever resolving.", "- Anything not listed above goes through `ORCA exec --page --command \"\"`, which takes agent-browser's own command syntax (for example `\"get text @e3\"` or `\"is visible @e3\"`) and prints JSON. It has no `help` command.", + "- `upload` takes the file input itself or the button that opens a file picker, in any frame, including cross-origin iframes; Orca catches the picker and sets the files on its input. Use `ORCA upload`, not `ORCA exec --command \"upload ...\"`. On server-hosted pages pass the file input's ref.", "- `fill` sets a field's whole value in one step, at any size. `type` presses real keys into the focused element, so `minlength`/`maxlength`, key handlers, Enter and Tab act as they would for a person; `ORCA focus` the element first. Use `type` when the page reacts to keystrokes.", "- On server-hosted pages `fill` and `type` still insert text the engine's own way. If either does nothing on a custom input there, try `ORCA focus --page --element @e1 --json` then `ORCA inserttext --page --text \"text\" --json`.", "- A client-hosted page renders in the paired desktop's browser engine, so every command against it needs that desktop online and returns `browser_host_unavailable` while it is closed, asleep, or disconnected. Server-hosted pages run with no desktop attached; prefer them for long or unattended automation.", @@ -902,7 +903,7 @@ const ORCA_CLI_BROWSER_REFERENCE_MARKDOWN = [ "", "- `browser_no_tab`: open a tab with `ORCA tab create --url --json`.", "- `browser_stale_ref`: run `ORCA snapshot --page --json` and retry on the same page with fresh refs.", - "- `browser_element_not_interactable`: the element can't take that command, and the message says why (not a text field, disabled, read-only, not a ``, opened no file picker). Re-snapshot and pick the right ref; for a custom dropdown, click it and then click the option.", "- `browser_tab_not_found`: run `ORCA tab list --json` before switching or closing.", "- `browser_dialog_open` / `browser_dialog_opened`: the page has a JavaScript alert, confirm or prompt open, and the error quotes it. Answer it with `ORCA dialog accept` (`--text` for a prompt; without it a prompt takes its default) or `ORCA dialog dismiss` before any other command on that page; a command that opened it may already have taken effect, so check before repeating it.", "- `browser_host_unavailable`: the desktop hosting the page is offline. Bring it back, or recreate the page with server placement if the work must outlive the desktop session.", diff --git a/src/cli/handlers/browser-interact.ts b/src/cli/handlers/browser-interact.ts index 7f5f946e0fc..c0833524b88 100644 --- a/src/cli/handlers/browser-interact.ts +++ b/src/cli/handlers/browser-interact.ts @@ -133,7 +133,12 @@ export const BROWSER_INTERACT_HANDLERS: Record = { files, ...target }) - printResult(result, json, (v) => `Uploaded ${v.uploaded} file(s)`) + printResult( + result, + json, + (v) => + `Uploaded ${v.uploaded} file(s)${v.frameUrl ? ` to a file input in ${v.frameUrl}` : ''}` + ) }, scrollintoview: async ({ flags, client, cwd, json }) => { const element = getRequiredStringFlag(flags, 'element') diff --git a/src/cli/root-help-text-secondary.ts b/src/cli/root-help-text-secondary.ts index 9e46cd66af3..f056bbfaa41 100644 --- a/src/cli/root-help-text-secondary.ts +++ b/src/cli/root-help-text-secondary.ts @@ -3,7 +3,7 @@ export const ROOT_HELP_TEXT_SECONDARY = [ ' focus Focus an element by --element ref', ' clear Clear an input by --element ref', ' drag Drag --from ref to --to ref', - ' upload Upload --files to a file input by --element ref', + ' upload Upload --files to a file input or picker button by --element ref', ' dblclick Double-click element by --element ref', ' forward Navigate forward in browser history', ' scrollintoview Scroll --element into view', diff --git a/src/cli/specs/browser-basic.ts b/src/cli/specs/browser-basic.ts index 49fd29c2623..6365c7a18de 100644 --- a/src/cli/specs/browser-basic.ts +++ b/src/cli/specs/browser-basic.ts @@ -190,7 +190,7 @@ export const BROWSER_BASIC_COMMAND_SPECS: CommandSpec[] = withBrowserPageTarget( }, { path: ['upload'], - summary: 'Upload files to a file input element', + summary: 'Upload files to a file input, or to the one a picker button opens', usage: 'orca upload --element --files [--worktree ] [--json]', allowedFlags: [...GLOBAL_FLAGS, 'element', 'files', 'worktree'] }, diff --git a/src/main/browser/agent-browser-bridge-element-calls.ts b/src/main/browser/agent-browser-bridge-element-calls.ts new file mode 100644 index 00000000000..3446f7706e6 --- /dev/null +++ b/src/main/browser/agent-browser-bridge-element-calls.ts @@ -0,0 +1,118 @@ +import type { WebContents } from 'electron' +import { BrowserError } from './cdp-bridge' +import type { CapturedElement } from './cdp-element-capture' +import { sendGuestCdpCommand } from './guest-cdp-command' +import { readRecord, readString } from './guest-frame-input-sessions' +import { AgentBrowserBridgeExecution } from './agent-browser-bridge-execution' + +export type CapturedTarget = { wc: WebContents; captured: CapturedElement } + +export abstract class AgentBrowserBridgeElementCalls extends AgentBrowserBridgeExecution { + /** Runs a browser-element-functions action on the element the helper resolves, in its own frame. */ + protected async actOnElement( + sessionName: string, + element: string, + action: string, + functionDeclaration: string, + args: unknown[] = [] + ): Promise { + const target = await this.captureElement(sessionName, element) + return await this.callElementFunction( + sessionName, + target, + element, + action, + functionDeclaration, + { + args + } + ) + } + + /** Resolves a ref or selector to the element the helper found, with the CDP session of its frame. */ + protected async captureElement(sessionName: string, element: string): Promise { + const session = this.sessions.get(sessionName) + if (!session) { + throw this.createPageUnavailableError(sessionName) + } + session.proxy.elementCapture.start() + session.proxy.pointAtNextNode(true) + let captured: CapturedElement | null + try { + // Any element read makes the helper resolve the ref or selector, in whichever frame owns it. + await this.execAgentBrowser(sessionName, ['get', 'attr', element, 'id']) + } finally { + session.proxy.pointAtNextNode(false) + captured = session.proxy.elementCapture.stop() + } + const wc = this.getWebContents(session.webContentsId) + if (!wc) { + throw this.createPageUnavailableError(sessionName) + } + if (!captured) { + throw new BrowserError('browser_error', `Could not resolve ${element} in the page`) + } + return { wc, captured } + } + + protected async callElementFunction( + sessionName: string, + { wc, captured }: CapturedTarget, + element: string, + action: string, + functionDeclaration: string, + { args = [], userGesture = false }: { args?: unknown[]; userGesture?: boolean } = {} + ): Promise { + const response = await this.sendElementCdpCommand( + sessionName, + wc, + 'Runtime.callFunctionOn', + { + objectId: captured.objectId, + functionDeclaration, + arguments: args.map((value) => ({ value })), + returnByValue: true, + ...(userGesture ? { userGesture } : {}) + }, + captured.sessionId + ) + const exception = readRecord(response, 'exceptionDetails') + if (exception) { + const description = readString(readRecord(exception, 'exception'), 'description') + throw new BrowserError( + 'browser_eval_error', + description ?? readString(exception, 'text') ?? 'Page function failed' + ) + } + const outcome = readRecord(response, 'result')?.value + if (typeof outcome === 'string') { + throw new BrowserError( + 'browser_element_not_interactable', + `Cannot ${action} ${element}: ${outcome}` + ) + } + return outcome + } + + protected sendElementCdpCommand( + sessionName: string, + wc: WebContents, + method: string, + params: Record, + sessionId: string | undefined + ): Promise { + return sendGuestCdpCommand(wc, method, params, sessionId).catch((error: unknown) => { + if (error instanceof BrowserError) { + throw error + } + const session = this.sessions.get(sessionName) + if (!session || !this.getWebContents(session.webContentsId)) { + throw this.createPageUnavailableError(sessionName) + } + throw new BrowserError( + 'browser_cdp_error', + error instanceof Error ? error.message : String(error) + ) + }) + } +} diff --git a/src/main/browser/agent-browser-bridge-execution.ts b/src/main/browser/agent-browser-bridge-execution.ts index 8ef0aa15d6c..c80660131ac 100644 --- a/src/main/browser/agent-browser-bridge-execution.ts +++ b/src/main/browser/agent-browser-bridge-execution.ts @@ -1,9 +1,6 @@ import { execFile } from 'node:child_process' import type { WebContents } from 'electron' import { BrowserError } from './cdp-bridge' -import type { CapturedElement } from './cdp-element-capture' -import { sendGuestCdpCommand } from './guest-cdp-command' -import { readRecord, readString } from './guest-frame-input-sessions' import { isTabClosedTransportError, pageUnavailableMessageForSession @@ -167,75 +164,6 @@ export abstract class AgentBrowserBridgeExecution extends AgentBrowserBridgeTabs return this.runAgentBrowserRaw(sessionName, args, execOptions) } - /** Runs a browser-element-functions action on the element the helper resolves, in its own frame. */ - protected async actOnElement( - sessionName: string, - element: string, - action: string, - functionDeclaration: string, - args: unknown[] = [] - ): Promise { - const session = this.sessions.get(sessionName) - if (!session) { - throw this.createPageUnavailableError(sessionName) - } - session.proxy.elementCapture.start() - session.proxy.pointAtNextNode(true) - let captured: CapturedElement | null - try { - // Any element read makes the helper resolve the ref or selector, in whichever frame owns it. - await this.execAgentBrowser(sessionName, ['get', 'attr', element, 'id']) - } finally { - session.proxy.pointAtNextNode(false) - captured = session.proxy.elementCapture.stop() - } - const wc = this.getWebContents(session.webContentsId) - if (!wc) { - throw this.createPageUnavailableError(sessionName) - } - if (!captured) { - throw new BrowserError('browser_error', `Could not resolve ${element} in the page`) - } - const response = await sendGuestCdpCommand( - wc, - 'Runtime.callFunctionOn', - { - objectId: captured.objectId, - functionDeclaration, - arguments: args.map((value) => ({ value })), - returnByValue: true - }, - captured.sessionId - ).catch((error: unknown) => { - if (error instanceof BrowserError) { - throw error - } - if (!this.getWebContents(session.webContentsId)) { - throw this.createPageUnavailableError(sessionName) - } - throw new BrowserError( - 'browser_cdp_error', - error instanceof Error ? error.message : String(error) - ) - }) - const exception = readRecord(response, 'exceptionDetails') - if (exception) { - const description = readString(readRecord(exception, 'exception'), 'description') - throw new BrowserError( - 'browser_eval_error', - description ?? readString(exception, 'text') ?? 'Page function failed' - ) - } - const outcome = readRecord(response, 'result')?.value - if (typeof outcome === 'string') { - throw new BrowserError( - 'browser_element_not_interactable', - `Cannot ${action} ${element}: ${outcome}` - ) - } - return outcome - } - protected createPageUnavailableError(sessionName: string): BrowserError { return new BrowserError('browser_tab_not_found', pageUnavailableMessageForSession(sessionName)) } diff --git a/src/main/browser/agent-browser-bridge-interaction-commands.ts b/src/main/browser/agent-browser-bridge-interaction-commands.ts index dd089b0fa93..dedfc6b752a 100644 --- a/src/main/browser/agent-browser-bridge-interaction-commands.ts +++ b/src/main/browser/agent-browser-bridge-interaction-commands.ts @@ -11,11 +11,16 @@ import type { } from '../../shared/runtime-types' import { BrowserError } from './cdp-bridge' import { - FILE_INPUT_TARGET_FUNCTION, + FILE_INPUT_FRAME_URL_FUNCTION, + FILE_UPLOAD_TARGET_FUNCTION, FILL_ELEMENT_FUNCTION, SELECT_ALL_ELEMENT_FUNCTION } from './browser-element-functions' import { assertUploadFilesExist } from './browser-upload-files' +import type { WebContents } from 'electron' +import type { CapturedElement } from './cdp-element-capture' +import { interceptFileChooser, type OpenedFileChooser } from './guest-file-chooser' +import { readRecord, readString } from './guest-frame-input-sessions' import { agentBrowserWaitArguments, type AgentBrowserWaitOptions @@ -67,17 +72,79 @@ export abstract class AgentBrowserBridgeInteractionCommands extends AgentBrowser browserPageId, async (sessionName) => { await assertUploadFilesExist(filePaths) - await this.actOnElement(sessionName, element, 'upload to', FILE_INPUT_TARGET_FUNCTION) - return (await this.execAgentBrowser(sessionName, [ - 'upload', - element, - ...filePaths - ])) as BrowserUploadResult + const target = await this.captureElement(sessionName, element) + const { wc, captured } = target + const chooser = await interceptFileChooser(wc, captured.sessionId) + try { + const isFileInput = await this.callElementFunction( + sessionName, + target, + element, + 'upload to', + FILE_UPLOAD_TARGET_FUNCTION, + { userGesture: true } + ) + const input = + isFileInput === true + ? captured + : await this.openedFileInput(sessionName, wc, element, await chooser.next()) + const frame = await this.sendElementCdpCommand( + sessionName, + wc, + 'Runtime.callFunctionOn', + { + objectId: input.objectId, + functionDeclaration: FILE_INPUT_FRAME_URL_FUNCTION, + returnByValue: true + }, + input.sessionId + ) + const frameUrl = readRecord(frame, 'result')?.value + await this.sendElementCdpCommand( + sessionName, + wc, + 'DOM.setFileInputFiles', + { files: filePaths, objectId: input.objectId }, + input.sessionId + ) + return { + uploaded: filePaths.length, + ...(typeof frameUrl === 'string' ? { frameUrl } : {}) + } + } finally { + await chooser.release() + } }, { elements: [element] } ) } + private async openedFileInput( + sessionName: string, + wc: WebContents, + element: string, + opened: OpenedFileChooser | null + ): Promise { + if (!opened?.backendNodeId) { + throw new BrowserError( + 'browser_element_not_interactable', + `Cannot upload to ${element}: ${opened ? 'its file picker has no file input' : 'clicking it opened no file picker'}` + ) + } + const resolved = await this.sendElementCdpCommand( + sessionName, + wc, + 'DOM.resolveNode', + { backendNodeId: opened.backendNodeId }, + opened.sessionId + ) + const objectId = readString(readRecord(resolved, 'object'), 'objectId') + if (!objectId) { + throw new BrowserError('browser_error', `Could not resolve the file input ${element} opened`) + } + return { objectId, sessionId: opened.sessionId } + } + async wait( options?: AgentBrowserWaitOptions, worktreeId?: string, diff --git a/src/main/browser/agent-browser-bridge-raw-process.ts b/src/main/browser/agent-browser-bridge-raw-process.ts index 43773e5af66..68ba8908f16 100644 --- a/src/main/browser/agent-browser-bridge-raw-process.ts +++ b/src/main/browser/agent-browser-bridge-raw-process.ts @@ -2,7 +2,7 @@ import { execFile, type ChildProcess } from 'node:child_process' import { BrowserError } from './cdp-bridge' import { classifyErrorCode } from './agent-browser-bridge-process' import { parseAgentBrowserJsonOutput } from './agent-browser-batch-command' -import { AgentBrowserBridgeExecution } from './agent-browser-bridge-execution' +import { AgentBrowserBridgeElementCalls } from './agent-browser-bridge-element-calls' import { AGENT_BROWSER_EXIT_DRAIN_MS, CONSECUTIVE_TIMEOUT_LIMIT, @@ -10,7 +10,7 @@ import { type AgentBrowserExecOptions } from './agent-browser-bridge-types' -export abstract class AgentBrowserBridgeRawProcess extends AgentBrowserBridgeExecution { +export abstract class AgentBrowserBridgeRawProcess extends AgentBrowserBridgeElementCalls { protected abstract destroySession( sessionName: string, options?: { closeTimeoutMs?: number } diff --git a/src/main/browser/agent-browser-bridge-test-harness.ts b/src/main/browser/agent-browser-bridge-test-harness.ts index f637c891674..255027577de 100644 --- a/src/main/browser/agent-browser-bridge-test-harness.ts +++ b/src/main/browser/agent-browser-bridge-test-harness.ts @@ -50,7 +50,7 @@ export function mockBrowserManager( } as unknown as BrowserManager } -type MockEmitterListener = (...args: never[]) => void +type MockEmitterListener = (...args: any[]) => void export type MockWebContentsDebugger = { isAttached: Mock<() => boolean> diff --git a/src/main/browser/agent-browser-bridge-text-input.test.ts b/src/main/browser/agent-browser-bridge-text-input.test.ts index ad7c5b1e087..aee00c09e6b 100644 --- a/src/main/browser/agent-browser-bridge-text-input.test.ts +++ b/src/main/browser/agent-browser-bridge-text-input.test.ts @@ -1,3 +1,4 @@ +import { EventEmitter } from 'node:events' import { describe, it, expect, vi, beforeEach } from 'vitest' const { execFileMock, webContentsFromIdMock, existsSyncMock, readFileSyncMock, stdinWrites } = @@ -78,7 +79,11 @@ import { resetAgentBrowserBridgeMocks, type MockWebContents } from './agent-browser-bridge-test-harness' -import { FILL_ELEMENT_FUNCTION } from './browser-element-functions' +import { + FILE_INPUT_FRAME_URL_FUNCTION, + FILE_UPLOAD_TARGET_FUNCTION, + FILL_ELEMENT_FUNCTION +} from './browser-element-functions' overrideBridgeWebContentsLookup(AgentBrowserBridge.prototype, webContentsFromIdMock) @@ -189,7 +194,59 @@ describe('AgentBrowserBridge', () => { await expect(bridge.upload('@e19', ['/nonexistent/orca-upload.txt'])).rejects.toMatchObject({ code: 'invalid_argument' }) - expect(agentBrowserCommandsIn(stdinWrites).map(([name]) => name)).not.toContain('upload') + expect(agentBrowserCommandsIn(stdinWrites)).toEqual([]) + }) + + it.each([ + ['its cross-site frame', 'frame-session'], + ['the page its cross-site frame messages', undefined] + ])('sets the files on the input a picker button opens in %s', async (_case, chooserSession) => { + const file = import.meta.filename + const debuggerEvents = new EventEmitter() + wc.debugger.on.mockImplementation((event, listener) => void debuggerEvents.on(event, listener)) + wc.debugger.removeListener.mockImplementation( + (event, listener) => void debuggerEvents.removeListener(event, listener) + ) + wc.debugger.sendCommand.mockImplementation(async (method, params) => { + const declaration = + typeof params === 'object' && params !== null && 'functionDeclaration' in params + ? params.functionDeclaration + : null + if (declaration === FILE_UPLOAD_TARGET_FUNCTION) { + // The page reports the picker its click opened on the frame's own session. + const session = chooserSession ?? '' + debuggerEvents.emit('message', {}, 'Page.fileChooserOpened', { backendNodeId: 7 }, session) + return { result: { value: false } } + } + if (method === 'DOM.resolveNode') { + return { object: { objectId: 'input-7' } } + } + if (declaration === FILE_INPUT_FRAME_URL_FUNCTION) { + return { result: { value: 'https://widgets.example/picker' } } + } + return {} + }) + succeedWith({ value: null }) + + await expect(bridge.upload('@e6', [file])).resolves.toEqual({ + uploaded: 1, + frameUrl: 'https://widgets.example/picker' + }) + + const sent = wc.debugger.sendCommand.mock.calls + expect(sent).toContainEqual([ + 'DOM.setFileInputFiles', + { files: [file], objectId: 'input-7' }, + // The page session is Electron's default, so its commands carry no session id. + ...(chooserSession ? [chooserSession] : []) + ]) + expect(sent).toContainEqual(['Page.setInterceptFileChooserDialog', { enabled: false }]) + expect(sent).toContainEqual([ + 'Page.setInterceptFileChooserDialog', + { enabled: false }, + 'frame-session' + ]) + expect(debuggerEvents.listenerCount('message')).toBe(0) }) it.each([ diff --git a/src/main/browser/browser-element-functions.test.ts b/src/main/browser/browser-element-functions.test.ts index 9c7cc504f75..2acceab4806 100644 --- a/src/main/browser/browser-element-functions.test.ts +++ b/src/main/browser/browser-element-functions.test.ts @@ -1,7 +1,7 @@ // @vitest-environment happy-dom import { describe, it, expect, vi, beforeEach } from 'vitest' import { - FILE_INPUT_TARGET_FUNCTION, + FILE_UPLOAD_TARGET_FUNCTION, FILL_ELEMENT_FUNCTION, SELECT_ALL_ELEMENT_FUNCTION, SELECT_OPTION_TARGET_FUNCTION @@ -131,17 +131,43 @@ describe('element target checks', () => { expect(callOn(element, SELECT_OPTION_TARGET_FUNCTION)).toMatch(reason) }) - it('accepts select on an enabled ') + it('accepts select on an enabled ').querySelector('select')! - expect(callOn(body.querySelector('select')!, SELECT_OPTION_TARGET_FUNCTION)).toBeNull() - expect(callOn(body.querySelector('input')!, FILE_INPUT_TARGET_FUNCTION)).toBeNull() + expect(callOn(select, SELECT_OPTION_TARGET_FUNCTION)).toBeNull() }) - it('refuses upload on an element that is not a file input', () => { - const button = mount('').querySelector('button')! + it('takes a file input for upload as is and clicks any other element to open its picker', () => { + const body = mount('') + const input = body.querySelector('input')! + const button = body.querySelector('button')! + const clicks: string[] = [] + input.addEventListener('click', () => clicks.push('input')) + button.addEventListener('click', () => clicks.push('button')) - expect(callOn(button, FILE_INPUT_TARGET_FUNCTION)).toBe('it is not a file input') + expect(callOn(input, FILE_UPLOAD_TARGET_FUNCTION)).toBe(true) + expect(callOn(button, FILE_UPLOAD_TARGET_FUNCTION)).toBe(false) + expect(clicks).toEqual(['button']) + }) + + it('clicks an SVG picker control, which has no click() of its own', () => { + const svg = mount('').querySelector('svg')! + let clicked = false + document.body.addEventListener('click', () => (clicked = true)) + + expect(callOn(svg, FILE_UPLOAD_TARGET_FUNCTION)).toBe(false) + expect(clicked).toBe(true) + }) + + it('refuses upload on a disabled control without clicking it', () => { + const body = mount('') + const button = body.querySelector('button')! + let clicked = false + button.addEventListener('click', () => (clicked = true)) + + expect(callOn(body.querySelector('input')!, FILE_UPLOAD_TARGET_FUNCTION)).toBe('it is disabled') + expect(callOn(button, FILE_UPLOAD_TARGET_FUNCTION)).toBe('it is disabled') + expect(clicked).toBe(false) }) it('selects all of an input value and reports that it holds focus', () => { diff --git a/src/main/browser/browser-element-functions.ts b/src/main/browser/browser-element-functions.ts index 48efd8467b4..0d6db9715f8 100644 --- a/src/main/browser/browser-element-functions.ts +++ b/src/main/browser/browser-element-functions.ts @@ -75,7 +75,16 @@ export const SELECT_OPTION_TARGET_FUNCTION = `function () { return this.matches(':disabled') ? 'it is disabled' : null; }` -export const FILE_INPUT_TARGET_FUNCTION = `function () { - if (this.localName !== 'input' || this.type !== 'file') return 'it is not a file input'; - return this.matches(':disabled') ? 'it is disabled' : null; +// Returns true for a file input; anything else is clicked so the picker it opens can be caught. +export const FILE_UPLOAD_TARGET_FUNCTION = `function () { + if (this.matches(':disabled')) return 'it is disabled'; + if (this.localName === 'input' && this.type === 'file') return true; + // SVG controls have no click(); a dispatched click still reaches their handlers. + if (typeof this.click === 'function') this.click(); + else this.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, composed: true })); + return false; +}` + +export const FILE_INPUT_FRAME_URL_FUNCTION = `function () { + return this.ownerDocument.URL; }` diff --git a/src/main/browser/guest-file-chooser.ts b/src/main/browser/guest-file-chooser.ts new file mode 100644 index 00000000000..85c6f5fc4e9 --- /dev/null +++ b/src/main/browser/guest-file-chooser.ts @@ -0,0 +1,74 @@ +import type { WebContents } from 'electron' +import { withTimeout } from '../../shared/promise-timeout-fallback' +import { sendGuestCdpCommand } from './guest-cdp-command' + +// A click's user activation lasts 5s in Chromium, so its picker cannot open any later. +const FILE_CHOOSER_OPEN_TIMEOUT_MS = 5_000 + +export type OpenedFileChooser = { backendNodeId: number | null; sessionId: string | undefined } + +export type FileChooserInterception = { + /** The next chooser the page opened, or null when none opened in time. */ + next(): Promise + release(): Promise +} + +/** + * Catches file choosers opened from a frame's CDP session, or from the page's (a click activates + * ancestor frames too), instead of showing the OS dialog. + */ +export async function interceptFileChooser( + wc: WebContents, + frameSessionId: string | undefined +): Promise { + const sessionIds = [...new Set([undefined, frameSessionId])] + const { promise: opened, resolve: resolveOpened } = Promise.withResolvers() + const onMessage = ( + _event: unknown, + method: string, + params: { backendNodeId?: number } | undefined, + eventSessionId?: string + ): void => { + const sessionId = eventSessionId || undefined + if (method === 'Page.fileChooserOpened' && sessionIds.includes(sessionId)) { + resolveOpened({ backendNodeId: params?.backendNodeId ?? null, sessionId }) + } + } + wc.debugger.on('message', onMessage) + const release = async (): Promise => { + wc.debugger.removeListener('message', onMessage) + await Promise.all( + sessionIds.map((sessionId) => + sendGuestCdpCommand( + wc, + 'Page.setInterceptFileChooserDialog', + { enabled: false }, + sessionId + ).catch(() => {}) + ) + ) + } + try { + await Promise.all( + sessionIds.map(async (sessionId) => { + // Cross-site frame sessions report choosers only with Page on; the page session has it. + if (sessionId !== undefined) { + await sendGuestCdpCommand(wc, 'Page.enable', {}, sessionId) + } + await sendGuestCdpCommand( + wc, + 'Page.setInterceptFileChooserDialog', + { enabled: true }, + sessionId + ) + }) + ) + } catch (error) { + await release() + throw error + } + return { + next: () => withTimeout(opened, FILE_CHOOSER_OPEN_TIMEOUT_MS, null), + release + } +} diff --git a/src/shared/runtime-browser-contracts.ts b/src/shared/runtime-browser-contracts.ts index 1de512ef75e..3c83e511951 100644 --- a/src/shared/runtime-browser-contracts.ts +++ b/src/shared/runtime-browser-contracts.ts @@ -156,7 +156,8 @@ export type BrowserProfileImportFromBrowserResult = BrowserCookieImportResult & export type BrowserProfileClearDefaultCookiesResult = { cleared: boolean } export type BrowserHoverResult = { hovered: string } export type BrowserDragResult = { dragged: { from: string; to: string } } -export type BrowserUploadResult = { uploaded: number } +// Older hosts and server-hosted pages report only the count. +export type BrowserUploadResult = { uploaded: number; frameUrl?: string } export type BrowserWaitResult = { waited: boolean } export type BrowserCheckResult = { checked: boolean } export type BrowserFocusResult = { focused: string }