mirror of
https://github.com/stablyai/orca.git
synced 2026-11-13 17:37:40 +00:00
feat(browser): upload through file-picker buttons in any frame, including cross-origin iframes (#36390)
* feat(browser): upload through picker buttons in any frame * fix(browser): catch pickers the page opens for a frame button and click SVG controls * docs(cli): note server-hosted pages need the file input ref for upload * test(browser): assert both picker interceptions are released * refactor(browser): flatten picker upload and arm frame sessions in parallel * test(browser): emit picker events through a real emitter instead of Reflect.apply
This commit is contained in:
@@ -75,6 +75,7 @@ Browser rules:
|
||||
- Use `ORCA set device` or `ORCA viewport --mobile`, not `ORCA exec --command "set device ..."`, so the page also gets touch input and `pointer: coarse`. Each page load (reload, `goto`, link click) resets it, so run it again after navigating.
|
||||
- Prefer `wait --text`, `--url`, `--selector`, or `--load` after async page changes instead of bare timeouts. Avoid `--load networkidle` except on pages known to go quiet: polling, websockets and streaming keep it from ever resolving.
|
||||
- Anything not listed above goes through `ORCA exec --page <browserPageId> --command "<agent-browser command>"`, which takes agent-browser's own command syntax (for example `"get text @e3"` or `"is visible @e3"`) and prints JSON. It has no `help` command.
|
||||
- `upload` takes the file input itself or the button that opens a file picker, in any frame, including cross-origin iframes; Orca catches the picker and sets the files on its input. Use `ORCA upload`, not `ORCA exec --command "upload ..."`. On server-hosted pages pass the file input's ref.
|
||||
- `fill` sets a field's whole value in one step, at any size. `type` presses real keys into the focused element, so `minlength`/`maxlength`, key handlers, Enter and Tab act as they would for a person; `ORCA focus` the element first. Use `type` when the page reacts to keystrokes.
|
||||
- On server-hosted pages `fill` and `type` still insert text the engine's own way. If either does nothing on a custom input there, try `ORCA focus --page <browserPageId> --element @e1 --json` then `ORCA inserttext --page <browserPageId> --text "text" --json`.
|
||||
- A client-hosted page renders in the paired desktop's browser engine, so every command against it needs that desktop online and returns `browser_host_unavailable` while it is closed, asleep, or disconnected. Server-hosted pages run with no desktop attached; prefer them for long or unattended automation.
|
||||
@@ -83,7 +84,7 @@ Common recoveries:
|
||||
|
||||
- `browser_no_tab`: open a tab with `ORCA tab create --url <url> --json`.
|
||||
- `browser_stale_ref`: run `ORCA snapshot --page <browserPageId> --json` and retry on the same page with fresh refs.
|
||||
- `browser_element_not_interactable`: the element can't take that command, and the message says why (not a text field, disabled, read-only, not a `<select>`, not a file input). Re-snapshot and pick the right ref; for a custom dropdown, click it and then click the option.
|
||||
- `browser_element_not_interactable`: the element can't take that command, and the message says why (not a text field, disabled, read-only, not a `<select>`, opened no file picker). Re-snapshot and pick the right ref; for a custom dropdown, click it and then click the option.
|
||||
- `browser_tab_not_found`: run `ORCA tab list --json` before switching or closing.
|
||||
- `browser_dialog_open` / `browser_dialog_opened`: the page has a JavaScript alert, confirm or prompt open, and the error quotes it. Answer it with `ORCA dialog accept` (`--text` for a prompt; without it a prompt takes its default) or `ORCA dialog dismiss` before any other command on that page; a command that opened it may already have taken effect, so check before repeating it.
|
||||
- `browser_host_unavailable`: the desktop hosting the page is offline. Bring it back, or recreate the page with server placement if the work must outlive the desktop session.
|
||||
|
||||
@@ -894,6 +894,7 @@ const ORCA_CLI_BROWSER_REFERENCE_MARKDOWN = [
|
||||
"- Use `ORCA set device` or `ORCA viewport --mobile`, not `ORCA exec --command \"set device ...\"`, so the page also gets touch input and `pointer: coarse`. Each page load (reload, `goto`, link click) resets it, so run it again after navigating.",
|
||||
"- Prefer `wait --text`, `--url`, `--selector`, or `--load` after async page changes instead of bare timeouts. Avoid `--load networkidle` except on pages known to go quiet: polling, websockets and streaming keep it from ever resolving.",
|
||||
"- Anything not listed above goes through `ORCA exec --page <browserPageId> --command \"<agent-browser command>\"`, which takes agent-browser's own command syntax (for example `\"get text @e3\"` or `\"is visible @e3\"`) and prints JSON. It has no `help` command.",
|
||||
"- `upload` takes the file input itself or the button that opens a file picker, in any frame, including cross-origin iframes; Orca catches the picker and sets the files on its input. Use `ORCA upload`, not `ORCA exec --command \"upload ...\"`. On server-hosted pages pass the file input's ref.",
|
||||
"- `fill` sets a field's whole value in one step, at any size. `type` presses real keys into the focused element, so `minlength`/`maxlength`, key handlers, Enter and Tab act as they would for a person; `ORCA focus` the element first. Use `type` when the page reacts to keystrokes.",
|
||||
"- On server-hosted pages `fill` and `type` still insert text the engine's own way. If either does nothing on a custom input there, try `ORCA focus --page <browserPageId> --element @e1 --json` then `ORCA inserttext --page <browserPageId> --text \"text\" --json`.",
|
||||
"- A client-hosted page renders in the paired desktop's browser engine, so every command against it needs that desktop online and returns `browser_host_unavailable` while it is closed, asleep, or disconnected. Server-hosted pages run with no desktop attached; prefer them for long or unattended automation.",
|
||||
@@ -902,7 +903,7 @@ const ORCA_CLI_BROWSER_REFERENCE_MARKDOWN = [
|
||||
"",
|
||||
"- `browser_no_tab`: open a tab with `ORCA tab create --url <url> --json`.",
|
||||
"- `browser_stale_ref`: run `ORCA snapshot --page <browserPageId> --json` and retry on the same page with fresh refs.",
|
||||
"- `browser_element_not_interactable`: the element can't take that command, and the message says why (not a text field, disabled, read-only, not a `<select>`, not a file input). Re-snapshot and pick the right ref; for a custom dropdown, click it and then click the option.",
|
||||
"- `browser_element_not_interactable`: the element can't take that command, and the message says why (not a text field, disabled, read-only, not a `<select>`, opened no file picker). Re-snapshot and pick the right ref; for a custom dropdown, click it and then click the option.",
|
||||
"- `browser_tab_not_found`: run `ORCA tab list --json` before switching or closing.",
|
||||
"- `browser_dialog_open` / `browser_dialog_opened`: the page has a JavaScript alert, confirm or prompt open, and the error quotes it. Answer it with `ORCA dialog accept` (`--text` for a prompt; without it a prompt takes its default) or `ORCA dialog dismiss` before any other command on that page; a command that opened it may already have taken effect, so check before repeating it.",
|
||||
"- `browser_host_unavailable`: the desktop hosting the page is offline. Bring it back, or recreate the page with server placement if the work must outlive the desktop session.",
|
||||
|
||||
@@ -133,7 +133,12 @@ export const BROWSER_INTERACT_HANDLERS: Record<string, CommandHandler> = {
|
||||
files,
|
||||
...target
|
||||
})
|
||||
printResult(result, json, (v) => `Uploaded ${v.uploaded} file(s)`)
|
||||
printResult(
|
||||
result,
|
||||
json,
|
||||
(v) =>
|
||||
`Uploaded ${v.uploaded} file(s)${v.frameUrl ? ` to a file input in ${v.frameUrl}` : ''}`
|
||||
)
|
||||
},
|
||||
scrollintoview: async ({ flags, client, cwd, json }) => {
|
||||
const element = getRequiredStringFlag(flags, 'element')
|
||||
|
||||
@@ -3,7 +3,7 @@ export const ROOT_HELP_TEXT_SECONDARY = [
|
||||
' focus Focus an element by --element ref',
|
||||
' clear Clear an input by --element ref',
|
||||
' drag Drag --from ref to --to ref',
|
||||
' upload Upload --files to a file input by --element ref',
|
||||
' upload Upload --files to a file input or picker button by --element ref',
|
||||
' dblclick Double-click element by --element ref',
|
||||
' forward Navigate forward in browser history',
|
||||
' scrollintoview Scroll --element into view',
|
||||
|
||||
@@ -190,7 +190,7 @@ export const BROWSER_BASIC_COMMAND_SPECS: CommandSpec[] = withBrowserPageTarget(
|
||||
},
|
||||
{
|
||||
path: ['upload'],
|
||||
summary: 'Upload files to a file input element',
|
||||
summary: 'Upload files to a file input, or to the one a picker button opens',
|
||||
usage: 'orca upload --element <ref> --files <path,...> [--worktree <selector>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'element', 'files', 'worktree']
|
||||
},
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import type { WebContents } from 'electron'
|
||||
import { BrowserError } from './cdp-bridge'
|
||||
import type { CapturedElement } from './cdp-element-capture'
|
||||
import { sendGuestCdpCommand } from './guest-cdp-command'
|
||||
import { readRecord, readString } from './guest-frame-input-sessions'
|
||||
import { AgentBrowserBridgeExecution } from './agent-browser-bridge-execution'
|
||||
|
||||
export type CapturedTarget = { wc: WebContents; captured: CapturedElement }
|
||||
|
||||
export abstract class AgentBrowserBridgeElementCalls extends AgentBrowserBridgeExecution {
|
||||
/** Runs a browser-element-functions action on the element the helper resolves, in its own frame. */
|
||||
protected async actOnElement(
|
||||
sessionName: string,
|
||||
element: string,
|
||||
action: string,
|
||||
functionDeclaration: string,
|
||||
args: unknown[] = []
|
||||
): Promise<unknown> {
|
||||
const target = await this.captureElement(sessionName, element)
|
||||
return await this.callElementFunction(
|
||||
sessionName,
|
||||
target,
|
||||
element,
|
||||
action,
|
||||
functionDeclaration,
|
||||
{
|
||||
args
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
/** Resolves a ref or selector to the element the helper found, with the CDP session of its frame. */
|
||||
protected async captureElement(sessionName: string, element: string): Promise<CapturedTarget> {
|
||||
const session = this.sessions.get(sessionName)
|
||||
if (!session) {
|
||||
throw this.createPageUnavailableError(sessionName)
|
||||
}
|
||||
session.proxy.elementCapture.start()
|
||||
session.proxy.pointAtNextNode(true)
|
||||
let captured: CapturedElement | null
|
||||
try {
|
||||
// Any element read makes the helper resolve the ref or selector, in whichever frame owns it.
|
||||
await this.execAgentBrowser(sessionName, ['get', 'attr', element, 'id'])
|
||||
} finally {
|
||||
session.proxy.pointAtNextNode(false)
|
||||
captured = session.proxy.elementCapture.stop()
|
||||
}
|
||||
const wc = this.getWebContents(session.webContentsId)
|
||||
if (!wc) {
|
||||
throw this.createPageUnavailableError(sessionName)
|
||||
}
|
||||
if (!captured) {
|
||||
throw new BrowserError('browser_error', `Could not resolve ${element} in the page`)
|
||||
}
|
||||
return { wc, captured }
|
||||
}
|
||||
|
||||
protected async callElementFunction(
|
||||
sessionName: string,
|
||||
{ wc, captured }: CapturedTarget,
|
||||
element: string,
|
||||
action: string,
|
||||
functionDeclaration: string,
|
||||
{ args = [], userGesture = false }: { args?: unknown[]; userGesture?: boolean } = {}
|
||||
): Promise<unknown> {
|
||||
const response = await this.sendElementCdpCommand(
|
||||
sessionName,
|
||||
wc,
|
||||
'Runtime.callFunctionOn',
|
||||
{
|
||||
objectId: captured.objectId,
|
||||
functionDeclaration,
|
||||
arguments: args.map((value) => ({ value })),
|
||||
returnByValue: true,
|
||||
...(userGesture ? { userGesture } : {})
|
||||
},
|
||||
captured.sessionId
|
||||
)
|
||||
const exception = readRecord(response, 'exceptionDetails')
|
||||
if (exception) {
|
||||
const description = readString(readRecord(exception, 'exception'), 'description')
|
||||
throw new BrowserError(
|
||||
'browser_eval_error',
|
||||
description ?? readString(exception, 'text') ?? 'Page function failed'
|
||||
)
|
||||
}
|
||||
const outcome = readRecord(response, 'result')?.value
|
||||
if (typeof outcome === 'string') {
|
||||
throw new BrowserError(
|
||||
'browser_element_not_interactable',
|
||||
`Cannot ${action} ${element}: ${outcome}`
|
||||
)
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
|
||||
protected sendElementCdpCommand(
|
||||
sessionName: string,
|
||||
wc: WebContents,
|
||||
method: string,
|
||||
params: Record<string, unknown>,
|
||||
sessionId: string | undefined
|
||||
): Promise<unknown> {
|
||||
return sendGuestCdpCommand(wc, method, params, sessionId).catch((error: unknown) => {
|
||||
if (error instanceof BrowserError) {
|
||||
throw error
|
||||
}
|
||||
const session = this.sessions.get(sessionName)
|
||||
if (!session || !this.getWebContents(session.webContentsId)) {
|
||||
throw this.createPageUnavailableError(sessionName)
|
||||
}
|
||||
throw new BrowserError(
|
||||
'browser_cdp_error',
|
||||
error instanceof Error ? error.message : String(error)
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,6 @@
|
||||
import { execFile } from 'node:child_process'
|
||||
import type { WebContents } from 'electron'
|
||||
import { BrowserError } from './cdp-bridge'
|
||||
import type { CapturedElement } from './cdp-element-capture'
|
||||
import { sendGuestCdpCommand } from './guest-cdp-command'
|
||||
import { readRecord, readString } from './guest-frame-input-sessions'
|
||||
import {
|
||||
isTabClosedTransportError,
|
||||
pageUnavailableMessageForSession
|
||||
@@ -167,75 +164,6 @@ export abstract class AgentBrowserBridgeExecution extends AgentBrowserBridgeTabs
|
||||
return this.runAgentBrowserRaw(sessionName, args, execOptions)
|
||||
}
|
||||
|
||||
/** Runs a browser-element-functions action on the element the helper resolves, in its own frame. */
|
||||
protected async actOnElement(
|
||||
sessionName: string,
|
||||
element: string,
|
||||
action: string,
|
||||
functionDeclaration: string,
|
||||
args: unknown[] = []
|
||||
): Promise<unknown> {
|
||||
const session = this.sessions.get(sessionName)
|
||||
if (!session) {
|
||||
throw this.createPageUnavailableError(sessionName)
|
||||
}
|
||||
session.proxy.elementCapture.start()
|
||||
session.proxy.pointAtNextNode(true)
|
||||
let captured: CapturedElement | null
|
||||
try {
|
||||
// Any element read makes the helper resolve the ref or selector, in whichever frame owns it.
|
||||
await this.execAgentBrowser(sessionName, ['get', 'attr', element, 'id'])
|
||||
} finally {
|
||||
session.proxy.pointAtNextNode(false)
|
||||
captured = session.proxy.elementCapture.stop()
|
||||
}
|
||||
const wc = this.getWebContents(session.webContentsId)
|
||||
if (!wc) {
|
||||
throw this.createPageUnavailableError(sessionName)
|
||||
}
|
||||
if (!captured) {
|
||||
throw new BrowserError('browser_error', `Could not resolve ${element} in the page`)
|
||||
}
|
||||
const response = await sendGuestCdpCommand(
|
||||
wc,
|
||||
'Runtime.callFunctionOn',
|
||||
{
|
||||
objectId: captured.objectId,
|
||||
functionDeclaration,
|
||||
arguments: args.map((value) => ({ value })),
|
||||
returnByValue: true
|
||||
},
|
||||
captured.sessionId
|
||||
).catch((error: unknown) => {
|
||||
if (error instanceof BrowserError) {
|
||||
throw error
|
||||
}
|
||||
if (!this.getWebContents(session.webContentsId)) {
|
||||
throw this.createPageUnavailableError(sessionName)
|
||||
}
|
||||
throw new BrowserError(
|
||||
'browser_cdp_error',
|
||||
error instanceof Error ? error.message : String(error)
|
||||
)
|
||||
})
|
||||
const exception = readRecord(response, 'exceptionDetails')
|
||||
if (exception) {
|
||||
const description = readString(readRecord(exception, 'exception'), 'description')
|
||||
throw new BrowserError(
|
||||
'browser_eval_error',
|
||||
description ?? readString(exception, 'text') ?? 'Page function failed'
|
||||
)
|
||||
}
|
||||
const outcome = readRecord(response, 'result')?.value
|
||||
if (typeof outcome === 'string') {
|
||||
throw new BrowserError(
|
||||
'browser_element_not_interactable',
|
||||
`Cannot ${action} ${element}: ${outcome}`
|
||||
)
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
|
||||
protected createPageUnavailableError(sessionName: string): BrowserError {
|
||||
return new BrowserError('browser_tab_not_found', pageUnavailableMessageForSession(sessionName))
|
||||
}
|
||||
|
||||
@@ -11,11 +11,16 @@ import type {
|
||||
} from '../../shared/runtime-types'
|
||||
import { BrowserError } from './cdp-bridge'
|
||||
import {
|
||||
FILE_INPUT_TARGET_FUNCTION,
|
||||
FILE_INPUT_FRAME_URL_FUNCTION,
|
||||
FILE_UPLOAD_TARGET_FUNCTION,
|
||||
FILL_ELEMENT_FUNCTION,
|
||||
SELECT_ALL_ELEMENT_FUNCTION
|
||||
} from './browser-element-functions'
|
||||
import { assertUploadFilesExist } from './browser-upload-files'
|
||||
import type { WebContents } from 'electron'
|
||||
import type { CapturedElement } from './cdp-element-capture'
|
||||
import { interceptFileChooser, type OpenedFileChooser } from './guest-file-chooser'
|
||||
import { readRecord, readString } from './guest-frame-input-sessions'
|
||||
import {
|
||||
agentBrowserWaitArguments,
|
||||
type AgentBrowserWaitOptions
|
||||
@@ -67,17 +72,79 @@ export abstract class AgentBrowserBridgeInteractionCommands extends AgentBrowser
|
||||
browserPageId,
|
||||
async (sessionName) => {
|
||||
await assertUploadFilesExist(filePaths)
|
||||
await this.actOnElement(sessionName, element, 'upload to', FILE_INPUT_TARGET_FUNCTION)
|
||||
return (await this.execAgentBrowser(sessionName, [
|
||||
'upload',
|
||||
element,
|
||||
...filePaths
|
||||
])) as BrowserUploadResult
|
||||
const target = await this.captureElement(sessionName, element)
|
||||
const { wc, captured } = target
|
||||
const chooser = await interceptFileChooser(wc, captured.sessionId)
|
||||
try {
|
||||
const isFileInput = await this.callElementFunction(
|
||||
sessionName,
|
||||
target,
|
||||
element,
|
||||
'upload to',
|
||||
FILE_UPLOAD_TARGET_FUNCTION,
|
||||
{ userGesture: true }
|
||||
)
|
||||
const input =
|
||||
isFileInput === true
|
||||
? captured
|
||||
: await this.openedFileInput(sessionName, wc, element, await chooser.next())
|
||||
const frame = await this.sendElementCdpCommand(
|
||||
sessionName,
|
||||
wc,
|
||||
'Runtime.callFunctionOn',
|
||||
{
|
||||
objectId: input.objectId,
|
||||
functionDeclaration: FILE_INPUT_FRAME_URL_FUNCTION,
|
||||
returnByValue: true
|
||||
},
|
||||
input.sessionId
|
||||
)
|
||||
const frameUrl = readRecord(frame, 'result')?.value
|
||||
await this.sendElementCdpCommand(
|
||||
sessionName,
|
||||
wc,
|
||||
'DOM.setFileInputFiles',
|
||||
{ files: filePaths, objectId: input.objectId },
|
||||
input.sessionId
|
||||
)
|
||||
return {
|
||||
uploaded: filePaths.length,
|
||||
...(typeof frameUrl === 'string' ? { frameUrl } : {})
|
||||
}
|
||||
} finally {
|
||||
await chooser.release()
|
||||
}
|
||||
},
|
||||
{ elements: [element] }
|
||||
)
|
||||
}
|
||||
|
||||
private async openedFileInput(
|
||||
sessionName: string,
|
||||
wc: WebContents,
|
||||
element: string,
|
||||
opened: OpenedFileChooser | null
|
||||
): Promise<CapturedElement> {
|
||||
if (!opened?.backendNodeId) {
|
||||
throw new BrowserError(
|
||||
'browser_element_not_interactable',
|
||||
`Cannot upload to ${element}: ${opened ? 'its file picker has no file input' : 'clicking it opened no file picker'}`
|
||||
)
|
||||
}
|
||||
const resolved = await this.sendElementCdpCommand(
|
||||
sessionName,
|
||||
wc,
|
||||
'DOM.resolveNode',
|
||||
{ backendNodeId: opened.backendNodeId },
|
||||
opened.sessionId
|
||||
)
|
||||
const objectId = readString(readRecord(resolved, 'object'), 'objectId')
|
||||
if (!objectId) {
|
||||
throw new BrowserError('browser_error', `Could not resolve the file input ${element} opened`)
|
||||
}
|
||||
return { objectId, sessionId: opened.sessionId }
|
||||
}
|
||||
|
||||
async wait(
|
||||
options?: AgentBrowserWaitOptions,
|
||||
worktreeId?: string,
|
||||
|
||||
@@ -2,7 +2,7 @@ import { execFile, type ChildProcess } from 'node:child_process'
|
||||
import { BrowserError } from './cdp-bridge'
|
||||
import { classifyErrorCode } from './agent-browser-bridge-process'
|
||||
import { parseAgentBrowserJsonOutput } from './agent-browser-batch-command'
|
||||
import { AgentBrowserBridgeExecution } from './agent-browser-bridge-execution'
|
||||
import { AgentBrowserBridgeElementCalls } from './agent-browser-bridge-element-calls'
|
||||
import {
|
||||
AGENT_BROWSER_EXIT_DRAIN_MS,
|
||||
CONSECUTIVE_TIMEOUT_LIMIT,
|
||||
@@ -10,7 +10,7 @@ import {
|
||||
type AgentBrowserExecOptions
|
||||
} from './agent-browser-bridge-types'
|
||||
|
||||
export abstract class AgentBrowserBridgeRawProcess extends AgentBrowserBridgeExecution {
|
||||
export abstract class AgentBrowserBridgeRawProcess extends AgentBrowserBridgeElementCalls {
|
||||
protected abstract destroySession(
|
||||
sessionName: string,
|
||||
options?: { closeTimeoutMs?: number }
|
||||
|
||||
@@ -50,7 +50,7 @@ export function mockBrowserManager(
|
||||
} as unknown as BrowserManager
|
||||
}
|
||||
|
||||
type MockEmitterListener = (...args: never[]) => void
|
||||
type MockEmitterListener = (...args: any[]) => void
|
||||
|
||||
export type MockWebContentsDebugger = {
|
||||
isAttached: Mock<() => boolean>
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
|
||||
const { execFileMock, webContentsFromIdMock, existsSyncMock, readFileSyncMock, stdinWrites } =
|
||||
@@ -78,7 +79,11 @@ import {
|
||||
resetAgentBrowserBridgeMocks,
|
||||
type MockWebContents
|
||||
} from './agent-browser-bridge-test-harness'
|
||||
import { FILL_ELEMENT_FUNCTION } from './browser-element-functions'
|
||||
import {
|
||||
FILE_INPUT_FRAME_URL_FUNCTION,
|
||||
FILE_UPLOAD_TARGET_FUNCTION,
|
||||
FILL_ELEMENT_FUNCTION
|
||||
} from './browser-element-functions'
|
||||
|
||||
overrideBridgeWebContentsLookup(AgentBrowserBridge.prototype, webContentsFromIdMock)
|
||||
|
||||
@@ -189,7 +194,59 @@ describe('AgentBrowserBridge', () => {
|
||||
await expect(bridge.upload('@e19', ['/nonexistent/orca-upload.txt'])).rejects.toMatchObject({
|
||||
code: 'invalid_argument'
|
||||
})
|
||||
expect(agentBrowserCommandsIn(stdinWrites).map(([name]) => name)).not.toContain('upload')
|
||||
expect(agentBrowserCommandsIn(stdinWrites)).toEqual([])
|
||||
})
|
||||
|
||||
it.each([
|
||||
['its cross-site frame', 'frame-session'],
|
||||
['the page its cross-site frame messages', undefined]
|
||||
])('sets the files on the input a picker button opens in %s', async (_case, chooserSession) => {
|
||||
const file = import.meta.filename
|
||||
const debuggerEvents = new EventEmitter()
|
||||
wc.debugger.on.mockImplementation((event, listener) => void debuggerEvents.on(event, listener))
|
||||
wc.debugger.removeListener.mockImplementation(
|
||||
(event, listener) => void debuggerEvents.removeListener(event, listener)
|
||||
)
|
||||
wc.debugger.sendCommand.mockImplementation(async (method, params) => {
|
||||
const declaration =
|
||||
typeof params === 'object' && params !== null && 'functionDeclaration' in params
|
||||
? params.functionDeclaration
|
||||
: null
|
||||
if (declaration === FILE_UPLOAD_TARGET_FUNCTION) {
|
||||
// The page reports the picker its click opened on the frame's own session.
|
||||
const session = chooserSession ?? ''
|
||||
debuggerEvents.emit('message', {}, 'Page.fileChooserOpened', { backendNodeId: 7 }, session)
|
||||
return { result: { value: false } }
|
||||
}
|
||||
if (method === 'DOM.resolveNode') {
|
||||
return { object: { objectId: 'input-7' } }
|
||||
}
|
||||
if (declaration === FILE_INPUT_FRAME_URL_FUNCTION) {
|
||||
return { result: { value: 'https://widgets.example/picker' } }
|
||||
}
|
||||
return {}
|
||||
})
|
||||
succeedWith({ value: null })
|
||||
|
||||
await expect(bridge.upload('@e6', [file])).resolves.toEqual({
|
||||
uploaded: 1,
|
||||
frameUrl: 'https://widgets.example/picker'
|
||||
})
|
||||
|
||||
const sent = wc.debugger.sendCommand.mock.calls
|
||||
expect(sent).toContainEqual([
|
||||
'DOM.setFileInputFiles',
|
||||
{ files: [file], objectId: 'input-7' },
|
||||
// The page session is Electron's default, so its commands carry no session id.
|
||||
...(chooserSession ? [chooserSession] : [])
|
||||
])
|
||||
expect(sent).toContainEqual(['Page.setInterceptFileChooserDialog', { enabled: false }])
|
||||
expect(sent).toContainEqual([
|
||||
'Page.setInterceptFileChooserDialog',
|
||||
{ enabled: false },
|
||||
'frame-session'
|
||||
])
|
||||
expect(debuggerEvents.listenerCount('message')).toBe(0)
|
||||
})
|
||||
|
||||
it.each([
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// @vitest-environment happy-dom
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import {
|
||||
FILE_INPUT_TARGET_FUNCTION,
|
||||
FILE_UPLOAD_TARGET_FUNCTION,
|
||||
FILL_ELEMENT_FUNCTION,
|
||||
SELECT_ALL_ELEMENT_FUNCTION,
|
||||
SELECT_OPTION_TARGET_FUNCTION
|
||||
@@ -131,17 +131,43 @@ describe('element target checks', () => {
|
||||
expect(callOn(element, SELECT_OPTION_TARGET_FUNCTION)).toMatch(reason)
|
||||
})
|
||||
|
||||
it('accepts select on an enabled <select> and upload on a file input', () => {
|
||||
const body = mount('<select></select><input type="file">')
|
||||
it('accepts select on an enabled <select>', () => {
|
||||
const select = mount('<select></select>').querySelector('select')!
|
||||
|
||||
expect(callOn(body.querySelector('select')!, SELECT_OPTION_TARGET_FUNCTION)).toBeNull()
|
||||
expect(callOn(body.querySelector('input')!, FILE_INPUT_TARGET_FUNCTION)).toBeNull()
|
||||
expect(callOn(select, SELECT_OPTION_TARGET_FUNCTION)).toBeNull()
|
||||
})
|
||||
|
||||
it('refuses upload on an element that is not a file input', () => {
|
||||
const button = mount('<button>Attach</button>').querySelector('button')!
|
||||
it('takes a file input for upload as is and clicks any other element to open its picker', () => {
|
||||
const body = mount('<input type="file"><button>Attach</button>')
|
||||
const input = body.querySelector('input')!
|
||||
const button = body.querySelector('button')!
|
||||
const clicks: string[] = []
|
||||
input.addEventListener('click', () => clicks.push('input'))
|
||||
button.addEventListener('click', () => clicks.push('button'))
|
||||
|
||||
expect(callOn(button, FILE_INPUT_TARGET_FUNCTION)).toBe('it is not a file input')
|
||||
expect(callOn(input, FILE_UPLOAD_TARGET_FUNCTION)).toBe(true)
|
||||
expect(callOn(button, FILE_UPLOAD_TARGET_FUNCTION)).toBe(false)
|
||||
expect(clicks).toEqual(['button'])
|
||||
})
|
||||
|
||||
it('clicks an SVG picker control, which has no click() of its own', () => {
|
||||
const svg = mount('<svg role="button"></svg>').querySelector('svg')!
|
||||
let clicked = false
|
||||
document.body.addEventListener('click', () => (clicked = true))
|
||||
|
||||
expect(callOn(svg, FILE_UPLOAD_TARGET_FUNCTION)).toBe(false)
|
||||
expect(clicked).toBe(true)
|
||||
})
|
||||
|
||||
it('refuses upload on a disabled control without clicking it', () => {
|
||||
const body = mount('<input type="file" disabled><button disabled>Attach</button>')
|
||||
const button = body.querySelector('button')!
|
||||
let clicked = false
|
||||
button.addEventListener('click', () => (clicked = true))
|
||||
|
||||
expect(callOn(body.querySelector('input')!, FILE_UPLOAD_TARGET_FUNCTION)).toBe('it is disabled')
|
||||
expect(callOn(button, FILE_UPLOAD_TARGET_FUNCTION)).toBe('it is disabled')
|
||||
expect(clicked).toBe(false)
|
||||
})
|
||||
|
||||
it('selects all of an input value and reports that it holds focus', () => {
|
||||
|
||||
@@ -75,7 +75,16 @@ export const SELECT_OPTION_TARGET_FUNCTION = `function () {
|
||||
return this.matches(':disabled') ? 'it is disabled' : null;
|
||||
}`
|
||||
|
||||
export const FILE_INPUT_TARGET_FUNCTION = `function () {
|
||||
if (this.localName !== 'input' || this.type !== 'file') return 'it is not a file input';
|
||||
return this.matches(':disabled') ? 'it is disabled' : null;
|
||||
// Returns true for a file input; anything else is clicked so the picker it opens can be caught.
|
||||
export const FILE_UPLOAD_TARGET_FUNCTION = `function () {
|
||||
if (this.matches(':disabled')) return 'it is disabled';
|
||||
if (this.localName === 'input' && this.type === 'file') return true;
|
||||
// SVG controls have no click(); a dispatched click still reaches their handlers.
|
||||
if (typeof this.click === 'function') this.click();
|
||||
else this.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, composed: true }));
|
||||
return false;
|
||||
}`
|
||||
|
||||
export const FILE_INPUT_FRAME_URL_FUNCTION = `function () {
|
||||
return this.ownerDocument.URL;
|
||||
}`
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import type { WebContents } from 'electron'
|
||||
import { withTimeout } from '../../shared/promise-timeout-fallback'
|
||||
import { sendGuestCdpCommand } from './guest-cdp-command'
|
||||
|
||||
// A click's user activation lasts 5s in Chromium, so its picker cannot open any later.
|
||||
const FILE_CHOOSER_OPEN_TIMEOUT_MS = 5_000
|
||||
|
||||
export type OpenedFileChooser = { backendNodeId: number | null; sessionId: string | undefined }
|
||||
|
||||
export type FileChooserInterception = {
|
||||
/** The next chooser the page opened, or null when none opened in time. */
|
||||
next(): Promise<OpenedFileChooser | null>
|
||||
release(): Promise<void>
|
||||
}
|
||||
|
||||
/**
|
||||
* Catches file choosers opened from a frame's CDP session, or from the page's (a click activates
|
||||
* ancestor frames too), instead of showing the OS dialog.
|
||||
*/
|
||||
export async function interceptFileChooser(
|
||||
wc: WebContents,
|
||||
frameSessionId: string | undefined
|
||||
): Promise<FileChooserInterception> {
|
||||
const sessionIds = [...new Set([undefined, frameSessionId])]
|
||||
const { promise: opened, resolve: resolveOpened } = Promise.withResolvers<OpenedFileChooser>()
|
||||
const onMessage = (
|
||||
_event: unknown,
|
||||
method: string,
|
||||
params: { backendNodeId?: number } | undefined,
|
||||
eventSessionId?: string
|
||||
): void => {
|
||||
const sessionId = eventSessionId || undefined
|
||||
if (method === 'Page.fileChooserOpened' && sessionIds.includes(sessionId)) {
|
||||
resolveOpened({ backendNodeId: params?.backendNodeId ?? null, sessionId })
|
||||
}
|
||||
}
|
||||
wc.debugger.on('message', onMessage)
|
||||
const release = async (): Promise<void> => {
|
||||
wc.debugger.removeListener('message', onMessage)
|
||||
await Promise.all(
|
||||
sessionIds.map((sessionId) =>
|
||||
sendGuestCdpCommand(
|
||||
wc,
|
||||
'Page.setInterceptFileChooserDialog',
|
||||
{ enabled: false },
|
||||
sessionId
|
||||
).catch(() => {})
|
||||
)
|
||||
)
|
||||
}
|
||||
try {
|
||||
await Promise.all(
|
||||
sessionIds.map(async (sessionId) => {
|
||||
// Cross-site frame sessions report choosers only with Page on; the page session has it.
|
||||
if (sessionId !== undefined) {
|
||||
await sendGuestCdpCommand(wc, 'Page.enable', {}, sessionId)
|
||||
}
|
||||
await sendGuestCdpCommand(
|
||||
wc,
|
||||
'Page.setInterceptFileChooserDialog',
|
||||
{ enabled: true },
|
||||
sessionId
|
||||
)
|
||||
})
|
||||
)
|
||||
} catch (error) {
|
||||
await release()
|
||||
throw error
|
||||
}
|
||||
return {
|
||||
next: () => withTimeout<OpenedFileChooser | null>(opened, FILE_CHOOSER_OPEN_TIMEOUT_MS, null),
|
||||
release
|
||||
}
|
||||
}
|
||||
@@ -156,7 +156,8 @@ export type BrowserProfileImportFromBrowserResult = BrowserCookieImportResult &
|
||||
export type BrowserProfileClearDefaultCookiesResult = { cleared: boolean }
|
||||
export type BrowserHoverResult = { hovered: string }
|
||||
export type BrowserDragResult = { dragged: { from: string; to: string } }
|
||||
export type BrowserUploadResult = { uploaded: number }
|
||||
// Older hosts and server-hosted pages report only the count.
|
||||
export type BrowserUploadResult = { uploaded: number; frameUrl?: string }
|
||||
export type BrowserWaitResult = { waited: boolean }
|
||||
export type BrowserCheckResult = { checked: boolean }
|
||||
export type BrowserFocusResult = { focused: string }
|
||||
|
||||
Reference in New Issue
Block a user