Files
archived-orca/src/shared/runtime-browser-contracts.ts
T
Kelvin Amoaba dcaeaaa1fc feat(browser): typed snapshot flags, screenshots saved as files, and a guide that keeps browser work in Orca (#36257)
* feat(browser): typed snapshot flags and screenshots saved as files for agents

orca snapshot takes --interactive, --compact, --depth, --selector, --urls,
--delta and --full, passed to agent-browser on every route. The header
now shows the page title and URL. orca screenshot and full-screenshot save
the image where the CLI runs and print its path, in CSS pixels, with
--if-changed and --annotate. Flag help for exec, wait and download is
written for those commands, and the orca-cli guide sends web pages to
Orca's browser unless the user picked another tool.

* fix(browser): scale screenshots by the scrollbar-free axis and keep snapshot metadata behind SSH admission

* fix(browser): upscale low-DPR screenshots and wrap a torn-down tab's screenshot error

* fix(browser): give each screenshot its own file and keep typed screenshot errors

* refactor(browser): share the agent-browser screenshot reader between Electron and orcad

* refactor(browser): move snapshot and screenshot params under the line cap
2026-10-30 15:07:05 +00:00

250 lines
7.9 KiB
TypeScript

import type {
BrowserCertificateFailure,
BrowserCookieImportResult,
BrowserLoadError,
BrowserSessionProfile,
BrowserSessionProfileSource
} from './browser-workspace-types'
import type { BROWSER_UNAVAILABLE_ERROR_CODE } from './runtime-session-contracts'
export type BrowserSnapshotRef = { ref: string; role: string; name: string }
// agent-browser's `snapshot --delta` answer; `full` carries the tree, the others change the last one.
export type BrowserSnapshotDelta =
| { kind: 'full'; revision: number; tree: string }
| { kind: 'unchanged'; revision: number; baseRevision: number }
| {
kind: 'delta'
revision: number
baseRevision: number
treeChange?: { startLine: number; deleteCount: number; lines: string[] }
}
export type BrowserSnapshotResult = {
browserPageId: string
snapshot: string | BrowserSnapshotDelta
refs: BrowserSnapshotRef[]
url: string
title: string
}
/** What an agent's page action caused; absent for people's input and from runtimes that predate it. */
export type BrowserActionEffects = {
/** Set only when the action changed them. */
url?: string
title?: string
/** A navigation was still loading at the settle cap, so no page changes are listed. */
loading?: boolean
loadError?: string
/** No browserPageId: the tab was requested but had not opened by the settle cap. */
newTabs?: { browserPageId?: string; url: string }[]
popups?: { action: 'blocked' | 'opened-external' | 'window'; origin: string }[]
downloads?: { filename: string; origin: string }[]
consoleErrors?: string[]
/** Snapshot lines changed since the last reading; `full` is the whole new tree. Absent when unread. */
page?: 'unchanged' | { lines: string[]; removedLines: number; full: boolean; truncated: boolean }
/** False when the page was still changing at the settle cap. */
settled: boolean
}
export type BrowserClickResult = { clicked: string }
export type BrowserGotoResult = { url: string; title: string }
export type BrowserFillResult = { filled: string }
export type BrowserTypeResult = { typed: boolean }
export type BrowserSelectResult = { selected: string }
export type BrowserScrollResult = { scrolled: 'up' | 'down' }
export type BrowserBackResult = { url: string; title: string }
export type BrowserReloadResult = { url: string; title: string }
export type BrowserScreenshotAnnotation = {
ref: string
number: number
role: string
name?: string
box: { x: number; y: number; width: number; height: number }
}
export type BrowserScreenshotResult = {
data: string
format: 'png' | 'jpeg'
// Set only for --if-changed; false means nothing was captured and data is empty.
changed?: boolean
annotations?: BrowserScreenshotAnnotation[]
}
export type BrowserScreencastReadyResult = {
type: 'ready'
subscriptionId: string
browserPageId: string
format: 'jpeg' | 'png'
tab: BrowserTabInfo
}
export type BrowserScreencastEndResult = { type: 'end'; subscriptionId: string }
export type BrowserScreencastDialogResult = { type: 'dialog'; dialogType: string; message: string }
export type BrowserScreencastDialogClosedResult = { type: 'dialogClosed' }
export type BrowserScreencastErrorResult = { type: 'error'; message: string }
export type BrowserScreencastResult =
| BrowserScreencastReadyResult
| BrowserScreencastEndResult
| BrowserScreencastDialogResult
| BrowserScreencastDialogClosedResult
| BrowserScreencastErrorResult
export type BrowserEvalResult = { result: string; origin: string }
export type BrowserTabInfo = {
browserPageId: string
index: number
url: string
title: string
active: boolean
loadError?: BrowserLoadError | null
certificateFailure?: BrowserCertificateFailure | null
worktreeId?: string | null
profileId?: string | null
profileLabel?: string | null
}
export type BrowserTabListResult = { tabs: BrowserTabInfo[] }
export type BrowserTabSwitchResult = { switched: number; browserPageId: string }
export type BrowserTabSetProfileResult = {
browserPageId: string
profileId: string | null
profileLabel: string | null
}
export type BrowserTabShowResult = { tab: BrowserTabInfo }
export type BrowserTabCurrentResult = { tab: BrowserTabInfo }
export type BrowserTabProfileShowResult = {
browserPageId: string
worktreeId: string | null
profileId: string | null
profileLabel: string | null
}
export type BrowserTabProfileCloneResult = {
browserPageId: string
sourceBrowserPageId: string
profileId: string | null
profileLabel: string | null
}
export type BrowserProfileListResult = {
profiles: BrowserSessionProfile[]
executorRuntimeId?: string
}
export type BrowserProfileCreateResult = { profile: BrowserSessionProfile | null }
export type BrowserProfileDeleteResult = { deleted: boolean; profileId: string }
export type BrowserDetectedProfileInfo = { name: string; directory: string }
export type BrowserDetectedInfo = {
family: BrowserSessionProfileSource['browserFamily']
label: string
profiles: BrowserDetectedProfileInfo[]
selectedProfile: string
}
export type BrowserDetectProfilesResult = { browsers: BrowserDetectedInfo[] }
export type BrowserProfileImportFromBrowserResult = BrowserCookieImportResult & {
executorRuntimeId?: string
}
export type BrowserProfileClearDefaultCookiesResult = { cleared: boolean }
export type BrowserHoverResult = { hovered: string }
export type BrowserDragResult = { dragged: { from: string; to: string } }
export type BrowserUploadResult = { uploaded: number }
export type BrowserWaitResult = { waited: boolean }
export type BrowserCheckResult = { checked: boolean }
export type BrowserFocusResult = { focused: string }
export type BrowserClearResult = { cleared: string }
export type BrowserSelectAllResult = { selected: string }
export type BrowserKeypressResult = { pressed: string }
export type BrowserPdfResult = { data: string }
export type BrowserCookie = {
name: string
value: string
domain: string
path: string
expires: number
httpOnly: boolean
secure: boolean
sameSite: string
}
export type BrowserCookieGetResult = { cookies: BrowserCookie[] }
export type BrowserCookieSetResult = { success: boolean }
export type BrowserCookieDeleteResult = { deleted: boolean }
export type BrowserViewportResult = {
width: number
height: number
deviceScaleFactor: number
mobile: boolean
}
export type BrowserGeolocationResult = {
latitude: number
longitude: number
accuracy: number
}
export type BrowserInterceptedRequest = {
id: string
url: string
method: string
headers: Record<string, string>
resourceType: string
}
export type BrowserInterceptEnableResult = { enabled: boolean; patterns: string[] }
export type BrowserInterceptDisableResult = { disabled: boolean }
export type BrowserConsoleEntry = {
level: string
text: string
timestamp?: number
url?: string
line?: number
}
export type BrowserConsoleResult = { entries: BrowserConsoleEntry[]; truncated: boolean }
export type BrowserNetworkEntry = {
url: string
method: string
// Absent while a request is pending or after it failed.
status?: number
mimeType?: string
size?: number
timestamp?: number
}
export type BrowserNetworkLogResult = { entries: BrowserNetworkEntry[]; truncated: boolean }
export type BrowserCaptureStartResult = { capturing: boolean }
export type BrowserCaptureStopResult = { stopped: boolean }
export type BrowserTabCreateResult = { browserPageId: string }
export type BrowserErrorCode =
| typeof BROWSER_UNAVAILABLE_ERROR_CODE
| 'browser_command_unavailable'
| 'browser_profile_unavailable'
| 'browser_screencast_unavailable'
| 'browser_certificate_trust_unavailable'
| 'browser_no_tab'
| 'browser_tab_not_found'
| 'browser_tab_closed'
| 'browser_tab_changed'
| 'browser_owner_unavailable'
| 'browser_stale_ref'
| 'browser_ref_not_found'
| 'browser_navigation_failed'
| 'browser_element_not_interactable'
| 'browser_eval_error'
| 'browser_cdp_error'
| 'browser_debugger_detached'
| 'browser_timeout'
| 'browser_error'