Files
orca/src/main/runtime/runtime-browser-commands-browser-click.ts
T
Jinwoo Hong 08c8d30f10 fix(e2ee): stop dropping terminal output on a backed-up legacy connection (#26528)
* fix(browser): retry a screencast frame a viewer's socket refused

An idle page sends no next frame, so a refused last frame left the viewer on a stale picture. The
desktop's relay leg to a server uses legacy E2EE, which drops on a full socket where a direct v2 phone
queues; retrying the newest pending frame restores parity. Retries never count toward ghost eviction.

Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb

* fix(e2ee): queue legacy binary replies with text instead of dropping them

Legacy E2EE (every direct phone, the desktop's server link) sent binary straight to the socket and
silently dropped it once the outbound budget was full, so terminals lost output and binary could
overtake queued text. Binary now shares the ordered text queue and closes on overflow like v2; the
lossy screencast opts into dropWhenBacklogged and its retry delivers the newest frame after a drain.

Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb

* fix(e2ee): keep a backlogged screencast viewer instead of evicting it as a ghost

A lossy frame skipped on a busy socket returned false like a dead socket, so a long backlog evicted a
live viewer. Skips now return 'backlogged', which retries without counting toward eviction; legacy
judges idleness on the sealed size before encrypting, so skipped frames cost no encryption.

Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb

* test(browser): expect the ready gate to forward screencast send options

The pre-ready gate now passes the per-frame send options through to the
socket sender, so the replayed startup frame arrives with an options slot.

Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb

* test(browser): share the streamed-guest double across screencast tests

Main's renderer throttle lease now calls setBackgroundThrottling on the guest,
so the bare { isDestroyed } guest in two branch tests threw. One shared double
keeps all three screencast suites on the shape the lease needs.

Claude-Session: https://claude.ai/code/session_01JNnE9qzUZMMnqpZWCqM3nb
2026-10-11 02:54:27 -04:00

177 lines
6.3 KiB
TypeScript

// @ts-nocheck -- mechanically split class members.
import { RuntimeBrowserCommandsWithActiveScreencastsByPageId } from './runtime-browser-commands-active-screencasts-by-page-id'
import type {
ActiveBrowserScreencastPage,
BrowserCommandTargetParams
} from './runtime-browser-commands-browser-command-target-params'
import {
applySharedScreencastFrameBudget,
hasScreencastViewportSize
} from './runtime-browser-commands-browser-command-target-params'
import type {
BrowserBackResult,
BrowserClickResult,
BrowserFillResult,
BrowserGotoResult,
BrowserReloadResult,
BrowserScreenshotResult,
BrowserScrollResult,
BrowserSelectResult,
BrowserTypeResult
} from '../../shared/runtime-types'
import type { BrowserScreencastSession } from '../browser/browser-screencast-stream-types'
import { cancelScreencastSubscriberFrameRetry } from './browser-screencast-subscriber-frame-delivery'
export class RuntimeBrowserCommandsWithBrowserClick extends RuntimeBrowserCommandsWithActiveScreencastsByPageId {
async browserClick(
params: { element: string } & BrowserCommandTargetParams
): Promise<BrowserClickResult> {
const target = await this.resolveBrowserCommandTarget(params)
const bridge = this.requireAgentBrowserBridge()
const result = await bridge.click(params.element, target.worktreeId, target.browserPageId)
// Why: clicks can trigger navigation, so push the tab's live URL/title to the renderer even when automation targeted a non-active page.
const page = bridge.getPageInfo(target.worktreeId, target.browserPageId)
if (page) {
this.notifyRendererNavigation(page.browserPageId, page.url, page.title)
}
return result
}
async browserGoto(
params: { url: string } & BrowserCommandTargetParams
): Promise<BrowserGotoResult> {
const target = await this.resolveBrowserCommandTarget(params)
const bridge = this.requireAgentBrowserBridge()
const result = await bridge.goto(params.url, target.worktreeId, target.browserPageId)
const pageId = bridge.getActivePageId(target.worktreeId, target.browserPageId)
if (pageId) {
this.notifyRendererNavigation(pageId, result.url, result.title)
}
if (!this.host.getAvailableAuthoritativeWindow() && target.worktreeId) {
this.host.notifyHeadlessBrowserSessionTabsChanged?.(target.worktreeId)
}
return result
}
async browserFill(
params: {
element: string
value: string
} & BrowserCommandTargetParams
): Promise<BrowserFillResult> {
const target = await this.resolveBrowserCommandTarget(params)
return this.requireAgentBrowserBridge().fill(
params.element,
params.value,
target.worktreeId,
target.browserPageId
)
}
async browserType(
params: { input: string } & BrowserCommandTargetParams
): Promise<BrowserTypeResult> {
const target = await this.resolveBrowserCommandTarget(params)
return this.requireAgentBrowserBridge().type(
params.input,
target.worktreeId,
target.browserPageId
)
}
async browserSelect(
params: {
element: string
value: string
} & BrowserCommandTargetParams
): Promise<BrowserSelectResult> {
const target = await this.resolveBrowserCommandTarget(params)
return this.requireAgentBrowserBridge().select(
params.element,
params.value,
target.worktreeId,
target.browserPageId
)
}
async browserScroll(
params: { direction: 'up' | 'down'; amount?: number } & BrowserCommandTargetParams
): Promise<BrowserScrollResult> {
const target = await this.resolveBrowserCommandTarget(params)
return this.requireAgentBrowserBridge().scroll(
params.direction,
params.amount,
target.worktreeId,
target.browserPageId
)
}
async browserBack(params: BrowserCommandTargetParams): Promise<BrowserBackResult> {
const target = await this.resolveBrowserCommandTarget(params)
const bridge = this.requireAgentBrowserBridge()
const result = await bridge.back(target.worktreeId, target.browserPageId)
const pageId = bridge.getActivePageId(target.worktreeId, target.browserPageId)
if (pageId) {
this.notifyRendererNavigation(pageId, result.url, result.title)
}
return result
}
async browserReload(params: BrowserCommandTargetParams): Promise<BrowserReloadResult> {
const target = await this.resolveBrowserCommandTarget(params)
const bridge = this.requireAgentBrowserBridge()
const result = await bridge.reload(target.worktreeId, target.browserPageId)
const pageId = bridge.getActivePageId(target.worktreeId, target.browserPageId)
if (pageId) {
this.notifyRendererNavigation(pageId, result.url, result.title)
}
return result
}
async browserScreenshot(
params: {
format?: 'png' | 'jpeg'
} & BrowserCommandTargetParams
): Promise<BrowserScreenshotResult> {
const target = await this.resolveBrowserCommandTarget(params)
return this.requireAgentBrowserBridge().screenshot(
params.format,
target.worktreeId,
target.browserPageId
)
}
// The single leave path: an explicit stop, a ghost eviction and a same-device replacement all
// unwind through here, so viewport hand-off, budget release and stream teardown cannot drift.
private leaveScreencastSubscriber(
active: ActiveBrowserScreencastPage,
subscriptionId: string,
session: BrowserScreencastSession
): void {
const subscriber = active.subscribers.get(subscriptionId)
if (!subscriber) {
return
}
active.subscribers.delete(subscriptionId)
cancelScreencastSubscriberFrameRetry(subscriber)
subscriber.resolveDone()
if (active.viewportOwnerSubscriptionId === subscriptionId) {
const fallback = Array.from(active.subscribers.entries()).findLast(([, candidate]) =>
hasScreencastViewportSize(candidate.viewport)
)
active.viewportOwnerSubscriptionId = fallback?.[0] ?? null
if (fallback) {
void session.updateViewport(fallback[1].viewport).catch(() => {})
}
}
if (active.subscribers.size === 0) {
active.stopping = true
session.stop()
return
}
// Why: a departed subscriber's caps would otherwise pin the shared stream for
// the rest of its life, long after the client that asked for them is gone.
void applySharedScreencastFrameBudget(active, session).catch(() => {})
}
}