feat: computer use via agent-browser CDP bridge (#856)

This commit is contained in:
Jinwoo Hong
2026-04-20 20:56:14 -07:00
committed by GitHub
parent 22ae50e0ae
commit 7a9bc4ef6d
35 changed files with 13560 additions and 43 deletions
@@ -999,6 +999,7 @@ function BrowserPagePane({
void window.api.browser.registerGuest({
browserPageId: browserTab.id,
workspaceId,
worktreeId,
webContentsId
})
}
+515 -3
View File
@@ -152,6 +152,10 @@ describe('useIpcEvents updater integration', () => {
onJumpToWorktreeIndex: () => () => {},
onActivateWorktree: () => () => {},
onNewBrowserTab: () => () => {},
onRequestTabCreate: () => () => {},
replyTabCreate: () => {},
onRequestTabClose: () => () => {},
replyTabClose: () => {},
onNewTerminalTab: () => () => {},
onCloseActiveTab: () => () => {},
onSwitchTab: () => () => {},
@@ -171,7 +175,9 @@ describe('useIpcEvents updater integration', () => {
},
browser: {
onGuestLoadFailed: () => () => {},
onOpenLinkInOrcaTab: () => () => {}
onOpenLinkInOrcaTab: () => () => {},
onNavigationUpdate: () => () => {},
onActivateView: () => () => {}
},
rateLimits: {
get: () => Promise.resolve({ limits: {}, lastUpdatedAt: Date.now() }),
@@ -314,6 +320,10 @@ describe('useIpcEvents updater integration', () => {
onJumpToWorktreeIndex: () => () => {},
onActivateWorktree: () => () => {},
onNewBrowserTab: () => () => {},
onRequestTabCreate: () => () => {},
replyTabCreate: () => {},
onRequestTabClose: () => () => {},
replyTabClose: () => {},
onNewTerminalTab: () => () => {},
onCloseActiveTab: () => () => {},
onSwitchTab: () => () => {},
@@ -330,7 +340,9 @@ describe('useIpcEvents updater integration', () => {
},
browser: {
onGuestLoadFailed: () => () => {},
onOpenLinkInOrcaTab: () => () => {}
onOpenLinkInOrcaTab: () => () => {},
onNavigationUpdate: () => () => {},
onActivateView: () => () => {}
},
rateLimits: {
get: () => Promise.resolve({ limits: {}, lastUpdatedAt: Date.now() }),
@@ -372,6 +384,500 @@ describe('useIpcEvents updater integration', () => {
})
})
describe('useIpcEvents browser tab close routing', () => {
beforeEach(() => {
vi.resetModules()
vi.unstubAllGlobals()
})
it('closes the active browser tab for the requested worktree when main does not provide a tab id', async () => {
const closeBrowserTab = vi.fn()
const closeBrowserPage = vi.fn()
const replyTabClose = vi.fn()
const tabCloseListenerRef: {
current:
| ((data: { requestId: string; tabId: string | null; worktreeId?: string }) => void)
| null
} = {
current: null
}
vi.doMock('react', async () => {
const actual = await vi.importActual<typeof ReactModule>('react')
return {
...actual,
useEffect: (effect: () => void | (() => void)) => {
effect()
}
}
})
vi.doMock('../store', () => ({
useAppStore: {
getState: () => ({
setUpdateStatus: vi.fn(),
fetchRepos: vi.fn(),
fetchWorktrees: vi.fn(),
setActiveView: vi.fn(),
activeModal: null,
closeModal: vi.fn(),
openModal: vi.fn(),
activeWorktreeId: 'wt-1',
activeView: 'terminal',
setActiveRepo: vi.fn(),
setActiveWorktree: vi.fn(),
revealWorktreeInSidebar: vi.fn(),
setIsFullScreen: vi.fn(),
updateBrowserTabPageState: vi.fn(),
activeTabType: 'browser',
editorFontZoomLevel: 0,
setEditorFontZoomLevel: vi.fn(),
setRateLimitsFromPush: vi.fn(),
setSshConnectionState: vi.fn(),
setSshTargetLabels: vi.fn(),
enqueueSshCredentialRequest: vi.fn(),
removeSshCredentialRequest: vi.fn(),
settings: { terminalFontSize: 13 },
activeBrowserTabId: 'workspace-global',
activeBrowserTabIdByWorktree: {
'wt-1': 'workspace-global',
'wt-2': 'workspace-target'
},
browserTabsByWorktree: {
'wt-1': [{ id: 'workspace-global' }],
'wt-2': [{ id: 'workspace-target' }]
},
browserPagesByWorkspace: {},
closeBrowserTab,
closeBrowserPage
})
}
}))
vi.doMock('@/lib/ui-zoom', () => ({
applyUIZoom: vi.fn()
}))
vi.doMock('@/lib/worktree-activation', () => ({
activateAndRevealWorktree: vi.fn(),
ensureWorktreeHasInitialTerminal: vi.fn()
}))
vi.doMock('@/components/sidebar/visible-worktrees', () => ({
getVisibleWorktreeIds: () => []
}))
vi.doMock('@/lib/editor-font-zoom', () => ({
nextEditorFontZoomLevel: vi.fn(() => 0),
computeEditorFontSize: vi.fn(() => 13)
}))
vi.doMock('@/components/settings/SettingsConstants', () => ({
zoomLevelToPercent: vi.fn(() => 100),
ZOOM_MIN: -3,
ZOOM_MAX: 3
}))
vi.doMock('@/lib/zoom-events', () => ({
dispatchZoomLevelChanged: vi.fn()
}))
vi.stubGlobal('window', {
dispatchEvent: vi.fn(),
api: {
repos: { onChanged: () => () => {} },
worktrees: { onChanged: () => () => {} },
ui: {
onOpenSettings: () => () => {},
onToggleLeftSidebar: () => () => {},
onToggleRightSidebar: () => () => {},
onToggleWorktreePalette: () => () => {},
onOpenQuickOpen: () => () => {},
onJumpToWorktreeIndex: () => () => {},
onActivateWorktree: () => () => {},
onNewBrowserTab: () => () => {},
onRequestTabCreate: () => () => {},
replyTabCreate: () => {},
onRequestTabClose: (
listener: (data: {
requestId: string
tabId: string | null
worktreeId?: string
}) => void
) => {
tabCloseListenerRef.current = listener
return () => {}
},
replyTabClose,
onNewTerminalTab: () => () => {},
onCloseActiveTab: () => () => {},
onSwitchTab: () => () => {},
onToggleStatusBar: () => () => {},
onFullscreenChanged: () => () => {},
onTerminalZoom: () => () => {},
getZoomLevel: () => 0,
set: vi.fn()
},
updater: {
getStatus: () => Promise.resolve({ state: 'idle' }),
onStatus: () => () => {},
onClearDismissal: () => () => {}
},
browser: {
onGuestLoadFailed: () => () => {},
onOpenLinkInOrcaTab: () => () => {},
onNavigationUpdate: () => () => {},
onActivateView: () => () => {}
},
rateLimits: {
get: () => Promise.resolve({ limits: {}, lastUpdatedAt: Date.now() }),
onUpdate: () => () => {}
},
ssh: {
listTargets: () => Promise.resolve([]),
getState: () => Promise.resolve(null),
onStateChanged: () => () => {},
onCredentialRequest: () => () => {},
onCredentialResolved: () => () => {}
}
}
})
const { useIpcEvents } = await import('./useIpcEvents')
useIpcEvents()
expect(tabCloseListenerRef.current).toBeTypeOf('function')
tabCloseListenerRef.current?.({
requestId: 'req-1',
tabId: null,
worktreeId: 'wt-2'
})
expect(closeBrowserTab).toHaveBeenCalledWith('workspace-target')
expect(closeBrowserPage).not.toHaveBeenCalled()
expect(replyTabClose).toHaveBeenCalledWith({ requestId: 'req-1' })
})
it('closes only the requested browser page when a workspace has multiple pages', async () => {
const closeBrowserTab = vi.fn()
const closeBrowserPage = vi.fn()
const replyTabClose = vi.fn()
const tabCloseListenerRef: {
current:
| ((data: { requestId: string; tabId: string | null; worktreeId?: string }) => void)
| null
} = {
current: null
}
vi.doMock('react', async () => {
const actual = await vi.importActual<typeof ReactModule>('react')
return {
...actual,
useEffect: (effect: () => void | (() => void)) => {
effect()
}
}
})
vi.doMock('../store', () => ({
useAppStore: {
getState: () => ({
setUpdateStatus: vi.fn(),
fetchRepos: vi.fn(),
fetchWorktrees: vi.fn(),
setActiveView: vi.fn(),
activeModal: null,
closeModal: vi.fn(),
openModal: vi.fn(),
activeWorktreeId: 'wt-1',
activeView: 'terminal',
setActiveRepo: vi.fn(),
setActiveWorktree: vi.fn(),
revealWorktreeInSidebar: vi.fn(),
setIsFullScreen: vi.fn(),
updateBrowserTabPageState: vi.fn(),
activeTabType: 'browser',
editorFontZoomLevel: 0,
setEditorFontZoomLevel: vi.fn(),
setRateLimitsFromPush: vi.fn(),
setSshConnectionState: vi.fn(),
setSshTargetLabels: vi.fn(),
enqueueSshCredentialRequest: vi.fn(),
removeSshCredentialRequest: vi.fn(),
settings: { terminalFontSize: 13 },
activeBrowserTabId: 'workspace-1',
activeBrowserTabIdByWorktree: { 'wt-1': 'workspace-1' },
browserTabsByWorktree: {
'wt-1': [{ id: 'workspace-1' }]
},
browserPagesByWorkspace: {
'workspace-1': [
{ id: 'page-1', workspaceId: 'workspace-1' },
{ id: 'page-2', workspaceId: 'workspace-1' }
]
},
closeBrowserTab,
closeBrowserPage
})
}
}))
vi.doMock('@/lib/ui-zoom', () => ({
applyUIZoom: vi.fn()
}))
vi.doMock('@/lib/worktree-activation', () => ({
activateAndRevealWorktree: vi.fn(),
ensureWorktreeHasInitialTerminal: vi.fn()
}))
vi.doMock('@/components/sidebar/visible-worktrees', () => ({
getVisibleWorktreeIds: () => []
}))
vi.doMock('@/lib/editor-font-zoom', () => ({
nextEditorFontZoomLevel: vi.fn(() => 0),
computeEditorFontSize: vi.fn(() => 13)
}))
vi.doMock('@/components/settings/SettingsConstants', () => ({
zoomLevelToPercent: vi.fn(() => 100),
ZOOM_MIN: -3,
ZOOM_MAX: 3
}))
vi.doMock('@/lib/zoom-events', () => ({
dispatchZoomLevelChanged: vi.fn()
}))
vi.stubGlobal('window', {
dispatchEvent: vi.fn(),
api: {
repos: { onChanged: () => () => {} },
worktrees: { onChanged: () => () => {} },
ui: {
onOpenSettings: () => () => {},
onToggleLeftSidebar: () => () => {},
onToggleRightSidebar: () => () => {},
onToggleWorktreePalette: () => () => {},
onOpenQuickOpen: () => () => {},
onJumpToWorktreeIndex: () => () => {},
onActivateWorktree: () => () => {},
onNewBrowserTab: () => () => {},
onRequestTabCreate: () => () => {},
replyTabCreate: () => {},
onRequestTabClose: (
listener: (data: {
requestId: string
tabId: string | null
worktreeId?: string
}) => void
) => {
tabCloseListenerRef.current = listener
return () => {}
},
replyTabClose,
onNewTerminalTab: () => () => {},
onCloseActiveTab: () => () => {},
onSwitchTab: () => () => {},
onToggleStatusBar: () => () => {},
onFullscreenChanged: () => () => {},
onTerminalZoom: () => () => {},
getZoomLevel: () => 0,
set: vi.fn()
},
updater: {
getStatus: () => Promise.resolve({ state: 'idle' }),
onStatus: () => () => {},
onClearDismissal: () => () => {}
},
browser: {
onGuestLoadFailed: () => () => {},
onOpenLinkInOrcaTab: () => () => {},
onNavigationUpdate: () => () => {},
onActivateView: () => () => {}
},
rateLimits: {
get: () => Promise.resolve({ limits: {}, lastUpdatedAt: Date.now() }),
onUpdate: () => () => {}
},
ssh: {
listTargets: () => Promise.resolve([]),
getState: () => Promise.resolve(null),
onStateChanged: () => () => {},
onCredentialRequest: () => () => {},
onCredentialResolved: () => () => {}
}
}
})
const { useIpcEvents } = await import('./useIpcEvents')
useIpcEvents()
tabCloseListenerRef.current?.({
requestId: 'req-2',
tabId: 'page-2'
})
expect(closeBrowserPage).toHaveBeenCalledWith('page-2')
expect(closeBrowserTab).not.toHaveBeenCalled()
expect(replyTabClose).toHaveBeenCalledWith({ requestId: 'req-2' })
})
it('rejects explicit unknown browser page ids instead of reporting success', async () => {
const closeBrowserTab = vi.fn()
const closeBrowserPage = vi.fn()
const replyTabClose = vi.fn()
const tabCloseListenerRef: {
current:
| ((data: { requestId: string; tabId: string | null; worktreeId?: string }) => void)
| null
} = {
current: null
}
vi.doMock('react', async () => {
const actual = await vi.importActual<typeof ReactModule>('react')
return {
...actual,
useEffect: (effect: () => void | (() => void)) => {
effect()
}
}
})
vi.doMock('../store', () => ({
useAppStore: {
getState: () => ({
setUpdateStatus: vi.fn(),
fetchRepos: vi.fn(),
fetchWorktrees: vi.fn(),
setActiveView: vi.fn(),
activeModal: null,
closeModal: vi.fn(),
openModal: vi.fn(),
activeWorktreeId: 'wt-1',
activeView: 'terminal',
setActiveRepo: vi.fn(),
setActiveWorktree: vi.fn(),
revealWorktreeInSidebar: vi.fn(),
setIsFullScreen: vi.fn(),
updateBrowserTabPageState: vi.fn(),
activeTabType: 'browser',
editorFontZoomLevel: 0,
setEditorFontZoomLevel: vi.fn(),
setRateLimitsFromPush: vi.fn(),
setSshConnectionState: vi.fn(),
setSshTargetLabels: vi.fn(),
enqueueSshCredentialRequest: vi.fn(),
removeSshCredentialRequest: vi.fn(),
settings: { terminalFontSize: 13 },
activeBrowserTabId: 'workspace-1',
activeBrowserTabIdByWorktree: { 'wt-1': 'workspace-1' },
browserTabsByWorktree: {
'wt-1': [{ id: 'workspace-1' }]
},
browserPagesByWorkspace: {
'workspace-1': [{ id: 'page-1', workspaceId: 'workspace-1' }]
},
closeBrowserTab,
closeBrowserPage
})
}
}))
vi.doMock('@/lib/ui-zoom', () => ({
applyUIZoom: vi.fn()
}))
vi.doMock('@/lib/worktree-activation', () => ({
activateAndRevealWorktree: vi.fn(),
ensureWorktreeHasInitialTerminal: vi.fn()
}))
vi.doMock('@/components/sidebar/visible-worktrees', () => ({
getVisibleWorktreeIds: () => []
}))
vi.doMock('@/lib/editor-font-zoom', () => ({
nextEditorFontZoomLevel: vi.fn(() => 0),
computeEditorFontSize: vi.fn(() => 13)
}))
vi.doMock('@/components/settings/SettingsConstants', () => ({
zoomLevelToPercent: vi.fn(() => 100),
ZOOM_MIN: -3,
ZOOM_MAX: 3
}))
vi.doMock('@/lib/zoom-events', () => ({
dispatchZoomLevelChanged: vi.fn()
}))
vi.stubGlobal('window', {
dispatchEvent: vi.fn(),
api: {
repos: { onChanged: () => () => {} },
worktrees: { onChanged: () => () => {} },
ui: {
onOpenSettings: () => () => {},
onToggleLeftSidebar: () => () => {},
onToggleRightSidebar: () => () => {},
onToggleWorktreePalette: () => () => {},
onOpenQuickOpen: () => () => {},
onJumpToWorktreeIndex: () => () => {},
onActivateWorktree: () => () => {},
onNewBrowserTab: () => () => {},
onRequestTabCreate: () => () => {},
replyTabCreate: () => {},
onRequestTabClose: (
listener: (data: {
requestId: string
tabId: string | null
worktreeId?: string
}) => void
) => {
tabCloseListenerRef.current = listener
return () => {}
},
replyTabClose,
onNewTerminalTab: () => () => {},
onCloseActiveTab: () => () => {},
onSwitchTab: () => () => {},
onToggleStatusBar: () => () => {},
onFullscreenChanged: () => () => {},
onTerminalZoom: () => () => {},
getZoomLevel: () => 0,
set: vi.fn()
},
updater: {
getStatus: () => Promise.resolve({ state: 'idle' }),
onStatus: () => () => {},
onClearDismissal: () => () => {}
},
browser: {
onGuestLoadFailed: () => () => {},
onOpenLinkInOrcaTab: () => () => {},
onNavigationUpdate: () => () => {},
onActivateView: () => () => {}
},
rateLimits: {
get: () => Promise.resolve({ limits: {}, lastUpdatedAt: Date.now() }),
onUpdate: () => () => {}
},
ssh: {
listTargets: () => Promise.resolve([]),
getState: () => Promise.resolve(null),
onStateChanged: () => () => {},
onCredentialRequest: () => () => {},
onCredentialResolved: () => () => {}
}
}
})
const { useIpcEvents } = await import('./useIpcEvents')
useIpcEvents()
tabCloseListenerRef.current?.({
requestId: 'req-3',
tabId: 'missing-page'
})
expect(closeBrowserPage).not.toHaveBeenCalled()
expect(closeBrowserTab).not.toHaveBeenCalled()
expect(replyTabClose).toHaveBeenCalledWith({
requestId: 'req-3',
error: 'Browser tab missing-page not found'
})
})
})
describe('useIpcEvents shortcut hint clearing', () => {
beforeEach(() => {
vi.resetModules()
@@ -485,6 +991,10 @@ describe('useIpcEvents shortcut hint clearing', () => {
},
onActivateWorktree: () => () => {},
onNewBrowserTab: () => () => {},
onRequestTabCreate: () => () => {},
replyTabCreate: () => {},
onRequestTabClose: () => () => {},
replyTabClose: () => {},
onNewTerminalTab: () => () => {},
onCloseActiveTab: () => () => {},
onSwitchTab: () => () => {},
@@ -501,7 +1011,9 @@ describe('useIpcEvents shortcut hint clearing', () => {
},
browser: {
onGuestLoadFailed: () => () => {},
onOpenLinkInOrcaTab: () => () => {}
onOpenLinkInOrcaTab: () => () => {},
onNavigationUpdate: () => () => {},
onActivateView: () => () => {}
},
rateLimits: {
get: () => Promise.resolve({ limits: {}, lastUpdatedAt: Date.now() }),
+108
View File
@@ -156,6 +156,28 @@ export function useIpcEvents(): void {
})
)
// Why: agent-browser drives navigation via CDP, bypassing Electron's webview
// event system. The renderer's did-navigate listener never fires for those
// navigations, so the Zustand store (address bar, tab title) stays stale.
// This IPC pushes the live URL/title from main after goto/click/back/reload.
unsubs.push(
window.api.browser.onNavigationUpdate(({ browserPageId, url, title }) => {
const store = useAppStore.getState()
store.setBrowserPageUrl(browserPageId, url)
store.updateBrowserPageState(browserPageId, { title, loading: false })
})
)
// Why: browser webviews only start their guest process when the container
// has display != none. After app restart, activeTabType defaults to 'terminal'
// so persisted browser tabs never mount. The main process sends this IPC
// before browser commands so the webview can start and registerGuest fires.
unsubs.push(
window.api.browser.onActivateView(() => {
useAppStore.getState().setActiveTabType('browser')
})
)
unsubs.push(
window.api.browser.onOpenLinkInOrcaTab(({ browserPageId, url }) => {
const store = useAppStore.getState()
@@ -187,6 +209,92 @@ export function useIpcEvents(): void {
})
)
// Why: CLI-driven tab creation sends a request with a specific worktreeId and
// url. The renderer creates the tab and replies with the workspace ID so the
// main process can wait for registerGuest before returning to the CLI.
unsubs.push(
window.api.ui.onRequestTabCreate((data) => {
try {
const store = useAppStore.getState()
const worktreeId = data.worktreeId ?? store.activeWorktreeId
if (!worktreeId) {
window.api.ui.replyTabCreate({ requestId: data.requestId, error: 'No active worktree' })
return
}
const workspace = store.createBrowserTab(worktreeId, data.url, { title: data.url })
// Why: registerGuest fires with the page ID (not workspace ID) as
// browserPageId. Return the page ID so waitForTabRegistration can
// correlate correctly.
const pages = useAppStore.getState().browserPagesByWorkspace[workspace.id] ?? []
const browserPageId = pages[0]?.id ?? workspace.id
window.api.ui.replyTabCreate({ requestId: data.requestId, browserPageId })
} catch (err) {
window.api.ui.replyTabCreate({
requestId: data.requestId,
error: err instanceof Error ? err.message : 'Tab creation failed'
})
}
})
)
unsubs.push(
window.api.ui.onRequestTabClose((data) => {
try {
const store = useAppStore.getState()
const explicitTargetId = data.tabId ?? null
let tabToClose =
explicitTargetId ??
(data.worktreeId
? (store.activeBrowserTabIdByWorktree?.[data.worktreeId] ?? null)
: store.activeBrowserTabId)
if (!tabToClose) {
window.api.ui.replyTabClose({
requestId: data.requestId,
error: 'No active browser tab to close'
})
return
}
// Why: the bridge stores tabs keyed by browserPageId (which is the page
// ID from registerGuest), but closeBrowserTab expects a workspace ID. If
// tabToClose is a page ID, close only that page unless it is the
// last page in its workspace. The CLI's `tab close --page` contract
// targets one browser page, not the entire workspace tab.
const isWorkspaceId = Object.values(store.browserTabsByWorktree)
.flat()
.some((ws) => ws.id === tabToClose)
if (!isWorkspaceId) {
const owningWorkspace = Object.entries(store.browserPagesByWorkspace).find(
([, pages]) => pages.some((p) => p.id === tabToClose)
)
if (owningWorkspace) {
const [workspaceId, pages] = owningWorkspace
if (pages.length <= 1) {
store.closeBrowserTab(workspaceId)
} else {
store.closeBrowserPage(tabToClose)
}
window.api.ui.replyTabClose({ requestId: data.requestId })
return
}
}
if (explicitTargetId) {
window.api.ui.replyTabClose({
requestId: data.requestId,
error: `Browser tab ${explicitTargetId} not found`
})
return
}
store.closeBrowserTab(tabToClose)
window.api.ui.replyTabClose({ requestId: data.requestId })
} catch (err) {
window.api.ui.replyTabClose({
requestId: data.requestId,
error: err instanceof Error ? err.message : 'Tab close failed'
})
}
})
)
unsubs.push(
window.api.ui.onNewTerminalTab(() => {
const store = useAppStore.getState()
+17
View File
@@ -595,6 +595,17 @@ export const createBrowserSlice: StateCreator<AppState, [], [], BrowserSlice> =
}
})
// Why: notify the CDP bridge which guest webContents is now active so
// subsequent agent commands (snapshot, click, etc.) target the correct tab.
// registerGuest uses page IDs (not workspace IDs), so we resolve the active
// page within the workspace to find the correct browserPageId.
const workspace = findWorkspace(get().browserTabsByWorktree, tabId)
if (workspace?.activePageId && typeof window !== 'undefined' && window.api?.browser) {
window.api.browser
.notifyActiveTabChanged({ browserPageId: workspace.activePageId })
.catch(() => {})
}
const item = Object.values(get().unifiedTabsByWorktree)
.flat()
.find((entry) => entry.contentType === 'browser' && entry.entityId === tabId)
@@ -796,6 +807,12 @@ export const createBrowserSlice: StateCreator<AppState, [], [], BrowserSlice> =
}
})
// Why: switching the active page within a workspace changes which guest
// webContents the CDP bridge should target for agent commands.
if (typeof window !== 'undefined' && window.api?.browser) {
window.api.browser.notifyActiveTabChanged({ browserPageId: pageId }).catch(() => {})
}
const workspace = findWorkspace(get().browserTabsByWorktree, workspaceId)
if (!workspace) {
return