Files
orca/tests/e2e/runtime-host-status-recovery.spec.ts
Neil 23808bb73f refactor(renderer): organize host and terminal owners and remove forwarding shells (#39320)
* refactor(renderer): organize host and terminal implementation owners

* fix(refactor): align active adoption reliability references

* refactor(terminal): remove forwarding shells and stale mocks

* fix(renderer): consolidate direct owner imports

* fix(refactor): route incoming issue trust type to its owner

* test(runtime): give independent status viewers separate pairing tickets

* style(refactor): normalize rebased terminal mock formatting
2026-11-07 03:31:13 -08:00

255 lines
10 KiB
TypeScript

import { createConnection, createServer, type Socket, type AddressInfo } from 'node:net'
import type { Page } from '@stablyai/playwright-test'
import {
decodePairingOffer,
encodePairingOffer
} from '../../src/shared/remote-runtime/pairing/pairing'
import type { PairingCreateOfferResult } from '../../src/shared/remote-runtime/pairing/pairing-create-offer-contract'
import { expect, test } from './helpers/orca-app'
import {
createRuntimeDesktopPairingOffer,
launchPairedElectronClient,
launchPairedWebClient,
type RuntimeDesktopPairingOffer
} from './helpers/paired-electron-client'
import { launchHeadlessPairedRuntimeHost } from './helpers/headless-paired-runtime-host'
async function interruptibleHost(offer: RuntimeDesktopPairingOffer) {
const pairing = decodePairingOffer(offer.pairingUrl)
const endpoint = new URL(pairing.endpoint)
const sockets = new Set<Socket>()
let online = true
const server = createServer((client) => {
if (!online) {
client.destroy()
return
}
const host = createConnection({ host: endpoint.hostname, port: Number(endpoint.port) })
for (const socket of [client, host]) {
sockets.add(socket)
socket.on('error', () => {
client.destroy()
host.destroy()
})
socket.on('close', () => {
sockets.delete(socket)
client.destroy()
host.destroy()
})
}
client.pipe(host).pipe(client)
})
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
const address = server.address() as AddressInfo
function proxyPairingOffer(candidate: RuntimeDesktopPairingOffer) {
const pairingUrl = encodePairingOffer({
...decodePairingOffer(candidate.pairingUrl),
endpoint: `ws://127.0.0.1:${address.port}`
})
let webClientUrl: string | undefined
if (candidate.webClientUrl) {
const url = new URL(candidate.webClientUrl)
url.search = ''
url.hash = new URLSearchParams({ pairing: pairingUrl }).toString()
webClientUrl = url.href
}
return { pairingUrl, webClientUrl }
}
return {
offer: proxyPairingOffer(offer),
proxyPairingOffer,
setOnline(value: boolean) {
online = value
if (!online) {
sockets.forEach((socket) => socket.destroy())
}
},
async close() {
sockets.forEach((socket) => socket.destroy())
await new Promise<void>((resolve) => server.close(() => resolve()))
}
}
}
async function statusEvidence(page: Page, environmentId?: string) {
return page.evaluate((id) => {
const entries = window.__store?.getState().runtimeStatusByEnvironmentId
const entry = id ? entries?.get(id) : entries?.values().next().value
return entry?.snapshot
? {
verification: entry.snapshot.verification,
transport: entry.snapshot.transport,
runtimeId: entry.status?.runtimeId,
sequence: entry.snapshot.sequence
}
: null
}, environmentId)
}
async function expectWorkspaceHostAppearance(
page: Page,
disconnected: boolean,
hostLabel?: string
) {
const cards = page.locator('[data-worktree-card-surface="true"]')
const card = (
hostLabel ? cards.filter({ has: page.getByText(hostLabel, { exact: true }) }) : cards
).first()
await expect(card).toBeVisible()
await expect(card).toHaveCSS('opacity', disconnected ? '0.6' : '1')
const icon = card.locator(disconnected ? 'svg.lucide-server-off' : 'svg.lucide-server').first()
await expect(icon).toBeVisible()
await expect(
card.locator(disconnected ? 'svg.lucide-server' : 'svg.lucide-server-off')
).toHaveCount(0)
await expect(icon).toHaveClass(disconnected ? /text-destructive/ : /text-muted-foreground/)
await icon.hover()
// Name the host: the previous card's tooltip can still be closing in the DOM.
const tooltipName = hostLabel
? disconnected
? `${hostLabel} disconnected`
: `Project on ${hostLabel}`
: disconnected
? /disconnected/i
: /Project on/
await expect(page.getByRole('tooltip', { name: tooltipName, exact: true })).toBeVisible()
await page.mouse.move(900, 600)
}
for (const topology of ['desktop', 'headless'] as const) {
test(`connection-owned status recovers with a ${topology} host and independent viewers`, async ({
electronApp,
orcaPage: page,
testRepoPath
}, testInfo) => {
test.setTimeout(180_000)
let headless: Awaited<ReturnType<typeof launchHeadlessPairedRuntimeHost>> | null = null
let proxy: Awaited<ReturnType<typeof interruptibleHost>> | undefined
let client: Awaited<ReturnType<typeof launchPairedElectronClient>> | undefined
let browser: Awaited<ReturnType<typeof launchPairedWebClient>> | undefined
try {
// Why two servers: a client saves one entry per server, so the steady host must be another one.
headless = await launchHeadlessPairedRuntimeHost({ pinnedServePort: true })
await headless.client.call('repo.add', { path: testRepoPath })
const desktopOffer = await createRuntimeDesktopPairingOffer(page)
const [recoveringOffer, directOffer] =
topology === 'headless' ? [headless.offer, desktopOffer] : [desktopOffer, headless.offer]
proxy = await interruptibleHost(recoveringOffer)
client = await launchPairedElectronClient(directOffer, testInfo, 'Direct host')
proxy.setOnline(false)
const offlineId = await client.page.evaluate(async (pairingCode) => {
const { environment } = await window.api.runtimeEnvironments.addFromPairingCode({
name: 'Recovering host',
pairingCode
})
const store = window.__store!.getState()
store.setRuntimeEnvironments(await window.api.runtimeEnvironments.list())
await store.refreshRuntimeEnvironmentStatus(environment.id, 1_000)
return environment.id
}, proxy.offer.pairingUrl)
await expect
.poll(() => statusEvidence(client!.page, offlineId))
.toMatchObject({ verification: 'unavailable' })
expect(await statusEvidence(client!.page, client.environmentId)).toMatchObject({
verification: 'verified'
})
proxy.setOnline(true)
await expect
.poll(() => statusEvidence(client!.page, offlineId), { timeout: 30_000 })
.toMatchObject({ verification: 'verified', transport: 'ready' })
const initial = await statusEvidence(client!.page, offlineId)
await expectWorkspaceHostAppearance(client.page, false, 'Recovering host')
await expect(client.page.getByText('Recovering host', { exact: true }).first()).toBeVisible()
await client.page.screenshot({ path: testInfo.outputPath(`${topology}-recovered.png`) })
// Each viewer needs its own ticket; browser key migration retires its pairing ticket.
let browserOffer: RuntimeDesktopPairingOffer
if (topology === 'desktop') {
browserOffer = await createRuntimeDesktopPairingOffer(page)
} else {
const { result } = await headless.client.call<PairingCreateOfferResult>(
'pairing.createOffer',
{
scope: 'runtime',
address: '127.0.0.1',
rotate: true
}
)
if (!result.available) {
throw new Error(result.guidance)
}
browserOffer = {
pairingUrl: result.pairingUrl,
...(result.webClientUrl ? { webClientUrl: result.webClientUrl } : {})
}
}
browser = await launchPairedWebClient(electronApp, proxy.proxyPairingOffer(browserOffer))
await expect
.poll(() => statusEvidence(browser!.page), { timeout: 30_000 })
.toMatchObject({ verification: 'verified', transport: 'ready' })
await expectWorkspaceHostAppearance(browser.page, false)
proxy.setOnline(false)
await expect
.poll(() => statusEvidence(client!.page, offlineId))
.toMatchObject({ transport: 'disconnected' })
await expect
.poll(() => statusEvidence(browser!.page), { timeout: 30_000 })
.toMatchObject({ transport: 'disconnected' })
expect(await statusEvidence(client!.page, client.environmentId)).toMatchObject({
verification: 'verified',
transport: 'ready'
})
await expectWorkspaceHostAppearance(client.page, false, 'Recovering host')
await expectWorkspaceHostAppearance(client.page, false, 'Direct host')
await expectWorkspaceHostAppearance(browser.page, false)
await client.page.screenshot({
path: testInfo.outputPath(`${topology}-sidebar-reconnecting.png`)
})
await browser.page.screenshot({
path: testInfo.outputPath(`${topology}-browser-reconnecting.png`)
})
proxy.setOnline(true)
await expect
.poll(() => statusEvidence(client!.page, offlineId), { timeout: 30_000 })
.toMatchObject({ verification: 'verified', transport: 'ready' })
await expect
.poll(() => statusEvidence(browser!.page), { timeout: 30_000 })
.toMatchObject({ verification: 'verified', transport: 'ready' })
expect((await statusEvidence(client!.page, offlineId))!.sequence).toBeGreaterThan(
initial!.sequence
)
await expectWorkspaceHostAppearance(client.page, false, 'Recovering host')
await expectWorkspaceHostAppearance(browser.page, false)
await browser.page.screenshot({
path: testInfo.outputPath(`${topology}-browser-recovered.png`)
})
await client.page.evaluate(async (selector) => {
await window.api.runtimeEnvironments.disconnect({ selector })
}, offlineId)
await expect
.poll(() => statusEvidence(client!.page, offlineId))
.toMatchObject({ verification: 'blocked', transport: 'disconnected' })
await expectWorkspaceHostAppearance(client.page, true, 'Recovering host')
await expectWorkspaceHostAppearance(client.page, false, 'Direct host')
await client.page.screenshot({
path: testInfo.outputPath(`${topology}-sidebar-disconnected.png`)
})
await client.page.evaluate(async (selector) => {
await window.api.runtimeEnvironments.connect({ selector })
}, offlineId)
await expect
.poll(() => statusEvidence(client!.page, offlineId), { timeout: 30_000 })
.toMatchObject({ verification: 'verified', transport: 'ready' })
await expectWorkspaceHostAppearance(client.page, false, 'Recovering host')
await expectWorkspaceHostAppearance(browser.page, false)
await client.page.screenshot({
path: testInfo.outputPath(`${topology}-sidebar-restored.png`)
})
} finally {
await browser?.dispose()
await client?.dispose()
await proxy?.close()
await headless?.dispose()
}
})
}