From 2f001fe880be709166c3a8f716f07b8323aaba45 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:02:29 -0400 Subject: [PATCH] fix(remote-browser): host client browser pages again when a paired host comes back after an outage (#25827) * test(remote-browser): reproduce a client-hosted browser tab left unavailable after a network outage A paired desktop that loses its host for longer than the browser-host lease grace (25s reset cut, or a 100s silent cut the host's heartbeat reaps) tears down its client-hosted pages and never hosts them again, leaving 'Client-hosted browser unavailable'. Red on main. Extends the freezable TCP proxy with a full silent or reset link cut. * fix(remote-browser): host client browser pages again when a paired host comes back after an outage When the network between a paired desktop and its Orca host dropped for longer than the browser-host lease grace, the desktop tore down its client-hosted pages and never offered to host them again, so the tab stayed on 'Client-hosted browser unavailable' until the app was relaunched. Regaining contact with a runtime now re-claims browser hosting the same way a runtime restart already did: if this desktop still has client-placed pages for that host, it starts its browser client host again and the host's existing recovery re-places them, reloading at their last URL. The notice now says the host is offline (and that the page reloads when it reconnects), names another desktop only when the placement belongs to one, and otherwise says the page is not available here right now. * fix(remote-browser): claim offline only for a host that can reconnect, and re-host from the notice The unavailable notice said the host was offline for any null status, which also covers a retired or refused pairing and a host whose runtime did not answer. It now uses the host connection state and excludes revoked verdicts. When the control link returns while the old browser host is still inside its lease reconnect, the contact-edge ensure is a no-op; if that reconnect then fails, nothing re-hosts. The notice now re-claims hosting on mount while the host is reachable, which is exactly that stranded state. Plain regained contact no longer queues a second preparation behind one already in flight; only a runtime restart does. * fix(remote-browser): re-prepare browser hosting on every regained contact A preparation still in flight when contact returns started while the host was unreachable, so it is the one most likely to fail. Coalescing the contact edge into it dropped the only trigger that could succeed and left the page unavailable. Always queue one follow-up preparation instead. --- .../ClientHostedBrowserPagePane.tsx | 1 + ...hosted-browser-unavailable-notice.test.tsx | 105 ++++++++++ ...ient-hosted-browser-unavailable-notice.tsx | 60 +++++- src/renderer/src/i18n/locales/en.json | 4 +- src/renderer/src/i18n/locales/es.json | 4 +- src/renderer/src/i18n/locales/fr.json | 4 +- src/renderer/src/i18n/locales/ja.json | 4 +- src/renderer/src/i18n/locales/ko.json | 4 +- src/renderer/src/i18n/locales/zh.json | 4 +- ...tored-client-hosted-browser-host-attach.ts | 16 +- ...hosted-browser-host-restart-attach.test.ts | 62 ++++-- .../slices/runtime-status-snapshot.test.ts | 10 +- .../src/store/slices/runtime-status.ts | 15 +- tests/e2e/helpers/freezable-tcp-proxy.ts | 119 ++++++++++++ ...ient-hosted-browser-network-outage.spec.ts | 183 ++++++++++++++++++ 15 files changed, 551 insertions(+), 44 deletions(-) create mode 100644 src/renderer/src/components/browser-pane/client-hosted-browser-unavailable-notice.test.tsx create mode 100644 tests/e2e/helpers/freezable-tcp-proxy.ts create mode 100644 tests/e2e/paired-client-hosted-browser-network-outage.spec.ts diff --git a/src/renderer/src/components/browser-pane/ClientHostedBrowserPagePane.tsx b/src/renderer/src/components/browser-pane/ClientHostedBrowserPagePane.tsx index bcc72d82308..a956a35a642 100644 --- a/src/renderer/src/components/browser-pane/ClientHostedBrowserPagePane.tsx +++ b/src/renderer/src/components/browser-pane/ClientHostedBrowserPagePane.tsx @@ -419,6 +419,7 @@ export function ClientHostedBrowserPagePane({ runtimeEnvironmentId={runtimeEnvironmentId} worktreeId={worktreeId} lastCommittedUrl={browserTab.url} + placementHostClientId={browserHostClientId} /> ) : null} diff --git a/src/renderer/src/components/browser-pane/client-hosted-browser-unavailable-notice.test.tsx b/src/renderer/src/components/browser-pane/client-hosted-browser-unavailable-notice.test.tsx new file mode 100644 index 00000000000..5befddce167 --- /dev/null +++ b/src/renderer/src/components/browser-pane/client-hosted-browser-unavailable-notice.test.tsx @@ -0,0 +1,105 @@ +// @vitest-environment happy-dom +import '@testing-library/jest-dom/vitest' +import { cleanup, render, screen } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { RuntimeEnvironmentStatus } from '../../../../shared/runtime-host-status' +import { createCompatibleRuntimeStatusResponse } from '@/runtime/runtime-compatibility-test-fixture' + +const mocks = vi.hoisted(() => { + const state = { + runtimeStatusByEnvironmentId: new Map(), + runtimeEnvironments: [{ id: 'env-a', name: 'Studio PC' }], + remoteBrowserPageHandlesByPageId: {} + } + const useAppStore = (selector: (s: typeof state) => T): T => selector(state) + useAppStore.getState = () => state + return { state, useAppStore, ensure: vi.fn(), ownHostClientId: 'desktop-self' } +}) + +vi.mock('@/store', () => ({ useAppStore: mocks.useAppStore })) +vi.mock('@/runtime/restored-client-hosted-browser-host-attach', () => ({ + ensureBrowserClientHostOnRuntimeContact: mocks.ensure +})) +vi.mock('@/runtime/browser-client-host-identity', () => ({ + readBrowserClientHostId: () => mocks.ownHostClientId +})) +vi.mock('./browser-reopen-on-server', () => ({ reopenBrowserPageOnServer: vi.fn() })) +vi.mock('sonner', () => ({ toast: { error: vi.fn() } })) + +import { ClientHostedBrowserUnavailableNotice } from './client-hosted-browser-unavailable-notice' + +afterEach(() => { + cleanup() + vi.clearAllMocks() +}) + +function renderNotice( + entry: RuntimeEnvironmentStatus | undefined, + placementHostClientId: string | null = 'desktop-self' +): void { + mocks.state.runtimeStatusByEnvironmentId = new Map(entry ? [['env-a', entry]] : []) + render( + + ) +} + +const liveResponse = createCompatibleRuntimeStatusResponse('rt-1') +const live: RuntimeEnvironmentStatus = { + status: liveResponse.ok ? liveResponse.result : null, + checkedAt: 1 +} + +describe('ClientHostedBrowserUnavailableNotice', () => { + it('says the host is offline while contact is lost', () => { + renderNotice({ status: null, checkedAt: 1 }) + + expect( + screen.getByText('Studio PC is offline. This page will reload here once it reconnects.') + ).toBeInTheDocument() + expect(mocks.ensure).not.toHaveBeenCalled() + }) + + it('does not promise a reconnect for a retired pairing', () => { + renderNotice({ + status: null, + checkedAt: 1, + snapshot: { + environmentId: 'env-a', + pairingRevision: 1, + sequence: 1, + checkedAt: 1, + transport: 'disconnected', + verification: 'unavailable', + status: null, + retired: true + } + }) + + expect( + screen.getByText("This page isn't available on this desktop right now.") + ).toBeInTheDocument() + }) + + it('names another desktop only when the placement belongs to one', () => { + renderNotice(live, 'desktop-other') + + expect(screen.getByText('This page is open on another desktop.')).toBeInTheDocument() + expect(mocks.ensure).not.toHaveBeenCalled() + }) + + // Why: a lease reconnect that fails after contact already returned has no later edge to re-host. + it('shows the generic copy and re-claims hosting once while the host is reachable', () => { + renderNotice(live) + + expect( + screen.getByText("This page isn't available on this desktop right now.") + ).toBeInTheDocument() + expect(mocks.ensure).toHaveBeenCalledTimes(1) + expect(mocks.ensure).toHaveBeenCalledWith(expect.anything(), 'env-a') + }) +}) diff --git a/src/renderer/src/components/browser-pane/client-hosted-browser-unavailable-notice.tsx b/src/renderer/src/components/browser-pane/client-hosted-browser-unavailable-notice.tsx index fbd82cc7fdc..ca0fdf2aa9f 100644 --- a/src/renderer/src/components/browser-pane/client-hosted-browser-unavailable-notice.tsx +++ b/src/renderer/src/components/browser-pane/client-hosted-browser-unavailable-notice.tsx @@ -1,5 +1,14 @@ +import { useEffect } from 'react' import { Globe } from 'lucide-react' import { translate } from '@/i18n/i18n' +import { useAppStore } from '@/store' +import { readBrowserClientHostId } from '@/runtime/browser-client-host-identity' +import { runtimeHostConnectionStateForEntry } from '@/runtime/runtime-host-connection-state' +import { ensureBrowserClientHostOnRuntimeContact } from '@/runtime/restored-client-hosted-browser-host-attach' +import { + isRuntimeHostContactRevokedVerdict, + runtimeHostContactForEntry +} from '../../../../shared/runtime-host-contact' import { ReopenBrowserPageOnServerButton, reopenOnServerCaveat @@ -13,12 +22,54 @@ import { export function ClientHostedBrowserUnavailableNotice({ runtimeEnvironmentId, worktreeId, - lastCommittedUrl + lastCommittedUrl, + placementHostClientId }: { runtimeEnvironmentId: string worktreeId: string lastCommittedUrl: string + placementHostClientId: string | null }): React.JSX.Element { + const offlineHostName = useAppStore((s) => { + const entry = s.runtimeStatusByEnvironmentId.get(runtimeEnvironmentId) + const state = runtimeHostConnectionStateForEntry(entry) + // Why the revoked check: a retired or refused pairing also reads 'disconnected', but never + // reconnects. + const offline = + (state === 'reconnecting' || state === 'disconnected') && + !isRuntimeHostContactRevokedVerdict(runtimeHostContactForEntry(entry)) + return offline + ? (s.runtimeEnvironments.find((environment) => environment.id === runtimeEnvironmentId) + ?.name ?? null) + : null + }) + const ownHostClientId = readBrowserClientHostId() + const onOtherDesktop = Boolean( + placementHostClientId && ownHostClientId && placementHostClientId !== ownHostClientId + ) + // Why on mount: a lease reconnect that fails after the control link already came back leaves + // this desktop unhosted with no later contact edge to re-claim it. + useEffect(() => { + const state = useAppStore.getState() + if (!onOtherDesktop && state.runtimeStatusByEnvironmentId.get(runtimeEnvironmentId)?.status) { + void ensureBrowserClientHostOnRuntimeContact(state, runtimeEnvironmentId) + } + }, [onOtherDesktop, runtimeEnvironmentId]) + const description = onOtherDesktop + ? translate( + 'browser.clientHosted.unavailableOtherDesktopDescription', + 'This page is open on another desktop.' + ) + : offlineHostName + ? translate( + 'browser.clientHosted.unavailableHostOfflineDescription', + '{{host}} is offline. This page will reload here once it reconnects.', + { host: offlineHostName } + ) + : translate( + 'browser.clientHosted.unavailableDescription', + "This page isn't available on this desktop right now." + ) return (
@@ -26,12 +77,7 @@ export function ClientHostedBrowserUnavailableNotice({
{translate('browser.clientHosted.unavailableTitle', 'Client-hosted browser unavailable')}
-
- {translate( - 'browser.clientHosted.unavailableDescription', - 'This page is attached to a different desktop or is no longer available.' - )} -
+
{description}
{reopenOnServerCaveat()}
{ if (!hasLiveClientHostedPage(state, environmentId)) { return } + // Why re-prepare: an in-flight preparation began before this contact, so it may have failed. await prepareBrowserClientHost(environmentId, true) } diff --git a/src/renderer/src/runtime/restored-client-hosted-browser-host-restart-attach.test.ts b/src/renderer/src/runtime/restored-client-hosted-browser-host-restart-attach.test.ts index 5daf0c70cdf..a01cb2e640a 100644 --- a/src/renderer/src/runtime/restored-client-hosted-browser-host-restart-attach.test.ts +++ b/src/renderer/src/runtime/restored-client-hosted-browser-host-restart-attach.test.ts @@ -9,7 +9,7 @@ import { } from '../store/slices/runtime-status' import { createCompatibleRuntimeStatusResponse } from './runtime-compatibility-test-fixture' import { - ensureBrowserClientHostForRestartedRuntime, + ensureBrowserClientHostOnRuntimeContact, ensureBrowserClientHostsForRestoredPages, resetRestoredBrowserClientHostAttachForTests } from './restored-client-hosted-browser-host-attach' @@ -60,7 +60,7 @@ function preparedEnvironmentIds(): string[] { return prepareBrowserClientHostPlacement.mock.calls.map((call) => call[0].selector) } -describe('ensureBrowserClientHostForRestartedRuntime', () => { +describe('ensureBrowserClientHostOnRuntimeContact', () => { beforeEach(() => { resetRestoredBrowserClientHostAttachForTests() prepareBrowserClientHostPlacement.mockClear() @@ -75,7 +75,7 @@ describe('ensureBrowserClientHostForRestartedRuntime', () => { // Why: the guests are still alive in this desktop's webviews, so re-preparing drives the registry // down replaceAuthority and hands the replacement runtime an inventory it can adopt. it('prepares the host for an environment with a live client-placed page', async () => { - await ensureBrowserClientHostForRestartedRuntime( + await ensureBrowserClientHostOnRuntimeContact( stateWith({ 'page-1': { environmentId: 'env-a', placement: CLIENT_PLACEMENT } }), 'env-a' ) @@ -89,7 +89,7 @@ describe('ensureBrowserClientHostForRestartedRuntime', () => { // Why: a staged page has no host-minted placement yet, but this desktop is already its host. it('prepares the host for a staged client-hosted page with no placement yet', async () => { - await ensureBrowserClientHostForRestartedRuntime( + await ensureBrowserClientHostOnRuntimeContact( stateWith({ 'page-1': { environmentId: 'env-a', staged: true, stagedClientHosted: true } }), 'env-a' ) @@ -99,7 +99,7 @@ describe('ensureBrowserClientHostForRestartedRuntime', () => { // Why: a session-restored row that adoption has not spent yet is client-hosted just as surely. it('prepares the host for a restored client-hosted page with no placement yet', async () => { - await ensureBrowserClientHostForRestartedRuntime( + await ensureBrowserClientHostOnRuntimeContact( stateWith({ 'page-1': { environmentId: 'env-a', restoredFromSession: true, restoredClientHosted: true } }), @@ -110,7 +110,7 @@ describe('ensureBrowserClientHostForRestartedRuntime', () => { }) it('prepares no host when the environment has no handles at all', async () => { - await ensureBrowserClientHostForRestartedRuntime(stateWith({}), 'env-a') + await ensureBrowserClientHostOnRuntimeContact(stateWith({}), 'env-a') expect(prepareBrowserClientHostPlacement).not.toHaveBeenCalled() }) @@ -118,7 +118,7 @@ describe('ensureBrowserClientHostForRestartedRuntime', () => { // Why: a server-hosted-only environment loses nothing to a runtime restart, so claiming hosting // duty for it would be work this desktop was never asked to do. it('prepares no host when the environment only has server-hosted pages', async () => { - await ensureBrowserClientHostForRestartedRuntime( + await ensureBrowserClientHostOnRuntimeContact( stateWith({ 'page-1': { environmentId: 'env-a', placement: { kind: 'server' } } }), 'env-a' ) @@ -128,7 +128,7 @@ describe('ensureBrowserClientHostForRestartedRuntime', () => { // Why: one runtime restarting must not start a host on every other environment this client knows. it('prepares no host when the only client-hosted page belongs to another environment', async () => { - await ensureBrowserClientHostForRestartedRuntime( + await ensureBrowserClientHostOnRuntimeContact( stateWith({ 'page-1': { environmentId: 'env-b', placement: CLIENT_PLACEMENT } }), 'env-a' ) @@ -137,7 +137,7 @@ describe('ensureBrowserClientHostForRestartedRuntime', () => { }) it('prepares only the restarted environment when several have client-hosted pages', async () => { - await ensureBrowserClientHostForRestartedRuntime( + await ensureBrowserClientHostOnRuntimeContact( stateWith({ 'page-1': { environmentId: 'env-a', placement: CLIENT_PLACEMENT }, 'page-2': { environmentId: 'env-b', placement: CLIENT_PLACEMENT } @@ -155,7 +155,7 @@ describe('ensureBrowserClientHostForRestartedRuntime', () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) await expect( - ensureBrowserClientHostForRestartedRuntime( + ensureBrowserClientHostOnRuntimeContact( stateWith({ 'page-1': { environmentId: 'env-a', placement: CLIENT_PLACEMENT } }), 'env-a' ) @@ -168,7 +168,7 @@ describe('ensureBrowserClientHostForRestartedRuntime', () => { // Why: a restart can be observed before the browser slice has published any handle map. it('tolerates a state with no handle map at all', async () => { - await expect(ensureBrowserClientHostForRestartedRuntime({}, 'env-a')).resolves.toBeUndefined() + await expect(ensureBrowserClientHostOnRuntimeContact({}, 'env-a')).resolves.toBeUndefined() expect(prepareBrowserClientHostPlacement).not.toHaveBeenCalled() }) @@ -188,12 +188,33 @@ describe('ensureBrowserClientHostForRestartedRuntime', () => { const restored = ensureBrowserClientHostsForRestoredPages( stateWith({ 'page-1': { environmentId: 'env-a', restoredClientHosted: true } }) ) - const restarted = ensureBrowserClientHostForRestartedRuntime(state, 'env-a') + const restarted = ensureBrowserClientHostOnRuntimeContact(state, 'env-a') releaseFirst() await Promise.all([restored, restarted]) expect(preparedEnvironmentIds()).toEqual(['env-a', 'env-a']) }) + + // Why: a preparation in flight when contact returns began while the host was unreachable. + it('re-prepares when contact returns during a preparation that then fails', async () => { + const state = stateWith({ 'page-1': { environmentId: 'env-a', placement: CLIENT_PLACEMENT } }) + let releaseFirst = (): void => {} + prepareBrowserClientHostPlacement.mockImplementationOnce( + async () => + new Promise((resolve) => { + releaseFirst = () => resolve({ kind: 'server' as const }) + }) + ) + + const restored = ensureBrowserClientHostsForRestoredPages( + stateWith({ 'page-1': { environmentId: 'env-a', restoredClientHosted: true } }) + ) + const regained = ensureBrowserClientHostOnRuntimeContact(state, 'env-a') + releaseFirst() + await Promise.all([restored, regained]) + + expect(preparedEnvironmentIds()).toEqual(['env-a', 'env-a']) + }) }) function statusFor(runtimeId: string, checkedAt = 1): RuntimeEnvironmentStatus { @@ -209,7 +230,7 @@ function storeWithClientHostedPage() { })) } -describe('setRuntimeEnvironmentStatus runtime-restart detection', () => { +describe('setRuntimeEnvironmentStatus runtime-contact detection', () => { beforeEach(() => { clearRuntimeEnvironmentConnectionGenerationsForTests() resetRestoredBrowserClientHostAttachForTests() @@ -245,6 +266,21 @@ describe('setRuntimeEnvironmentStatus runtime-restart detection', () => { expect(prepareBrowserClientHostPlacement).not.toHaveBeenCalled() }) + // Why: an outage past the lease grace tears this desktop's host down with its guests, and nothing + // else ever starts it again, so the tab stays "unavailable" for good. + it('re-attaches when the same runtime answers again after contact was lost', async () => { + const store = storeWithClientHostedPage() + + store.getState().setRuntimeEnvironmentStatus('env-a', statusFor('runtime-1')) + store.getState().setRuntimeEnvironmentStatus('env-a', { status: null, checkedAt: 2 }) + await Promise.resolve() + expect(prepareBrowserClientHostPlacement).not.toHaveBeenCalled() + store.getState().setRuntimeEnvironmentStatus('env-a', statusFor('runtime-1', 3)) + await Promise.resolve() + + expect(preparedEnvironmentIds()).toEqual(['env-a']) + }) + it('does not re-attach when a re-probe reports the same runtime id', async () => { const store = storeWithClientHostedPage() diff --git a/src/renderer/src/store/slices/runtime-status-snapshot.test.ts b/src/renderer/src/store/slices/runtime-status-snapshot.test.ts index 48973fedf93..83d4628585e 100644 --- a/src/renderer/src/store/slices/runtime-status-snapshot.test.ts +++ b/src/renderer/src/store/slices/runtime-status-snapshot.test.ts @@ -10,12 +10,12 @@ import type { RuntimeHostStatusSnapshot } from '../../../../shared/runtime-host- import type { RuntimeStatus } from '../../../../shared/runtime-types' import type { PublicKnownRuntimeEnvironment } from '../../../../shared/runtime-environments' import { runtimeHostConnectionStateForEntry } from '@/runtime/runtime-host-connection-state' -import { ensureBrowserClientHostForRestartedRuntime } from '@/runtime/restored-client-hosted-browser-host-attach' +import { ensureBrowserClientHostOnRuntimeContact } from '@/runtime/restored-client-hosted-browser-host-attach' vi.mock('sonner', () => ({ toast: { warning: vi.fn(), dismiss: vi.fn() } })) vi.mock('@/runtime/restored-client-hosted-browser-host-attach', () => ({ ensureBrowserClientHostsForRestoredPages: vi.fn(), - ensureBrowserClientHostForRestartedRuntime: vi.fn() + ensureBrowserClientHostOnRuntimeContact: vi.fn() })) vi.mock('@/runtime/client-hosted-browser-close-intent-replay', () => ({ replayClientHostedBrowserCloseIntents: vi.fn() @@ -84,7 +84,7 @@ it('represents failed verification honestly without manufacturing a session rest viewer.getState().applyRuntimeHostStatusSnapshot(snapshot(3)) // Regaining contact on the same runtime is neither a new connection nor a new session: the // generation holds so the session mirror is not rebuilt (#19647), and only the contact epoch - // — the mirror's resubscribe trigger — moves. No restart hook, no toast. + // — the mirror's resubscribe trigger — moves. Hosting is re-claimed, but no toast. expect(viewer.getState().runtimeStatusByEnvironmentId.get('env-a')?.connectionGeneration).toBe( generation ) @@ -92,7 +92,7 @@ it('represents failed verification honestly without manufacturing a session rest expect(viewer.getState().runtimeStatusByEnvironmentId.get('env-a')?.status?.runtimeId).toBe( 'rt-1' ) - expect(ensureBrowserClientHostForRestartedRuntime).not.toHaveBeenCalled() + expect(ensureBrowserClientHostOnRuntimeContact).toHaveBeenCalledTimes(1) expect(toast.warning).not.toHaveBeenCalled() const reconnectedGeneration = viewer .getState() @@ -104,7 +104,7 @@ it('represents failed verification honestly without manufacturing a session rest expect(viewer.getState().runtimeStatusByEnvironmentId.get('env-a')?.connectionGeneration).toBe( (reconnectedGeneration ?? 0) + 1 ) - expect(ensureBrowserClientHostForRestartedRuntime).toHaveBeenCalled() + expect(ensureBrowserClientHostOnRuntimeContact).toHaveBeenCalledTimes(2) }) it('retains disconnect ordering and rejects publications for removed or replaced pairings', () => { diff --git a/src/renderer/src/store/slices/runtime-status.ts b/src/renderer/src/store/slices/runtime-status.ts index ee61d2aa002..2d2512c4e41 100644 --- a/src/renderer/src/store/slices/runtime-status.ts +++ b/src/renderer/src/store/slices/runtime-status.ts @@ -21,7 +21,7 @@ import { refreshRuntimeEnvironmentStatus } from './runtime-status-refresh' import * as runtimeStatusConnectionGeneration from './runtime-status-connection-generation' import { replayClientHostedBrowserCloseIntents } from '@/runtime/client-hosted-browser-close-intent-replay' import { - ensureBrowserClientHostForRestartedRuntime, + ensureBrowserClientHostOnRuntimeContact, ensureBrowserClientHostsForRestoredPages } from '@/runtime/restored-client-hosted-browser-host-attach' import { applyRuntimeHostStatusSnapshot } from './runtime-status-snapshot' @@ -181,6 +181,9 @@ export const createRuntimeStatusSlice: StateCreator number + /** + * Takes the whole link down until `restore`. `silent` freezes open connections and holds new ones + * unanswered, like a sleeping laptop; `reset` closes everything and refuses new connections. + */ + cut: (mode: 'silent' | 'reset') => void + /** Ends a `cut`: drops connections that lived through it, as a returning network does, and relays new ones. */ + restore: () => void + close: () => Promise +} + +type ProxyPair = { client: net.Socket; upstream: net.Socket | null; frozen: boolean } + +/** A loopback TCP proxy whose established connections can go half-open while new ones still pass. */ +export async function startFreezableTcpProxy( + targetHost: string, + targetPort: number +): Promise { + const pairs = new Set() + const events: string[] = [] + let nextId = 0 + let cutMode: 'silent' | 'reset' | null = null + const destroyPair = (pair: ProxyPair): void => { + pair.client.destroy() + pair.upstream?.destroy() + pairs.delete(pair) + } + const server = net.createServer((client) => { + const id = nextId++ + events.push(`${Date.now()} open#${id}${cutMode ? ` (during ${cutMode} cut)` : ''}`) + client.on('close', () => events.push(`${Date.now()} close#${id}`)) + client.on('error', () => undefined) + if (cutMode === 'reset') { + client.destroy() + return + } + if (cutMode === 'silent') { + // Why held, not refused: an unreachable host answers nothing; the dial must time out on its own. + client.pause() + pairs.add({ client, upstream: null, frozen: true }) + return + } + const upstream = net.connect(targetPort, targetHost) + const pair: ProxyPair = { client, upstream, frozen: false } + pairs.add(pair) + client.pipe(upstream) + upstream.pipe(client) + const end = (): void => { + // Why: a frozen pair must not relay a close either; the far side would learn of the drop. + if (!pair.frozen) { + destroyPair(pair) + } + } + client.on('close', end) + upstream.on('close', end) + upstream.on('error', () => undefined) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + if (!address || typeof address === 'string') { + throw new Error('Proxy did not bind a TCP port') + } + const freezeExisting = (direction: 'both' | 'to-client' = 'both'): number => { + let frozen = 0 + for (const pair of pairs) { + if (pair.frozen || !pair.upstream) { + continue + } + pair.frozen = true + // Why one-way: requests still reach the host while its replies and publications vanish. + pair.upstream.unpipe(pair.client) + pair.upstream.pause() + if (direction === 'both') { + pair.client.unpipe(pair.upstream) + pair.client.pause() + } + frozen += 1 + } + return frozen + } + return { + port: address.port, + events, + freezeExisting, + cut: (mode) => { + cutMode = mode + events.push(`${Date.now()} cut:${mode}`) + if (mode === 'silent') { + freezeExisting('both') + return + } + for (const pair of pairs) { + destroyPair(pair) + } + }, + restore: () => { + cutMode = null + events.push(`${Date.now()} restore`) + for (const pair of pairs) { + if (pair.frozen) { + destroyPair(pair) + } + } + }, + close: async () => { + for (const pair of pairs) { + destroyPair(pair) + } + await new Promise((resolve) => server.close(() => resolve())) + } + } +} diff --git a/tests/e2e/paired-client-hosted-browser-network-outage.spec.ts b/tests/e2e/paired-client-hosted-browser-network-outage.spec.ts new file mode 100644 index 00000000000..c4b626ab529 --- /dev/null +++ b/tests/e2e/paired-client-hosted-browser-network-outage.spec.ts @@ -0,0 +1,183 @@ +import type { Page } from '@stablyai/playwright-test' +import { expect, test } from './helpers/orca-app' +import { launchHeadlessPairedRuntimeHost } from './helpers/headless-paired-runtime-host' +import { cleanupE2EDaemons } from './helpers/electron-process-shutdown' +import { + launchPairedElectronClient, + type PairedElectronClient +} from './helpers/paired-electron-client' +import { + findMirroredBrowserPage, + focusClientBrowserRow, + navigateGuest, + readClientBrowserRows, + openClientHostedFixturePage, + readClientWebviewMarker, + selectPairedWorktreeGroup, + startClientHostedMarkerFixture, + waitForPairedWorktreeId, + waitForRenderedClientWebview +} from './helpers/client-hosted-browser-fixture' +import { startFreezableTcpProxy, type FreezableTcpProxy } from './helpers/freezable-tcp-proxy' +import { decodePairingOffer, encodePairingOffer } from '../../src/shared/pairing' + +/** + * Long enough for the host to void the lease. A reset outage starts the 15s lease grace at once; a + * silent one first needs the host's heartbeat to reap the socket (three missed 15s probes). + */ +const OUTAGE_MS = { reset: 25_000, silent: 100_000 } as const + +async function readPageHostGeneration( + client: PairedElectronClient, + localPageId: string +): Promise { + return client.page.evaluate((pageId) => { + const placement = window.__store?.getState().remoteBrowserPageHandlesByPageId[pageId]?.placement + return placement?.kind === 'client' ? placement.pageHostGeneration : null + }, localPageId) +} + +/** Leaves the browser tab for the terminal and comes back, so the pane remounts its guest. */ +async function remountBrowserTab( + page: Page, + worktreeId: string, + localPageId: string +): Promise { + await page.evaluate((worktreeId) => { + const state = window.__store!.getState() + const terminal = (state.unifiedTabsByWorktree[worktreeId] ?? []).find( + (tab) => tab.contentType === 'terminal' + ) + if (terminal) { + state.activateTab(terminal.id) + } + }, worktreeId) + await page.waitForTimeout(1_000) + await focusClientBrowserRow(page, worktreeId, localPageId) + await page.waitForTimeout(3_000) + return page.getByText('Client-hosted browser unavailable').count() +} + +/** + * Neither Orca process restarts; only the network between them goes away for longer than the lease + * grace. When it returns, the same desktop must host its page again on its own, instead of leaving + * the tab on "Client-hosted browser unavailable … attached to a different desktop" for good. + */ +for (const mode of ['silent', 'reset'] as const) { + test(`the same desktop re-hosts its browser tab after a ${mode} network outage longer than the lease grace`, async ({ + testRepoPath + }, testInfo) => { + test.setTimeout(420_000) + const fixture = await startClientHostedMarkerFixture({ created: 'outage-survivor', moved: 'x' }) + const host = await launchHeadlessPairedRuntimeHost() + let client: PairedElectronClient | null = null + let proxy: FreezableTcpProxy | null = null + try { + await host.client.call('repo.add', { path: testRepoPath, kind: 'git' }) + const decoded = decodePairingOffer(host.offer.pairingUrl) + if (!('endpoint' in decoded)) { + throw new Error('expected a direct pairing offer') + } + const endpoint = new URL(decoded.endpoint) + proxy = await startFreezableTcpProxy(endpoint.hostname, Number(endpoint.port)) + endpoint.port = String(proxy.port) + client = await launchPairedElectronClient( + { pairingUrl: encodePairingOffer({ ...decoded, endpoint: endpoint.toString() }) }, + testInfo, + `client-hosted ${mode} outage` + ) + const worktreeId = await waitForPairedWorktreeId(client.page, testRepoPath) + await selectPairedWorktreeGroup(client.page, client.environmentId, worktreeId) + const opened = await openClientHostedFixturePage(client, worktreeId, fixture.markerUrl) + expect( + await waitForRenderedClientWebview( + client.page, + { urlPrefix: fixture.markerUrl, remotePageId: opened.remotePageId }, + 'client-hosted guest never rendered the fixture' + ) + ).toBe('outage-survivor') + const hostPid = host.app.process().pid + const generationBefore = await readPageHostGeneration(client, opened.localPageId) + expect(generationBefore).not.toBeNull() + + proxy.cut(mode) + await new Promise((resolve) => setTimeout(resolve, OUTAGE_MS[mode])) + proxy.restore() + + // Why the generation: only a newly minted placement proves the page was hosted again, rather + // than read before the voided lease's guest was torn down. + try { + await expect + .poll(() => readPageHostGeneration(client!, opened.localPageId), { + timeout: 90_000, + message: 'the same desktop never hosted its page again after the network returned' + }) + .not.toBe(generationBefore) + } catch (error) { + const evidence = { + generationBefore, + generationAfter: await readPageHostGeneration(client, opened.localPageId), + unavailableNotices: await client.page + .getByText('Client-hosted browser unavailable') + .count(), + liveGuestMarker: await readClientWebviewMarker(client.page, { + urlPrefix: fixture.markerUrl, + remotePageId: opened.remotePageId + }), + noticesAfterRemount: await remountBrowserTab(client.page, worktreeId, opened.localPageId), + clientRowsAfter: await readClientBrowserRows(client.page, worktreeId), + markerAfterNavigating: await navigateGuest( + client.page, + fixture.markerUrl, + fixture.movedUrl + ) + .then(() => client!.page.waitForTimeout(10_000)) + .then(() => + readClientWebviewMarker(client!.page, { + urlPrefix: fixture.origin, + remotePageId: opened.remotePageId + }) + ) + .catch((navigationError: unknown) => `navigation failed: ${String(navigationError)}`), + proxyEvents: proxy.events + } + console.log('[outage] evidence', JSON.stringify(evidence)) + testInfo.annotations.push({ + type: 'outage-evidence', + description: JSON.stringify(evidence) + }) + await client.page.screenshot({ path: testInfo.outputPath('after-outage.png') }) + throw error + } + expect( + await waitForRenderedClientWebview( + client.page, + { urlPrefix: fixture.markerUrl, remotePageId: opened.remotePageId }, + 'the re-hosted page never rendered' + ) + ).toBe('outage-survivor') + await expect(client.page.getByText('Client-hosted browser unavailable')).toHaveCount(0) + expect( + (await findMirroredBrowserPage(client.page, worktreeId, fixture.markerUrl))?.placementKind + ).toBe('client') + // The re-hosted guest must be a working page, not just a rendered one. + await navigateGuest(client.page, fixture.markerUrl, fixture.movedUrl) + expect( + await waitForRenderedClientWebview( + client.page, + { urlPrefix: fixture.movedUrl, remotePageId: opened.remotePageId }, + 'the re-hosted page never navigated' + ) + ).toBe('x') + expect(host.app.process().pid, 'the host must not have restarted').toBe(hostPid) + } finally { + if (client) { + await cleanupE2EDaemons(client.userDataDir).catch(() => undefined) + await client.dispose() + } + await proxy?.close() + await host.dispose() + await fixture.close() + } + }) +}