From eb0ec3924254500b443bdbc77aec2fa256178756 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 18 Aug 2026 00:35:16 -0700 Subject: [PATCH] fix(runtime): stop one unreachable relay from freezing every workspace as active (STA-517) (#14649) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(runtime): stop one unreachable relay from freezing every workspace as active The worktree.ps liveness refresh is the only thing that retires an exited PTY, and mobile renders "active" straight off the summary it produces. Its aggregate inventory ran every provider through Promise.all with no per-provider deadline, so a single SSH relay that rejected — or simply did not answer inside the 3s budget, since a relay list runs to the mux's own 30s default — cost the runtime the whole inventory. Nothing was ever proven dead, so every retained pane kept reporting hasHostSidebarActivity/liveTerminalCount, and the SSH workspaces stayed "active" on mobile for as long as the connection stayed unreachable. Settle each SSH provider independently and forward the caller's deadline, so local and healthy relays are still reconciled. A provider that does not answer is unknown, not empty: the runtime's existing hasPty rescue keeps its panes. A local failure still fails the aggregate, matching pty:listSessions. The restored-orchestration-authority sweep now runs after that rescue, so a pane the controller still vouches for keeps its handle instead of losing it to a listing that merely omitted it. STA-517 * test(runtime): assert the provider scope, not the exact arity, of inventory calls These assertions exist to prove which provider scope the inventory asked for. Forwarding the caller's deadline added a second argument, which broke them on arity alone. Match the scope argument and require a numeric deadline beside it, so the intent is preserved and the budget is covered too. * test(runtime): type the inventory mock's scope parameter A bare `async () =>` mock types mock.calls as an empty tuple, so reading the scope argument off it fails typecheck. Declare the parameter the runtime actually passes. --- .../pty-controller-process-inventory.test.ts | 195 ++++++++++++++++++ .../ipc/pty-runtime-kill-and-exit.test.ts | 3 + src/main/ipc/pty.ts | 23 ++- src/main/runtime/orca-runtime.test.ts | 25 ++- src/main/runtime/orca-runtime.ts | 58 ++++-- ...y-inventory-partial-relay-liveness.test.ts | 166 +++++++++++++++ 6 files changed, 432 insertions(+), 38 deletions(-) create mode 100644 src/main/ipc/pty-controller-process-inventory.test.ts create mode 100644 src/main/runtime/pty-inventory-partial-relay-liveness.test.ts diff --git a/src/main/ipc/pty-controller-process-inventory.test.ts b/src/main/ipc/pty-controller-process-inventory.test.ts new file mode 100644 index 00000000000..e89d888978a --- /dev/null +++ b/src/main/ipc/pty-controller-process-inventory.test.ts @@ -0,0 +1,195 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +const { handleMock, onMock, removeHandlerMock, removeAllListenersMock } = vi.hoisted(() => ({ + handleMock: vi.fn(), + onMock: vi.fn(), + removeHandlerMock: vi.fn(), + removeAllListenersMock: vi.fn() +})) + +vi.mock('electron', () => ({ + app: { + isPackaged: true, + getPath: vi.fn().mockReturnValue('/tmp/orca-test-userdata') + }, + ipcMain: { + handle: handleMock, + on: onMock, + removeHandler: removeHandlerMock, + removeAllListeners: removeAllListenersMock + }, + powerMonitor: { + on: vi.fn() + } +})) + +vi.mock('fs', () => ({ + existsSync: () => true, + statSync: () => ({ isDirectory: () => true, mode: 0o755 }), + accessSync: () => undefined, + mkdirSync: vi.fn(), + readFileSync: vi.fn(() => ''), + writeFileSync: vi.fn(), + chmodSync: vi.fn(), + constants: { X_OK: 1 } +})) + +vi.mock('node-pty', () => ({ + spawn: vi.fn().mockReturnValue({ + onData: vi.fn(), + onExit: vi.fn(), + write: vi.fn(), + resize: vi.fn(), + kill: vi.fn(), + process: 'zsh', + pid: 12345 + }) +})) + +vi.mock('../opencode/hook-service', () => ({ + openCodeHookService: { buildPtyEnv: () => ({}), clearPty: vi.fn() } +})) + +vi.mock('../pi/titlebar-extension-service', () => ({ + piTitlebarExtensionService: { buildPtyEnv: () => ({}), clearPty: vi.fn() } +})) + +import { + registerPtyHandlers, + registerSshPtyProvider, + setLocalPtyProvider, + unregisterSshPtyProvider +} from './pty' +import type { IPtyProvider, PtyProcessInfo } from '../providers/types' + +// The runtime's worktree.ps liveness refresh calls the aggregate inventory (no connectionId) +// under a 3s budget, and only a returned inventory can retire an exited PTY. STA-517: one +// unreachable relay made the whole aggregate fail, so no PTY was ever proven dead and every +// retained pane — the SSH ones above all — kept reporting "active" to mobile indefinitely. + +type ListCall = { opts: { deadlineMs?: number } | undefined } + +function createProvider( + sessions: PtyProcessInfo[], + behavior: 'ok' | 'reject' = 'ok' +): { provider: IPtyProvider; calls: ListCall[] } { + const calls: ListCall[] = [] + const provider = { + onData: vi.fn().mockReturnValue(() => {}), + onRejectedData: vi.fn().mockReturnValue(() => {}), + onReplay: vi.fn().mockReturnValue(() => {}), + onExit: vi.fn().mockReturnValue(() => {}), + listProcesses: vi.fn(async (opts?: { deadlineMs?: number }) => { + calls.push({ opts }) + if (behavior === 'reject') { + throw new Error('relay unreachable') + } + return sessions + }) + } as unknown as IPtyProvider + return { provider, calls } +} + +function session(id: string): PtyProcessInfo { + return { id, cwd: '/tmp', title: id } as unknown as PtyProcessInfo +} + +const mainWindow = { + isDestroyed: () => false, + webContents: { on: vi.fn(), send: vi.fn(), removeListener: vi.fn() } +} + +function captureController(): { + listProcesses: ( + connectionId?: string | null, + opts?: { deadlineMs?: number } + ) => Promise +} { + handleMock.mockReset() + onMock.mockReset() + handleMock.mockImplementation(() => {}) + onMock.mockImplementation(() => {}) + let controller: { listProcesses?: unknown } | undefined + const runtime = { + setPtyController: vi.fn((next: { listProcesses?: unknown }) => { + controller = next + }), + createPreAllocatedTerminalHandle: vi.fn(() => 'term_test'), + registerPreAllocatedHandleForPty: vi.fn(), + registerPty: vi.fn() + } + registerPtyHandlers(mainWindow as never, runtime as never) + if (typeof controller?.listProcesses !== 'function') { + throw new Error('PTY controller listProcesses was not registered') + } + return controller as never +} + +describe('aggregate PTY process inventory', () => { + const registered: string[] = [] + + function register(connectionId: string, provider: IPtyProvider): void { + registerSshPtyProvider(connectionId, provider) + registered.push(connectionId) + } + + afterEach(() => { + for (const connectionId of registered.splice(0)) { + unregisterSshPtyProvider(connectionId) + } + }) + + it('still reports local and healthy relays when one SSH relay rejects', async () => { + const local = createProvider([session('local-pty')]) + const healthy = createProvider([session('ssh:conn-ok@@pty')]) + const broken = createProvider([], 'reject') + setLocalPtyProvider(local.provider) + register('conn-ok', healthy.provider) + register('conn-broken', broken.provider) + const controller = captureController() + + const sessions = await controller.listProcesses() + + // Pre-fix this rejected: Promise.all surfaced the broken relay's error, the runtime + // read it as "no inventory", and no PTY anywhere was retired. + expect(sessions.map((entry) => entry.id).sort()).toEqual(['local-pty', 'ssh:conn-ok@@pty']) + }) + + it('bounds every relay list by the caller deadline instead of the mux default', async () => { + const local = createProvider([session('local-pty')]) + const remote = createProvider([session('ssh:conn-a@@pty')]) + setLocalPtyProvider(local.provider) + register('conn-a', remote.provider) + const controller = captureController() + const deadlineMs = Date.now() + 2500 + + await controller.listProcesses(undefined, { deadlineMs }) + + // Without a forwarded deadline an unanswered relay list runs to the SSH mux's own + // 30s default, far past the runtime's 3s budget for the whole refresh. + expect(remote.calls).toEqual([{ opts: { deadlineMs } }]) + }) + + it('forwards the caller deadline on a targeted single-connection list', async () => { + const local = createProvider([session('local-pty')]) + const remote = createProvider([session('ssh:conn-a@@pty')]) + setLocalPtyProvider(local.provider) + register('conn-a', remote.provider) + const controller = captureController() + const deadlineMs = Date.now() + 1200 + + await controller.listProcesses('conn-a', { deadlineMs }) + + expect(remote.calls).toEqual([{ opts: { deadlineMs } }]) + }) + + it('fails the aggregate when the local provider cannot list', async () => { + const local = createProvider([], 'reject') + setLocalPtyProvider(local.provider) + const controller = captureController() + + // A local failure is a real controller fault, not one unreachable host: the runtime must + // keep treating it as "no inventory" rather than proving every local PTY dead. + await expect(controller.listProcesses()).rejects.toThrow('relay unreachable') + }) +}) diff --git a/src/main/ipc/pty-runtime-kill-and-exit.test.ts b/src/main/ipc/pty-runtime-kill-and-exit.test.ts index 2da8b80d4b8..c4dec001542 100644 --- a/src/main/ipc/pty-runtime-kill-and-exit.test.ts +++ b/src/main/ipc/pty-runtime-kill-and-exit.test.ts @@ -126,6 +126,9 @@ describe('registerPtyHandlers', () => { expect(sshAList).toHaveBeenCalledOnce() expect(sshBList).not.toHaveBeenCalled() + // STA-517: the aggregate used to propagate ssh-b's failure, which cost the runtime the + // whole liveness inventory — so no PTY was ever proven dead and mobile kept every + // retained pane "active". One unreachable relay now drops out of the answer instead. await expect(controller.listProcesses()).resolves.toEqual([ { id: 'local-pty', title: 'Local', cwd: '/local' }, { id: 'ssh-a-pty' } diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 0414cd514fa..ad1a4fb544c 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -269,8 +269,15 @@ function registeredPtyProviders(): RegisteredPtyProvider[] { ] } +// Why: settling each provider separately only bounds a relay that *rejects*. An +// unanswered relay list runs to the mux's own 30s default — far past the caller's +// budget — so the aggregate still expired and the runtime lost the inventory it +// needs to retire exited PTYs, freezing every retained pane as "active" (STA-517). +// Forward the caller's deadline so a silent relay fails fast and lands in the +// unavailable branch below: unknown, not empty. async function listRegisteredPtyProcessesWithHostScope( - onSshInventoryUnavailable?: (connectionId: string, error: unknown) => void + onSshInventoryUnavailable?: (connectionId: string, error: unknown) => void, + opts?: { deadlineMs?: number } ): Promise<{ processes: PtyProcessInfo[] hostIds: ExecutionHostId[] @@ -283,7 +290,8 @@ async function listRegisteredPtyProcessesWithHostScope( ? toSshExecutionHostId(connectionId) : LOCAL_EXECUTION_HOST_ID return { - processes: await provider.listProcesses(), + // Why: the deadline only applies to relay round-trips; the local provider answers in-process. + processes: await (connectionId ? provider.listProcesses(opts) : provider.listProcesses()), hostId } } catch (error) { @@ -5909,22 +5917,23 @@ export function registerPtyHandlers( return null } }, - listProcesses: async (connectionId) => { + listProcesses: async (connectionId, opts) => { if (connectionId === null) { return localProvider.listProcesses() } if (connectionId !== undefined) { try { - return await getProvider(connectionId).listProcesses() + return await getProvider(connectionId).listProcesses(opts) } catch (error) { markSshInventoryUnverifiable(connectionId, error) throw error } } - return (await listRegisteredPtyProcessesWithHostScope(markSshInventoryUnverifiable)).processes + return (await listRegisteredPtyProcessesWithHostScope(markSshInventoryUnverifiable, opts)) + .processes }, - listProcessesWithHostScope: () => - listRegisteredPtyProcessesWithHostScope(markSshInventoryUnverifiable), + listProcessesWithHostScope: (opts) => + listRegisteredPtyProcessesWithHostScope(markSshInventoryUnverifiable, opts), serializeBuffer: (ptyId, opts) => { // Why: mobile xterm must start from the desktop's exact screen state/dimensions before live TUI chunks render correctly. return requestSerializedBuffer(ptyId, opts) diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 122300b514a..49f1fbac345 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -1004,6 +1004,9 @@ async function referenceStatusFrameLines( } const TEST_WINDOW_ID = 1 +// The inventory refresh forwards its own budget so a relay cannot outlive it (STA-517). +// These assertions are about which provider scope was asked, so the deadline stays loose. +const LIST_PROVIDER_DEADLINE = expect.objectContaining({ deadlineMs: expect.any(Number) }) const TEST_REPO_ID = 'repo-1' const TEST_REPO_PATH = '/tmp/repo' const TEST_WORKTREE_PATH = '/tmp/worktree-a' @@ -20306,7 +20309,7 @@ describe('OrcaRuntimeService', () => { paneKey: makePaneKey('tab-agent', HEADLESS_LEAF_ID) }) expect(listProcesses).toHaveBeenCalledTimes(inventoryCount + 1) - expect(listProcesses).toHaveBeenLastCalledWith(null) + expect(listProcesses).toHaveBeenLastCalledWith(null, LIST_PROVIDER_DEADLINE) expect( (await runtime.listTerminals()).terminals.find((terminal) => terminal.ptyId === 'pty-agent') ).toMatchObject({ @@ -20316,7 +20319,7 @@ describe('OrcaRuntimeService', () => { leafId: HEADLESS_LEAF_ID }) expect(listProcesses).toHaveBeenCalledTimes(inventoryCount + 2) - expect(listProcesses).toHaveBeenLastCalledWith(undefined) + expect(listProcesses).toHaveBeenLastCalledWith(undefined, LIST_PROVIDER_DEADLINE) await expect( runtime.adoptTerminalOrphans({ worktree: `id:${TEST_WORKTREE_ID}`, @@ -21582,7 +21585,7 @@ describe('OrcaRuntimeService', () => { expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() expect(getSession().sleepingAgentSessionsByPaneKey?.[secondWorkerPaneKey]).toBeUndefined() expect(listProcesses).toHaveBeenCalledTimes(3) - expect(listProcesses).toHaveBeenCalledWith(null) + expect(listProcesses).toHaveBeenCalledWith(null, LIST_PROVIDER_DEADLINE) expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([ expect.objectContaining({ id: 'legacy-worker-two', ptyId: 'pty-exited-two' }) ]) @@ -21697,7 +21700,7 @@ describe('OrcaRuntimeService', () => { await vi.advanceTimersByTimeAsync(1_000) expect(listProcesses).toHaveBeenCalledTimes(4) - expect(listProcesses.mock.calls).toEqual([[null], [null], [null], [null]]) + expect(listProcesses.mock.calls.map((call) => call[0])).toEqual([null, null, null, null]) expect(hasPty).not.toHaveBeenCalled() expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([ expect.objectContaining({ id: 'legacy-worker', ptyId: 'pty-inventory-unavailable' }) @@ -21888,7 +21891,7 @@ describe('OrcaRuntimeService', () => { deferredDispatchIds: ['dispatch-missing', 'dispatch-ambiguous'] }) expect(listProcesses).toHaveBeenCalledOnce() - expect(listProcesses).toHaveBeenCalledWith(null) + expect(listProcesses).toHaveBeenCalledWith(null, LIST_PROVIDER_DEADLINE) for (const { name, leafId } of cases.slice(0, 2)) { expect( getSession().sleepingAgentSessionsByPaneKey?.[`legacy-${name}:${leafId}`] @@ -21998,7 +22001,7 @@ describe('OrcaRuntimeService', () => { ) expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() expect(listProcesses).toHaveBeenCalledTimes(3) - expect(listProcesses).toHaveBeenCalledWith(null) + expect(listProcesses).toHaveBeenCalledWith(null, LIST_PROVIDER_DEADLINE) expect(revealTerminalSession).toHaveBeenCalledWith(TEST_FOLDER_WORKSPACE_KEY, { ptyId: 'pty-folder-legacy', title: 'Folder worker', @@ -22140,7 +22143,7 @@ describe('OrcaRuntimeService', () => { expect(getWorkspaceSession).toHaveBeenCalledWith(`ssh:${connectionId}`) expect(setWorkspaceSession).toHaveBeenCalledWith(expect.any(Object), `ssh:${connectionId}`) expect(listProcesses).toHaveBeenCalledTimes(3) - expect(listProcesses).toHaveBeenCalledWith(connectionId) + expect(listProcesses).toHaveBeenCalledWith(connectionId, LIST_PROVIDER_DEADLINE) expect(sshSession.tabsByWorktree[TEST_FOLDER_WORKSPACE_KEY]).toContainEqual( expect.objectContaining({ id: 'legacy-ssh-folder-worker', @@ -22369,7 +22372,7 @@ describe('OrcaRuntimeService', () => { exitedDispatchIds: [], deferredDispatchIds: [] }) - expect(listProcesses).toHaveBeenLastCalledWith(connectionId) + expect(listProcesses).toHaveBeenLastCalledWith(connectionId, LIST_PROVIDER_DEADLINE) } finally { unregisterSshGitProvider(connectionId) } @@ -22455,7 +22458,9 @@ describe('OrcaRuntimeService', () => { } ] } as unknown as OrchestrationDb) - const listProcesses = vi.fn(async () => [ + // Declares the scope parameter so mock.calls keeps it — the runtime passes a deadline + // alongside it, and a bare `async () =>` would type the call tuple as empty. + const listProcesses = vi.fn(async (_connectionId?: string | null) => [ { id: 'pty-wsl-legacy', incarnationId, @@ -22506,7 +22511,7 @@ describe('OrcaRuntimeService', () => { expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() expect(revealTerminalSession).toHaveBeenCalledOnce() expect(listProcesses).toHaveBeenCalledTimes(5) - expect(listProcesses.mock.calls).toEqual([[null], [null], [null], [null], [null]]) + expect(listProcesses.mock.calls.map((call) => call[0])).toEqual([null, null, null, null, null]) }) it('restores orphan pane and group topology without replacing a newer host-owned tab', async () => { diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 3872f25d03e..1fb99bdb02b 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -1911,8 +1911,13 @@ type RuntimePtyController = { resize?(ptyId: string, cols: number, rows: number): boolean // Why: exact-id mobile polls should not enumerate every local and SSH PTY. hasPty?(ptyId: string): boolean | null - listProcesses?(connectionId?: string | null): Promise - listProcessesWithHostScope?(): Promise<{ + // Why: the caller's budget has to reach the relay. Without it an SSH list runs to + // the mux's own 30s default and blows every inventory refresh (STA-517). + listProcesses?( + connectionId?: string | null, + opts?: { deadlineMs?: number } + ): Promise + listProcessesWithHostScope?(opts?: { deadlineMs?: number }): Promise<{ processes: PtyProcessInfo[] hostIds: ExecutionHostId[] }> @@ -32038,10 +32043,20 @@ export class OrcaRuntimeService { } else { this.ptyControllerInventoryGenerationByProvider.set(providerKey, inventoryGeneration) } + const listBudgetMs = + deadline === undefined + ? PTY_CONTROLLER_LIST_TIMEOUT_MS + : Math.max(1, Math.min(PTY_CONTROLLER_LIST_TIMEOUT_MS, deadline - Date.now())) + // Why: give each provider a deadline strictly inside our own, so a relay that + // never answers still leaves the aggregate time to return the providers that did + // — expiring at the same instant would discard the whole inventory instead. + const providerListOpts = { + deadlineMs: Date.now() + Math.max(1, listBudgetMs - PTY_CONTROLLER_LIST_PROVIDER_MARGIN_MS) + } const processInventory = connectionId === undefined && this.ptyController.listProcessesWithHostScope - ? this.ptyController.listProcessesWithHostScope() - : this.ptyController.listProcesses(connectionId).then((processes) => { + ? this.ptyController.listProcessesWithHostScope(providerListOpts) + : this.ptyController.listProcesses(connectionId, providerListOpts).then((processes) => { const hostIds = new Set() if (connectionId === undefined || connectionId === null) { hostIds.add(LOCAL_EXECUTION_HOST_ID) @@ -32062,12 +32077,7 @@ export class OrcaRuntimeService { } return { processes, hostIds: [...hostIds] } }) - const sessionsResult = await withTimeoutResult( - processInventory, - deadline === undefined - ? PTY_CONTROLLER_LIST_TIMEOUT_MS - : Math.max(1, Math.min(PTY_CONTROLLER_LIST_TIMEOUT_MS, deadline - Date.now())) - ) + const sessionsResult = await withTimeoutResult(processInventory, listBudgetMs) if (!sessionsResult.ok) { // Why: a transient controller failure is not evidence that retained PTYs exited. return null @@ -32229,17 +32239,6 @@ export class OrcaRuntimeService { // Why: fire-and-forget so this listing hot path doesn't serialize a relay round-trip per session and a throw can't abort the sweep below. this.refreshPtyForegroundAgent(session.id) } - for (const [ptyId, receipt] of this.restoredOrchestrationAuthorityByPtyId) { - const inScope = - connectionId === undefined || - (connectionId === null && receipt.hostScope.kind !== 'ssh') || - (typeof connectionId === 'string' && - receipt.hostScope.kind === 'ssh' && - receipt.hostScope.targetId === connectionId) - if (inScope && !allLivePtyIds.has(ptyId)) { - this.restoredOrchestrationAuthorityByPtyId.delete(ptyId) - } - } for (const pty of this.ptysById.values()) { if (connectionId !== undefined && pty.connectionId !== connectionId) { continue @@ -32282,6 +32281,20 @@ export class OrcaRuntimeService { } } } + // Why: runs after the hasPty rescue so a still-addressable pane keeps its receipt. + // A provider that failed to list is absent from `sessions`, and dropping authority on + // that silence would retire an orchestration handle the relay can still reach. + for (const [ptyId, receipt] of this.restoredOrchestrationAuthorityByPtyId) { + const inScope = + connectionId === undefined || + (connectionId === null && receipt.hostScope.kind !== 'ssh') || + (typeof connectionId === 'string' && + receipt.hostScope.kind === 'ssh' && + receipt.hostScope.targetId === connectionId) + if (inScope && !allLivePtyIds.has(ptyId)) { + this.restoredOrchestrationAuthorityByPtyId.delete(ptyId) + } + } this.pruneDisconnectedPtyRecords() return { livePtyIds: targetWorktreeId ? selectedLivePtyIds : allLivePtyIds, @@ -38059,6 +38072,9 @@ export function resolveWorktreeScanCacheTtlMs(repo: Pick unknown + refreshPtyWorktreeRecordsWithControllerInventory: ( + resolvedWorktrees: unknown[], + targetWorktreeId?: string | null, + deadline?: number + ) => Promise + recordPtyWorktree: ( + ptyId: string, + worktreeId: string, + state?: Record + ) => Record + ptysById: Map + restoredOrchestrationAuthorityByPtyId: Map +} + +type ListCall = { connectionId: string | null | undefined; deadlineMs: number | undefined } + +function createRuntime(options: { sessions?: unknown[]; vouchesForRetainedPty?: boolean } = {}): { + internals: RuntimeInternals + calls: ListCall[] +} { + const meta: Record> = { [WORKSPACE]: { hostId: 'local' } } + const store = { + getRepos: () => [REPO], + getRepo: (id: string) => (id === REPO_ID ? REPO : undefined), + getAllWorktreeMeta: () => meta, + getWorktreeMeta: (worktreeId: string) => meta[worktreeId], + setWorktreeMeta: (worktreeId: string, patch: Record) => { + meta[worktreeId] = { ...meta[worktreeId], ...patch } + return meta[worktreeId] + }, + getWorkspaceSession: () => getDefaultWorkspaceSession(), + setWorkspaceSession: () => {}, + flushOrThrow: () => {} + } as never + const calls: ListCall[] = [] + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + // Why: the controller's own liveness vouch, which the runtime consults for a PTY the + // listing omitted. A relay that failed to list still owns its panes. + hasPty: (ptyId: string) => + options.vouchesForRetainedPty && ptyId === RETAINED_PTY ? true : null, + listProcesses: async (connectionId?: string | null, opts?: { deadlineMs?: number }) => { + calls.push({ connectionId, deadlineMs: opts?.deadlineMs }) + return options.sessions ?? [] + } + } as never) + return { internals: runtime as unknown as RuntimeInternals, calls } +} + +describe('pty inventory refresh against a partially answering relay set', () => { + it('gives the providers a deadline strictly inside its own list budget', async () => { + const { internals, calls } = createRuntime() + const before = Date.now() + + await internals.refreshPtyWorktreeRecordsWithControllerInventory([ + internals.buildResolvedWorktreeFromId(WORKSPACE) + ]) + + expect(calls).toHaveLength(1) + const { deadlineMs } = calls[0]! + // Unbounded, an SSH list runs to the mux's 30s default and the whole refresh expires, so + // no inventory ever arrives and nothing is retired. + expect(deadlineMs).toBeDefined() + expect(deadlineMs!).toBeGreaterThan(before) + expect(deadlineMs!).toBeLessThan(before + LIST_BUDGET_MS) + }) + + it('honours a caller deadline tighter than the list budget', async () => { + const { internals, calls } = createRuntime() + const callerDeadline = Date.now() + 400 + + await internals.refreshPtyWorktreeRecordsWithControllerInventory( + [internals.buildResolvedWorktreeFromId(WORKSPACE)], + null, + callerDeadline + ) + + expect(calls[0]!.deadlineMs!).toBeLessThanOrEqual(callerDeadline) + }) + + it('keeps orchestration authority for a pane the controller still vouches for', async () => { + const { internals } = createRuntime({ vouchesForRetainedPty: true }) + internals.recordPtyWorktree(RETAINED_PTY, WORKSPACE, { + connected: true, + tabId: TAB_ID, + paneKey: PANE_KEY + }) + internals.restoredOrchestrationAuthorityByPtyId.set(RETAINED_PTY, { + ptyId: RETAINED_PTY, + worktreeId: WORKSPACE, + terminalHandle: 'term_retained', + paneKey: PANE_KEY, + processIncarnation: `${RETAINED_PTY}:inc-1`, + hostScope: { kind: 'local' } + }) + + await internals.refreshPtyWorktreeRecordsWithControllerInventory([ + internals.buildResolvedWorktreeFromId(WORKSPACE) + ]) + + // A listing that omits a still-addressable pane is silence, not proof of exit: the + // authority sweep has to read the rescued live set, not the raw listing. + expect(internals.restoredOrchestrationAuthorityByPtyId.has(RETAINED_PTY)).toBe(true) + expect(internals.ptysById.get(RETAINED_PTY)?.connected).toBe(true) + }) + + it('retires a pane no provider vouches for', async () => { + const { internals } = createRuntime({ vouchesForRetainedPty: false }) + internals.recordPtyWorktree(RETAINED_PTY, WORKSPACE, { + connected: true, + tabId: TAB_ID, + paneKey: PANE_KEY + }) + internals.restoredOrchestrationAuthorityByPtyId.set(RETAINED_PTY, { + ptyId: RETAINED_PTY, + worktreeId: WORKSPACE, + terminalHandle: 'term_retained', + paneKey: PANE_KEY, + processIncarnation: `${RETAINED_PTY}:inc-1`, + hostScope: { kind: 'local' } + }) + + await internals.refreshPtyWorktreeRecordsWithControllerInventory([ + internals.buildResolvedWorktreeFromId(WORKSPACE) + ]) + + // The other half of the contract: an answered inventory that omits an unvouched pane is + // what lets the workspace stop reporting itself active on mobile. + expect(internals.ptysById.get(RETAINED_PTY)?.connected).toBe(false) + expect(internals.restoredOrchestrationAuthorityByPtyId.has(RETAINED_PTY)).toBe(false) + }) +})