Merge remote-tracking branch 'origin/main' into brennanb2025/explain-orca-crash-restart

This commit is contained in:
Brennan Benson
2026-08-04 02:11:29 -07:00
52 changed files with 4218 additions and 372 deletions
+28 -3
View File
@@ -26,6 +26,23 @@ const PLAIN_NODE_ENTRY_NAMES = [
'codex/codex-app-server-grant-entry'
] as const
// Entries executed as worker threads of the main process. Electron's module is
// not registered on worker threads, so require("electron") throws
// "Cannot find module 'electron'" there too (verified on Electron 43) and kills
// the worker at startup. These carry hand-written "must stay electron-free"
// comments, which is convention, not enforcement — and the port-scan worker in
// particular sits one import away from a client module that deliberately does
// require electron.
const WORKER_THREAD_ENTRY_NAMES = [
'stt-worker',
'warp-theme-parser-worker',
'session-scanner-opencode-sqlite-worker-entry',
'main-thread-hang-watchdog-entry',
'port-scan-command-worker-entry'
] as const
type EntryRuntime = 'plain-Node process' | 'worker thread'
const ELECTRON_REQUIRE_RE = /require\(\s*["']electron["']\s*\)/
function collectReachableChunks(
@@ -56,13 +73,14 @@ function collectReachableChunks(
function assertNoElectronRequire(
entryName: string,
entry: OutputChunk,
byFileName: Map<string, OutputChunk>
byFileName: Map<string, OutputChunk>,
runtime: EntryRuntime = 'plain-Node process'
): void {
for (const chunk of collectReachableChunks(entry, byFileName)) {
if (ELECTRON_REQUIRE_RE.test(chunk.code)) {
throw new Error(
`[plain-node-entry-guard] "${entryName}" reaches chunk "${chunk.fileName}" that ` +
`requires electron. "${entryName}" runs as a plain-Node process, where ` +
`requires electron. "${entryName}" runs as a ${runtime}, where ` +
`require("electron") throws MODULE_NOT_FOUND and kills it at startup (the ` +
`v1.4.129-rc.1 daemon outage). Keep electron imports out of its module graph.`
)
@@ -125,7 +143,14 @@ export function createPlainNodeEntryGuardPlugin(): Plugin {
for (const entryName of PLAIN_NODE_ENTRY_NAMES) {
const entry = entryByName.get(entryName)
if (entry) {
assertNoElectronRequire(entryName, entry, byFileName)
assertNoElectronRequire(entryName, entry, byFileName, 'plain-Node process')
}
}
for (const entryName of WORKER_THREAD_ENTRY_NAMES) {
const entry = entryByName.get(entryName)
if (entry) {
assertNoElectronRequire(entryName, entry, byFileName, 'worker thread')
}
}
+1
View File
@@ -10,6 +10,7 @@
"src/main/speech/stt-worker.ts",
"src/main/warp-themes/warp-theme-parser-worker.ts",
"src/main/ai-vault/session-scanner-opencode-sqlite-worker-entry.ts",
"src/main/ports/port-scan-command-worker-entry.ts",
"src/main/ipc/parcel-watcher-process-entry.ts",
"src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts",
"src/main/codex/codex-app-server-grant-entry.ts",
@@ -14,6 +14,21 @@ import { describe, expect, it } from 'vitest'
const itCrossHost = process.platform === 'win32' ? it.skip : it
const projectRoot = resolve(import.meta.dirname, '../..')
const WINDOWS_LOCK_CODES = ['EBUSY', 'ENOTEMPTY', 'EPERM']
// Why: Windows releases the image handle on a just-executed exe (and finishes the
// AV scan of the freshly compiled one) after the process exits, so tearing down the
// fixture races those locks. Retry, then leave the temp tree rather than reporting a
// teardown lock as a launcher failure.
function removeFixtureTree(path) {
try {
rmSync(path, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 })
} catch (error) {
if (process.platform !== 'win32' || !WINDOWS_LOCK_CODES.includes(error?.code)) {
throw error
}
}
}
// Why: cold csc.exe startup exceeds Vitest's 5s unit budget on hosted Windows;
// keep the larger allowance scoped to the real compiler integration test.
function itWindows(name, test) {
@@ -35,7 +50,7 @@ describe('Windows CLI launcher', () => {
expect(result.stderr).toContain('Windows CLI launcher')
expect(result.stderr).toContain('Windows host')
} finally {
rmSync(outputRoot, { recursive: true, force: true })
removeFixtureTree(outputRoot)
}
})
@@ -108,7 +123,7 @@ describe('Windows CLI launcher', () => {
orcaNodeOptions: '--no-warnings'
})
} finally {
rmSync(appRoot, { recursive: true, force: true })
removeFixtureTree(appRoot)
}
})
@@ -161,7 +176,7 @@ describe('Windows CLI launcher', () => {
pathKeys: ['PATH', 'Path']
})
} finally {
rmSync(appRoot, { recursive: true, force: true })
removeFixtureTree(appRoot)
}
})
})
@@ -85,3 +85,91 @@ describe('plain Node entry guard', () => {
)
})
})
// Why (#11161): Electron's module is not registered on worker threads either —
// require("electron") throws "Cannot find module 'electron'" inside a
// main-process worker and kills it at startup. The worker entries carried only
// hand-written "must stay electron-free" comments, and the port-scan worker sits
// one import away from a client that deliberately does require electron.
describe('worker thread entry guard', () => {
function runWorkerWriteBundle(plugin: Plugin, bundle: Rollup.OutputBundle): void {
const hook = plugin.writeBundle
if (typeof hook !== 'function') {
throw new Error('Expected writeBundle hook')
}
hook.call(
{ meta: { watchMode: false } } as never,
{ dir: createOutputDir() } as Rollup.NormalizedOutputOptions,
bundle
)
}
function workerChunk(name: string, code: string, imports: string[] = []): Rollup.OutputChunk {
return {
type: 'chunk',
code,
dynamicImports: [],
fileName: `${name}.js`,
imports,
isEntry: true,
name
} as Rollup.OutputChunk
}
it('rejects an Electron require reachable from a worker entry', () => {
const plugin = createPlainNodeEntryGuardPlugin()
const bundle = {
'port-scan-command-worker-entry.js': workerChunk(
'port-scan-command-worker-entry',
'require("electron")'
)
} as Rollup.OutputBundle
expect(() => runWorkerWriteBundle(plugin, bundle)).toThrow('requires electron')
})
it('names the worker-thread runtime so the failure is actionable', () => {
const plugin = createPlainNodeEntryGuardPlugin()
const bundle = {
'stt-worker.js': workerChunk('stt-worker', 'require("electron")')
} as Rollup.OutputBundle
expect(() => runWorkerWriteBundle(plugin, bundle)).toThrow('runs as a worker thread')
})
// The real risk is transitive: a worker entry importing a shared chunk that
// reaches the electron-requiring client, not a direct import anyone would spot.
it('follows shared chunks out of a worker entry', () => {
const plugin = createPlainNodeEntryGuardPlugin()
const bundle = {
'session-scanner-opencode-sqlite-worker-entry.js': workerChunk(
'session-scanner-opencode-sqlite-worker-entry',
'require("./chunks/shared.js")',
['chunks/shared.js']
),
'chunks/shared.js': {
type: 'chunk',
code: 'require("electron")',
dynamicImports: [],
fileName: 'chunks/shared.js',
imports: [],
isEntry: false,
name: 'shared'
} as Rollup.OutputChunk
} as Rollup.OutputBundle
expect(() => runWorkerWriteBundle(plugin, bundle)).toThrow('chunks/shared.js')
})
it('passes a clean worker entry', () => {
const plugin = createPlainNodeEntryGuardPlugin()
const bundle = {
'warp-theme-parser-worker.js': workerChunk(
'warp-theme-parser-worker',
'require("node:worker_threads")'
)
} as Rollup.OutputBundle
expect(() => runWorkerWriteBundle(plugin, bundle)).not.toThrow()
})
})
+5
View File
@@ -216,6 +216,11 @@ export const electronViteConfig: UserConfig = {
'session-scanner-opencode-sqlite-worker-entry': resolve(
'src/main/ai-vault/session-scanner-opencode-sqlite-worker-entry.ts'
),
// Why: libuv spawns processes inline on the calling loop, so the port
// scan's probe commands run on a worker thread instead of the UI one.
'port-scan-command-worker-entry': resolve(
'src/main/ports/port-scan-command-worker-entry.ts'
),
// Why: forked with ELECTRON_RUN_AS_NODE so @parcel/watcher faults
// can't take down the main process (issue #7547).
'parcel-watcher-process-entry': resolve('src/main/ipc/parcel-watcher-process-entry.ts'),
+4 -2
View File
@@ -253,7 +253,8 @@ import { buildHeadlessAutomationWorktreeCreateArgs } from './automations/headles
import { AgentAwakeService } from './agent-awake-service'
import { registerSystemResumeBroadcast } from './system-resume-broadcast'
import { settleTeardownWithinDeadline } from './quit-teardown-deadline'
import { QuitTeardownStartGate } from './quit-teardown-start-gate'
import { quitTeardownStartGate } from './quit-teardown-start-gate'
import { detachAllSshSessionsForShutdown } from './ipc/ssh'
import { PluginService } from './plugins/plugin-service'
import { PluginKillListService } from './plugins/plugin-kill-list-service'
import { getPluginsDataDir } from './plugins/plugin-discovery'
@@ -2978,7 +2979,6 @@ app.on('before-quit', () => {
// Why: will-quit fires twice — first pass preventDefaults and runs teardown; second pass exits.
let daemonDisconnectDone = false
const quitTeardownStartGate = new QuitTeardownStartGate()
app.on('will-quit', (e) => {
// Why return instead of re-running teardown: the second pass is Electron re-firing after
// our own app.quit(), so every step below already ran and every durable write already
@@ -3030,6 +3030,7 @@ app.on('will-quit', (e) => {
runtime?.getOffscreenBrowserBackend()?.destroyAll?.()
browserManager.setBrowserGuestStateChangedListener(null)
const emulatorShutdown = runtime?.getEmulatorBridge()?.destroyAllSessions() ?? Promise.resolve()
const sshShutdown = detachAllSshSessionsForShutdown()
killAllPty()
const watcherShutdown = shutdownWatchersOnce()
const storeFlush = store?.flushAsync() ?? Promise.resolve()
@@ -3072,6 +3073,7 @@ app.on('will-quit', (e) => {
{ name: 'runtime-rpc', promise: rpcStopAndClear },
{ name: 'watchers', promise: watcherShutdown },
{ name: 'emulator', promise: emulatorShutdown },
{ name: 'ssh', promise: sshShutdown },
{ name: 'plugin-hosts', promise: pluginHostShutdown },
{ name: 'usage-cache', promise: usageCacheFlush },
{ name: 'stats', promise: statsFlush },
+5 -1
View File
@@ -42,7 +42,11 @@ async function assertAllowedTarget(store: Store, targetUrl: string): Promise<voi
// Why: URL drops the port for protocol defaults (e.g. http://host/ on 80),
// so compare against the effective port rather than the raw (empty) string.
const targetPort = parsed.port || (parsed.protocol === 'https:' ? '443' : '80')
const scan = await scanWorkspacePortProbes(getStoreWorkspacePortProbes(store))
// Why (#11161): a metadata-skipped scan drops advertisedUrl, which would
// silently narrow this allowlist on an EDR-hooked host.
const scan = await scanWorkspacePortProbes(getStoreWorkspacePortProbes(store), {
requireMetadata: true
})
const matches = scan.ports.some((port) => {
if (String(port.port) !== targetPort) {
return false
+295 -1
View File
@@ -45,6 +45,7 @@ const {
mockConnectionManager: {
connect: vi.fn(),
disconnect: vi.fn(),
disconnectConnection: vi.fn(),
reconnect: vi.fn(),
getConnection: vi.fn(),
getState: vi.fn(),
@@ -257,7 +258,13 @@ vi.mock('../ssh/ssh-port-scanner', () => ({
}
}))
import { getSshConnectionManager, registerSshHandlers, resetSshHandlerStateForTests } from './ssh'
import {
detachAllSshSessionsForShutdown,
getActiveMultiplexer,
getSshConnectionManager,
registerSshHandlers,
resetSshHandlerStateForTests
} from './ssh'
import { RelayVersionMismatchError } from '../ssh/ssh-relay-version-mismatch-error'
import {
SSH_RELAY_CONFIGURE_GRACE_TIME_METHOD,
@@ -276,6 +283,7 @@ import {
} from './pty'
import { assertSshMutationExpectation } from '../ssh/ssh-connection-generation'
import { getSshPtyConsumerRecovery } from '../ssh/ssh-pty-consumer-recovery'
import { quitTeardownStartGate } from '../quit-teardown-start-gate'
describe('SSH IPC handlers', () => {
const relayBuildId = '0.1.0+ipc-test'
@@ -312,6 +320,7 @@ describe('SSH IPC handlers', () => {
const createConnectionManagerMock = () => ({
connect: vi.fn(),
disconnect: vi.fn(),
disconnectConnection: vi.fn(),
reconnect: vi.fn(),
getConnection: vi.fn(),
getState: vi.fn(),
@@ -381,6 +390,7 @@ describe('SSH IPC handlers', () => {
mockConnectionManager.connect.mockReset()
mockConnectionManager.disconnect.mockReset()
mockConnectionManager.disconnectConnection.mockReset().mockResolvedValue(undefined)
mockConnectionManager.reconnect.mockReset()
mockConnectionManager.getConnection.mockReset()
mockConnectionManager.getState.mockReset()
@@ -557,6 +567,145 @@ describe('SSH IPC handlers', () => {
expect(mockSshStore.removeTarget).toHaveBeenCalledWith('ssh-1')
})
it('detaches active SSH sessions during app shutdown without terminating recovery', async () => {
const target: SshTarget = {
id: 'ssh-1',
label: 'Server',
host: 'example.com',
port: 22,
username: 'deploy'
}
mockSshStore.getTarget.mockReturnValue(target)
mockConnectionManager.connect.mockResolvedValue({})
mockConnectionManager.getState.mockReturnValue({
targetId: 'ssh-1',
status: 'connected',
error: null,
reconnectAttempt: 0
})
await handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })
mockMux.dispose.mockClear()
mockConnectionManager.disconnectAll.mockClear().mockResolvedValue(undefined)
await detachAllSshSessionsForShutdown()
expect(mockPortForwardManager.removeAllForwards).toHaveBeenCalledWith('ssh-1')
expect(mockMux.dispose).toHaveBeenCalledWith('connection_lost')
expect(mockStore.markSshRemotePtyLeasesAsync).toHaveBeenCalledWith('ssh-1', 'detached')
expect(mockStore.removeSshPtyConsumerRecovery).not.toHaveBeenCalled()
expect(mockConnectionManager.disconnectAll).toHaveBeenCalled()
})
it('refuses the replacement session a paused connect would publish after shutdown began', async () => {
const target: SshTarget = {
id: 'ssh-1',
label: 'Server',
host: 'example.com',
port: 22,
username: 'deploy'
}
mockSshStore.getTarget.mockReturnValue(target)
mockConnectionManager.connect.mockResolvedValue({})
mockConnectionManager.getState.mockReturnValue({
targetId: 'ssh-1',
status: 'connected',
error: null,
reconnectAttempt: 0
})
mockConnectionManager.disconnectAll.mockResolvedValue(undefined)
await handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })
// Why here: the old session's lease flush is the one window where doConnect resumes straight into
// publishing its replacement session without re-checking authority.
let releaseDetach = (): void => {}
const detachFlush = new Promise<void>((resolve) => {
releaseDetach = resolve
})
let signalEnteredDetach = (): void => {}
const enteredDetach = new Promise<void>((resolve) => {
signalEnteredDetach = resolve
})
mockStore.markSshRemotePtyLeasesAsync.mockImplementationOnce(() => {
signalEnteredDetach()
return detachFlush
})
const replacement = handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })
// Why await the flush entry and not ticks: shutdown must not snapshot what to drain until the
// replacement connect is parked in the lease flush.
await enteredDetach
expect(mockStore.markSshRemotePtyLeasesAsync).toHaveBeenCalledWith('ssh-1', 'detached')
mockConnectionManager.disconnectAll.mockClear()
// Why latch the gate here: the committed quit path owns it, so the drain alone must not fence.
quitTeardownStartGate.tryStart({ preventDefault() {} })
const shutdown = detachAllSshSessionsForShutdown()
releaseDetach()
await expect(replacement).rejects.toThrow('closed for app shutdown')
await shutdown
// Why one detach: the fence sits at the publication point, so the resumed connect never registers a
// replacement session — only the old session it had already torn down was detached.
const detaches = mockStore.markSshRemotePtyLeasesAsync.mock.calls.filter(
(call) => call[1] === 'detached'
)
expect(detaches).toHaveLength(1)
// Why twice: once for the drain's snapshot, once after joining the connect that was still in flight.
expect(mockConnectionManager.disconnectAll).toHaveBeenCalledTimes(2)
expect(getActiveMultiplexer('ssh-1')).toBeUndefined()
await expect(handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })).rejects.toThrow(
'closed for app shutdown'
)
})
it('joins an in-flight test-connection probe before the final shutdown disconnect', async () => {
const target: SshTarget = {
id: 'ssh-probe',
label: 'Server',
host: 'example.com',
port: 22,
username: 'deploy'
}
mockSshStore.getTarget.mockReturnValue(target)
mockConnectionManager.getState.mockReturnValue({
targetId: 'ssh-probe',
status: 'connected',
error: null,
reconnectAttempt: 0
})
mockConnectionManager.disconnectAll.mockResolvedValue(undefined)
mockConnectionManager.disconnect.mockClear().mockResolvedValue(undefined)
// Why: a probe holds a transport no session owns, so shutdown has to wait for it to hand it back.
const probeState = {
targetId: 'ssh-probe',
status: 'connected',
error: null,
reconnectAttempt: 0
}
let openProbeTransport = (): void => {}
mockConnectionManager.connect.mockImplementationOnce(
() =>
new Promise((resolve) => {
openProbeTransport = () => resolve({ getState: () => probeState })
})
)
const probe = handlers.get('ssh:testConnection')!(null, { targetId: 'ssh-probe' })
for (let tick = 0; tick < 5; tick++) {
await Promise.resolve()
}
expect(mockConnectionManager.connect).toHaveBeenCalledWith(target)
quitTeardownStartGate.tryStart({ preventDefault() {} })
const shutdown = detachAllSshSessionsForShutdown()
openProbeTransport()
await shutdown
expect(await probe).toMatchObject({ success: true })
expect(mockConnectionManager.disconnect).toHaveBeenCalledWith('ssh-probe')
})
it('ssh:importConfig returns imported targets', async () => {
const imported: SshTarget[] = [
{ id: 'ssh-imp', label: 'staging', host: 'staging.com', port: 22, username: '' }
@@ -1886,6 +2035,106 @@ describe('SSH IPC handlers', () => {
expect(mockDeployAndLaunchRelay).toHaveBeenCalledTimes(1)
})
it('closes the transport a cancelled connect opened after the disconnect finished', async () => {
const target: SshTarget = {
id: 'ssh-1',
label: 'Server',
host: 'example.com',
port: 22,
username: 'deploy'
}
const lateConn = { id: 'late-transport' }
let resolveStaleConnect!: (connection: unknown) => void
mockSshStore.getTarget.mockReturnValue(target)
mockConnectionManager.getConnection.mockReturnValue(undefined)
mockConnectionManager.connect.mockReturnValueOnce(
new Promise((resolve) => {
resolveStaleConnect = resolve
})
)
mockConnectionManager.disconnect.mockResolvedValue(undefined)
const staleConnect = handlers.get('ssh:connect')!(null, {
targetId: 'ssh-1'
}) as Promise<SshConnectionState>
await vi.waitFor(() => expect(mockConnectionManager.connect).toHaveBeenCalledTimes(1))
// Why await the whole disconnect: the leak only exists once its teardown has already run, so
// nothing else is left to close the transport this attempt opens afterwards.
await handlers.get('ssh:disconnect')!(null, { targetId: 'ssh-1' })
resolveStaleConnect(lateConn)
await expect(staleConnect).rejects.toThrow('SSH connection attempt was cancelled')
expect(mockConnectionManager.disconnectConnection).toHaveBeenCalledWith('ssh-1', lateConn)
expect(getActiveMultiplexer('ssh-1')).toBeUndefined()
})
it('closes the transport when establish resumes after the connect was invalidated', async () => {
const target: SshTarget = {
id: 'ssh-1',
label: 'Server',
host: 'example.com',
port: 22,
username: 'deploy'
}
const conn = { id: 'establishing-transport' }
let releaseRelayLaunch = (): void => {}
mockSshStore.getTarget.mockReturnValue(target)
mockConnectionManager.getConnection.mockReturnValue(undefined)
mockConnectionManager.connect.mockResolvedValue(conn)
mockConnectionManager.disconnect.mockResolvedValue(undefined)
mockDeployAndLaunchRelay.mockImplementationOnce(
() =>
new Promise((resolve) => {
releaseRelayLaunch = () => resolve(createRelayLaunchResult())
})
)
const connect = handlers.get('ssh:connect')!(null, {
targetId: 'ssh-1'
}) as Promise<SshConnectionState>
await vi.waitFor(() => expect(mockDeployAndLaunchRelay).toHaveBeenCalledTimes(1))
await handlers.get('ssh:disconnect')!(null, { targetId: 'ssh-1' })
releaseRelayLaunch()
await expect(connect).rejects.toThrow('SSH connection attempt was cancelled')
expect(mockConnectionManager.disconnectConnection).toHaveBeenCalledWith('ssh-1', conn)
expect(getActiveMultiplexer('ssh-1')).toBeUndefined()
})
it('leaves a reused transport to its replacement when the connect is cancelled', async () => {
const target: SshTarget = {
id: 'ssh-1',
label: 'Server',
host: 'example.com',
port: 22,
username: 'deploy'
}
// Why: connect() hands back the already-open transport, so this attempt never owned it.
const sharedConn = { id: 'shared-transport' }
let resolveStaleConnect!: (connection: unknown) => void
mockSshStore.getTarget.mockReturnValue(target)
mockConnectionManager.getConnection.mockReturnValue(sharedConn)
mockConnectionManager.connect.mockReturnValueOnce(
new Promise((resolve) => {
resolveStaleConnect = resolve
})
)
mockConnectionManager.disconnect.mockResolvedValue(undefined)
const staleConnect = handlers.get('ssh:connect')!(null, {
targetId: 'ssh-1'
}) as Promise<SshConnectionState>
await vi.waitFor(() => expect(mockConnectionManager.connect).toHaveBeenCalledTimes(1))
await handlers.get('ssh:disconnect')!(null, { targetId: 'ssh-1' })
resolveStaleConnect(sharedConn)
await expect(staleConnect).rejects.toThrow('SSH connection attempt was cancelled')
expect(mockConnectionManager.disconnectConnection).not.toHaveBeenCalled()
})
it('keeps reconnect behind transport disconnect when forward teardown fails', async () => {
const target: SshTarget = {
id: 'ssh-1',
@@ -2374,6 +2623,51 @@ describe('SSH IPC handlers', () => {
expect(mockConnectionManager.disconnect).toHaveBeenCalledWith('ssh-1')
})
it('ssh:resetRelay does not open a transport when shutdown starts while it waits for a connect', async () => {
const target: SshTarget = {
id: 'ssh-1',
label: 'Server',
host: 'example.com',
port: 22,
username: 'deploy'
}
let failConnect!: (error: unknown) => void
const connectResult = new Promise((_resolve, reject) => {
failConnect = reject
})
mockSshStore.getTarget.mockReturnValue(target)
mockConnectionManager.connect.mockReturnValue(connectResult)
// Why undefined: reset must fall through to opening its own transport, which is the call under test.
mockConnectionManager.getConnection.mockReturnValue(undefined)
mockConnectionManager.getState.mockReturnValue({
targetId: 'ssh-1',
status: 'connected',
error: null,
reconnectAttempt: 0
})
const connectPromise = handlers.get('ssh:connect')!(null, {
targetId: 'ssh-1'
}) as Promise<unknown>
await vi.waitFor(() => expect(mockConnectionManager.connect).toHaveBeenCalledTimes(1))
// Why admitted first: the gate latches only after reset is already parked behind the connect, so
// the entry fence cannot be what stops it.
const resetPromise = handlers.get('ssh:resetRelay')!(null, {
targetId: 'ssh-1'
}) as Promise<void>
await Promise.resolve()
quitTeardownStartGate.tryStart({ preventDefault() {} })
failConnect(new Error('transport lost'))
await expect(connectPromise).rejects.toThrow('transport lost')
await expect(resetPromise).rejects.toThrow('closed for app shutdown')
// Why once: the resumed reset must not open a second transport that would outlive the drain.
expect(mockConnectionManager.connect).toHaveBeenCalledTimes(1)
expect(mockForceStopRelayForTarget).not.toHaveBeenCalled()
})
it('ssh:connect waits for an in-flight reset before starting a new connection', async () => {
const target: SshTarget = {
id: 'ssh-1',
+132 -4
View File
@@ -7,7 +7,7 @@ import {
listUserSshConfigHostSummaries,
resolveUserSshConfigHost
} from '../ssh/ssh-config-host-picker'
import type { SshConnectionCallbacks } from '../ssh/ssh-connection'
import type { SshConnection, SshConnectionCallbacks } from '../ssh/ssh-connection'
import { SshConnectionManager } from '../ssh/ssh-connection-manager'
import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer'
import { SshRelaySession, type SshRelayAiVaultHostInfo } from '../ssh/ssh-relay-session'
@@ -25,6 +25,7 @@ import type {
DirectSshAuthority
} from '../../shared/ssh-types'
import { SSH_TERMINATE_RECONNECT_REQUIRED } from '../../shared/constants'
import { quitTeardownStartGate } from '../quit-teardown-start-gate'
import { isRuntimeOwnedSshTargetId } from '../../shared/execution-host'
import { isAuthError } from '../ssh/ssh-connection-utils'
import { createCancelledConnectAttemptError } from '../ssh/ssh-connect-attempt-cancellation'
@@ -292,6 +293,33 @@ async function abandonFailedSshSession(targetId: string, session: SshRelaySessio
}
}
// Why: a connect cancelled after its transport already opened still owns that transport and its
// unpublished session — nothing else will reach them, so it has to close them itself. Why only the
// connection it minted, and by identity: disconnecting by target id (or closing a transport it merely
// reused) would tear down the replacement connect's live transport.
async function abandonCancelledConnectAttempt(
targetId: string,
session: SshRelaySession,
mintedConnection: SshConnection | null
): Promise<void> {
// Why the identity guard: every path that removes a session from activeSessions detaches it first,
// so re-detaching a superseded session would only clobber the replacement's lease state.
if (activeSessions.get(targetId) === session) {
await abandonFailedSshSession(targetId, session)
}
if (!mintedConnection) {
return
}
try {
await connectionManager!.disconnectConnection(targetId, mintedConnection)
} catch (error) {
// Why: the caller is about to throw the cancellation; a teardown throw must not replace it.
console.warn(
`[ssh] Failed to disconnect cancelled connect transport for ${targetId}: ${error instanceof Error ? error.message : String(error)}`
)
}
}
function relayGracePeriodForTarget(target: SshTarget | null | undefined): number | undefined {
return target?.relayGracePeriodSeconds
}
@@ -306,6 +334,15 @@ type ConnectAttempt = {
const connectInFlight = new Map<string, ConnectAttempt>()
const pendingTransportReconnects = new Set<string>()
// Why the quit gate rather than a local latch: "the committed quit has begun" already has an owner,
// and a private copy could be set by something that is not actually quitting — leaving SSH connects
// refused for the rest of the process lifetime.
function assertSshConnectsNotFenced(): void {
if (quitTeardownStartGate.hasStarted()) {
throw new Error('SSH connects are closed for app shutdown')
}
}
function invalidateConnectAttempt(targetId: string): void {
rotateSshProviderAuthority(targetId)
pendingTransportReconnects.delete(targetId)
@@ -322,6 +359,7 @@ const resetRelayInFlight = new Map<string, Promise<void>>()
// Why: ssh:testConnection connects then disconnects; suppressing broadcasts during the test avoids worktree cards flashing connected → disconnected.
const testingTargets = new Set<string>()
const testConnectionProbes = new Set<Promise<unknown>>()
// Why: without backoff, a relay channel that keeps dying reconnects as fast as the network allows, hammering local + remote sshd; track attempts and back off to end the loop recoverably.
type RelayLostBackoffState = {
@@ -1042,6 +1080,9 @@ export function registerSshHandlers(
if (!isCurrentSshProviderAuthority(observedAuthority)) {
throw createCancelledConnectAttemptError()
}
// Why: the shutdown drain fences and snapshots synchronously, so a connect either registers in
// connectInFlight below (and gets joined) or fails here — it can never slip between the two.
assertSshConnectsNotFenced()
pendingTransportReconnects.delete(targetId)
const promise = doConnect(targetId, replacePendingTransport)
@@ -1130,6 +1171,11 @@ export function registerSshHandlers(
}
}
// Why here and not only at entry: this is the publication point, and it is the last statement
// before the transport opens. Checking it in the same synchronous block as activeSessions.set
// means a connect either registers before the shutdown drain snapshots, or registers never and
// owns nothing to clean up.
assertSshConnectsNotFenced()
// Why: create the session early so onStateChange sees it in 'deploying' and skips reconnect logic.
const session = new SshRelaySession(
targetId,
@@ -1144,6 +1190,12 @@ export function registerSshHandlers(
const ownsSession = (): boolean =>
isCurrentConnectAttempt(targetId, authority) && activeSessions.get(targetId) === session
// Why captured here and not with existingState: connect() reuses an already-connected transport,
// and only a transport this attempt opened is this attempt's to close when it loses the race.
const priorConnection = connectionManager!.getConnection(targetId)
const mintedConnection = (): SshConnection | null =>
conn && conn !== priorConnection ? conn : null
try {
conn = await connectionManager!.connect(target)
if (!ownsSession()) {
@@ -1154,6 +1206,7 @@ export function registerSshHandlers(
const errObj = err instanceof Error ? err : new Error(String(err))
const status: SshConnectionStatus = isAuthError(errObj) ? 'auth-failed' : 'error'
if (!ownsSession()) {
await abandonCancelledConnectAttempt(targetId, session, mintedConnection())
throw createCancelledConnectAttemptError()
}
// Why: clear this failed connect's flag so a later non-prompting connect isn't deferred.
@@ -1194,6 +1247,7 @@ export function registerSshHandlers(
})
} catch (err) {
if (!ownsSession()) {
await abandonCancelledConnectAttempt(targetId, session, mintedConnection())
throw createCancelledConnectAttemptError()
}
await abandonFailedSshSession(targetId, session)
@@ -1300,7 +1354,13 @@ export function registerSshHandlers(
}
const existingConn = connectionManager!.getConnection(targetId)
const conn = existingConn ?? (await connectionManager!.connect(target))
let conn = existingConn
if (!conn) {
// Why re-check: admission fenced this reset before it parked on the in-flight connect, so shutdown
// may have started (and drained) while we waited — opening a transport now would outlive the drain.
assertSshConnectsNotFenced()
conn = await connectionManager!.connect(target)
}
try {
await forceStopRelayForTarget(conn, targetId)
} finally {
@@ -1333,6 +1393,8 @@ export function registerSshHandlers(
if (!target) {
throw new Error(`SSH target "${args.targetId}" not found`)
}
// Why: reset opens its own transport, so it must be fenced by shutdown the same way connect is.
assertSshConnectsNotFenced()
let resetPromise: Promise<void>
resetPromise = runTargetLifecycle(args.targetId, () =>
@@ -1392,17 +1454,26 @@ export function registerSshHandlers(
}
testingTargets.add(args.targetId)
try {
// Why a tracked promise and not just the id: a probe holds a real transport that no session owns,
// so shutdown has to be able to join it before the final drain disconnects what is left.
const probe = (async () => {
// Why: a probe transport opened after the shutdown drain would outlive orderly teardown.
assertSshConnectsNotFenced()
const conn = await connectionManager!.connect(target)
const state = conn.getState()
await connectionManager!.disconnect(args.targetId)
return { success: true, state }
return state
})()
testConnectionProbes.add(probe)
try {
return { success: true, state: await probe }
} catch (err) {
return {
success: false,
error: err instanceof Error ? err.message : String(err)
}
} finally {
testConnectionProbes.delete(probe)
testingTargets.delete(args.targetId)
// Why: clear so a test's credential prompt doesn't leave lastRequiredPassphrase=true and defer this target at startup.
credentialRequestedForTarget.delete(args.targetId)
@@ -1510,6 +1581,61 @@ export function getSshConnectionManager(): SshConnectionManager | null {
return connectionManager
}
// Why: an invalidated connect only observes its cancellation at the next checkpoint, and one blocked
// in the transport handshake can sit there for the whole SSH timeout. Bound the join so the final
// drain always runs well inside the quit deadline instead of racing it.
const SSH_SHUTDOWN_INFLIGHT_JOIN_MS = 2_000
async function settleWithinMs(work: readonly Promise<unknown>[], timeoutMs: number): Promise<void> {
if (work.length === 0) {
return
}
await Promise.race([
Promise.allSettled(work),
new Promise<void>((resolve) => {
const timer = setTimeout(resolve, timeoutMs)
timer.unref?.()
})
])
}
// Why no fence latch here: the committed quit path sets it before calling this, so it is already on
// for the snapshot below. Called without that gate (tests), this degrades to a plain drain.
export async function detachAllSshSessionsForShutdown(): Promise<void> {
const inFlightWork: Promise<unknown>[] = [
...[...connectInFlight.values()].map((attempt) => attempt.promise),
...resetRelayInFlight.values(),
...testConnectionProbes
]
for (const targetId of Array.from(connectInFlight.keys())) {
invalidateConnectAttempt(targetId)
}
const errors: unknown[] = []
const drain = async (): Promise<void> => {
const results = await Promise.allSettled([
...[...activeSessions.keys()].map((targetId) =>
teardownActiveSshSession(targetId, (session) => session.detachAndPersist())
),
connectionManager?.disconnectAll() ?? Promise.resolve()
])
for (const result of results) {
if (result.status === 'rejected') {
errors.push(result.reason as unknown)
}
}
}
await drain()
// Why: a connect paused in old-session teardown still publishes its replacement session and opens a
// transport before it reaches the cancellation checkpoint, so the first drain can miss both.
await settleWithinMs(inFlightWork, SSH_SHUTDOWN_INFLIGHT_JOIN_MS)
await drain()
if (errors.length > 0) {
throw new AggregateError(errors, 'SSH shutdown detach failed')
}
}
export async function resetSshHandlerStateForTests(): Promise<void> {
advertisedUrlWatcherUnsubscribe?.()
advertisedUrlWatcherUnsubscribe = null
@@ -1536,7 +1662,9 @@ export async function resetSshHandlerStateForTests(): Promise<void> {
resetSshProviderAuthorities()
resetRelayInFlight.clear()
testingTargets.clear()
testConnectionProbes.clear()
credentialRequestedForTarget.clear()
quitTeardownStartGate.resetForTests()
await connectionManager?.disconnectAll()
portForwardManager?.dispose()
+30 -22
View File
@@ -102,14 +102,18 @@ describe('registerWorkspacePortHandlers', () => {
worktrees: [{ id: 'attacker', path: '/tmp/not-authorized', repoId: 'local-repo' }]
})
expect(scanWorkspacePortsMock).toHaveBeenCalledWith([
{
id: 'local-repo::/workspace/repo',
repoId: 'local-repo',
displayName: 'Primary',
path: '/workspace/repo'
}
])
expect(scanWorkspacePortsMock).toHaveBeenCalledWith(
[
{
id: 'local-repo::/workspace/repo',
repoId: 'local-repo',
displayName: 'Primary',
path: '/workspace/repo'
}
],
undefined,
undefined
)
})
it('deduplicates concurrent scans for the same store-derived probe set', async () => {
@@ -139,20 +143,24 @@ describe('registerWorkspacePortHandlers', () => {
await handlers.get('workspacePorts:scan')?.(null, undefined)
expect(scanWorkspacePortsMock).toHaveBeenCalledWith([
{
id: 'local-repo::/workspace/repo',
repoId: 'local-repo',
displayName: 'Primary',
path: '/workspace/repo'
},
{
id: 'other-repo::/workspace/other',
repoId: 'other-repo',
displayName: 'Other',
path: '/workspace/other'
}
])
expect(scanWorkspacePortsMock).toHaveBeenCalledWith(
[
{
id: 'local-repo::/workspace/repo',
repoId: 'local-repo',
displayName: 'Primary',
path: '/workspace/repo'
},
{
id: 'other-repo::/workspace/other',
repoId: 'other-repo',
displayName: 'Other',
path: '/workspace/other'
}
],
undefined,
undefined
)
})
it('stops a process only after the current scan proves the pid owns a workspace port', async () => {
@@ -1,4 +1,4 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { afterEach, describe, expect, it, vi, type Mock } from 'vitest'
import path from 'node:path'
import {
attributePortToWorkspace,
@@ -9,13 +9,21 @@ import {
resetWorkspacePortScanTimeoutBackoffForTests,
scanWorkspacePorts
} from './local-workspace-port-scanner'
import { PortScanCommandTimeoutError } from './port-scan-command-protocol'
const execFileMock = vi.hoisted(() => vi.fn())
const runPortScanCommandMock = vi.hoisted(() => vi.fn())
vi.mock('child_process', () => ({
execFile: execFileMock
vi.mock('./port-scan-command-client', () => ({
runPortScanCommand: runPortScanCommandMock,
isPortScanWorkerUnavailableError: () => false
}))
const LSOF_LISTEN_OUTPUT = ['p123', 'cnode', 'n127.0.0.1:5173'].join('\n')
function urlWatcherStub(): { lookup: () => undefined; reconcileScan: Mock } {
return { lookup: () => undefined, reconcileScan: vi.fn() }
}
const worktrees = [
{
id: 'repo::/repo',
@@ -177,52 +185,42 @@ describe('scanWorkspacePorts attribution work', () => {
afterEach(() => {
resetWorkspacePortScanTimeoutBackoffForTests()
vi.restoreAllMocks()
execFileMock.mockReset()
runPortScanCommandMock.mockReset()
})
it('normalizes worktree paths once per scan instead of once per port phase', async () => {
vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin')
const win32ResolveSpy = vi.spyOn(path.win32, 'resolve')
const posixResolveSpy = vi.spyOn(path.posix, 'resolve')
const invokeCallback = (callback: unknown, stdout: string): void => {
if (typeof callback !== 'function') {
throw new Error('missing execFile callback')
}
const execCallback = callback as (error: Error | null, stdout: string) => void
execCallback(null, stdout)
}
execFileMock.mockImplementation(
(command: string, args: string[], _options: unknown, callback: unknown) => {
if (command === 'lsof' && args.includes('-iTCP')) {
invokeCallback(
callback,
['p123', 'cnode', 'n127.0.0.1:3000', 'p124', 'cnode', 'n127.0.0.1:3001'].join('\n')
)
} else if (command === 'lsof') {
invokeCallback(
callback,
['p123', 'n/repo/service', 'p124', 'n/repo/worktrees/feature/app'].join('\n')
)
} else if (command === 'ps') {
invokeCallback(
callback,
[
'123 node /repo/service/server.js',
'124 node /repo/worktrees/feature/app/server.js'
].join('\n')
)
} else {
invokeCallback(callback, '')
runPortScanCommandMock.mockImplementation(async (command: string, args: string[]) => {
if (command === 'lsof' && args.includes('-iTCP')) {
return {
stdout: ['p123', 'cnode', 'n127.0.0.1:3000', 'p124', 'cnode', 'n127.0.0.1:3001'].join(
'\n'
),
spawnMs: 5
}
return { kill: vi.fn() }
}
)
const scan = await scanWorkspacePorts(worktrees, {
lookup: () => undefined,
reconcileScan: vi.fn()
if (command === 'lsof') {
return {
stdout: ['p123', 'n/repo/service', 'p124', 'n/repo/worktrees/feature/app'].join('\n'),
spawnMs: 5
}
}
if (command === 'ps') {
return {
stdout: [
'123 node /repo/service/server.js',
'124 node /repo/worktrees/feature/app/server.js'
].join('\n'),
spawnMs: 5
}
}
return { stdout: '', spawnMs: 5 }
})
const scan = await scanWorkspacePorts(worktrees, urlWatcherStub())
expect(scan.ports.filter((port) => port.kind === 'workspace')).toHaveLength(2)
const win32WorktreePathResolveCalls = win32ResolveSpy.mock.calls.filter(
([input]) => input === '/repo' || input === '/repo/worktrees/feature'
@@ -240,62 +238,39 @@ describe('scanWorkspacePorts command timeout', () => {
vi.useRealTimers()
resetWorkspacePortScanTimeoutBackoffForTests()
vi.restoreAllMocks()
execFileMock.mockReset()
runPortScanCommandMock.mockReset()
})
it('returns an unavailable scan when lsof never reports completion', async () => {
vi.useFakeTimers()
vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin')
const killMock = vi.fn()
execFileMock.mockImplementation(() => ({ kill: killMock }))
runPortScanCommandMock.mockRejectedValue(
new PortScanCommandTimeoutError('lsof timed out after 4000ms')
)
let settled = false
const scanPromise = scanWorkspacePorts([], {
lookup: () => undefined,
reconcileScan: vi.fn()
}).then((scan) => {
settled = true
return scan
})
await vi.advanceTimersByTimeAsync(4_000)
expect(settled).toBe(true)
await expect(scanPromise).resolves.toMatchObject({
await expect(scanWorkspacePorts([], urlWatcherStub())).resolves.toMatchObject({
platform: 'darwin',
ports: [],
unavailableReason: 'Port scanning is unavailable on darwin.'
})
expect(killMock).toHaveBeenCalled()
})
it('backs off after a command timeout instead of launching lsof on every scan tick', async () => {
vi.useFakeTimers()
vi.setSystemTime(1_000)
vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin')
const killMock = vi.fn()
execFileMock.mockImplementation(() => ({ kill: killMock }))
runPortScanCommandMock.mockRejectedValue(
new PortScanCommandTimeoutError('lsof timed out after 4000ms')
)
const firstScanPromise = scanWorkspacePorts([], {
lookup: () => undefined,
reconcileScan: vi.fn()
})
await vi.advanceTimersByTimeAsync(4_000)
await expect(firstScanPromise).resolves.toMatchObject({
await expect(scanWorkspacePorts([], urlWatcherStub())).resolves.toMatchObject({
platform: 'darwin',
ports: [],
unavailableReason: 'Port scanning is unavailable on darwin.'
})
expect(execFileMock).toHaveBeenCalledTimes(1)
expect(runPortScanCommandMock).toHaveBeenCalledTimes(1)
const cooldownScans = await Promise.all(
Array.from({ length: 10 }, () =>
scanWorkspacePorts([], {
lookup: () => undefined,
reconcileScan: vi.fn()
})
)
Array.from({ length: 10 }, () => scanWorkspacePorts([], urlWatcherStub()))
)
expect(cooldownScans).toHaveLength(10)
@@ -306,25 +281,172 @@ describe('scanWorkspacePorts command timeout', () => {
expect(
cooldownScans.every((scan) => scan.unavailableReason?.includes('temporarily paused'))
).toBe(true)
expect(execFileMock).toHaveBeenCalledTimes(1)
expect(runPortScanCommandMock).toHaveBeenCalledTimes(1)
vi.setSystemTime(65_001)
await vi.advanceTimersByTimeAsync(0)
execFileMock.mockImplementation(
(_command: string, args: string[], _options: unknown, callback: unknown) => {
const execCallback = callback as (error: Error | null, stdout: string) => void
const output = args.includes('-iTCP') ? 'p123\ncnode\nn127.0.0.1:3000' : ''
execCallback(null, output)
return { kill: vi.fn() }
}
)
runPortScanCommandMock.mockImplementation(async (_command: string, args: string[]) => ({
stdout: args.includes('-iTCP') ? 'p123\ncnode\nn127.0.0.1:3000' : '',
spawnMs: 5
}))
const recoveredScan = await scanWorkspacePorts([], {
lookup: () => undefined,
reconcileScan: vi.fn()
})
const recoveredScan = await scanWorkspacePorts([], urlWatcherStub())
expect(recoveredScan.unavailableReason).toBeUndefined()
expect(execFileMock).toHaveBeenCalledTimes(4)
expect(runPortScanCommandMock).toHaveBeenCalledTimes(4)
})
})
describe('scanWorkspacePorts with delayed process creation', () => {
afterEach(() => {
resetWorkspacePortScanTimeoutBackoffForTests()
vi.restoreAllMocks()
runPortScanCommandMock.mockReset()
})
// Regression for #11161: an endpoint-security hook makes CreateProcessW take
// seconds, so the command's own budget must not be charged for the spawn.
it('does not report a command timeout when only process creation was delayed', async () => {
vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin')
runPortScanCommandMock.mockResolvedValue({ stdout: LSOF_LISTEN_OUTPUT, spawnMs: 4_200 })
const first = await scanWorkspacePorts([], urlWatcherStub())
const second = await scanWorkspacePorts([], urlWatcherStub())
expect(first.unavailableReason).toBeUndefined()
expect(second.unavailableReason).toBeUndefined()
expect(first.ports).toHaveLength(1)
})
it('skips the optional metadata commands for one cycle after a stalled spawn', async () => {
vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin')
runPortScanCommandMock.mockResolvedValue({ stdout: LSOF_LISTEN_OUTPUT, spawnMs: 4_200 })
const scan = await scanWorkspacePorts([], urlWatcherStub())
expect(runPortScanCommandMock).toHaveBeenCalledTimes(1)
expect(scan.ports).toHaveLength(1)
})
// Regression for #11161 review: a metadata-less scan must not be reconciled as
// "the listener vanished" — that evicts advertised URLs only a PTY can restore.
it('does not reconcile advertised URLs for a scan that skipped metadata', async () => {
vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin')
runPortScanCommandMock.mockResolvedValue({ stdout: LSOF_LISTEN_OUTPUT, spawnMs: 4_200 })
const watcher = urlWatcherStub()
await scanWorkspacePorts(worktrees, watcher)
expect(watcher.reconcileScan).not.toHaveBeenCalled()
})
it('re-probes metadata on the scan after a skip instead of degrading forever', async () => {
vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin')
runPortScanCommandMock.mockImplementation(async (command: string, args: string[]) => {
if (command === 'lsof' && args.includes('-iTCP')) {
return { stdout: LSOF_LISTEN_OUTPUT, spawnMs: 4_200 }
}
if (command === 'lsof') {
return { stdout: ['p123', 'n/repo'].join('\n'), spawnMs: 4_200 }
}
return { stdout: '123 node /repo/server.js', spawnMs: 4_200 }
})
const watcher = urlWatcherStub()
const skipped = await scanWorkspacePorts(worktrees, watcher)
const recovered = await scanWorkspacePorts(worktrees, watcher)
expect(skipped.ports[0]?.kind).toBe('external')
expect(recovered.ports[0]).toMatchObject({ kind: 'workspace' })
expect(runPortScanCommandMock).toHaveBeenCalledTimes(4)
expect(watcher.reconcileScan).toHaveBeenCalledTimes(worktrees.length)
})
it('still collects process metadata when process creation was fast', async () => {
vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin')
runPortScanCommandMock.mockImplementation(async (_command: string, args: string[]) => ({
stdout: args.includes('-iTCP') ? LSOF_LISTEN_OUTPUT : '',
spawnMs: 5
}))
await scanWorkspacePorts([], urlWatcherStub())
expect(runPortScanCommandMock).toHaveBeenCalledTimes(3)
})
// Regression for #11161 review: the skip parity is driven by the 30s poller,
// so a one-shot user action would otherwise land on a random parity.
it('keeps probing metadata for callers that require attribution', async () => {
vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin')
mockStalledDarwinScan()
const scan = await scanWorkspacePorts(worktrees, urlWatcherStub(), { requireMetadata: true })
expect(scan.ports[0]).toMatchObject({ kind: 'workspace' })
expect(runPortScanCommandMock).toHaveBeenCalledTimes(3)
})
it('does not let a required-metadata scan reset the background skip parity', async () => {
vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin')
mockStalledDarwinScan()
await scanWorkspacePorts(worktrees, urlWatcherStub())
await scanWorkspacePorts(worktrees, urlWatcherStub(), { requireMetadata: true })
await scanWorkspacePorts(worktrees, urlWatcherStub())
// 1 skipped + 3 required + 3 recovered; a reset parity would skip twice.
expect(runPortScanCommandMock).toHaveBeenCalledTimes(7)
})
// Regression for #11161 review: without carry-forward the panel moves every
// workspace port into External on each skipped cycle.
it('carries the previous cycle attribution through a skipped scan', async () => {
vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin')
let listenSpawnMs = 5
runPortScanCommandMock.mockImplementation(async (command: string, args: string[]) => {
if (command === 'lsof' && args.includes('-iTCP')) {
return { stdout: LSOF_LISTEN_OUTPUT, spawnMs: listenSpawnMs }
}
return { stdout: command === 'lsof' ? ['p123', 'n/repo'].join('\n') : '', spawnMs: 5 }
})
const full = await scanWorkspacePorts(worktrees, urlWatcherStub())
listenSpawnMs = 4_200
const skipped = await scanWorkspacePorts(worktrees, urlWatcherStub())
expect(full.ports[0]).toMatchObject({ kind: 'workspace' })
expect(skipped.ports[0]).toMatchObject({ kind: 'workspace', processName: 'node' })
})
it('does not hand carried-forward metadata to a different listener', async () => {
vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin')
let listenOutput = LSOF_LISTEN_OUTPUT
let listenSpawnMs = 5
runPortScanCommandMock.mockImplementation(async (command: string, args: string[]) => {
if (command === 'lsof' && args.includes('-iTCP')) {
return { stdout: listenOutput, spawnMs: listenSpawnMs }
}
return { stdout: command === 'lsof' ? ['p123', 'n/repo'].join('\n') : '', spawnMs: 5 }
})
await scanWorkspacePorts(worktrees, urlWatcherStub())
listenOutput = ['p123', 'cnode', 'n127.0.0.1:9999'].join('\n')
listenSpawnMs = 4_200
const skipped = await scanWorkspacePorts(worktrees, urlWatcherStub())
expect(skipped.ports[0]?.kind).toBe('external')
})
})
/** Darwin scan where every spawn stalls past the metadata-skip threshold. */
function mockStalledDarwinScan(): void {
runPortScanCommandMock.mockImplementation(async (command: string, args: string[]) => {
if (command === 'lsof' && args.includes('-iTCP')) {
return { stdout: LSOF_LISTEN_OUTPUT, spawnMs: 4_200 }
}
if (command === 'lsof') {
return { stdout: ['p123', 'n/repo'].join('\n'), spawnMs: 4_200 }
}
return { stdout: '123 node /repo/server.js', spawnMs: 4_200 }
})
}
+158 -77
View File
@@ -1,6 +1,5 @@
/* eslint-disable max-lines -- Why: the platform-specific scan paths share parsing,
attribution, and normalization rules that must stay in lockstep. */
import { execFile } from 'node:child_process'
import { readFile, readdir, readlink } from 'node:fs/promises'
import path from 'node:path'
import type {
@@ -11,14 +10,36 @@ import type {
} from '../../shared/workspace-ports'
import { getProcessOutputFields } from '../../shared/process-output-field-scanner'
import { advertisedUrlWatcher, type AdvertisedUrlWatcher } from './advertised-url-watcher'
import { isPortScanWorkerUnavailableError, runPortScanCommand } from './port-scan-command-client'
import { PortScanCommandTimeoutError } from './port-scan-command-protocol'
import { WorkspacePortScanTimeoutBackoff } from './workspace-port-scan-timeout-backoff'
const COMMAND_TIMEOUT_MS = 4_000
// Why (#11161): on an EDR-hooked host process creation alone can take seconds.
// Past this, skip the scan's optional metadata commands for one cycle so a scan
// costs roughly one stall instead of three.
const SLOW_SPAWN_SKIP_METADATA_MS = 2_000
const MAX_PORTS = 200
const HTTP_PORTS = new Set([80, 3000, 3001, 4200, 5000, 5173, 5174, 8000, 8080, 8888])
const HTTPS_PORTS = new Set([443, 8443])
const commandTimeoutBackoff = new WorkspacePortScanTimeoutBackoff()
let loggedWorkerUnavailable = false
// Why (#11161): a hooked host stalls every spawn, so gating only on the current
// scan would drop metadata forever. Never skip twice running, so attribution —
// and the Stop action and advertised-URL matching that ride on it — recovers on
// the next tick.
let skippedMetadataOnLastScan = false
// Why (#11161): a skipped cycle would otherwise report every listener as
// external, so the panel flip-flops and the Stop action loses its owner. Carry
// the previous cycle's metadata forward, keyed tightly enough that a recycled
// pid cannot inherit it.
let lastListenerMetadata = new Map<string, ProcessMetadata>()
export type WorkspacePortScanOptions = {
/** Set by attribution-dependent callers (Stop, the localhost-label allowlist)
* that must never trade owner metadata for scan latency. */
requireMetadata?: boolean
}
type RawListeningPort = {
host: string
@@ -40,9 +61,16 @@ type NormalizedWorkspacePortProbe = {
normalizedPath: string
}
type PlatformListeningPortScan = {
ports: RawListeningPort[]
/** False when a stalled spawn made the scan skip its cwd/command-line probes. */
metadataAvailable: boolean
}
export async function scanWorkspacePorts(
worktrees: WorkspacePortProbe[],
urlWatcher: Pick<AdvertisedUrlWatcher, 'lookup' | 'reconcileScan'> = advertisedUrlWatcher
urlWatcher: Pick<AdvertisedUrlWatcher, 'lookup' | 'reconcileScan'> = advertisedUrlWatcher,
options: WorkspacePortScanOptions = {}
): Promise<WorkspacePortScanResult> {
const cooldown = commandTimeoutBackoff.snapshot()
if (cooldown.isCoolingDown) {
@@ -54,10 +82,15 @@ export async function scanWorkspacePorts(
}
try {
const rawPorts = await scanPlatformListeningPorts()
const { ports: rawPorts, metadataAvailable } = await scanPlatformListeningPorts(options)
commandTimeoutBackoff.recordSuccess()
const normalizedWorktrees = normalizeWorkspacePortProbes(worktrees)
reconcileAdvertisedUrls(rawPorts, normalizedWorktrees, urlWatcher)
// Why (#11161): without cwd/command-line every port looks unattributed, and
// reconciling that would read as "the listener vanished" and evict cached
// advertised URLs that only live PTY output can ever restore.
if (metadataAvailable) {
reconcileAdvertisedUrls(rawPorts, normalizedWorktrees, urlWatcher)
}
const ports = rawPorts
.map((port) => enrichPort(port, normalizedWorktrees, urlWatcher))
.sort(compareWorkspacePorts)
@@ -67,13 +100,66 @@ export async function scanWorkspacePorts(
if (isCommandTimeoutError(error)) {
commandTimeoutBackoff.recordTimeout()
}
console.warn('[workspace-ports] scan failed', error)
warnScanFailure(error)
return makeUnavailableScan(`Port scanning is unavailable on ${process.platform}.`)
}
}
export function resetWorkspacePortScanTimeoutBackoffForTests(): void {
commandTimeoutBackoff.reset()
loggedWorkerUnavailable = false
skippedMetadataOnLastScan = false
lastListenerMetadata = new Map()
}
function shouldSkipMetadataCommands(spawnMs: number, opts: WorkspacePortScanOptions): boolean {
if (opts.requireMetadata) {
// Leave the skip parity alone: it belongs to the background scan cadence,
// and a one-shot user action must not shift which tick degrades.
return false
}
const skip = spawnMs > SLOW_SPAWN_SKIP_METADATA_MS && !skippedMetadataOnLastScan
skippedMetadataOnLastScan = skip
return skip
}
/** Identity tight enough that a recycled pid cannot inherit stale metadata. */
function listenerMetadataKey(port: RawListeningPort): string {
return `${port.pid ?? 'unknown'}:${port.host}:${port.port}`
}
function rememberListenerMetadata(ports: readonly RawListeningPort[]): void {
lastListenerMetadata = new Map(
ports.map((port) => [
listenerMetadataKey(port),
{ processName: port.processName, commandLine: port.commandLine, cwd: port.cwd }
])
)
}
function recallListenerMetadata(port: RawListeningPort): RawListeningPort {
const remembered = lastListenerMetadata.get(listenerMetadataKey(port))
if (!remembered) {
return port
}
return {
...port,
processName: port.processName ?? remembered.processName,
commandLine: port.commandLine ?? remembered.commandLine,
cwd: port.cwd ?? remembered.cwd
}
}
// Why: a mispackaged probe worker fails identically forever, so logging it on
// every 30s scan tick is pure noise.
function warnScanFailure(error: unknown): void {
if (isPortScanWorkerUnavailableError(error)) {
if (loggedWorkerUnavailable) {
return
}
loggedWorkerUnavailable = true
}
console.warn('[workspace-ports] scan failed', error)
}
function makeUnavailableScan(reason: string): WorkspacePortScanResult {
@@ -194,38 +280,73 @@ export function parseProcNetTcp(content: string): { host: string; port: number;
return results
}
async function scanPlatformListeningPorts(): Promise<RawListeningPort[]> {
async function scanPlatformListeningPorts(
options: WorkspacePortScanOptions
): Promise<PlatformListeningPortScan> {
const scan = await dispatchPlatformListeningPortScan(options)
if (scan.metadataAvailable) {
rememberListenerMetadata(scan.ports)
return scan
}
return { ...scan, ports: scan.ports.map(recallListenerMetadata) }
}
async function dispatchPlatformListeningPortScan(
options: WorkspacePortScanOptions
): Promise<PlatformListeningPortScan> {
if (process.platform === 'linux') {
return scanLinuxProcPorts()
}
if (process.platform === 'darwin') {
return scanDarwinLsofPorts()
return scanDarwinLsofPorts(options)
}
if (process.platform === 'win32') {
return scanWindowsNetstatPorts()
return scanWindowsNetstatPorts(options)
}
throw new Error(`Port scanning is not supported on ${process.platform}`)
}
async function scanDarwinLsofPorts(): Promise<RawListeningPort[]> {
const { stdout } = await runCommand('lsof', ['-nP', '-iTCP', '-sTCP:LISTEN', '-F', 'pcn'])
async function scanDarwinLsofPorts(
options: WorkspacePortScanOptions
): Promise<PlatformListeningPortScan> {
const { stdout, spawnMs } = await runPortScanCommand('lsof', [
'-nP',
'-iTCP',
'-sTCP:LISTEN',
'-F',
'pcn'
])
const ports = parseLsofListeningOutput(stdout)
if (shouldSkipMetadataCommands(spawnMs, options)) {
return { ports, metadataAvailable: false }
}
const metadata = await loadDarwinProcessMetadata(
new Set(ports.flatMap((p) => (p.pid ? [p.pid] : [])))
)
return ports.map((port) => ({ ...metadata.get(port.pid ?? -1), ...port }))
return {
ports: ports.map((port) => ({ ...metadata.get(port.pid ?? -1), ...port })),
metadataAvailable: true
}
}
async function scanWindowsNetstatPorts(): Promise<RawListeningPort[]> {
const { stdout } = await runCommand('netstat', ['-ano', '-p', 'tcp'])
async function scanWindowsNetstatPorts(
options: WorkspacePortScanOptions
): Promise<PlatformListeningPortScan> {
const { stdout, spawnMs } = await runPortScanCommand('netstat', ['-ano', '-p', 'tcp'])
const ports = parseNetstatListeningOutput(stdout)
if (shouldSkipMetadataCommands(spawnMs, options)) {
return { ports, metadataAvailable: false }
}
const metadata = await loadWindowsProcessMetadata(
new Set(ports.flatMap((p) => (p.pid ? [p.pid] : [])))
)
return ports.map((port) => ({ ...metadata.get(port.pid ?? -1), ...port }))
return {
ports: ports.map((port) => ({ ...metadata.get(port.pid ?? -1), ...port })),
metadataAvailable: true
}
}
async function scanLinuxProcPorts(): Promise<RawListeningPort[]> {
async function scanLinuxProcPorts(): Promise<PlatformListeningPortScan> {
const [tcp4, tcp6] = await Promise.all([
readProcNet('/proc/net/tcp'),
readProcNet('/proc/net/tcp6')
@@ -248,7 +369,7 @@ async function scanLinuxProcPorts(): Promise<RawListeningPort[]> {
})
}
return dedupeRawPorts(rawPorts)
return { ports: dedupeRawPorts(rawPorts), metadataAvailable: true }
}
async function readProcNet(
@@ -321,10 +442,24 @@ async function loadDarwinProcessMetadata(pids: Set<number>): Promise<Map<number,
return result
}
const [cwdOutput, commandOutput] = await Promise.all([
runCommand('lsof', ['-a', '-p', pidList, '-d', 'cwd', '-Fn']).catch(() => null),
runCommand('ps', ['-p', pidList, '-o', 'pid=', '-o', 'command=']).catch(() => null)
])
// Why (#11161): sequential, not Promise.all — the probe worker dispatches one
// command at a time, so issuing both at once would only queue the second.
const cwdOutput = await runPortScanCommand('lsof', [
'-a',
'-p',
pidList,
'-d',
'cwd',
'-Fn'
]).catch(() => null)
const commandOutput = await runPortScanCommand('ps', [
'-p',
pidList,
'-o',
'pid=',
'-o',
'command='
]).catch(() => null)
let currentPid: number | null = null
for (const line of cwdOutput?.stdout.split('\n') ?? []) {
@@ -360,7 +495,7 @@ async function loadWindowsProcessMetadata(
.filter(Number.isFinite)
.map((pid) => `ProcessId=${pid}`)
.join(' OR ')
const { stdout } = await runCommand('powershell.exe', [
const { stdout } = await runPortScanCommand('powershell.exe', [
'-NoProfile',
'-Command',
`Get-CimInstance Win32_Process -Filter "${pidFilter}" | Select-Object ProcessId,Name,CommandLine | ConvertTo-Json -Compress`
@@ -382,62 +517,8 @@ async function loadWindowsProcessMetadata(
return result
}
async function runCommand(command: string, args: string[]): Promise<{ stdout: string }> {
return await new Promise((resolve, reject) => {
let settled = false
let child: ReturnType<typeof execFile> | undefined
const timer = setTimeout(() => {
if (settled) {
return
}
settled = true
child?.kill()
reject(new CommandTimeoutError(command, COMMAND_TIMEOUT_MS))
}, COMMAND_TIMEOUT_MS)
const settle = (callback: () => void): void => {
if (settled) {
return
}
settled = true
clearTimeout(timer)
callback()
}
// Why: Node's execFile timeout only signals the child; if the callback
// never arrives, the workspace port scan would otherwise hang forever.
try {
child = execFile(
command,
args,
{
timeout: COMMAND_TIMEOUT_MS,
maxBuffer: 2 * 1024 * 1024,
windowsHide: true
},
(error, stdout) => {
if (error) {
settle(() => reject(error))
return
}
settle(() => resolve({ stdout: String(stdout) }))
}
)
} catch (error) {
settle(() => reject(error))
}
})
}
class CommandTimeoutError extends Error {
constructor(command: string, timeoutMs: number) {
super(`${command} timed out after ${timeoutMs}ms`)
this.name = 'CommandTimeoutError'
}
}
function isCommandTimeoutError(error: unknown): boolean {
return error instanceof CommandTimeoutError
return error instanceof PortScanCommandTimeoutError
}
async function readTextIfAvailable(filePath: string): Promise<string | undefined> {
@@ -0,0 +1,300 @@
import { readFileSync } from 'node:fs'
import { join, sep } from 'node:path'
import { Worker } from 'node:worker_threads'
import { afterEach, describe, expect, it, vi } from 'vitest'
import {
CALL_DEADLINE_MS,
MAX_QUEUED_CALLS,
PortScanCommandClient,
isPortScanWorkerUnavailableError,
resolveWorkerEntryPath
} from './port-scan-command-client'
import {
PortScanCommandTimeoutError,
type PortScanCommandRequest
} from './port-scan-command-protocol'
type PortScanCommandResponseBody =
| { ok: true; stdout: string; spawnMs: number }
| { ok: false; timedOut: boolean; error: string }
const execFileMock = vi.hoisted(() => vi.fn())
vi.mock('child_process', () => ({
execFile: execFileMock
}))
// A worker_threads stand-in the tests drive directly: it records posted requests
// and lets a test emit message/error/exit without a built worker bundle.
class FakeWorker {
postedRequests: PortScanCommandRequest[] = []
terminated = false
private listeners = new Map<string, Set<(arg?: unknown) => void>>()
on(event: string, listener: (arg?: unknown) => void): this {
const set = this.listeners.get(event) ?? new Set()
set.add(listener)
this.listeners.set(event, set)
return this
}
off(event: string, listener: (arg?: unknown) => void): this {
this.listeners.get(event)?.delete(listener)
return this
}
removeAllListeners(): void {
this.listeners.clear()
}
unref(): void {}
async terminate(): Promise<number> {
this.terminated = true
return 1
}
postMessage(request: PortScanCommandRequest): void {
this.postedRequests.push(request)
}
emit(event: string, arg?: unknown): void {
// Copy first: the client removes its listeners synchronously during a fault.
for (const listener of Array.from(this.listeners.get(event) ?? [])) {
listener(arg)
}
}
respond(body: PortScanCommandResponseBody): void {
const last = this.postedRequests.at(-1)
if (!last) {
throw new Error('no request posted to fake worker')
}
this.emit('message', { id: last.id, ...body })
}
}
function makeFactory(workers: FakeWorker[]): () => Worker {
return () => {
const worker = new FakeWorker()
workers.push(worker)
return worker as unknown as Worker
}
}
function makeClient(workers: FakeWorker[]): PortScanCommandClient {
return new PortScanCommandClient({ workerFactory: makeFactory(workers), log() {} })
}
describe('PortScanCommandClient', () => {
afterEach(() => {
vi.useRealTimers()
execFileMock.mockReset()
})
it('dispatches one command at a time so a stalled spawn cannot fan out', async () => {
const workers: FakeWorker[] = []
const client = makeClient(workers)
const first = client.run('lsof', ['-nP'])
const second = client.run('ps', ['-p', '1'])
await Promise.resolve()
// Why (#11161): a second in-flight request would have its deadline armed
// while the worker's loop is still blocked inside the first uv_spawn.
expect(workers[0].postedRequests.map((request) => request.command)).toEqual(['lsof'])
workers[0].respond({ ok: true, stdout: 'lsof-out', spawnMs: 12 })
await expect(first).resolves.toEqual({ stdout: 'lsof-out', spawnMs: 12 })
expect(workers[0].postedRequests.map((request) => request.command)).toEqual(['lsof', 'ps'])
workers[0].respond({ ok: true, stdout: 'ps-out', spawnMs: 9 })
await expect(second).resolves.toEqual({ stdout: 'ps-out', spawnMs: 9 })
})
it('ignores responses that do not correlate with the active request', async () => {
const workers: FakeWorker[] = []
const client = makeClient(workers)
const pending = client.run('lsof', [])
await Promise.resolve()
workers[0].emit('message', { id: 999, ok: true, stdout: 'stale', spawnMs: 1 })
workers[0].respond({ ok: true, stdout: 'fresh', spawnMs: 3 })
await expect(pending).resolves.toMatchObject({ stdout: 'fresh' })
})
it('rehydrates a worker-side command timeout so the scan can back off', async () => {
const workers: FakeWorker[] = []
const client = makeClient(workers)
const pending = client.run('lsof', [])
await Promise.resolve()
workers[0].respond({ ok: false, timedOut: true, error: 'lsof timed out after 4000ms' })
await expect(pending).rejects.toBeInstanceOf(PortScanCommandTimeoutError)
})
it('reports a worker crash as a non-timeout error and respawns for the next call', async () => {
const workers: FakeWorker[] = []
const client = makeClient(workers)
const pending = client.run('lsof', [])
await Promise.resolve()
workers[0].emit('error', new Error('worker blew up'))
const error = await pending.catch((err: unknown) => err)
expect(error).toBeInstanceOf(Error)
expect(error).not.toBeInstanceOf(PortScanCommandTimeoutError)
expect(workers[0].terminated).toBe(true)
const next = client.run('ps', [])
await Promise.resolve()
expect(workers).toHaveLength(2)
workers[1].respond({ ok: true, stdout: 'ps-out', spawnMs: 2 })
await expect(next).resolves.toMatchObject({ stdout: 'ps-out' })
})
it('terminates a silent worker at the call deadline without arming the backoff', async () => {
vi.useFakeTimers()
const workers: FakeWorker[] = []
const client = makeClient(workers)
const pending = client.run('netstat', ['-ano'])
const settled = pending.catch((err: unknown) => err)
await vi.advanceTimersByTimeAsync(CALL_DEADLINE_MS)
const error = await settled
expect(error).toBeInstanceOf(Error)
expect(error).not.toBeInstanceOf(PortScanCommandTimeoutError)
expect(workers[0].terminated).toBe(true)
})
it('rejects overflow instead of growing the queue without bound', async () => {
const workers: FakeWorker[] = []
const client = makeClient(workers)
const accepted = Array.from({ length: MAX_QUEUED_CALLS + 1 }, () => client.run('lsof', []))
const overflow = client.run('lsof', [])
const error = await overflow.catch((err: unknown) => err)
expect(error).toBeInstanceOf(Error)
expect(error).not.toBeInstanceOf(PortScanCommandTimeoutError)
for (let i = 0; i < accepted.length; i++) {
workers[0].respond({ ok: true, stdout: 'drained', spawnMs: 1 })
}
await expect(Promise.all(accepted)).resolves.toHaveLength(MAX_QUEUED_CALLS + 1)
})
it('fails closed instead of spawning the command on this thread', async () => {
const client = new PortScanCommandClient({
workerFactory: () => {
throw new Error('worker entry not found')
},
log() {}
})
const error = await client.run('lsof', []).catch((err: unknown) => err)
expect(isPortScanWorkerUnavailableError(error)).toBe(true)
expect(execFileMock).not.toHaveBeenCalled()
})
})
// Why: the packaged branch never runs in dev or e2e (both take the __dirname
// path), so it is pinned here at the path-construction level. Whether Electron's
// asar shim can load a Worker entry from inside app.asar is not testable here —
// that still needs a packaged smoke test.
describe('resolveWorkerEntryPath', () => {
const WORKER_ENTRY_FILENAME = 'port-scan-command-worker-entry.js'
it('resolves a packaged build under resourcesPath/app.asar/out/main', () => {
const resourcesPath = join(sep, 'Applications', 'Orca.app', 'Contents', 'Resources')
const resolved = resolveWorkerEntryPath({
isPackaged: true,
resourcesPath,
moduleDir: join(sep, 'unpackaged', 'out', 'main')
})
expect(resolved.startsWith(`${resourcesPath}${sep}`)).toBe(true)
expect(resolved.slice(resourcesPath.length + 1).split(sep)).toEqual([
'app.asar',
'out',
'main',
WORKER_ENTRY_FILENAME
])
})
it('ignores resourcesPath when the app is not packaged', () => {
const moduleDir = join(sep, 'repo', 'out', 'main')
const resolved = resolveWorkerEntryPath({
isPackaged: false,
resourcesPath: join(sep, 'Applications', 'Orca.app', 'Contents', 'Resources'),
moduleDir
})
expect(resolved).toBe(join(moduleDir, WORKER_ENTRY_FILENAME))
expect(resolved).not.toContain('app.asar')
})
// A rename in the build config would leave both branches pointing at a file
// that is never emitted, and only the packaged one fails silently.
it('names the entry the main build actually emits', () => {
const config = readFileSync(
join(import.meta.dirname, '..', '..', '..', 'electron.vite.config.ts'),
'utf8'
)
expect(config).toContain("'port-scan-command-worker-entry': resolve(")
})
})
const REAL_WORKER_BLOCK_MS = 800
// Mirrors the worker entry's protocol but blocks its own thread first, standing
// in for the endpoint-security hook that makes CreateProcessW take seconds.
const BLOCKING_WORKER_SCRIPT = `
const { parentPort } = require('node:worker_threads')
const { execFile } = require('node:child_process')
parentPort.on('message', (request) => {
const startedAt = Date.now()
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ${REAL_WORKER_BLOCK_MS})
execFile(process.execPath, ['-e', ''], () => {
parentPort.postMessage({
id: request.id,
ok: true,
stdout: 'ok',
spawnMs: Date.now() - startedAt
})
})
})
`
describe('PortScanCommandClient on a real worker thread', () => {
it('keeps the calling event loop responsive while a spawn stalls', async () => {
const client = new PortScanCommandClient({
workerFactory: () => new Worker(BLOCKING_WORKER_SCRIPT, { eval: true }),
log() {}
})
let last = Date.now()
let maxStallMs = 0
const probe = setInterval(() => {
const now = Date.now()
maxStallMs = Math.max(maxStallMs, now - last - 10)
last = now
}, 10)
try {
const results = await Promise.all([client.run('lsof', []), client.run('ps', [])])
expect(results.map((result) => result.stdout)).toEqual(['ok', 'ok'])
expect(results[0].spawnMs).toBeGreaterThanOrEqual(REAL_WORKER_BLOCK_MS - 100)
expect(maxStallMs).toBeLessThan(400)
} finally {
clearInterval(probe)
}
}, 30_000)
})
+364
View File
@@ -0,0 +1,364 @@
import { existsSync } from 'node:fs'
import { join } from 'node:path'
import { Worker } from 'node:worker_threads'
import {
PORT_SCAN_COMMAND_TIMEOUT_MS,
PortScanCommandTimeoutError,
type PortScanCommandRequest,
type PortScanCommandResponse
} from './port-scan-command-protocol'
// Why (#11161): a lazily-spawned, unref'd worker runs the port scan's probe
// spawns off the Electron main-process event loop, because libuv performs
// process creation inline on the calling thread. Lifecycle (FIFO one-at-a-time
// dispatch, per-call deadlines, respawn-on-fault, idle teardown, fail-closed)
// mirrors src/main/ai-vault/session-scanner-opencode-sqlite-worker-client.ts;
// the duplicated ~150 lines are cheaper than a premature shared abstraction, so
// a third adopter should extract one.
//
// This module contains the literal text require('electron'), so it must never
// become reachable from a plain-Node fork entry (build-plugins/
// plain-node-entry-guard.ts fails the build on that text, try/catch or not).
// Why: the worker's own loop absorbs the spawn stall, so the client only needs
// a backstop for a wedged thread. Kept at 30s because a scan sits on the
// user-blocking localhost-label allowlist path (src/main/ipc/
// localhost-worktree-labels.ts).
export const WORKER_STALL_GRACE_MS = 26_000
export const CALL_DEADLINE_MS = PORT_SCAN_COMMAND_TIMEOUT_MS + WORKER_STALL_GRACE_MS
// Deliberately far longer than the 30s scan cadence so a visible window does not
// re-create the worker every tick; the renderer stops the interval when hidden,
// so this is effectively the hidden-window teardown.
export const IDLE_TEARDOWN_MS = 5 * 60_000
export const MAX_CONSECUTIVE_DEATHS = 3
// One scan issues at most three commands; anything beyond this is pile-up.
export const MAX_QUEUED_CALLS = 8
export type PortScanCommandResult = { stdout: string; spawnMs: number }
export type PortScanWorkerFactory = () => Worker
// Distinguishes "no worker at all" from a timeout or crash so the scanner can
// log it once and callers never mistake it for a command timeout.
class PortScanWorkerUnavailableError extends Error {}
/** True when a scan failed because the probe worker could not be started. */
export function isPortScanWorkerUnavailableError(error: unknown): boolean {
return error instanceof PortScanWorkerUnavailableError
}
type PendingCall = {
request: PortScanCommandRequest
resolve: (value: PortScanCommandResult) => void
reject: (error: Error) => void
timer: NodeJS.Timeout | null
}
/**
* Main-thread bridge that runs port-scan probe commands on a persistent worker
* thread. Dispatches one command at a time (FIFO), times each call out from
* dispatch, respawns after faults (capped by `MAX_CONSECUTIVE_DEATHS`), tears
* the worker down after `IDLE_TEARDOWN_MS`, and fails closed when no worker can
* be spawned rather than moving process creation back onto the main thread.
*/
export class PortScanCommandClient {
private worker: Worker | null = null
private active: PendingCall | null = null
private queue: PendingCall[] = []
private idleTimer: NodeJS.Timeout | null = null
private consecutiveDeaths = 0
private nextId = 1
private loggedWorkerUnavailable = false
private cleanupWorkerListeners: (() => void) | null = null
private readonly workerFactory: PortScanWorkerFactory
private readonly log: (message: string) => void
constructor(options: { workerFactory: PortScanWorkerFactory; log?: (message: string) => void }) {
this.workerFactory = options.workerFactory
this.log = options.log ?? ((message) => console.warn(message))
}
/**
* Run one probe command on the worker.
* @param command - Executable name (lsof, ps, netstat, powershell.exe).
* @param args - Argument vector passed verbatim to execFile.
* @returns The command's stdout plus its measured process-creation latency.
*/
run(command: string, args: string[]): Promise<PortScanCommandResult> {
return new Promise((resolve, reject) => {
if (this.queue.length >= MAX_QUEUED_CALLS) {
reject(new Error(`Port scan command queue is full; dropped ${command}.`))
return
}
// A fresh burst from full idle starts a new scan: clear any death count
// carried from a prior scan so the respawn cap can't drain this scan early.
if (!this.active && this.queue.length === 0) {
this.consecutiveDeaths = 0
}
this.queue.push({
request: { id: this.nextId++, command, args },
resolve,
reject,
timer: null
})
this.pump()
})
}
private pump(): void {
if (this.active || this.queue.length === 0) {
return
}
const worker = this.ensureWorker()
if (!worker) {
this.failQueuedAsUnavailable()
return
}
const call = this.queue.shift()
if (!call) {
return
}
this.active = call
this.clearIdleTimer()
// Why (#11161): one at a time. uv_spawn blocks the worker's own loop, so a
// second concurrent request would have its deadline armed while the first
// spawn is still stalling the thread, producing a false timeout.
call.timer = setTimeout(() => this.onDeadline(call), CALL_DEADLINE_MS)
call.timer.unref?.()
worker.postMessage(call.request)
}
private ensureWorker(): Worker | null {
if (this.worker) {
return this.worker
}
try {
const worker = this.workerFactory()
const onMessage = (response: PortScanCommandResponse): void => this.onMessage(response)
const onError = (error: Error): void => this.onWorkerFault(error)
const onExit = (code: number): void => this.onWorkerExit(code)
worker.on('message', onMessage)
worker.on('error', onError)
worker.on('exit', onExit)
this.cleanupWorkerListeners = () => {
worker.off('message', onMessage)
worker.off('error', onError)
worker.off('exit', onExit)
}
// Never keep the app alive for a port scan.
worker.unref?.()
this.worker = worker
return worker
} catch (err) {
// Why (#11161): never fall back to in-process execFile here; a missing
// bundle must report port scanning as unavailable rather than reintroduce
// the main-thread freeze this worker boundary exists to prevent.
if (!this.loggedWorkerUnavailable) {
this.loggedWorkerUnavailable = true
this.log(`[workspace-ports] probe worker unavailable. ${errorMessage(err)}`)
}
return null
}
}
private onMessage(response: PortScanCommandResponse): void {
const call = this.active
if (!call || call.request.id !== response.id) {
return
}
this.consecutiveDeaths = 0
if (response.ok) {
this.settle(call, () => call.resolve({ stdout: response.stdout, spawnMs: response.spawnMs }))
} else {
const error = response.timedOut
? new PortScanCommandTimeoutError(response.error)
: new Error(response.error)
this.settle(call, () => call.reject(error))
}
this.afterSettle()
}
private onDeadline(call: PendingCall): void {
if (this.active !== call) {
return
}
// Plain Error on purpose: a wedged worker is not a command timeout and must
// never feed the scanner's timeout backoff.
this.onWorkerFault(new Error(`Port scan probe worker stalled after ${CALL_DEADLINE_MS}ms`))
}
private onWorkerExit(code: number): void {
// A clean self-exit is not a death, but the stale handle must be dropped or
// the next dispatch would post into a dead worker and stall to its deadline.
if (code === 0 && !this.active && this.queue.length === 0) {
this.destroyWorker()
return
}
this.onWorkerFault(new Error(`Port scan probe worker exited with code ${code}`))
}
private onWorkerFault(error: Error): void {
const failed = this.active
this.destroyWorker()
this.consecutiveDeaths++
if (failed) {
this.settle(failed, () => failed.reject(error))
}
if (this.consecutiveDeaths >= MAX_CONSECUTIVE_DEATHS) {
this.drainQueueAfterCrashLoop(error)
return
}
if (this.queue.length > 0) {
this.pump()
}
}
private drainQueueAfterCrashLoop(error: Error): void {
const pending = this.queue
this.queue = []
this.consecutiveDeaths = 0
const drainError = new Error(`Port scan probe worker crashed repeatedly (${error.message})`)
for (const call of pending) {
this.settle(call, () => call.reject(drainError))
}
}
private failQueuedAsUnavailable(): void {
const pending = this.queue
this.queue = []
for (const call of pending) {
this.settle(call, () =>
call.reject(new PortScanWorkerUnavailableError('port scan probe worker spawn failed'))
)
}
}
private settle(call: PendingCall, run: () => void): void {
if (call.timer) {
clearTimeout(call.timer)
call.timer = null
}
if (this.active === call) {
this.active = null
}
run()
}
private afterSettle(): void {
if (this.queue.length > 0) {
this.pump()
} else {
this.scheduleIdleTeardown()
}
}
private scheduleIdleTeardown(): void {
this.clearIdleTimer()
if (!this.worker) {
return
}
this.idleTimer = setTimeout(() => this.teardownIfIdle(), IDLE_TEARDOWN_MS)
this.idleTimer.unref?.()
}
private teardownIfIdle(): void {
this.idleTimer = null
// Only tear down with nothing active AND nothing queued: a request arriving
// as the timer fires must never be lost to a self-exiting worker.
if (this.active || this.queue.length > 0) {
return
}
this.destroyWorker()
}
private clearIdleTimer(): void {
if (this.idleTimer) {
clearTimeout(this.idleTimer)
this.idleTimer = null
}
}
private destroyWorker(): void {
this.clearIdleTimer()
const worker = this.worker
this.worker = null
if (!worker) {
return
}
this.cleanupWorkerListeners?.()
this.cleanupWorkerListeners = null
worker.removeAllListeners()
// Terminating can orphan a probe child mid-spawn; the worker reaps what it
// can on exit, and every probe here is short-lived.
void worker.terminate().catch(() => undefined)
}
}
function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error)
}
const WORKER_ENTRY_FILENAME = 'port-scan-command-worker-entry.js'
/** Where the built worker entry can live: packaged resources or the build dir. */
export type WorkerEntryLayout = {
isPackaged: boolean
resourcesPath: string
moduleDir: string
}
/**
* Resolve the built worker entry for one runtime layout.
* @param layout - Packaged flag plus both candidate roots.
* @returns Path passed to `new Worker()`.
*/
export function resolveWorkerEntryPath(layout: WorkerEntryLayout): string {
// Packaged builds leave this entry inside app.asar — only forked child
// processes are asarUnpack'd — so it resolves off resourcesPath rather than
// the bundler's __dirname, matching the shipped stt/warp/opencode workers.
// Split out from the electron read so the packaged branch is testable without
// a packaged build.
if (layout.isPackaged) {
return join(layout.resourcesPath, 'app.asar', 'out', 'main', WORKER_ENTRY_FILENAME)
}
return join(layout.moduleDir, WORKER_ENTRY_FILENAME)
}
function currentWorkerEntryLayout(): WorkerEntryLayout {
let app: { isPackaged: boolean } | null = null
try {
app = require('electron').app ?? null
} catch {
app = null
}
return {
isPackaged: app?.isPackaged === true,
resourcesPath: process.resourcesPath,
moduleDir: __dirname
}
}
function defaultWorkerFactory(): Worker {
const workerPath = resolveWorkerEntryPath(currentWorkerEntryLayout())
// Why: a missing built entry must throw synchronously so the client can fail
// closed before it waits on a worker that can never post a result.
if (!existsSync(workerPath)) {
throw new Error(`Port scan command worker entry not found: ${workerPath}`)
}
return new Worker(workerPath)
}
let sharedClient: PortScanCommandClient | null = null
/**
* Run a port-scan probe command through the process-wide worker client.
* @param command - Executable name (lsof, ps, netstat, powershell.exe).
* @param args - Argument vector passed verbatim to execFile.
* @returns The command's stdout plus its measured process-creation latency.
*/
export function runPortScanCommand(
command: string,
args: string[]
): Promise<PortScanCommandResult> {
sharedClient ??= new PortScanCommandClient({ workerFactory: defaultWorkerFactory })
return sharedClient.run(command, args)
}
@@ -0,0 +1,97 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { runPortScanCommandInProcess } from './port-scan-command-execution'
import {
PORT_SCAN_COMMAND_TIMEOUT_MS,
PortScanCommandTimeoutError,
WATCHDOG_GRACE_MS
} from './port-scan-command-protocol'
const execFileMock = vi.hoisted(() => vi.fn())
vi.mock('child_process', () => ({
execFile: execFileMock
}))
// Why (#11161): must outlast the whole watchdog budget, otherwise a watchdog
// armed before execFile still survives the stall and the ordering goes unpinned.
const SPAWN_STALL_MS = PORT_SCAN_COMMAND_TIMEOUT_MS + WATCHDOG_GRACE_MS + 200
const LSOF_OUTPUT = ['p123', 'cnode', 'n127.0.0.1:5173'].join('\n')
/** Emulates a hooked CreateProcessW: blocks the calling thread inside uv_spawn. */
function blockCallingThread(ms: number): void {
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms)
}
describe('runPortScanCommandInProcess', () => {
afterEach(() => {
vi.useRealTimers()
execFileMock.mockReset()
})
it('arms the watchdog only after process creation returns', async () => {
execFileMock.mockImplementation(
(_command: string, _args: string[], _options: unknown, callback: unknown) => {
blockCallingThread(SPAWN_STALL_MS)
// The command itself is healthy once it finally starts.
setTimeout(() => (callback as (e: null, out: string) => void)(null, LSOF_OUTPUT), 5)
return { kill: vi.fn() }
}
)
const result = await runPortScanCommandInProcess('lsof', ['-nP', '-iTCP'])
expect(result.stdout).toBe(LSOF_OUTPUT)
expect(result.spawnMs).toBeGreaterThanOrEqual(SPAWN_STALL_MS - 100)
})
it('kills the child and times out when the callback never arrives', async () => {
vi.useFakeTimers()
const killMock = vi.fn()
execFileMock.mockImplementation(() => ({ kill: killMock }))
let settled = false
const promise = runPortScanCommandInProcess('lsof', []).catch((error: unknown) => {
settled = true
return error
})
await vi.advanceTimersByTimeAsync(PORT_SCAN_COMMAND_TIMEOUT_MS)
expect(settled).toBe(false)
await vi.advanceTimersByTimeAsync(WATCHDOG_GRACE_MS)
expect(await promise).toBeInstanceOf(PortScanCommandTimeoutError)
expect(killMock).toHaveBeenCalled()
})
it("classifies Node's own execFile timeout kill as a command timeout", async () => {
execFileMock.mockImplementation(
(_command: string, _args: string[], _options: unknown, callback: unknown) => {
const killed = Object.assign(new Error('Command failed: lsof'), {
killed: true,
signal: 'SIGTERM'
})
setTimeout(() => (callback as (e: Error) => void)(killed), 0)
return { kill: vi.fn() }
}
)
await expect(runPortScanCommandInProcess('lsof', [])).rejects.toBeInstanceOf(
PortScanCommandTimeoutError
)
})
it('leaves a genuine command failure unclassified so the scan does not back off', async () => {
execFileMock.mockImplementation(
(_command: string, _args: string[], _options: unknown, callback: unknown) => {
setTimeout(() => (callback as (e: Error) => void)(new Error('spawn ENOENT')), 0)
return { kill: vi.fn() }
}
)
const error = await runPortScanCommandInProcess('lsof', []).catch((err: unknown) => err)
expect(error).toBeInstanceOf(Error)
expect(error).not.toBeInstanceOf(PortScanCommandTimeoutError)
})
})
@@ -0,0 +1,115 @@
import { execFile, type ChildProcess } from 'node:child_process'
import {
PORT_SCAN_COMMAND_TIMEOUT_MS,
PortScanCommandTimeoutError,
WATCHDOG_GRACE_MS,
portScanCommandTimeoutMessage
} from './port-scan-command-protocol'
// Why (#11161): libuv runs uv_spawn inline on whichever event loop calls it, so
// an endpoint-security hook on CreateProcessW stalls that thread for the whole
// spawn. This module is only ever entered from the worker thread
// (port-scan-command-worker-entry.ts); nothing on the main thread may import it.
const WATCHDOG_TIMEOUT_MS = PORT_SCAN_COMMAND_TIMEOUT_MS + WATCHDOG_GRACE_MS
const activeChildren = new Set<ChildProcess>()
/**
* Run a port-scan probe command and report how long process creation took.
* @param command - Executable name resolved against the worker's PATH.
* @param args - Argument vector passed verbatim, never shell-interpolated.
* @returns The command's stdout plus `spawnMs`, the measured process-creation
* latency callers use to skip optional follow-up commands on a stalled host.
* @throws PortScanCommandTimeoutError when the command itself outran its budget.
*/
export async function runPortScanCommandInProcess(
command: string,
args: string[]
): Promise<{ stdout: string; spawnMs: number }> {
return await new Promise((resolve, reject) => {
let settled = false
let timer: NodeJS.Timeout | null = null
let spawnMs = 0
let child: ChildProcess | undefined
const settle = (callback: () => void): void => {
if (settled) {
return
}
settled = true
if (timer) {
clearTimeout(timer)
}
if (child) {
activeChildren.delete(child)
}
callback()
}
const startedAt = Date.now()
try {
child = execFile(
command,
args,
{
timeout: PORT_SCAN_COMMAND_TIMEOUT_MS,
maxBuffer: 2 * 1024 * 1024,
windowsHide: true
},
(error, stdout) => {
if (error) {
// Node kills its own execFile timeout with a signal, which surfaces
// as killed:true — the only way to tell a timeout from a real error.
const timedOut = (error as { killed?: boolean }).killed === true
settle(() =>
reject(
timedOut
? new PortScanCommandTimeoutError(
portScanCommandTimeoutMessage(command, PORT_SCAN_COMMAND_TIMEOUT_MS)
)
: error
)
)
return
}
settle(() => resolve({ stdout: String(stdout), spawnMs }))
}
)
} catch (error) {
settle(() => reject(error))
return
}
// Why (#11161): measured after execFile returns, because that call blocks
// for the whole of process creation. Arming the watchdog earlier would
// charge the spawn stall against the command's budget and fire immediately.
spawnMs = Date.now() - startedAt
if (settled || !child) {
return
}
activeChildren.add(child)
timer = setTimeout(() => {
settle(() => {
child?.kill()
reject(
new PortScanCommandTimeoutError(
portScanCommandTimeoutMessage(command, WATCHDOG_TIMEOUT_MS)
)
)
})
}, WATCHDOG_TIMEOUT_MS)
})
}
/** Best-effort reap so a terminated worker does not orphan an in-flight probe. */
export function killActivePortScanCommands(): void {
for (const child of activeChildren) {
try {
child.kill()
} catch {
// Already exited; nothing to reap.
}
}
activeChildren.clear()
}
@@ -0,0 +1,18 @@
import { readFileSync } from 'node:fs'
import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
// Why (#11161): libuv runs process creation inline on the calling event loop,
// so the port scan only stays off CrBrowserMain while these main-thread modules
// spawn nothing themselves. Spawning belongs to port-scan-command-execution.ts,
// which only the worker entry imports.
const MAIN_THREAD_MODULES = ['local-workspace-port-scanner.ts', 'port-scan-command-client.ts']
describe('port scan main-thread spawn boundary', () => {
it.each(MAIN_THREAD_MODULES)('keeps %s free of child_process', (fileName) => {
const source = readFileSync(join(import.meta.dirname, fileName), 'utf8')
expect(source).not.toMatch(/from\s+['"](node:)?child_process['"]/)
expect(source).not.toMatch(/require\(\s*['"](node:)?child_process['"]\s*\)/)
})
})
@@ -0,0 +1,36 @@
// Why (#11161): request/response shapes plus the timeout error shared by the
// port-scan worker entry and its main-thread client. Kept free of Electron,
// node:worker_threads and node:child_process so importing it from either side
// can never drag the other side's dependencies across the boundary.
export const PORT_SCAN_COMMAND_TIMEOUT_MS = 4_000
// Node's own execFile timeout is the primary killer; the manual watchdog only
// covers "the callback never arrived", so it must fire strictly later.
export const WATCHDOG_GRACE_MS = 1_000
export type PortScanCommandRequest = {
id: number
command: string
args: string[]
}
export type PortScanCommandResponse =
| { id: number; ok: true; stdout: string; spawnMs: number }
| { id: number; ok: false; timedOut: boolean; error: string }
/**
* Raised when a port-scan command was killed after exceeding its budget. Errors
* do not survive structured clone as subclasses, so the worker reports a
* `timedOut` flag and the client reconstructs this type from it.
*/
export class PortScanCommandTimeoutError extends Error {
constructor(message: string) {
super(message)
this.name = 'PortScanCommandTimeoutError'
}
}
/** Shared wording so worker-side and client-side timeouts read identically. */
export function portScanCommandTimeoutMessage(command: string, timeoutMs: number): string {
return `${command} timed out after ${timeoutMs}ms`
}
@@ -0,0 +1,53 @@
import { parentPort } from 'node:worker_threads'
import {
killActivePortScanCommands,
runPortScanCommandInProcess
} from './port-scan-command-execution'
import {
PortScanCommandTimeoutError,
type PortScanCommandRequest,
type PortScanCommandResponse
} from './port-scan-command-protocol'
// Why (#11161): process creation blocks the event loop that issues it. Running
// the port scan's probe commands on this worker thread keeps an EDR-hooked
// CreateProcessW off CrBrowserMain. The client dispatches one request at a
// time, so this loop stays serial; imports must remain electron-free.
if (!parentPort) {
throw new Error('Port scan command worker must run with a parent port.')
}
const port = parentPort
process.once('exit', killActivePortScanCommands)
async function handleRequest(request: PortScanCommandRequest): Promise<PortScanCommandResponse> {
try {
const { stdout, spawnMs } = await runPortScanCommandInProcess(request.command, request.args)
return { id: request.id, ok: true, stdout, spawnMs }
} catch (err) {
return {
id: request.id,
ok: false,
timedOut: err instanceof PortScanCommandTimeoutError,
error: err instanceof Error ? err.message : String(err)
}
}
}
port.on('message', (request: PortScanCommandRequest) => {
void handleRequest(request).then((response) => {
try {
port.postMessage(response)
} catch {
// A non-cloneable result would otherwise post nothing and leave the client
// waiting out its deadline; fail that request fast instead.
port.postMessage({
id: request.id,
ok: false,
timedOut: false,
error: 'Port scan command result could not be serialized.'
})
}
})
})
@@ -0,0 +1,57 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { killWorkspacePort } from './workspace-port-ownership'
const scanWorkspacePortsMock = vi.hoisted(() => vi.fn())
vi.mock('./local-workspace-port-scanner', () => ({
scanWorkspacePorts: scanWorkspacePortsMock
}))
const worktrees = [{ id: 'repo::/repo', repoId: 'repo', displayName: 'main', path: '/repo' }]
describe('killWorkspacePort', () => {
afterEach(() => {
scanWorkspacePortsMock.mockReset()
vi.restoreAllMocks()
})
// Regression for #11161 review: on an EDR-hooked host the background poller
// alternates the metadata skip, so an unscoped skip would fail Stop with
// "Only workspace-owned local processes can be stopped here" every other try.
it('requires owner metadata from the authorizing re-scan', async () => {
scanWorkspacePortsMock.mockResolvedValue({ platform: 'darwin', scannedAt: 0, ports: [] })
await killWorkspacePort(worktrees, { pid: 123, port: 5173 })
expect(scanWorkspacePortsMock).toHaveBeenCalledWith(worktrees, undefined, {
requireMetadata: true
})
})
it('refuses a pid the re-scan does not attribute to a workspace', async () => {
const killSpy = vi.spyOn(process, 'kill').mockImplementation(() => true)
scanWorkspacePortsMock.mockResolvedValue({
platform: 'darwin',
scannedAt: 0,
ports: [
{
id: '127.0.0.1:5173:123',
bindHost: '127.0.0.1',
connectHost: '127.0.0.1',
port: 5173,
pid: 123,
protocol: 'http',
kind: 'external'
}
]
})
const result = await killWorkspacePort(worktrees, { pid: 123, port: 5173 })
expect(result).toEqual({
ok: false,
reason: 'Only workspace-owned local processes can be stopped here.'
})
expect(killSpy).not.toHaveBeenCalled()
})
})
+7 -4
View File
@@ -8,7 +8,7 @@ import type {
WorkspacePortProbe,
WorkspacePortScanResult
} from '../../shared/workspace-ports'
import { scanWorkspacePorts } from './local-workspace-port-scanner'
import { scanWorkspacePorts, type WorkspacePortScanOptions } from './local-workspace-port-scanner'
export type WorkspacePortProbeInput = WorkspacePortProbe & {
connectionId?: string | null
@@ -82,7 +82,9 @@ export async function killWorkspacePort(
return { ok: false, reason: 'Invalid process or port.' }
}
const scan = await scanWorkspacePorts([...worktrees])
// Why (#11161): this re-scan is the authorization check for SIGTERM, so it
// must never land on a cycle that skipped the owner-attribution metadata.
const scan = await scanWorkspacePorts([...worktrees], undefined, { requireMetadata: true })
const port = scan.ports.find(
(candidate) => candidate.pid === args.pid && candidate.port === args.port
)
@@ -113,7 +115,8 @@ export async function killWorkspacePort(
}
export async function scanWorkspacePortProbes(
worktrees: readonly WorkspacePortProbe[]
worktrees: readonly WorkspacePortProbe[],
options?: WorkspacePortScanOptions
): Promise<WorkspacePortScanResult> {
return scanWorkspacePorts([...worktrees])
return scanWorkspacePorts([...worktrees], undefined, options)
}
+12
View File
@@ -9,4 +9,16 @@ export class QuitTeardownStartGate {
this.started = true
return true
}
hasStarted(): boolean {
return this.started
}
resetForTests(): void {
this.started = false
}
}
// Why shared, not per-caller: subsystems fence new work on "the committed quit has begun", and a
// second instance would answer no while the real teardown is already draining what it snapshotted.
export const quitTeardownStartGate = new QuitTeardownStartGate()
@@ -1,4 +1,4 @@
import { describe, expect, it } from 'vitest'
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { MobilePairingConnectionContext } from '../runtime-rpc'
import { DesktopRelayService, pairingAuthorizationForContext } from './desktop-relay-service'
@@ -61,6 +61,47 @@ describe('pairingAuthorizationForContext', () => {
})
})
describe('liveness safety net lifecycle', () => {
afterEach(() => vi.useRealTimers())
it('fences hard: the tick stops on fence and re-arms on the next auth mutation', () => {
// Why: a tick surviving the pre-sign-out fence could catch the window
// before the profile wipe and briefly resurrect a broker.
vi.useFakeTimers()
const coordinator = {
reconcile: vi.fn(),
ensureLive: vi.fn(),
fenceAndCloseNow: vi.fn(),
stop: vi.fn()
}
const service = Object.create(DesktopRelayService.prototype) as DesktopRelayService
Object.assign(service, {
coordinator,
demandLedger: { nextPendingExpiry: () => null },
stopped: false,
livenessTimer: null,
demandExpiryTimer: null
})
service.start()
vi.advanceTimersByTime(5 * 60_000)
expect(coordinator.ensureLive).toHaveBeenCalledTimes(1)
service.fenceAndCloseNow()
expect(coordinator.fenceAndCloseNow).toHaveBeenCalledOnce()
vi.advanceTimersByTime(30 * 60_000)
expect(coordinator.ensureLive).toHaveBeenCalledTimes(1)
service.authMutated()
vi.advanceTimersByTime(5 * 60_000)
expect(coordinator.ensureLive).toHaveBeenCalledTimes(2)
service.stop()
vi.advanceTimersByTime(30 * 60_000)
expect(coordinator.ensureLive).toHaveBeenCalledTimes(2)
})
})
describe('local-only mobile pairing', () => {
it('refuses endpoint discovery and provisioning without opening Relay demand', async () => {
const registry = {
@@ -96,9 +96,6 @@ export class DesktopRelayService {
start(): void {
this.refreshDemand()
if (!this.livenessTimer) {
this.livenessTimer = setInterval(() => this.ensureLive(), RELAY_LIVENESS_INTERVAL_MS)
}
}
// Safe to call from any wake signal (power resume, network change).
@@ -113,6 +110,13 @@ export class DesktopRelayService {
}
fenceAndCloseNow(): void {
// Why: a fence must be hard — a surviving liveness tick could catch the
// window between the pre-sign-out fence and the profile wipe and briefly
// resurrect a broker. The next auth mutation re-arms via refreshDemand.
if (this.livenessTimer) {
clearInterval(this.livenessTimer)
this.livenessTimer = null
}
this.coordinator.fenceAndCloseNow()
}
@@ -307,6 +311,9 @@ export class DesktopRelayService {
if (this.stopped) {
return
}
if (!this.livenessTimer) {
this.livenessTimer = setInterval(() => this.ensureLive(), RELAY_LIVENESS_INTERVAL_MS)
}
if (this.demandExpiryTimer) {
clearTimeout(this.demandExpiryTimer)
this.demandExpiryTimer = null
@@ -94,4 +94,36 @@ describe('SshConnectionManager', () => {
expect(await manager.connect(target)).toBe(replacement)
expect(manager.getConnection(target.id)).toBe(replacement)
})
it('closes a late-resolved connection without evicting the replacement', async () => {
let resolveFirst!: () => void
mockState.connectResults.push(
new Promise<void>((resolve) => {
resolveFirst = resolve
}),
Promise.resolve()
)
const manager = new SshConnectionManager({ onStateChange: vi.fn() })
const firstConnect = manager.connect(target)
await manager.disconnect(target.id)
const replacement = await manager.connect(target)
resolveFirst()
const late = await firstConnect
await manager.disconnectConnection(target.id, late)
expect(mockState.instances[0].disconnect).toHaveBeenCalledTimes(2)
expect(manager.getConnection(target.id)).toBe(replacement)
})
it('clears the pool entry when the closed connection is still the registered one', async () => {
const manager = new SshConnectionManager({ onStateChange: vi.fn() })
const conn = await manager.connect(target)
await manager.disconnectConnection(target.id, conn)
expect(mockState.instances[0].disconnect).toHaveBeenCalledOnce()
expect(manager.getConnection(target.id)).toBeUndefined()
})
})
+12
View File
@@ -76,6 +76,18 @@ export class SshConnectionManager {
}
}
/**
* Close one specific connection, clearing the pool entry only when it is still the registered one.
* Why: a cancelled connect whose transport opened late owns that exact connection — disconnecting
* by target id would tear down the replacement's live transport instead.
*/
async disconnectConnection(targetId: string, conn: SshConnection): Promise<void> {
await conn.disconnect()
if (this.connections.get(targetId) === conn) {
this.connections.delete(targetId)
}
}
async reconnect(targetId: string): Promise<void> {
const conn = this.connections.get(targetId)
if (!conn) {
@@ -0,0 +1,106 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import {
PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR,
PTY_CONSUMER_OWNER_RECOVERY_SUPERSEDED_ERROR
} from '../../shared/pty-consumer-session'
import { retrySshOwnerRecoveryWhileBlocked } from './ssh-owner-recovery-retry'
function publicationPendingError(): Error & { code: number } {
return Object.assign(new Error('Owner grant publication is still pending'), {
code: PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR
})
}
function supersededError(): Error & { code: number } {
return Object.assign(new Error('Owner recovery generation was superseded'), {
code: PTY_CONSUMER_OWNER_RECOVERY_SUPERSEDED_ERROR
})
}
function openGate() {
return {
isCurrent: () => true,
onClosed: () => () => {}
}
}
describe('SSH owner recovery retry', () => {
afterEach(() => {
vi.useRealTimers()
})
it('recovers once the incumbent grant publication settles', async () => {
vi.useFakeTimers()
const attempt = vi
.fn<() => Promise<string>>()
.mockRejectedValueOnce(publicationPendingError())
.mockRejectedValueOnce(publicationPendingError())
.mockResolvedValue('recovered')
const recovery = retrySshOwnerRecoveryWhileBlocked(attempt, openGate())
await vi.advanceTimersByTimeAsync(75)
await expect(recovery).resolves.toBe('recovered')
expect(attempt).toHaveBeenCalledTimes(3)
})
it('stops at the publication-settlement deadline', async () => {
vi.useFakeTimers()
const error = publicationPendingError()
const attempt = vi.fn<() => Promise<never>>().mockRejectedValue(error)
const recovery = retrySshOwnerRecoveryWhileBlocked(attempt, openGate(), 60)
const rejection = expect(recovery).rejects.toBe(error)
await vi.advanceTimersByTimeAsync(60)
await rejection
expect(attempt).toHaveBeenCalledTimes(3)
})
it('does not retry unrelated failures', async () => {
const error = Object.assign(new Error('stale owner'), { code: -32041 })
const attempt = vi.fn<() => Promise<never>>().mockRejectedValue(error)
await expect(retrySshOwnerRecoveryWhileBlocked(attempt, openGate())).rejects.toBe(error)
expect(attempt).toHaveBeenCalledOnce()
})
it('retries while a superseded transport is closing', async () => {
vi.useFakeTimers()
const attempt = vi
.fn<() => Promise<string>>()
.mockRejectedValueOnce(supersededError())
.mockResolvedValue('recovered')
const recovery = retrySshOwnerRecoveryWhileBlocked(attempt, openGate())
await vi.advanceTimersByTimeAsync(25)
await expect(recovery).resolves.toBe('recovered')
expect(attempt).toHaveBeenCalledTimes(2)
})
it('stops waiting when the relay channel closes', async () => {
vi.useFakeTimers()
let current = true
let close: (() => void) | undefined
const error = publicationPendingError()
const attempt = vi.fn<() => Promise<never>>().mockRejectedValue(error)
const recovery = retrySshOwnerRecoveryWhileBlocked(attempt, {
isCurrent: () => current,
onClosed: (listener) => {
close = listener
return () => {
close = undefined
}
}
})
await vi.advanceTimersByTimeAsync(0)
expect(close).toBeTypeOf('function')
current = false
close?.()
await expect(recovery).rejects.toBe(error)
expect(attempt).toHaveBeenCalledOnce()
})
})
+69
View File
@@ -0,0 +1,69 @@
import {
PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR,
PTY_CONSUMER_OWNER_RECOVERY_SUPERSEDED_ERROR
} from '../../shared/pty-consumer-session'
// Why: bound polling when publication is settling or a superseded attempt is closing its transport.
export const SSH_OWNER_RECOVERY_WAIT_MS = 3_000
const SSH_OWNER_RECOVERY_INITIAL_DELAY_MS = 25
const SSH_OWNER_RECOVERY_MAX_DELAY_MS = 250
type SshOwnerRecoveryRetryGate = {
isCurrent: () => boolean
onClosed: (listener: () => void) => () => void
}
function waitForRetry(delayMs: number, gate: SshOwnerRecoveryRetryGate): Promise<void> {
return new Promise((resolve) => {
let settled = false
let timer: ReturnType<typeof setTimeout> | undefined
let unsubscribe = (): void => {}
const finish = (): void => {
if (settled) {
return
}
settled = true
clearTimeout(timer)
unsubscribe()
resolve()
}
unsubscribe = gate.onClosed(finish)
timer = setTimeout(finish, delayMs)
timer.unref?.()
if (!gate.isCurrent()) {
finish()
}
})
}
export async function retrySshOwnerRecoveryWhileBlocked<T>(
attempt: () => Promise<T>,
gate: SshOwnerRecoveryRetryGate,
waitMs: number = SSH_OWNER_RECOVERY_WAIT_MS
): Promise<T> {
const deadline = Date.now() + waitMs
let delayMs = SSH_OWNER_RECOVERY_INITIAL_DELAY_MS
while (true) {
try {
return await attempt()
} catch (error) {
const code = (error as { code?: unknown } | null | undefined)?.code
if (
(code !== PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR &&
code !== PTY_CONSUMER_OWNER_RECOVERY_SUPERSEDED_ERROR) ||
!gate.isCurrent()
) {
throw error
}
const remainingMs = deadline - Date.now()
if (remainingMs <= 0) {
throw error
}
await waitForRetry(Math.min(delayMs, remainingMs), gate)
if (!gate.isCurrent()) {
throw error
}
delayMs = Math.min(delayMs * 2, SSH_OWNER_RECOVERY_MAX_DELAY_MS)
}
}
}
@@ -1,4 +1,8 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import {
PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR,
PTY_CONSUMER_OWNER_RECOVERY_SUPERSEDED_ERROR
} from '../../shared/pty-consumer-session'
import { SshRelaySession } from './ssh-relay-session'
import {
createMismatchedOwnerRecoveryError,
@@ -154,6 +158,82 @@ describe('SshRelaySession consumer recovery durability', () => {
session.dispose()
})
it('retries a pending incumbent publication before recovering its persisted lease', async () => {
vi.useFakeTimers()
try {
const targetId = 'target-owner-publication-pending'
const deps = createMockDeps()
vi.mocked(deps.mockStore.getSshPtyConsumerRecovery).mockReturnValue({
targetId,
clientInstanceId: 'persisted-client',
serverBuildId: 'test-relay-build',
clientGeneration: 1,
ownerGeneration: 1,
ownerLease: 'persisted-owner'
})
openConsumerSessionMock.mockRejectedValueOnce(
Object.assign(new Error('Owner grant publication is still pending'), {
code: PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR
})
)
const session = new SshRelaySession(
targetId,
deps.getMainWindow,
deps.mockStore,
deps.mockPortForward
)
const establishing = session.establish(deps.mockConn)
await vi.advanceTimersByTimeAsync(25)
await establishing
expect(openConsumerSessionMock).toHaveBeenCalledTimes(2)
expect(openConsumerSessionMock.mock.calls[0]?.[1]).toMatchObject({
clientInstanceId: 'persisted-client',
resume: { ownerGeneration: 1, ownerLease: 'persisted-owner' }
})
session.dispose()
} finally {
vi.useRealTimers()
}
})
it('preserves persisted recovery while a superseding transport is still live', async () => {
vi.useFakeTimers()
try {
const targetId = 'target-owner-generation-superseded'
const deps = createMockDeps()
vi.mocked(deps.mockStore.getSshPtyConsumerRecovery).mockReturnValue({
targetId,
clientInstanceId: 'persisted-client',
serverBuildId: 'test-relay-build',
clientGeneration: 1,
ownerGeneration: 1,
ownerLease: 'persisted-owner'
})
const superseded = Object.assign(new Error('Owner recovery generation was superseded'), {
code: PTY_CONSUMER_OWNER_RECOVERY_SUPERSEDED_ERROR
})
openConsumerSessionMock.mockRejectedValue(superseded)
const session = new SshRelaySession(
targetId,
deps.getMainWindow,
deps.mockStore,
deps.mockPortForward
)
const failed = expect(session.establish(deps.mockConn)).rejects.toBe(superseded)
await vi.advanceTimersByTimeAsync(3_000)
await failed
expect(openConsumerSessionMock.mock.calls.length).toBeGreaterThan(1)
expect(deps.mockStore.removeSshPtyConsumerRecovery).not.toHaveBeenCalled()
session.dispose()
} finally {
vi.useRealTimers()
}
})
it('keeps destructive disposal pending until consumer recovery is removed', async () => {
const { mockStore, mockPortForward, getMainWindow } = createMockDeps()
vi.mocked(mockStore.getSshPtyConsumerRecovery).mockReturnValue({
@@ -232,6 +312,46 @@ describe('SshRelaySession consumer recovery durability', () => {
expect(openConsumerSessionMock).toHaveBeenCalledTimes(1)
})
it('leaves recovery state alone when a newer owner already claimed the target record', async () => {
const targetId = 'target-stale-owner-loser'
const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps()
vi.mocked(mockStore.getSshPtyConsumerRecovery).mockReturnValue({
targetId,
clientInstanceId: 'persisted-client',
serverBuildId: 'test-relay-build',
clientGeneration: 1,
ownerGeneration: 1,
ownerLease: 'stale-owner'
})
const winner = {
mode: 'negotiated' as const,
clientInstanceId: 'persisted-client',
clientGeneration: 2,
ownerGeneration: 5,
ownerLease: 'winner-owner'
}
openConsumerSessionMock.mockImplementationOnce(() => {
// Why inside the rejection: the record is target-scoped, so the winner can land while this
// attempt is still unwinding its own resume.
const record = getSshPtyConsumerRecovery(targetId)!
record.owner = winner
record.checkpointsByAppPtyId.set('pty-1', {
id: 'pty-1'
} as unknown as never)
return Promise.reject(createMismatchedOwnerRecoveryError())
})
openConsumerSessionMock.mockRejectedValueOnce(new Error('fresh open failed'))
const session = new SshRelaySession(targetId, getMainWindow, mockStore, mockPortForward)
await expect(session.establish(mockConn)).rejects.toThrow('fresh open failed')
const record = getSshPtyConsumerRecovery(targetId)!
expect(record.owner).toBe(winner)
expect(record.checkpointsByAppPtyId.has('pty-1')).toBe(true)
expect(mockStore.removeSshPtyConsumerRecovery).not.toHaveBeenCalled()
session.dispose()
})
it('does not remember a consumer opened after establish was disposed', async () => {
const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps()
let signalOpenStarted!: () => void
+46 -28
View File
@@ -81,6 +81,7 @@ import type { Store } from '../persistence'
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract'
import { PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR } from '../../shared/pty-consumer-session'
import { retrySshOwnerRecoveryWhileBlocked } from './ssh-owner-recovery-retry'
import { runRemoteOrcaCli } from './ssh-remote-orca-cli'
import {
acknowledgeRemoteOrcaCliPostOutput,
@@ -1027,17 +1028,24 @@ export class SshRelaySession {
outputFlowControl: { requestedWindowSu: DEFAULT_PTY_SOURCE_WINDOW_SU }
}
try {
return await openSshPtyConsumerSession(mux, {
...options,
...(previousOwner
? {
resume: {
ownerGeneration: previousOwner.ownerGeneration,
ownerLease: previousOwner.ownerLease
}
}
: {})
})
return await retrySshOwnerRecoveryWhileBlocked(
() =>
openSshPtyConsumerSession(mux, {
...options,
...(previousOwner
? {
resume: {
ownerGeneration: previousOwner.ownerGeneration,
ownerLease: previousOwner.ownerLease
}
}
: {})
}),
{
isCurrent: () => ownsAttempt() && !mux.isDisposed(),
onClosed: (listener) => mux.onDispose(listener)
}
)
} catch (error) {
if (
!previousOwner ||
@@ -1046,26 +1054,36 @@ export class SshRelaySession {
throw error
}
const recovery = getSshPtyConsumerRecovery(this.targetId)
if (recovery) {
delete recovery.owner
recovery.checkpointsByAppPtyId.clear()
for (const [ptyId, migration] of recovery.modelMigrationsByAppPtyId) {
recovery.modelMigrationsByAppPtyId.set(
ptyId,
migration.then(() =>
Object.freeze({
status: 'checkpoint-unavailable' as const,
reason: 'completion-failed' as const
})
// Why identity-guarded: the record is target-scoped and its clientInstanceId is shared by every
// session for that target, so only a record still describing the owner this attempt tried to
// resume is ours to drop — otherwise a loser wipes the winner's checkpoints.
const ownsRecoveryRecord =
ownsAttempt() &&
(!recovery?.owner ||
(recovery.owner.ownerGeneration === previousOwner.ownerGeneration &&
recovery.owner.ownerLease === previousOwner.ownerLease))
if (ownsRecoveryRecord) {
if (recovery) {
delete recovery.owner
recovery.checkpointsByAppPtyId.clear()
for (const [ptyId, migration] of recovery.modelMigrationsByAppPtyId) {
recovery.modelMigrationsByAppPtyId.set(
ptyId,
migration.then(() =>
Object.freeze({
status: 'checkpoint-unavailable' as const,
reason: 'completion-failed' as const
})
)
)
)
}
}
await removeSshPtyConsumerOwnerRecovery(
this.targetId,
this.ptyConsumerClientInstanceId,
this.store
)
}
await removeSshPtyConsumerOwnerRecovery(
this.targetId,
this.ptyConsumerClientInstanceId,
this.store
)
if (!ownsAttempt()) {
throw new Error('Session disposed during establish')
}
@@ -0,0 +1,333 @@
import os from 'node:os'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type * as TracerModule from './observability/tracer'
import type * as UpdaterModule from './updater'
const {
appMock,
browserWindowMock,
nativeUpdaterMock,
autoUpdaterMock,
isMock,
killAllPtyMock,
recordUpdaterLifecycleMock
} = vi.hoisted(() => {
const appEventHandlers = new Map<string, ((...args: unknown[]) => void)[]>()
const eventHandlers = new Map<string, ((...args: unknown[]) => void)[]>()
const appOn = vi.fn((event: string, handler: (...args: unknown[]) => void) => {
const handlers = appEventHandlers.get(event) ?? []
handlers.push(handler)
appEventHandlers.set(event, handlers)
return appMock
})
const on = vi.fn((event: string, handler: (...args: unknown[]) => void) => {
const handlers = eventHandlers.get(event) ?? []
handlers.push(handler)
eventHandlers.set(event, handlers)
return autoUpdaterMock
})
const emit = (event: string, ...args: unknown[]) => {
for (const handler of eventHandlers.get(event) ?? []) {
handler(...args)
}
}
const reset = () => {
appEventHandlers.clear()
appOn.mockClear()
eventHandlers.clear()
on.mockClear()
autoUpdaterMock.checkForUpdates.mockReset()
autoUpdaterMock.downloadUpdate.mockReset()
autoUpdaterMock.quitAndInstall.mockReset()
autoUpdaterMock.setFeedURL.mockClear()
}
const autoUpdaterMock = {
autoDownload: false,
autoInstallOnAppQuit: false,
on,
checkForUpdates: vi.fn(),
downloadUpdate: vi.fn(),
quitAndInstall: vi.fn(),
setFeedURL: vi.fn(),
emit,
reset
}
return {
appMock: {
isPackaged: true,
getVersion: vi.fn(() => '1.4.162'),
on: appOn,
quit: vi.fn(),
exit: vi.fn()
},
browserWindowMock: { getAllWindows: vi.fn(() => []) },
nativeUpdaterMock: { on: vi.fn() },
autoUpdaterMock,
isMock: { dev: false },
killAllPtyMock: vi.fn(),
recordUpdaterLifecycleMock: vi.fn()
}
})
vi.mock('electron', () => ({
app: appMock,
BrowserWindow: browserWindowMock,
autoUpdater: nativeUpdaterMock,
powerMonitor: { on: vi.fn() },
shell: { openExternal: vi.fn() },
net: { fetch: vi.fn() }
}))
vi.mock('electron-updater', () => ({ autoUpdater: autoUpdaterMock }))
vi.mock('./electron-updater-loader', () => ({
loadElectronAutoUpdater: () => autoUpdaterMock
}))
vi.mock('@electron-toolkit/utils', () => ({ is: isMock }))
vi.mock('./ipc/pty', () => ({ killAllPty: killAllPtyMock }))
vi.mock('./updater-changelog', () => ({
fetchChangelog: vi.fn().mockResolvedValue(null)
}))
vi.mock('./updater-nudge', () => ({
fetchNudge: vi.fn().mockResolvedValue(null),
shouldApplyNudge: vi.fn().mockReturnValue(false)
}))
vi.mock('./updater-lifecycle-diagnostics', () => ({
recordUpdaterLifecycle: recordUpdaterLifecycleMock
}))
// The real electron-updater DebUpdater failure text when elevation is impossible.
const DEB_ELEVATION_ERROR =
'Error: Command failed: /usr/bin/pkexec --disable-internal-agent "/bin/bash" "-c" "dpkg -i \'/home/u/.cache/orca-updater/pending/orca-ide_1.4.163_amd64.deb\'"\npkexec must be setuid root'
// electron-updater's ERR_UPDATER_INVALID_SIGNATURE text, which drives its own card in UpdateCard.
const WINDOWS_SIGNATURE_MISMATCH_ERROR =
'New version 1.4.163 is not signed by the application owner: publisherNames: Orca, Inc.'
type CapturedSpan = {
readonly name: string
readonly exit: { readonly _tag: string; readonly cause?: string }
}
const originalPlatform = process.platform
let spans: CapturedSpan[]
let tracer: typeof TracerModule | null = null
function capturingSink(): TracerModule.TracerSink {
return {
push(record) {
spans.push(record as CapturedSpan)
},
flush() {
/* no-op */
},
close() {
/* no-op */
}
}
}
function installSpan(): CapturedSpan | undefined {
return spans.find((span) => span.name === 'updater.install')
}
/** Drives the updater to `downloaded`, the only state `quitAndInstall` acts on. */
async function reachDownloaded(): Promise<typeof UpdaterModule> {
const mainWindow = { webContents: { send: vi.fn() } }
autoUpdaterMock.checkForUpdates.mockResolvedValue(undefined)
// Why: resetModules gives each test a fresh module graph, so the sink must be installed on the
// same tracer instance updater.ts will import.
tracer = await import('./observability/tracer')
tracer.setActiveSink(capturingSink())
const updater = await import('./updater')
updater.setupAutoUpdater(mainWindow as never)
await vi.waitFor(() => {
expect(autoUpdaterMock.checkForUpdates).toHaveBeenCalledTimes(1)
})
autoUpdaterMock.emit('checking-for-update')
autoUpdaterMock.emit('update-available', { version: '1.4.163' })
await new Promise((resolve) => setTimeout(resolve, 0))
autoUpdaterMock.emit('update-downloaded', { version: '1.4.163' })
expect(updater.getUpdateStatus().state).toBe('downloaded')
return updater
}
/**
* On a `.deb` Linux host electron-updater's `install()` catches the failed elevation and
* re-dispatches it through the 'error' event *synchronously* inside `quitAndInstall()`. Orca
* recovers the app state, so the payload has to survive on the status and the span has to exit
* Failure — otherwise the only record of why the install never ran is destroyed (#11906).
*/
describe('quitAndInstall failure carries the updater cause', () => {
beforeEach(() => {
vi.resetModules()
autoUpdaterMock.reset()
nativeUpdaterMock.on.mockReset()
browserWindowMock.getAllWindows.mockReset()
browserWindowMock.getAllWindows.mockReturnValue([])
appMock.getVersion.mockReset()
appMock.getVersion.mockReturnValue('1.4.162')
appMock.quit.mockReset()
appMock.isPackaged = true
isMock.dev = false
killAllPtyMock.mockReset()
recordUpdaterLifecycleMock.mockReset()
Object.defineProperty(process, 'platform', {
value: 'linux',
configurable: true
})
vi.useRealTimers()
spans = []
})
afterEach(() => {
Object.defineProperty(process, 'platform', {
value: originalPlatform,
configurable: true
})
tracer?._resetTracerForTests()
tracer = null
})
it('reports the underlying elevation failure rather than a generic restart message', async () => {
const { quitAndInstall, getUpdateStatus } = await reachDownloaded()
autoUpdaterMock.quitAndInstall.mockImplementation(() => {
autoUpdaterMock.emit('error', new Error(DEB_ELEVATION_ERROR))
})
quitAndInstall()
await vi.waitFor(() => {
expect(autoUpdaterMock.quitAndInstall).toHaveBeenCalledTimes(1)
})
const status = getUpdateStatus()
expect(status.state).toBe('error')
// The only place the real cause exists is this event payload; dropping it leaves nothing
// in the logs and nothing actionable in the client's remote-server update dialog.
const message = status.state === 'error' ? status.message : ''
expect(message).toContain('pkexec')
expect(message).toContain('Could not start the update installer.')
})
it('keeps the durable breadcrumb to classification only', async () => {
const { quitAndInstall } = await reachDownloaded()
autoUpdaterMock.quitAndInstall.mockImplementation(() => {
autoUpdaterMock.emit('error', new Error(DEB_ELEVATION_ERROR))
})
quitAndInstall()
await vi.waitFor(() => {
expect(autoUpdaterMock.quitAndInstall).toHaveBeenCalledTimes(1)
})
expect(recordUpdaterLifecycleMock).toHaveBeenCalledWith(
'quit_and_install_failed_via_event',
{ errorType: 'Error' },
expect.anything()
)
})
it('exits the install span Failure with the cause', async () => {
const { quitAndInstall } = await reachDownloaded()
autoUpdaterMock.quitAndInstall.mockImplementation(() => {
autoUpdaterMock.emit('error', new Error(DEB_ELEVATION_ERROR))
})
quitAndInstall()
await vi.waitFor(() => {
expect(installSpan()).toBeDefined()
})
expect(installSpan()?.exit._tag).toBe('Failure')
expect(installSpan()?.exit.cause).toContain('pkexec')
})
it('exits the install span Success when the installer takes over', async () => {
const { quitAndInstall } = await reachDownloaded()
autoUpdaterMock.quitAndInstall.mockImplementation(() => {
// The installer runs and the old process is left to exit; no 'error' comes back.
})
quitAndInstall()
await vi.waitFor(() => {
expect(installSpan()).toBeDefined()
})
expect(installSpan()?.exit._tag).toBe('Success')
})
it('keeps a Windows signature verdict unprefixed so its own card still renders', async () => {
Object.defineProperty(process, 'platform', {
value: 'win32',
configurable: true
})
const { quitAndInstall, getUpdateStatus } = await reachDownloaded()
autoUpdaterMock.quitAndInstall.mockImplementation(() => {
autoUpdaterMock.emit('error', new Error(WINDOWS_SIGNATURE_MISMATCH_ERROR))
})
quitAndInstall()
await vi.waitFor(() => {
expect(autoUpdaterMock.quitAndInstall).toHaveBeenCalledTimes(1)
})
const status = getUpdateStatus()
// Prefixing this would put two contradictory instructions on one card.
expect(status.state === 'error' ? status.message : '').toBe(WINDOWS_SIGNATURE_MISMATCH_ERROR)
})
it('redacts the cause the same way the retained-package card does', async () => {
const { quitAndInstall, getUpdateStatus } = await reachDownloaded()
const home = os.homedir()
const escape = String.fromCharCode(27)
autoUpdaterMock.quitAndInstall.mockImplementation(() => {
autoUpdaterMock.emit(
'error',
new Error(`${escape}[31mdpkg -i '${home}/.cache/orca-updater/pending/orca.deb'${escape}[0m`)
)
})
quitAndInstall()
await vi.waitFor(() => {
expect(autoUpdaterMock.quitAndInstall).toHaveBeenCalledTimes(1)
})
const status = getUpdateStatus()
const message = status.state === 'error' ? status.message : ''
expect(message).not.toContain(home)
expect(message).not.toContain(escape)
expect(message).toContain("<home>/.cache/orca-updater/pending/orca.deb'")
})
it('carries the cause when quitAndInstall throws instead of dispatching an error event', async () => {
const { quitAndInstall, getUpdateStatus } = await reachDownloaded()
// Squirrel/NSIS can reject the request by throwing out of the native call; the event path never runs.
autoUpdaterMock.quitAndInstall.mockImplementation(() => {
throw new Error('Squirrel.framework is missing from the app bundle')
})
quitAndInstall()
await vi.waitFor(() => {
expect(getUpdateStatus().state).toBe('error')
})
const status = getUpdateStatus()
const message = status.state === 'error' ? status.message : ''
expect(message).toContain('Squirrel.framework is missing from the app bundle')
expect(message).toContain('Could not start the update installer.')
})
})
+10 -10
View File
@@ -159,6 +159,12 @@ const { getLinuxRootPackageTypeMock, recordUpdaterLifecycleMock } = vi.hoisted((
recordUpdaterLifecycleMock: vi.fn()
}))
// Why: macOS keeps the restart advice because quitting does re-stage a Squirrel update.
const PRE_COMMIT_INSTALL_FAILURE =
process.platform === 'darwin'
? 'Could not restart to install the update. Quit and reopen Orca, then try again.'
: 'Could not start the update installer. Orca remains open.'
// Why: only the marker resolver is faked so the real artifact capture/redaction path stays under test.
vi.mock('./linux-update-package-type', () => ({
getLinuxRootPackageType: getLinuxRootPackageTypeMock
@@ -1829,10 +1835,8 @@ describe('updater', () => {
state: 'error',
// Why: a pre-commit install failure is not fixed by restarting, so the copy must not
// suggest it — except on macOS, where quitting does re-stage a Squirrel update.
message:
process.platform === 'darwin'
? 'Could not restart to install the update. Quit and reopen Orca, then try again.'
: 'Could not start the update installer. Orca remains open.'
// The updater's own text is appended because it is the only record of why the install never ran.
message: `${PRE_COMMIT_INSTALL_FAILURE} (No update filepath provided, can't quit and install)`
})
)
})
@@ -3911,11 +3915,7 @@ describe('updater', () => {
const lastStatus = (send: ReturnType<typeof vi.fn>): UpdateStatus | undefined =>
send.mock.calls.findLast(([channel]) => channel === 'updater:status')?.[1]
// Why: macOS keeps the restart advice because quitting does re-stage a Squirrel update.
const PRE_COMMIT_FAILURE_MESSAGE =
process.platform === 'darwin'
? 'Could not restart to install the update. Quit and reopen Orca, then try again.'
: 'Could not start the update installer. Orca remains open.'
const PRE_COMMIT_FAILURE_MESSAGE = PRE_COMMIT_INSTALL_FAILURE
const AGENT_STDERR =
'pkexec: Error executing command as another user: No authentication agent found.'
@@ -4081,7 +4081,7 @@ describe('updater', () => {
expect(send).toHaveBeenCalledWith('updater:status', {
state: 'error',
message: PRE_COMMIT_FAILURE_MESSAGE
message: `${PRE_COMMIT_FAILURE_MESSAGE} (${EXIT_127})`
})
expect(recordUpdaterLifecycleMock).not.toHaveBeenCalledWith(
'linux_package_install_failed',
+51 -7
View File
@@ -13,7 +13,10 @@ import type {
RemoteServerUpdaterSnapshot,
RemoteServerUpdateSupport
} from '../shared/remote-server-update'
import { isWindowsSignatureCheckUnavailableFailure } from '../shared/updater-windows-signature-check'
import {
isWindowsSignatureCheckUnavailableFailure,
isWindowsSignatureMismatchFailure
} from '../shared/updater-windows-signature-check'
import { killAllPty } from './ipc/pty'
import { withUpdaterSpan } from './observability/instrumentation'
import { loadElectronAutoUpdater, type ElectronAutoUpdater } from './electron-updater-loader'
@@ -763,6 +766,8 @@ async function performQuitAndInstall(): Promise<void> {
true
)
resetQuitForUpdateState()
// Why: a bare return would exit this span Success and hide the aborted install from tracing.
span.fail('Could not persist the supervised serve update handoff')
return
}
@@ -790,6 +795,12 @@ async function performQuitAndInstall(): Promise<void> {
// Why: quitAndInstall can synchronously clear quitAndInstallInProgress via recovery (Win/Linux dispatchError); skip destructive prep if it already ran.
if (!quitAndInstallInProgress) {
// Why: recovery already wrote the reason to currentStatus; a bare return would exit this span Success.
span.fail(
currentStatus.state === 'error'
? currentStatus.message
: 'quitAndInstall returned without invoking the installer'
)
return
}
@@ -853,8 +864,9 @@ async function performQuitAndInstall(): Promise<void> {
recoveryStatus ?? {
state: 'error',
// Why: past the native invoke this is the same pre-commit failure the event path reports, so it gets the same copy; only a pre-native exception can be helped by a restart.
// A synchronous throw out of quitAndInstall carries the same installer text the 'error' event would have.
message: quitAndInstallNativeInvokedBeforeReset
? getPreCommitInstallFailureMessage()
? withInstallFailureCause(getPreCommitInstallFailureMessage(), error)
: 'Could not restart to install the update. Quit and reopen Orca, then try again.'
}
)
@@ -890,6 +902,33 @@ function sendInstallFailureStatus(status: UpdateStatus): void {
sendStatus(status, { force: true })
}
const INSTALL_FAILURE_CAUSE_MAX_LENGTH = 200
/**
* Appends the updater's own text to the generic install-failure copy. Without it the only record of
* why the install never started is destroyed — on Linux that text carries the exact `dpkg -i <path>`
* command the user has to run by hand, and remote clients get nothing but "it didn't come back".
*/
function withInstallFailureCause(baseMessage: string, error: unknown): string {
const raw = error instanceof Error ? error.message : typeof error === 'string' ? error : ''
// Why: the retained-package card runs its text through this same sanitizer, so a home directory,
// user name, or terminal escape must not reach the card merely because no artifact was tracked.
const redacted =
redactLinuxPackageInstallText(raw, getTrackedLinuxPackageArtifact()?.path ?? null) ?? ''
const cause = redacted.slice(0, INSTALL_FAILURE_CAUSE_MAX_LENGTH)
if (!cause || cause === 'Unknown error') {
return baseMessage
}
// Why: UpdateCard picks the whole card off this string, so a signature verdict must not be prefixed by contradictory restart advice.
if (
isWindowsSignatureCheckUnavailableFailure(cause) ||
isWindowsSignatureMismatchFailure(cause)
) {
return cause
}
return `${baseMessage} (${cause})`
}
/**
* The recovery status for a failed `.deb`/`.rpm` install, or null when no retained package can
* recover it. Must run before `resetQuitForUpdateState()` clears the attempt diagnostic.
@@ -945,14 +984,19 @@ function handleQuitAndInstallFailure(error?: unknown): boolean {
const recoveryStatus = buildLinuxPackageInstallFailureStatus(error)
failServeUpdateHandoff('The native updater rejected the install request.')
resetQuitForUpdateState()
recordUpdaterLifecycle('quit_and_install_failed_via_event', undefined, {
level: 'warn',
message: 'Update install could not start; recovered app state'
})
// Durable data carries classification only — the cause text stays on the status the user can read.
recordUpdaterLifecycle(
'quit_and_install_failed_via_event',
{ errorType: error instanceof Error ? error.name : typeof error },
{
level: 'warn',
message: 'Update install could not start; recovered app state'
}
)
sendInstallFailureStatus(
recoveryStatus ?? {
state: 'error',
message: getPreCommitInstallFailureMessage()
message: withInstallFailureCause(getPreCommitInstallFailureMessage(), error)
}
)
return true
+10
View File
@@ -161,6 +161,16 @@ export class RelayDispatcher {
this.closeClient(client, new Error('Relay client detached'), true)
}
// Why: a displaced owner must lose its transport whichever client holds it, and the launch channel is
// often the half-open one. The primary keeps its id for setWrite() revival, so invalidate it instead.
releaseDisplacedClient(clientId: number): void {
if (clientId === this.primaryClient.id) {
this.invalidateClient()
return
}
this.detachClient(clientId)
}
feedClient(clientId: number, data: Buffer): void {
const client = this.clients.get(clientId)
if (!client) {
@@ -0,0 +1,368 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import {
RelayDispatcher,
type RelayClientSessionIdentity,
type SinkWriteSettlement
} from './dispatcher'
import { encodeJsonRpcFrame, MessageType } from './protocol'
import { SshPtyConsumerSessionAdapter } from './ssh-pty-consumer-session-adapter'
import {
PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR,
PTY_CONSUMER_OWNER_RECOVERY_SUPERSEDED_ERROR
} from '../shared/pty-consumer-session'
const endpointIdentity: RelayClientSessionIdentity = {
principal: 'endpoint-principal',
authenticated: true,
allowSessionOwner: true,
authenticationKind: 'endpoint-credential'
}
function requestFrame(id: number, method: string, params: Record<string, unknown>): Buffer {
return encodeJsonRpcFrame({ jsonrpc: '2.0', id, method, params }, id, 0)
}
function message(buffer: Buffer): Record<string, unknown> | null {
if (buffer[0] !== MessageType.Regular) {
return null
}
const length = buffer.readUInt32BE(9)
return JSON.parse(buffer.subarray(13, 13 + length).toString('utf8'))
}
function response(writes: readonly Buffer[], id: number): Record<string, unknown> | undefined {
return writes.map(message).find((entry) => entry?.id === id) as
| Record<string, unknown>
| undefined
}
function ownerHelloParams(resume?: Record<string, unknown>): Record<string, unknown> {
return {
protocolVersion: 1,
clientInstanceId: 'client-1',
requestedRole: 'session-owner',
capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } },
...(resume ? { resume } : {})
}
}
async function flushRequests(): Promise<void> {
await new Promise((resolve) => setImmediate(resolve))
}
describe('relay PTY consumer owner displacement', () => {
let dispatcher: RelayDispatcher | null = null
afterEach(() => {
dispatcher?.dispose()
dispatcher = null
})
it('displaces a half-open owner and retains its deliveries for the reconnected client', async () => {
// Why no settle deferral: a half-open peer is indistinguishable from a live one at the sink — writes
// still "succeed" locally, so the relay never learns the owner is gone.
const staleWrites: Buffer[] = []
const closeStaleTransport = vi.fn()
dispatcher = new RelayDispatcher(
(data, settle) => {
staleWrites.push(Buffer.from(data))
settle({ ok: true })
return true
},
{ supportsWriteCallback: true, close: closeStaleTransport },
endpointIdentity
)
const detached: number[] = []
dispatcher.onClientDetached((clientId) => detached.push(clientId))
const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a')
dispatcher.feed(requestFrame(1, 'pty.openClient', ownerHelloParams()))
await flushRequests()
const staleGrant = response(staleWrites, 1)!.result as Record<string, unknown>
expect(adapter.deliveryMode(1)).toBe('source-owner')
const staleDelivery = adapter.openDelivery(1, 'pty-1', 'incarnation-1')!
adapter.appendSource(staleDelivery, {
spanId: 'span-1',
data: 'tail',
displayStart: 0,
displayEnd: 4,
splittable: true,
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true }
})
const reconnectWrites: Buffer[] = []
const reconnectClientId = dispatcher.attachClient(
(data, settle) => {
reconnectWrites.push(Buffer.from(data))
settle({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
dispatcher.feedClient(
reconnectClientId,
requestFrame(
2,
'pty.openClient',
ownerHelloParams({
ownerGeneration: staleGrant.ownerGeneration,
ownerLease: staleGrant.ownerLease
})
)
)
await flushRequests()
expect(response(reconnectWrites, 2)!.result).toMatchObject({
role: 'session-owner',
ownerGeneration: 2
})
expect(adapter.deliveryMode(reconnectClientId)).toBe('source-owner')
// Why: revocation is what bounds the takeover — the stale owner cannot open or drive deliveries again
// even though its socket never closed.
expect(adapter.deliveryMode(1)).toBe('subscriber')
expect(adapter.openDelivery(1, 'pty-2', 'incarnation-1')).toBeNull()
expect(detached).toContain(1)
expect(closeStaleTransport).toHaveBeenCalledOnce()
// Why: retained, not closed — the reconnected owner still has to rotate the stale delivery forward.
expect(adapter.getDebugSnapshot()).toMatchObject({ deliveryTokens: 1, graceTimers: 1 })
const rotation = adapter.rotateDelivery(staleDelivery, reconnectClientId, 0)
expect(rotation.identity).toMatchObject({ ownerGeneration: 2, clientGeneration: 2 })
expect(rotation.recovery.map((span) => span.data)).toEqual(['tail'])
})
it('retains the incumbent delivery once when its socket closes mid-publication', async () => {
const incumbentWrites: Buffer[] = []
dispatcher = new RelayDispatcher(
(data, settle) => {
incumbentWrites.push(Buffer.from(data))
settle({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a')
dispatcher.feed(requestFrame(1, 'pty.openClient', ownerHelloParams()))
await flushRequests()
const incumbentGrant = response(incumbentWrites, 1)!.result as Record<string, unknown>
const incumbentDelivery = adapter.openDelivery(1, 'pty-1', 'incarnation-1')!
adapter.appendSource(incumbentDelivery, {
spanId: 'span-1',
data: 'tail',
displayStart: 0,
displayEnd: 4,
splittable: true,
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true }
})
const reconnectWrites: Buffer[] = []
let grantSettlement: ((result: SinkWriteSettlement) => void) | undefined
const reconnectClientId = dispatcher.attachClient(
(data, settle) => {
reconnectWrites.push(Buffer.from(data))
grantSettlement = settle
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
dispatcher.feedClient(
reconnectClientId,
requestFrame(
2,
'pty.openClient',
ownerHelloParams({
ownerGeneration: incumbentGrant.ownerGeneration,
ownerLease: incumbentGrant.ownerLease
})
)
)
await flushRequests()
// Why this ordering: the incumbent's socket close and the replacement's grant write are independent
// events, so detach-then-commit has to leave exactly one retention, not two competing ones.
dispatcher.invalidateClient()
grantSettlement!({ ok: true })
expect(adapter.deliveryMode(reconnectClientId)).toBe('source-owner')
expect(adapter.getDebugSnapshot()).toMatchObject({ deliveryTokens: 1, graceTimers: 1 })
const rotation = adapter.rotateDelivery(incumbentDelivery, reconnectClientId, 0)
expect(rotation.identity).toMatchObject({ ownerGeneration: 2, clientGeneration: 2 })
expect(adapter.getDebugSnapshot()).toMatchObject({ graceTimers: 0 })
})
it('keeps the incumbent owner when the replacement grant fails to publish', async () => {
const incumbentWrites: Buffer[] = []
dispatcher = new RelayDispatcher(
(data, settle) => {
incumbentWrites.push(Buffer.from(data))
settle({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
const detached: number[] = []
dispatcher.onClientDetached((clientId) => detached.push(clientId))
const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a')
dispatcher.feed(requestFrame(1, 'pty.openClient', ownerHelloParams()))
await flushRequests()
const incumbentGrant = response(incumbentWrites, 1)!.result as Record<string, unknown>
const reconnectWrites: Buffer[] = []
let grantSettlement: ((result: SinkWriteSettlement) => void) | undefined
const reconnectClientId = dispatcher.attachClient(
(data, settle) => {
reconnectWrites.push(Buffer.from(data))
// Why: the takeover must not commit until the replacement grant is on the wire.
grantSettlement = settle
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
dispatcher.feedClient(
reconnectClientId,
requestFrame(
2,
'pty.openClient',
ownerHelloParams({
ownerGeneration: incumbentGrant.ownerGeneration,
ownerLease: incumbentGrant.ownerLease
})
)
)
await flushRequests()
expect(adapter.deliveryMode(1)).toBe('source-owner')
expect(detached).not.toContain(1)
grantSettlement!({ ok: false, error: new Error('reconnect socket closed mid-response') })
expect(adapter.deliveryMode(1)).toBe('source-owner')
expect(adapter.deliveryMode(reconnectClientId)).toBe('subscriber')
// Why: the rolled-back attempt must leave the incumbent's lease reclaimable by the next reconnect.
const retryWrites: Buffer[] = []
const retryClientId = dispatcher.attachClient(
(data, settle) => {
retryWrites.push(Buffer.from(data))
settle({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
dispatcher.feedClient(
retryClientId,
requestFrame(
3,
'pty.openClient',
ownerHelloParams({
ownerGeneration: incumbentGrant.ownerGeneration,
ownerLease: incumbentGrant.ownerLease
})
)
)
await flushRequests()
// Why generation 3: the rolled-back attempt consumed generation 2, but not the incumbent's lease.
expect(response(retryWrites, 3)!.result).toMatchObject({
role: 'session-owner',
ownerGeneration: 3
})
expect(adapter.deliveryMode(1)).toBe('subscriber')
expect(detached).toContain(1)
})
it('fences an old proof while its replacement is live', async () => {
const incumbentWrites: Buffer[] = []
dispatcher = new RelayDispatcher(
(data, settle) => {
incumbentWrites.push(Buffer.from(data))
settle({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a')
dispatcher.feed(requestFrame(1, 'pty.openClient', ownerHelloParams()))
await flushRequests()
const incumbentGrant = response(incumbentWrites, 1)!.result as Record<string, unknown>
const resume = {
ownerGeneration: incumbentGrant.ownerGeneration,
ownerLease: incumbentGrant.ownerLease
}
let firstReconnectSettlement: ((result: SinkWriteSettlement) => void) | undefined
const firstReconnectWrites: Buffer[] = []
const firstReconnectClientId = dispatcher.attachClient(
(data, settle) => {
firstReconnectWrites.push(Buffer.from(data))
// Why ??=: the test settles the grant response, not whatever frame the dispatcher wrote last.
firstReconnectSettlement ??= settle
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
dispatcher.feedClient(
firstReconnectClientId,
requestFrame(2, 'pty.openClient', ownerHelloParams(resume))
)
await flushRequests()
const retryWrites: Buffer[] = []
const retryClientId = dispatcher.attachClient(
(data, settle) => {
retryWrites.push(Buffer.from(data))
settle({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
dispatcher.feedClient(
retryClientId,
requestFrame(3, 'pty.openClient', ownerHelloParams(resume))
)
await flushRequests()
expect(response(retryWrites, 3)!.error).toMatchObject({
code: PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR
})
firstReconnectSettlement!({ ok: true })
dispatcher.feedClient(
retryClientId,
requestFrame(4, 'pty.openClient', ownerHelloParams(resume))
)
await flushRequests()
expect(response(retryWrites, 4)!.error).toMatchObject({
code: PTY_CONSUMER_OWNER_RECOVERY_SUPERSEDED_ERROR
})
expect(adapter.deliveryMode(firstReconnectClientId)).toBe('source-owner')
expect(adapter.deliveryMode(retryClientId)).toBe('subscriber')
dispatcher.detachClient(firstReconnectClientId)
dispatcher.feedClient(
retryClientId,
requestFrame(5, 'pty.openClient', ownerHelloParams(resume))
)
await flushRequests()
expect(response(retryWrites, 5)!.result).toMatchObject({
role: 'session-owner',
ownerGeneration: 3,
ownerLease: incumbentGrant.ownerLease
})
expect(adapter.deliveryMode(retryClientId)).toBe('source-owner')
})
})
+34 -1
View File
@@ -10,7 +10,15 @@
import { createServer, createConnection, type Socket, type Server } from 'node:net'
import { join } from 'node:path'
import { unlinkSync, existsSync, statSync, readFileSync, chmodSync } from 'node:fs'
import {
unlinkSync,
existsSync,
statSync,
readFileSync,
chmodSync,
closeSync,
openSync
} from 'node:fs'
import {
RELAY_SENTINEL,
FrameDecoder,
@@ -597,6 +605,31 @@ async function main(): Promise<void> {
}
stdoutDrainWaiters.add(cb)
return () => stdoutDrainWaiters.delete(cb)
},
close: () => {
stdoutAlive = false
flushStdoutDrainWaiters()
// Why close then re-pin: the SSH peer must see EOF, but a long-lived daemon that
// frees fds 0/1 lets accept()/open() recycle them while Node still treats
// process.stdin/stdout as those numbers — corrupting socket clients and shutdown.
for (const fd of [process.stdin.fd, process.stdout.fd]) {
try {
closeSync(fd)
} catch {
// Already closed by the peer.
}
}
const devNull = process.platform === 'win32' ? 'NUL' : '/dev/null'
try {
openSync(devNull, 'r')
} catch {
/* best-effort pin of the lowest free fd (normally 0) */
}
try {
openSync(devNull, 'w')
} catch {
/* best-effort pin of the next free fd (normally 1) */
}
}
},
undefined,
+20 -4
View File
@@ -1,6 +1,7 @@
import {
PTY_CONSUMER_SESSION_PROTOCOL_VERSION,
PtyConsumerSession,
type PtyConsumerSessionAdmission,
type PtyConsumerSessionGrant,
type PtyConsumerSessionHello
} from '../shared/pty-consumer-session'
@@ -74,7 +75,7 @@ export class SshPtyConsumerSessionAdapter {
private readonly pausedDeliveryByPty = new Map<string, PtySourceDeliveryIdentity>()
constructor(
dispatcher: RelayDispatcher,
private readonly dispatcher: RelayDispatcher,
serverBuildId: string,
private readonly setDeliveryPaused?: (id: string, paused: boolean) => void,
onSourceCreditAvailable?: (id: string) => void
@@ -268,15 +269,30 @@ export class SshPtyConsumerSessionAdapter {
throw new Error('SSH PTY consumer response publication fence is unavailable')
}
context.onResponseSettled((result) => {
if (result.ok) {
admission.commitPublication()
} else {
if (!result.ok) {
admission.rollbackPublication()
return
}
admission.commitPublication()
this.closeDisplacedOwner(admission.displacedOwner)
})
return admission.grant
}
// Why: only after the replacement grant is published — until then the admission can still roll back
// onto the incumbent. Its deliveries are retained for the new owner to rotate, exactly as on detach.
private closeDisplacedOwner(displaced: PtyConsumerSessionAdmission['displacedOwner']): void {
if (!displaced) {
return
}
this.clearPausedForGrant(displaced.grant)
this.sourceCredit.retainOrCloseOnDetach(displaced.grant)
const clientId = Number(displaced.connectionId)
if (Number.isSafeInteger(clientId)) {
this.dispatcher.releaseDisplacedClient(clientId)
}
}
private clearPausedIdentity(identity: PtySourceDeliveryIdentity): void {
if (this.pausedDeliveryByPty.get(identity.id) === identity) {
this.pausedDeliveryByPty.delete(identity.id)
@@ -127,6 +127,19 @@ describe('UpdateCard Windows signature failures', () => {
'true'
)
})
// An install failure now carries the updater's own text, so it can reach these branches too.
it('routes a signature verdict raised during install to the security-stop card', () => {
const message =
'New version 1.4.200 is not signed by the application owner: publisherNames: Orca'
renderAfterAvailableStatus()
act(() => useAppStore.getState().setUpdateStatus({ state: 'error', message }))
expect(screen.getByText("Update Wasn't Installed")).toBeTruthy()
// The generic restart advice must not be prefixed onto a security stop.
expect(screen.queryByText(/Quit and reopen Orca/)).toBeNull()
})
})
describe('UpdateCard hourly builds', () => {
@@ -260,6 +273,18 @@ describe('UpdateCard Linux package-install recovery', () => {
expect(download).toHaveBeenCalledTimes(1)
})
it('shows the appended install cause behind the generic card details', () => {
const message =
'Could not start the update installer. Orca remains open. (Command failed: pkexec must be setuid root)'
renderAfterAvailableStatus()
act(() => useAppStore.getState().setUpdateStatus({ state: 'error', message }))
expect(screen.getByText('Update Error')).toBeTruthy()
fireEvent.click(screen.getByRole('button', { name: 'Show details' }))
expect(screen.getByText(message)).toBeTruthy()
})
it('leaves the HTTP/1.1 compatibility branch untouched', () => {
renderAfterAvailableStatus()
@@ -93,7 +93,9 @@ function ServerUpdateRow({
{help ? (
<p
className={
entry.phase === 'failed' ? 'text-xs text-destructive' : 'text-xs text-muted-foreground'
entry.phase === 'failed'
? 'text-xs break-words text-destructive'
: 'text-xs break-words text-muted-foreground'
}
>
{help}
@@ -0,0 +1,104 @@
import { describe, expect, it, vi } from 'vitest'
import type {
RemoteServerUpdateInstallResult,
RemoteServerUpdaterSnapshot
} from '../../../shared/remote-server-update'
import type { RuntimeStatus } from '../../../shared/runtime-types'
import { readRemoteServerInstallFailure } from './remote-server-install-failure-probe'
const support = {
installMode: 'supervised-headless-serve',
automatic: true,
reason: 'available'
} as const
const install: RemoteServerUpdateInstallResult = {
accepted: true,
fromVersion: '1.4.0',
targetVersion: '1.5.0',
runtimeId: 'runtime-old'
}
function runtime(runtimeId = 'runtime-old'): RuntimeStatus {
return {
runtimeId,
rendererGraphEpoch: 0,
graphStatus: 'ready',
authoritativeWindowId: null,
liveTabCount: 2,
liveLeafCount: 1,
capabilities: ['updater.remote-control.v1'],
appVersion: '1.4.0',
remoteUpdateSupport: support
}
}
function snapshot(
status: RemoteServerUpdaterSnapshot['status'],
runtimeId = 'runtime-old'
): RemoteServerUpdaterSnapshot {
return { appVersion: '1.4.0', runtimeId, support, status }
}
describe('readRemoteServerInstallFailure', () => {
it('returns the message the original process is publishing', async () => {
const getUpdaterStatus = vi.fn(async () =>
snapshot({ state: 'error', message: 'pkexec must be setuid root' })
)
await expect(
readRemoteServerInstallFailure('server-1', { getUpdaterStatus }, install, runtime())
).resolves.toBe('pkexec must be setuid root')
})
it('bounds the status call by the deadline the caller passes', async () => {
const getUpdaterStatus = vi.fn(async () =>
snapshot({ state: 'error', message: 'pkexec must be setuid root' })
)
await readRemoteServerInstallFailure(
'server-1',
{ getUpdaterStatus },
install,
runtime(),
4_000
)
// Without this the call inherits the RPC client's 15s default and can outlast the reconnect budget.
expect(getUpdaterStatus).toHaveBeenCalledWith('server-1', 4_000)
})
it('ignores an error published by a different process', async () => {
const getUpdaterStatus = vi.fn(async () =>
snapshot({ state: 'error', message: 'pkexec must be setuid root' }, 'runtime-new')
)
await expect(
readRemoteServerInstallFailure('server-1', { getUpdaterStatus }, install, runtime())
).resolves.toBeNull()
})
it('ignores a snapshot that is not an error', async () => {
const getUpdaterStatus = vi.fn(async () => snapshot({ state: 'downloaded', version: '1.5.0' }))
await expect(
readRemoteServerInstallFailure('server-1', { getUpdaterStatus }, install, runtime())
).resolves.toBeNull()
})
it('costs no round trip once the replacement runtime answers', async () => {
const getUpdaterStatus = vi.fn(async () =>
snapshot({ state: 'error', message: 'pkexec must be setuid root' })
)
await expect(
readRemoteServerInstallFailure(
'server-1',
{ getUpdaterStatus },
install,
runtime('runtime-new')
)
).resolves.toBeNull()
expect(getUpdaterStatus).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,36 @@
import type {
RemoteServerUpdateInstallResult,
RemoteServerUpdaterSnapshot
} from '../../../shared/remote-server-update'
import type { RuntimeStatus } from '../../../shared/runtime-types'
type InstallFailureProbeTransport = {
getUpdaterStatus: (
environmentId: string,
timeoutMs?: number
) => Promise<RemoteServerUpdaterSnapshot>
}
/**
* The install RPC is accepted before the installer runs, so a server that failed to restart keeps
* publishing the reason through `updater.getStatus`. Reading it turns a three-minute reconnect
* timeout into the actual cause. Both runtime-id guards matter: a snapshot from a process that is
* not the one we asked to install describes some other attempt.
*/
export async function readRemoteServerInstallFailure(
environmentId: string,
transport: InstallFailureProbeTransport,
install: RemoteServerUpdateInstallResult,
runtime: RuntimeStatus,
timeoutMs?: number
): Promise<string | null> {
if (runtime.runtimeId !== install.runtimeId) {
return null
}
// Why: without an explicit deadline this inherits the RPC client's 15s default and can outlast the caller's own wait.
const snapshot = await transport.getUpdaterStatus(environmentId, timeoutMs)
if (snapshot.runtimeId !== install.runtimeId || snapshot.status.state !== 'error') {
return null
}
return snapshot.status.message
}
@@ -0,0 +1,178 @@
import { describe, expect, it, vi } from 'vitest'
import type {
RemoteServerUpdateInstallResult,
RemoteServerUpdaterSnapshot
} from '../../../shared/remote-server-update'
import type { RuntimeStatus } from '../../../shared/runtime-types'
import { waitForReplacementRuntime } from './remote-server-restart-wait'
const support = {
installMode: 'supervised-headless-serve',
automatic: true,
reason: 'available'
} as const
const install: RemoteServerUpdateInstallResult = {
accepted: true,
fromVersion: '1.4.0',
targetVersion: '1.5.0',
runtimeId: 'runtime-old'
}
const idleSnapshot: RemoteServerUpdaterSnapshot = {
appVersion: '1.4.0',
runtimeId: 'runtime-old',
support,
status: { state: 'downloaded', version: '1.5.0' }
}
function runtime(version: string, runtimeId: string): RuntimeStatus {
return {
runtimeId,
rendererGraphEpoch: 0,
graphStatus: 'ready',
authoritativeWindowId: null,
liveTabCount: 2,
liveLeafCount: 1,
capabilities: ['updater.remote-control.v1'],
appVersion: version,
remoteUpdateSupport: support
}
}
describe('waitForReplacementRuntime', () => {
it('returns the replacement process once it answers on the target version', async () => {
let clock = 0
let ticks = 0
await expect(
waitForReplacementRuntime(
'server-1',
{
now: () => clock,
wait: async (milliseconds) => {
clock += milliseconds
},
getUpdaterStatus: async () => idleSnapshot,
getRuntimeStatus: async () => {
ticks += 1
return ticks > 1 ? runtime('1.5.0', 'runtime-new') : runtime('1.4.0', 'runtime-old')
}
},
install,
{ reconnectTimeoutMs: 1_000, pollIntervalMs: 10 }
)
).resolves.toMatchObject({ runtimeId: 'runtime-new', appVersion: '1.5.0' })
})
it('never lets a poll RPC outlive the reconnect deadline', async () => {
let clock = 0
const reconnectTimeoutMs = 25_000
const pollIntervalMs = 500
await expect(
waitForReplacementRuntime(
'server-1',
{
now: () => clock,
wait: async (milliseconds) => {
clock += milliseconds
},
getUpdaterStatus: async () => idleSnapshot,
// A wedged SSH link burns the whole allowance it is given before rejecting.
getRuntimeStatus: async (_environmentId, timeoutMs) => {
clock += timeoutMs ?? 0
throw new Error('runtime status timed out')
}
},
install,
{ reconnectTimeoutMs, pollIntervalMs }
)
).rejects.toThrow('remote_update_reconnect_timeout')
// A tick allowed its full 10s past the deadline would push this well beyond the budget.
expect(clock).toBeLessThanOrEqual(reconnectTimeoutMs)
})
it('keeps the total wait inside the budget when the status RPC eats it before the probe', async () => {
let clock = 0
const reconnectTimeoutMs = 20_000
await expect(
waitForReplacementRuntime(
'server-1',
{
now: () => clock,
wait: async (milliseconds) => {
clock += milliseconds
},
// Both RPCs burn everything they are handed, so a stale budget doubles the tick.
getUpdaterStatus: async (_environmentId, timeoutMs) => {
clock += timeoutMs ?? 0
return idleSnapshot
},
getRuntimeStatus: async (_environmentId, timeoutMs) => {
clock += timeoutMs ?? 0
return runtime('1.4.0', 'runtime-old')
}
},
install,
{ reconnectTimeoutMs, pollIntervalMs: 500 }
)
).rejects.toThrow('remote_update_reconnect_timeout')
expect(clock).toBeLessThanOrEqual(reconnectTimeoutMs)
})
it('charges the failure probe only the budget left after the status RPC', async () => {
let clock = 0
const getUpdaterStatus = vi.fn(async () => idleSnapshot)
await expect(
waitForReplacementRuntime(
'server-1',
{
now: () => clock,
wait: async (milliseconds) => {
clock += milliseconds
},
getUpdaterStatus,
getRuntimeStatus: async () => {
clock += 2_000
return runtime('1.4.0', 'runtime-old')
}
},
install,
{ reconnectTimeoutMs: 9_000, pollIntervalMs: 1_000 }
)
).rejects.toThrow('remote_update_reconnect_timeout')
// First probe runs on the second tick: it starts at 3s, the status RPC spends 2s more,
// so 4s of the 9s budget is left — not the 6s the tick began with.
expect(getUpdaterStatus).toHaveBeenNthCalledWith(1, 'server-1', 4_000)
})
it('bounds the failure probe by the same remaining budget', async () => {
let clock = 0
const getUpdaterStatus = vi.fn(async () => idleSnapshot)
await expect(
waitForReplacementRuntime(
'server-1',
{
now: () => clock,
wait: async (milliseconds) => {
clock += milliseconds
},
getUpdaterStatus,
getRuntimeStatus: async () => runtime('1.4.0', 'runtime-old')
},
install,
{ reconnectTimeoutMs: 4_000, pollIntervalMs: 1_000 }
)
).rejects.toThrow('remote_update_reconnect_timeout')
// First tick is the probe's warm-up; the second runs with 3s of the 4s budget left.
expect(getUpdaterStatus).toHaveBeenCalledWith('server-1', 3_000)
})
})
@@ -0,0 +1,81 @@
import { hasReachedAppVersion } from '../../../shared/app-version'
import type {
RemoteServerUpdateInstallResult,
RemoteServerUpdaterSnapshot
} from '../../../shared/remote-server-update'
import type { RuntimeStatus } from '../../../shared/runtime-types'
import { readRemoteServerInstallFailure } from './remote-server-install-failure-probe'
type RestartWaitTransport = {
getRuntimeStatus: (environmentId: string, timeoutMs?: number) => Promise<RuntimeStatus>
getUpdaterStatus: (
environmentId: string,
timeoutMs?: number
) => Promise<RemoteServerUpdaterSnapshot>
wait: (milliseconds: number) => Promise<void>
now?: () => number
}
type RestartWaitTiming = {
reconnectTimeoutMs: number
pollIntervalMs: number
}
/** Per-call ceiling for the two poll RPCs; each tick also clamps to the time left. */
const RESTART_WAIT_RPC_TIMEOUT_MS = 10_000
/**
* Waits for the replacement process to answer on the target version and returns its runtime status.
* Throws the server's own install-failure text as soon as the *original* process publishes one —
* without that, a server that could not restart is indistinguishable from a slow one until the
* reconnect budget runs out, and the user is told the wrong thing.
*/
export async function waitForReplacementRuntime(
environmentId: string,
transport: RestartWaitTransport,
install: RemoteServerUpdateInstallResult,
timing: RestartWaitTiming
): Promise<RuntimeStatus> {
const now = transport.now ?? Date.now
const deadline = now() + timing.reconnectTimeoutMs
// Why: the install RPC returns before the installer fires, so an error on the first tick belongs to an earlier attempt.
let probed = false
while (now() < deadline) {
// Why: an RPC allowed to outlive the deadline drags the whole wait past the budget the caller set.
const rpcTimeoutMs = Math.min(RESTART_WAIT_RPC_TIMEOUT_MS, deadline - now())
let installFailure: string | null = null
try {
const status = await transport.getRuntimeStatus(environmentId, rpcTimeoutMs)
const version = status.appVersion?.trim() ?? ''
const reachedTarget = hasReachedAppVersion(version, install.targetVersion)
if (status.runtimeId !== install.runtimeId && reachedTarget) {
return status
}
// Why: the status RPC already spent part of rpcTimeoutMs, so the probe re-reads what is left
// rather than starting a second full budget of its own.
const probeTimeoutMs = Math.min(RESTART_WAIT_RPC_TIMEOUT_MS, deadline - now())
installFailure =
probed && probeTimeoutMs > 0
? await readRemoteServerInstallFailure(
environmentId,
transport,
install,
status,
probeTimeoutMs
)
: null
probed = true
} catch {
// A refused connection is expected while the server process is being replaced.
}
// Why: thrown outside the try so the loop's own catch cannot swallow the reason we came for.
if (installFailure !== null) {
throw new Error(installFailure)
}
const waitMs = Math.min(timing.pollIntervalMs, deadline - now())
if (waitMs > 0) {
await transport.wait(waitMs)
}
}
throw new Error('remote_update_reconnect_timeout')
}
@@ -44,6 +44,10 @@ function status(version: string, runtimeId = 'runtime-old', automatic = true): R
}
}
// The main process's own copy once quitAndInstall fails; it must survive to the client verbatim.
const INSTALL_FAILURE =
"Could not start the update installer. Orca remains open. (Command failed: pkexec dpkg -i '/home/u/.cache/orca-updater/pending/orca-ide_1.5.0_amd64.deb' pkexec must be setuid root)"
const availableSnapshot: RemoteServerUpdaterSnapshot = {
appVersion: '1.4.0',
runtimeId: 'runtime-old',
@@ -275,6 +279,91 @@ describe('remote server update execution', () => {
})
})
it('repeats the install failure a still-running server is publishing', async () => {
const snapshots = [
availableSnapshot,
{ ...availableSnapshot, status: { state: 'downloaded', version: '1.5.0' } }
] satisfies RemoteServerUpdaterSnapshot[]
let installed = false
const result = await runRemoteServerUpdate(
availableEntry(),
transport({
getUpdaterStatus: async () =>
installed
? { ...availableSnapshot, status: { state: 'error', message: INSTALL_FAILURE } }
: (snapshots.shift() ?? availableSnapshot),
install: async (): Promise<RemoteServerUpdateInstallResult> => {
installed = true
return {
accepted: true,
fromVersion: '1.4.0',
targetVersion: '1.5.0',
runtimeId: 'runtime-old'
}
},
// The server never restarts: same runtimeId, same version, still perfectly reachable.
getRuntimeStatus: async () => status('1.4.0', 'runtime-old')
}),
() => undefined,
{ timing: { operationTimeoutMs: 10, reconnectTimeoutMs: 60, pollIntervalMs: 1 } }
)
expect(result).toMatchObject({ phase: 'failed', error: INSTALL_FAILURE })
})
it('ignores an updater error that predates the install request', async () => {
const snapshots = [
availableSnapshot,
{ ...availableSnapshot, status: { state: 'downloaded', version: '1.5.0' } }
] satisfies RemoteServerUpdaterSnapshot[]
let reconnectTicks = 0
const result = await runRemoteServerUpdate(
availableEntry(),
transport({
// A background check that errored inside the 100ms before the installer fires.
getUpdaterStatus: async () =>
snapshots.shift() ?? {
...availableSnapshot,
status: { state: 'error', message: 'a background check failed' }
},
getRuntimeStatus: async () => {
reconnectTicks += 1
return reconnectTicks > 1
? status('1.5.0', 'runtime-new')
: status('1.4.0', 'runtime-old')
}
}),
() => undefined,
{ timing: { operationTimeoutMs: 10, reconnectTimeoutMs: 60, pollIntervalMs: 1 } }
)
expect(result).toMatchObject({ phase: 'updated', currentVersion: '1.5.0' })
})
it('keeps waiting when the failure probe itself cannot be reached', async () => {
const snapshots = [
availableSnapshot,
{ ...availableSnapshot, status: { state: 'downloaded', version: '1.5.0' } }
] satisfies RemoteServerUpdaterSnapshot[]
const result = await runRemoteServerUpdate(
availableEntry(),
transport({
getUpdaterStatus: async () => {
const next = snapshots.shift()
if (!next) {
throw new Error('connection refused')
}
return next
},
getRuntimeStatus: async () => status('1.4.0', 'runtime-old')
}),
() => undefined,
{ timing: { operationTimeoutMs: 10, reconnectTimeoutMs: 6, pollIntervalMs: 1 } }
)
expect(result).toMatchObject({
phase: 'failed',
error: 'The server did not reconnect on the updated version.'
})
})
it('turns a capability race into manual update guidance', async () => {
const result = await runRemoteServerUpdate(
availableEntry(),
@@ -1,5 +1,6 @@
import {
compareAppVersions,
hasReachedAppVersion,
isPerfPrereleaseAppVersion,
isPrereleaseAppVersion,
isValidAppVersion
@@ -13,6 +14,7 @@ import type {
import type { PublicKnownRuntimeEnvironment } from '../../../shared/runtime-environments'
import type { RuntimeStatus } from '../../../shared/runtime-types'
import type { UpdateCheckOptions } from '../../../shared/types'
import { waitForReplacementRuntime } from './remote-server-restart-wait'
import { remoteServerUpdateErrorMessage } from './remote-server-update-errors'
import { pollRemoteServerUpdater } from './remote-server-updater-polling'
@@ -45,7 +47,10 @@ export type RemoteServerUpdateEntry = {
export type RemoteServerUpdateTransport = {
getRuntimeStatus: (environmentId: string, timeoutMs?: number) => Promise<RuntimeStatus>
getUpdaterStatus: (environmentId: string) => Promise<RemoteServerUpdaterSnapshot>
getUpdaterStatus: (
environmentId: string,
timeoutMs?: number
) => Promise<RemoteServerUpdaterSnapshot>
check: (
environmentId: string,
options: UpdateCheckOptions
@@ -213,12 +218,7 @@ export async function runRemoteServerUpdate(
if (available.status.state === 'not-available') {
const status = await transport.getRuntimeStatus(entry.environmentId, 10_000)
const currentVersion = status.appVersion?.trim() ?? ''
const reachedTarget =
entry.targetVersion !== null &&
isValidAppVersion(currentVersion) &&
isValidAppVersion(entry.targetVersion) &&
compareAppVersions(currentVersion, entry.targetVersion) >= 0
if (!reachedTarget) {
if (!hasReachedAppVersion(currentVersion, entry.targetVersion)) {
throw new Error('remote_update_requested_version_unavailable')
}
next = {
@@ -267,34 +267,22 @@ export async function runRemoteServerUpdate(
}
onProgress(next)
const now = transport.now ?? Date.now
const reconnectDeadline = now() + timing.reconnectTimeoutMs
while (now() < reconnectDeadline) {
try {
const status = await transport.getRuntimeStatus(entry.environmentId, 10_000)
const version = status.appVersion?.trim() ?? ''
const reachedTarget =
isValidAppVersion(version) &&
isValidAppVersion(install.targetVersion) &&
compareAppVersions(version, install.targetVersion) >= 0
if (status.runtimeId !== install.runtimeId && reachedTarget) {
next = {
...next,
phase: 'updated',
currentVersion: version,
runtimeId: status.runtimeId,
liveTabCount: status.liveTabCount,
liveLeafCount: status.liveLeafCount
}
onProgress(next)
return next
}
} catch {
// A refused connection is expected while the server process is being replaced.
}
await transport.wait(timing.pollIntervalMs)
const replacement = await waitForReplacementRuntime(
entry.environmentId,
transport,
install,
timing
)
next = {
...next,
phase: 'updated',
currentVersion: replacement.appVersion?.trim() ?? '',
runtimeId: replacement.runtimeId,
liveTabCount: replacement.liveTabCount,
liveLeafCount: replacement.liveLeafCount
}
throw new Error('remote_update_reconnect_timeout')
onProgress(next)
return next
} catch (error) {
next = {
...next,
@@ -33,8 +33,13 @@ const transport: RemoteServerUpdateTransport = {
window.api.runtimeEnvironments
.getStatus({ selector: environmentId, timeoutMs })
.then((response) => unwrapRuntimeRpcResult<RuntimeStatus>(response)),
getUpdaterStatus: (environmentId) =>
callRemoteUpdater<RemoteServerUpdaterSnapshot>(environmentId, 'updater.getStatus'),
getUpdaterStatus: (environmentId, timeoutMs) =>
callRemoteUpdater<RemoteServerUpdaterSnapshot>(
environmentId,
'updater.getStatus',
undefined,
timeoutMs
),
check: (environmentId, options) =>
callRemoteUpdater<RemoteServerUpdaterSnapshot>(environmentId, 'updater.check', options),
download: (environmentId) =>
+10
View File
@@ -94,3 +94,13 @@ export function compareAppVersions(left: string, right: string): number {
return 0
}
/** True when `current` is at or past `target`. Unparseable or absent versions never qualify. */
export function hasReachedAppVersion(current: string, target: string | null): boolean {
return (
target !== null &&
isValidAppVersion(current) &&
isValidAppVersion(target) &&
compareAppVersions(current, target) >= 0
)
}
+63
View File
@@ -0,0 +1,63 @@
import {
PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR,
PTY_CONSUMER_OWNER_RECOVERY_SUPERSEDED_ERROR,
PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR,
type PtyConsumerAuthentication,
type PtyConsumerSessionHello
} from './pty-consumer-session-contract'
type IncumbentOwner = {
principal: string
clientInstanceId: string
generation: number
lease: string
state: 'pending' | 'active' | 'disconnected'
replaces?: { generation: number }
}
function throwRecoveryError(message: string, code: number): never {
throw Object.assign(new Error(message), { code })
}
export function assertPtyConsumerOwnerRecovery(
hello: PtyConsumerSessionHello,
authentication: PtyConsumerAuthentication,
current: IncumbentOwner
): void {
const resume = hello.resume
if (!resume) {
throw new Error('Owner recovery proof is required')
}
if (
resume.ownerLease !== current.lease ||
hello.clientInstanceId !== current.clientInstanceId ||
authentication.principal !== current.principal
) {
throwRecoveryError(
'Owner recovery lease is stale or belongs to another principal',
PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR
)
}
if (current.state === 'active' && resume.ownerGeneration < current.generation) {
throwRecoveryError(
'Owner recovery generation was superseded',
PTY_CONSUMER_OWNER_RECOVERY_SUPERSEDED_ERROR
)
}
const generationMatches =
resume.ownerGeneration === current.generation ||
(current.state === 'pending' && resume.ownerGeneration === current.replaces?.generation) ||
(current.state === 'disconnected' && resume.ownerGeneration < current.generation)
if (!generationMatches) {
throwRecoveryError(
'Owner recovery generation is stale',
PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR
)
}
if (current.state === 'pending') {
throwRecoveryError(
'Owner grant publication is still pending',
PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR
)
}
}
@@ -1,6 +1,10 @@
export const PTY_CONSUMER_SESSION_PROTOCOL_VERSION = 1
export const PTY_CONSUMER_OWNER_GRACE_MS = 30_000
export const PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR = -32041
// Why: recovery is blocked only while the incumbent owner's grant publication is still settling — a
// window bounded by one response write, so the client may retry within a short budget.
export const PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR = -32042
export const PTY_CONSUMER_OWNER_RECOVERY_SUPERSEDED_ERROR = -32043
export type PtyConsumerRole = 'session-owner' | 'subscriber'
@@ -41,8 +45,16 @@ export type PtyConsumerAuthentication = {
allowSessionOwner: boolean
}
export type PtyConsumerDisplacedOwner = {
connectionId: string
grant: Readonly<PtyConsumerSessionGrant>
}
export type PtyConsumerSessionAdmission = {
grant: Readonly<PtyConsumerSessionGrant>
// Why: set when this admission takes over a still-attached owner. The transport layer owns closing
// that connection and releasing its deliveries — do it only once the new grant has been published.
displacedOwner?: Readonly<PtyConsumerDisplacedOwner>
commitPublication: () => void
rollbackPublication: () => void
}
+52
View File
@@ -0,0 +1,52 @@
import type { PtyConsumerSessionHello } from './pty-consumer-session-contract'
export const MAX_CAPABILITY_VERSIONS = 8
export function assertNonEmptyString(value: unknown, name: string): asserts value is string {
if (typeof value !== 'string' || value.length === 0 || value.length > 512) {
throw new Error(`${name} must be a non-empty string of at most 512 characters`)
}
}
export function validateHello(hello: PtyConsumerSessionHello): void {
assertNonEmptyString(hello.clientInstanceId, 'clientInstanceId')
if (hello.requestedRole !== 'session-owner' && hello.requestedRole !== 'subscriber') {
throw new Error('requestedRole must be session-owner or subscriber')
}
if (hello.resume) {
if (!Number.isSafeInteger(hello.resume.ownerGeneration) || hello.resume.ownerGeneration <= 0) {
throw new Error('resume.ownerGeneration must be a positive safe integer')
}
assertNonEmptyString(hello.resume.ownerLease, 'resume.ownerLease')
}
const flow = hello.capabilities?.outputFlowControl
if (!flow) {
return
}
if (
!Array.isArray(flow.versions) ||
flow.versions.length > MAX_CAPABILITY_VERSIONS ||
flow.versions.some((version) => !Number.isSafeInteger(version) || version <= 0)
) {
throw new Error('outputFlowControl.versions must contain positive safe integers')
}
if (!Number.isSafeInteger(flow.requestedWindowSu) || flow.requestedWindowSu <= 0) {
throw new Error('outputFlowControl.requestedWindowSu must be a positive safe integer')
}
}
// Why: a duplicate open on one connection is only idempotent if it asks for exactly the same thing.
export function helloFingerprint(hello: PtyConsumerSessionHello): string {
const flow = hello.capabilities?.outputFlowControl
return JSON.stringify({
clientInstanceId: hello.clientInstanceId,
requestedRole: hello.requestedRole,
resume: hello.resume,
outputFlowControl: flow
? {
versions: [...flow.versions].sort((a, b) => a - b),
requestedWindowSu: flow.requestedWindowSu
}
: undefined
})
}
+181 -5
View File
@@ -1,5 +1,7 @@
import { describe, expect, it } from 'vitest'
import {
PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR,
PTY_CONSUMER_OWNER_RECOVERY_SUPERSEDED_ERROR,
PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR,
PtyConsumerSession,
type PtyConsumerAuthentication,
@@ -121,7 +123,7 @@ describe('PtyConsumerSession', () => {
).toThrow('authentication required')
})
it('rotates the lease and increments owner generation on valid recovery', () => {
it('keeps the lease stable and increments owner generation on valid recovery', () => {
const session = createSession()
const first = session.admit(ownerHello(), auth('connection-1'))
first.commitPublication()
@@ -141,15 +143,97 @@ describe('PtyConsumerSession', () => {
expect(recovered.grant).toMatchObject({
role: 'session-owner',
ownerGeneration: 2,
ownerLease: 'lease-2'
ownerLease: 'lease-1'
})
})
it('rejects recovery while the current owner is still active', () => {
it('displaces a still-attached owner that a matching resume proof reclaims', () => {
const session = createSession()
const first = session.admit(ownerHello(), auth('connection-1'))
first.commitPublication()
const recovered = session.admit(
ownerHello({
resume: {
ownerGeneration: first.grant.ownerGeneration!,
ownerLease: first.grant.ownerLease!
}
}),
auth('connection-2')
)
expect(recovered.displacedOwner).toEqual({
connectionId: 'connection-1',
grant: first.grant
})
// Why: the incumbent keeps authority until the replacement grant is actually published.
expect(session.activeGrant('connection-1')).toBe(first.grant)
recovered.commitPublication()
expect(recovered.grant).toMatchObject({ role: 'session-owner', ownerGeneration: 2 })
expect(session.activeGrant('connection-1')).toBeNull()
expect(session.activeGrant('connection-2')).toBe(recovered.grant)
})
it('restores the displaced owner when the replacement publication rolls back', () => {
const session = createSession()
const first = session.admit(ownerHello(), auth('connection-1'))
first.commitPublication()
const recovered = session.admit(
ownerHello({
resume: {
ownerGeneration: first.grant.ownerGeneration!,
ownerLease: first.grant.ownerLease!
}
}),
auth('connection-2')
)
recovered.rollbackPublication()
expect(session.activeGrant('connection-1')).toBe(first.grant)
expect(session.activeGrant('connection-2')).toBeNull()
// Why: the restored incumbent must still hold the lease it was admitted with.
const reclaimed = session.admit(
ownerHello({
resume: {
ownerGeneration: first.grant.ownerGeneration!,
ownerLease: first.grant.ownerLease!
}
}),
auth('connection-3')
)
expect(reclaimed.displacedOwner?.connectionId).toBe('connection-1')
})
it('expires a displaced owner restored after its connection already closed', () => {
let now = 10
const session = createSession({ now: () => now })
const first = session.admit(ownerHello(), auth('connection-1'))
first.commitPublication()
const recovered = session.admit(
ownerHello({
resume: {
ownerGeneration: first.grant.ownerGeneration!,
ownerLease: first.grant.ownerLease!
}
}),
auth('connection-2')
)
session.close('connection-1')
recovered.rollbackPublication()
now += 30_000
session.sweepExpired()
const fresh = session.admit(ownerHello(), auth('connection-4'))
expect(fresh.grant.role).toBe('session-owner')
})
it('refuses recovery while the incumbent grant publication is still settling', () => {
const session = createSession()
const first = session.admit(ownerHello(), auth('connection-1'))
expect(() =>
session.admit(
ownerHello({
@@ -160,7 +244,99 @@ describe('PtyConsumerSession', () => {
}),
auth('connection-2')
)
).toThrow('Active owner')
).toThrow(
expect.objectContaining({
code: PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR,
message: expect.stringContaining('still pending')
})
)
})
it('fences an old recovery generation after its replacement commits', () => {
const session = createSession()
const first = session.admit(ownerHello(), auth('connection-1'))
first.commitPublication()
const resume = {
ownerGeneration: first.grant.ownerGeneration!,
ownerLease: first.grant.ownerLease!
}
const replacement = session.admit(ownerHello({ resume }), auth('connection-2'))
expect(() => session.admit(ownerHello({ resume }), auth('connection-3'))).toThrow(
expect.objectContaining({ code: PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR })
)
replacement.commitPublication()
expect(() => session.admit(ownerHello({ resume }), auth('connection-3'))).toThrow(
expect.objectContaining({ code: PTY_CONSUMER_OWNER_RECOVERY_SUPERSEDED_ERROR })
)
const retry = session.admit(
ownerHello({
resume: {
ownerGeneration: replacement.grant.ownerGeneration!,
ownerLease: replacement.grant.ownerLease!
}
}),
auth('connection-3')
)
expect(retry.grant).toMatchObject({
role: 'session-owner',
ownerGeneration: 3,
ownerLease: first.grant.ownerLease
})
expect(retry.displacedOwner?.connectionId).toBe('connection-2')
// Why: the committed replacement already retired the incumbent, so only connection-2 is left to displace.
expect(session.activeGrant('connection-1')).toBeNull()
})
it('accepts an older stable proof after its replacement disconnects', () => {
const session = createSession()
const first = session.admit(ownerHello(), auth('connection-1'))
first.commitPublication()
const resume = {
ownerGeneration: first.grant.ownerGeneration!,
ownerLease: first.grant.ownerLease!
}
const replacement = session.admit(ownerHello({ resume }), auth('connection-2'))
replacement.commitPublication()
session.close('connection-2')
const retry = session.admit(ownerHello({ resume }), auth('connection-3'))
expect(retry.grant).toMatchObject({
role: 'session-owner',
ownerGeneration: 3,
ownerLease: first.grant.ownerLease
})
})
it('retries overlapping recovery against the incumbent after publication rolls back', () => {
const session = createSession()
const first = session.admit(ownerHello(), auth('connection-1'))
first.commitPublication()
const resume = {
ownerGeneration: first.grant.ownerGeneration!,
ownerLease: first.grant.ownerLease!
}
const replacement = session.admit(ownerHello({ resume }), auth('connection-2'))
expect(() => session.admit(ownerHello({ resume }), auth('connection-3'))).toThrow(
expect.objectContaining({ code: PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR })
)
replacement.rollbackPublication()
const retry = session.admit(ownerHello({ resume }), auth('connection-3'))
expect(retry.grant).toMatchObject({
role: 'session-owner',
ownerGeneration: 3,
ownerLease: first.grant.ownerLease
})
expect(retry.displacedOwner?.connectionId).toBe('connection-1')
// Why: the rolled-back replacement never published, so it holds no grant the retry could displace.
expect(session.activeGrant('connection-2')).toBeNull()
})
it('types mismatched recovery without disturbing principal or lease ownership', () => {
@@ -201,7 +377,7 @@ describe('PtyConsumerSession', () => {
expect(recovered.grant).toMatchObject({
role: 'session-owner',
ownerGeneration: 2,
ownerLease: 'lease-2'
ownerLease: 'lease-1'
})
})
+59 -71
View File
@@ -4,22 +4,28 @@ import {
PTY_CONSUMER_SESSION_PROTOCOL_VERSION,
PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR,
type PtyConsumerAuthentication,
type PtyConsumerDisplacedOwner,
type PtyConsumerSessionAdmission,
type PtyConsumerSessionGrant,
type PtyConsumerSessionHello,
type PtyConsumerSessionOptions
} from './pty-consumer-session-contract'
import {
assertNonEmptyString,
helloFingerprint,
MAX_CAPABILITY_VERSIONS,
validateHello
} from './pty-consumer-session-hello'
import { assertPtyConsumerOwnerRecovery } from './pty-consumer-owner-recovery'
export * from './pty-consumer-session-contract'
const MAX_CAPABILITY_VERSIONS = 8
type ClientRecord = {
fingerprint: string
principal: string
clientInstanceId: string
grant: Readonly<PtyConsumerSessionGrant>
state: 'pending' | 'active'
state: 'pending' | 'active' | 'displaced'
publicationState: 'pending' | 'committed' | 'rolled-back'
}
@@ -34,54 +40,6 @@ type OwnerRecord = {
replaces?: OwnerRecord
}
function assertNonEmptyString(value: unknown, name: string): asserts value is string {
if (typeof value !== 'string' || value.length === 0 || value.length > 512) {
throw new Error(`${name} must be a non-empty string of at most 512 characters`)
}
}
function validateHello(hello: PtyConsumerSessionHello): void {
assertNonEmptyString(hello.clientInstanceId, 'clientInstanceId')
if (hello.requestedRole !== 'session-owner' && hello.requestedRole !== 'subscriber') {
throw new Error('requestedRole must be session-owner or subscriber')
}
if (hello.resume) {
if (!Number.isSafeInteger(hello.resume.ownerGeneration) || hello.resume.ownerGeneration <= 0) {
throw new Error('resume.ownerGeneration must be a positive safe integer')
}
assertNonEmptyString(hello.resume.ownerLease, 'resume.ownerLease')
}
const flow = hello.capabilities?.outputFlowControl
if (!flow) {
return
}
if (
!Array.isArray(flow.versions) ||
flow.versions.length > MAX_CAPABILITY_VERSIONS ||
flow.versions.some((version) => !Number.isSafeInteger(version) || version <= 0)
) {
throw new Error('outputFlowControl.versions must contain positive safe integers')
}
if (!Number.isSafeInteger(flow.requestedWindowSu) || flow.requestedWindowSu <= 0) {
throw new Error('outputFlowControl.requestedWindowSu must be a positive safe integer')
}
}
function helloFingerprint(hello: PtyConsumerSessionHello): string {
const flow = hello.capabilities?.outputFlowControl
return JSON.stringify({
clientInstanceId: hello.clientInstanceId,
requestedRole: hello.requestedRole,
resume: hello.resume,
outputFlowControl: flow
? {
versions: [...flow.versions].sort((a, b) => a - b),
requestedWindowSu: flow.requestedWindowSu
}
: undefined
})
}
export class PtyConsumerSession {
private readonly clients = new Map<string, ClientRecord>()
private readonly now: () => number
@@ -160,7 +118,7 @@ export class PtyConsumerSession {
if (owner) {
this.owner = owner
}
return this.admissionFor(client)
return this.admissionFor(client, this.displacedOwnerFor(owner))
}
close(connectionId: string): void {
@@ -170,6 +128,17 @@ export class PtyConsumerSession {
}
this.clients.delete(connectionId)
if (this.owner?.connectionId !== connectionId) {
// Why: a pending replacement can still roll back onto the owner it is displacing; restoring an
// 'active' record whose connection has since closed would wedge an owner that can never expire.
if (
this.owner?.replaces?.connectionId === connectionId &&
this.owner.replaces.state === 'active'
) {
this.owner = {
...this.owner,
replaces: { ...this.owner.replaces, state: 'disconnected', disconnectedAt: this.now() }
}
}
return
}
if (this.owner.state === 'pending') {
@@ -192,9 +161,13 @@ export class PtyConsumerSession {
return client?.state === 'active' ? client.grant : null
}
private admissionFor(client: ClientRecord): PtyConsumerSessionAdmission {
private admissionFor(
client: ClientRecord,
displacedOwner?: Readonly<PtyConsumerDisplacedOwner>
): PtyConsumerSessionAdmission {
return {
grant: client.grant,
...(displacedOwner ? { displacedOwner } : {}),
commitPublication: () => {
if (client.publicationState !== 'pending') {
return
@@ -206,6 +179,7 @@ export class PtyConsumerSession {
client.state = 'active'
const owner = this.owner
if (owner?.connectionId === this.connectionIdFor(client) && owner.state === 'pending') {
this.retireDisplacedOwner(owner.replaces)
this.owner = { ...owner, state: 'active', replaces: undefined }
}
},
@@ -254,32 +228,46 @@ export class PtyConsumerSession {
if (!hello.resume) {
return null
}
const recoveryMatches =
hello.resume.ownerGeneration === current.generation &&
hello.resume.ownerLease === current.lease &&
hello.clientInstanceId === current.clientInstanceId &&
authentication.principal === current.principal
if (!recoveryMatches) {
throw Object.assign(
new Error('Owner recovery lease is stale or belongs to another principal'),
{ code: PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR }
)
}
if (current.state === 'pending') {
throw new Error('Owner grant publication is still pending')
}
if (current.state !== 'disconnected') {
throw new Error('Active owner cannot be replaced by recovery')
}
assertPtyConsumerOwnerRecovery(hello, authentication, current)
// Why an active owner is displaced rather than refused: the resume proof matched this owner's
// generation, lease, client instance, and principal on a *different* transport, so the requester is
// the same logical owner reconnecting. Waiting for the incumbent's socket to close is unbounded —
// a half-open connection after sleep/resume or NAT loss never gets there.
return this.newOwner(hello, authentication, current)
}
private displacedOwnerFor(
owner: OwnerRecord | null
): Readonly<PtyConsumerDisplacedOwner> | undefined {
const replaced = owner?.replaces
if (replaced?.state !== 'active') {
return undefined
}
const client = this.clients.get(replaced.connectionId)
if (client?.state !== 'active') {
return undefined
}
return Object.freeze({ connectionId: replaced.connectionId, grant: client.grant })
}
// Why: the displaced connection may still be writable (half-open), so revoke its grant the moment the
// replacement is published — a stale owner must not keep driving deliveries under the old generation.
private retireDisplacedOwner(replaced: OwnerRecord | undefined): void {
if (replaced?.state !== 'active') {
return
}
const client = this.clients.get(replaced.connectionId)
if (client?.state === 'active') {
client.state = 'displaced'
}
}
private newOwner(
hello: PtyConsumerSessionHello,
authentication: PtyConsumerAuthentication,
replaces: OwnerRecord | null
): OwnerRecord {
const lease = this.createLease()
const lease = replaces?.lease ?? this.createLease()
assertNonEmptyString(lease, 'ownerLease')
return {
connectionId: authentication.connectionId,