Files
orca/src/main/ssh/ssh-remote-cli-host-passthrough.test.ts
T
Brennan Benson 913509edeb fix(orchestration): prevent slow worker-start stalls (#16300)
* Extend orchestration agent submission timing budgets

* fix(orchestration): preserve mutation recovery identity

* fix(orchestration): preserve recovery executable identity

* fix(orchestration): keep worker starts and recovery commands safe

* test(orchestration): cover federated worker preflight

* fix(orchestration): harden mutation recovery

* fix(orchestration): redact dispatch recovery credentials

* chore: preserve upstream skill dialog formatting

* test(orchestration): stabilize agent prompt submit e2e

* fix(orchestration): validate federated start receipts

* perf(runtime): cache unchanged prompt verification tail

* fix(orchestration): reject worker-start timer overflow

* fix(orchestration): normalize worker-start timeout defaults

* fix(orchestration): normalize worker-start readiness budgets

* fix(orchestration): normalize federated readiness timeout

* test(runtime): tolerate current-main degradation exports

* chore: preserve current-main orcad formatting

* chore: drop unrelated formatting carryover
2026-08-27 15:25:30 -07:00

403 lines
14 KiB
TypeScript

import { EventEmitter } from 'node:events'
import { join } from 'node:path'
import { describe, expect, it, vi } from 'vitest'
vi.mock('electron', () => ({
app: {
isPackaged: false,
getAppPath: () => '/host/app'
}
}))
vi.mock('../persistence', () => ({
getCanonicalUserDataPath: () => '/host/user-data'
}))
import {
HostCliUnavailableError,
buildHostCliEnv,
resolveHostCliEntryPath,
resolveHostCliKillTimeoutMs,
runHostOrcaCliPassthrough
} from './ssh-remote-cli-host-passthrough'
import { resolveOrchestrationAskClientTimeoutMs } from '../../shared/orchestration-ask-timeout'
import { remoteCliRequestTimeoutMs } from '../../relay/remote-cli-timeout'
import { MAX_TIMER_DELAY_MS } from '../../shared/timer-delay'
import {
ORCHESTRATION_COMPATIBILITY_ATTACHMENT_ENV,
ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV,
ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV,
ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV
} from '../../shared/orchestration-compatibility-evidence'
import { REMOTE_ARTIFACT_INPUT_ENV } from '../../shared/artifact-cli-bridge'
type FakeChild = EventEmitter & {
stdout: EventEmitter
stderr: EventEmitter
stdin: { end: ReturnType<typeof vi.fn>; on: ReturnType<typeof vi.fn> }
kill: ReturnType<typeof vi.fn>
}
function createFakeChild(): FakeChild {
const child = new EventEmitter() as FakeChild
child.stdout = new EventEmitter()
child.stderr = new EventEmitter()
child.stdin = { end: vi.fn(), on: vi.fn() }
child.kill = vi.fn()
return child
}
const BASE_OPTIONS = {
execPath: '/host/electron',
cliEntryPath: '/host/app/out/cli/index.js',
userDataPath: '/host/user-data',
entryExists: () => true
}
describe('resolveHostCliEntryPath', () => {
it('uses the in-repo entry for dev builds and the unpacked asar entry when packaged', () => {
expect(
resolveHostCliEntryPath({ isPackaged: false, resourcesPath: '/r', appPath: '/host/app' })
).toBe(join('/host/app', 'out', 'cli', 'index.js'))
expect(
resolveHostCliEntryPath({ isPackaged: true, resourcesPath: '/r', appPath: '/host/app' })
).toBe(join('/r', 'app.asar.unpacked', 'out', 'cli', 'index.js'))
})
})
describe('buildHostCliEnv', () => {
it('forwards only Orca terminal-context vars from the remote env', () => {
const env = buildHostCliEnv({
hostEnv: { PATH: '/host/bin', NODE_OPTIONS: '--inspect' },
remoteEnv: {
ORCA_TERMINAL_HANDLE: 'term_remote',
ORCA_WORKTREE_ID: 'repo::/home/alice/wt',
ORCA_PANE_KEY: 'pane-9',
ORCA_AGENT_LAUNCH_TOKEN: 'launch-secret',
ORCA_WORKSPACE_ID: 'ws-1',
[ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV]: 'wsl',
[ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV]: 'caller-host',
[ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV]: 'caller-incarnation',
[ORCHESTRATION_COMPATIBILITY_ATTACHMENT_ENV]: 'caller-attachment',
// Why: these are remote-machine paths and must not leak into the host
// subprocess (PATH would break host binary lookup; user-data would
// retarget the CLI at a different local instance).
PATH: '/remote/bin',
ORCA_USER_DATA_PATH: '/remote/user-data'
},
userDataPath: '/host/user-data',
remoteCwd: '/home/alice/wt/sub',
runtimeAuthority: {
kind: 'ssh',
targetId: 'saved-target',
connectionIncarnation: 'connection-incarnation',
attachmentId: 'runtime-attachment'
}
})
expect(env.ORCA_TERMINAL_HANDLE).toBe('term_remote')
expect(env.ORCA_WORKTREE_ID).toBe('repo::/home/alice/wt')
expect(env.ORCA_PANE_KEY).toBe('pane-9')
expect(env.ORCA_AGENT_LAUNCH_TOKEN).toBe('launch-secret')
expect(env.ORCA_WORKSPACE_ID).toBe('ws-1')
expect(env[ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV]).toBe('ssh')
expect(env[ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV]).toBe('saved-target')
expect(env[ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV]).toBe('connection-incarnation')
expect(env[ORCHESTRATION_COMPATIBILITY_ATTACHMENT_ENV]).toBe('runtime-attachment')
expect(env.PATH).toBe('/host/bin')
expect(env.ORCA_USER_DATA_PATH).toBe('/host/user-data')
expect(env.ORCA_CLI_CWD).toBe('/home/alice/wt/sub')
expect(env.ORCA_CLI_COMMAND).toBe('orca')
expect(env.ELECTRON_RUN_AS_NODE).toBe('1')
expect(env.NODE_OPTIONS).toBeUndefined()
expect(env.ORCA_NODE_OPTIONS).toBe('--inspect')
})
it.each([
['dev host', { ORCA_DEV_REPO_ROOT: '/repo', ORCA_CLI_COMMAND: 'orca-dev' }],
['packaged Linux host', { ORCA_CLI_COMMAND: 'orca-ide' }],
['local host', {}],
['WSL host', { WSL_DISTRO_NAME: 'Ubuntu', ORCA_CLI_COMMAND: 'orca-ide' }],
['Windows host', { ComSpec: 'C:\\Windows\\System32\\cmd.exe' }]
])('pins %s recovery to the remote shim', (_name, hostEnv) => {
const env = buildHostCliEnv({
hostEnv,
remoteEnv: { ORCA_CLI_COMMAND: 'untrusted-remote-command' },
userDataPath: '/host/user-data',
remoteCwd: '/srv/repo'
})
expect(env.ORCA_CLI_COMMAND).toBe('orca')
})
it('namespaces identical remote artifact paths by stable SSH target', () => {
const artifactInput = {
sourceKey: '/srv/repo/report.html',
fileName: 'report.html',
contentType: 'text/html' as const
}
const build = (targetId: string) =>
buildHostCliEnv({
hostEnv: {},
remoteEnv: {},
userDataPath: '/host/user-data',
remoteCwd: '/srv/repo',
runtimeAuthority: {
kind: 'ssh',
targetId,
connectionIncarnation: 'ephemeral-connection',
attachmentId: 'ephemeral-attachment'
},
artifactInput
})[REMOTE_ARTIFACT_INPUT_ENV]
const first = JSON.parse(String(build('host-a')))
const second = JSON.parse(String(build('host-b')))
expect(first.sourceKey).not.toBe(second.sourceKey)
expect(JSON.parse(first.sourceKey)).toEqual(['ssh', 'host-a', '/srv/repo/report.html'])
expect(JSON.parse(second.sourceKey)).toEqual(['ssh', 'host-b', '/srv/repo/report.html'])
expect(first.fileName).toBe('report.html')
})
})
describe('resolveHostCliKillTimeoutMs', () => {
it('extends the kill timer past an explicit --timeout-ms budget', () => {
expect(resolveHostCliKillTimeoutMs(['terminal', 'wait', '--timeout-ms', '1800000'])).toBe(
1_920_000
)
expect(resolveHostCliKillTimeoutMs(['orchestration', 'check', '--timeout-ms=5000'])).toBe(
600_000
)
expect(resolveHostCliKillTimeoutMs(['worktree', 'list'])).toBe(600_000)
})
it.each([
[[], 720_000],
[['--timeout-ms', String(Number.MAX_SAFE_INTEGER)], 1_920_000],
[['--timeout-ms', String(Number.MAX_SAFE_INTEGER + 1)], 720_000],
[['--timeout-ms', '9007199254740991.1'], 720_000],
[['--timeout-ms', '1', '--timeout-ms=1800000'], 1_920_000],
[['--timeout-ms=1800000', '--timeout-ms', '1'], 600_000],
[['--timeout-ms', '1800000', '--timeout-ms'], 720_000],
[['--timeout-ms=1800000', '--timeout-ms='], 720_000],
[['--timeout-ms=1800000', '--timeout-ms', 'bad'], 720_000],
[['--timeout-ms', 'bad', '--timeout-ms=1800000'], 1_920_000],
[['--timeout-ms=bad', '--timeout-ms', '1800000'], 1_920_000]
])('bounds ask child timers with last-wins flags %#', (timeoutArgs, expected) => {
expect(resolveHostCliKillTimeoutMs(['orchestration', '--json', 'ask', ...timeoutArgs])).toBe(
expected
)
})
it('does not apply the ask maximum to other commands', () => {
expect(resolveHostCliKillTimeoutMs(['terminal', 'wait', '--timeout-ms', '1800001'])).toBe(
1_920_001
)
})
it.each(['+1000000', '1000000.0', '1e6'])(
'extends non-ask child timers using CLI-compatible integer syntax %s',
(raw) => {
expect(resolveHostCliKillTimeoutMs(['terminal', 'wait', '--timeout-ms', raw])).toBe(1_120_000)
}
)
it.each([
'Infinity',
'1.5',
'-1',
'bad',
String(Number.MAX_SAFE_INTEGER),
String(MAX_TIMER_DELAY_MS - 120_000 + 1)
])('falls back to the default kill timer when a non-ask --timeout-ms %s is unusable', (raw) => {
expect(resolveHostCliKillTimeoutMs(['terminal', 'wait', '--timeout-ms', raw])).toBe(600_000)
})
it('keeps the largest non-ask kill timer that stays inside the timer range', () => {
expect(
resolveHostCliKillTimeoutMs([
'terminal',
'wait',
'--timeout-ms',
String(MAX_TIMER_DELAY_MS - 120_000)
])
).toBe(MAX_TIMER_DELAY_MS)
})
it.each<[string[], number | undefined]>([
[[], undefined],
[['--timeout-ms', '1'], 1],
[['--timeout-ms', String(Number.MAX_SAFE_INTEGER)], Number.MAX_SAFE_INTEGER],
[['--timeout-ms', String(Number.MAX_SAFE_INTEGER + 1)], undefined]
])('keeps inner, host, and relay ask deadlines ordered %#', (timeoutArgs, parsedTimeout) => {
const argv = ['orchestration', 'ask', '--to', 'term_x', ...timeoutArgs]
const innerTimeout = resolveOrchestrationAskClientTimeoutMs(parsedTimeout)
const hostTimeout = resolveHostCliKillTimeoutMs(argv)
const relayTimeout = remoteCliRequestTimeoutMs({ argv })
expect(innerTimeout).toBeLessThan(hostTimeout)
expect(hostTimeout).toBeLessThan(relayTimeout!)
})
})
describe('runHostOrcaCliPassthrough', () => {
it('spawns the bundled CLI entry with the remote argv and returns captured output', async () => {
const child = createFakeChild()
const spawn = vi.fn(() => child)
const resultPromise = runHostOrcaCliPassthrough(
{
argv: ['orchestration', 'task-create', '--spec', 'do the thing', '--json'],
cwd: '/home/alice/wt',
env: { ORCA_TERMINAL_HANDLE: 'term_remote' }
},
{ ...BASE_OPTIONS, spawn: spawn as never }
)
await Promise.resolve()
child.stdout.emit('data', Buffer.from('{"ok":true}\n'))
child.stderr.emit('data', Buffer.from('warn\n'))
child.emit('close', 0)
const result = await resultPromise
expect(result).toEqual({ stdout: '{"ok":true}\n', stderr: 'warn\n', exitCode: 0 })
expect(spawn).toHaveBeenCalledTimes(1)
const [execPath, args, options] = spawn.mock.calls[0] as unknown as [
string,
string[],
{ env: NodeJS.ProcessEnv }
]
expect(execPath).toBe('/host/electron')
expect(args).toEqual([
'/host/app/out/cli/index.js',
'orchestration',
'task-create',
'--spec',
'do the thing',
'--json'
])
expect(options.env.ELECTRON_RUN_AS_NODE).toBe('1')
expect(options.env.ORCA_CLI_CWD).toBe('/home/alice/wt')
expect(options.env.ORCA_TERMINAL_HANDLE).toBe('term_remote')
// Why: stdin must be closed even without a payload so CLI handlers that
// stream stdin see EOF instead of hanging forever.
expect(child.stdin.end).toHaveBeenCalledWith()
})
it('pipes a stdin payload to the CLI subprocess', async () => {
const child = createFakeChild()
const spawn = vi.fn(() => child)
const resultPromise = runHostOrcaCliPassthrough(
{
argv: ['linear', 'comment', 'add', 'ENG-1', '--body-file', '-'],
cwd: '/home/alice/wt',
env: {},
stdin: 'comment body'
},
{ ...BASE_OPTIONS, spawn: spawn as never }
)
await Promise.resolve()
child.emit('close', 0)
await resultPromise
expect(child.stdin.end).toHaveBeenCalledWith('comment body')
})
it('propagates non-zero exit codes', async () => {
const child = createFakeChild()
const spawn = vi.fn(() => child)
const resultPromise = runHostOrcaCliPassthrough(
{ argv: ['worktree', 'show'], cwd: '/', env: {} },
{ ...BASE_OPTIONS, spawn: spawn as never }
)
await Promise.resolve()
child.stderr.emit('data', Buffer.from('boom\n'))
child.emit('close', 3)
await expect(resultPromise).resolves.toEqual({ stdout: '', stderr: 'boom\n', exitCode: 3 })
})
it('throws HostCliUnavailableError when the CLI entry is missing', async () => {
const spawn = vi.fn()
await expect(
runHostOrcaCliPassthrough(
{ argv: ['status'], cwd: '/', env: {} },
{ ...BASE_OPTIONS, entryExists: () => false, spawn: spawn as never }
)
).rejects.toBeInstanceOf(HostCliUnavailableError)
expect(spawn).not.toHaveBeenCalled()
})
it('rejects an invalid injected kill timeout before spawning', async () => {
const spawn = vi.fn()
await expect(
runHostOrcaCliPassthrough(
{ argv: ['status'], cwd: '/', env: {} },
{ ...BASE_OPTIONS, spawn: spawn as never, killTimeoutMs: 2_147_483_648 }
)
).rejects.toBeInstanceOf(RangeError)
expect(spawn).not.toHaveBeenCalled()
})
it('throws HostCliUnavailableError when the subprocess fails to launch', async () => {
const child = createFakeChild()
const spawn = vi.fn(() => child)
const resultPromise = runHostOrcaCliPassthrough(
{ argv: ['status'], cwd: '/', env: {} },
{ ...BASE_OPTIONS, spawn: spawn as never }
)
await Promise.resolve()
child.emit('error', new Error('spawn ENOENT'))
await expect(resultPromise).rejects.toBeInstanceOf(HostCliUnavailableError)
})
it('kills the subprocess and reports an error when the kill timeout elapses', async () => {
vi.useFakeTimers()
try {
const child = createFakeChild()
const spawn = vi.fn(() => child)
const resultPromise = runHostOrcaCliPassthrough(
{ argv: ['terminal', 'wait', '--for', 'exit'], cwd: '/', env: {} },
{ ...BASE_OPTIONS, spawn: spawn as never, killTimeoutMs: 1000 }
)
await vi.advanceTimersByTimeAsync(1001)
const result = await resultPromise
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
expect(result.exitCode).toBe(1)
expect(result.stderr).toContain('timed out')
} finally {
vi.useRealTimers()
}
})
it('caps runaway output instead of buffering it unbounded', async () => {
const child = createFakeChild()
const spawn = vi.fn(() => child)
const resultPromise = runHostOrcaCliPassthrough(
{ argv: ['terminal', 'read'], cwd: '/', env: {} },
{ ...BASE_OPTIONS, spawn: spawn as never }
)
await Promise.resolve()
const chunk = Buffer.alloc(3 * 1024 * 1024, 97)
for (let i = 0; i < 4; i += 1) {
child.stdout.emit('data', chunk)
}
child.emit('close', 0)
const result = await resultPromise
expect(result.stdout.length).toBeLessThanOrEqual(8 * 1024 * 1024 + 64)
expect(result.stdout).toContain('output truncated')
})
})