Files
orca/src/cli/orchestration-mutation-recovery.test.ts
T
Brennan Benson 913509edeb fix(orchestration): prevent slow worker-start stalls (#16300)
* Extend orchestration agent submission timing budgets

* fix(orchestration): preserve mutation recovery identity

* fix(orchestration): preserve recovery executable identity

* fix(orchestration): keep worker starts and recovery commands safe

* test(orchestration): cover federated worker preflight

* fix(orchestration): harden mutation recovery

* fix(orchestration): redact dispatch recovery credentials

* chore: preserve upstream skill dialog formatting

* test(orchestration): stabilize agent prompt submit e2e

* fix(orchestration): validate federated start receipts

* perf(runtime): cache unchanged prompt verification tail

* fix(orchestration): reject worker-start timer overflow

* fix(orchestration): normalize worker-start timeout defaults

* fix(orchestration): normalize worker-start readiness budgets

* fix(orchestration): normalize federated readiness timeout

* test(runtime): tolerate current-main degradation exports

* chore: preserve current-main orcad formatting

* chore: drop unrelated formatting carryover
2026-08-27 15:25:30 -07:00

270 lines
8.5 KiB
TypeScript

import { describe, expect, it } from 'vitest'
import { runProcess } from '../shared/child-process/run-process'
import {
orchestrationMutationRecoveryError,
renderCommand
} from './orchestration-mutation-recovery'
import { RuntimeClientError } from './runtime-client'
describe('orchestration mutation recovery', () => {
it('queries a known dispatch before issuing the keyed retry', () => {
const result = orchestrationMutationRecoveryError(
new RuntimeClientError('runtime_timeout', 'request timed out', {
orchestrationRequestId: 'request_1',
dispatchId: 'dispatch_1',
originalCommand: ['orca', 'orchestration', 'worker-start', '--task', 'task_1']
})
) as RuntimeClientError
expect(result.data).toMatchObject({
recovery: {
orchestrationRequestId: 'request_1',
dispatchId: 'dispatch_1',
queryCommand: [
'orca',
'orchestration',
'worker-show',
'--dispatch',
'dispatch_1',
'--json'
],
retryCommand: [
'orca',
'orchestration',
'worker-start',
'--task',
'task_1',
'--retry-request',
'request_1'
],
workerDeathInferred: false
}
})
expect(result.message.indexOf('orca orchestration worker-show')).toBeLessThan(
result.message.indexOf('orca orchestration worker-start')
)
expect((result.data as { nextSteps?: string[] }).nextSteps).toEqual([
'Run orca orchestration worker-show --dispatch dispatch_1 --json before retrying.',
'Run orca orchestration worker-start --task task_1 --retry-request request_1.'
])
})
it('does not invent a dispatch for an old-client-shaped error', () => {
const result = orchestrationMutationRecoveryError(
new RuntimeClientError('runtime_timeout', 'request timed out', {
orchestrationRequestId: 'request_2',
originalCommand: ['orca', 'orchestration', 'worker-start', '--task', 'task_2']
})
) as RuntimeClientError
expect(result.data).toMatchObject({
recovery: {
orchestrationRequestId: 'request_2',
retryCommand: expect.arrayContaining(['--retry-request', 'request_2']),
workerDeathInferred: false
}
})
expect((result.data as Record<string, unknown>).recovery).not.toHaveProperty('dispatchId')
expect(result.message).not.toContain('worker death')
})
it('renders the exact executable and safely quotes original arguments', () => {
const result = orchestrationMutationRecoveryError(
new RuntimeClientError('runtime_timeout', 'request timed out', {
orchestrationRequestId: 'request_3',
dispatchId: 'dispatch_3',
originalCommand: [
'orca-dev',
'orchestration',
'worker-start',
'--task',
'task 3',
'--comment',
'literal $(do-not-run)'
]
})
) as RuntimeClientError
expect((result.data as { nextSteps?: string[] }).nextSteps).toEqual([
'Run orca-dev orchestration worker-show --dispatch dispatch_3 --json before retrying.',
"Run orca-dev orchestration worker-start --task 'task 3' --comment 'literal $(do-not-run)' --retry-request request_3."
])
expect(result.message).toContain("'literal $(do-not-run)'")
})
it('parses legacy command text without losing quoted arguments', () => {
const result = orchestrationMutationRecoveryError(
new RuntimeClientError('runtime_timeout', 'request timed out', {
orchestrationRequestId: 'request_4',
originalCommand:
'orca-ide orchestration worker-stop --dispatch dispatch_4 --comment "quoted value"'
})
) as RuntimeClientError
expect(
(result.data as { recovery?: { retryCommand?: string[] } }).recovery?.retryCommand
).toEqual([
'orca-ide',
'orchestration',
'worker-stop',
'--dispatch',
'dispatch_4',
'--comment',
'quoted value',
'--retry-request',
'request_4'
])
})
it.each([
[
'gate-create',
['orca', 'orchestration', 'gate-create', '--task', 'task_1', '--question', 'ship?']
],
[
'worker-retain',
['orca', 'orchestration', 'worker-retain', '--dispatch', 'dispatch_1', '--json']
]
])('replays exact %s argv with the keyed retry', (_name, originalCommand) => {
const result = orchestrationMutationRecoveryError(
new RuntimeClientError('runtime_timeout', 'request timed out', {
orchestrationRequestId: 'request_exact',
originalCommand
})
) as RuntimeClientError
expect(
(result.data as { recovery?: { retryCommand?: string[] } }).recovery?.retryCommand
).toEqual([...originalCommand, '--retry-request', 'request_exact'])
})
it('reuses the request identity without duplicating an existing retry flag', () => {
const result = orchestrationMutationRecoveryError(
new RuntimeClientError('runtime_timeout', 'request timed out', {
orchestrationRequestId: 'request_reused',
originalCommand: [
'orca',
'orchestration',
'worker-retain',
'--dispatch',
'dispatch_1',
'--retry-request=request_reused'
]
})
) as RuntimeClientError
expect(
(result.data as { recovery?: { retryCommand?: string[] } }).recovery?.retryCommand
).toEqual([
'orca',
'orchestration',
'worker-retain',
'--dispatch',
'dispatch_1',
'--retry-request',
'request_reused'
])
})
it('renders Windows cmd recovery guidance without quote drift or percent expansion', () => {
expect(
renderCommand(
['orca', 'orchestration', 'worker-start', '--comment', 'literal "quoted" %PATH% & safe'],
'win32',
{ ComSpec: 'C:\\Windows\\System32\\cmd.exe' }
)
).toBe(
'"orca" "orchestration" "worker-start" "--comment" "literal ""quoted"" "^%"PATH"^%" & safe"'
)
})
it('keeps PowerShell and POSIX recovery guidance literal', () => {
expect(
renderCommand(['orca', 'literal "quoted" $HOME'], 'win32', {
ComSpec: 'powershell.exe'
})
).toBe("& 'orca' 'literal \\\"quoted\\\" $HOME'")
expect(renderCommand(['orca', 'literal $(do-not-run)'], 'darwin')).toBe(
"orca 'literal $(do-not-run)'"
)
})
it.runIf(process.platform !== 'win32')(
'round trips POSIX recovery argv through /bin/sh',
async () => {
const values = ['with spaces', "apostrophe's", 'literal $(do-not-run)', 'line one\nline two']
const command = renderCommand(
[
process.execPath,
'-e',
'process.stdout.write(JSON.stringify(process.argv.slice(1)))',
...values
],
'darwin'
)
const result = await runProcess({ program: '/bin/sh', args: ['-c', command] })
expect(result).toMatchObject({ code: 0, stderr: '', timedOut: false })
expect(JSON.parse(result.stdout)).toEqual(values)
}
)
it.each([
[
'split',
['orca', 'orchestration', 'send', '--pairing-code', 'split-secret', '--subject', 'status'],
'split-secret'
],
[
'equals',
['orca', 'orchestration', 'send', '--pairing-code=equals-secret', '--subject', 'status'],
'equals-secret'
],
[
'dispatch split',
[
'orca',
'orchestration',
'send',
'--dispatch-capability',
'split-dispatch-secret',
'--subject',
'status'
],
'split-dispatch-secret'
],
[
'dispatch equals',
[
'orca',
'orchestration',
'send',
'--dispatch-capability=equals-dispatch-secret',
'--subject',
'status'
],
'equals-dispatch-secret'
]
])('blocks recovery and removes %s credentials', (_name, originalCommand, secret) => {
const result = orchestrationMutationRecoveryError(
new RuntimeClientError('runtime_timeout', 'request timed out', {
orchestrationRequestId: 'request_secret',
originalCommand
})
) as RuntimeClientError
const output = JSON.stringify({ message: result.message, data: result.data })
expect(output).not.toContain(secret)
expect(result.message).toContain('Recovery is blocked')
expect(result.data).toMatchObject({
recovery: {
orchestrationRequestId: 'request_secret',
recoveryBlocked: true
}
})
expect(result.data).not.toHaveProperty('originalCommand')
expect((result.data as { recovery: object }).recovery).not.toHaveProperty('retryCommand')
})
})