fix(orchestration): complete legacy worker cutover recovery

This commit is contained in:
OrcaWin
2026-07-29 04:39:02 -07:00
parent f58353c83d
commit efd06382ae
61 changed files with 5185 additions and 1117 deletions
@@ -71,8 +71,15 @@ describe('orchestration skill guidance', () => {
expect(migration).toContain(
'It must not spawn, write, signal, stop, switch, focus, split, or inject a terminal.'
)
expect(migration).toContain('task-list --run run_legacy_local')
expect(migration).not.toContain('task-list --run run_legacy_local')
expect(migration).toContain('run_legacy_local is an empty audit tombstone')
expect(migration).toContain('Recovered orchestration work from a contract update')
expect(migration).toContain('run-show --id <adopted_run_id>')
expect(migration).toContain('task-list --run <adopted_run_id>')
expect(migration).toContain('Legacy inspection remains available without consuming mail')
expect(migration).toContain('run-use --id <adopted_run_id> --takeover-legacy')
expect(migration).toContain('Takeover fences only the old coordinator')
expect(migration).toContain('Live legacy workers keep their original Tasks, Dispatches')
expect(migration).toContain(
'keep the original worker as the only editor until it reaches a stable handoff point'
)
+14 -1
View File
@@ -74,14 +74,27 @@ Legacy inspection remains available without consuming mail:
```bash
orca orchestration run-list --json
# run_legacy_local is an empty audit tombstone after adoption.
orca orchestration run-show --id run_legacy_local --json
orca orchestration task-list --run run_legacy_local --json
# In run-list, find the ordinary Run whose objective is:
# "Recovered orchestration work from a contract update"
orca orchestration run-show --id <adopted_run_id> --json
orca orchestration task-list --run <adopted_run_id> --json
orca orchestration inbox --full --json
orca orchestration check --terminal <legacy_handle> --peek --format --json
orca terminal read --terminal <legacy_handle> --json
orca terminal wait --terminal <legacy_handle> --for tui-idle --timeout-ms 60000 --json
```
If the original coordinator is unavailable or cannot prove its retained authority, a current coordinator may explicitly take over the adopted Run:
```bash
orca orchestration run-use --id <adopted_run_id> --takeover-legacy --json
orca orchestration check --run <adopted_run_id> --json
```
Takeover fences only the old coordinator, binds the current one, and moves pending worker mail into current Run Delivery. Live legacy workers keep their original Tasks, Dispatches, processes, filesystems, and old prompt commands; their later questions, escalations, and completion reports route to the current coordinator. Do not use takeover while the original coordinator is still actively coordinating, because its later lifecycle mutations are rejected.
Do not launch a replacement editor merely because the desktop app or runtime was updated. If adoption cannot prove continuing authority, keep the original worker as the only editor until it reaches a stable handoff point, then use a new current Dispatch in a conflict-free placement for any remaining work.
## Ownership
File diff suppressed because one or more lines are too long
@@ -71,4 +71,24 @@ describe('orchestration check identity', () => {
})
)
})
it('preserves the pinned legacy --inject check signature', async () => {
process.env.ORCA_TERMINAL_HANDLE = 'term_legacy_worker'
await invokeCheck(
new Map<string, string | boolean>([
['unread', true],
['inject', true]
])
)
expect(callMock).toHaveBeenCalledWith(
'orchestration.check',
expect.objectContaining({
terminal: 'term_legacy_worker',
unread: true,
inject: true
})
)
})
})
@@ -179,6 +179,7 @@ describe('legacy orchestration CLI inspection', () => {
id: 'msg_current',
run_id: 'run_current',
from_handle: 'term_current',
to_handle: 'run:run_current',
subject: 'current question',
type: 'question',
body: 'May I continue?',
@@ -217,7 +218,8 @@ describe('legacy orchestration CLI inspection', () => {
expect(response.result.formatted).not.toContain('RUNTIME_SENTINEL')
expect(
response.result.formatted.split('\n').filter((line) => line.startsWith('[Reply:'))
).toEqual(['[Reply: orca orchestration reply --id msg_current --from term_coord --body "..."]'])
).toEqual(['[Reply: orca orchestration reply --id msg_current --body "..."]'])
expect(response.result.formatted).not.toContain('--from run:run_current')
})
it('preserves runtime formatting when every message belongs to a current Run', async () => {
@@ -62,6 +62,28 @@ describe('lightweight Run CLI handlers', () => {
from: 'term_coord'
})
})
it('passes explicit legacy takeover only when requested', async () => {
callMock.mockResolvedValue({
result: { run: { id: 'run_adopted', objective: 'Recovered work' } }
})
await ORCHESTRATION_HANDLERS['orchestration run-use']({
flags: new Map<string, string | boolean>([
['id', 'run_adopted'],
['from', 'term_current'],
['takeover-legacy', true]
]),
client: { call: callMock },
cwd: '/tmp/repo',
json: true
} as never)
expect(callMock).toHaveBeenCalledWith('orchestration.runUse', {
id: 'run_adopted',
from: 'term_current',
takeoverLegacy: true
})
})
})
describe('orchestration reset CLI handler', () => {
+14 -178
View File
@@ -21,10 +21,14 @@ import type {
} from '../../shared/orchestration-worker-output'
import type { NativeChatMessage } from '../../shared/native-chat-types'
import type { RuntimeTerminalRead } from '../../shared/runtime-types'
import { orchestrationMigrationData } from '../../shared/orchestration-rpc-contract'
import {
ORCHESTRATION_LEGACY_RUN_ID,
orchestrationMigrationData
} from '../../shared/orchestration-rpc-contract'
formatMessageReadOnlyTag,
formatOrchestrationCheckText,
prepareOrchestrationCheckOutput,
type LegacyCompatibilityResult,
type OrchestrationMessageSummary as MessageSummary
} from '../../shared/orchestration-check-output'
// Why: 15 s is well under Claude Code's ~2 min Bash-tool silence budget while keeping log volume low. See design doc §3.4.
const DEFAULT_KEEPALIVE_INTERVAL_MS = 15_000
@@ -74,93 +78,6 @@ const TASK_STATUS_VALUES = [
'blocked'
] as const
type MessageSummary = {
id: string
run_id?: string
delivery_contract?: 'legacy_direct' | 'current_delivery' | 'audit_only'
from_handle: string
to_handle?: string
subject?: string
type?: string
body?: string
payload?: string | null
priority?: string
read?: number
}
function formatMessageReadOnlyTag(
message: MessageSummary,
legacyCompatibilityActive = false
): string {
return isLegacyReadOnlyMessage(message, legacyCompatibilityActive) ? ' [legacy, read-only]' : ''
}
function isLegacyReadOnlyMessage(
message: MessageSummary,
legacyCompatibilityActive = false
): boolean {
return (
message.run_id === ORCHESTRATION_LEGACY_RUN_ID ||
(message.delivery_contract === 'legacy_direct' && !legacyCompatibilityActive) ||
message.delivery_contract === 'audit_only'
)
}
function formatMessagePriorityTag(message: MessageSummary): string {
return message.priority === 'urgent' ? ' [URGENT]' : message.priority === 'high' ? ' [HIGH]' : ''
}
function escapeTerminalControlCharacters(value: string): string {
return [...value]
.map((character) => {
const code = character.charCodeAt(0)
if (character === '\n' || (code >= 0x20 && code < 0x7f) || code > 0x9f) {
return character
}
return `\\x${code.toString(16).padStart(2, '0')}`
})
.join('')
}
function formatQuotedMessageField(label: string, value?: string): string {
return `[${label}]\n${escapeTerminalControlCharacters(value ?? '')
.split('\n')
.map((line) => ` ${line}`)
.join('\n')}`
}
function formatLegacyAwareCheckMessages(
messages: MessageSummary[],
checkedTerminal: string,
legacyCompatibilityActive = false
): string {
return messages
.map((message) => {
const legacyReadOnly = isLegacyReadOnlyMessage(message, legacyCompatibilityActive)
const lines = [
`${message.id}${formatMessageReadOnlyTag(message, legacyCompatibilityActive)}${formatMessagePriorityTag(message)} [${message.type ?? 'status'}] from=${message.from_handle}`,
formatQuotedMessageField('subject', message.subject)
]
if (legacyReadOnly) {
lines.push('[Inspection only: reply and acknowledgment are unavailable.]')
}
if (message.body) {
lines.push(formatQuotedMessageField('body', message.body))
}
if (message.payload) {
lines.push(formatQuotedMessageField('payload', message.payload))
}
if (!legacyReadOnly) {
const replyFrom = message.to_handle ?? checkedTerminal
lines.push(
`[Reply: orca orchestration reply --id ${message.id} --from ${replyFrom} --body "..."]`
)
}
return lines.join('\n')
})
.join('\n\n')
}
type LifecycleSendRejection = {
action: 'rejected'
code: string
@@ -181,23 +98,6 @@ type OrchestrationSendResult =
lifecycle?: { action: 'completed' | 'failed' }
}
type LegacyCompatibilityResult = {
recovery?: boolean
readOnly?: boolean
ackMessageIds?: string[]
answerAcknowledgement?: {
questionId: string
answerMessageId: string
}
currentDelivery?: {
runId: string
checkCommand: string
ackCommand: string
}
resumeRequired?: boolean
resumeCommand?: string
}
function resolveCompatibilityCliCommand(): 'orca' | 'orca-ide' | 'orca-dev' {
const configured = process.env.ORCA_CLI_COMMAND
if (configured === 'orca' || configured === 'orca-ide' || configured === 'orca-dev') {
@@ -206,19 +106,6 @@ function resolveCompatibilityCliCommand(): 'orca' | 'orca-ide' | 'orca-dev' {
return process.platform === 'linux' ? 'orca-ide' : 'orca'
}
function formatCurrentDeliveryNotice(
delivery: LegacyCompatibilityResult['currentDelivery']
): string {
if (!delivery) {
return ''
}
return (
`\n[CURRENT RUN MAIL WAITING]\n` +
`Read: ${delivery.checkCommand}\n` +
`Then acknowledge the delivery ID it prints: ${delivery.ackCommand}`
)
}
function resolvePackagedWindowsCompatibilityCommand(): 'orca' | 'orca-ide' | undefined {
if (process.env.ORCA_WINDOWS_PACKAGED_CLI_LAUNCHER !== '1') {
return undefined
@@ -560,7 +447,8 @@ export const ORCHESTRATION_HANDLERS: Record<string, CommandHandler> = {
run: { id: string; objective: string; consumer_generation: number }
}>(client, flags, 'orchestration.runUse', {
id: getRequiredStringFlag(flags, 'id'),
from
from,
...(flags.has('takeover-legacy') ? { takeoverLegacy: true } : {})
})
printResult(result, json, (r) => `Using Run ${r.run.id}: ${r.run.objective}`)
},
@@ -715,6 +603,7 @@ export const ORCHESTRATION_HANDLERS: Record<string, CommandHandler> = {
all: flags.has('all') ? true : undefined,
types: getOptionalStringFlag(flags, 'types'),
format: flags.has('format') ? true : undefined,
inject: flags.has('inject') ? true : undefined,
compatibilityCliCommand: resolveCompatibilityCliCommand(),
run: getOptionalStringFlag(flags, 'run'),
ack: getOptionalStringFlag(flags, 'ack'),
@@ -752,64 +641,11 @@ export const ORCHESTRATION_HANDLERS: Record<string, CommandHandler> = {
}
}
}
const compatibilityActive = Boolean(
result.result.legacyCompatibility && !result.result.legacyCompatibility.readOnly
)
if (
flags.has('format') &&
result.result.messages.some((message) =>
isLegacyReadOnlyMessage(message, compatibilityActive)
)
) {
// Why: formatted is one opaque batch with untrusted bodies, so selective banner parsing cannot safely remove legacy actions.
result = {
...result,
result: {
...result.result,
formatted: formatLegacyAwareCheckMessages(
result.result.messages,
terminal,
compatibilityActive
)
}
}
result = {
...result,
result: prepareOrchestrationCheckOutput(result.result, terminal, flags.has('format'))
}
printResult(result, json, (r) => {
const legacyHeader = r.legacyCompatibility
? r.legacyCompatibility.readOnly
? '[LEGACY READ-ONLY]\n'
: r.legacyCompatibility.recovery
? '[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]\n'
: '[LEGACY COMPATIBILITY]\n'
: ''
const deliveryNotice = formatCurrentDeliveryNotice(r.legacyCompatibility?.currentDelivery)
if (r.formatted) {
return `${legacyHeader}${r.formatted}${deliveryNotice}`
}
if (r.count === 0) {
if (r.timedOut) {
return `${legacyHeader}Wait timed out; no messages were consumed.${deliveryNotice}`
}
if (r.cancelled) {
const cancelled = r.connectionLost
? 'Wait cancelled because the connection closed; no messages were consumed.'
: 'Wait cancelled; no messages were consumed.'
return `${legacyHeader}${cancelled}${deliveryNotice}`
}
return `${legacyHeader}No messages.${deliveryNotice}`
}
const rendered = r.messages
.map(
(m) =>
`${m.id}${formatMessageReadOnlyTag(
m,
Boolean(r.legacyCompatibility && !r.legacyCompatibility.readOnly)
)} [${m.type ?? 'status'}] from=${m.from_handle} "${m.subject}"`
)
.join('\n')
const output = r.deliveryId ? `Delivery ${r.deliveryId}\n${rendered}` : rendered
return `${legacyHeader}${output}${deliveryNotice}`
})
printResult(result, json, (r) => formatOrchestrationCheckText(r, terminal))
const compatibilityAck = result.result.legacyCompatibility?.ackMessageIds
if (compatibilityAck && compatibilityAck.length > 0) {
await flushStdout()
+6 -2
View File
@@ -16,8 +16,12 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
{
path: ['orchestration', 'run-use'],
summary: 'Bind this coordinator terminal to an existing Run',
usage: 'orca orchestration run-use --id <run_id> [--from <handle>] [--json]',
allowedFlags: [...GLOBAL_FLAGS, 'id', 'from', 'retry-request']
usage:
'orca orchestration run-use --id <run_id> [--from <handle>] [--takeover-legacy] [--json]',
allowedFlags: [...GLOBAL_FLAGS, 'id', 'from', 'takeover-legacy', 'retry-request'],
notes: [
'--takeover-legacy explicitly replaces an unavailable legacy coordinator while preserving live worker assignments.'
]
},
{
path: ['orchestration', 'run-current'],
-2
View File
@@ -2220,8 +2220,6 @@ void app.whenReady().then(async () => {
getAgentProviderSessionSnapshot: () => agentHookServer.getStatusSnapshot(),
getAgentProviderSessionRowsForPane: (paneKey) =>
agentHookServer.getStatusSnapshotForPane(paneKey),
getHydratedAgentHookAuthority: () => agentHookServer.getHydratedAuthorityCommitments(),
getCurrentAgentHookAuthority: () => agentHookServer.getCurrentAuthorityObservations(),
attestAgentHookCompatibilityAuthority: (candidate) =>
agentHookServer.attestCompatibilityAuthority(candidate),
canRecoverPersistentLocalPtys: () => getDaemonProvider() !== null,
+31
View File
@@ -5374,6 +5374,37 @@ describe('registerPtyHandlers', () => {
expect(listProcesses).not.toHaveBeenCalled()
})
it('scopes runtime inventories to the requested provider', async () => {
const localList = vi
.spyOn(getLocalPtyProvider(), 'listProcesses')
.mockResolvedValue([{ id: 'local-pty', title: 'Local', cwd: '/local' }])
const sshAList = vi.fn(async () => [{ id: 'ssh-a-pty' }])
const sshBList = vi.fn(async () => {
throw new Error('ssh-b unavailable')
})
registerSshPtyProvider('ssh-a', { listProcesses: sshAList } as never)
registerSshPtyProvider('ssh-b', { listProcesses: sshBList } as never)
const runtime = { setPtyController: vi.fn() }
handlers.clear()
registerPtyHandlers(mainWindow as never, runtime as never)
const controller = runtime.setPtyController.mock.calls[0]?.[0] as {
listProcesses(connectionId?: string | null): Promise<{ id: string }[]>
}
await expect(controller.listProcesses(null)).resolves.toEqual([
{ id: 'local-pty', title: 'Local', cwd: '/local' }
])
expect(localList).toHaveBeenCalledOnce()
expect(sshAList).not.toHaveBeenCalled()
expect(sshBList).not.toHaveBeenCalled()
await expect(controller.listProcesses('ssh-a')).resolves.toEqual([{ id: 'ssh-a-pty' }])
expect(sshAList).toHaveBeenCalledOnce()
expect(sshBList).not.toHaveBeenCalled()
await expect(controller.listProcesses()).rejects.toThrow('ssh-b unavailable')
})
it('returns unavailable runtime confirmation for unsupported or missing providers', async () => {
registerSshPtyProvider('ssh-1', {} as never)
setPtyOwnership('unsupported-pty', 'ssh-1')
+7 -1
View File
@@ -4253,7 +4253,13 @@ export function registerPtyHandlers(
return null
}
},
listProcesses: async () => {
listProcesses: async (connectionId) => {
if (connectionId === null) {
return localProvider.listProcesses()
}
if (connectionId !== undefined) {
return getProvider(connectionId).listProcesses()
}
const providerSessions = await Promise.all([
localProvider.listProcesses(),
...Array.from(sshProviders.values(), (provider) => provider.listProcesses())
+696 -61
View File
@@ -16337,6 +16337,17 @@ describe('OrcaRuntimeService', () => {
['pty-shell', 'inc-shell', 'term_shell', 'Shell']
] as const
const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow: vi.fn() } as never)
const listProcesses = vi.fn(async () =>
processes.map(([id, incarnationId, terminalHandle, title]) => ({
id,
incarnationId,
terminalHandle,
title,
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID,
wslDistro: null
}))
)
runtime.setPtyController({
write: (ptyId, data) => {
writes.push([ptyId, data])
@@ -16345,16 +16356,7 @@ describe('OrcaRuntimeService', () => {
resize,
kill: () => true,
getForegroundProcess: async () => null,
listProcesses: async () =>
processes.map(([id, incarnationId, terminalHandle, title]) => ({
id,
incarnationId,
terminalHandle,
title,
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID,
wslDistro: null
}))
listProcesses
})
const before = await runtime.listTerminals(`id:${TEST_WORKTREE_ID}`)
expect(before.terminals.map((terminal) => terminal.tabId)).toEqual(
@@ -16442,6 +16444,14 @@ describe('OrcaRuntimeService', () => {
tail: ['legacy output']
})
const inventoryCount = listProcesses.mock.calls.length
const agentPty = (
runtime as unknown as {
ptysById: Map<string, { tabId: string | null; paneKey: string | null }>
}
).ptysById.get('pty-agent')!
agentPty.tabId = null
agentPty.paneKey = null
const secondClient = await runtime.adoptTerminalOrphans({
worktree: `id:${TEST_WORKTREE_ID}`,
expectedTopologyRevision: 0,
@@ -16454,6 +16464,22 @@ describe('OrcaRuntimeService', () => {
}))
})
expect(secondClient).toMatchObject({ adopted: false, topologyRevision: 1 })
expect(agentPty).toMatchObject({
tabId: 'tab-agent',
paneKey: makePaneKey('tab-agent', HEADLESS_LEAF_ID)
})
expect(listProcesses).toHaveBeenCalledTimes(inventoryCount + 1)
expect(listProcesses).toHaveBeenLastCalledWith(null)
expect(
(await runtime.listTerminals()).terminals.find((terminal) => terminal.ptyId === 'pty-agent')
).toMatchObject({
handle: 'term_agent',
orphaned: false,
tabId: 'tab-agent',
leafId: HEADLESS_LEAF_ID
})
expect(listProcesses).toHaveBeenCalledTimes(inventoryCount + 2)
expect(listProcesses).toHaveBeenLastCalledWith(undefined)
await expect(
runtime.adoptTerminalOrphans({
worktree: `id:${TEST_WORKTREE_ID}`,
@@ -16560,6 +16586,7 @@ describe('OrcaRuntimeService', () => {
getForegroundProcess: async () => null,
hasRendererSerializer: () => true,
serializeBuffer,
hasPty: (ptyId) => ptyId === 'pty-legacy',
listProcesses: async () => [
{
id: 'pty-legacy',
@@ -16606,7 +16633,11 @@ describe('OrcaRuntimeService', () => {
presentation: 'background',
tabId: 'legacy-worker',
leafId: workerLeafId,
focus: false
focus: false,
expectedProcessIdentity: {
terminalHandle: 'term_legacy',
incarnationId
}
})
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted')
expect(write).not.toHaveBeenCalled()
@@ -16683,6 +16714,7 @@ describe('OrcaRuntimeService', () => {
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
hasPty: (ptyId) => ptyId === 'pty-reveal-retry',
listProcesses: async () => [
{
id: 'pty-reveal-retry',
@@ -16716,6 +16748,269 @@ describe('OrcaRuntimeService', () => {
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted')
})
function makePostRevealWorkerRecoveryHarness(hasPty: (ptyId: string) => boolean | null): {
runtime: OrcaRuntimeService
getSession: () => WorkspaceSessionState
workerPaneKey: string
ptyId: string
incarnationId: string
terminalHandle: string
revealTerminalSession: ReturnType<typeof vi.fn>
resolveLegacyWorkerTerminalRecovery: ReturnType<typeof vi.fn>
} {
const workerPaneKey = `legacy-post-reveal:${HEADLESS_LEAF_ID}`
const ptyId = 'pty-post-reveal'
const incarnationId = '45454545-4545-4545-8545-454545454545'
const terminalHandle = 'term_post_reveal'
const session: WorkspaceSessionState = {
...getDefaultWorkspaceSession(),
tabsByWorktree: { [TEST_WORKTREE_ID]: [] },
sleepingAgentSessionsByPaneKey: {
[workerPaneKey]: {
paneKey: workerPaneKey,
tabId: 'legacy-post-reveal',
worktreeId: TEST_WORKTREE_ID,
agent: 'codex',
providerSession: { key: 'session_id', id: 'legacy-post-reveal-session' },
prompt: 'continue',
state: 'working',
capturedAt: 1,
updatedAt: 1,
origin: 'live'
}
}
}
const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session)
const runtime = new OrcaRuntimeService(
{ ...runtimeStore, flushOrThrow: vi.fn() } as never,
undefined,
{ canRecoverPersistentLocalPtys: () => true }
)
runtime.setOrchestrationDb({
listLegacyWorkerTerminalRecoveryRows: () => [
{
dispatch_id: 'dispatch-post-reveal',
task_id: 'task-post-reveal',
dispatch_status: 'completed',
contract_version: 0,
assignee_handle: 'term_post_reveal',
assignee_pane_key: workerPaneKey,
process_incarnation: `${ptyId}:${incarnationId}`,
worker_state: 'ready',
worktree_id: TEST_WORKTREE_ID,
agent_terminal_handle: 'term_post_reveal'
}
]
} as unknown as OrchestrationDb)
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
hasPty,
listProcesses: async () => [
{
id: ptyId,
incarnationId,
terminalHandle: 'term_post_reveal',
title: 'Post-reveal worker',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID,
wslDistro: null
}
]
})
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-post-reveal' })
const resolveLegacyWorkerTerminalRecovery = vi.fn()
runtime.setNotifier({
revealTerminalSession,
resolveLegacyWorkerTerminalRecovery
} as never)
return {
runtime,
getSession,
workerPaneKey,
ptyId,
incarnationId,
terminalHandle,
revealTerminalSession,
resolveLegacyWorkerTerminalRecovery
}
}
it('keeps a revealed worker fenced until its exact renderer graph is published', async () => {
vi.useFakeTimers()
try {
const harness = makePostRevealWorkerRecoveryHarness(() => true)
const identity = {
worktreeId: TEST_WORKTREE_ID,
tabId: 'legacy-post-reveal',
leafId: HEADLESS_LEAF_ID,
ptyId: harness.ptyId
}
harness.revealTerminalSession.mockResolvedValue({
tabId: identity.tabId,
identity
})
await expect(
harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true })
).resolves.toMatchObject({
adoptedDispatchIds: [],
exitedDispatchIds: [],
deferredDispatchIds: ['dispatch-post-reveal']
})
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
expect(
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
).toBeDefined()
expect(harness.resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled()
harness.runtime.attachWindow(1)
harness.runtime.syncWindowGraph(1, {
tabs: [
{
tabId: identity.tabId,
worktreeId: identity.worktreeId,
title: 'Post-reveal worker',
activeLeafId: identity.leafId,
layout: null
}
],
leaves: [
{
tabId: identity.tabId,
worktreeId: identity.worktreeId,
leafId: identity.leafId,
paneRuntimeId: 1,
ptyId: identity.ptyId
}
]
})
await vi.advanceTimersByTimeAsync(1_000)
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
expect(
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
).toBeUndefined()
expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(
harness.workerPaneKey,
'adopted'
)
expect(
(await harness.runtime.listTerminals()).terminals.filter(
(terminal) => terminal.ptyId === identity.ptyId
)
).toEqual([
expect.objectContaining({
handle: harness.terminalHandle,
incarnationId: harness.incarnationId,
orphaned: false,
worktreeId: identity.worktreeId,
tabId: identity.tabId,
leafId: identity.leafId
})
])
} finally {
vi.useRealTimers()
}
})
it('retires the exact worker when it exits after renderer reveal', async () => {
const hasPty = vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(false)
const harness = makePostRevealWorkerRecoveryHarness(hasPty)
await expect(
harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true })
).resolves.toMatchObject({
adoptedDispatchIds: [],
exitedDispatchIds: ['dispatch-post-reveal'],
deferredDispatchIds: []
})
expect(hasPty).toHaveBeenCalledTimes(2)
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([])
expect(
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
).toBeUndefined()
expect(
(
harness.runtime as unknown as {
ptysById: Map<string, { connected: boolean; incarnationId?: string }>
}
).ptysById.get(harness.ptyId)
).toMatchObject({ connected: false, incarnationId: '45454545-4545-4545-8545-454545454545' })
expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(
harness.workerPaneKey,
'exited'
)
})
it.each([
{
name: 'null',
makeHasPty: () =>
vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(null).mockReturnValue(true)
},
{
name: 'throw',
makeHasPty: () =>
vi
.fn()
.mockReturnValueOnce(true)
.mockImplementationOnce(() => {
throw new Error('provider unavailable')
})
.mockReturnValue(true)
}
])(
'retries post-reveal $name liveness without duplicating the worker surface',
async ({ makeHasPty }) => {
vi.useFakeTimers()
try {
const hasPty = makeHasPty()
const harness = makePostRevealWorkerRecoveryHarness(hasPty)
await expect(
harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true })
).resolves.toMatchObject({
adoptedDispatchIds: [],
exitedDispatchIds: [],
deferredDispatchIds: ['dispatch-post-reveal']
})
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([
expect.objectContaining({ id: 'legacy-post-reveal', ptyId: harness.ptyId })
])
expect(
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
?.automaticResumeBlockedBy
).toBe('legacy-orchestration-worker')
await vi.advanceTimersByTimeAsync(1_000)
expect(hasPty).toHaveBeenCalledTimes(4)
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([
expect.objectContaining({ id: 'legacy-post-reveal', ptyId: harness.ptyId })
])
expect(
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
).toBeUndefined()
expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(
harness.workerPaneKey,
'adopted'
)
await vi.advanceTimersByTimeAsync(30_000)
expect(hasPty).toHaveBeenCalledTimes(4)
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
} finally {
vi.useRealTimers()
}
}
)
it('keeps the legacy worker resume fence in memory when persistence fails', async () => {
const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}`
const incarnationId = '99999999-9999-4999-8999-999999999999'
@@ -16768,6 +17063,7 @@ describe('OrcaRuntimeService', () => {
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
hasPty: (ptyId) => ptyId === 'pty-persistence-failure',
listProcesses: async () => [
{
id: 'pty-persistence-failure',
@@ -16804,8 +17100,9 @@ describe('OrcaRuntimeService', () => {
warn.mockRestore()
})
it('unblocks provider resume only after authoritative inventory proves the legacy PTY exited', async () => {
it('retires provider resume only after authoritative inventory proves the legacy PTY exited', async () => {
const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}`
const secondWorkerPaneKey = `legacy-worker-two:${HEADLESS_SECOND_LEAF_ID}`
const incarnationId = '33333333-3333-4333-8333-333333333333'
const session: WorkspaceSessionState = {
...getDefaultWorkspaceSession(),
@@ -16822,6 +17119,18 @@ describe('OrcaRuntimeService', () => {
capturedAt: 1,
updatedAt: 1,
origin: 'live'
},
[secondWorkerPaneKey]: {
paneKey: secondWorkerPaneKey,
tabId: 'legacy-worker-two',
worktreeId: TEST_WORKTREE_ID,
agent: 'codex',
providerSession: { key: 'session_id', id: 'legacy-codex-session-two' },
prompt: 'continue',
state: 'working',
capturedAt: 1,
updatedAt: 1,
origin: 'live'
}
}
}
@@ -16844,17 +17153,50 @@ describe('OrcaRuntimeService', () => {
worker_state: 'ready',
worktree_id: TEST_WORKTREE_ID,
agent_terminal_handle: 'term_exited'
},
{
dispatch_id: 'dispatch-exited-two',
task_id: 'task-exited-two',
dispatch_status: 'completed',
contract_version: 0,
assignee_handle: 'term_exited_two',
assignee_pane_key: secondWorkerPaneKey,
process_incarnation: `pty-exited-two:${incarnationId}`,
worker_state: 'ready',
worktree_id: TEST_WORKTREE_ID,
agent_terminal_handle: 'term_exited_two'
}
]
} as unknown as OrchestrationDb)
const listProcesses = vi.fn(async (connectionId?: string | null) => {
if (connectionId !== null) {
throw new Error('unrelated SSH inventory must not run')
}
return [
{
id: 'pty-exited-two',
incarnationId,
terminalHandle: 'term_exited_two',
title: 'Exited worker',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID,
wslDistro: null
}
]
})
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
listProcesses: async () => []
hasPty: () => false,
listProcesses
})
const revealTerminalSession = vi.fn()
const resolveLegacyWorkerTerminalRecovery = vi.fn()
runtime.setNotifier({ resolveLegacyWorkerTerminalRecovery } as never)
runtime.setNotifier({
revealTerminalSession,
resolveLegacyWorkerTerminalRecovery
} as never)
runtime.prepareLegacyWorkerTerminalRecovery()
expect(
@@ -16863,16 +17205,27 @@ describe('OrcaRuntimeService', () => {
await expect(runtime.reconcileLegacyWorkerTerminals()).resolves.toMatchObject({
adoptedDispatchIds: [],
exitedDispatchIds: ['dispatch-exited'],
exitedDispatchIds: ['dispatch-exited', 'dispatch-exited-two'],
deferredDispatchIds: []
})
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
expect(getSession().sleepingAgentSessionsByPaneKey?.[secondWorkerPaneKey]).toBeUndefined()
expect(listProcesses).toHaveBeenCalledOnce()
expect(listProcesses).toHaveBeenCalledWith(null)
expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([])
expect(revealTerminalSession).not.toHaveBeenCalled()
expect(
getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy
).toBeUndefined()
(
runtime as unknown as {
ptysById: Map<string, { connected: boolean }>
}
).ptysById.get('pty-exited-two')
).toMatchObject({ connected: false })
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'exited')
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(secondWorkerPaneKey, 'exited')
})
it('keeps provider resume fenced when authoritative PTY inventory is unavailable', async () => {
it('retries inventory and unknown liveness without revealing a ghost worker', async () => {
const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}`
const incarnationId = '55555555-5555-4555-8555-555555555555'
const session: WorkspaceSessionState = {
@@ -16915,23 +17268,263 @@ describe('OrcaRuntimeService', () => {
}
]
} as unknown as OrchestrationDb)
const listProcesses = vi
.fn()
.mockRejectedValueOnce(new Error('local provider unavailable'))
.mockResolvedValue([
{
id: 'pty-inventory-unavailable',
incarnationId,
terminalHandle: 'term_inventory_unavailable',
title: 'Recovered legacy worker',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID,
wslDistro: null
}
])
const hasPty = vi.fn().mockReturnValueOnce(null).mockReturnValue(true)
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null
getForegroundProcess: async () => null,
hasPty,
listProcesses
})
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-worker' })
const resolveLegacyWorkerTerminalRecovery = vi.fn()
runtime.setNotifier({ resolveLegacyWorkerTerminalRecovery } as never)
runtime.setNotifier({
revealTerminalSession,
resolveLegacyWorkerTerminalRecovery
} as never)
vi.useFakeTimers()
try {
await expect(
runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true })
).resolves.toMatchObject({
adoptedDispatchIds: [],
exitedDispatchIds: [],
deferredDispatchIds: ['dispatch-inventory-unavailable']
})
expect(
getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy
).toBe('legacy-orchestration-worker')
expect(resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled()
expect(listProcesses).toHaveBeenCalledOnce()
expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([])
expect(revealTerminalSession).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(1_000)
expect(listProcesses).toHaveBeenCalledTimes(2)
expect(listProcesses.mock.calls).toEqual([[null], [null]])
expect(hasPty).toHaveBeenCalledOnce()
expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([])
expect(revealTerminalSession).not.toHaveBeenCalled()
expect(
getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy
).toBe('legacy-orchestration-worker')
await vi.advanceTimersByTimeAsync(2_000)
expect(listProcesses).toHaveBeenCalledTimes(3)
expect(hasPty).toHaveBeenCalledTimes(3)
expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([
expect.objectContaining({ id: 'legacy-worker', ptyId: 'pty-inventory-unavailable' })
])
expect(revealTerminalSession).toHaveBeenCalledOnce()
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted')
await vi.advanceTimersByTimeAsync(30_000)
expect(listProcesses).toHaveBeenCalledTimes(3)
expect(revealTerminalSession).toHaveBeenCalledOnce()
} finally {
vi.useRealTimers()
}
})
it('cancels a coalesced SSH worker recovery retry when its provider disconnects', async () => {
vi.useFakeTimers()
try {
const runtime = new OrcaRuntimeService(store)
const reconcile = vi.spyOn(runtime, 'reconcileLegacyWorkerTerminals').mockResolvedValue({
blockedPaneCount: 1,
adoptedDispatchIds: [],
exitedDispatchIds: [],
deferredDispatchIds: []
})
const retryInternals = runtime as unknown as {
updateLegacyWorkerTerminalRecoveryRetry: (
plan: {
candidates: { dispatchId: string; ptyId: string }[]
},
deferredDispatchIds: ReadonlySet<string>,
options: { connectionId?: string; materializeRenderer?: boolean }
) => void
}
const plan = {
candidates: [
{
dispatchId: 'dispatch-ssh-retry',
ptyId: 'ssh:ssh-retry@@pty-worker'
}
]
}
const deferred = new Set(['dispatch-ssh-retry'])
retryInternals.updateLegacyWorkerTerminalRecoveryRetry(plan, deferred, {
connectionId: 'ssh-retry',
materializeRenderer: true
})
retryInternals.updateLegacyWorkerTerminalRecoveryRetry(plan, deferred, {
connectionId: 'ssh-retry',
materializeRenderer: true
})
runtime.notifySshStateChanged('ssh-retry', {
targetId: 'ssh-retry',
status: 'disconnected',
error: null,
reconnectAttempt: 0
})
await vi.advanceTimersByTimeAsync(30_000)
expect(reconcile).not.toHaveBeenCalled()
} finally {
vi.useRealTimers()
}
})
it('keeps live workers fenced without exact controller identity evidence', async () => {
const incarnationId = '56565656-5656-4656-8656-565656565656'
const cases = [
{
name: 'missing',
leafId: '11111111-1111-4111-8111-111111111111',
terminalHandle: 'term_missing'
},
{
name: 'ambiguous',
leafId: '22222222-2222-4222-8222-222222222222',
terminalHandle: 'term_ambiguous'
},
{
name: 'wrong-handle',
leafId: '33333333-3333-4333-8333-333333333333',
terminalHandle: 'term_wrong_handle'
},
{
name: 'wrong-incarnation',
leafId: '44444444-4444-4444-8444-444444444444',
terminalHandle: 'term_wrong_incarnation'
}
] as const
const sleepingAgentSessionsByPaneKey = Object.fromEntries(
cases.map(({ name, leafId }) => {
const paneKey = `legacy-${name}:${leafId}`
return [
paneKey,
{
paneKey,
tabId: `legacy-${name}`,
worktreeId: TEST_WORKTREE_ID,
agent: 'codex',
providerSession: { key: 'session_id', id: `session-${name}` },
prompt: 'continue',
state: 'working',
capturedAt: 1,
updatedAt: 1,
origin: 'live'
}
]
})
) as WorkspaceSessionState['sleepingAgentSessionsByPaneKey']
const session: WorkspaceSessionState = {
...getDefaultWorkspaceSession(),
tabsByWorktree: { [TEST_WORKTREE_ID]: [] },
sleepingAgentSessionsByPaneKey
}
const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session)
const runtime = new OrcaRuntimeService(
{ ...runtimeStore, flushOrThrow: vi.fn() } as never,
undefined,
{ canRecoverPersistentLocalPtys: () => true }
)
runtime.setOrchestrationDb({
listLegacyWorkerTerminalRecoveryRows: () =>
cases.map(({ name, leafId, terminalHandle }) => ({
dispatch_id: `dispatch-${name}`,
task_id: `task-${name}`,
dispatch_status: 'completed',
contract_version: 0,
assignee_handle: terminalHandle,
assignee_pane_key: `legacy-${name}:${leafId}`,
process_incarnation: `pty-${name}:${incarnationId}`,
worker_state: 'ready',
worktree_id: TEST_WORKTREE_ID,
agent_terminal_handle: terminalHandle
}))
} as unknown as OrchestrationDb)
const listProcesses = vi.fn(async () => [
{
id: 'pty-missing',
title: 'Missing identity',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID
},
{
id: 'pty-ambiguous',
incarnationId,
terminalHandle: 'term_ambiguous',
title: 'Ambiguous identity',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID
},
{
id: 'pty-ambiguous-other',
incarnationId,
terminalHandle: 'term_ambiguous',
title: 'Ambiguous identity duplicate',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID
},
{
id: 'pty-wrong-handle',
incarnationId,
terminalHandle: 'term_other',
title: 'Wrong handle',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID
},
{
id: 'pty-wrong-incarnation',
incarnationId: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
terminalHandle: 'term_wrong_incarnation',
title: 'Wrong incarnation',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID
}
])
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
hasPty: (candidate) => candidate === 'pty-folder-legacy',
listProcesses
})
await expect(runtime.reconcileLegacyWorkerTerminals()).resolves.toMatchObject({
adoptedDispatchIds: [],
exitedDispatchIds: [],
deferredDispatchIds: ['dispatch-inventory-unavailable']
deferredDispatchIds: cases.map(({ name }) => `dispatch-${name}`)
})
expect(
getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy
).toBe('legacy-orchestration-worker')
expect(resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled()
expect(listProcesses).toHaveBeenCalledOnce()
expect(listProcesses).toHaveBeenCalledWith(null)
for (const { name, leafId } of cases) {
expect(
getSession().sleepingAgentSessionsByPaneKey?.[`legacy-${name}:${leafId}`]
?.automaticResumeBlockedBy
).toBe('legacy-orchestration-worker')
}
})
it('adopts an exact live legacy worker in a folder workspace', async () => {
@@ -16986,21 +17579,23 @@ describe('OrcaRuntimeService', () => {
}
]
} as unknown as OrchestrationDb)
const listProcesses = vi.fn(async () => [
{
id: 'pty-folder-legacy',
incarnationId,
terminalHandle: 'term_folder',
title: 'Folder worker',
cwd: folderPath,
worktreeId: TEST_FOLDER_WORKSPACE_KEY,
wslDistro: null
}
])
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
listProcesses: async () => [
{
id: 'pty-folder-legacy',
incarnationId,
terminalHandle: 'term_folder',
title: 'Folder worker',
cwd: folderPath,
worktreeId: TEST_FOLDER_WORKSPACE_KEY,
wslDistro: null
}
]
hasPty: (candidate) => candidate === 'pty-folder-legacy',
listProcesses
})
const revealTerminalSession = vi.fn().mockResolvedValue({
tabId: 'legacy-folder-worker'
@@ -17022,6 +17617,8 @@ describe('OrcaRuntimeService', () => {
})
)
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
expect(listProcesses).toHaveBeenCalledOnce()
expect(listProcesses).toHaveBeenCalledWith(null)
expect(revealTerminalSession).toHaveBeenCalledWith(TEST_FOLDER_WORKSPACE_KEY, {
ptyId: 'pty-folder-legacy',
title: 'Folder worker',
@@ -17029,7 +17626,11 @@ describe('OrcaRuntimeService', () => {
presentation: 'background',
tabId: 'legacy-folder-worker',
leafId: HEADLESS_LEAF_ID,
focus: false
focus: false,
expectedProcessIdentity: {
terminalHandle: 'term_folder',
incarnationId
}
})
})
@@ -17101,21 +17702,23 @@ describe('OrcaRuntimeService', () => {
}
]
} as unknown as OrchestrationDb)
const listProcesses = vi.fn(async () => [
{
id: ptyId,
incarnationId,
terminalHandle: 'term_ssh_folder',
title: 'SSH folder worker',
cwd: folderPath,
worktreeId: TEST_FOLDER_WORKSPACE_KEY,
wslDistro: null
}
])
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
listProcesses: async () => [
{
id: ptyId,
incarnationId,
terminalHandle: 'term_ssh_folder',
title: 'SSH folder worker',
cwd: folderPath,
worktreeId: TEST_FOLDER_WORKSPACE_KEY,
wslDistro: null
}
]
hasPty: (candidate) => candidate === ptyId,
listProcesses
})
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-ssh-folder-worker' })
runtime.setNotifier({ revealTerminalSession } as never)
@@ -17149,6 +17752,8 @@ describe('OrcaRuntimeService', () => {
expect(localSession.sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
expect(getWorkspaceSession).toHaveBeenCalledWith(`ssh:${connectionId}`)
expect(setWorkspaceSession).toHaveBeenCalledWith(expect.any(Object), `ssh:${connectionId}`)
expect(listProcesses).toHaveBeenCalledOnce()
expect(listProcesses).toHaveBeenCalledWith(connectionId)
expect(sshSession.tabsByWorktree[TEST_FOLDER_WORKSPACE_KEY]).toContainEqual(
expect.objectContaining({
id: 'legacy-ssh-folder-worker',
@@ -17163,7 +17768,11 @@ describe('OrcaRuntimeService', () => {
presentation: 'background',
tabId: 'legacy-ssh-folder-worker',
leafId: HEADLESS_LEAF_ID,
focus: false
focus: false,
expectedProcessIdentity: {
terminalHandle: 'term_ssh_folder',
incarnationId
}
})
})
@@ -17321,6 +17930,7 @@ describe('OrcaRuntimeService', () => {
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
hasPty: (candidate) => candidate === ptyId,
listProcesses
})
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-ssh-worker' })
@@ -17365,12 +17975,29 @@ describe('OrcaRuntimeService', () => {
exitedDispatchIds: [],
deferredDispatchIds: []
})
await expect(
runtime.adoptTerminalOrphans({
worktree: `id:${TEST_WORKTREE_ID}`,
expectedTopologyRevision: 0,
claims: [
{
terminal: 'term_ssh_legacy',
ptyId,
incarnationId,
tabId: 'legacy-ssh-worker',
leafId: HEADLESS_LEAF_ID
}
]
})
).resolves.toMatchObject({ adopted: false })
expect(listProcesses).toHaveBeenLastCalledWith(connectionId)
} finally {
unregisterSshGitProvider(connectionId)
}
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
expect(setWorkspaceSession).toHaveBeenCalledWith(expect.any(Object), `ssh:${connectionId}`)
expect(listProcesses).toHaveBeenCalledTimes(2)
expect(revealTerminalSession).toHaveBeenCalledWith(TEST_WORKTREE_ID, {
ptyId,
title: 'SSH legacy worker',
@@ -17378,7 +18005,11 @@ describe('OrcaRuntimeService', () => {
presentation: 'background',
tabId: 'legacy-ssh-worker',
leafId: HEADLESS_LEAF_ID,
focus: false
focus: false,
expectedProcessIdentity: {
terminalHandle: 'term_ssh_legacy',
incarnationId
}
})
})
@@ -17445,21 +18076,23 @@ describe('OrcaRuntimeService', () => {
}
]
} as unknown as OrchestrationDb)
const listProcesses = vi.fn(async () => [
{
id: 'pty-wsl-legacy',
incarnationId,
terminalHandle: 'term_wsl_legacy',
title: 'WSL legacy worker',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID,
wslDistro: observedDistro
}
])
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
listProcesses: async () => [
{
id: 'pty-wsl-legacy',
incarnationId,
terminalHandle: 'term_wsl_legacy',
title: 'WSL legacy worker',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID,
wslDistro: observedDistro
}
]
hasPty: (ptyId) => ptyId === 'pty-wsl-legacy',
listProcesses
})
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-wsl-worker' })
runtime.setNotifier({ revealTerminalSession } as never)
@@ -17486,6 +18119,8 @@ describe('OrcaRuntimeService', () => {
})
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
expect(revealTerminalSession).toHaveBeenCalledOnce()
expect(listProcesses).toHaveBeenCalledTimes(2)
expect(listProcesses.mock.calls).toEqual([[null], [null]])
})
it('restores orphan pane and group topology without replacing a newer host-owned tab', async () => {
+456 -176
View File
@@ -36,10 +36,7 @@ import {
type AgentStatusEntry
} from '../../shared/agent-status-types'
import { indexAgentStatusRowsByPaneKey } from '../agent-hooks/agent-status-pane-index'
import type {
AgentHookAuthorityAttestation,
AgentHookAuthorityEvidence
} from '../agent-hooks/server'
import type { AgentHookAuthorityAttestation } from '../agent-hooks/server'
import type {
AgentSessionClaimedSpawnResult,
AgentSessionExecutionClaim,
@@ -110,6 +107,7 @@ import {
createSetupCompletionScanner
} from './orchestration/setup-completion-signal'
import type { RuntimeOrchestrationEnvelope } from '../../shared/runtime-rpc-envelope'
import type { TerminalRevealIdentity } from '../../shared/terminal-reveal-identity'
import type {
OrchestrationCompatibilityEvidence,
OrchestrationCompatibilityHostStamp
@@ -1506,7 +1504,7 @@ type RuntimePtyController = {
resize?(ptyId: string, cols: number, rows: number): boolean
// Why: exact-id mobile polls should not enumerate every local and SSH PTY.
hasPty?(ptyId: string): boolean | null
listProcesses?(): Promise<PtyProcessInfo[]>
listProcesses?(connectionId?: string | null): Promise<PtyProcessInfo[]>
serializeBuffer?(
ptyId: string,
opts?: { scrollbackRows?: number; altScreenForcesZeroRows?: boolean }
@@ -1530,6 +1528,17 @@ type RuntimePtyController = {
getSize?(ptyId: string): { cols: number; rows: number } | null
}
type PtyControllerTerminalIdentity = Readonly<{
handle: string
incarnationId: string
wslDistro?: string | null
}>
type PtyControllerInventory = Readonly<{
livePtyIds: ReadonlySet<string>
terminalIdentityByPtyId: ReadonlyMap<string, PtyControllerTerminalIdentity>
}>
type WorktreeStartupDraftPaste = {
agent: TuiAgent
content: string
@@ -1645,10 +1654,14 @@ type RuntimeNotifier = {
splitDirection?: 'horizontal' | 'vertical'
splitTelemetrySource?: TerminalPaneSplitSource
focus?: boolean
expectedProcessIdentity?: {
terminalHandle: string
incarnationId: string
}
}
):
| Promise<{ tabId: string; title?: string | null }>
| { tabId: string; title?: string | null }
| Promise<{ tabId: string; title?: string | null; identity?: TerminalRevealIdentity }>
| { tabId: string; title?: string | null; identity?: TerminalRevealIdentity }
| void
resolveLegacyWorkerTerminalRecovery?(paneKey: string, resolution: 'adopted' | 'exited'): void
splitTerminal(
@@ -1733,11 +1746,6 @@ export type OrchestrationCompatibilityTerminalAuthority = {
| { kind: 'ssh'; targetId: string }
}
export type OrchestrationCompatibilityAuthorityInputs = {
hydratedCommitments: readonly AgentHookAuthorityEvidence[]
currentObservations: readonly AgentHookAuthorityEvidence[]
}
export type LegacyWorkerTerminalRecoveryResult = {
blockedPaneCount: number
adoptedDispatchIds: string[]
@@ -2615,10 +2623,6 @@ export class OrcaRuntimeService {
private handles = new Map<string, TerminalHandleRecord>()
private handleByLeafKey = new Map<string, string>()
private handleByPtyId = new Map<string, string>()
private controllerTerminalIdentityByPtyId = new Map<
string,
{ handle: string; incarnationId: string; wslDistro?: string | null }
>()
private detachedPreAllocatedLeaves = new Map<string, RuntimeLeafRecord>()
private graphSyncCallbacks: (() => void)[] = []
private waitersByHandle = new Map<string, Set<TerminalWaiter>>()
@@ -3011,12 +3015,6 @@ export class OrcaRuntimeService {
private readonly getAgentProviderSessionRowsForPaneFn:
| ((paneKey: string) => AgentStatusIpcPayload[])
| null
private readonly getHydratedAgentHookAuthorityFn:
| (() => readonly AgentHookAuthorityEvidence[])
| null
private readonly getCurrentAgentHookAuthorityFn:
| (() => readonly AgentHookAuthorityEvidence[])
| null
private readonly attestAgentHookCompatibilityAuthorityFn:
| ((candidate: {
paneKey: string
@@ -3038,7 +3036,17 @@ export class OrcaRuntimeService {
>()
private sshRelayRecoveryGenerationByTargetId = new Map<string, number>()
private legacyWorkerTerminalRecoveryQueue: Promise<void> = Promise.resolve()
private legacyWorkerTerminalRecoveryRetries = new Map<
string,
{
attempt: number
connectionId?: string
materializeRenderer: boolean
timer: ReturnType<typeof setTimeout> | null
}
>()
private legacyWorkerTerminalMaterializedPanes = new Set<string>()
private legacyWorkerTerminalIdentityReceiptPanes = new Set<string>()
private legacyWorkerRecoveredPtys = new Set<string>()
private accountServices: RuntimeAccountServices | null = null
private commitMessageAgentEnv: CommitMessageAgentEnvironmentResolvers | null = null
@@ -3074,8 +3082,6 @@ export class OrcaRuntimeService {
* only carrier of the provider session a transcript is addressed by. */
getAgentProviderSessionSnapshot?: () => AgentStatusIpcPayload[]
getAgentProviderSessionRowsForPane?: (paneKey: string) => AgentStatusIpcPayload[]
getHydratedAgentHookAuthority?: () => readonly AgentHookAuthorityEvidence[]
getCurrentAgentHookAuthority?: () => readonly AgentHookAuthorityEvidence[]
attestAgentHookCompatibilityAuthority?: (candidate: {
paneKey: string
launchTokenHash: string
@@ -3114,8 +3120,6 @@ export class OrcaRuntimeService {
this.getAgentProviderSessionSnapshotFn =
deps?.getAgentProviderSessionSnapshot ?? deps?.getAgentStatusSnapshot ?? null
this.getAgentProviderSessionRowsForPaneFn = deps?.getAgentProviderSessionRowsForPane ?? null
this.getHydratedAgentHookAuthorityFn = deps?.getHydratedAgentHookAuthority ?? null
this.getCurrentAgentHookAuthorityFn = deps?.getCurrentAgentHookAuthority ?? null
this.attestAgentHookCompatibilityAuthorityFn =
deps?.attestAgentHookCompatibilityAuthority ?? null
this.canRecoverPersistentLocalPtysFn = deps?.canRecoverPersistentLocalPtys ?? (() => true)
@@ -3659,6 +3663,70 @@ export class OrcaRuntimeService {
return result
}
private hasExactTerminalSurfaceIdentity(expected: {
worktreeId: string
tabId: string
leafId: string
ptyId: string
terminalHandle: string
incarnationId: string
}): boolean {
if (this.graphStatus !== 'ready') {
return false
}
const pty = this.ptysById.get(expected.ptyId)
if (
!pty?.connected ||
pty.incarnationId !== expected.incarnationId ||
pty.tabId !== expected.tabId ||
pty.paneKey !== makePaneKey(expected.tabId, expected.leafId) ||
!runtimeWorktreeIdsEqual(pty.worktreeId, expected.worktreeId) ||
this.handleByPtyId.get(expected.ptyId) !== expected.terminalHandle
) {
return false
}
const tab = this.tabs.get(expected.tabId)
const leaf = this.leaves.get(this.getLeafKey(expected.tabId, expected.leafId))
const ptyLeaves = this.getLeavesForPty(expected.ptyId)
return (
Boolean(tab && runtimeWorktreeIdsEqual(tab.worktreeId, expected.worktreeId)) &&
Boolean(
leaf &&
leaf.ptyId === expected.ptyId &&
runtimeWorktreeIdsEqual(leaf.worktreeId, expected.worktreeId)
) &&
ptyLeaves.length === 1 &&
ptyLeaves[0]?.tabId === expected.tabId &&
ptyLeaves[0]?.leafId === expected.leafId
)
}
private hasExactPersistedTerminalSurfaceIdentity(expected: {
worktreeId: string
tabId: string
leafId: string
ptyId: string
incarnationId: string
}): boolean {
const session = this.getWorkspaceSessionForWorktree(expected.worktreeId)
const sessionWorktreeId = session
? resolveTerminalSessionWorktreeId(session, expected.worktreeId)
: null
if (!session || !sessionWorktreeId) {
return false
}
const tab = session.tabsByWorktree[sessionWorktreeId]?.find(
(candidate) => candidate.id === expected.tabId
)
const paneKey = makePaneKey(expected.tabId, expected.leafId)
return Boolean(
tab &&
session.terminalLayoutsByTabId[expected.tabId]?.ptyIdsByLeafId?.[expected.leafId] ===
expected.ptyId &&
session.terminalPtyIncarnationsByPaneKey?.[paneKey] === expected.incarnationId
)
}
private persistLegacyWorkerTerminalRecoveryResolution(
candidate: LegacyWorkerTerminalRecoveryPlan['candidates'][number],
resolution: 'adopted' | 'exited'
@@ -3672,23 +3740,8 @@ export class OrcaRuntimeService {
if (!record || !runtimeWorktreeIdsEqual(record.worktreeId, candidate.worktreeId)) {
return true
}
if (
resolution === 'exited' &&
record.automaticResumeBlockedBy !== 'legacy-orchestration-worker'
) {
return true
}
const next = structuredClone(session)
if (resolution === 'adopted') {
delete next.sleepingAgentSessionsByPaneKey?.[candidate.paneKey]
} else {
const unblocked = { ...record }
delete unblocked.automaticResumeBlockedBy
next.sleepingAgentSessionsByPaneKey = {
...next.sleepingAgentSessionsByPaneKey,
[candidate.paneKey]: unblocked
}
}
delete next.sleepingAgentSessionsByPaneKey?.[candidate.paneKey]
try {
this.setWorkspaceSessionForWorktree(candidate.worktreeId, next)
store.flushOrThrow()
@@ -3704,6 +3757,85 @@ export class OrcaRuntimeService {
}
}
private updateLegacyWorkerTerminalRecoveryRetry(
plan: LegacyWorkerTerminalRecoveryPlan,
deferredDispatchIds: ReadonlySet<string>,
options: { connectionId?: string; materializeRenderer?: boolean }
): void {
const scopeKey = options.connectionId ? `ssh:${options.connectionId}` : 'local'
const hasDeferredWorker = plan.candidates.some((candidate) => {
const sshPty = parseAppSshPtyId(candidate.ptyId)
const inScope = options.connectionId
? sshPty?.connectionId === options.connectionId
: sshPty === null
return inScope && deferredDispatchIds.has(candidate.dispatchId)
})
if (!hasDeferredWorker) {
this.cancelLegacyWorkerTerminalRecoveryRetry(scopeKey)
return
}
const existing = this.legacyWorkerTerminalRecoveryRetries.get(scopeKey)
const retry = existing ?? {
attempt: 0,
...(options.connectionId ? { connectionId: options.connectionId } : {}),
materializeRenderer: options.materializeRenderer === true,
timer: null
}
retry.materializeRenderer ||= options.materializeRenderer === true
this.legacyWorkerTerminalRecoveryRetries.set(scopeKey, retry)
this.armLegacyWorkerTerminalRecoveryRetry(scopeKey, retry)
}
private cancelLegacyWorkerTerminalRecoveryRetry(scopeKey: string): void {
const retry = this.legacyWorkerTerminalRecoveryRetries.get(scopeKey)
if (retry?.timer) {
clearTimeout(retry.timer)
}
this.legacyWorkerTerminalRecoveryRetries.delete(scopeKey)
}
private getLegacyWorkerControllerPtyLiveness(
candidate: LegacyWorkerTerminalRecoveryPlan['candidates'][number]
): boolean | null {
try {
return this.ptyController?.hasPty?.(candidate.ptyId) ?? null
} catch {
return null
}
}
private armLegacyWorkerTerminalRecoveryRetry(
scopeKey: string,
retry: {
attempt: number
connectionId?: string
materializeRenderer: boolean
timer: ReturnType<typeof setTimeout> | null
}
): void {
if (retry.timer) {
return
}
const delayMs = Math.min(1_000 * 2 ** retry.attempt, 30_000)
retry.attempt += 1
retry.timer = setTimeout(() => {
retry.timer = null
void this.reconcileLegacyWorkerTerminals({
...(retry.connectionId ? { connectionId: retry.connectionId } : {}),
materializeRenderer: retry.materializeRenderer
}).catch((error) => {
console.warn('[orchestration] worker terminal recovery retry failed', {
scope: scopeKey,
error
})
if (this.legacyWorkerTerminalRecoveryRetries.get(scopeKey) === retry) {
this.armLegacyWorkerTerminalRecoveryRetry(scopeKey, retry)
}
})
}, delayMs)
retry.timer.unref?.()
}
private async reconcileLegacyWorkerTerminalsNow(options: {
connectionId?: string
materializeRenderer?: boolean
@@ -3712,142 +3844,251 @@ export class OrcaRuntimeService {
const adoptedDispatchIds: string[] = []
const exitedDispatchIds: string[] = []
const deferredDispatchIds = new Set(plan.ambiguousDispatchIds)
for (const candidate of plan.candidates) {
let workspace: TerminalWorkspaceLaunchScope
try {
workspace = await this.resolveTerminalWorkspaceLaunchScope(`id:${candidate.worktreeId}`)
} catch {
deferredDispatchIds.add(candidate.dispatchId)
continue
const recoveryCandidatesByProvider = new Map<
string,
{
connectionId: string | null
entries: {
candidate: (typeof plan.candidates)[number]
workspace: TerminalWorkspaceLaunchScope
resolvedWorkspace: ResolvedWorktree
}[]
}
const sshPty = parseAppSshPtyId(candidate.ptyId)
if (workspace.connectionId) {
if (
options.connectionId !== workspace.connectionId ||
sshPty?.connectionId !== workspace.connectionId
>()
for (const candidate of plan.candidates) {
try {
const workspace = await this.resolveTerminalWorkspaceLaunchScope(
`id:${candidate.worktreeId}`
)
const sshPty = parseAppSshPtyId(candidate.ptyId)
if (workspace.connectionId) {
if (
options.connectionId !== workspace.connectionId ||
sshPty?.connectionId !== workspace.connectionId
) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
} else if (
options.connectionId !== undefined ||
sshPty !== null ||
!this.canRecoverPersistentLocalPtysFn()
) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
} else if (
options.connectionId !== undefined ||
sshPty !== null ||
!this.canRecoverPersistentLocalPtysFn()
) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
const resolvedWorkspace = workspace.folderWorkspace
? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace)
: await this.resolveWorktreeSelector(`id:${workspace.id}`)
const livePtyIds = await this.refreshPtyWorktreeRecordsFromController([resolvedWorkspace])
if (!livePtyIds) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
if (!livePtyIds.has(candidate.ptyId)) {
if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) {
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited')
exitedDispatchIds.push(candidate.dispatchId)
} else {
deferredDispatchIds.add(candidate.dispatchId)
const resolvedWorkspace = workspace.folderWorkspace
? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace)
: await this.resolveWorktreeSelector(`id:${workspace.id}`)
const connectionId = workspace.connectionId ?? null
const providerKey = connectionId === null ? 'local' : `ssh:${connectionId}`
const provider = recoveryCandidatesByProvider.get(providerKey) ?? {
connectionId,
entries: []
}
continue
}
const controllerIdentity = this.controllerTerminalIdentityByPtyId.get(candidate.ptyId)
if (
controllerIdentity?.handle !== candidate.terminalHandle ||
controllerIdentity.incarnationId !== candidate.incarnationId
) {
provider.entries.push({ candidate, workspace, resolvedWorkspace })
recoveryCandidatesByProvider.set(providerKey, provider)
} catch {
deferredDispatchIds.add(candidate.dispatchId)
}
}
for (const provider of recoveryCandidatesByProvider.values()) {
const resolvedWorktrees = [
...new Map(
provider.entries.map(({ resolvedWorkspace }) => [resolvedWorkspace.id, resolvedWorkspace])
).values()
]
const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory(
resolvedWorktrees,
null,
undefined,
provider.connectionId
)
if (!inventory) {
provider.entries.forEach(({ candidate }) => deferredDispatchIds.add(candidate.dispatchId))
continue
}
const session = this.getWorkspaceSessionForWorktree(candidate.worktreeId)
const sessionWorktreeId = session
? resolveTerminalSessionWorktreeId(session, candidate.worktreeId)
: null
const activeTabId = sessionWorktreeId
? session?.activeTabIdByWorktree?.[sessionWorktreeId]
: undefined
const activeGroupId = sessionWorktreeId
? session?.activeGroupIdByWorktree?.[sessionWorktreeId]
: undefined
try {
await this.adoptTerminalOrphans({
worktree: `id:${candidate.worktreeId}`,
expectedTopologyRevision: this.getTerminalTopologyRevision(candidate.worktreeId),
...(activeTabId ? { activeTabId } : {}),
...(activeGroupId ? { activeGroupId } : {}),
claims: [
{
terminal: candidate.terminalHandle,
ptyId: candidate.ptyId,
incarnationId: candidate.incarnationId,
tabId: candidate.tabId,
leafId: candidate.leafId
}
]
})
} catch (error) {
console.warn('[orchestration] legacy worker terminal adoption deferred', {
dispatchId: candidate.dispatchId,
error
})
deferredDispatchIds.add(candidate.dispatchId)
continue
}
let rendererMaterialized =
options.materializeRenderer !== true ||
this.legacyWorkerTerminalMaterializedPanes.has(candidate.paneKey)
const pty = this.ptysById.get(candidate.ptyId)
if (
options.materializeRenderer &&
!rendererMaterialized &&
pty &&
this.notifier?.revealTerminalSession
) {
for (let attempt = 0; attempt < 2 && !rendererMaterialized; attempt += 1) {
for (const { candidate, workspace } of provider.entries) {
if (!inventory.livePtyIds.has(candidate.ptyId)) {
if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) {
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited')
exitedDispatchIds.push(candidate.dispatchId)
} else {
deferredDispatchIds.add(candidate.dispatchId)
}
continue
}
const controllerIdentity = inventory.terminalIdentityByPtyId.get(candidate.ptyId)
if (
controllerIdentity?.handle !== candidate.terminalHandle ||
controllerIdentity.incarnationId !== candidate.incarnationId
) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
const controllerLiveness = this.getLegacyWorkerControllerPtyLiveness(candidate)
if (controllerLiveness === false) {
this.onPtyExit(candidate.ptyId, 0, candidate.incarnationId)
if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) {
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited')
exitedDispatchIds.push(candidate.dispatchId)
} else {
deferredDispatchIds.add(candidate.dispatchId)
}
continue
}
if (controllerLiveness === null) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
const session = this.getWorkspaceSessionForWorktree(candidate.worktreeId)
const sessionWorktreeId = session
? resolveTerminalSessionWorktreeId(session, candidate.worktreeId)
: null
const activeTabId = sessionWorktreeId
? session?.activeTabIdByWorktree?.[sessionWorktreeId]
: undefined
const activeGroupId = sessionWorktreeId
? session?.activeGroupIdByWorktree?.[sessionWorktreeId]
: undefined
const exactSurfaceAlreadyPublished =
this.hasExactPersistedTerminalSurfaceIdentity(candidate) &&
this.hasExactTerminalSurfaceIdentity(candidate)
if (!exactSurfaceAlreadyPublished) {
try {
await this.notifier.revealTerminalSession(candidate.worktreeId, {
ptyId: candidate.ptyId,
title: getLatestPtyTitle(pty) ?? pty.controllerTitle,
activate: false,
presentation: 'background',
tabId: candidate.tabId,
leafId: candidate.leafId,
focus: false
})
rendererMaterialized = true
this.legacyWorkerTerminalMaterializedPanes.add(candidate.paneKey)
await this.adoptTerminalOrphansFromInventory(
{
worktree: `id:${candidate.worktreeId}`,
expectedTopologyRevision: this.getTerminalTopologyRevision(candidate.worktreeId),
...(activeTabId ? { activeTabId } : {}),
...(activeGroupId ? { activeGroupId } : {}),
claims: [
{
terminal: candidate.terminalHandle,
ptyId: candidate.ptyId,
incarnationId: candidate.incarnationId,
tabId: candidate.tabId,
leafId: candidate.leafId
}
]
},
workspace,
inventory
)
} catch (error) {
if (attempt === 0) {
await new Promise<void>((resolve) => setTimeout(resolve, 100))
continue
}
console.warn('[orchestration] adopted legacy worker was not revealed', {
console.warn('[orchestration] legacy worker terminal adoption deferred', {
dispatchId: candidate.dispatchId,
error
})
deferredDispatchIds.add(candidate.dispatchId)
continue
}
}
let rendererMaterialized =
options.materializeRenderer !== true ||
this.legacyWorkerTerminalMaterializedPanes.has(candidate.paneKey)
const pty = this.ptysById.get(candidate.ptyId)
if (
options.materializeRenderer &&
!rendererMaterialized &&
pty &&
this.notifier?.revealTerminalSession
) {
for (let attempt = 0; attempt < 2 && !rendererMaterialized; attempt += 1) {
try {
const reveal = await this.notifier.revealTerminalSession(candidate.worktreeId, {
ptyId: candidate.ptyId,
title: getLatestPtyTitle(pty) ?? pty.controllerTitle,
activate: false,
presentation: 'background',
tabId: candidate.tabId,
leafId: candidate.leafId,
focus: false,
expectedProcessIdentity: {
terminalHandle: candidate.terminalHandle,
incarnationId: candidate.incarnationId
}
})
if (reveal?.identity) {
const identity = reveal.identity
if (
!runtimeWorktreeIdsEqual(identity.worktreeId, candidate.worktreeId) ||
identity.tabId !== candidate.tabId ||
identity.leafId !== candidate.leafId ||
identity.ptyId !== candidate.ptyId
) {
throw new Error('terminal_reveal_identity_mismatch')
}
this.legacyWorkerTerminalIdentityReceiptPanes.add(candidate.paneKey)
}
rendererMaterialized = true
this.legacyWorkerTerminalMaterializedPanes.add(candidate.paneKey)
} catch (error) {
if (attempt === 0) {
await new Promise<void>((resolve) => setTimeout(resolve, 100))
continue
}
console.warn('[orchestration] adopted legacy worker was not revealed', {
dispatchId: candidate.dispatchId,
error
})
}
}
}
if (!rendererMaterialized) {
this.legacyWorkerTerminalMaterializedPanes.delete(candidate.paneKey)
this.legacyWorkerTerminalIdentityReceiptPanes.delete(candidate.paneKey)
deferredDispatchIds.add(candidate.dispatchId)
continue
}
if (
this.legacyWorkerTerminalIdentityReceiptPanes.has(candidate.paneKey) &&
!this.hasExactTerminalSurfaceIdentity({
worktreeId: candidate.worktreeId,
tabId: candidate.tabId,
leafId: candidate.leafId,
ptyId: candidate.ptyId,
terminalHandle: candidate.terminalHandle,
incarnationId: candidate.incarnationId
})
) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
const postRevealLiveness = this.getLegacyWorkerControllerPtyLiveness(candidate)
if (postRevealLiveness === false) {
this.legacyWorkerTerminalMaterializedPanes.delete(candidate.paneKey)
this.legacyWorkerTerminalIdentityReceiptPanes.delete(candidate.paneKey)
this.onPtyExit(candidate.ptyId, 0, candidate.incarnationId)
if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) {
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited')
exitedDispatchIds.push(candidate.dispatchId)
} else {
deferredDispatchIds.add(candidate.dispatchId)
}
continue
}
if (postRevealLiveness === null) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
if (!this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'adopted')) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
this.legacyWorkerRecoveredPtys.add(candidate.ptyId)
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'adopted')
adoptedDispatchIds.push(candidate.dispatchId)
}
if (
!rendererMaterialized ||
!this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'adopted')
) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
this.legacyWorkerRecoveredPtys.add(candidate.ptyId)
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'adopted')
adoptedDispatchIds.push(candidate.dispatchId)
}
return {
const result = {
blockedPaneCount: plan.blockedPanes.length,
adoptedDispatchIds,
exitedDispatchIds,
deferredDispatchIds: [...deferredDispatchIds]
}
this.updateLegacyWorkerTerminalRecoveryRetry(plan, deferredDispatchIds, options)
return result
}
setAutomationService(service: AutomationService): void {
@@ -4213,6 +4454,9 @@ export class OrcaRuntimeService {
notifySshStateChanged(targetId: string, state: SshConnectionState): void {
this.bumpSshRelayRecoveryGeneration(targetId)
this.invalidateSshWorktreeScanCache(targetId)
if (state.status !== 'connected') {
this.cancelLegacyWorkerTerminalRecoveryRetry(`ssh:${targetId}`)
}
this.emitClientEvent({ type: 'sshStateChanged', targetId, state: getPublicSshState(state)! })
}
@@ -10345,13 +10589,6 @@ export class OrcaRuntimeService {
return 'local'
}
getOrchestrationCompatibilityAuthorityInputs(): OrchestrationCompatibilityAuthorityInputs {
return {
hydratedCommitments: this.getHydratedAgentHookAuthorityFn?.() ?? [],
currentObservations: this.getCurrentAgentHookAuthorityFn?.() ?? []
}
}
registerOrchestrationCompatibilitySshAttachment(
targetId: string,
connectionIncarnation: string
@@ -13554,13 +13791,24 @@ export class OrcaRuntimeService {
const resolvedWorkspace = workspace.folderWorkspace
? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace)
: await this.resolveWorktreeSelector(`id:${workspace.id}`)
const livePtyIds = await this.refreshPtyWorktreeRecordsFromController(
const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory(
[resolvedWorkspace],
workspace.id
workspace.id,
undefined,
workspace.connectionId ?? null
)
if (!livePtyIds) {
if (!inventory) {
throw new Error('terminal_liveness_unavailable')
}
return this.adoptTerminalOrphansFromInventory(request, workspace, inventory)
}
private async adoptTerminalOrphansFromInventory(
request: RuntimeTerminalOrphanAdoptionRequest,
workspace: TerminalWorkspaceLaunchScope,
inventory: PtyControllerInventory
): Promise<RuntimeTerminalOrphanAdoptionResult> {
const { livePtyIds, terminalIdentityByPtyId } = inventory
const store = this.store
const session = this.getWorkspaceSessionForWorktree(workspace.id)
if (!store?.setWorkspaceSession || !store.flushOrThrow || !session) {
@@ -13593,7 +13841,7 @@ export class OrcaRuntimeService {
seenPaneKeys.add(paneKey)
const live = this.getLivePtyForHandle(claim.terminal)
const pty = live?.pty
const controllerIdentity = this.controllerTerminalIdentityByPtyId.get(claim.ptyId)
const controllerIdentity = terminalIdentityByPtyId.get(claim.ptyId)
if (
!pty ||
pty.ptyId !== claim.ptyId ||
@@ -13681,10 +13929,14 @@ export class OrcaRuntimeService {
)
})
if (isExactPersisted && sessionWorktreeId === workspace.id) {
for (const { claim, pty, paneKey } of validated) {
pty.tabId = claim.tabId
pty.paneKey = paneKey
}
return {
adopted: false,
topologyRevision: currentRevision,
snapshot: await this.listMobileSessionTabs(`id:${workspace.id}`)
snapshot: this.getTerminalOrphanAdoptionSnapshot(workspace.id)
}
}
if (currentRevision !== request.expectedTopologyRevision) {
@@ -13999,10 +14251,19 @@ export class OrcaRuntimeService {
return {
adopted: true,
topologyRevision: persisted.terminalTopologyRevisionByRepoId?.[repoId] ?? currentRevision + 1,
snapshot: await this.listMobileSessionTabs(`id:${workspace.id}`)
snapshot: this.getTerminalOrphanAdoptionSnapshot(workspace.id)
}
}
private getTerminalOrphanAdoptionSnapshot(worktreeId: string): RuntimeMobileSessionTabsResult {
this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId, {
allowAttachedWindow: true,
onlyRuntimeOwnedTerminals: true
})
this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId)
return this.getMobileSessionTabsForWorktree(worktreeId)
}
private buildTerminalVisualLayouts(
terminals: RuntimeTerminalSummary[],
worktreesById: Map<string, ResolvedWorktree>,
@@ -26302,17 +26563,34 @@ export class OrcaRuntimeService {
targetWorktreeId: string | null = null,
deadline?: number
): Promise<Set<string> | null> {
const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory(
resolvedWorktrees,
targetWorktreeId,
deadline
)
return inventory ? new Set(inventory.livePtyIds) : null
}
private async refreshPtyWorktreeRecordsWithControllerInventory(
resolvedWorktrees: ResolvedWorktree[],
targetWorktreeId: string | null = null,
deadline?: number,
connectionId?: string | null
): Promise<PtyControllerInventory | null> {
if (targetWorktreeId === FLOATING_TERMINAL_WORKTREE_ID) {
const targetedLiveness = this.refreshFloatingWorkspacePtyLiveness()
if (targetedLiveness !== null) {
return targetedLiveness
return {
livePtyIds: targetedLiveness,
terminalIdentityByPtyId: new Map()
}
}
}
if (!this.ptyController?.listProcesses) {
return null
}
const sessionsResult = await withTimeoutResult(
this.ptyController.listProcesses(),
this.ptyController.listProcesses(connectionId),
deadline === undefined
? PTY_CONTROLLER_LIST_TIMEOUT_MS
: Math.max(1, Math.min(PTY_CONTROLLER_LIST_TIMEOUT_MS, deadline - Date.now()))
@@ -26322,10 +26600,7 @@ export class OrcaRuntimeService {
return null
}
const sessions = sessionsResult.value
const controllerIdentityByPtyId = new Map<
string,
{ handle: string; incarnationId: string; wslDistro?: string | null }
>()
const controllerIdentityByPtyId = new Map<string, PtyControllerTerminalIdentity>()
const ptyIdByControllerHandle = new Map<string, string>()
const ambiguousControllerPtyIds = new Set<string>()
for (const session of sessions) {
@@ -26356,7 +26631,6 @@ export class OrcaRuntimeService {
for (const ptyId of ambiguousControllerPtyIds) {
controllerIdentityByPtyId.delete(ptyId)
}
this.controllerTerminalIdentityByPtyId = controllerIdentityByPtyId
const persistedWorktreeIdByPtyId = indexPersistedPtyWorktreeBindings(
this.store?.getWorkspaceSession?.()
)
@@ -26432,6 +26706,9 @@ export class OrcaRuntimeService {
this.refreshPtyForegroundAgent(session.id)
}
for (const pty of this.ptysById.values()) {
if (connectionId !== undefined && pty.connectionId !== connectionId) {
continue
}
if (!allLivePtyIds.has(pty.ptyId) && !this.leafExistsForPty(pty.ptyId)) {
if (this.ptyController.hasPty?.(pty.ptyId) === true) {
// Why: an SSH spawn can become addressable before an overlapping relay list includes it.
@@ -26451,7 +26728,10 @@ export class OrcaRuntimeService {
}
}
this.pruneDisconnectedPtyRecords()
return targetWorktreeId ? selectedLivePtyIds : allLivePtyIds
return {
livePtyIds: targetWorktreeId ? selectedLivePtyIds : allLivePtyIds,
terminalIdentityByPtyId: controllerIdentityByPtyId
}
}
private refreshFloatingWorkspacePtyLiveness(): Set<string> | null {
+261 -52
View File
@@ -171,10 +171,10 @@ function legacyMessageMatchesQuestion(
message: MessageRow,
question: string,
options: string[],
recipientHandle: string
recipientHandles: readonly string[]
): boolean {
if (
message.to_handle !== recipientHandle ||
!recipientHandles.includes(message.to_handle) ||
normalizeLegacyQuestionText(message.body) !== normalizeLegacyQuestionText(question)
) {
return false
@@ -1212,6 +1212,13 @@ export class OrchestrationDb {
`The ${params.role} compatibility principal is already committed to different proof.`
)
}
if (existing.status === 'revoked') {
throw new OrchestrationError(
'legacy_read_only',
`The ${params.role} compatibility principal has been revoked. No effects were applied.`,
{ effectsApplied: false }
)
}
this.db.exec('COMMIT')
return { principal: existing, duplicate: true }
}
@@ -1263,6 +1270,15 @@ export class OrchestrationDb {
.all(runId) as LegacyCompatibilityPrincipalRow[]
}
getLegacyCoordinatorPrincipal(runId: string): LegacyCompatibilityPrincipalRow | undefined {
return this.db
.prepare(
`SELECT * FROM legacy_compatibility_principals
WHERE run_id = ? AND role = 'coordinator'`
)
.get(runId) as LegacyCompatibilityPrincipalRow | undefined
}
resolveLegacyCompatibilityPrincipalByIdentity(params: {
runId: string
role: LegacyPrincipalRole
@@ -1362,6 +1378,10 @@ export class OrchestrationDb {
}
isLegacyCoordinatorHandle(runId: string, terminalHandle: string): boolean {
const principal = this.getLegacyCoordinatorPrincipal(runId)
if (principal?.terminal_handle === terminalHandle) {
return true
}
return Boolean(
this.db
.prepare(
@@ -1403,18 +1423,24 @@ export class OrchestrationDb {
if (!principal || principal.role !== 'worker' || !principal.dispatch_id) {
throw new OrchestrationError('request_mismatch', 'Legacy worker principal was not found.')
}
const runAddress = `run:${principal.run_id}`
const rows = this.db
.prepare(
`SELECT * FROM messages
WHERE run_id = ? AND delivery_contract = 'legacy_direct'
AND from_handle = ? AND to_handle = ? AND type = 'worker_done'
WHERE run_id = ?
AND (
(delivery_contract = 'legacy_direct' AND to_handle = ?) OR
(delivery_contract = 'current_delivery' AND to_handle = ?)
)
AND from_handle = ? AND type = 'worker_done'
AND subject = ? AND body = ? AND payload IS ?
ORDER BY sequence`
)
.all(
principal.run_id,
principal.terminal_handle,
params.recipientHandle,
runAddress,
principal.terminal_handle,
params.subject,
params.body,
params.payload
@@ -1444,10 +1470,11 @@ export class OrchestrationDb {
this.db
.prepare(
`SELECT 1 FROM messages
WHERE run_id = ? AND delivery_contract = 'current_delivery' AND read = 0
WHERE run_id = ? AND to_handle = ?
AND delivery_contract = 'current_delivery' AND read = 0
LIMIT 1`
)
.get(runId)
.get(runId, `run:${runId}`)
)
}
@@ -1832,7 +1859,14 @@ export class OrchestrationDb {
runId: string
coordinatorHandle: string
coordinatorPaneKey: string
allowLegacyCompatibility?: boolean
takeoverLegacy?: boolean
legacyCoordinatorAuthority?: {
runId: string
principalId: string | null
terminalHandle: string
paneKey: string
consumerGeneration: number
}
}): RunRow | undefined {
this.db.exec('BEGIN IMMEDIATE')
try {
@@ -1845,30 +1879,95 @@ export class OrchestrationDb {
run.coordinator_pane_key !== null &&
isEquivalentPaneKey(run.coordinator_pane_key, params.coordinatorPaneKey)
const adoption = this.getLegacyAdoption()
const adoptedRun = adoption?.adopted_run_id === params.runId
const legacyAuthority = params.legacyCoordinatorAuthority
const legacyPrincipalId = legacyAuthority?.principalId
const legacyPrincipal = legacyPrincipalId
? this.getLegacyCompatibilityPrincipal(legacyPrincipalId)
: undefined
const provenLegacyBinding = Boolean(
adoptedRun &&
legacyAuthority &&
legacyAuthority.principalId !== null &&
legacyAuthority.runId === params.runId &&
legacyAuthority.consumerGeneration === run.consumer_generation &&
legacyPrincipal?.run_id === params.runId &&
legacyPrincipal.role === 'coordinator' &&
legacyPrincipal.status === 'committed' &&
legacyPrincipal.terminal_handle === legacyAuthority.terminalHandle &&
isEquivalentPaneKey(legacyPrincipal.pane_key, legacyAuthority.paneKey) &&
params.coordinatorHandle === legacyAuthority.terminalHandle &&
isEquivalentPaneKey(params.coordinatorPaneKey, legacyAuthority.paneKey)
)
if (legacyAuthority && !provenLegacyBinding) {
throw new OrchestrationError(
'legacy_read_only',
'This retained legacy coordinator no longer has lifecycle authority. No effects were applied.',
{ effectsApplied: false }
)
}
const activeLegacyAssignment =
adoption?.adopted_run_id === params.runId &&
adoptedRun &&
Boolean(
this.db
.prepare(
`SELECT 1 FROM dispatch_contexts
WHERE run_id = ? AND contract_version = ?
AND status IN ('pending', 'dispatched')
UNION ALL
SELECT 1 FROM legacy_compatibility_principals
WHERE run_id = ? AND status = 'committed'
LIMIT 1`
)
.get(params.runId, LEGACY_CONTRACT_VERSION, params.runId)
.get(params.runId, LEGACY_CONTRACT_VERSION)
)
if (activeLegacyAssignment && !sameBinding && !params.allowLegacyCompatibility) {
const coordinatorPrincipal = adoptedRun
? this.getLegacyCoordinatorPrincipal(params.runId)
: undefined
const retainedCoordinatorHandle =
coordinatorPrincipal?.terminal_handle ??
run.coordinator_handle ??
this.getUniqueLegacyTaskCreatorHandle(params.runId)
const takeoverAlreadyApplied = Boolean(
params.takeoverLegacy &&
sameBinding &&
run.coordinator_handle === params.coordinatorHandle &&
coordinatorPrincipal?.status !== 'committed'
)
if (params.takeoverLegacy && !adoptedRun) {
throw new OrchestrationError(
'invalid_argument',
'Legacy takeover is only available for the automatically adopted Run.'
)
}
if (
activeLegacyAssignment &&
!sameBinding &&
!provenLegacyBinding &&
!params.takeoverLegacy
) {
throw new OrchestrationError(
'consumer_fenced',
'This adopted Run still has a live legacy principal; only its attested coordinator may rebind it.',
{ effectsApplied: false }
'This adopted Run still has live legacy work. Its attested coordinator may rebind it, or a current coordinator may explicitly use run-use --takeover-legacy.',
{
effectsApplied: false,
recoveryCommand: `orca orchestration run-use --id ${params.runId} --takeover-legacy`
}
)
}
this.unbindOtherRunsForPane(params.coordinatorPaneKey, params.runId)
if (!sameBinding || run.coordinator_handle !== params.coordinatorHandle) {
if (
(params.takeoverLegacy && !takeoverAlreadyApplied) ||
!sameBinding ||
run.coordinator_handle !== params.coordinatorHandle
) {
if (adoptedRun && (params.takeoverLegacy || !activeLegacyAssignment)) {
if (
coordinatorPrincipal?.status === 'committed' &&
(params.takeoverLegacy ||
coordinatorPrincipal.terminal_handle !== params.coordinatorHandle ||
!isEquivalentPaneKey(coordinatorPrincipal.pane_key, params.coordinatorPaneKey))
) {
this.setLegacyCompatibilityPrincipalStatus(coordinatorPrincipal.id, 'revoked')
}
}
this.db
.prepare(
`UPDATE runs
@@ -1879,6 +1978,9 @@ export class OrchestrationDb {
)
.run(params.coordinatorHandle, params.coordinatorPaneKey, params.runId)
this.fenceOutstandingDelivery(params.runId)
if (params.takeoverLegacy) {
this.promoteLegacyCoordinatorMailForTakeover(params.runId, retainedCoordinatorHandle)
}
}
this.db.exec('COMMIT')
} catch (error) {
@@ -1953,6 +2055,51 @@ export class OrchestrationDb {
.run(runId)
}
private promoteLegacyCoordinatorMailForTakeover(
runId: string,
retainedCoordinatorHandle: string | null
): void {
if (!retainedCoordinatorHandle) {
return
}
this.db
.prepare(
`UPDATE messages
SET to_handle = ?, delivery_contract = 'current_delivery',
read = 0, delivered_at = NULL
WHERE run_id = ? AND delivery_contract = 'legacy_direct'
AND to_handle = ?
AND EXISTS(
SELECT 1 FROM dispatch_contexts d
WHERE d.run_id = messages.run_id
AND d.contract_version = ?
AND (
messages.from_handle = d.assignee_handle OR
messages.from_handle = 'dispatch:' || d.id
)
)
AND (
read = 0 OR EXISTS(
SELECT 1 FROM question_threads q
WHERE q.message_id = messages.id AND q.status = 'pending'
)
)`
)
.run(`run:${runId}`, runId, retainedCoordinatorHandle, LEGACY_CONTRACT_VERSION)
}
private getUniqueLegacyTaskCreatorHandle(runId: string): string | null {
const rows = this.db
.prepare(
`SELECT DISTINCT created_by_terminal_handle AS handle
FROM tasks
WHERE run_id = ? AND created_by_terminal_handle IS NOT NULL
LIMIT 2`
)
.all(runId) as { handle: string }[]
return rows.length === 1 ? (rows[0]?.handle ?? null) : null
}
private requireCurrentConsumer(runId: string, consumerGeneration: number): RunRow {
const run = this.getRunRaw(runId)
if (!run || run.legacy === 1 || run.consumer_generation !== consumerGeneration) {
@@ -2269,12 +2416,20 @@ export class OrchestrationDb {
let message = params.message.existingId
? this.getMessageById(params.message.existingId)
: undefined
const delivery = this.resolveLegacyWorkerCoordinatorDelivery(
principal.run_id,
params.message.to
)
if (params.message.existingId) {
const matchesOriginalLegacyRoute =
message?.delivery_contract === 'legacy_direct' && message.to_handle === params.message.to
const matchesCurrentRoute =
message?.delivery_contract === delivery.contract && message.to_handle === delivery.to
if (
!message ||
message.run_id !== principal.run_id ||
message.delivery_contract !== 'legacy_direct' ||
message.to_handle !== params.message.to
message.from_handle !== principal.terminal_handle ||
(!matchesOriginalLegacyRoute && !matchesCurrentRoute)
) {
throw new OrchestrationError(
'request_mismatch',
@@ -2284,7 +2439,7 @@ export class OrchestrationDb {
} else {
message = this.insertMessage({
from: principal.terminal_handle,
to: params.message.to,
to: delivery.to,
subject: params.message.subject,
body: params.message.body,
type: params.message.type,
@@ -2292,7 +2447,7 @@ export class OrchestrationDb {
payload: params.message.payload,
senderPaneKey: principal.pane_key,
runId: principal.run_id,
deliveryContract: 'legacy_direct'
deliveryContract: delivery.contract
})
}
@@ -2300,12 +2455,24 @@ export class OrchestrationDb {
if (params.lifecycle.kind === 'heartbeat') {
this.recordHeartbeat(dispatchId, params.lifecycle.at)
} else if (params.lifecycle.kind === 'worker_report') {
settlement = this.settleWorkerReportInTransaction({
taskId: params.lifecycle.taskId,
dispatchId,
outcome: params.lifecycle.outcome,
result: params.lifecycle.result
})
const persistedOutcome =
params.message.existingId &&
dispatch.task_id === params.lifecycle.taskId &&
dispatch.status === 'completed'
? 'succeeded'
: params.message.existingId &&
dispatch.task_id === params.lifecycle.taskId &&
dispatch.status === 'failed'
? 'failed'
: undefined
settlement = persistedOutcome
? { action: 'settled', outcome: persistedOutcome, duplicate: true }
: this.settleWorkerReportInTransaction({
taskId: params.lifecycle.taskId,
dispatchId,
outcome: params.lifecycle.outcome,
result: params.lifecycle.result
})
if (settlement.action === 'rejected') {
throw new OrchestrationError(settlement.code, settlement.reason)
}
@@ -2380,24 +2547,36 @@ export class OrchestrationDb {
)
}
const existingQuestionId =
params.existingQuestionId &&
!this.db
.prepare(
`SELECT 1 FROM legacy_operation_receipts
WHERE principal_id = ? AND method = 'orchestration.ask' AND effect_id = ?
LIMIT 1`
)
.get(principal.id, params.existingQuestionId)
? params.existingQuestionId
: undefined
let question: QuestionRow
let message: MessageRow
if (params.existingQuestionId) {
const existingQuestion = this.getQuestion(params.existingQuestionId)
const existingMessage = this.getMessageById(params.existingQuestionId)
const delivery = this.resolveLegacyWorkerCoordinatorDelivery(
principal.run_id,
params.recipientHandle
)
if (existingQuestionId) {
const existingQuestion = this.getQuestion(existingQuestionId)
const existingMessage = this.getMessageById(existingQuestionId)
if (
!existingQuestion ||
!existingMessage ||
existingQuestion.run_id !== principal.run_id ||
existingQuestion.dispatch_id !== dispatchId ||
existingQuestion.status !== 'pending' ||
existingMessage.delivery_contract !== 'legacy_direct' ||
!legacyMessageMatchesQuestion(
existingMessage,
params.question,
params.options ?? [],
params.recipientHandle
)
existingMessage.delivery_contract !== delivery.contract ||
!legacyMessageMatchesQuestion(existingMessage, params.question, params.options ?? [], [
delivery.to
])
) {
throw new OrchestrationError(
'request_mismatch',
@@ -2409,7 +2588,7 @@ export class OrchestrationDb {
} else {
message = this.insertMessage({
from: principal.terminal_handle,
to: params.recipientHandle,
to: delivery.to,
subject: 'Question',
body: params.question,
type: 'question',
@@ -2421,7 +2600,7 @@ export class OrchestrationDb {
}),
senderPaneKey: principal.pane_key,
runId: principal.run_id,
deliveryContract: 'legacy_direct'
deliveryContract: delivery.contract
})
this.db
.prepare('UPDATE messages SET thread_id = ? WHERE id = ?')
@@ -2473,21 +2652,37 @@ export class OrchestrationDb {
question: QuestionRow
message: MessageRow
answerAcknowledged: boolean
claimedByOperation: boolean
}[] {
const principal = this.requireCommittedLegacyPrincipal(params.principalId, 'worker')
const runAddress = `run:${principal.run_id}`
const rows = this.db
.prepare(
`SELECT q.*, m.id AS source_message_id
`SELECT q.*, m.id AS source_message_id,
EXISTS(
SELECT 1 FROM legacy_operation_receipts lor
WHERE lor.principal_id = ? AND lor.method = 'orchestration.ask'
AND lor.effect_id = q.message_id
) AS claimed_by_operation
FROM question_threads q
INNER JOIN messages m ON m.id = q.message_id
WHERE q.run_id = ? AND q.dispatch_id = ?
AND m.delivery_contract = 'legacy_direct'
AND m.to_handle = ?
AND (
(m.delivery_contract = 'legacy_direct' AND m.to_handle = ?) OR
(m.delivery_contract = 'current_delivery' AND m.to_handle = ?)
)
ORDER BY m.sequence
LIMIT 501`
)
.all(principal.run_id, principal.dispatch_id, params.recipientHandle) as (QuestionRow & {
.all(
principal.id,
principal.run_id,
principal.dispatch_id,
params.recipientHandle,
runAddress
) as (QuestionRow & {
source_message_id: string
claimed_by_operation: number
})[]
if (rows.length > 500) {
throw new OrchestrationError(
@@ -2500,17 +2695,16 @@ export class OrchestrationDb {
const message = this.getMessageById(row.source_message_id)
return Boolean(
message &&
legacyMessageMatchesQuestion(
message,
params.question,
params.options ?? [],
params.recipientHandle
)
legacyMessageMatchesQuestion(message, params.question, params.options ?? [], [
params.recipientHandle,
runAddress
])
)
})
.map((row) => ({
question: exposeQuestionTimestamps(row),
message: this.getMessageById(row.message_id) as MessageRow,
claimedByOperation: row.claimed_by_operation === 1,
answerAcknowledged: row.answer_message_id
? Boolean(
this.db
@@ -2525,6 +2719,22 @@ export class OrchestrationDb {
}))
}
private resolveLegacyWorkerCoordinatorDelivery(
runId: string,
retainedCoordinatorHandle: string
): { to: string; contract: MessageDeliveryContract } {
const run = this.getRunRaw(runId)
const principal = this.getLegacyCoordinatorPrincipal(runId)
const takenOver =
run?.coordinator_handle !== null &&
(principal?.status === 'revoked' ||
(run?.coordinator_handle !== undefined &&
run.coordinator_handle !== retainedCoordinatorHandle))
return takenOver
? { to: `run:${runId}`, contract: 'current_delivery' }
: { to: retainedCoordinatorHandle, contract: 'legacy_direct' }
}
commitLegacyReplyOperation(params: {
principalId: string
operationKey: string
@@ -3658,11 +3868,10 @@ export class OrchestrationDb {
wd.agent_terminal_handle
FROM dispatch_contexts dc
INNER JOIN worker_dispatches wd ON wd.dispatch_id = dc.id
WHERE dc.contract_version = ?
AND wd.state IN ('starting', 'ready', 'start_unknown', 'stopping', 'stop_unknown')
WHERE wd.state IN ('starting', 'ready', 'start_unknown', 'stopping', 'stop_unknown')
ORDER BY dc.rowid`
)
.all(LEGACY_CONTRACT_VERSION) as LegacyWorkerTerminalRecoveryRow[]
.all() as LegacyWorkerTerminalRecoveryRow[]
}
getFederatedDispatch(dispatchId: string): FederatedDispatchRow | undefined {
@@ -77,6 +77,14 @@ describe('formatMessageBanner', () => {
)
})
it('lets the CLI resolve the live sender for Run and Dispatch addresses', () => {
for (const to_handle of ['run:run_test', 'dispatch:dispatch_test']) {
const banner = formatMessageBanner(makeMessage({ to_handle }))
expect(banner).toContain('[Reply: orca orchestration reply --id msg_test1 --body "..."]')
expect(banner).not.toContain(`--from ${to_handle}`)
}
})
it('marks legacy messages read-only without reply or acknowledgment affordances', () => {
const banner = formatMessageBanner(
makeMessage({ id: 'msg_legacy', run_id: 'run_legacy_local' })
+5 -4
View File
@@ -86,10 +86,11 @@ export function formatMessageBanner(
}
if (authority === 'current') {
// Why: older shells can lack Orca's terminal identity environment.
lines.push(
`[Reply: orca orchestration reply --id ${msg.id} --from ${msg.to_handle} --body "..."]`
)
const explicitFrom =
msg.to_handle.startsWith('run:') || msg.to_handle.startsWith('dispatch:')
? ''
: ` --from ${msg.to_handle}`
lines.push(`[Reply: orca orchestration reply --id ${msg.id}${explicitFrom} --body "..."]`)
}
lines.push(SEPARATOR)
@@ -1,6 +1,4 @@
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { rmSync } from 'node:fs'
import { afterEach, describe, expect, it } from 'vitest'
import Database from '../../sqlite/sync-database'
import {
@@ -9,21 +7,10 @@ import {
LEGACY_RUN_ID,
OrchestrationDb
} from './db'
type CutoverFixture = {
dbPath: string
currentRunId: string
unrelatedRunId: string
currentDispatchId: string
legacyTaskId: string
legacyDispatchId: string
legacyGateId: string
legacyMessageIds: string[]
legacyQuestionId: string
legacyDeliveryId: string
rejectionMessageId: string
lookalikeMessageId: string
}
import {
createLegacyStorageCutoverFixture,
type LegacyStorageCutoverFixture
} from './orchestration-legacy-storage-test-fixture'
describe('OrchestrationDb legacy contract storage', () => {
let db: OrchestrationDb | undefined
@@ -38,129 +25,14 @@ describe('OrchestrationDb legacy contract storage', () => {
}
})
function createCutoverFixture(): CutoverFixture {
tempDir = mkdtempSync(join(tmpdir(), 'orca-legacy-storage-'))
const dbPath = join(tempDir, 'orchestration.db')
const first = new OrchestrationDb(dbPath)
const currentRun = first.createRun({
objective: 'Current work',
coordinatorHandle: 'term_current_coord',
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111'
})
const currentTask = first.createTask({ spec: 'current', runId: currentRun.id })
const unrelatedRun = first.createRun({
objective: 'Unrelated current work',
coordinatorHandle: 'term_unrelated_coord',
coordinatorPaneKey: 'tab_unrelated:55555555-5555-4555-8555-555555555555'
})
const currentDispatch = first.createDispatchContext(
currentTask.id,
'term_current_worker',
'tab_current:22222222-2222-4222-9222-222222222222',
'current_launch_hash'
)
first.insertMessage({
runId: currentRun.id,
from: 'term_current_worker',
to: `run:${currentRun.id}`,
subject: 'current mail'
})
const legacyTask = first.createTask({
spec: 'legacy',
createdByTerminalHandle: 'term_legacy_coord'
})
first.createDispatchContext(
legacyTask.id,
'term_legacy_worker',
'tab_legacy:33333333-3333-4333-8333-333333333333'
)
const legacyGate = first.createGate({
taskId: legacyTask.id,
question: 'Retained gate?'
})
first.resolveGate(legacyGate.id, 'continue')
const retryDispatch = first.createDispatchContext(
legacyTask.id,
'term_legacy_worker',
'tab_legacy:33333333-3333-4333-8333-333333333333'
)
const legacyMessages = [
first.insertMessage({
from: 'term_legacy_coord',
to: 'term_legacy_worker',
subject: 'read worker mail'
}),
first.insertMessage({
from: 'term_legacy_worker',
to: 'term_legacy_coord',
subject: 'read coordinator mail'
}),
first.insertMessage({
from: 'term_legacy_coord',
to: 'term_legacy_worker',
subject: 'second worker page'
})
]
first.markAsRead(legacyMessages.map((message) => message.id))
const question = first.createQuestion({
runId: LEGACY_RUN_ID,
dispatchId: retryDispatch.id,
askerHandle: 'term_legacy_worker',
question: 'Retained question?'
})
const rejection = first.insertMessage({
from: 'term_legacy_worker',
to: 'term_legacy_coord',
subject: 'Rejected heartbeat',
type: 'heartbeat',
payload: JSON.stringify({ _orcaLifecycleRejection: { code: 'migration', reason: 'cutover' } })
})
const lookalike = first.insertMessage({
from: 'term_legacy_worker',
to: 'term_legacy_coord',
subject: 'Ordinary legacy mail',
payload: JSON.stringify({ '.orcaLifecycleRejection': 'not an audit marker' })
})
first.close()
const raw = new Database(dbPath)
const legacyDeliveryId = 'delivery_legacy_outstanding'
raw
.prepare(
`INSERT INTO deliveries (
id, run_id, consumer_generation, message_ids, status
) VALUES (?, ?, 0, ?, 'outstanding')`
)
.run(legacyDeliveryId, LEGACY_RUN_ID, JSON.stringify([legacyMessages[0].id]))
raw.exec(`
DROP INDEX IF EXISTS idx_messages_delivery_contract;
DROP TABLE legacy_mail_receipts;
DROP TABLE legacy_operation_receipts;
DROP TABLE legacy_compatibility_principals;
DROP TABLE legacy_adoptions;
`)
raw.pragma('user_version = 18')
raw.close()
return {
dbPath,
currentRunId: currentRun.id,
unrelatedRunId: unrelatedRun.id,
currentDispatchId: currentDispatch.id,
legacyTaskId: legacyTask.id,
legacyDispatchId: retryDispatch.id,
legacyGateId: legacyGate.id,
legacyMessageIds: legacyMessages.map((message) => message.id),
legacyQuestionId: question.message.id,
legacyDeliveryId,
rejectionMessageId: rejection.id,
lookalikeMessageId: lookalike.id
}
function createCutoverFixture(): LegacyStorageCutoverFixture {
const created = createLegacyStorageCutoverFixture()
tempDir = created.tempDir
return created.fixture
}
function openAdoptedFixture(): {
fixture: CutoverFixture
fixture: LegacyStorageCutoverFixture
adoptedRunId: string
workerPrincipalId: string
coordinatorPrincipalId: string
@@ -411,13 +283,19 @@ describe('OrchestrationDb legacy contract storage', () => {
coordinatorHandle: 'term_other',
coordinatorPaneKey: 'tab_other:55555555-5555-4555-8555-555555555555'
})
).toThrow(/only its attested coordinator/)
).toThrow(/attested coordinator may rebind/)
const bound = db!.bindRun({
runId: state.adoptedRunId,
coordinatorHandle: 'term_legacy_coord',
coordinatorPaneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
allowLegacyCompatibility: true
legacyCoordinatorAuthority: {
runId: state.adoptedRunId,
principalId: state.coordinatorPrincipalId,
terminalHandle: 'term_legacy_coord',
paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
consumerGeneration: 0
}
})
expect(bound).toMatchObject({ coordinator_handle: 'term_legacy_coord' })
expect(
@@ -429,6 +307,99 @@ describe('OrchestrationDb legacy contract storage', () => {
).toMatchObject({ coordinator_handle: 'term_legacy_coord' })
})
it.each([
['generation', { consumerGeneration: 1 }],
['principal', { principalId: 'legacy_principal_wrong' }],
['handle', { terminalHandle: 'term_wrong' }],
['pane', { paneKey: 'tab_wrong:99999999-9999-4999-8999-999999999999' }]
] as const)('rejects stale legacy coordinator %s proof inside Run binding', (_label, patch) => {
const state = openAdoptedFixture()
const before = db!.getRun(state.adoptedRunId)
expect(() =>
db!.bindRun({
runId: state.adoptedRunId,
coordinatorHandle: 'term_legacy_coord',
coordinatorPaneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
legacyCoordinatorAuthority: {
runId: state.adoptedRunId,
principalId: state.coordinatorPrincipalId,
terminalHandle: 'term_legacy_coord',
paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
consumerGeneration: 0,
...patch
}
})
).toThrow(/no longer has lifecycle authority/)
expect(db!.getRun(state.adoptedRunId)).toEqual(before)
})
it('revokes the legacy coordinator when a current coordinator takes over after settlement', () => {
const state = openAdoptedFixture()
db!.settleWorkerReport({
taskId: state.fixture.legacyTaskId,
dispatchId: state.fixture.legacyDispatchId,
outcome: 'succeeded',
result: 'done'
})
expect(
db!.bindRun({
runId: state.adoptedRunId,
coordinatorHandle: 'term_current_coord',
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111'
})
).toMatchObject({ coordinator_handle: 'term_current_coord' })
expect(db!.getLegacyCompatibilityPrincipal(state.coordinatorPrincipalId)?.status).toBe(
'revoked'
)
expect(() =>
db!.commitLegacyCompatibilityPrincipal({
runId: state.adoptedRunId,
role: 'coordinator',
hostScope: 'local:runtime_1',
terminalHandle: 'term_legacy_coord',
paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
launchTokenHash: 'coord_launch_hash',
processIncarnation: 'process_coord'
})
).toThrow(/revoked/)
})
it('promotes only mail addressed to the replaced legacy coordinator', () => {
const state = openAdoptedFixture()
const coordinatorMail = db!.insertMessage({
runId: state.adoptedRunId,
deliveryContract: 'legacy_direct',
from: 'term_legacy_worker',
to: 'term_legacy_coord',
subject: 'coordinator outcome'
})
const workerMail = db!.insertMessage({
runId: state.adoptedRunId,
deliveryContract: 'legacy_direct',
from: 'term_legacy_worker',
to: 'term_other_worker',
subject: 'worker-only guidance'
})
db!.bindRun({
runId: state.adoptedRunId,
coordinatorHandle: 'term_current_coord',
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111',
takeoverLegacy: true
})
expect(db!.getMessageById(coordinatorMail.id)).toMatchObject({
to_handle: `run:${state.adoptedRunId}`,
delivery_contract: 'current_delivery'
})
expect(db!.getMessageById(workerMail.id)).toMatchObject({
to_handle: 'term_other_worker',
delivery_contract: 'legacy_direct'
})
})
it('commits legacy messages, lifecycle effects, and invocation receipts exactly once', () => {
const state = openAdoptedFixture()
const params = {
@@ -508,6 +479,65 @@ describe('OrchestrationDb legacy contract storage', () => {
expect(db!.getLegacyCompatibilityPrincipal(state.workerPrincipalId)?.status).toBe('settled')
})
it('reconstructs a read pre-takeover completion through its original legacy route', () => {
const state = openAdoptedFixture()
const taskId = db!.getDispatchContextById(state.fixture.legacyDispatchId)!.task_id
const accepted = db!.insertMessage({
runId: state.adoptedRunId,
deliveryContract: 'legacy_direct',
from: 'term_legacy_worker',
to: 'term_legacy_coord',
subject: 'Completed',
body: 'accepted before takeover',
type: 'worker_done'
})
db!.markAsRead([accepted.id])
db!.settleWorkerReport({
taskId,
dispatchId: state.fixture.legacyDispatchId,
outcome: 'succeeded',
result: 'accepted before takeover'
})
db!.bindRun({
runId: state.adoptedRunId,
coordinatorHandle: 'term_current_coord',
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111',
takeoverLegacy: true
})
const beforeIds = db!.getInbox(100).map((message) => message.id)
const reconstructed = db!.commitLegacyLifecycleOperation({
principalId: state.workerPrincipalId,
operationKey: 'read_completion_after_takeover',
method: 'orchestration.send',
payloadHash: 'read_completion_after_takeover_payload',
message: {
existingId: accepted.id,
to: 'term_legacy_coord',
subject: 'Completed',
body: 'accepted before takeover',
type: 'worker_done'
},
lifecycle: {
kind: 'worker_report',
taskId,
outcome: 'succeeded',
result: 'accepted before takeover'
}
})
expect(reconstructed).toMatchObject({
duplicate: false,
message: {
id: accepted.id,
to_handle: 'term_legacy_coord',
delivery_contract: 'legacy_direct'
},
settlement: { action: 'settled', outcome: 'succeeded', duplicate: true }
})
expect(db!.getInbox(100).map((message) => message.id)).toEqual(beforeIds)
})
it('rejects cross-cutover completion reconstruction for another recipient', () => {
const state = openAdoptedFixture()
const taskId = db!.getDispatchContextById(state.fixture.legacyDispatchId)!.task_id
@@ -625,6 +655,38 @@ describe('OrchestrationDb legacy contract storage', () => {
})
).toHaveLength(2)
const inherited = db!.createQuestion({
runId: state.adoptedRunId,
dispatchId: state.fixture.legacyDispatchId,
askerHandle: 'term_legacy_worker',
question: 'Inherited?'
})
const sqlite = (db as unknown as { db: Database.Database }).db
sqlite
.prepare(
`UPDATE messages
SET from_handle = 'term_legacy_worker', to_handle = 'term_legacy_coord',
delivery_contract = 'legacy_direct'
WHERE id = ?`
)
.run(inherited.message.id)
const inheritedAsk = {
...ask,
operationKey: 'ask_inherited_1',
payloadHash: 'ask_inherited_payload_1',
question: 'Inherited?',
options: [],
existingQuestionId: inherited.message.id
}
const adopted = db!.commitLegacyAskOperation(inheritedAsk)
const distinct = db!.commitLegacyAskOperation({
...inheritedAsk,
operationKey: 'ask_inherited_2',
payloadHash: 'ask_inherited_payload_2'
})
expect(adopted.message.id).toBe(inherited.message.id)
expect(distinct.message.id).not.toBe(inherited.message.id)
const reply = {
principalId: state.coordinatorPrincipalId,
operationKey: 'reply_invocation_1',
@@ -0,0 +1,147 @@
import { mkdtempSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import Database from '../../sqlite/sync-database'
import { LEGACY_RUN_ID, OrchestrationDb } from './db'
export type LegacyStorageCutoverFixture = {
dbPath: string
currentRunId: string
unrelatedRunId: string
currentDispatchId: string
legacyTaskId: string
legacyDispatchId: string
legacyGateId: string
legacyMessageIds: string[]
legacyQuestionId: string
legacyDeliveryId: string
rejectionMessageId: string
lookalikeMessageId: string
}
export function createLegacyStorageCutoverFixture(): {
fixture: LegacyStorageCutoverFixture
tempDir: string
} {
const tempDir = mkdtempSync(join(tmpdir(), 'orca-legacy-storage-'))
const dbPath = join(tempDir, 'orchestration.db')
const first = new OrchestrationDb(dbPath)
const currentRun = first.createRun({
objective: 'Current work',
coordinatorHandle: 'term_current_coord',
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111'
})
const currentTask = first.createTask({ spec: 'current', runId: currentRun.id })
const unrelatedRun = first.createRun({
objective: 'Unrelated current work',
coordinatorHandle: 'term_unrelated_coord',
coordinatorPaneKey: 'tab_unrelated:55555555-5555-4555-8555-555555555555'
})
const currentDispatch = first.createDispatchContext(
currentTask.id,
'term_current_worker',
'tab_current:22222222-2222-4222-9222-222222222222',
'current_launch_hash'
)
first.insertMessage({
runId: currentRun.id,
from: 'term_current_worker',
to: `run:${currentRun.id}`,
subject: 'current mail'
})
const legacyTask = first.createTask({
spec: 'legacy',
createdByTerminalHandle: 'term_legacy_coord'
})
first.createDispatchContext(
legacyTask.id,
'term_legacy_worker',
'tab_legacy:33333333-3333-4333-8333-333333333333'
)
const legacyGate = first.createGate({
taskId: legacyTask.id,
question: 'Retained gate?'
})
first.resolveGate(legacyGate.id, 'continue')
const retryDispatch = first.createDispatchContext(
legacyTask.id,
'term_legacy_worker',
'tab_legacy:33333333-3333-4333-8333-333333333333'
)
const legacyMessages = [
first.insertMessage({
from: 'term_legacy_coord',
to: 'term_legacy_worker',
subject: 'read worker mail'
}),
first.insertMessage({
from: 'term_legacy_worker',
to: 'term_legacy_coord',
subject: 'read coordinator mail'
}),
first.insertMessage({
from: 'term_legacy_coord',
to: 'term_legacy_worker',
subject: 'second worker page'
})
]
first.markAsRead(legacyMessages.map((message) => message.id))
const question = first.createQuestion({
runId: LEGACY_RUN_ID,
dispatchId: retryDispatch.id,
askerHandle: 'term_legacy_worker',
question: 'Retained question?'
})
const rejection = first.insertMessage({
from: 'term_legacy_worker',
to: 'term_legacy_coord',
subject: 'Rejected heartbeat',
type: 'heartbeat',
payload: JSON.stringify({ _orcaLifecycleRejection: { code: 'migration', reason: 'cutover' } })
})
const lookalike = first.insertMessage({
from: 'term_legacy_worker',
to: 'term_legacy_coord',
subject: 'Ordinary legacy mail',
payload: JSON.stringify({ '.orcaLifecycleRejection': 'not an audit marker' })
})
first.close()
const raw = new Database(dbPath)
const legacyDeliveryId = 'delivery_legacy_outstanding'
raw
.prepare(
`INSERT INTO deliveries (
id, run_id, consumer_generation, message_ids, status
) VALUES (?, ?, 0, ?, 'outstanding')`
)
.run(legacyDeliveryId, LEGACY_RUN_ID, JSON.stringify([legacyMessages[0].id]))
raw.exec(`
DROP INDEX IF EXISTS idx_messages_delivery_contract;
DROP TABLE legacy_mail_receipts;
DROP TABLE legacy_operation_receipts;
DROP TABLE legacy_compatibility_principals;
DROP TABLE legacy_adoptions;
`)
raw.pragma('user_version = 18')
raw.close()
return {
tempDir,
fixture: {
dbPath,
currentRunId: currentRun.id,
unrelatedRunId: unrelatedRun.id,
currentDispatchId: currentDispatch.id,
legacyTaskId: legacyTask.id,
legacyDispatchId: retryDispatch.id,
legacyGateId: legacyGate.id,
legacyMessageIds: legacyMessages.map((message) => message.id),
legacyQuestionId: question.message.id,
legacyDeliveryId,
rejectionMessageId: rejection.id,
lookalikeMessageId: lookalike.id
}
}
}
@@ -29,7 +29,8 @@ describe('legacy worker terminal recovery planning', () => {
blockedPanes: [
{
worktreeId: 'repo::/workspace',
paneKey: `tab-worker:${LEAF_ID}`
paneKey: `tab-worker:${LEAF_ID}`,
contractVersion: 0
}
],
candidates: [
@@ -50,7 +51,8 @@ describe('legacy worker terminal recovery planning', () => {
blockedPanes: [
{
worktreeId: 'repo::/workspace',
paneKey: `tab-worker:${LEAF_ID}`
paneKey: `tab-worker:${LEAF_ID}`,
contractVersion: 0
}
],
candidates: [],
@@ -4,6 +4,7 @@ import type { LegacyWorkerTerminalRecoveryRow } from './types'
export type LegacyWorkerTerminalRecoveryCandidate = {
dispatchId: string
contractVersion: number
taskId: string
worktreeId: string
terminalHandle: string
@@ -16,7 +17,7 @@ export type LegacyWorkerTerminalRecoveryCandidate = {
}
export type LegacyWorkerTerminalRecoveryPlan = {
blockedPanes: { worktreeId: string; paneKey: string }[]
blockedPanes: { worktreeId: string; paneKey: string; contractVersion: number }[]
candidates: LegacyWorkerTerminalRecoveryCandidate[]
ambiguousDispatchIds: string[]
}
@@ -48,14 +49,21 @@ function countCandidateKeys(
export function planLegacyWorkerTerminalRecovery(
rows: readonly LegacyWorkerTerminalRecoveryRow[]
): LegacyWorkerTerminalRecoveryPlan {
const blockedPanes = new Map<string, { worktreeId: string; paneKey: string }>()
const blockedPanes = new Map<
string,
{ worktreeId: string; paneKey: string; contractVersion: number }
>()
const parsedCandidates: LegacyWorkerTerminalRecoveryCandidate[] = []
for (const row of rows) {
const worktreeId = row.worktree_id?.trim()
const paneKey = row.assignee_pane_key?.trim()
const pane = paneKey ? parsePaneKey(paneKey) : null
if (worktreeId && paneKey && pane) {
blockedPanes.set(`${worktreeId}\0${paneKey}`, { worktreeId, paneKey })
blockedPanes.set(`${worktreeId}\0${paneKey}`, {
worktreeId,
paneKey,
contractVersion: row.contract_version
})
}
const terminalHandle = row.assignee_handle?.trim()
const workerHandle = row.agent_terminal_handle?.trim()
@@ -74,6 +82,7 @@ export function planLegacyWorkerTerminalRecovery(
}
parsedCandidates.push({
dispatchId: row.dispatch_id,
contractVersion: row.contract_version,
taskId: row.task_id,
worktreeId,
terminalHandle,
@@ -1,5 +1,5 @@
import { afterEach, describe, expect, it } from 'vitest'
import { OrchestrationDb } from './db'
import { CURRENT_CONTRACT_VERSION, OrchestrationDb } from './db'
describe('OrchestrationDb worker Dispatch state', () => {
let db: OrchestrationDb | undefined
@@ -44,6 +44,14 @@ describe('OrchestrationDb worker Dispatch state', () => {
status: 'dispatched',
assignee_handle: 'term_worker'
})
expect(d.listLegacyWorkerTerminalRecoveryRows()).toEqual([
expect.objectContaining({
dispatch_id: started.dispatch.id,
contract_version: CURRENT_CONTRACT_VERSION,
worker_state: 'ready',
agent_terminal_handle: 'term_worker'
})
])
})
it('commits worker-start mutation acceptance with the starting Dispatch', () => {
+10
View File
@@ -53,6 +53,14 @@ export type RpcRequest = {
orchestrationCompatibilityEvidence?: OrchestrationCompatibilityEvidence
}
export type LegacyCoordinatorAuthorityProof = Readonly<{
runId: string
principalId: string | null
terminalHandle: string
paneKey: string
consumerGeneration: number
}>
export type RpcContext = {
runtime: OrcaRuntimeService
// Why: lets long-poll handlers release immediately on client disconnect instead of running down timeoutMs. See design doc §3.1.
@@ -82,6 +90,8 @@ export type RpcContext = {
}
// Why: only the compatibility authority router can set this trusted scope; user params cannot bypass Run consumer binding.
legacyCoordinatorRunId?: string
legacyCoordinatorAuthority?: LegacyCoordinatorAuthorityProof
revalidateLegacyCoordinator?: () => string
// Why: federation pins the authenticated saved-environment caller without exposing its token to handlers or storage.
authenticatedCallerFingerprint?: string
pairing?: PairingRpcContext
@@ -46,6 +46,10 @@ function makeRuntime(ui: PersistedUIState = getDefaultUIState()): OrcaRuntimeSer
updateUIState: vi.fn((updates: Partial<PersistedUIState>) => {
currentUI = { ...currentUI, ...updates }
return currentUI
}),
getOrchestrationDb: () => ({
getLegacyAdoption: () => undefined,
resolveLegacyWorkerCandidate: () => undefined
})
} as unknown as OrcaRuntimeService
}
@@ -0,0 +1,18 @@
import type { RuntimeCapability } from '../../../shared/protocol-version'
import type { TerminalStreamFrame } from '../../../shared/terminal-stream-protocol'
import type { PairingRpcContext } from './core'
export type RpcDispatchStreamingOptions = {
connectionId?: string
signal?: AbortSignal
clientId?: string
pairedDeviceId?: string
clientKind?: 'mobile' | 'runtime'
clientCapabilities?: readonly RuntimeCapability[]
pairing?: PairingRpcContext
sendBinary?: (bytes: Uint8Array<ArrayBufferLike>) => boolean | void
registerBinaryStreamHandler?: (
streamId: number,
handler: (frame: TerminalStreamFrame) => void
) => () => void
}
+49 -58
View File
@@ -10,13 +10,11 @@ import {
isStreamingMethod,
type RpcAnyMethod,
type RpcEnvelopeMeta,
type PairingRpcContext,
type RpcRegistry,
type RpcRequest,
type RpcResponse
} from './core'
import type { TerminalStreamFrame } from '../../../shared/terminal-stream-protocol'
import type { FeatureInteractionId } from '../../../shared/feature-interactions'
import {
computerErrorData,
@@ -29,20 +27,18 @@ import {
import { ALL_RPC_METHODS } from './methods'
import { emulatorProbe, emulatorProbeError } from '../../emulator/emulator-probe'
import type { OrcaRuntimeService } from '../orca-runtime'
import type { RuntimeCapability } from '../../../shared/protocol-version'
import {
OrchestrationMutationExecutor,
authenticatedCallerFingerprint,
getOrchestrationMutationExecutor,
type OrchestrationMutationExecutor,
type DurableMutationInvocation
} from './orchestration-mutation-executor'
import { orchestrationMigrationFence } from './orchestration-contract-fence'
import { getRuntimeFeatureInteractionId } from './runtime-feature-interaction'
import { recordRuntimeFeatureInteraction } from './runtime-feature-interaction'
import { OrchestrationLegacyCompatibility } from './orchestration-legacy-compatibility'
import type { RpcDispatchStreamingOptions } from './dispatcher-stream-options'
export type DispatcherOptions = {
runtime: OrcaRuntimeService
methods?: readonly RpcAnyMethod[]
}
export type DispatcherOptions = { runtime: OrcaRuntimeService; methods?: readonly RpcAnyMethod[] }
export class RpcDispatcher {
private readonly runtime: OrcaRuntimeService
@@ -53,7 +49,7 @@ export class RpcDispatcher {
constructor({ runtime, methods = ALL_RPC_METHODS }: DispatcherOptions) {
this.runtime = runtime
this.registry = buildRegistry(methods)
this.orchestrationMutations = new OrchestrationMutationExecutor(runtime)
this.orchestrationMutations = getOrchestrationMutationExecutor(runtime)
this.legacyOrchestration = new OrchestrationLegacyCompatibility(runtime)
}
@@ -91,8 +87,7 @@ export class RpcDispatcher {
)
}
const isEmulator = request.method.startsWith('emulator.')
if (isEmulator) {
if (request.method.startsWith('emulator.')) {
emulatorProbe(`rpc ${request.method}`, request.params)
}
try {
@@ -105,8 +100,13 @@ export class RpcDispatcher {
return successResponse(request.id, meta, compatibility.result)
}
const effectiveParams = compatibility.params ?? parsedParams.value
const invoke = (mutation?: DurableMutationInvocation) =>
method.handler(effectiveParams, {
const legacyCoordinator = this.legacyOrchestration.createCoordinatorInvocation(
request,
compatibility.legacyCoordinatorAuthority
)
const invoke = (mutation?: DurableMutationInvocation) => {
const legacyCoordinatorRunId = legacyCoordinator?.revalidate()
return method.handler(effectiveParams, {
runtime: this.runtime,
signal: options?.signal,
requestId: request.id,
@@ -114,13 +114,22 @@ export class RpcDispatcher {
authenticatedCallerFingerprint: authenticatedCallerFingerprint(request),
recordMutationReceipt: mutation?.recordReceipt,
orchestrationMutation: mutation?.identity,
legacyCoordinatorRunId: compatibility.legacyCoordinatorRunId
legacyCoordinatorRunId,
legacyCoordinatorAuthority: legacyCoordinator?.authority,
revalidateLegacyCoordinator: legacyCoordinator?.revalidate
})
}
const result = await this.orchestrationMutations.run(request, effectiveParams, invoke)
this.recordRuntimeFeatureInteraction(request.method, result, undefined, request.params)
recordRuntimeFeatureInteraction(
this.runtime,
request.method,
result,
undefined,
request.params
)
return successResponse(request.id, meta, result)
} catch (error) {
if (isEmulator) {
if (request.method.startsWith('emulator.')) {
emulatorProbeError(`rpc ${request.method}`, error, { params: request.params })
}
return this.mapError(request, meta, error)
@@ -133,20 +142,7 @@ export class RpcDispatcher {
async dispatchStreaming(
request: RpcRequest,
reply: (response: string) => void,
options?: {
connectionId?: string
signal?: AbortSignal
clientId?: string
pairedDeviceId?: string
clientKind?: 'mobile' | 'runtime'
clientCapabilities?: readonly RuntimeCapability[]
pairing?: PairingRpcContext
sendBinary?: (bytes: Uint8Array<ArrayBufferLike>) => boolean | void
registerBinaryStreamHandler?: (
streamId: number,
handler: (frame: TerminalStreamFrame) => void
) => () => void
}
options?: RpcDispatchStreamingOptions
): Promise<void> {
const meta = this.meta()
const method = this.registry.get(request.method)
@@ -183,8 +179,13 @@ export class RpcDispatcher {
return
}
const effectiveParams = compatibility.params ?? parsedParams.value
const invoke = (mutation?: DurableMutationInvocation) =>
method.handler(effectiveParams, {
const legacyCoordinator = this.legacyOrchestration.createCoordinatorInvocation(
request,
compatibility.legacyCoordinatorAuthority
)
const invoke = (mutation?: DurableMutationInvocation) => {
const legacyCoordinatorRunId = legacyCoordinator?.revalidate()
return method.handler(effectiveParams, {
runtime: this.runtime,
signal: options?.signal,
requestId: request.id,
@@ -200,10 +201,19 @@ export class RpcDispatcher {
pairing: options?.pairing,
sendBinary: options?.sendBinary,
registerBinaryStreamHandler: options?.registerBinaryStreamHandler,
legacyCoordinatorRunId: compatibility.legacyCoordinatorRunId
legacyCoordinatorRunId,
legacyCoordinatorAuthority: legacyCoordinator?.authority,
revalidateLegacyCoordinator: legacyCoordinator?.revalidate
})
}
const result = await this.orchestrationMutations.run(request, effectiveParams, invoke)
this.recordRuntimeFeatureInteraction(request.method, result, undefined, request.params)
recordRuntimeFeatureInteraction(
this.runtime,
request.method,
result,
undefined,
request.params
)
reply(JSON.stringify(successResponse(request.id, meta, result)))
} catch (error) {
reply(JSON.stringify(this.mapError(request, meta, error)))
@@ -213,7 +223,8 @@ export class RpcDispatcher {
const recordedStreamingFeatureInteractions = new Set<FeatureInteractionId>()
const emit = (result: unknown): void => {
this.recordRuntimeFeatureInteraction(
recordRuntimeFeatureInteraction(
this.runtime,
request.method,
result,
recordedStreamingFeatureInteractions,
@@ -243,7 +254,8 @@ export class RpcDispatcher {
},
emit
)
this.recordRuntimeFeatureInteraction(
recordRuntimeFeatureInteraction(
this.runtime,
request.method,
result,
recordedStreamingFeatureInteractions,
@@ -310,25 +322,4 @@ export class RpcDispatcher {
private meta(): RpcEnvelopeMeta {
return { runtimeId: this.runtime.getRuntimeId() }
}
private recordRuntimeFeatureInteraction(
method: string,
result: unknown,
alreadyRecorded?: Set<FeatureInteractionId>,
rawParams?: unknown
): void {
const id = getRuntimeFeatureInteractionId(method, result, rawParams)
if (!id) {
return
}
if (alreadyRecorded?.has(id)) {
return
}
try {
this.runtime.recordFeatureInteraction(id)
alreadyRecorded?.add(id)
} catch {
// Best-effort education state must not break runtime tools.
}
}
}
@@ -1,6 +1,6 @@
import { z } from 'zod'
import { defineMethod, type RpcMethod } from '../core'
import { OptionalString, requiredString } from '../schemas'
import { OptionalBoolean, OptionalString, requiredString } from '../schemas'
import type { OrcaRuntimeService } from '../../orca-runtime'
import { OrchestrationError } from '../../orchestration/orchestration-error'
@@ -11,7 +11,8 @@ const RunCreateParams = z.object({
const RunUseParams = z.object({
id: requiredString('Missing --id'),
from: requiredString('Missing coordinator terminal')
from: requiredString('Missing coordinator terminal'),
takeoverLegacy: OptionalBoolean
})
const RunCurrentParams = z.object({ from: requiredString('Missing coordinator terminal') })
@@ -51,7 +52,7 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [
defineMethod({
name: 'orchestration.runUse',
params: RunUseParams,
handler: (params, { runtime, legacyCoordinatorRunId }) => {
handler: (params, { runtime, legacyCoordinatorAuthority }) => {
const paneKey = requireCallerPane(runtime, params.from)
const db = runtime.getOrchestrationDb()
const priorRun = db.getCurrentRunForPane(paneKey)
@@ -59,7 +60,8 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [
runId: params.id,
coordinatorHandle: params.from,
coordinatorPaneKey: paneKey,
allowLegacyCompatibility: legacyCoordinatorRunId === params.id
takeoverLegacy: params.takeoverLegacy,
legacyCoordinatorAuthority
})
if (!run) {
throw new OrchestrationError(
+99 -6
View File
@@ -2,7 +2,12 @@
import { z } from 'zod'
import { defineMethod, type RpcMethod } from '../core'
import { OptionalFiniteNumber, OptionalString, OptionalBoolean, requiredString } from '../schemas'
import type { MessageType, MessagePriority, TaskStatus } from '../../orchestration/db'
import {
LEGACY_CONTRACT_VERSION,
type MessageType,
type MessagePriority,
type TaskStatus
} from '../../orchestration/db'
import { MESSAGE_TYPES } from '../../orchestration/types'
import { buildDispatchPreamble } from '../../orchestration/preamble'
import { formatMessageBanner } from '../../orchestration/formatter'
@@ -379,6 +384,49 @@ function resolveMessageRun(
return { run, dispatchId: dispatch?.id ?? dispatchId }
}
function legacyWorkerDeliveryContract(
runtime: OrcaRuntimeService,
runId: string | undefined,
recipient: string
): 'legacy_direct' | undefined {
if (!runId) {
return undefined
}
if (!recipient.startsWith('dispatch:')) {
return runtime
.getOrchestrationDb()
.resolveLegacyWorkerCandidate({ runId, terminalHandle: recipient })
? 'legacy_direct'
: undefined
}
const dispatch = runtime
.getOrchestrationDb()
.getDispatchContextById(recipient.slice('dispatch:'.length))
return dispatch?.run_id === runId &&
dispatch.contract_version === LEGACY_CONTRACT_VERSION &&
(dispatch.status === 'pending' || dispatch.status === 'dispatched')
? 'legacy_direct'
: undefined
}
function interruptedAcknowledgedCheck(
runId: string,
acknowledged: string,
reason: 'consumer_fenced' | 'outcome_unknown' | 'waiter_exists'
): Record<string, unknown> {
return {
runId,
deliveryId: null,
messages: [],
count: 0,
acknowledged,
timedOut: false,
cancelled: false,
connectionLost: false,
waitInterrupted: reason
}
}
function rejectFederatedExplicitTarget(params: { to?: string; run?: string }): void {
if (params.to || params.run) {
throw new OrchestrationError(
@@ -395,7 +443,10 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
defineMethod({
name: 'orchestration.send',
params: SendParams,
handler: async (params, { runtime, orchestrationCapability }) => {
handler: async (
params,
{ runtime, orchestrationCapability, legacyCoordinatorRunId, revalidateLegacyCoordinator }
) => {
const db = runtime.getOrchestrationDb()
const from = params.from ?? 'unknown'
// Why: caller-supplied pane fields are only compatibility metadata; lifecycle authority uses the runtime-observed pane plus capability.
@@ -536,6 +587,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
'Coordinator-to-worker control mail cannot report worker lifecycle.'
)
}
revalidateLegacyCoordinator?.()
const relay = db.enqueueFederationRelay({
dispatchId,
direction: 'to_worker',
@@ -562,6 +614,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
}
}
// Point-to-point — existing single-recipient behavior
revalidateLegacyCoordinator?.()
const msg = db.insertMessage({
from,
to,
@@ -572,7 +625,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
threadId: params.threadId,
payload: params.payload,
senderPaneKey,
runId: routing.run?.id
runId: routing.run?.id,
deliveryContract: legacyWorkerDeliveryContract(
runtime,
routing.run?.id ?? legacyCoordinatorRunId,
to
)
})
const dispatch = routing.dispatchId
? db.getDispatchContextById(routing.dispatchId)
@@ -629,6 +687,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
throw new Error(`No recipients resolved for group address: ${to}`)
}
revalidateLegacyCoordinator?.()
const threadId = params.threadId ?? `thread_${Date.now()}`
const messages = handles.map((handle) =>
db.insertMessage({
@@ -641,7 +700,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
threadId,
payload: params.payload,
senderPaneKey,
runId: routing.run?.id
runId: routing.run?.id,
deliveryContract: legacyWorkerDeliveryContract(
runtime,
routing.run?.id ?? legacyCoordinatorRunId,
handle
)
})
)
for (const message of messages) {
@@ -655,7 +719,16 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
defineMethod({
name: 'orchestration.check',
params: CheckParams,
handler: async (params, { runtime, signal, legacyCoordinatorRunId }) => {
handler: async (
params,
{
runtime,
signal,
legacyCoordinatorRunId,
revalidateLegacyCoordinator,
recordMutationReceipt
}
) => {
const db = runtime.getOrchestrationDb()
const handle = params.terminal ?? 'unknown'
const typeFilter = parseMessageTypes(params.types)
@@ -682,6 +755,11 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
deliveryId: params.ack
})
: undefined
if (acknowledged) {
recordMutationReceipt?.(
interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'outcome_unknown')
)
}
if (params.peek || params.all || params.unread === false) {
const history = db.getRunMailboxHistory(run.id, 100, typeFilter)
const messages =
@@ -745,14 +823,28 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
signal,
exclusive: true
})
try {
revalidateLegacyCoordinator?.()
} catch (error) {
if (!acknowledged) {
throw error
}
return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'consumer_fenced')
}
const latestRun = db.getRun(run.id)
if (!latestRun || latestRun.consumer_generation !== generation) {
if (acknowledged) {
return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'consumer_fenced')
}
throw new OrchestrationError(
'consumer_fenced',
'This mailbox consumer was replaced while waiting.'
)
}
if (waitResult === 'waiter_exists') {
if (acknowledged) {
return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'waiter_exists')
}
throw new OrchestrationError(
'waiter_exists',
`Run ${run.id} already has an active actionable waiter.`
@@ -1119,7 +1211,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
defineMethod({
name: 'orchestration.dispatch',
params: DispatchParams,
handler: async (params, { runtime, legacyCoordinatorRunId }) => {
handler: async (params, { runtime, legacyCoordinatorRunId, revalidateLegacyCoordinator }) => {
const db = runtime.getOrchestrationDb()
const task = db.getTask(params.task)
if (!task) {
@@ -1184,6 +1276,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
)
}
revalidateLegacyCoordinator?.()
const ctx = db.createDispatchContext(params.task, to, assigneePaneKey)
const dispatchCapability = params.inject
? db.mintDispatchCapability({
@@ -1,4 +1,3 @@
import { parsePaneKey } from '../../../shared/stable-pane-id'
import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence'
import type {
DispatchContextRow,
@@ -6,9 +5,14 @@ import type {
LegacyPrincipalRole
} from '../orchestration/types'
import { OrchestrationError } from '../orchestration/orchestration-error'
import type { OrcaRuntimeService } from '../orca-runtime'
import type { OrcaRuntimeService, OrchestrationCompatibilityCallerAuthority } from '../orca-runtime'
import { LEGACY_CONTRACT_VERSION } from '../orchestration/db'
import type { RpcRequest } from './core'
import {
equivalentLegacyPaneKey,
legacyCoordinatorReadOnly,
legacyReadOnlyError
} from './orchestration-legacy-process-identity'
export type LegacyPrincipalCandidate = {
runId: string
@@ -22,8 +26,9 @@ export function resolveAttestedLegacyPrincipal(args: {
runtime: OrcaRuntimeService
evidence?: OrchestrationCompatibilityEvidence
candidate: LegacyPrincipalCandidate
authority?: OrchestrationCompatibilityCallerAuthority
}): LegacyCompatibilityPrincipalRow {
const authority = verifyAttestedLegacyCandidate(args)
const authority = args.authority ?? verifyAttestedLegacyCandidate(args)
return args.runtime.getOrchestrationDb().commitLegacyCompatibilityPrincipal({
runId: args.candidate.runId,
dispatchId: args.candidate.dispatchId,
@@ -36,7 +41,7 @@ export function resolveAttestedLegacyPrincipal(args: {
}).principal
}
function verifyAttestedLegacyCandidate(args: {
export function verifyAttestedLegacyCandidate(args: {
runtime: OrcaRuntimeService
evidence?: OrchestrationCompatibilityEvidence
candidate: LegacyPrincipalCandidate
@@ -44,7 +49,7 @@ function verifyAttestedLegacyCandidate(args: {
const authority = args.runtime.verifyOrchestrationCompatibilityCaller(args.evidence)
if (
!authority ||
!equivalentPaneKey(args.candidate.paneKey, authority.paneKey) ||
!equivalentLegacyPaneKey(args.candidate.paneKey, authority.paneKey) ||
args.candidate.terminalHandle !== authority.terminalHandle
) {
throw legacyReadOnlyError()
@@ -221,37 +226,6 @@ export class LegacyCompatibilityAuthority {
})
return principal
}
resolveProvenCoordinatorScope(request: RpcRequest, requestedRunId?: string): string | undefined {
const db = this.runtime.getOrchestrationDb()
const adoption = db.getLegacyAdoption()
const requestedRun = requestedRunId ?? adoption?.adopted_run_id
if (!adoption || requestedRun !== adoption.adopted_run_id) {
return undefined
}
const candidate = db.resolveLegacyCoordinatorCandidate({
runId: adoption.adopted_run_id,
terminalHandle: request.orchestrationCompatibilityEvidence?.terminalHandle,
paneKey: request.orchestrationCompatibilityEvidence?.paneKey
})
if (!candidate) {
if (request.orchestrationCompatibilityEvidence) {
throw legacyCoordinatorReadOnly()
}
return undefined
}
verifyAttestedLegacyCandidate({
runtime: this.runtime,
evidence: request.orchestrationCompatibilityEvidence,
candidate: {
runId: adoption.adopted_run_id,
role: 'coordinator',
terminalHandle: candidate.terminalHandle,
paneKey: candidate.paneKey
}
})
return adoption.adopted_run_id
}
}
function candidateFromDispatch(dispatch: DispatchContextRow): LegacyPrincipalCandidate {
@@ -281,31 +255,3 @@ function candidateFromPrincipal(
paneKey: principal.pane_key
}
}
function equivalentPaneKey(a: string | null | undefined, b: string | null | undefined): boolean {
if (!a || !b) {
return false
}
if (a === b) {
return true
}
const aLeaf = parsePaneKey(a)?.leafId
const bLeaf = parsePaneKey(b)?.leafId
return Boolean(aLeaf && bLeaf && aLeaf === bLeaf)
}
function legacyReadOnlyError(): OrchestrationError {
return new OrchestrationError(
'legacy_read_only',
'This retained legacy assignment could not prove authority from its original live process. No effects were applied.',
{ effectsApplied: false }
)
}
function legacyCoordinatorReadOnly(): OrchestrationError {
return new OrchestrationError(
'legacy_read_only',
'This retained legacy coordinator could not prove its original process identity. No effects were applied.',
{ effectsApplied: false }
)
}
@@ -300,6 +300,82 @@ describe('legacy compatibility through RpcDispatcher', () => {
expect(harness.notify).toHaveBeenCalledOnce()
})
it('replays an A-era completion without touching a newer current attempt', async () => {
const harness = createHarness()
const payload = JSON.stringify({
taskId: harness.taskId,
dispatchId: harness.dispatchId
})
const completion = harness.db.insertMessage({
runId: harness.adoptedRunId,
from: WORKER_HANDLE,
to: COORDINATOR_HANDLE,
subject: 'Failed: pinned A treated this as complete',
body: 'persisted result',
type: 'worker_done',
payload,
senderPaneKey: WORKER_PANE,
deliveryContract: 'legacy_direct'
})
harness.db.settleWorkerReport({
taskId: harness.taskId,
dispatchId: harness.dispatchId,
outcome: 'succeeded',
result: 'persisted result'
})
harness.db.commitLegacyCompatibilityPrincipal({
runId: harness.adoptedRunId,
dispatchId: harness.dispatchId,
role: 'worker',
hostScope: JSON.stringify({ kind: 'local', hostId: 'local' }),
terminalHandle: WORKER_HANDLE,
paneKey: WORKER_PANE,
launchTokenHash: createHash('sha256').update('worker-token').digest('hex'),
processIncarnation: 'process-1'
})
harness.db.updateTaskStatus(harness.taskId, 'ready')
const currentDispatch = harness.db.createDispatchContext(
harness.taskId,
'term_current_worker',
'tab_current_worker:77777777-7777-4777-8777-777777777777',
'current-launch-hash'
)
const currentTaskBefore = harness.db.getTask(harness.taskId)
const currentDispatchBefore = harness.db.getDispatchContextById(currentDispatch.id)
const before = counts(harness.db)
const response = await harness.dispatcher.dispatch(
request(
'orchestration.send',
{
from: WORKER_HANDLE,
to: COORDINATOR_HANDLE,
subject: completion.subject,
body: completion.body,
type: 'worker_done',
payload
},
evidence('worker'),
'reconstruct-pinned-a-completion'
)
)
expect(response).toMatchObject({
ok: true,
result: {
message: { id: completion.id },
lifecycle: { action: 'settled', outcome: 'succeeded', duplicate: true }
}
})
expect(harness.db.getTask(harness.taskId)).toEqual(currentTaskBefore)
expect(harness.db.getDispatchContextById(harness.dispatchId)?.status).toBe('completed')
expect(harness.db.getDispatchContextById(currentDispatch.id)).toEqual(currentDispatchBefore)
expect(counts(harness.db)).toEqual({
...before,
legacy_operation_receipts: before.legacy_operation_receipts + 1
})
})
it('rejects a legacy lifecycle recipient outside the adopted Run with zero effects', async () => {
const harness = createHarness()
const before = counts(harness.db)
@@ -367,6 +443,35 @@ describe('legacy compatibility through RpcDispatcher', () => {
expect(harness.notify).not.toHaveBeenCalled()
})
it('keeps distinct compatibility ask invocations on distinct questions', async () => {
const harness = createHarness()
const ask = {
from: WORKER_HANDLE,
to: COORDINATOR_HANDLE,
question: 'Same question?',
options: ['yes', 'no'],
timeoutMs: 1
}
const first = await harness.dispatcher.dispatch(
request('orchestration.ask', ask, evidence('worker'), 'ask-one')
)
const replay = await harness.dispatcher.dispatch(
request('orchestration.ask', ask, evidence('worker'), 'ask-one')
)
const second = await harness.dispatcher.dispatch(
request('orchestration.ask', ask, evidence('worker'), 'ask-two')
)
expect(first).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: false } } })
expect(replay).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: true } } })
expect(second).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: false } } })
const firstId = (first as { result: { messageId: string } }).result.messageId
const replayId = (replay as { result: { messageId: string } }).result.messageId
const secondId = (second as { result: { messageId: string } }).result.messageId
expect(replayId).toBe(firstId)
expect(secondId).not.toBe(firstId)
})
it('does not infer an outcome for a current Dispatch when legacy adoption exists', async () => {
const harness = createHarness()
const run = harness.db.createRun({
@@ -524,6 +629,56 @@ describe('legacy compatibility through RpcDispatcher', () => {
expect(harness.notify).not.toHaveBeenCalled()
})
it('advertises only Run-addressed current delivery to a legacy coordinator', async () => {
const harness = createHarness()
harness.db.insertMessage({
runId: harness.adoptedRunId,
from: 'term_current_worker',
to: WORKER_HANDLE,
subject: 'worker-only current mail',
deliveryContract: 'current_delivery'
})
const workerOnly = await harness.dispatcher.dispatch(
request(
'orchestration.check',
{ terminal: COORDINATOR_HANDLE },
evidence('coordinator'),
'worker-current-mail'
)
)
expect(workerOnly).toMatchObject({
ok: true,
result: { legacyCompatibility: { currentDelivery: undefined } }
})
harness.db.insertMessage({
runId: harness.adoptedRunId,
from: 'term_current_worker',
to: `run:${harness.adoptedRunId}`,
subject: 'Run current mail',
deliveryContract: 'current_delivery'
})
const runMail = await harness.dispatcher.dispatch(
request(
'orchestration.check',
{ terminal: COORDINATOR_HANDLE },
evidence('coordinator'),
'run-current-mail'
)
)
expect(runMail).toMatchObject({
ok: true,
result: {
legacyCompatibility: {
currentDelivery: {
runId: harness.adoptedRunId,
checkCommand: expect.stringContaining(`--run ${harness.adoptedRunId}`)
}
}
}
})
})
it.each(['dispatch', 'websocket'] as const)(
'%s binds an attested coordinator once and rejects contradictory proof before binding',
async (transport) => {
@@ -588,4 +743,42 @@ describe('legacy compatibility through RpcDispatcher', () => {
expect(harness.db.getRun(harness.adoptedRunId)?.consumer_generation).toBe(0)
expect(counts(harness.db)).toEqual(before)
})
it('rejects stale legacy coordinator proof after current takeover', async () => {
const harness = createHarness()
const principal = harness.db.commitLegacyCompatibilityPrincipal({
runId: harness.adoptedRunId,
role: 'coordinator',
hostScope: JSON.stringify({ kind: 'local', hostId: 'local' }),
terminalHandle: COORDINATOR_HANDLE,
paneKey: COORDINATOR_PANE,
launchTokenHash: createHash('sha256').update('coordinator-token').digest('hex'),
processIncarnation: 'process-1'
}).principal
harness.db.settleWorkerReport({
taskId: harness.taskId,
dispatchId: harness.dispatchId,
outcome: 'succeeded',
result: 'done'
})
harness.db.bindRun({
runId: harness.adoptedRunId,
coordinatorHandle: 'term_current_coord',
coordinatorPaneKey: 'tab_current:55555555-5555-4555-8555-555555555555'
})
expect(harness.db.getLegacyCompatibilityPrincipal(principal.id)?.status).toBe('revoked')
const before = counts(harness.db)
const response = await harness.dispatcher.dispatch(
request(
'orchestration.taskList',
{},
evidence('coordinator'),
'stale-coordinator-after-takeover'
)
)
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
expect(counts(harness.db)).toEqual(before)
})
})
@@ -1,4 +1,4 @@
import type { RpcRequest } from './core'
import type { LegacyCoordinatorAuthorityProof, RpcRequest } from './core'
import type { OrcaRuntimeService } from '../orca-runtime'
import { LegacyCompatibilityAuthority } from './orchestration-legacy-authority'
import { handleLegacyLifecycleSend } from './orchestration-legacy-lifecycle'
@@ -10,6 +10,7 @@ import type {
LegacyReplyParams,
LegacySendParams
} from './orchestration-legacy-operation'
import { LegacyCoordinatorAuthority } from './orchestration-legacy-coordinator-authority'
const COORDINATOR_PREFLIGHT_METHODS = new Set([
'orchestration.taskCreate',
@@ -19,19 +20,31 @@ const COORDINATOR_PREFLIGHT_METHODS = new Set([
'orchestration.gateCreate',
'orchestration.gateResolve',
'orchestration.runUse',
'orchestration.send',
'orchestration.check',
'orchestration.reply'
])
export type LegacyCompatibilityRoute =
| { handled: true; result: unknown }
| { handled: false; params?: unknown; legacyCoordinatorRunId?: string }
| {
handled: false
params?: unknown
legacyCoordinatorAuthority?: LegacyCoordinatorAuthorityProof
}
export type LegacyCoordinatorInvocation = Readonly<{
authority: LegacyCoordinatorAuthorityProof
revalidate: () => string
}>
export class OrchestrationLegacyCompatibility {
private readonly authority: LegacyCompatibilityAuthority
private readonly coordinatorAuthority: LegacyCoordinatorAuthority
constructor(private readonly runtime: OrcaRuntimeService) {
this.authority = new LegacyCompatibilityAuthority(runtime)
this.coordinatorAuthority = new LegacyCoordinatorAuthority(runtime)
}
async tryHandle(
@@ -50,18 +63,40 @@ export class OrchestrationLegacyCompatibility {
return { handled: false }
}
const values = params as Record<string, unknown>
if (request.method === 'orchestration.runUse' && values.takeoverLegacy === true) {
return { handled: false }
}
const requestedRunId =
request.method === 'orchestration.runUse' ? stringValue(values.id) : stringValue(values.run)
const runId = this.authority.resolveProvenCoordinatorScope(request, requestedRunId)
return runId
const authority = this.coordinatorAuthority.resolve(request, requestedRunId)
return authority
? {
handled: false,
params: { ...values, run: runId },
legacyCoordinatorRunId: runId
params: { ...values, run: authority.runId },
legacyCoordinatorAuthority: authority
}
: { handled: false }
}
revalidateCoordinatorAuthority(
request: RpcRequest,
proof: LegacyCoordinatorAuthorityProof
): string {
return this.coordinatorAuthority.revalidate(request, proof)
}
createCoordinatorInvocation(
request: RpcRequest,
authority?: LegacyCoordinatorAuthorityProof
): LegacyCoordinatorInvocation | undefined {
return authority
? {
authority,
revalidate: () => this.revalidateCoordinatorAuthority(request, authority)
}
: undefined
}
private async route(
request: RpcRequest,
params: unknown,
@@ -0,0 +1,158 @@
import type { OrcaRuntimeService, OrchestrationCompatibilityCallerAuthority } from '../orca-runtime'
import type { LegacyCompatibilityPrincipalRow } from '../orchestration/types'
import type { LegacyCoordinatorAuthorityProof, RpcRequest } from './core'
import {
resolveAttestedLegacyPrincipal,
verifyAttestedLegacyCandidate,
type LegacyPrincipalCandidate
} from './orchestration-legacy-authority'
import {
equivalentLegacyPaneKey,
legacyCoordinatorReadOnly
} from './orchestration-legacy-process-identity'
export class LegacyCoordinatorAuthority {
constructor(private readonly runtime: OrcaRuntimeService) {}
resolve(
request: RpcRequest,
requestedRunId?: string
): LegacyCoordinatorAuthorityProof | undefined {
const db = this.runtime.getOrchestrationDb()
const adoption = db.getLegacyAdoption()
const requestedRun = requestedRunId ?? adoption?.adopted_run_id
if (!adoption || requestedRun !== adoption.adopted_run_id) {
return undefined
}
const candidate = db.resolveLegacyCoordinatorCandidate({
runId: adoption.adopted_run_id,
terminalHandle: request.orchestrationCompatibilityEvidence?.terminalHandle,
paneKey: request.orchestrationCompatibilityEvidence?.paneKey
})
if (!candidate) {
if (request.orchestrationCompatibilityEvidence) {
throw legacyCoordinatorReadOnly()
}
return undefined
}
const existing = db.getLegacyCoordinatorPrincipal(adoption.adopted_run_id)
const proofCandidate = existing
? this.candidate(adoption.adopted_run_id, existing)
: this.candidate(adoption.adopted_run_id, candidate)
const attestation = verifyAttestedLegacyCandidate({
runtime: this.runtime,
evidence: request.orchestrationCompatibilityEvidence,
candidate: proofCandidate
})
const run = db.getRun(adoption.adopted_run_id)
if (!run || !this.bindingMatches(run, proofCandidate)) {
throw legacyCoordinatorReadOnly()
}
if (existing && !this.principalMatchesAttestation(existing, attestation)) {
throw legacyCoordinatorReadOnly()
}
const principal = existing
? existing
: request.method === 'orchestration.runUse'
? resolveAttestedLegacyPrincipal({
runtime: this.runtime,
evidence: request.orchestrationCompatibilityEvidence,
candidate: proofCandidate,
authority: attestation
})
: undefined
return {
runId: run.id,
principalId: principal?.id ?? null,
terminalHandle: proofCandidate.terminalHandle,
paneKey: proofCandidate.paneKey,
consumerGeneration: run.consumer_generation
}
}
revalidate(request: RpcRequest, proof: LegacyCoordinatorAuthorityProof): string {
const db = this.runtime.getOrchestrationDb()
const adoption = db.getLegacyAdoption()
const run = db.getRun(proof.runId)
if (
adoption?.adopted_run_id !== proof.runId ||
!run ||
run.consumer_generation !== proof.consumerGeneration
) {
throw legacyCoordinatorReadOnly()
}
const candidate = this.candidate(proof.runId, proof)
const principal = db.getLegacyCoordinatorPrincipal(proof.runId)
if (proof.principalId) {
if (
principal?.id !== proof.principalId ||
principal.status !== 'committed' ||
principal.terminal_handle !== proof.terminalHandle ||
!equivalentLegacyPaneKey(principal.pane_key, proof.paneKey)
) {
throw legacyCoordinatorReadOnly()
}
} else if (
principal &&
(principal.status !== 'committed' ||
principal.terminal_handle !== proof.terminalHandle ||
!equivalentLegacyPaneKey(principal.pane_key, proof.paneKey))
) {
throw legacyCoordinatorReadOnly()
}
const attestation = verifyAttestedLegacyCandidate({
runtime: this.runtime,
evidence: request.orchestrationCompatibilityEvidence,
candidate
})
if (principal && !this.principalMatchesAttestation(principal, attestation)) {
throw legacyCoordinatorReadOnly()
}
if (!this.bindingMatches(run, candidate)) {
throw legacyCoordinatorReadOnly()
}
return run.id
}
private candidate(
runId: string,
identity: {
terminalHandle?: string
terminal_handle?: string
paneKey?: string
pane_key?: string
}
): LegacyPrincipalCandidate {
return {
runId,
role: 'coordinator',
terminalHandle: identity.terminalHandle ?? (identity.terminal_handle as string),
paneKey: identity.paneKey ?? (identity.pane_key as string)
}
}
private bindingMatches(
run: { coordinator_handle: string | null; coordinator_pane_key: string | null },
candidate: LegacyPrincipalCandidate
): boolean {
return (
!run.coordinator_pane_key ||
(run.coordinator_handle === candidate.terminalHandle &&
equivalentLegacyPaneKey(run.coordinator_pane_key, candidate.paneKey))
)
}
private principalMatchesAttestation(
principal: LegacyCompatibilityPrincipalRow,
attestation: OrchestrationCompatibilityCallerAuthority
): boolean {
return (
principal.status === 'committed' &&
principal.host_scope === JSON.stringify(attestation.hostScope) &&
principal.terminal_handle === attestation.terminalHandle &&
equivalentLegacyPaneKey(principal.pane_key, attestation.paneKey) &&
principal.launch_token_hash === attestation.launchTokenHash &&
principal.process_incarnation === attestation.processIncarnation
)
}
}
@@ -0,0 +1,849 @@
import { createHash } from 'node:crypto'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { ORCHESTRATION_CONTRACT_VERSION } from '../../../shared/protocol-version'
import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence'
import Database from '../../sqlite/sync-database'
import { OrcaRuntimeService } from '../orca-runtime'
import { OrchestrationDb } from '../orchestration/db'
import type { RpcRequest, RpcResponse } from './core'
import { RpcDispatcher } from './dispatcher'
import { ORCHESTRATION_METHODS } from './methods/orchestration'
const COORDINATOR_HANDLE = 'term_legacy_coord'
const COORDINATOR_PANE = 'tab_coord:44444444-4444-4444-8444-444444444444'
const WORKER_HANDLE = 'term_legacy_worker'
const WORKER_PANE = 'tab_worker:33333333-3333-4333-8333-333333333333'
type Transport = 'dispatch' | 'websocket'
type Harness = {
db: OrchestrationDb
dispatcher: RpcDispatcher
runtime: OrcaRuntimeService
adoptedRunId: string
taskId: string
dispatchId: string
}
const tempDirs: string[] = []
const databases: OrchestrationDb[] = []
afterEach(() => {
databases.splice(0).forEach((database) => database.close())
tempDirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }))
})
function createHarness(): Harness {
const dir = mkdtempSync(join(tmpdir(), 'orca-legacy-coordinator-race-'))
tempDirs.push(dir)
const dbPath = join(dir, 'orchestration.db')
const before = new OrchestrationDb(dbPath)
const task = before.createTask({
spec: 'legacy assignment',
createdByTerminalHandle: COORDINATOR_HANDLE
})
const dispatch = before.createDispatchContext(task.id, WORKER_HANDLE, WORKER_PANE)
before.close()
const raw = new Database(dbPath)
raw.exec(`
UPDATE dispatch_contexts SET process_incarnation = 'process-1';
DROP INDEX IF EXISTS idx_messages_delivery_contract;
DROP TABLE legacy_mail_receipts;
DROP TABLE legacy_operation_receipts;
DROP TABLE legacy_compatibility_principals;
DROP TABLE legacy_adoptions;
`)
raw.pragma('user_version = 18')
raw.close()
const db = new OrchestrationDb(dbPath)
databases.push(db)
const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string
const runtime = new OrcaRuntimeService()
runtime.setOrchestrationDb(db)
vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) =>
handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : handle === WORKER_HANDLE ? WORKER_PANE : null
)
vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation(
(compatibilityEvidence) => {
const coordinator =
compatibilityEvidence?.terminalHandle === COORDINATOR_HANDLE &&
compatibilityEvidence.paneKey === COORDINATOR_PANE &&
compatibilityEvidence.launchToken === 'coordinator-token'
const worker =
compatibilityEvidence?.terminalHandle === WORKER_HANDLE &&
compatibilityEvidence.paneKey === WORKER_PANE &&
compatibilityEvidence.launchToken === 'worker-token'
return coordinator || worker
? {
hostScope: { kind: 'local', hostId: 'local' },
terminalHandle: coordinator ? COORDINATOR_HANDLE : WORKER_HANDLE,
paneKey: coordinator ? COORDINATOR_PANE : WORKER_PANE,
processIncarnation: 'process-1',
launchTokenHash: createHash('sha256')
.update(coordinator ? 'coordinator-token' : 'worker-token')
.digest('hex')
}
: null
}
)
return {
db,
dispatcher: new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }),
runtime,
adoptedRunId,
taskId: task.id,
dispatchId: dispatch.id
}
}
function evidence(
role: 'coordinator' | 'worker' = 'coordinator'
): OrchestrationCompatibilityEvidence {
return {
terminalHandle: role === 'coordinator' ? COORDINATOR_HANDLE : WORKER_HANDLE,
paneKey: role === 'coordinator' ? COORDINATOR_PANE : WORKER_PANE,
launchToken: role === 'coordinator' ? 'coordinator-token' : 'worker-token'
}
}
function request(
method: string,
params: unknown,
invocationId: string,
callerEvidence = evidence()
): RpcRequest {
return {
id: `rpc_${invocationId}`,
authToken: 'caller-token',
method,
params,
orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION,
orchestrationRequestId: invocationId,
compatibilityInvocationId: invocationId,
orchestrationCompatibilityEvidence: callerEvidence
}
}
async function invoke(
dispatcher: RpcDispatcher,
rpcRequest: RpcRequest,
transport: Transport
): Promise<RpcResponse> {
if (transport === 'dispatch') {
return await dispatcher.dispatch(rpcRequest)
}
const replies: string[] = []
await dispatcher.dispatchStreaming(rpcRequest, (reply) => replies.push(reply))
expect(replies).toHaveLength(1)
return JSON.parse(replies[0]) as RpcResponse
}
function installTakeoverDuringMutationPreflight(harness: Harness): void {
const originalBegin = harness.db.beginMutationReceipt.bind(harness.db)
vi.spyOn(harness.db, 'beginMutationReceipt').mockImplementation((identity) => {
const begun = originalBegin(identity)
settleLegacyWorkerAndTakeOver(harness, 'settled during takeover')
return begun
})
}
function settleLegacyWorkerAndTakeOver(harness: Harness, result: string): void {
harness.db.settleWorkerReport({
taskId: harness.taskId,
dispatchId: harness.dispatchId,
outcome: 'succeeded',
result
})
harness.db.bindRun({
runId: harness.adoptedRunId,
coordinatorHandle: 'term_current_coord',
coordinatorPaneKey: 'tab_current:55555555-5555-4555-8555-555555555555'
})
}
function mutationReceiptCount(db: OrchestrationDb): number {
const sqlite = (db as unknown as { db: Database.Database }).db
return (
sqlite.prepare('SELECT COUNT(*) AS count FROM mutation_receipts').get() as { count: number }
).count
}
function messageCount(db: OrchestrationDb): number {
const sqlite = (db as unknown as { db: Database.Database }).db
return (sqlite.prepare('SELECT COUNT(*) AS count FROM messages').get() as { count: number }).count
}
describe('legacy coordinator takeover races', () => {
it.each(['dispatch', 'websocket'] as const)(
'%s routes a pre-bind coordinator send to the retained worker mailbox',
async (transport) => {
const harness = createHarness()
const response = await invoke(
harness.dispatcher,
request(
'orchestration.send',
{ from: COORDINATOR_HANDLE, to: WORKER_HANDLE, subject: 'continue safely' },
`send-direct-${transport}`
),
transport
)
expect(response).toMatchObject({
ok: true,
result: {
message: {
run_id: harness.adoptedRunId,
to_handle: WORKER_HANDLE,
delivery_contract: 'legacy_direct'
}
}
})
const workerCheck = await harness.dispatcher.dispatch(
request(
'orchestration.check',
{ terminal: WORKER_HANDLE },
`worker-check-${transport}`,
evidence('worker')
)
)
expect(workerCheck).toMatchObject({
ok: true,
result: {
messages: [
{
subject: 'continue safely',
run_id: harness.adoptedRunId,
delivery_contract: 'legacy_direct'
}
],
legacyCompatibility: { ackMessageIds: [expect.any(String)] }
}
})
expect(
harness.db.getInbox(100).some((message) => message.run_id === 'run_legacy_local')
).toBe(false)
}
)
it('routes an exact legacy Dispatch recipient to compatibility delivery', async () => {
const harness = createHarness()
const response = await harness.dispatcher.dispatch(
request(
'orchestration.send',
{
from: COORDINATOR_HANDLE,
to: `dispatch:${harness.dispatchId}`,
subject: 'dispatch guidance'
},
'send-dispatch'
)
)
expect(response).toMatchObject({
ok: true,
result: {
message: {
run_id: harness.adoptedRunId,
to_handle: `dispatch:${harness.dispatchId}`,
delivery_contract: 'legacy_direct'
}
}
})
const workerCheck = await harness.dispatcher.dispatch(
request(
'orchestration.check',
{ terminal: WORKER_HANDLE },
'worker-check-dispatch',
evidence('worker')
)
)
expect(workerCheck).toMatchObject({
ok: true,
result: { messages: [{ subject: 'dispatch guidance' }] }
})
})
it('partitions a coordinator group send by legacy recipient contract', async () => {
const harness = createHarness()
vi.spyOn(harness.runtime, 'listTerminals').mockResolvedValue({
terminals: [
{ handle: COORDINATOR_HANDLE },
{ handle: WORKER_HANDLE },
{ handle: 'term_current_worker' }
],
totalCount: 3,
truncated: false
} as never)
const response = await harness.dispatcher.dispatch(
request(
'orchestration.send',
{ from: COORDINATOR_HANDLE, to: '@all', subject: 'group guidance' },
'send-group'
)
)
expect(response).toMatchObject({ ok: true, result: { recipients: 2 } })
const messages = harness.db
.getInbox(100)
.filter((message) => message.subject === 'group guidance')
expect(messages).toEqual(
expect.arrayContaining([
expect.objectContaining({
run_id: harness.adoptedRunId,
to_handle: WORKER_HANDLE,
delivery_contract: 'legacy_direct'
}),
expect.objectContaining({
run_id: harness.adoptedRunId,
to_handle: 'term_current_worker',
delivery_contract: 'current_delivery'
})
])
)
})
it.each(['dispatch', 'websocket'] as const)(
'%s rejects a coordinator send after takeover during durable preflight',
async (transport) => {
const harness = createHarness()
installTakeoverDuringMutationPreflight(harness)
const before = messageCount(harness.db)
const response = await invoke(
harness.dispatcher,
request(
'orchestration.send',
{ from: COORDINATOR_HANDLE, to: WORKER_HANDLE, subject: 'must not send' },
`send-takeover-${transport}`
),
transport
)
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
expect(messageCount(harness.db)).toBe(before)
expect(mutationReceiptCount(harness.db)).toBe(0)
}
)
it('revalidates a group send after asynchronous terminal discovery', async () => {
const harness = createHarness()
let resolveTerminals:
| ((value: {
terminals: { handle: string }[]
totalCount: number
truncated: boolean
}) => void)
| undefined
let signalListingStarted: (() => void) | undefined
const listingStarted = new Promise<void>((resolve) => {
signalListingStarted = resolve
})
vi.spyOn(harness.runtime, 'listTerminals').mockImplementation(
() =>
new Promise((resolve) => {
resolveTerminals = resolve as typeof resolveTerminals
signalListingStarted?.()
}) as never
)
const before = messageCount(harness.db)
const pending = harness.dispatcher.dispatch(
request(
'orchestration.send',
{ from: COORDINATOR_HANDLE, to: '@all', subject: 'must remain unsent' },
'send-group-takeover'
)
)
await listingStarted
settleLegacyWorkerAndTakeOver(harness, 'settled during terminal discovery')
resolveTerminals?.({
terminals: [{ handle: COORDINATOR_HANDLE }, { handle: WORKER_HANDLE }],
totalCount: 2,
truncated: false
})
await expect(pending).resolves.toMatchObject({
ok: false,
error: { code: 'legacy_read_only' }
})
expect(messageCount(harness.db)).toBe(before)
expect(mutationReceiptCount(harness.db)).toBe(0)
})
it('reports a committed ACK when coordinator takeover interrupts its wait', async () => {
const harness = createHarness()
const incoming = harness.db.insertMessage({
runId: harness.adoptedRunId,
from: WORKER_HANDLE,
to: `run:${harness.adoptedRunId}`,
subject: 'ready to acknowledge'
})
const first = await harness.dispatcher.dispatch(
request(
'orchestration.check',
{ terminal: COORDINATOR_HANDLE, run: harness.adoptedRunId },
'check-delivery-before-takeover'
)
)
expect(first).toMatchObject({
ok: true,
result: { deliveryId: expect.any(String), messages: [{ id: incoming.id }] }
})
const deliveryId = (first as { result: { deliveryId: string } }).result.deliveryId
let resolveWait: (() => void) | undefined
let signalWaitStarted: (() => void) | undefined
const waitStarted = new Promise<void>((resolve) => {
signalWaitStarted = resolve
})
vi.spyOn(harness.runtime, 'waitForMessage').mockImplementation(
() =>
new Promise((resolve) => {
resolveWait = () => resolve('notified')
signalWaitStarted?.()
})
)
const pending = harness.dispatcher.dispatch(
request(
'orchestration.check',
{
terminal: COORDINATOR_HANDLE,
run: harness.adoptedRunId,
ack: deliveryId,
wait: true,
timeoutMs: 10_000
},
'check-ack-takeover'
)
)
await waitStarted
expect(harness.db.getMessageById(incoming.id)?.read).toBe(1)
settleLegacyWorkerAndTakeOver(harness, 'settled during acknowledged wait')
resolveWait?.()
await expect(pending).resolves.toMatchObject({
ok: true,
result: {
acknowledged: deliveryId,
count: 0,
waitInterrupted: 'consumer_fenced',
mutation: { requestId: 'check-ack-takeover', replayed: false }
}
})
const sqlite = (harness.db as unknown as { db: Database.Database }).db
expect(sqlite.prepare('SELECT state FROM mutation_receipts').get()).toEqual({
state: 'completed'
})
})
it('keeps concurrent duplicates on the live ACK-and-wait result', async () => {
const harness = createHarness()
harness.db.insertMessage({
runId: harness.adoptedRunId,
from: WORKER_HANDLE,
to: `run:${harness.adoptedRunId}`,
subject: 'first delivery'
})
const firstCheck = await harness.dispatcher.dispatch(
request(
'orchestration.check',
{ terminal: COORDINATOR_HANDLE, run: harness.adoptedRunId },
'check-first-delivery'
)
)
const deliveryId = (firstCheck as { result: { deliveryId: string } }).result.deliveryId
let resolveWait: (() => void) | undefined
let signalWaitStarted: (() => void) | undefined
const waitStarted = new Promise<void>((resolve) => {
signalWaitStarted = resolve
})
vi.spyOn(harness.runtime, 'waitForMessage').mockImplementation(
() =>
new Promise((resolve) => {
resolveWait = () => resolve('notified')
signalWaitStarted?.()
})
)
const ackRequest = request(
'orchestration.check',
{
terminal: COORDINATOR_HANDLE,
run: harness.adoptedRunId,
ack: deliveryId,
wait: true,
timeoutMs: 10_000
},
'check-ack-concurrent'
)
const original = harness.dispatcher.dispatch(ackRequest)
await waitStarted
const secondLiveDispatcher = new RpcDispatcher({
runtime: harness.runtime,
methods: ORCHESTRATION_METHODS
})
const duplicate = secondLiveDispatcher.dispatch({
...ackRequest,
id: 'rpc_check-ack-concurrent-duplicate'
})
let duplicateSettled = false
void duplicate.then(() => {
duplicateSettled = true
})
await Promise.resolve()
expect(duplicateSettled).toBe(false)
const arrived = harness.db.insertMessage({
runId: harness.adoptedRunId,
from: WORKER_HANDLE,
to: `run:${harness.adoptedRunId}`,
subject: 'arrived while waiting'
})
resolveWait?.()
const [originalResult, duplicateResult] = await Promise.all([original, duplicate])
expect(originalResult).toMatchObject({
ok: true,
result: {
messages: [{ id: arrived.id }],
mutation: { requestId: 'check-ack-concurrent', replayed: false }
}
})
expect(duplicateResult).toMatchObject({
ok: true,
result: {
messages: [{ id: arrived.id }],
mutation: { requestId: 'check-ack-concurrent', replayed: true }
}
})
})
it.each(['dispatch', 'websocket'] as const)(
'%s rejects run-use after takeover during durable preflight',
async (transport) => {
const harness = createHarness()
installTakeoverDuringMutationPreflight(harness)
const response = await invoke(
harness.dispatcher,
request(
'orchestration.runUse',
{ id: harness.adoptedRunId, from: COORDINATOR_HANDLE },
`run-use-takeover-${transport}`
),
transport
)
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
expect(harness.db.getRun(harness.adoptedRunId)).toMatchObject({
coordinator_handle: 'term_current_coord',
consumer_generation: 1
})
expect(mutationReceiptCount(harness.db)).toBe(0)
expect(harness.db.getLegacyCoordinatorPrincipal(harness.adoptedRunId)?.status).toBe('revoked')
}
)
it.each(['dispatch', 'websocket'] as const)(
'%s rejects a mutation after takeover during durable preflight',
async (transport) => {
const harness = createHarness()
const target = harness.db.createTask({
spec: 'must remain ready',
runId: harness.adoptedRunId
})
installTakeoverDuringMutationPreflight(harness)
const response = await invoke(
harness.dispatcher,
request(
'orchestration.taskUpdate',
{
id: target.id,
status: 'completed',
callerTerminalHandle: COORDINATOR_HANDLE
},
`task-update-takeover-${transport}`
),
transport
)
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
expect(harness.db.getTask(target.id)?.status).toBe('ready')
expect(mutationReceiptCount(harness.db)).toBe(0)
}
)
it('revalidates after asynchronous agent detection', async () => {
const harness = createHarness()
const target = harness.db.createTask({
spec: 'must not dispatch after takeover',
runId: harness.adoptedRunId
})
const targetHandle = 'term_current_worker'
const targetPane = 'tab_current_worker:77777777-7777-4777-8777-777777777777'
vi.spyOn(harness.runtime, 'getTerminalPaneKey').mockImplementation((handle: string) =>
handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : handle === targetHandle ? targetPane : null
)
vi.spyOn(harness.runtime, 'getTerminalProcessIncarnation').mockReturnValue('current-process')
let resolveDetection: ((detected: boolean) => void) | undefined
let signalDetectionStarted: (() => void) | undefined
const detectionStarted = new Promise<void>((resolve) => {
signalDetectionStarted = resolve
})
vi.spyOn(harness.runtime, 'isTerminalRunningAgent').mockImplementation(
() =>
new Promise<boolean>((resolve) => {
resolveDetection = resolve
signalDetectionStarted?.()
})
)
const sendPrompt = vi
.spyOn(harness.runtime, 'sendTerminalAgentPrompt')
.mockResolvedValue({ handle: targetHandle, accepted: true, bytesWritten: 1 })
const pending = harness.dispatcher.dispatch(
request(
'orchestration.dispatch',
{ task: target.id, to: targetHandle, from: COORDINATOR_HANDLE, inject: true },
'dispatch-detection-takeover'
)
)
await detectionStarted
settleLegacyWorkerAndTakeOver(harness, 'settled during detection')
resolveDetection?.(true)
const response = await pending
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
expect(harness.db.getTask(target.id)?.status).toBe('ready')
expect(harness.db.getDispatchContext(target.id)).toBeUndefined()
expect(sendPrompt).not.toHaveBeenCalled()
})
it('explicitly takes over an active adopted Run without stranding its legacy worker', async () => {
const harness = createHarness()
const currentHandle = 'term_current_coord'
const currentPane = 'tab_current:55555555-5555-4555-8555-555555555555'
vi.mocked(harness.runtime.getTerminalPaneKey).mockImplementation((handle) =>
handle === COORDINATOR_HANDLE
? COORDINATOR_PANE
: handle === WORKER_HANDLE
? WORKER_PANE
: handle === currentHandle
? currentPane
: null
)
await expect(
harness.dispatcher.dispatch(
request(
'orchestration.runUse',
{ id: harness.adoptedRunId, from: COORDINATOR_HANDLE },
'bind-original-coordinator'
)
)
).resolves.toMatchObject({ ok: true })
const pendingAsk = await harness.dispatcher.dispatch(
request(
'orchestration.ask',
{
from: WORKER_HANDLE,
to: COORDINATOR_HANDLE,
question: 'Continue with the migration?',
timeoutMs: 0
},
'pending-before-takeover',
evidence('worker')
)
)
const questionId = (pendingAsk as { result: { messageId: string } }).result.messageId
const unrelated = harness.db.insertMessage({
runId: harness.adoptedRunId,
from: WORKER_HANDLE,
to: 'term_peer_worker',
subject: 'peer-only',
deliveryContract: 'legacy_direct'
})
const currentRequest = (method: string, params: unknown, invocationId: string): RpcRequest => ({
...request(method, params, invocationId),
orchestrationCompatibilityEvidence: undefined
})
const takeoverParams = {
id: harness.adoptedRunId,
from: currentHandle,
takeoverLegacy: true
}
const takeover = await harness.dispatcher.dispatch(
currentRequest('orchestration.runUse', takeoverParams, 'explicit-takeover')
)
const repeated = await harness.dispatcher.dispatch(
currentRequest('orchestration.runUse', takeoverParams, 'explicit-takeover-repeat')
)
expect(takeover).toMatchObject({
ok: true,
result: { binding: { consumerGeneration: 2 } }
})
expect(repeated).toMatchObject({
ok: true,
result: { binding: { consumerGeneration: 2 } }
})
expect(harness.db.getDispatchContextById(harness.dispatchId)?.status).toBe('dispatched')
expect(harness.db.getLegacyCoordinatorPrincipal(harness.adoptedRunId)?.status).toBe('revoked')
expect(harness.db.getMessageById(unrelated.id)).toMatchObject({
to_handle: 'term_peer_worker',
delivery_contract: 'legacy_direct'
})
const promoted = await harness.dispatcher.dispatch(
currentRequest(
'orchestration.check',
{ terminal: currentHandle, run: harness.adoptedRunId, format: true },
'read-promoted-question'
)
)
expect(promoted).toMatchObject({
ok: true,
result: {
messages: [
{
id: questionId,
to_handle: `run:${harness.adoptedRunId}`,
delivery_contract: 'current_delivery'
}
],
formatted: expect.not.stringContaining(`--from run:${harness.adoptedRunId}`)
}
})
const deliveryId = (promoted as { result: { deliveryId: string } }).result.deliveryId
await expect(
harness.dispatcher.dispatch(
currentRequest(
'orchestration.reply',
{
id: questionId,
body: 'Yes, continue.',
from: currentHandle,
run: harness.adoptedRunId
},
'reply-after-takeover'
)
)
).resolves.toMatchObject({ ok: true, result: { question: { status: 'answered' } } })
await expect(
harness.dispatcher.dispatch(
request(
'orchestration.ask',
{ from: WORKER_HANDLE, resume: questionId, timeoutMs: 100 },
'resume-after-takeover',
evidence('worker')
)
)
).resolves.toMatchObject({ ok: true, result: { answer: 'Yes, continue.' } })
const followUp = await harness.dispatcher.dispatch(
currentRequest(
'orchestration.send',
{
from: currentHandle,
to: `dispatch:${harness.dispatchId}`,
subject: 'Continue from the new coordinator'
},
'current-coordinator-to-legacy-worker'
)
)
expect(followUp).toMatchObject({
ok: true,
result: {
message: {
run_id: harness.adoptedRunId,
to_handle: `dispatch:${harness.dispatchId}`,
delivery_contract: 'legacy_direct'
}
}
})
const workerFollowUp = await harness.dispatcher.dispatch(
request(
'orchestration.check',
{ terminal: WORKER_HANDLE },
'legacy-worker-check-after-takeover',
evidence('worker')
)
)
expect(workerFollowUp).toMatchObject({
ok: true,
result: {
messages: [{ subject: 'Continue from the new coordinator' }],
legacyCompatibility: { ackMessageIds: [expect.any(String)] }
}
})
const [followUpMessageId] = (
workerFollowUp as { result: { legacyCompatibility: { ackMessageIds: string[] } } }
).result.legacyCompatibility.ackMessageIds
await expect(
harness.dispatcher.dispatch(
request(
'orchestration.check',
{
terminal: WORKER_HANDLE,
compatibilityAck: JSON.stringify({ messageIds: [followUpMessageId] })
},
'legacy-worker-ack-after-takeover',
evidence('worker')
)
)
).resolves.toMatchObject({
ok: true,
result: {
acknowledged: [followUpMessageId],
legacyCompatibility: { acknowledged: true }
}
})
await harness.dispatcher.dispatch(
currentRequest(
'orchestration.check',
{ terminal: currentHandle, run: harness.adoptedRunId, ack: deliveryId },
'ack-promoted-question'
)
)
await expect(
harness.dispatcher.dispatch(
request(
'orchestration.send',
{
from: WORKER_HANDLE,
to: COORDINATOR_HANDLE,
subject: 'Need current coordinator',
type: 'escalation',
payload: JSON.stringify({ taskId: harness.taskId })
},
'escalation-after-takeover',
evidence('worker')
)
)
).resolves.toMatchObject({
ok: true,
result: {
message: {
to_handle: `run:${harness.adoptedRunId}`,
delivery_contract: 'current_delivery'
}
}
})
await expect(
harness.dispatcher.dispatch(
request(
'orchestration.taskList',
{ callerTerminalHandle: COORDINATOR_HANDLE },
'old-coordinator-fenced'
)
)
).resolves.toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
})
})
@@ -104,7 +104,7 @@ export async function handleLegacyLifecycleSend(args: {
: { kind: 'message_only' }
})
if (!committed.duplicate) {
runtime.notifyMessageArrived(params.to, committed.message.type)
runtime.notifyMessageArrived(committed.message.to_handle, committed.message.type)
}
return {
message: committed.message,
@@ -0,0 +1,33 @@
import { parsePaneKey } from '../../../shared/stable-pane-id'
import { OrchestrationError } from '../orchestration/orchestration-error'
export function equivalentLegacyPaneKey(
a: string | null | undefined,
b: string | null | undefined
): boolean {
if (!a || !b) {
return false
}
if (a === b) {
return true
}
const aLeaf = parsePaneKey(a)?.leafId
const bLeaf = parsePaneKey(b)?.leafId
return Boolean(aLeaf && bLeaf && aLeaf === bLeaf)
}
export function legacyReadOnlyError(): OrchestrationError {
return new OrchestrationError(
'legacy_read_only',
'This retained legacy assignment could not prove authority from its original live process. No effects were applied.',
{ effectsApplied: false }
)
}
export function legacyCoordinatorReadOnly(): OrchestrationError {
return new OrchestrationError(
'legacy_read_only',
'This retained legacy coordinator could not prove its original process identity. No effects were applied.',
{ effectsApplied: false }
)
}
@@ -65,8 +65,9 @@ export async function handleLegacyAsk(args: {
options,
recipientHandle: recipient
})
const pending = matches.filter((match) => match.question.status === 'pending')
const lostAnswer = matches.find(
const unclaimed = matches.filter((match) => !match.claimedByOperation)
const pending = unclaimed.filter((match) => match.question.status === 'pending')
const lostAnswer = unclaimed.find(
(match) => match.question.status === 'answered' && !match.answerAcknowledged
)
if (lostAnswer) {
@@ -98,7 +99,7 @@ export async function handleLegacyAsk(args: {
questionId = committed.question.message_id
duplicate = committed.duplicate || Boolean(existingQuestionId)
if (!duplicate) {
runtime.notifyMessageArrived(recipient, committed.message.type)
runtime.notifyMessageArrived(committed.message.to_handle, committed.message.type)
}
}
@@ -55,6 +55,10 @@ export class OrchestrationMutationExecutor {
: db.beginMutationReceipt(identity)
if (begun.disposition === 'completed') {
const active = this.inFlight.get(key)
if (active) {
return attachMutationReceipt(await active, requestId, true)
}
return attachMutationReceipt(JSON.parse(begun.row.receipt ?? 'null'), requestId, true)
}
if (begun.disposition === 'pending') {
@@ -102,6 +106,20 @@ export class OrchestrationMutationExecutor {
}
}
const executorsByRuntime = new WeakMap<OrcaRuntimeService, OrchestrationMutationExecutor>()
export function getOrchestrationMutationExecutor(
runtime: OrcaRuntimeService
): OrchestrationMutationExecutor {
const existing = executorsByRuntime.get(runtime)
if (existing) {
return existing
}
const executor = new OrchestrationMutationExecutor(runtime)
executorsByRuntime.set(runtime, executor)
return executor
}
export function authenticatedCallerFingerprint(request: RpcRequest): string {
const callerToken =
request.authToken ||
@@ -252,7 +252,10 @@ describe('durable orchestration mutation ledger', () => {
})
it('recovers a lost ask acceptance without creating a second question', async () => {
const db = new OrchestrationDb(':memory:')
const dir = mkdtempSync(join(tmpdir(), 'orca-mutation-ask-recovery-'))
paths.push(dir)
const dbPath = join(dir, 'orchestration.db')
const db = new OrchestrationDb(dbPath)
const runtime = new OrcaRuntimeService()
runtime.setOrchestrationDb(db)
vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_worker:leaf_worker')
@@ -290,7 +293,16 @@ describe('durable orchestration mutation ledger', () => {
const first = firstDispatcher.dispatch(askRequest, { signal: controller.signal })
await vi.waitFor(() => expect(db.getInbox(10)).toHaveLength(1))
const restartedDispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS })
const restartedDb = new OrchestrationDb(dbPath)
const restartedRuntime = new OrcaRuntimeService()
restartedRuntime.setOrchestrationDb(restartedDb)
vi.spyOn(restartedRuntime, 'getTerminalPaneKey').mockReturnValue('tab_worker:leaf_worker')
vi.spyOn(restartedRuntime, 'getTerminalProcessIncarnation').mockReturnValue('runtime:pty:1')
vi.spyOn(restartedRuntime, 'notifyMessageArrived').mockImplementation(() => {})
const restartedDispatcher = new RpcDispatcher({
runtime: restartedRuntime,
methods: ORCHESTRATION_METHODS
})
const recovered = await restartedDispatcher.dispatch({ ...askRequest, id: 'rpc_ask_2' })
expect(recovered).toMatchObject({
ok: true,
@@ -304,6 +316,7 @@ describe('durable orchestration mutation ledger', () => {
controller.abort()
await first
restartedDb.close()
db.close()
})
})
@@ -1,5 +1,6 @@
import type { FeatureInteractionId } from '../../../shared/feature-interactions'
import { isBrowserPaneUiRuntimeRpcParams } from '../../../shared/runtime-rpc-feature-interaction-source'
import type { OrcaRuntimeService } from '../orca-runtime'
export function getRuntimeFeatureInteractionId(
method: string,
@@ -37,6 +38,25 @@ export function getRuntimeFeatureInteractionId(
return method.startsWith('orchestration.') ? 'agent-orchestration' : null
}
export function recordRuntimeFeatureInteraction(
runtime: OrcaRuntimeService,
method: string,
result: unknown,
alreadyRecorded?: Set<FeatureInteractionId>,
rawParams?: unknown
): void {
const id = getRuntimeFeatureInteractionId(method, result, rawParams)
if (!id || alreadyRecorded?.has(id)) {
return
}
try {
runtime.recordFeatureInteraction(id)
alreadyRecorded?.add(id)
} catch {
// Best-effort education state must not break runtime tools.
}
}
function hasBooleanResult(value: unknown, key: string): boolean {
return (
value !== null && typeof value === 'object' && (value as Record<string, unknown>)[key] === true
+35
View File
@@ -66,6 +66,10 @@ import {
import type { Store } from '../persistence'
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
import { runRemoteOrcaCli } from './ssh-remote-orca-cli'
import {
acknowledgeRemoteOrcaCliPostOutput,
parseRemoteOrcaCliPostOutput
} from './ssh-remote-orchestration-post-output'
import { toSshExecutionHostId, type ExecutionHostId } from '../../shared/execution-host'
import { isTerminalLeafId, makePaneKey } from '../../shared/stable-pane-id'
import { isValidTerminalTabId } from '../../shared/terminal-tab-id'
@@ -716,6 +720,37 @@ export class SshRelaySession {
this.runtime.releaseOrchestrationCompatibilitySshAttachment(runtimeAuthority.attachmentId)
}
})
mux.onRequest('orca.cli.postOutput', async (params) => {
if (!this.runtime) {
throw new Error('Orca runtime is unavailable')
}
const rawEnv = params.env
const env =
rawEnv && typeof rawEnv === 'object' && !Array.isArray(rawEnv)
? Object.fromEntries(
Object.entries(rawEnv).filter(
(entry): entry is [string, string] =>
typeof entry[0] === 'string' && typeof entry[1] === 'string'
)
)
: {}
const runtimeAuthority = this.runtime.registerOrchestrationCompatibilitySshAttachment(
this.targetId,
connectionIncarnation
)
this.activeCompatibilityAttachmentIds.add(runtimeAuthority.attachmentId)
try {
await acknowledgeRemoteOrcaCliPostOutput(this.runtime, {
postOutput: parseRemoteOrcaCliPostOutput(params.postOutput),
env,
runtimeAuthority
})
return { acknowledged: true }
} finally {
this.activeCompatibilityAttachmentIds.delete(runtimeAuthority.attachmentId)
this.runtime.releaseOrchestrationCompatibilitySshAttachment(runtimeAuthority.attachmentId)
}
})
}
// Why: ship plugin/extension source from Orca so agent-event changes don't force a relay redeploy (agent-status-over-ssh.md §4/§8). Best-effort.
@@ -43,8 +43,23 @@ export type RemoteOrcaCliResult = {
stdout: string
stderr: string
exitCode: number
postOutput?: RemoteOrcaCliPostOutput
}
export type RemoteOrcaCliPostOutput =
| {
kind: 'legacy_check_ack'
terminal: string
messageIds: string[]
types?: string[]
}
| {
kind: 'legacy_question_ack'
terminal: string
questionId: string
answerMessageId: string
}
export type HostCliPassthroughOptions = {
execPath?: string
cliEntryPath?: string
@@ -0,0 +1,40 @@
import type { RpcResponse } from '../runtime/rpc/core'
import type { ParsedRemoteCli } from './ssh-remote-cli-argument-error'
import type { RemoteOrcaCliResult } from './ssh-remote-cli-host-passthrough'
import { formatRemoteCli } from './ssh-remote-cli-format'
import {
formatRemoteOrchestrationAsk,
getRemoteCliExitCode
} from './ssh-remote-orchestration-ask-output'
import { formatRemoteOrchestrationCheck } from './ssh-remote-orchestration-check-output'
import { getRemoteCliPostOutput } from './ssh-remote-orchestration-post-output'
import { resolveRemoteCliHandle } from './ssh-remote-cli-args'
export function formatInProcessRemoteCliResult(
parsed: ParsedRemoteCli,
env: Record<string, string>,
response: RpcResponse,
json: boolean
): RemoteOrcaCliResult {
const command = parsed.commandPath.join(' ')
const formatted =
command === 'orchestration check'
? formatRemoteOrchestrationCheck(
response,
json,
resolveRemoteCliHandle(parsed.flags, env, 'terminal'),
parsed.flags.has('format')
)
: command === 'orchestration ask'
? formatRemoteOrchestrationAsk(response, json)
: json
? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' }
: formatRemoteCli(response)
const postOutput = getRemoteCliPostOutput(parsed, env, response)
return {
stdout: formatted.stdout,
stderr: formatted.stderr,
exitCode: getRemoteCliExitCode(command, response),
...(postOutput ? { postOutput } : {})
}
}
-1
View File
@@ -24,7 +24,6 @@ const LEGACY_FALLBACK_OPTIONS: HostCliPassthroughOptions = {
userDataPath: '/host/user-data',
entryExists: () => false
}
type FakeChild = EventEmitter & {
stdout: EventEmitter
stderr: EventEmitter
+48 -18
View File
@@ -6,7 +6,6 @@ import { ORCHESTRATION_CONTRACT_VERSION } from '../../shared/protocol-version'
import { RpcDispatcher } from '../runtime/rpc/dispatcher'
import type { RpcResponse } from '../runtime/rpc/core'
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
import { formatRemoteCli } from './ssh-remote-cli-format'
import {
HostCliUnavailableError,
runHostOrcaCliPassthrough,
@@ -26,9 +25,9 @@ import { buildRemoteCliError } from './ssh-remote-cli-error-response'
import { getRemoteLinearHelp, tryDispatchRemoteLinearCli } from './ssh-remote-linear-cli'
import {
getRemoteOrchestrationPayload,
hasRemoteLifecycleRejection,
resolveRemoteOrchestrationSender
} from './ssh-remote-orchestration-send'
import { formatInProcessRemoteCliResult } from './ssh-remote-cli-in-process-result'
export type { RemoteOrcaCliRequest, RemoteOrcaCliResult } from './ssh-remote-cli-host-passthrough'
@@ -51,6 +50,7 @@ export async function runRemoteOrcaCli(
): Promise<RemoteOrcaCliResult> {
const parsed = parseRemoteCliArgs(request.argv)
const json = parsed.flags.has('json')
const command = parsed.commandPath.join(' ')
const interactiveMessage = HOST_INTERACTIVE_COMMANDS[parsed.commandPath[0] ?? '']
if (interactiveMessage) {
@@ -64,6 +64,17 @@ export async function runRemoteOrcaCli(
return { stdout: '', stderr: `${interactiveMessage}\n`, exitCode: 1 }
}
if (command === 'orchestration check' || command === 'orchestration ask') {
// Why: compatibility ACKs must wait until relay stdout is observable; a host CLI child can only flush into main's capture pipe.
return await runLegacyRemoteOrcaCli(
runtime,
request,
parsed,
json,
new HostCliUnavailableError('output-ordered orchestration bridge required')
)
}
let passthroughFailure: HostCliUnavailableError | null = null
try {
return await runHostOrcaCliPassthrough(request, passthroughOptions)
@@ -101,16 +112,7 @@ async function runLegacyRemoteOrcaCli(
passthroughFailure.message,
request.runtimeAuthority
)
const formatted = json
? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' }
: formatRemoteCli(response)
return {
stdout: formatted.stdout,
stderr: formatted.stderr,
// Why: the legacy SSH bridge bypasses the local CLI handler that turns
// a persisted lifecycle rejection into an unsuccessful command.
exitCode: response.ok && !hasRemoteLifecycleRejection(response.result) ? 0 : 1
}
return formatInProcessRemoteCliResult(parsed, request.env, response, json)
} catch (err) {
const message = err instanceof Error ? err.message : String(err)
const code =
@@ -141,13 +143,17 @@ async function dispatchRemoteCli(
runtimeAuthority: RemoteOrcaCliRequest['runtimeAuthority']
): Promise<RpcResponse> {
const command = parsed.commandPath.join(' ')
const compatibilityInvocationId = randomUUID()
const inheritedEvidence = readOrchestrationCompatibilityEvidence(env)
const orchestrationCompatibilityEvidence = runtimeAuthority
? { ...inheritedEvidence, host: runtimeAuthority }
: inheritedEvidence
const compatibilityEnvelope: RuntimeOrchestrationEnvelope = {
compatibilityInvocationId,
compatibilityInvocationId: randomUUID(),
orchestrationRequestId:
optionalRemoteCliString(parsed.flags, 'retry-request') ??
(command === 'orchestration check' || command === 'orchestration ask'
? randomUUID()
: undefined),
orchestrationCompatibilityEvidence
}
const linearResponse = await tryDispatchRemoteLinearCli(dispatcher, parsed, env, stdin)
@@ -201,8 +207,7 @@ async function dispatchRemoteCli(
},
{
...compatibilityEnvelope,
orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability'),
orchestrationRequestId: optionalRemoteCliString(parsed.flags, 'retry-request')
orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability')
}
)
}
@@ -215,15 +220,39 @@ async function dispatchRemoteCli(
terminalPaneKey: parsed.flags.has('terminal')
? undefined
: env.ORCA_PANE_KEY || undefined,
unread: parsed.flags.has('unread') ? true : undefined,
unread: parsed.flags.has('unread') ? true : parsed.flags.has('peek') ? false : undefined,
peek: parsed.flags.has('peek') ? true : undefined,
all: parsed.flags.has('all') ? true : undefined,
types: optionalRemoteCliString(parsed.flags, 'types'),
format: parsed.flags.has('format') ? true : undefined,
inject: parsed.flags.has('inject') ? true : undefined,
compatibilityCliCommand: 'orca',
run: optionalRemoteCliString(parsed.flags, 'run'),
ack: optionalRemoteCliString(parsed.flags, 'ack'),
wait: parsed.flags.has('wait') ? true : undefined,
timeoutMs: optionalRemoteCliNumber(parsed.flags, 'timeout-ms')
},
compatibilityEnvelope
)
case 'orchestration ask':
return await call(
dispatcher,
'orchestration.ask',
{
to: optionalRemoteCliString(parsed.flags, 'to'),
question: optionalRemoteCliString(parsed.flags, 'question'),
resume: optionalRemoteCliString(parsed.flags, 'resume'),
options: optionalRemoteCliString(parsed.flags, 'options'),
timeoutMs: optionalRemoteCliNumber(parsed.flags, 'timeout-ms'),
from: resolveRemoteCliHandle(parsed.flags, env, 'from'),
run: optionalRemoteCliString(parsed.flags, 'run'),
compatibilityCliCommand: 'orca'
},
{
...compatibilityEnvelope,
orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability')
}
)
case 'orchestration reply':
return await call(
dispatcher,
@@ -271,7 +300,8 @@ async function call(
? ORCHESTRATION_CONTRACT_VERSION
: undefined,
orchestrationRequestId: envelope?.orchestrationRequestId,
compatibilityInvocationId: envelope?.compatibilityInvocationId,
compatibilityInvocationId:
envelope?.orchestrationRequestId ?? envelope?.compatibilityInvocationId,
orchestrationCompatibilityEvidence: envelope?.orchestrationCompatibilityEvidence
})
}
@@ -0,0 +1,66 @@
import type { RpcResponse } from '../runtime/rpc/core'
import { formatRemoteCli } from './ssh-remote-cli-format'
import { hasRemoteLifecycleRejection } from './ssh-remote-orchestration-send'
export function formatRemoteOrchestrationAsk(
response: RpcResponse,
json: boolean
): { stdout: string; stderr: string } {
if (!response.ok || !isRecord(response.result)) {
return json
? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' }
: formatRemoteCli(response)
}
if (json) {
return { stdout: `${JSON.stringify(response.result)}\n`, stderr: '' }
}
if (isRecord(response.result.legacyCompatibility)) {
const compatibility = response.result.legacyCompatibility
if (compatibility.resumeRequired === true && typeof compatibility.resumeCommand === 'string') {
return {
stdout:
`Question ${String(response.result.messageId ?? 'unknown')} committed.\n` +
`Resume with: ${compatibility.resumeCommand}\n`,
stderr: ''
}
}
}
const answer = typeof response.result.answer === 'string' ? response.result.answer : ''
const thread = typeof response.result.threadId === 'string' ? response.result.threadId : 'unknown'
const timeoutMs =
typeof response.result.timeoutMs === 'number' ? response.result.timeoutMs : undefined
const stderr = response.result.timedOut
? `ask timeout after ${timeoutMs ?? 0}ms (thread ${thread})\n`
: response.result.cancelled
? response.result.connectionLost
? `ask connection closed (question ${String(response.result.messageId ?? 'unknown')})\n`
: `ask cancelled (question ${String(response.result.messageId ?? 'unknown')})\n`
: ''
return { stdout: answer ? `${answer}\n` : '', stderr }
}
export function getRemoteCliExitCode(command: string, response: RpcResponse): number {
if (!response.ok || hasRemoteLifecycleRejection(response.result)) {
return 1
}
if (
command === 'orchestration ask' &&
isRecord(response.result) &&
isRecord(response.result.legacyCompatibility) &&
response.result.legacyCompatibility.resumeRequired === true
) {
return 75
}
if (
command === 'orchestration ask' &&
isRecord(response.result) &&
(response.result.timedOut === true || response.result.cancelled === true)
) {
return 1
}
return 0
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
}
@@ -0,0 +1,48 @@
import {
formatOrchestrationCheckText,
prepareOrchestrationCheckOutput,
type OrchestrationCheckOutput,
type OrchestrationMessageSummary
} from '../../shared/orchestration-check-output'
import type { RpcResponse } from '../runtime/rpc/core'
import { formatRemoteCli } from './ssh-remote-cli-format'
export function formatRemoteOrchestrationCheck(
response: RpcResponse,
json: boolean,
terminal: string,
formattedRequested: boolean
): { stdout: string; stderr: string } {
if (!response.ok || !isRecord(response.result)) {
return json
? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' }
: formatRemoteCli(response)
}
const messages = Array.isArray(response.result.messages)
? response.result.messages.filter(isMessageSummary)
: []
const count = typeof response.result.count === 'number' ? response.result.count : messages.length
const result = prepareOrchestrationCheckOutput(
{ ...(response.result as OrchestrationCheckOutput), messages, count },
terminal,
formattedRequested
)
if (json) {
return {
stdout: `${JSON.stringify({ ...response, result }, null, 2)}\n`,
stderr: ''
}
}
return {
stdout: `${formatOrchestrationCheckText(result, terminal)}\n`,
stderr: ''
}
}
function isMessageSummary(value: unknown): value is OrchestrationMessageSummary {
return isRecord(value) && typeof value.id === 'string' && typeof value.from_handle === 'string'
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
}
@@ -0,0 +1,394 @@
import { createHash } from 'node:crypto'
import { describe, expect, it, vi } from 'vitest'
vi.mock('electron', () => ({
app: {
isPackaged: false,
getAppPath: () => '/host/app'
}
}))
vi.mock('../persistence', () => ({
getCanonicalUserDataPath: () => '/host/user-data'
}))
import { OrchestrationDb } from '../runtime/orchestration/db'
import { OrcaRuntimeService } from '../runtime/orca-runtime'
import type Database from '../sqlite/sync-database'
import type { HostCliPassthroughOptions } from './ssh-remote-cli-host-passthrough'
import { runRemoteOrcaCli } from './ssh-remote-orca-cli'
import { acknowledgeRemoteOrcaCliPostOutput } from './ssh-remote-orchestration-post-output'
const LEGACY_FALLBACK_OPTIONS: HostCliPassthroughOptions = {
execPath: '/host/electron',
cliEntryPath: '/host/app/out/cli/index.js',
userDataPath: '/host/user-data',
entryExists: () => false
}
const WORKER_HANDLE = 'term_legacy_ssh_worker'
const WORKER_PANE = 'tab_legacy_ssh:33333333-3333-4333-8333-333333333333'
const COORDINATOR_HANDLE = 'term_legacy_ssh_coord'
const COORDINATOR_PANE = 'tab_legacy_coord:44444444-4444-4444-8444-444444444444'
const WORKER_ENV = {
ORCA_TERMINAL_HANDLE: WORKER_HANDLE,
ORCA_PANE_KEY: WORKER_PANE,
ORCA_AGENT_LAUNCH_TOKEN: 'legacy-ssh-token'
}
const COORDINATOR_ENV = {
ORCA_TERMINAL_HANDLE: COORDINATOR_HANDLE,
ORCA_PANE_KEY: COORDINATOR_PANE,
ORCA_AGENT_LAUNCH_TOKEN: 'legacy-ssh-coordinator-token'
}
const RUNTIME_AUTHORITY = {
kind: 'ssh' as const,
targetId: 'saved-target',
connectionIncarnation: 'connection-1',
attachmentId: 'attachment-1'
}
function createLegacyRuntime() {
const db = new OrchestrationDb(':memory:')
const run = db.createRun({
objective: 'Adopted legacy SSH work',
coordinatorHandle: COORDINATOR_HANDLE,
coordinatorPaneKey: COORDINATOR_PANE
})
const task = db.createTask({
spec: 'legacy SSH assignment',
runId: run.id,
createdByTerminalHandle: COORDINATOR_HANDLE
})
const dispatch = db.createDispatchContext(task.id, WORKER_HANDLE, WORKER_PANE)
const sqlite = (db as unknown as { db: Database.Database }).db
sqlite
.prepare(
`UPDATE dispatch_contexts
SET contract_version = 0, process_incarnation = 'process-1'
WHERE id = ?`
)
.run(dispatch.id)
sqlite
.prepare(
`INSERT INTO legacy_adoptions (source_run_id, adopted_run_id, scheduler_state_lost)
VALUES ('run_legacy_local', ?, 1)`
)
.run(run.id)
const runtime = new OrcaRuntimeService()
runtime.setOrchestrationDb(db)
vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) =>
handle === WORKER_HANDLE ? WORKER_PANE : handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : null
)
vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation((evidence) => {
const worker =
evidence?.terminalHandle === WORKER_HANDLE &&
evidence.paneKey === WORKER_PANE &&
evidence.launchToken === WORKER_ENV.ORCA_AGENT_LAUNCH_TOKEN
const coordinator =
evidence?.terminalHandle === COORDINATOR_HANDLE &&
evidence.paneKey === COORDINATOR_PANE &&
evidence.launchToken === COORDINATOR_ENV.ORCA_AGENT_LAUNCH_TOKEN
if (
(!worker && !coordinator) ||
evidence.host?.kind !== 'ssh' ||
evidence.host.targetId !== RUNTIME_AUTHORITY.targetId ||
evidence.host.connectionIncarnation !== RUNTIME_AUTHORITY.connectionIncarnation
) {
return null
}
const identity = worker
? {
terminalHandle: WORKER_HANDLE,
paneKey: WORKER_PANE,
launchToken: WORKER_ENV.ORCA_AGENT_LAUNCH_TOKEN
}
: {
terminalHandle: COORDINATOR_HANDLE,
paneKey: COORDINATOR_PANE,
launchToken: COORDINATOR_ENV.ORCA_AGENT_LAUNCH_TOKEN
}
return {
hostScope: { kind: 'ssh', targetId: RUNTIME_AUTHORITY.targetId },
terminalHandle: identity.terminalHandle,
paneKey: identity.paneKey,
processIncarnation: 'process-1',
launchTokenHash: createHash('sha256').update(identity.launchToken).digest('hex')
}
})
vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {})
return { db, runtime, run, dispatch }
}
describe('legacy SSH orchestration fallback', () => {
it('acknowledges a consuming check only after remote output', async () => {
const { db, runtime, run } = createLegacyRuntime()
const message = db.insertMessage({
runId: run.id,
deliveryContract: 'legacy_direct',
from: COORDINATOR_HANDLE,
to: WORKER_HANDLE,
subject: 'retained SSH mail'
})
const request = {
argv: ['orchestration', 'check', '--unread', '--inject', '--json'],
cwd: '/home/alice/repo',
env: WORKER_ENV,
runtimeAuthority: RUNTIME_AUTHORITY
}
try {
const first = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
const replay = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
expect(JSON.parse(first.stdout)).toMatchObject({
result: {
messages: [{ id: message.id }],
formatted: expect.stringContaining('retained SSH mail')
}
})
expect(first.postOutput).toEqual({
kind: 'legacy_check_ack',
terminal: WORKER_HANDLE,
messageIds: [message.id]
})
expect(JSON.parse(replay.stdout)).toMatchObject({
result: { messages: [{ id: message.id }] }
})
expect(db.getMessageById(message.id)?.read).toBe(0)
await acknowledgeRemoteOrcaCliPostOutput(runtime, {
postOutput: first.postOutput!,
env: WORKER_ENV,
runtimeAuthority: RUNTIME_AUTHORITY
})
const afterAck = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
expect(JSON.parse(afterAck.stdout)).toMatchObject({ result: { messages: [], count: 0 } })
expect(afterAck.postOutput).toBeUndefined()
expect(db.getMessageById(message.id)?.read).toBe(1)
} finally {
db.close()
}
})
it('keeps peek formatted and non-consuming', async () => {
const { db, runtime, run } = createLegacyRuntime()
const message = db.insertMessage({
runId: run.id,
deliveryContract: 'legacy_direct',
from: COORDINATOR_HANDLE,
to: WORKER_HANDLE,
subject: 'inspect retained SSH mail'
})
try {
const peek = await runRemoteOrcaCli(
runtime,
{
argv: ['orchestration', 'check', '--peek', '--format', '--json'],
cwd: '/home/alice/repo',
env: WORKER_ENV,
runtimeAuthority: RUNTIME_AUTHORITY
},
LEGACY_FALLBACK_OPTIONS
)
const output = JSON.parse(peek.stdout) as {
result: { formatted: string }
}
expect(output).toMatchObject({
result: {
messages: [{ id: message.id }],
formatted: expect.stringContaining('inspect retained SSH mail'),
legacyCompatibility: { readOnly: true }
}
})
expect(output.result.formatted).toContain(`${message.id} [legacy, read-only]`)
expect(output.result.formatted).toContain(
'Inspection only: reply and acknowledgment are unavailable.'
)
expect(output.result.formatted).not.toContain('orchestration reply')
expect(peek.postOutput).toBeUndefined()
expect(db.getMessageById(message.id)?.read).toBe(0)
} finally {
db.close()
}
})
it('reads and acknowledges current Run delivery', async () => {
const { db, runtime, run } = createLegacyRuntime()
const message = db.insertMessage({
runId: run.id,
from: WORKER_HANDLE,
to: `run:${run.id}`,
subject: 'current Run mail'
})
const baseRequest = {
cwd: '/home/alice/repo',
env: COORDINATOR_ENV,
runtimeAuthority: RUNTIME_AUTHORITY
}
const spawn = vi.fn()
const hostCliAvailable = {
...LEGACY_FALLBACK_OPTIONS,
entryExists: () => true,
spawn: spawn as HostCliPassthroughOptions['spawn']
}
try {
const checked = await runRemoteOrcaCli(
runtime,
{
...baseRequest,
argv: ['orchestration', 'check', '--run', run.id]
},
hostCliAvailable
)
expect(checked.stdout).toContain('Delivery ')
expect(checked.stdout).toContain(`${message.id} [status] from=${WORKER_HANDLE}`)
expect(spawn).not.toHaveBeenCalled()
const checkedJson = await runRemoteOrcaCli(
runtime,
{
...baseRequest,
argv: ['orchestration', 'check', '--run', run.id, '--json']
},
hostCliAvailable
)
const result = JSON.parse(checkedJson.stdout) as {
result: { deliveryId: string; messages: { id: string }[] }
}
expect(result.result.messages).toEqual([expect.objectContaining({ id: message.id })])
const acknowledged = await runRemoteOrcaCli(
runtime,
{
...baseRequest,
argv: [
'orchestration',
'check',
'--run',
run.id,
'--ack',
result.result.deliveryId,
'--json'
]
},
hostCliAvailable
)
expect(JSON.parse(acknowledged.stdout)).toMatchObject({
result: { acknowledged: result.result.deliveryId }
})
expect(db.getMessageById(message.id)?.read).toBe(1)
} finally {
db.close()
}
})
it('replays an SSH legacy ask by retry request without creating another question', async () => {
const { db, runtime } = createLegacyRuntime()
const argv = [
'orchestration',
'ask',
'--to',
COORDINATOR_HANDLE,
'--question',
'Continue?',
'--timeout-ms',
'1',
'--retry-request',
'ssh-question-1',
'--json'
]
const request = {
argv,
cwd: '/home/alice/repo',
env: WORKER_ENV,
runtimeAuthority: RUNTIME_AUTHORITY
}
try {
const first = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
const replay = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
const sqlite = (db as unknown as { db: Database.Database }).db
const firstResult = JSON.parse(first.stdout) as { messageId: string; timedOut: boolean }
const replayResult = JSON.parse(replay.stdout) as { messageId: string; timedOut: boolean }
expect(first).toMatchObject({ exitCode: 1 })
expect(firstResult).toMatchObject({ messageId: expect.any(String), timedOut: true })
expect(replay).toMatchObject({ exitCode: 1 })
expect(replayResult).toMatchObject({ messageId: firstResult.messageId, timedOut: true })
expect(
(
sqlite.prepare('SELECT COUNT(*) AS count FROM question_threads').get() as {
count: number
}
).count
).toBe(1)
} finally {
db.close()
}
})
it('resumes and acknowledges a question answer', async () => {
const { db, runtime, run, dispatch } = createLegacyRuntime()
const pending = db.createQuestion({
runId: run.id,
dispatchId: dispatch.id,
askerHandle: WORKER_HANDLE,
question: 'Proceed?'
})
const answer = db.answerQuestion({
messageId: pending.question.message_id,
runId: run.id,
consumerGeneration: run.consumer_generation,
body: 'yes'
})
const sqlite = (db as unknown as { db: Database.Database }).db
sqlite
.prepare(
`UPDATE messages
SET delivery_contract = 'legacy_direct', read = 0
WHERE id = ?`
)
.run(answer.message.id)
try {
const result = await runRemoteOrcaCli(
runtime,
{
argv: ['orchestration', 'ask', '--resume', pending.question.message_id, '--json'],
cwd: '/home/alice/repo',
env: WORKER_ENV,
runtimeAuthority: RUNTIME_AUTHORITY
},
LEGACY_FALLBACK_OPTIONS
)
expect(result.exitCode).toBe(0)
expect(JSON.parse(result.stdout)).toMatchObject({
answer: 'yes',
messageId: pending.question.message_id,
timedOut: false
})
expect(result.postOutput).toEqual({
kind: 'legacy_question_ack',
terminal: WORKER_HANDLE,
questionId: pending.question.message_id,
answerMessageId: answer.message.id
})
expect(db.getMessageById(answer.message.id)?.read).toBe(0)
await acknowledgeRemoteOrcaCliPostOutput(runtime, {
postOutput: result.postOutput!,
env: WORKER_ENV,
runtimeAuthority: RUNTIME_AUTHORITY
})
expect(db.getMessageById(answer.message.id)?.read).toBe(1)
} finally {
db.close()
}
})
})
@@ -0,0 +1,143 @@
import { randomUUID } from 'node:crypto'
import { readOrchestrationCompatibilityEvidence } from '../../shared/orchestration-compatibility-evidence'
import { ORCHESTRATION_CONTRACT_VERSION } from '../../shared/protocol-version'
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
import type { RpcResponse } from '../runtime/rpc/core'
import { RpcDispatcher } from '../runtime/rpc/dispatcher'
import type {
RemoteOrcaCliPostOutput,
RemoteOrcaCliRequest
} from './ssh-remote-cli-host-passthrough'
import { RemoteCliArgumentError, type ParsedRemoteCli } from './ssh-remote-cli-argument-error'
import { optionalRemoteCliString, resolveRemoteCliHandle } from './ssh-remote-cli-args'
export async function acknowledgeRemoteOrcaCliPostOutput(
runtime: OrcaRuntimeService,
args: {
postOutput: RemoteOrcaCliPostOutput
env: Record<string, string>
runtimeAuthority?: RemoteOrcaCliRequest['runtimeAuthority']
}
): Promise<void> {
const inheritedEvidence = readOrchestrationCompatibilityEvidence(args.env)
const orchestrationCompatibilityEvidence = args.runtimeAuthority
? { ...inheritedEvidence, host: args.runtimeAuthority }
: inheritedEvidence
const params =
args.postOutput.kind === 'legacy_check_ack'
? {
terminal: args.postOutput.terminal,
compatibilityAck: JSON.stringify({
messageIds: args.postOutput.messageIds,
types: args.postOutput.types
})
}
: {
terminal: args.postOutput.terminal,
compatibilityQuestionAck: JSON.stringify({
questionId: args.postOutput.questionId,
answerMessageId: args.postOutput.answerMessageId
})
}
const response = await new RpcDispatcher({ runtime }).dispatch({
id: `remote-cli-post-output-${randomUUID()}`,
authToken: 'remote-cli',
method: 'orchestration.check',
params,
orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION,
compatibilityInvocationId: randomUUID(),
orchestrationCompatibilityEvidence
})
if (!response.ok) {
throw new Error(response.error.message)
}
}
export function parseRemoteOrcaCliPostOutput(value: unknown): RemoteOrcaCliPostOutput {
if (!isRecord(value) || typeof value.kind !== 'string' || typeof value.terminal !== 'string') {
throw invalidPostOutput()
}
if (
value.kind === 'legacy_check_ack' &&
Array.isArray(value.messageIds) &&
value.messageIds.every((id) => typeof id === 'string') &&
(value.types === undefined ||
(Array.isArray(value.types) && value.types.every((type) => typeof type === 'string')))
) {
return {
kind: value.kind,
terminal: value.terminal,
messageIds: value.messageIds,
...(value.types === undefined ? {} : { types: value.types })
}
}
if (
value.kind === 'legacy_question_ack' &&
typeof value.questionId === 'string' &&
typeof value.answerMessageId === 'string'
) {
return {
kind: value.kind,
terminal: value.terminal,
questionId: value.questionId,
answerMessageId: value.answerMessageId
}
}
throw invalidPostOutput()
}
export function getRemoteCliPostOutput(
parsed: ParsedRemoteCli,
env: Record<string, string>,
response: RpcResponse
): RemoteOrcaCliPostOutput | undefined {
if (!response.ok || !isRecord(response.result)) {
return undefined
}
const compatibility = response.result.legacyCompatibility
if (!isRecord(compatibility)) {
return undefined
}
const command = parsed.commandPath.join(' ')
if (
command === 'orchestration check' &&
Array.isArray(compatibility.ackMessageIds) &&
compatibility.ackMessageIds.length > 0 &&
compatibility.ackMessageIds.every((id) => typeof id === 'string')
) {
const types = optionalRemoteCliString(parsed.flags, 'types')
?.split(',')
.map((type) => type.trim())
.filter(Boolean)
return {
kind: 'legacy_check_ack',
terminal: resolveRemoteCliHandle(parsed.flags, env, 'terminal'),
messageIds: compatibility.ackMessageIds,
...(types ? { types } : {})
}
}
const acknowledgement = compatibility.answerAcknowledgement
if (
command === 'orchestration ask' &&
response.result.answer !== null &&
isRecord(acknowledgement) &&
typeof acknowledgement.questionId === 'string' &&
typeof acknowledgement.answerMessageId === 'string'
) {
return {
kind: 'legacy_question_ack',
terminal: resolveRemoteCliHandle(parsed.flags, env, 'from'),
questionId: acknowledgement.questionId,
answerMessageId: acknowledgement.answerMessageId
}
}
return undefined
}
function invalidPostOutput(): RemoteCliArgumentError {
return new RemoteCliArgumentError('invalid_argument', 'Invalid SSH CLI post-output action.')
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
}
@@ -829,4 +829,73 @@ describe('attachMainWindowServices', () => {
await expect(revealPromise).resolves.toEqual({ tabId: 'tab-1', title: 'SSH tmux' })
expect(removeListenerMock).toHaveBeenCalledWith('terminal:tabCreateReply', handler)
})
it('requires an exact renderer identity receipt for recovered worker reveals', async () => {
const sendMock = vi.fn()
const mainWindow = createMainWindow({ send: sendMock })
const runtime = createRuntime()
attachMainWindowServices(mainWindow as never, createStore(), runtime as never)
const notifier = runtime.setNotifier.mock.calls[0][0] as {
revealTerminalSession: (
worktreeId: string,
opts: {
ptyId: string
tabId: string
leafId: string
expectedProcessIdentity: { terminalHandle: string; incarnationId: string }
}
) => Promise<unknown>
}
const opts = {
ptyId: 'pty-worker',
tabId: 'tab-worker',
leafId: 'leaf-worker',
expectedProcessIdentity: {
terminalHandle: 'term_worker',
incarnationId: 'inc-worker'
}
}
const mismatch = notifier.revealTerminalSession('worktree-1', opts)
const mismatchPayload = sendMock.mock.calls.at(-1)?.[1]
const mismatchHandler = onMock.mock.calls.findLast(
([channel]) => channel === 'terminal:tabCreateReply'
)?.[1]
mismatchHandler?.(
{ sender: mainWindow.webContents },
{
requestId: mismatchPayload.requestId,
tabId: 'tab-worker',
identity: {
worktreeId: 'worktree-1',
tabId: 'tab-worker',
leafId: 'leaf-worker',
ptyId: 'pty-replacement'
}
}
)
await expect(mismatch).rejects.toThrow('terminal_reveal_identity_mismatch')
const exact = notifier.revealTerminalSession('worktree-1', opts)
const exactPayload = sendMock.mock.calls.at(-1)?.[1]
const exactHandler = onMock.mock.calls.findLast(
([channel]) => channel === 'terminal:tabCreateReply'
)?.[1]
const identity = {
worktreeId: 'worktree-1',
tabId: 'tab-worker',
leafId: 'leaf-worker',
ptyId: 'pty-worker'
}
exactHandler?.(
{ sender: mainWindow.webContents },
{ requestId: exactPayload.requestId, tabId: 'tab-worker', identity }
)
await expect(exact).resolves.toEqual({
tabId: 'tab-worker',
title: undefined,
identity
})
})
})
+27 -5
View File
@@ -49,6 +49,7 @@ import type {
RuntimeMarkdownSaveTabResult
} from '../../shared/mobile-markdown-document'
import type { RuntimeMobileSessionTabMove } from '../../shared/runtime-types'
import type { TerminalTabCreateReply } from '../../shared/terminal-reveal-identity'
import { isNativeFileDropPayload, type NativeFileDropPayload } from '../../shared/native-file-drop'
import { requestMobileMarkdownFromRenderer } from './mobile-markdown-request-relay'
import { requestTerminalTabCloseFromRenderer } from './terminal-tab-close-request-relay'
@@ -347,14 +348,20 @@ function registerRuntimeWindowLifecycle(
revealTerminalSession: (worktreeId, opts) =>
new Promise((resolve, reject) => {
const requestId = randomUUID()
const expectedIdentity = opts.expectedProcessIdentity
? opts.tabId && opts.leafId
? { worktreeId, tabId: opts.tabId, leafId: opts.leafId, ptyId: opts.ptyId }
: null
: undefined
if (expectedIdentity === null) {
reject(new Error('terminal_reveal_identity_required'))
return
}
const timer = setTimeout(() => {
ipcMain.removeListener('terminal:tabCreateReply', handler)
reject(new Error('Terminal reveal timed out'))
}, 10_000)
const handler = (
event: Electron.IpcMainEvent,
reply: { requestId: string; tabId?: string; title?: string; error?: string }
): void => {
const handler = (event: Electron.IpcMainEvent, reply: TerminalTabCreateReply): void => {
// Why: requestId is renderer-supplied, so only the targeted main window may satisfy the reveal.
if (event.sender !== mainWindow.webContents || reply.requestId !== requestId) {
return
@@ -365,7 +372,22 @@ function registerRuntimeWindowLifecycle(
reject(new Error(reply.error))
return
}
resolve({ tabId: reply.tabId!, title: reply.title })
if (
expectedIdentity &&
(!reply.identity ||
reply.identity.worktreeId !== expectedIdentity.worktreeId ||
reply.identity.tabId !== expectedIdentity.tabId ||
reply.identity.leafId !== expectedIdentity.leafId ||
reply.identity.ptyId !== expectedIdentity.ptyId)
) {
reject(new Error('terminal_reveal_identity_mismatch'))
return
}
resolve({
tabId: reply.tabId!,
title: reply.title,
...(reply.identity ? { identity: reply.identity } : {})
})
}
ipcMain.on('terminal:tabCreateReply', handler)
send('ui:createTerminal', {
+2 -6
View File
@@ -18,6 +18,7 @@ import type {
TerminalTabCloseRequest,
TerminalTabCloseResponse
} from '../shared/terminal-tab-close'
import type { TerminalTabCreateReply } from '../shared/terminal-reveal-identity'
import type {
LocalLogTailChangedPayload,
LocalLogTailReadArgs,
@@ -3149,12 +3150,7 @@ export type PreloadApi = {
onRequestTerminalTabMount: (
callback: (data: { worktreeId: string; tabId?: string; ptyId?: string }) => void
) => () => void
replyTerminalCreate: (reply: {
requestId: string
tabId?: string
title?: string
error?: string
}) => void
replyTerminalCreate: (reply: TerminalTabCreateReply) => void
onSplitTerminal: (
callback: (data: {
tabId: string
+2 -6
View File
@@ -13,6 +13,7 @@ import type { CliInstallStatus } from '../shared/cli-install-types'
import type { AgentHookInstallStatus } from '../shared/agent-hook-types'
import type { CodexConfigSyncStatus } from '../shared/codex-config-sync-types'
import type { TerminalPaneSplitSource } from '../shared/feature-education-telemetry'
import type { TerminalTabCreateReply } from '../shared/terminal-reveal-identity'
import type { ProjectExecutionRuntimeResolution } from '../shared/project-execution-runtime'
import type { StartupCommandDelivery } from '../shared/codex-startup-delivery'
import type {
@@ -3752,12 +3753,7 @@ const api = {
ipcRenderer.on('terminal:requestTabMount', listener)
return () => ipcRenderer.removeListener('terminal:requestTabMount', listener)
},
replyTerminalCreate: (reply: {
requestId: string
tabId?: string
title?: string
error?: string
}): void => {
replyTerminalCreate: (reply: TerminalTabCreateReply): void => {
ipcRenderer.send('terminal:tabCreateReply', reply)
},
onSplitTerminal: (
+85 -11
View File
@@ -198,6 +198,8 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise<void> {
let nextSeq = 1
let highestReceivedSeq = 0
const requestId = 1
const postOutputRequestId = 2
let initialExitCode = 0
const sendRequest = (): void => {
const env = pickRemoteCliEnv(process.env)
@@ -219,6 +221,59 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise<void> {
sock.write(frame)
}
const finish = (exitCode: number): void => {
sock.destroy()
process.exit(exitCode)
}
const sendPostOutput = (postOutput: unknown): void => {
sock.write(
encodeJsonRpcFrame(
{
jsonrpc: '2.0',
id: postOutputRequestId,
method: 'orca.cli.postOutput',
params: { postOutput, env: pickRemoteCliEnv(process.env) }
},
nextSeq++,
highestReceivedSeq
)
)
}
const writeOutput = (
result: { stdout?: unknown; stderr?: unknown },
onFlushed: (error?: Error) => void
): void => {
const writes = [
[process.stdout, result.stdout],
[process.stderr, result.stderr]
].filter((entry): entry is [NodeJS.WriteStream, string] => typeof entry[1] === 'string')
if (writes.length === 0) {
onFlushed()
return
}
let pending = writes.length
let completed = false
for (const [stream, output] of writes) {
stream.write(output, (error) => {
if (completed) {
return
}
if (error) {
completed = true
onFlushed(error)
return
}
pending -= 1
if (pending === 0) {
completed = true
onFlushed()
}
})
}
}
const decoder = new FrameDecoder((frame: DecodedFrame) => {
if (frame.id > highestReceivedSeq) {
highestReceivedSeq = frame.id
@@ -227,28 +282,41 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise<void> {
return
}
const msg = parseJsonRpcMessage(frame.payload)
if (!('id' in msg) || msg.id !== requestId || !('result' in msg || 'error' in msg)) {
if (
!('id' in msg) ||
(msg.id !== requestId && msg.id !== postOutputRequestId) ||
!('result' in msg || 'error' in msg)
) {
return
}
const response = msg as JsonRpcResponse
if (response.error) {
process.stderr.write(`${response.error.message}\n`)
sock.destroy()
process.exit(1)
finish(1)
return
}
if (response.id === postOutputRequestId) {
finish(initialExitCode)
return
}
const result = (response.result ?? {}) as {
stdout?: unknown
stderr?: unknown
exitCode?: unknown
postOutput?: unknown
}
if (typeof result.stdout === 'string' && result.stdout.length > 0) {
process.stdout.write(result.stdout)
}
if (typeof result.stderr === 'string' && result.stderr.length > 0) {
process.stderr.write(result.stderr)
}
sock.destroy()
process.exit(typeof result.exitCode === 'number' ? result.exitCode : 0)
initialExitCode = typeof result.exitCode === 'number' ? result.exitCode : 0
writeOutput(result, (error) => {
if (error) {
finish(1)
return
}
if (result.postOutput === undefined) {
finish(initialExitCode)
return
}
sendPostOutput(result.postOutput)
})
})
const connectTimeout = setTimeout(() => {
@@ -443,6 +511,12 @@ async function main(): Promise<void> {
timeoutMs: remoteCliRequestTimeoutMs(params)
})
})
dispatcher.onRequest('orca.cli.postOutput', async (params, context) => {
return await dispatcher.requestAnyClient('orca.cli.postOutput', params, {
excludeClientId: context.clientId,
timeoutMs: remoteCliRequestTimeoutMs(params)
})
})
function configureRelayGraceTime(params: Record<string, unknown>): { graceTimeMs: number } {
const seconds = Number(params.graceTimeSeconds)
+60 -10
View File
@@ -1504,7 +1504,6 @@ describe('useIpcEvents updater integration', () => {
openAgentTabsInChatByDefault: false
}
}
vi.doMock('react', async () => {
const actual = await vi.importActual<typeof ReactModule>('react')
return {
@@ -1789,7 +1788,11 @@ describe('useIpcEvents updater integration', () => {
})
it('surfaces terminal creates without stealing focus unless requested', async () => {
const createTab = vi.fn(() => ({ id: 'tab-new' }))
const createTab = vi.fn(
(_worktreeId: string, _groupId?: string, _tabType?: string, options?: { id?: string }) => ({
id: options?.id ?? 'tab-new'
})
)
const setActiveView = vi.fn()
const setActiveWorktree = vi.fn()
const markWorktreeVisited = vi.fn()
@@ -1876,6 +1879,20 @@ describe('useIpcEvents updater integration', () => {
activeRuntimeEnvironmentId: undefined as string | undefined
}
}
updateTabPtyId.mockImplementation((tabId: string, ptyId: string) => {
storeState.ptyIdsByTabId[tabId] = [
...new Set([...(storeState.ptyIdsByTabId[tabId] ?? []), ptyId])
]
for (const tabs of Object.values(storeState.tabsByWorktree)) {
const tab = tabs.find((candidate) => candidate.id === tabId)
if (tab) {
tab.ptyId = ptyId
}
}
})
setTabLayout.mockImplementation((tabId: string, layout: unknown) => {
storeState.terminalLayoutsByTabId[tabId] = layout
})
const createTerminalListenerRef: {
current:
| ((data: {
@@ -2280,6 +2297,7 @@ describe('useIpcEvents updater integration', () => {
setActiveTabType.mockClear()
setActiveTab.mockClear()
revealWorktreeInSidebar.mockClear()
dispatchEvent.mockClear()
createTerminalListenerRef.current({
worktreeId: 'wt-2',
title: 'Runner',
@@ -2588,6 +2606,7 @@ describe('useIpcEvents updater integration', () => {
setActiveTab.mockClear()
revealWorktreeInSidebar.mockClear()
replyTerminalCreate.mockClear()
dispatchEvent.mockClear()
requestTerminalCreateListenerRef.current({
requestId: 'req-renderer-backed-background',
worktreeId: 'wt-2',
@@ -2605,6 +2624,12 @@ describe('useIpcEvents updater integration', () => {
expect(setActiveTabType).not.toHaveBeenCalled()
expect(setActiveTab).not.toHaveBeenCalled()
expect(revealWorktreeInSidebar).not.toHaveBeenCalled()
expect(dispatchEvent).toHaveBeenCalledWith(
expect.objectContaining({
type: 'orca-background-mount-terminal-worktree',
detail: { worktreeId: 'wt-2', tabIds: ['tab-new'] }
})
)
expect(replyTerminalCreate).toHaveBeenCalledWith({
requestId: 'req-renderer-backed-background',
tabId: 'tab-new',
@@ -2708,6 +2733,7 @@ describe('useIpcEvents updater integration', () => {
setActiveTabType.mockClear()
setActiveTab.mockClear()
revealWorktreeInSidebar.mockClear()
dispatchEvent.mockClear()
createTerminalListenerRef.current({
worktreeId: 'wt-2',
ptyId: 'pty-bg-3',
@@ -2725,6 +2751,13 @@ describe('useIpcEvents updater integration', () => {
expect(setActiveTabType).not.toHaveBeenCalled()
expect(setActiveTab).not.toHaveBeenCalled()
expect(revealWorktreeInSidebar).not.toHaveBeenCalled()
expect(dispatchEvent).toHaveBeenCalledOnce()
expect(dispatchEvent).toHaveBeenCalledWith(
expect.objectContaining({
type: 'orca-background-mount-terminal-worktree',
detail: { worktreeId: 'wt-2', tabIds: ['tab-cli-bg-reveal'] }
})
)
createTab.mockClear()
setActiveView.mockClear()
@@ -2734,6 +2767,7 @@ describe('useIpcEvents updater integration', () => {
revealWorktreeInSidebar.mockClear()
focusRuntimeTerminalSurface.mockClear()
focusTerminalTabSurface.mockClear()
dispatchEvent.mockClear()
createTerminalListenerRef.current({
worktreeId: 'wt-2',
ptyId: 'pty-recovery-bg',
@@ -2754,6 +2788,13 @@ describe('useIpcEvents updater integration', () => {
expect(revealWorktreeInSidebar).not.toHaveBeenCalled()
expect(focusRuntimeTerminalSurface).not.toHaveBeenCalled()
expect(focusTerminalTabSurface).not.toHaveBeenCalled()
expect(dispatchEvent).toHaveBeenCalledOnce()
expect(dispatchEvent).toHaveBeenCalledWith(
expect.objectContaining({
type: 'orca-background-mount-terminal-worktree',
detail: { worktreeId: 'wt-2', tabIds: ['tab-recovery-bg'] }
})
)
storeState.tabsByWorktree = {
'wt-2': [{ id: 'tab-existing', ptyId: 'pty-bg', title: 'Terminal 1' }]
@@ -2836,7 +2877,13 @@ describe('useIpcEvents updater integration', () => {
expect(replyTerminalCreate).toHaveBeenCalledWith({
requestId: 'req-adopt-pending',
tabId: pendingTabId,
title: 'Terminal 3'
title: 'Terminal 3',
identity: {
worktreeId: 'wt-2',
tabId: pendingTabId,
leafId: pendingLeafId,
ptyId: 'serve-cf39bedb-a33a-417c-9ab6-f304dc27a6c0'
}
})
storeState.tabsByWorktree = {
@@ -2902,7 +2949,13 @@ describe('useIpcEvents updater integration', () => {
expect(replyTerminalCreate).toHaveBeenCalledWith({
requestId: 'req-split',
tabId: 'tab-existing',
title: 'Terminal 1'
title: 'Terminal 1',
identity: {
worktreeId: 'wt-2',
tabId: 'tab-existing',
leafId: 'leaf-split',
ptyId: 'pty-split'
}
})
storeState.terminalLayoutsByTabId = {
@@ -5234,7 +5287,7 @@ describe('useIpcEvents agent status snapshot integration', () => {
vi.unstubAllGlobals()
})
it('reconciles adopted and exited legacy worker recovery without broad record cleanup', async () => {
it('retires the exact sleeping record after adopted or exited legacy worker recovery', async () => {
const clearSleepingAgentSession = vi.fn()
const setSleepingAgentAutomaticResumeBlocked = vi.fn()
let listener:
@@ -5275,11 +5328,8 @@ describe('useIpcEvents agent status snapshot integration', () => {
clearSleepingAgentSession.mockClear()
listener?.({ paneKey: 'tab-exited:leaf-exited', resolution: 'exited' })
expect(clearSleepingAgentSession).not.toHaveBeenCalled()
expect(setSleepingAgentAutomaticResumeBlocked).toHaveBeenCalledWith(
'tab-exited:leaf-exited',
false
)
expect(clearSleepingAgentSession).toHaveBeenCalledWith('tab-exited:leaf-exited')
expect(setSleepingAgentAutomaticResumeBlocked).not.toHaveBeenCalled()
})
it.each([
+17 -8
View File
@@ -101,6 +101,7 @@ import { track } from '@/lib/telemetry'
import { singlePaneLayoutSnapshot } from '@/store/slices/terminal-helpers'
import { buildWorkspaceSessionPayload } from '@/lib/workspace-session'
import { persistWorkspaceSessionByHost } from '@/lib/workspace-session-host-persistence'
import { verifyTerminalRevealIdentity } from '@/lib/terminal-reveal-identity'
import { getLinearIssueWorkspaceName } from '../../../shared/workspace-name'
import type { RuntimeClientEvent } from '../../../shared/runtime-client-events'
import { applyHostWorktreeTerminalSleepState } from '@/components/terminal-pane/pty-shutdown-exit-deferral'
@@ -1576,11 +1577,24 @@ export function useIpcEvents(): void {
...(launchAgent ? { launchAgent } : {})
})
}
if (ptyId && terminalPresentation === 'background') {
requestBackgroundTerminalWorktreeMount({ worktreeId, tabIds: [tab.id] })
}
if (requestId) {
const identity =
ptyId && tabId && leafId
? verifyTerminalRevealIdentity(useAppStore.getState(), {
worktreeId,
tabId,
leafId,
ptyId
})
: undefined
window.api.ui.replyTerminalCreate({
requestId,
tabId: tab.id,
title: title ?? tab.title
title: title ?? tab.title,
...(identity ? { identity } : {})
})
}
} catch (err) {
@@ -3348,13 +3362,8 @@ export function useIpcEvents(): void {
unsubs.push(unsubscribeMigrationUnsupportedClear)
}
const unsubscribeLegacyWorkerTerminalRecovery =
window.api.agentStatus.onLegacyWorkerTerminalRecovery?.(({ paneKey, resolution }) => {
const store = useAppStore.getState()
if (resolution === 'adopted') {
store.clearSleepingAgentSession(paneKey)
return
}
store.setSleepingAgentAutomaticResumeBlocked(paneKey, false)
window.api.agentStatus.onLegacyWorkerTerminalRecovery?.(({ paneKey }) => {
useAppStore.getState().clearSleepingAgentSession(paneKey)
})
if (unsubscribeLegacyWorkerTerminalRecovery) {
unsubs.push(unsubscribeLegacyWorkerTerminalRecovery)
@@ -11,7 +11,7 @@ afterEach(() => {
})
describe('legacy worker sleeping-session recovery', () => {
it('keeps the session dormant until exited recovery removes only its resume block', () => {
it('never resumes a proven-exited legacy worker on workspace activation', () => {
const record: SleepingAgentSessionRecord = {
paneKey: 'tab-legacy:leaf-legacy',
tabId: 'tab-legacy',
@@ -46,16 +46,8 @@ describe('legacy worker sleeping-session recovery', () => {
expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(0)
expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey]).toBe(record)
useAppStore.getState().setSleepingAgentAutomaticResumeBlocked(record.paneKey, false)
const unblocked = useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey]
expect(unblocked).toMatchObject({
paneKey: record.paneKey,
providerSession: record.providerSession,
origin: 'live'
})
expect(unblocked?.automaticResumeBlockedBy).toBeUndefined()
expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(1)
useAppStore.getState().clearSleepingAgentSession(record.paneKey)
expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(0)
expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey]).toBeUndefined()
})
})
@@ -0,0 +1,65 @@
import { describe, expect, it } from 'vitest'
import { verifyTerminalRevealIdentity } from './terminal-reveal-identity'
const expected = {
worktreeId: 'worktree-1',
tabId: 'tab-1',
leafId: 'leaf-1',
ptyId: 'pty-1'
}
const mismatchCases: {
name: string
state: Parameters<typeof verifyTerminalRevealIdentity>[0]
}[] = [
{
name: 'worktree',
state: {
tabsByWorktree: { 'worktree-2': [{ id: 'tab-1' }] },
terminalLayoutsByTabId: {
'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-1' } }
}
}
},
{
name: 'leaf',
state: {
tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] },
terminalLayoutsByTabId: {
'tab-1': { ptyIdsByLeafId: { 'leaf-2': 'pty-1' } }
}
}
},
{
name: 'pty',
state: {
tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] },
terminalLayoutsByTabId: {
'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-2' } }
}
}
}
]
describe('verifyTerminalRevealIdentity', () => {
it('returns the exact fresh renderer binding', () => {
expect(
verifyTerminalRevealIdentity(
{
tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] },
terminalLayoutsByTabId: {
'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-1' } }
}
},
expected
)
).toEqual(expected)
})
it.each(mismatchCases)('rejects a mismatched $name owner', ({ state }) => {
expect(() => verifyTerminalRevealIdentity(state, expected)).toThrow(
'terminal_reveal_identity_mismatch'
)
})
})
@@ -0,0 +1,23 @@
import type { TerminalRevealIdentity } from '../../../shared/terminal-reveal-identity'
type TerminalRevealState = {
tabsByWorktree: Record<string, readonly { id: string }[]>
terminalLayoutsByTabId: Record<
string,
{ ptyIdsByLeafId?: Record<string, string | undefined> } | undefined
>
}
export function verifyTerminalRevealIdentity(
state: TerminalRevealState,
expected: TerminalRevealIdentity
): TerminalRevealIdentity {
const ownsTab = state.tabsByWorktree[expected.worktreeId]?.some(
(tab) => tab.id === expected.tabId
)
const boundPtyId = state.terminalLayoutsByTabId[expected.tabId]?.ptyIdsByLeafId?.[expected.leafId]
if (!ownsTab || boundPtyId !== expected.ptyId) {
throw new Error('terminal_reveal_identity_mismatch')
}
return expected
}
@@ -0,0 +1,42 @@
import { describe, expect, it } from 'vitest'
import { prepareOrchestrationCheckOutput } from './orchestration-check-output'
describe('prepareOrchestrationCheckOutput', () => {
it('keeps mixed read-only mail safe and current Run replies executable', () => {
const prepared = prepareOrchestrationCheckOutput(
{
count: 2,
messages: [
{
id: 'msg_current',
run_id: 'run_adopted',
delivery_contract: 'current_delivery',
from_handle: 'term_worker',
to_handle: 'run:run_adopted',
subject: 'Question'
},
{
id: 'msg_legacy',
run_id: 'run_legacy_local',
delivery_contract: 'audit_only',
from_handle: 'term_legacy',
to_handle: 'term_coord',
subject: 'Old reply'
}
],
formatted: '[Reply: unsafe stale formatter output]'
},
'term_current_coord',
true
)
expect(prepared.formatted).toContain(
'[Reply: orca orchestration reply --id msg_current --body "..."]'
)
expect(prepared.formatted).not.toContain('--from run:run_adopted')
expect(prepared.formatted).toContain(
'[Inspection only: reply and acknowledgment are unavailable.]'
)
expect(prepared.formatted).not.toContain('unsafe stale formatter output')
})
})
+199
View File
@@ -0,0 +1,199 @@
import { ORCHESTRATION_LEGACY_RUN_ID } from './orchestration-rpc-contract'
export type OrchestrationMessageSummary = {
id: string
run_id?: string
delivery_contract?: 'legacy_direct' | 'current_delivery' | 'audit_only'
from_handle: string
to_handle?: string
subject?: string
type?: string
body?: string
payload?: string | null
priority?: string
read?: number
}
export type LegacyCompatibilityResult = {
recovery?: boolean
readOnly?: boolean
ackMessageIds?: string[]
answerAcknowledgement?: {
questionId: string
answerMessageId: string
}
currentDelivery?: {
runId: string
checkCommand: string
ackCommand: string
}
resumeRequired?: boolean
resumeCommand?: string
}
export type OrchestrationCheckOutput = {
messages: OrchestrationMessageSummary[]
count: number
formatted?: string
deliveryId?: string | null
timedOut?: boolean
cancelled?: boolean
connectionLost?: boolean
legacyCompatibility?: LegacyCompatibilityResult
}
export function formatMessageReadOnlyTag(
message: OrchestrationMessageSummary,
legacyCompatibilityActive = false
): string {
return isLegacyReadOnlyMessage(message, legacyCompatibilityActive) ? ' [legacy, read-only]' : ''
}
export function isLegacyReadOnlyMessage(
message: OrchestrationMessageSummary,
legacyCompatibilityActive = false
): boolean {
return (
message.run_id === ORCHESTRATION_LEGACY_RUN_ID ||
(message.delivery_contract === 'legacy_direct' && !legacyCompatibilityActive) ||
message.delivery_contract === 'audit_only'
)
}
export function formatOrchestrationCheckText(
result: OrchestrationCheckOutput,
checkedTerminal: string
): string {
const prepared = prepareOrchestrationCheckOutput(
result,
checkedTerminal,
result.formatted !== undefined
)
const compatibilityActive = Boolean(
prepared.legacyCompatibility && !prepared.legacyCompatibility.readOnly
)
const legacyHeader = prepared.legacyCompatibility
? prepared.legacyCompatibility.readOnly
? '[LEGACY READ-ONLY]\n'
: prepared.legacyCompatibility.recovery
? '[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]\n'
: '[LEGACY COMPATIBILITY]\n'
: ''
const deliveryNotice = formatCurrentDeliveryNotice(prepared.legacyCompatibility?.currentDelivery)
if (prepared.formatted) {
return `${legacyHeader}${prepared.formatted}${deliveryNotice}`
}
if (prepared.count === 0) {
if (prepared.timedOut) {
return `${legacyHeader}Wait timed out; no messages were consumed.${deliveryNotice}`
}
if (prepared.cancelled) {
const cancelled = prepared.connectionLost
? 'Wait cancelled because the connection closed; no messages were consumed.'
: 'Wait cancelled; no messages were consumed.'
return `${legacyHeader}${cancelled}${deliveryNotice}`
}
return `${legacyHeader}No messages.${deliveryNotice}`
}
const rendered = prepared.messages
.map(
(message) =>
`${message.id}${formatMessageReadOnlyTag(
message,
compatibilityActive
)} [${message.type ?? 'status'}] from=${message.from_handle} "${message.subject}"`
)
.join('\n')
const output = prepared.deliveryId ? `Delivery ${prepared.deliveryId}\n${rendered}` : rendered
return `${legacyHeader}${output}${deliveryNotice}`
}
export function prepareOrchestrationCheckOutput<T extends OrchestrationCheckOutput>(
result: T,
checkedTerminal: string,
formattedRequested: boolean
): T {
const compatibilityActive = Boolean(
result.legacyCompatibility && !result.legacyCompatibility.readOnly
)
if (
!formattedRequested ||
!result.messages.some((message) => isLegacyReadOnlyMessage(message, compatibilityActive))
) {
return result
}
return {
...result,
formatted: formatLegacyAwareCheckMessages(result.messages, checkedTerminal, compatibilityActive)
}
}
function formatMessagePriorityTag(message: OrchestrationMessageSummary): string {
return message.priority === 'urgent' ? ' [URGENT]' : message.priority === 'high' ? ' [HIGH]' : ''
}
function escapeTerminalControlCharacters(value: string): string {
return [...value]
.map((character) => {
const code = character.charCodeAt(0)
if (character === '\n' || (code >= 0x20 && code < 0x7f) || code > 0x9f) {
return character
}
return `\\x${code.toString(16).padStart(2, '0')}`
})
.join('')
}
function formatQuotedMessageField(label: string, value?: string): string {
return `[${label}]\n${escapeTerminalControlCharacters(value ?? '')
.split('\n')
.map((line) => ` ${line}`)
.join('\n')}`
}
function formatLegacyAwareCheckMessages(
messages: OrchestrationMessageSummary[],
checkedTerminal: string,
legacyCompatibilityActive = false
): string {
return messages
.map((message) => {
const legacyReadOnly = isLegacyReadOnlyMessage(message, legacyCompatibilityActive)
const lines = [
`${message.id}${formatMessageReadOnlyTag(message, legacyCompatibilityActive)}${formatMessagePriorityTag(message)} [${message.type ?? 'status'}] from=${message.from_handle}`,
formatQuotedMessageField('subject', message.subject)
]
if (legacyReadOnly) {
lines.push('[Inspection only: reply and acknowledgment are unavailable.]')
}
if (message.body) {
lines.push(formatQuotedMessageField('body', message.body))
}
if (message.payload) {
lines.push(formatQuotedMessageField('payload', message.payload))
}
if (!legacyReadOnly) {
const replyTarget = message.to_handle ?? checkedTerminal
const replyFrom =
replyTarget.startsWith('run:') || replyTarget.startsWith('dispatch:')
? ''
: ` --from ${replyTarget}`
lines.push(`[Reply: orca orchestration reply --id ${message.id}${replyFrom} --body "..."]`)
}
return lines.join('\n')
})
.join('\n\n')
}
function formatCurrentDeliveryNotice(
delivery: LegacyCompatibilityResult['currentDelivery']
): string {
if (!delivery) {
return ''
}
return (
`\n[CURRENT RUN MAIL WAITING]\n` +
`Read: ${delivery.checkCommand}\n` +
`Then acknowledge the delivery ID it prints: ${delivery.ackCommand}`
)
}
+14
View File
@@ -0,0 +1,14 @@
export type TerminalRevealIdentity = {
worktreeId: string
tabId: string
leafId: string
ptyId: string
}
export type TerminalTabCreateReply = {
requestId: string
tabId?: string
title?: string
identity?: TerminalRevealIdentity
error?: string
}
@@ -16,7 +16,10 @@ import {
import { waitForActivePaneHookDescriptor, waitForActivePanePtyId } from './helpers/terminal'
import { RuntimeClient } from '../../src/cli/runtime-client'
import Database from '../../src/main/sqlite/sync-database'
import { LEGACY_CONTRACT_VERSION } from '../../src/main/runtime/orchestration/db'
import {
CURRENT_CONTRACT_VERSION,
LEGACY_CONTRACT_VERSION
} from '../../src/main/runtime/orchestration/db'
import { DEFAULT_LOCAL_ORCA_PROFILE_ID } from '../../src/shared/orca-profiles'
import type { RuntimeTerminalListResult, RuntimeTerminalRead } from '../../src/shared/runtime-types'
@@ -200,6 +203,44 @@ function stripLegacyWorkerRendererBinding(
writeFileSync(persistedDataPath(userDataDir), `${JSON.stringify(data, null, 2)}\n`, 'utf8')
}
function assertDispatchRemainsCurrent(
userDataDir: string,
input: {
dispatchId: string
terminalHandle: string
paneKey: string
processIncarnation: string
worktreeId: string
}
): void {
const db = new Database(path.join(userDataDir, 'orchestration.db'))
try {
const authority = db
.prepare(
`SELECT dc.status AS dispatch_status, dc.assignee_handle, dc.assignee_pane_key,
dc.process_incarnation, dc.contract_version, dc.capability_hash,
wd.state AS worker_state, wd.worktree_id, wd.agent_terminal_handle
FROM dispatch_contexts dc
INNER JOIN worker_dispatches wd ON wd.dispatch_id = dc.id
WHERE dc.id = ?`
)
.get(input.dispatchId)
expect(authority).toEqual({
dispatch_status: 'dispatched',
assignee_handle: input.terminalHandle,
assignee_pane_key: input.paneKey,
process_incarnation: input.processIncarnation,
contract_version: CURRENT_CONTRACT_VERSION,
capability_hash: expect.any(String),
worker_state: 'ready',
worktree_id: input.worktreeId,
agent_terminal_handle: input.terminalHandle
})
} finally {
db.close()
}
}
function markDispatchAsPreUpdateLegacy(
userDataDir: string,
input: {
@@ -233,9 +274,9 @@ function markDispatchAsPreUpdateLegacy(
})
db.prepare(
`UPDATE dispatch_contexts
SET contract_version = ?, capability_hash = NULL, capability_revoked_at = NULL,
launch_token_hash = NULL
WHERE id = ?`
SET contract_version = ?, capability_hash = NULL, capability_revoked_at = NULL,
launch_token_hash = NULL
WHERE id = ?`
).run(LEGACY_CONTRACT_VERSION, input.dispatchId)
} finally {
db.close()
@@ -248,285 +289,307 @@ test.afterAll(() => {
rmSync(fakeCliDir, { recursive: true, force: true })
})
test('adopts one live legacy worker after restart without replaying resume', async (// oxlint-disable-next-line no-empty-pattern -- This lifecycle test owns both Electron launches and intentionally opts out of the default app fixture.
{}, testInfo) => {
test.setTimeout(300_000)
const repoPath = existsSync(TEST_REPO_PATH_FILE)
? readFileSync(TEST_REPO_PATH_FILE, 'utf8').trim()
: ''
test.skip(!repoPath || !existsSync(repoPath), 'Global setup did not produce a seeded test repo')
for (const contractVersion of [LEGACY_CONTRACT_VERSION, CURRENT_CONTRACT_VERSION]) {
const contractLabel = contractVersion === LEGACY_CONTRACT_VERSION ? 'legacy' : 'current'
test(`adopts one live ${contractLabel} worker after restart without replaying resume`, async (// oxlint-disable-next-line no-empty-pattern -- This lifecycle test owns both Electron launches and intentionally opts out of the default app fixture.
{}, testInfo) => {
test.setTimeout(300_000)
rmSync(spawnLedgerPath, { force: true })
rmSync(interruptionLedgerPath, { force: true })
const repoPath = existsSync(TEST_REPO_PATH_FILE)
? readFileSync(TEST_REPO_PATH_FILE, 'utf8').trim()
: ''
test.skip(!repoPath || !existsSync(repoPath), 'Global setup did not produce a seeded test repo')
const session = createRestartSession(testInfo, {
PATH: `${fakeCliDir}${path.delimiter}${process.env.PATH ?? ''}`,
ORCA_E2E_SPAWN_LEDGER: spawnLedgerPath,
ORCA_E2E_INTERRUPTION_LEDGER: interruptionLedgerPath
})
let firstApp: ElectronApplication | null = null
let secondApp: ElectronApplication | null = null
const session = createRestartSession(testInfo, {
PATH: `${fakeCliDir}${path.delimiter}${process.env.PATH ?? ''}`,
ORCA_E2E_SPAWN_LEDGER: spawnLedgerPath,
ORCA_E2E_INTERRUPTION_LEDGER: interruptionLedgerPath
})
let firstApp: ElectronApplication | null = null
let secondApp: ElectronApplication | null = null
try {
const first = await session.launch()
firstApp = first.app
const worktreeId = await attachRepoAndOpenTerminal(first.page, repoPath)
await waitForSessionReady(first.page)
await ensureTerminalVisible(first.page)
const coordinatorTabId = await getActiveTabId(first.page)
expect(coordinatorTabId).toBeTruthy()
await waitForActivePanePtyId(first.page)
const coordinatorPane = await waitForActivePaneHookDescriptor(first.page)
const firstClient = new RuntimeClient(session.userDataDir, 30_000, null, null)
const coordinator = await firstClient.call<{ terminal: { handle: string } }>(
'terminal.resolvePane',
{ paneKey: coordinatorPane.paneKey }
)
const coordinatorTerminal = await firstClient.call<{
terminal: { worktreeId: string }
}>('terminal.show', { terminal: coordinator.result.terminal.handle })
await expect
.poll(async () => {
const listed = await firstClient.call<{ worktrees: { id: string }[] }>('worktree.list', {})
return listed.result.worktrees.some(
(candidate) => candidate.id === coordinatorTerminal.result.terminal.worktreeId
)
})
.toBe(true)
const run = await firstClient.call<{ run: { id: string } }>('orchestration.runCreate', {
objective: 'Legacy worker restart recovery',
from: coordinator.result.terminal.handle
})
const task = await firstClient.call<{ task: { id: string } }>('orchestration.taskCreate', {
spec: 'Respond ACK and remain idle',
run: run.result.run.id,
callerTerminalHandle: coordinator.result.terminal.handle
})
const started = await firstClient.call<{
effects: { kind: string; role?: string; id?: string }[]
}>('orchestration.workerStart', {
task: task.result.task.id,
from: coordinator.result.terminal.handle,
agent: 'codex',
timeoutMs: 15_000
})
const workerHandle = started.result.effects.find(
(effect) => effect.kind === 'terminal' && effect.role === 'agent'
)?.id
expect(workerHandle).toBeTruthy()
let worker = (
await firstClient.call<RuntimeTerminalListResult>('terminal.list')
).result.terminals.find((terminal) => terminal.title === 'Codex Ready')
await expect
.poll(async () => {
const listed = await firstClient.call<RuntimeTerminalListResult>('terminal.list')
worker = listed.result.terminals.find((terminal) => terminal.title === 'Codex Ready')
return worker?.ptyId ?? null
})
.toBeTruthy()
expect(worker?.incarnationId).toBeTruthy()
const workerPaneKey = `${worker!.tabId}:${worker!.leafId}`
await expect
.poll(async () => {
const read = await firstClient.call<{ terminal: RuntimeTerminalRead }>('terminal.read', {
terminal: worker!.handle,
limit: 200
try {
const first = await session.launch()
firstApp = first.app
const worktreeId = await attachRepoAndOpenTerminal(first.page, repoPath)
await waitForSessionReady(first.page)
await ensureTerminalVisible(first.page)
const coordinatorTabId = await getActiveTabId(first.page)
expect(coordinatorTabId).toBeTruthy()
await waitForActivePanePtyId(first.page)
const coordinatorPane = await waitForActivePaneHookDescriptor(first.page)
const firstClient = new RuntimeClient(session.userDataDir, 30_000, null, null)
const coordinator = await firstClient.call<{ terminal: { handle: string } }>(
'terminal.resolvePane',
{ paneKey: coordinatorPane.paneKey }
)
const coordinatorTerminal = await firstClient.call<{
terminal: { worktreeId: string }
}>('terminal.show', { terminal: coordinator.result.terminal.handle })
await expect
.poll(async () => {
const listed = await firstClient.call<{ worktrees: { id: string }[] }>(
'worktree.list',
{}
)
return listed.result.worktrees.some(
(candidate) => candidate.id === coordinatorTerminal.result.terminal.worktreeId
)
})
return read.result.terminal.tail.join('\n')
.toBe(true)
const run = await firstClient.call<{ run: { id: string } }>('orchestration.runCreate', {
objective: 'Legacy worker restart recovery',
from: coordinator.result.terminal.handle
})
.toContain('ACK')
const initialWorker = {
ptyId: worker!.ptyId,
incarnationId: worker!.incarnationId,
worktreeId: worker!.worktreeId,
tabId: worker!.tabId,
leafId: worker!.leafId
}
const initialDispatch = await firstClient.call<{
dispatch: {
id: string
task_id: string
assignee_handle: string
assignee_pane_key: string
process_incarnation: string
} | null
}>('orchestration.dispatchShow', { task: task.result.task.id })
expect(initialDispatch.result.dispatch).toEqual(
expect.objectContaining({
task_id: task.result.task.id,
assignee_pane_key: workerPaneKey,
process_incarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`
const task = await firstClient.call<{ task: { id: string } }>('orchestration.taskCreate', {
spec: 'Respond ACK and remain idle',
run: run.result.run.id,
callerTerminalHandle: coordinator.result.terminal.handle
})
)
const dispatchHandle = initialDispatch.result.dispatch!.assignee_handle
await expect.poll(() => readLedger(spawnLedgerPath)).toHaveLength(1)
const [initialSpawn] = readLedger(spawnLedgerPath)
expect(isProcessAlive(initialSpawn.pid)).toBe(true)
expect(readLedger(interruptionLedgerPath)).toEqual([])
const started = await firstClient.call<{
effects: { kind: string; role?: string; id?: string }[]
}>('orchestration.workerStart', {
task: task.result.task.id,
from: coordinator.result.terminal.handle,
agent: 'codex',
timeoutMs: 15_000
})
const workerHandle = started.result.effects.find(
(effect) => effect.kind === 'terminal' && effect.role === 'agent'
)?.id
expect(workerHandle).toBeTruthy()
const transcriptPath = session.seedCodexResumeRollout(PROVIDER_SESSION_ID, repoPath)
await first.page.evaluate(
({ paneKey, tabId, worktreeId: workerWorktreeId, terminalHandle, transcript }) => {
window.__store?.getState().setAgentStatus(
paneKey,
{ state: 'working', prompt: 'Respond ACK and remain idle', agentType: 'codex' },
'Codex Ready',
undefined,
{ tabId, worktreeId: workerWorktreeId, terminalHandle },
{
providerSession: {
key: 'session_id',
id: 'e2e-legacy-orchestration-worker',
transcriptPath: transcript
},
launchConfig: {
agentCommand: 'codex',
agentArgs: '--dangerously-bypass-approvals-and-sandbox',
agentEnv: {}
}
}
)
window.__store?.getState().captureAllSleepingAgentSessions('quit')
},
{
paneKey: workerPaneKey,
tabId: worker!.tabId,
let worker = (
await firstClient.call<RuntimeTerminalListResult>('terminal.list')
).result.terminals.find((terminal) => terminal.title === 'Codex Ready')
await expect
.poll(async () => {
const listed = await firstClient.call<RuntimeTerminalListResult>('terminal.list')
worker = listed.result.terminals.find((terminal) => terminal.title === 'Codex Ready')
return worker?.ptyId ?? null
})
.toBeTruthy()
expect(worker?.incarnationId).toBeTruthy()
const workerPaneKey = `${worker!.tabId}:${worker!.leafId}`
await expect
.poll(async () => {
const read = await firstClient.call<{ terminal: RuntimeTerminalRead }>('terminal.read', {
terminal: worker!.handle,
limit: 200
})
return read.result.terminal.tail.join('\n')
})
.toContain('ACK')
const initialWorker = {
ptyId: worker!.ptyId,
incarnationId: worker!.incarnationId,
worktreeId: worker!.worktreeId,
terminalHandle: worker!.handle,
transcript: transcriptPath
tabId: worker!.tabId,
leafId: worker!.leafId
}
)
await expect
.poll(() => hasPersistedResumeRecord(session.userDataDir, workerPaneKey), {
timeout: 30_000
})
.toBe(true)
await session.close(firstApp)
firstApp = null
expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn])
expect(readLedger(interruptionLedgerPath)).toEqual([])
expect(isProcessAlive(initialSpawn.pid)).toBe(true)
stripLegacyWorkerRendererBinding(session.userDataDir, {
worktreeId,
coordinatorTabId: coordinatorTabId!,
workerTabId: worker!.tabId,
workerPaneKey
})
markDispatchAsPreUpdateLegacy(session.userDataDir, {
dispatchId: initialDispatch.result.dispatch!.id,
terminalHandle: dispatchHandle,
paneKey: workerPaneKey,
processIncarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`,
worktreeId: initialWorker.worktreeId
})
const second = await session.launch()
secondApp = second.app
await waitForSessionReady(second.page)
expect(await waitForActiveWorktree(second.page)).toBe(worktreeId)
const secondClient = new RuntimeClient(session.userDataDir, 30_000, null, null)
let recovered = (
await secondClient.call<RuntimeTerminalListResult>('terminal.list')
).result.terminals.find((terminal) => terminal.ptyId === initialWorker.ptyId)
await expect
.poll(async () => {
const listed = await secondClient.call<RuntimeTerminalListResult>('terminal.list')
const matches = listed.result.terminals.filter(
(terminal) => terminal.ptyId === initialWorker.ptyId
)
recovered = matches[0]
return matches
})
.toEqual([
const initialDispatch = await firstClient.call<{
dispatch: {
id: string
task_id: string
assignee_handle: string
assignee_pane_key: string
process_incarnation: string
} | null
}>('orchestration.dispatchShow', { task: task.result.task.id })
expect(initialDispatch.result.dispatch).toEqual(
expect.objectContaining({
...initialWorker,
connected: true,
writable: true
task_id: task.result.task.id,
assignee_pane_key: workerPaneKey,
process_incarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`
})
])
)
const dispatchHandle = initialDispatch.result.dispatch!.assignee_handle
await expect.poll(() => readLedger(spawnLedgerPath)).toHaveLength(1)
const [initialSpawn] = readLedger(spawnLedgerPath)
expect(isProcessAlive(initialSpawn.pid)).toBe(true)
expect(readLedger(interruptionLedgerPath)).toEqual([])
const recoveredTab = second.page.locator(
`[data-testid="sortable-tab"][data-tab-id="${initialWorker.tabId}"]`
)
await expect(recoveredTab).toBeVisible()
await expect(recoveredTab).toHaveCount(1)
await expect(recoveredTab).toHaveAttribute('data-active', 'false')
await expect(
second.page.locator(`[data-testid="sortable-tab"][data-tab-id="${coordinatorTabId!}"]`)
).toHaveAttribute('data-active', 'true')
await expect
.poll(async () => {
const read = await secondClient.call<{ terminal: RuntimeTerminalRead }>('terminal.read', {
terminal: recovered!.handle,
limit: 200
const transcriptPath = session.seedCodexResumeRollout(PROVIDER_SESSION_ID, repoPath)
await first.page.evaluate(
({ paneKey, tabId, worktreeId: workerWorktreeId, terminalHandle, transcript }) => {
window.__store?.getState().setAgentStatus(
paneKey,
{ state: 'working', prompt: 'Respond ACK and remain idle', agentType: 'codex' },
'Codex Ready',
undefined,
{ tabId, worktreeId: workerWorktreeId, terminalHandle },
{
providerSession: {
key: 'session_id',
id: 'e2e-legacy-orchestration-worker',
transcriptPath: transcript
},
launchConfig: {
agentCommand: 'codex',
agentArgs: '--dangerously-bypass-approvals-and-sandbox',
agentEnv: {}
}
}
)
window.__store?.getState().captureAllSleepingAgentSessions('quit')
},
{
paneKey: workerPaneKey,
tabId: worker!.tabId,
worktreeId: worker!.worktreeId,
terminalHandle: worker!.handle,
transcript: transcriptPath
}
)
await expect
.poll(() => hasPersistedResumeRecord(session.userDataDir, workerPaneKey), {
timeout: 30_000
})
return read.result.terminal.tail.join('\n')
})
.toContain('ACK')
.toBe(true)
const restoredRun = await secondClient.call<{ run: { id: string } }>('orchestration.runShow', {
id: run.result.run.id
})
expect(restoredRun.result.run.id).toBe(run.result.run.id)
const tasks = await secondClient.call<{ tasks: { id: string }[] }>('orchestration.taskList', {
run: run.result.run.id
})
expect(tasks.result.tasks).toEqual(
expect.arrayContaining([expect.objectContaining({ id: task.result.task.id })])
)
const recoveredDispatch = await secondClient.call<{
dispatch: {
id: string
task_id: string
assignee_handle: string
assignee_pane_key: string
process_incarnation: string
contract_version: number
} | null
}>('orchestration.dispatchShow', { task: task.result.task.id })
expect(recoveredDispatch.result.dispatch).toEqual(
expect.objectContaining({
id: initialDispatch.result.dispatch!.id,
task_id: task.result.task.id,
assignee_handle: dispatchHandle,
assignee_pane_key: workerPaneKey,
process_incarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`,
contract_version: LEGACY_CONTRACT_VERSION
})
)
await expect
.poll(async () => ({
renderer: await readRendererRecoveryState(second.page, workerPaneKey, initialWorker.tabId),
persisted: hasPersistedResumeRecord(session.userDataDir, workerPaneKey)
}))
.toEqual({
renderer: { sleeping: false, resumeClaim: false, pendingStartup: false },
persisted: false
})
expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn])
expect(readLedger(interruptionLedgerPath)).toEqual([])
expect(isProcessAlive(initialSpawn.pid)).toBe(true)
await session.close(firstApp)
firstApp = null
expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn])
expect(readLedger(interruptionLedgerPath)).toEqual([])
expect(isProcessAlive(initialSpawn.pid)).toBe(true)
const otherWorktreeId = await switchToOtherWorktree(second.page, worktreeId)
expect(otherWorktreeId).toBeTruthy()
await switchToWorktree(second.page, worktreeId)
await expect(recoveredTab).toBeVisible()
await expect(recoveredTab).toHaveCount(1)
await expect(recoveredTab).toHaveAttribute('data-active', 'false')
await expect
.poll(async () => readRendererRecoveryState(second.page, workerPaneKey, initialWorker.tabId))
.toEqual({ sleeping: false, resumeClaim: false, pendingStartup: false })
expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn])
expect(readLedger(interruptionLedgerPath)).toEqual([])
expect(isProcessAlive(initialSpawn.pid)).toBe(true)
await expect(second.page.locator('body')).not.toContainText('Conversation interrupted')
} finally {
if (secondApp) {
await session.close(secondApp).catch(() => undefined)
stripLegacyWorkerRendererBinding(session.userDataDir, {
worktreeId,
coordinatorTabId: coordinatorTabId!,
workerTabId: worker!.tabId,
workerPaneKey
})
const dispatchIdentity = {
dispatchId: initialDispatch.result.dispatch!.id,
terminalHandle: dispatchHandle,
paneKey: workerPaneKey,
processIncarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`,
worktreeId: initialWorker.worktreeId
}
if (contractVersion === LEGACY_CONTRACT_VERSION) {
markDispatchAsPreUpdateLegacy(session.userDataDir, dispatchIdentity)
} else {
assertDispatchRemainsCurrent(session.userDataDir, dispatchIdentity)
}
const second = await session.launch()
secondApp = second.app
await waitForSessionReady(second.page)
expect(await waitForActiveWorktree(second.page)).toBe(worktreeId)
const secondClient = new RuntimeClient(session.userDataDir, 30_000, null, null)
let recovered = (
await secondClient.call<RuntimeTerminalListResult>('terminal.list')
).result.terminals.find((terminal) => terminal.ptyId === initialWorker.ptyId)
await expect
.poll(async () => {
const listed = await secondClient.call<RuntimeTerminalListResult>('terminal.list')
const matches = listed.result.terminals.filter(
(terminal) => terminal.ptyId === initialWorker.ptyId
)
recovered = matches[0]
return matches
})
.toEqual([
expect.objectContaining({
...initialWorker,
connected: true,
writable: true
})
])
const recoveredTab = second.page.locator(
`[data-testid="sortable-tab"][data-tab-id="${initialWorker.tabId}"]`
)
await expect(recoveredTab).toBeVisible()
await expect(recoveredTab).toHaveCount(1)
await expect(recoveredTab).toHaveAttribute('data-active', 'false')
await expect(
second.page.locator(`[data-testid="sortable-tab"][data-tab-id="${coordinatorTabId!}"]`)
).toHaveAttribute('data-active', 'true')
await expect
.poll(async () => {
const read = await secondClient.call<{ terminal: RuntimeTerminalRead }>('terminal.read', {
terminal: recovered!.handle,
limit: 200
})
return read.result.terminal.tail.join('\n')
})
.toContain('ACK')
const restoredRun = await secondClient.call<{ run: { id: string } }>(
'orchestration.runShow',
{
id: run.result.run.id
}
)
expect(restoredRun.result.run.id).toBe(run.result.run.id)
const tasks = await secondClient.call<{ tasks: { id: string }[] }>('orchestration.taskList', {
run: run.result.run.id
})
expect(tasks.result.tasks).toEqual(
expect.arrayContaining([expect.objectContaining({ id: task.result.task.id })])
)
const recoveredDispatch = await secondClient.call<{
dispatch: {
id: string
task_id: string
assignee_handle: string
assignee_pane_key: string
process_incarnation: string
contract_version: number
} | null
}>('orchestration.dispatchShow', { task: task.result.task.id })
expect(recoveredDispatch.result.dispatch).toEqual(
expect.objectContaining({
id: initialDispatch.result.dispatch!.id,
task_id: task.result.task.id,
assignee_handle: dispatchHandle,
assignee_pane_key: workerPaneKey,
process_incarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`,
contract_version: contractVersion
})
)
await expect
.poll(async () => ({
renderer: await readRendererRecoveryState(
second.page,
workerPaneKey,
initialWorker.tabId
),
persisted: hasPersistedResumeRecord(session.userDataDir, workerPaneKey)
}))
.toEqual({
renderer: { sleeping: false, resumeClaim: false, pendingStartup: false },
persisted: false
})
expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn])
expect(readLedger(interruptionLedgerPath)).toEqual([])
expect(isProcessAlive(initialSpawn.pid)).toBe(true)
const otherWorktreeId = await switchToOtherWorktree(second.page, worktreeId)
expect(otherWorktreeId).toBeTruthy()
await switchToWorktree(second.page, worktreeId)
await expect(recoveredTab).toBeVisible()
await expect(recoveredTab).toHaveCount(1)
await expect(recoveredTab).toHaveAttribute('data-active', 'false')
await expect
.poll(async () =>
readRendererRecoveryState(second.page, workerPaneKey, initialWorker.tabId)
)
.toEqual({ sleeping: false, resumeClaim: false, pendingStartup: false })
expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn])
expect(readLedger(interruptionLedgerPath)).toEqual([])
expect(isProcessAlive(initialSpawn.pid)).toBe(true)
await expect(second.page.locator('body')).not.toContainText('Conversation interrupted')
} finally {
if (secondApp) {
await session.close(secondApp).catch(() => undefined)
}
if (firstApp) {
await session.close(firstApp).catch(() => undefined)
}
await session.dispose()
}
if (firstApp) {
await session.close(firstApp).catch(() => undefined)
}
await session.dispose()
}
})
})
}