fix(orchestration): complete legacy worker cutover recovery

This commit is contained in:
OrcaWin
2026-07-29 04:39:02 -07:00
parent f58353c83d
commit efd06382ae
61 changed files with 5185 additions and 1117 deletions
File diff suppressed because one or more lines are too long
@@ -71,4 +71,24 @@ describe('orchestration check identity', () => {
})
)
})
it('preserves the pinned legacy --inject check signature', async () => {
process.env.ORCA_TERMINAL_HANDLE = 'term_legacy_worker'
await invokeCheck(
new Map<string, string | boolean>([
['unread', true],
['inject', true]
])
)
expect(callMock).toHaveBeenCalledWith(
'orchestration.check',
expect.objectContaining({
terminal: 'term_legacy_worker',
unread: true,
inject: true
})
)
})
})
@@ -179,6 +179,7 @@ describe('legacy orchestration CLI inspection', () => {
id: 'msg_current',
run_id: 'run_current',
from_handle: 'term_current',
to_handle: 'run:run_current',
subject: 'current question',
type: 'question',
body: 'May I continue?',
@@ -217,7 +218,8 @@ describe('legacy orchestration CLI inspection', () => {
expect(response.result.formatted).not.toContain('RUNTIME_SENTINEL')
expect(
response.result.formatted.split('\n').filter((line) => line.startsWith('[Reply:'))
).toEqual(['[Reply: orca orchestration reply --id msg_current --from term_coord --body "..."]'])
).toEqual(['[Reply: orca orchestration reply --id msg_current --body "..."]'])
expect(response.result.formatted).not.toContain('--from run:run_current')
})
it('preserves runtime formatting when every message belongs to a current Run', async () => {
@@ -62,6 +62,28 @@ describe('lightweight Run CLI handlers', () => {
from: 'term_coord'
})
})
it('passes explicit legacy takeover only when requested', async () => {
callMock.mockResolvedValue({
result: { run: { id: 'run_adopted', objective: 'Recovered work' } }
})
await ORCHESTRATION_HANDLERS['orchestration run-use']({
flags: new Map<string, string | boolean>([
['id', 'run_adopted'],
['from', 'term_current'],
['takeover-legacy', true]
]),
client: { call: callMock },
cwd: '/tmp/repo',
json: true
} as never)
expect(callMock).toHaveBeenCalledWith('orchestration.runUse', {
id: 'run_adopted',
from: 'term_current',
takeoverLegacy: true
})
})
})
describe('orchestration reset CLI handler', () => {
+14 -178
View File
@@ -21,10 +21,14 @@ import type {
} from '../../shared/orchestration-worker-output'
import type { NativeChatMessage } from '../../shared/native-chat-types'
import type { RuntimeTerminalRead } from '../../shared/runtime-types'
import { orchestrationMigrationData } from '../../shared/orchestration-rpc-contract'
import {
ORCHESTRATION_LEGACY_RUN_ID,
orchestrationMigrationData
} from '../../shared/orchestration-rpc-contract'
formatMessageReadOnlyTag,
formatOrchestrationCheckText,
prepareOrchestrationCheckOutput,
type LegacyCompatibilityResult,
type OrchestrationMessageSummary as MessageSummary
} from '../../shared/orchestration-check-output'
// Why: 15 s is well under Claude Code's ~2 min Bash-tool silence budget while keeping log volume low. See design doc §3.4.
const DEFAULT_KEEPALIVE_INTERVAL_MS = 15_000
@@ -74,93 +78,6 @@ const TASK_STATUS_VALUES = [
'blocked'
] as const
type MessageSummary = {
id: string
run_id?: string
delivery_contract?: 'legacy_direct' | 'current_delivery' | 'audit_only'
from_handle: string
to_handle?: string
subject?: string
type?: string
body?: string
payload?: string | null
priority?: string
read?: number
}
function formatMessageReadOnlyTag(
message: MessageSummary,
legacyCompatibilityActive = false
): string {
return isLegacyReadOnlyMessage(message, legacyCompatibilityActive) ? ' [legacy, read-only]' : ''
}
function isLegacyReadOnlyMessage(
message: MessageSummary,
legacyCompatibilityActive = false
): boolean {
return (
message.run_id === ORCHESTRATION_LEGACY_RUN_ID ||
(message.delivery_contract === 'legacy_direct' && !legacyCompatibilityActive) ||
message.delivery_contract === 'audit_only'
)
}
function formatMessagePriorityTag(message: MessageSummary): string {
return message.priority === 'urgent' ? ' [URGENT]' : message.priority === 'high' ? ' [HIGH]' : ''
}
function escapeTerminalControlCharacters(value: string): string {
return [...value]
.map((character) => {
const code = character.charCodeAt(0)
if (character === '\n' || (code >= 0x20 && code < 0x7f) || code > 0x9f) {
return character
}
return `\\x${code.toString(16).padStart(2, '0')}`
})
.join('')
}
function formatQuotedMessageField(label: string, value?: string): string {
return `[${label}]\n${escapeTerminalControlCharacters(value ?? '')
.split('\n')
.map((line) => ` ${line}`)
.join('\n')}`
}
function formatLegacyAwareCheckMessages(
messages: MessageSummary[],
checkedTerminal: string,
legacyCompatibilityActive = false
): string {
return messages
.map((message) => {
const legacyReadOnly = isLegacyReadOnlyMessage(message, legacyCompatibilityActive)
const lines = [
`${message.id}${formatMessageReadOnlyTag(message, legacyCompatibilityActive)}${formatMessagePriorityTag(message)} [${message.type ?? 'status'}] from=${message.from_handle}`,
formatQuotedMessageField('subject', message.subject)
]
if (legacyReadOnly) {
lines.push('[Inspection only: reply and acknowledgment are unavailable.]')
}
if (message.body) {
lines.push(formatQuotedMessageField('body', message.body))
}
if (message.payload) {
lines.push(formatQuotedMessageField('payload', message.payload))
}
if (!legacyReadOnly) {
const replyFrom = message.to_handle ?? checkedTerminal
lines.push(
`[Reply: orca orchestration reply --id ${message.id} --from ${replyFrom} --body "..."]`
)
}
return lines.join('\n')
})
.join('\n\n')
}
type LifecycleSendRejection = {
action: 'rejected'
code: string
@@ -181,23 +98,6 @@ type OrchestrationSendResult =
lifecycle?: { action: 'completed' | 'failed' }
}
type LegacyCompatibilityResult = {
recovery?: boolean
readOnly?: boolean
ackMessageIds?: string[]
answerAcknowledgement?: {
questionId: string
answerMessageId: string
}
currentDelivery?: {
runId: string
checkCommand: string
ackCommand: string
}
resumeRequired?: boolean
resumeCommand?: string
}
function resolveCompatibilityCliCommand(): 'orca' | 'orca-ide' | 'orca-dev' {
const configured = process.env.ORCA_CLI_COMMAND
if (configured === 'orca' || configured === 'orca-ide' || configured === 'orca-dev') {
@@ -206,19 +106,6 @@ function resolveCompatibilityCliCommand(): 'orca' | 'orca-ide' | 'orca-dev' {
return process.platform === 'linux' ? 'orca-ide' : 'orca'
}
function formatCurrentDeliveryNotice(
delivery: LegacyCompatibilityResult['currentDelivery']
): string {
if (!delivery) {
return ''
}
return (
`\n[CURRENT RUN MAIL WAITING]\n` +
`Read: ${delivery.checkCommand}\n` +
`Then acknowledge the delivery ID it prints: ${delivery.ackCommand}`
)
}
function resolvePackagedWindowsCompatibilityCommand(): 'orca' | 'orca-ide' | undefined {
if (process.env.ORCA_WINDOWS_PACKAGED_CLI_LAUNCHER !== '1') {
return undefined
@@ -560,7 +447,8 @@ export const ORCHESTRATION_HANDLERS: Record<string, CommandHandler> = {
run: { id: string; objective: string; consumer_generation: number }
}>(client, flags, 'orchestration.runUse', {
id: getRequiredStringFlag(flags, 'id'),
from
from,
...(flags.has('takeover-legacy') ? { takeoverLegacy: true } : {})
})
printResult(result, json, (r) => `Using Run ${r.run.id}: ${r.run.objective}`)
},
@@ -715,6 +603,7 @@ export const ORCHESTRATION_HANDLERS: Record<string, CommandHandler> = {
all: flags.has('all') ? true : undefined,
types: getOptionalStringFlag(flags, 'types'),
format: flags.has('format') ? true : undefined,
inject: flags.has('inject') ? true : undefined,
compatibilityCliCommand: resolveCompatibilityCliCommand(),
run: getOptionalStringFlag(flags, 'run'),
ack: getOptionalStringFlag(flags, 'ack'),
@@ -752,64 +641,11 @@ export const ORCHESTRATION_HANDLERS: Record<string, CommandHandler> = {
}
}
}
const compatibilityActive = Boolean(
result.result.legacyCompatibility && !result.result.legacyCompatibility.readOnly
)
if (
flags.has('format') &&
result.result.messages.some((message) =>
isLegacyReadOnlyMessage(message, compatibilityActive)
)
) {
// Why: formatted is one opaque batch with untrusted bodies, so selective banner parsing cannot safely remove legacy actions.
result = {
...result,
result: {
...result.result,
formatted: formatLegacyAwareCheckMessages(
result.result.messages,
terminal,
compatibilityActive
)
}
}
result = {
...result,
result: prepareOrchestrationCheckOutput(result.result, terminal, flags.has('format'))
}
printResult(result, json, (r) => {
const legacyHeader = r.legacyCompatibility
? r.legacyCompatibility.readOnly
? '[LEGACY READ-ONLY]\n'
: r.legacyCompatibility.recovery
? '[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]\n'
: '[LEGACY COMPATIBILITY]\n'
: ''
const deliveryNotice = formatCurrentDeliveryNotice(r.legacyCompatibility?.currentDelivery)
if (r.formatted) {
return `${legacyHeader}${r.formatted}${deliveryNotice}`
}
if (r.count === 0) {
if (r.timedOut) {
return `${legacyHeader}Wait timed out; no messages were consumed.${deliveryNotice}`
}
if (r.cancelled) {
const cancelled = r.connectionLost
? 'Wait cancelled because the connection closed; no messages were consumed.'
: 'Wait cancelled; no messages were consumed.'
return `${legacyHeader}${cancelled}${deliveryNotice}`
}
return `${legacyHeader}No messages.${deliveryNotice}`
}
const rendered = r.messages
.map(
(m) =>
`${m.id}${formatMessageReadOnlyTag(
m,
Boolean(r.legacyCompatibility && !r.legacyCompatibility.readOnly)
)} [${m.type ?? 'status'}] from=${m.from_handle} "${m.subject}"`
)
.join('\n')
const output = r.deliveryId ? `Delivery ${r.deliveryId}\n${rendered}` : rendered
return `${legacyHeader}${output}${deliveryNotice}`
})
printResult(result, json, (r) => formatOrchestrationCheckText(r, terminal))
const compatibilityAck = result.result.legacyCompatibility?.ackMessageIds
if (compatibilityAck && compatibilityAck.length > 0) {
await flushStdout()
+6 -2
View File
@@ -16,8 +16,12 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
{
path: ['orchestration', 'run-use'],
summary: 'Bind this coordinator terminal to an existing Run',
usage: 'orca orchestration run-use --id <run_id> [--from <handle>] [--json]',
allowedFlags: [...GLOBAL_FLAGS, 'id', 'from', 'retry-request']
usage:
'orca orchestration run-use --id <run_id> [--from <handle>] [--takeover-legacy] [--json]',
allowedFlags: [...GLOBAL_FLAGS, 'id', 'from', 'takeover-legacy', 'retry-request'],
notes: [
'--takeover-legacy explicitly replaces an unavailable legacy coordinator while preserving live worker assignments.'
]
},
{
path: ['orchestration', 'run-current'],
-2
View File
@@ -2220,8 +2220,6 @@ void app.whenReady().then(async () => {
getAgentProviderSessionSnapshot: () => agentHookServer.getStatusSnapshot(),
getAgentProviderSessionRowsForPane: (paneKey) =>
agentHookServer.getStatusSnapshotForPane(paneKey),
getHydratedAgentHookAuthority: () => agentHookServer.getHydratedAuthorityCommitments(),
getCurrentAgentHookAuthority: () => agentHookServer.getCurrentAuthorityObservations(),
attestAgentHookCompatibilityAuthority: (candidate) =>
agentHookServer.attestCompatibilityAuthority(candidate),
canRecoverPersistentLocalPtys: () => getDaemonProvider() !== null,
+31
View File
@@ -5374,6 +5374,37 @@ describe('registerPtyHandlers', () => {
expect(listProcesses).not.toHaveBeenCalled()
})
it('scopes runtime inventories to the requested provider', async () => {
const localList = vi
.spyOn(getLocalPtyProvider(), 'listProcesses')
.mockResolvedValue([{ id: 'local-pty', title: 'Local', cwd: '/local' }])
const sshAList = vi.fn(async () => [{ id: 'ssh-a-pty' }])
const sshBList = vi.fn(async () => {
throw new Error('ssh-b unavailable')
})
registerSshPtyProvider('ssh-a', { listProcesses: sshAList } as never)
registerSshPtyProvider('ssh-b', { listProcesses: sshBList } as never)
const runtime = { setPtyController: vi.fn() }
handlers.clear()
registerPtyHandlers(mainWindow as never, runtime as never)
const controller = runtime.setPtyController.mock.calls[0]?.[0] as {
listProcesses(connectionId?: string | null): Promise<{ id: string }[]>
}
await expect(controller.listProcesses(null)).resolves.toEqual([
{ id: 'local-pty', title: 'Local', cwd: '/local' }
])
expect(localList).toHaveBeenCalledOnce()
expect(sshAList).not.toHaveBeenCalled()
expect(sshBList).not.toHaveBeenCalled()
await expect(controller.listProcesses('ssh-a')).resolves.toEqual([{ id: 'ssh-a-pty' }])
expect(sshAList).toHaveBeenCalledOnce()
expect(sshBList).not.toHaveBeenCalled()
await expect(controller.listProcesses()).rejects.toThrow('ssh-b unavailable')
})
it('returns unavailable runtime confirmation for unsupported or missing providers', async () => {
registerSshPtyProvider('ssh-1', {} as never)
setPtyOwnership('unsupported-pty', 'ssh-1')
+7 -1
View File
@@ -4253,7 +4253,13 @@ export function registerPtyHandlers(
return null
}
},
listProcesses: async () => {
listProcesses: async (connectionId) => {
if (connectionId === null) {
return localProvider.listProcesses()
}
if (connectionId !== undefined) {
return getProvider(connectionId).listProcesses()
}
const providerSessions = await Promise.all([
localProvider.listProcesses(),
...Array.from(sshProviders.values(), (provider) => provider.listProcesses())
+696 -61
View File
@@ -16337,6 +16337,17 @@ describe('OrcaRuntimeService', () => {
['pty-shell', 'inc-shell', 'term_shell', 'Shell']
] as const
const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow: vi.fn() } as never)
const listProcesses = vi.fn(async () =>
processes.map(([id, incarnationId, terminalHandle, title]) => ({
id,
incarnationId,
terminalHandle,
title,
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID,
wslDistro: null
}))
)
runtime.setPtyController({
write: (ptyId, data) => {
writes.push([ptyId, data])
@@ -16345,16 +16356,7 @@ describe('OrcaRuntimeService', () => {
resize,
kill: () => true,
getForegroundProcess: async () => null,
listProcesses: async () =>
processes.map(([id, incarnationId, terminalHandle, title]) => ({
id,
incarnationId,
terminalHandle,
title,
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID,
wslDistro: null
}))
listProcesses
})
const before = await runtime.listTerminals(`id:${TEST_WORKTREE_ID}`)
expect(before.terminals.map((terminal) => terminal.tabId)).toEqual(
@@ -16442,6 +16444,14 @@ describe('OrcaRuntimeService', () => {
tail: ['legacy output']
})
const inventoryCount = listProcesses.mock.calls.length
const agentPty = (
runtime as unknown as {
ptysById: Map<string, { tabId: string | null; paneKey: string | null }>
}
).ptysById.get('pty-agent')!
agentPty.tabId = null
agentPty.paneKey = null
const secondClient = await runtime.adoptTerminalOrphans({
worktree: `id:${TEST_WORKTREE_ID}`,
expectedTopologyRevision: 0,
@@ -16454,6 +16464,22 @@ describe('OrcaRuntimeService', () => {
}))
})
expect(secondClient).toMatchObject({ adopted: false, topologyRevision: 1 })
expect(agentPty).toMatchObject({
tabId: 'tab-agent',
paneKey: makePaneKey('tab-agent', HEADLESS_LEAF_ID)
})
expect(listProcesses).toHaveBeenCalledTimes(inventoryCount + 1)
expect(listProcesses).toHaveBeenLastCalledWith(null)
expect(
(await runtime.listTerminals()).terminals.find((terminal) => terminal.ptyId === 'pty-agent')
).toMatchObject({
handle: 'term_agent',
orphaned: false,
tabId: 'tab-agent',
leafId: HEADLESS_LEAF_ID
})
expect(listProcesses).toHaveBeenCalledTimes(inventoryCount + 2)
expect(listProcesses).toHaveBeenLastCalledWith(undefined)
await expect(
runtime.adoptTerminalOrphans({
worktree: `id:${TEST_WORKTREE_ID}`,
@@ -16560,6 +16586,7 @@ describe('OrcaRuntimeService', () => {
getForegroundProcess: async () => null,
hasRendererSerializer: () => true,
serializeBuffer,
hasPty: (ptyId) => ptyId === 'pty-legacy',
listProcesses: async () => [
{
id: 'pty-legacy',
@@ -16606,7 +16633,11 @@ describe('OrcaRuntimeService', () => {
presentation: 'background',
tabId: 'legacy-worker',
leafId: workerLeafId,
focus: false
focus: false,
expectedProcessIdentity: {
terminalHandle: 'term_legacy',
incarnationId
}
})
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted')
expect(write).not.toHaveBeenCalled()
@@ -16683,6 +16714,7 @@ describe('OrcaRuntimeService', () => {
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
hasPty: (ptyId) => ptyId === 'pty-reveal-retry',
listProcesses: async () => [
{
id: 'pty-reveal-retry',
@@ -16716,6 +16748,269 @@ describe('OrcaRuntimeService', () => {
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted')
})
function makePostRevealWorkerRecoveryHarness(hasPty: (ptyId: string) => boolean | null): {
runtime: OrcaRuntimeService
getSession: () => WorkspaceSessionState
workerPaneKey: string
ptyId: string
incarnationId: string
terminalHandle: string
revealTerminalSession: ReturnType<typeof vi.fn>
resolveLegacyWorkerTerminalRecovery: ReturnType<typeof vi.fn>
} {
const workerPaneKey = `legacy-post-reveal:${HEADLESS_LEAF_ID}`
const ptyId = 'pty-post-reveal'
const incarnationId = '45454545-4545-4545-8545-454545454545'
const terminalHandle = 'term_post_reveal'
const session: WorkspaceSessionState = {
...getDefaultWorkspaceSession(),
tabsByWorktree: { [TEST_WORKTREE_ID]: [] },
sleepingAgentSessionsByPaneKey: {
[workerPaneKey]: {
paneKey: workerPaneKey,
tabId: 'legacy-post-reveal',
worktreeId: TEST_WORKTREE_ID,
agent: 'codex',
providerSession: { key: 'session_id', id: 'legacy-post-reveal-session' },
prompt: 'continue',
state: 'working',
capturedAt: 1,
updatedAt: 1,
origin: 'live'
}
}
}
const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session)
const runtime = new OrcaRuntimeService(
{ ...runtimeStore, flushOrThrow: vi.fn() } as never,
undefined,
{ canRecoverPersistentLocalPtys: () => true }
)
runtime.setOrchestrationDb({
listLegacyWorkerTerminalRecoveryRows: () => [
{
dispatch_id: 'dispatch-post-reveal',
task_id: 'task-post-reveal',
dispatch_status: 'completed',
contract_version: 0,
assignee_handle: 'term_post_reveal',
assignee_pane_key: workerPaneKey,
process_incarnation: `${ptyId}:${incarnationId}`,
worker_state: 'ready',
worktree_id: TEST_WORKTREE_ID,
agent_terminal_handle: 'term_post_reveal'
}
]
} as unknown as OrchestrationDb)
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
hasPty,
listProcesses: async () => [
{
id: ptyId,
incarnationId,
terminalHandle: 'term_post_reveal',
title: 'Post-reveal worker',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID,
wslDistro: null
}
]
})
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-post-reveal' })
const resolveLegacyWorkerTerminalRecovery = vi.fn()
runtime.setNotifier({
revealTerminalSession,
resolveLegacyWorkerTerminalRecovery
} as never)
return {
runtime,
getSession,
workerPaneKey,
ptyId,
incarnationId,
terminalHandle,
revealTerminalSession,
resolveLegacyWorkerTerminalRecovery
}
}
it('keeps a revealed worker fenced until its exact renderer graph is published', async () => {
vi.useFakeTimers()
try {
const harness = makePostRevealWorkerRecoveryHarness(() => true)
const identity = {
worktreeId: TEST_WORKTREE_ID,
tabId: 'legacy-post-reveal',
leafId: HEADLESS_LEAF_ID,
ptyId: harness.ptyId
}
harness.revealTerminalSession.mockResolvedValue({
tabId: identity.tabId,
identity
})
await expect(
harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true })
).resolves.toMatchObject({
adoptedDispatchIds: [],
exitedDispatchIds: [],
deferredDispatchIds: ['dispatch-post-reveal']
})
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
expect(
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
).toBeDefined()
expect(harness.resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled()
harness.runtime.attachWindow(1)
harness.runtime.syncWindowGraph(1, {
tabs: [
{
tabId: identity.tabId,
worktreeId: identity.worktreeId,
title: 'Post-reveal worker',
activeLeafId: identity.leafId,
layout: null
}
],
leaves: [
{
tabId: identity.tabId,
worktreeId: identity.worktreeId,
leafId: identity.leafId,
paneRuntimeId: 1,
ptyId: identity.ptyId
}
]
})
await vi.advanceTimersByTimeAsync(1_000)
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
expect(
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
).toBeUndefined()
expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(
harness.workerPaneKey,
'adopted'
)
expect(
(await harness.runtime.listTerminals()).terminals.filter(
(terminal) => terminal.ptyId === identity.ptyId
)
).toEqual([
expect.objectContaining({
handle: harness.terminalHandle,
incarnationId: harness.incarnationId,
orphaned: false,
worktreeId: identity.worktreeId,
tabId: identity.tabId,
leafId: identity.leafId
})
])
} finally {
vi.useRealTimers()
}
})
it('retires the exact worker when it exits after renderer reveal', async () => {
const hasPty = vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(false)
const harness = makePostRevealWorkerRecoveryHarness(hasPty)
await expect(
harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true })
).resolves.toMatchObject({
adoptedDispatchIds: [],
exitedDispatchIds: ['dispatch-post-reveal'],
deferredDispatchIds: []
})
expect(hasPty).toHaveBeenCalledTimes(2)
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([])
expect(
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
).toBeUndefined()
expect(
(
harness.runtime as unknown as {
ptysById: Map<string, { connected: boolean; incarnationId?: string }>
}
).ptysById.get(harness.ptyId)
).toMatchObject({ connected: false, incarnationId: '45454545-4545-4545-8545-454545454545' })
expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(
harness.workerPaneKey,
'exited'
)
})
it.each([
{
name: 'null',
makeHasPty: () =>
vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(null).mockReturnValue(true)
},
{
name: 'throw',
makeHasPty: () =>
vi
.fn()
.mockReturnValueOnce(true)
.mockImplementationOnce(() => {
throw new Error('provider unavailable')
})
.mockReturnValue(true)
}
])(
'retries post-reveal $name liveness without duplicating the worker surface',
async ({ makeHasPty }) => {
vi.useFakeTimers()
try {
const hasPty = makeHasPty()
const harness = makePostRevealWorkerRecoveryHarness(hasPty)
await expect(
harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true })
).resolves.toMatchObject({
adoptedDispatchIds: [],
exitedDispatchIds: [],
deferredDispatchIds: ['dispatch-post-reveal']
})
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([
expect.objectContaining({ id: 'legacy-post-reveal', ptyId: harness.ptyId })
])
expect(
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
?.automaticResumeBlockedBy
).toBe('legacy-orchestration-worker')
await vi.advanceTimersByTimeAsync(1_000)
expect(hasPty).toHaveBeenCalledTimes(4)
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([
expect.objectContaining({ id: 'legacy-post-reveal', ptyId: harness.ptyId })
])
expect(
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
).toBeUndefined()
expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(
harness.workerPaneKey,
'adopted'
)
await vi.advanceTimersByTimeAsync(30_000)
expect(hasPty).toHaveBeenCalledTimes(4)
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
} finally {
vi.useRealTimers()
}
}
)
it('keeps the legacy worker resume fence in memory when persistence fails', async () => {
const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}`
const incarnationId = '99999999-9999-4999-8999-999999999999'
@@ -16768,6 +17063,7 @@ describe('OrcaRuntimeService', () => {
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
hasPty: (ptyId) => ptyId === 'pty-persistence-failure',
listProcesses: async () => [
{
id: 'pty-persistence-failure',
@@ -16804,8 +17100,9 @@ describe('OrcaRuntimeService', () => {
warn.mockRestore()
})
it('unblocks provider resume only after authoritative inventory proves the legacy PTY exited', async () => {
it('retires provider resume only after authoritative inventory proves the legacy PTY exited', async () => {
const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}`
const secondWorkerPaneKey = `legacy-worker-two:${HEADLESS_SECOND_LEAF_ID}`
const incarnationId = '33333333-3333-4333-8333-333333333333'
const session: WorkspaceSessionState = {
...getDefaultWorkspaceSession(),
@@ -16822,6 +17119,18 @@ describe('OrcaRuntimeService', () => {
capturedAt: 1,
updatedAt: 1,
origin: 'live'
},
[secondWorkerPaneKey]: {
paneKey: secondWorkerPaneKey,
tabId: 'legacy-worker-two',
worktreeId: TEST_WORKTREE_ID,
agent: 'codex',
providerSession: { key: 'session_id', id: 'legacy-codex-session-two' },
prompt: 'continue',
state: 'working',
capturedAt: 1,
updatedAt: 1,
origin: 'live'
}
}
}
@@ -16844,17 +17153,50 @@ describe('OrcaRuntimeService', () => {
worker_state: 'ready',
worktree_id: TEST_WORKTREE_ID,
agent_terminal_handle: 'term_exited'
},
{
dispatch_id: 'dispatch-exited-two',
task_id: 'task-exited-two',
dispatch_status: 'completed',
contract_version: 0,
assignee_handle: 'term_exited_two',
assignee_pane_key: secondWorkerPaneKey,
process_incarnation: `pty-exited-two:${incarnationId}`,
worker_state: 'ready',
worktree_id: TEST_WORKTREE_ID,
agent_terminal_handle: 'term_exited_two'
}
]
} as unknown as OrchestrationDb)
const listProcesses = vi.fn(async (connectionId?: string | null) => {
if (connectionId !== null) {
throw new Error('unrelated SSH inventory must not run')
}
return [
{
id: 'pty-exited-two',
incarnationId,
terminalHandle: 'term_exited_two',
title: 'Exited worker',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID,
wslDistro: null
}
]
})
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
listProcesses: async () => []
hasPty: () => false,
listProcesses
})
const revealTerminalSession = vi.fn()
const resolveLegacyWorkerTerminalRecovery = vi.fn()
runtime.setNotifier({ resolveLegacyWorkerTerminalRecovery } as never)
runtime.setNotifier({
revealTerminalSession,
resolveLegacyWorkerTerminalRecovery
} as never)
runtime.prepareLegacyWorkerTerminalRecovery()
expect(
@@ -16863,16 +17205,27 @@ describe('OrcaRuntimeService', () => {
await expect(runtime.reconcileLegacyWorkerTerminals()).resolves.toMatchObject({
adoptedDispatchIds: [],
exitedDispatchIds: ['dispatch-exited'],
exitedDispatchIds: ['dispatch-exited', 'dispatch-exited-two'],
deferredDispatchIds: []
})
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
expect(getSession().sleepingAgentSessionsByPaneKey?.[secondWorkerPaneKey]).toBeUndefined()
expect(listProcesses).toHaveBeenCalledOnce()
expect(listProcesses).toHaveBeenCalledWith(null)
expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([])
expect(revealTerminalSession).not.toHaveBeenCalled()
expect(
getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy
).toBeUndefined()
(
runtime as unknown as {
ptysById: Map<string, { connected: boolean }>
}
).ptysById.get('pty-exited-two')
).toMatchObject({ connected: false })
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'exited')
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(secondWorkerPaneKey, 'exited')
})
it('keeps provider resume fenced when authoritative PTY inventory is unavailable', async () => {
it('retries inventory and unknown liveness without revealing a ghost worker', async () => {
const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}`
const incarnationId = '55555555-5555-4555-8555-555555555555'
const session: WorkspaceSessionState = {
@@ -16915,23 +17268,263 @@ describe('OrcaRuntimeService', () => {
}
]
} as unknown as OrchestrationDb)
const listProcesses = vi
.fn()
.mockRejectedValueOnce(new Error('local provider unavailable'))
.mockResolvedValue([
{
id: 'pty-inventory-unavailable',
incarnationId,
terminalHandle: 'term_inventory_unavailable',
title: 'Recovered legacy worker',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID,
wslDistro: null
}
])
const hasPty = vi.fn().mockReturnValueOnce(null).mockReturnValue(true)
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null
getForegroundProcess: async () => null,
hasPty,
listProcesses
})
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-worker' })
const resolveLegacyWorkerTerminalRecovery = vi.fn()
runtime.setNotifier({ resolveLegacyWorkerTerminalRecovery } as never)
runtime.setNotifier({
revealTerminalSession,
resolveLegacyWorkerTerminalRecovery
} as never)
vi.useFakeTimers()
try {
await expect(
runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true })
).resolves.toMatchObject({
adoptedDispatchIds: [],
exitedDispatchIds: [],
deferredDispatchIds: ['dispatch-inventory-unavailable']
})
expect(
getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy
).toBe('legacy-orchestration-worker')
expect(resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled()
expect(listProcesses).toHaveBeenCalledOnce()
expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([])
expect(revealTerminalSession).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(1_000)
expect(listProcesses).toHaveBeenCalledTimes(2)
expect(listProcesses.mock.calls).toEqual([[null], [null]])
expect(hasPty).toHaveBeenCalledOnce()
expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([])
expect(revealTerminalSession).not.toHaveBeenCalled()
expect(
getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy
).toBe('legacy-orchestration-worker')
await vi.advanceTimersByTimeAsync(2_000)
expect(listProcesses).toHaveBeenCalledTimes(3)
expect(hasPty).toHaveBeenCalledTimes(3)
expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([
expect.objectContaining({ id: 'legacy-worker', ptyId: 'pty-inventory-unavailable' })
])
expect(revealTerminalSession).toHaveBeenCalledOnce()
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted')
await vi.advanceTimersByTimeAsync(30_000)
expect(listProcesses).toHaveBeenCalledTimes(3)
expect(revealTerminalSession).toHaveBeenCalledOnce()
} finally {
vi.useRealTimers()
}
})
it('cancels a coalesced SSH worker recovery retry when its provider disconnects', async () => {
vi.useFakeTimers()
try {
const runtime = new OrcaRuntimeService(store)
const reconcile = vi.spyOn(runtime, 'reconcileLegacyWorkerTerminals').mockResolvedValue({
blockedPaneCount: 1,
adoptedDispatchIds: [],
exitedDispatchIds: [],
deferredDispatchIds: []
})
const retryInternals = runtime as unknown as {
updateLegacyWorkerTerminalRecoveryRetry: (
plan: {
candidates: { dispatchId: string; ptyId: string }[]
},
deferredDispatchIds: ReadonlySet<string>,
options: { connectionId?: string; materializeRenderer?: boolean }
) => void
}
const plan = {
candidates: [
{
dispatchId: 'dispatch-ssh-retry',
ptyId: 'ssh:ssh-retry@@pty-worker'
}
]
}
const deferred = new Set(['dispatch-ssh-retry'])
retryInternals.updateLegacyWorkerTerminalRecoveryRetry(plan, deferred, {
connectionId: 'ssh-retry',
materializeRenderer: true
})
retryInternals.updateLegacyWorkerTerminalRecoveryRetry(plan, deferred, {
connectionId: 'ssh-retry',
materializeRenderer: true
})
runtime.notifySshStateChanged('ssh-retry', {
targetId: 'ssh-retry',
status: 'disconnected',
error: null,
reconnectAttempt: 0
})
await vi.advanceTimersByTimeAsync(30_000)
expect(reconcile).not.toHaveBeenCalled()
} finally {
vi.useRealTimers()
}
})
it('keeps live workers fenced without exact controller identity evidence', async () => {
const incarnationId = '56565656-5656-4656-8656-565656565656'
const cases = [
{
name: 'missing',
leafId: '11111111-1111-4111-8111-111111111111',
terminalHandle: 'term_missing'
},
{
name: 'ambiguous',
leafId: '22222222-2222-4222-8222-222222222222',
terminalHandle: 'term_ambiguous'
},
{
name: 'wrong-handle',
leafId: '33333333-3333-4333-8333-333333333333',
terminalHandle: 'term_wrong_handle'
},
{
name: 'wrong-incarnation',
leafId: '44444444-4444-4444-8444-444444444444',
terminalHandle: 'term_wrong_incarnation'
}
] as const
const sleepingAgentSessionsByPaneKey = Object.fromEntries(
cases.map(({ name, leafId }) => {
const paneKey = `legacy-${name}:${leafId}`
return [
paneKey,
{
paneKey,
tabId: `legacy-${name}`,
worktreeId: TEST_WORKTREE_ID,
agent: 'codex',
providerSession: { key: 'session_id', id: `session-${name}` },
prompt: 'continue',
state: 'working',
capturedAt: 1,
updatedAt: 1,
origin: 'live'
}
]
})
) as WorkspaceSessionState['sleepingAgentSessionsByPaneKey']
const session: WorkspaceSessionState = {
...getDefaultWorkspaceSession(),
tabsByWorktree: { [TEST_WORKTREE_ID]: [] },
sleepingAgentSessionsByPaneKey
}
const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session)
const runtime = new OrcaRuntimeService(
{ ...runtimeStore, flushOrThrow: vi.fn() } as never,
undefined,
{ canRecoverPersistentLocalPtys: () => true }
)
runtime.setOrchestrationDb({
listLegacyWorkerTerminalRecoveryRows: () =>
cases.map(({ name, leafId, terminalHandle }) => ({
dispatch_id: `dispatch-${name}`,
task_id: `task-${name}`,
dispatch_status: 'completed',
contract_version: 0,
assignee_handle: terminalHandle,
assignee_pane_key: `legacy-${name}:${leafId}`,
process_incarnation: `pty-${name}:${incarnationId}`,
worker_state: 'ready',
worktree_id: TEST_WORKTREE_ID,
agent_terminal_handle: terminalHandle
}))
} as unknown as OrchestrationDb)
const listProcesses = vi.fn(async () => [
{
id: 'pty-missing',
title: 'Missing identity',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID
},
{
id: 'pty-ambiguous',
incarnationId,
terminalHandle: 'term_ambiguous',
title: 'Ambiguous identity',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID
},
{
id: 'pty-ambiguous-other',
incarnationId,
terminalHandle: 'term_ambiguous',
title: 'Ambiguous identity duplicate',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID
},
{
id: 'pty-wrong-handle',
incarnationId,
terminalHandle: 'term_other',
title: 'Wrong handle',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID
},
{
id: 'pty-wrong-incarnation',
incarnationId: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
terminalHandle: 'term_wrong_incarnation',
title: 'Wrong incarnation',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID
}
])
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
hasPty: (candidate) => candidate === 'pty-folder-legacy',
listProcesses
})
await expect(runtime.reconcileLegacyWorkerTerminals()).resolves.toMatchObject({
adoptedDispatchIds: [],
exitedDispatchIds: [],
deferredDispatchIds: ['dispatch-inventory-unavailable']
deferredDispatchIds: cases.map(({ name }) => `dispatch-${name}`)
})
expect(
getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy
).toBe('legacy-orchestration-worker')
expect(resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled()
expect(listProcesses).toHaveBeenCalledOnce()
expect(listProcesses).toHaveBeenCalledWith(null)
for (const { name, leafId } of cases) {
expect(
getSession().sleepingAgentSessionsByPaneKey?.[`legacy-${name}:${leafId}`]
?.automaticResumeBlockedBy
).toBe('legacy-orchestration-worker')
}
})
it('adopts an exact live legacy worker in a folder workspace', async () => {
@@ -16986,21 +17579,23 @@ describe('OrcaRuntimeService', () => {
}
]
} as unknown as OrchestrationDb)
const listProcesses = vi.fn(async () => [
{
id: 'pty-folder-legacy',
incarnationId,
terminalHandle: 'term_folder',
title: 'Folder worker',
cwd: folderPath,
worktreeId: TEST_FOLDER_WORKSPACE_KEY,
wslDistro: null
}
])
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
listProcesses: async () => [
{
id: 'pty-folder-legacy',
incarnationId,
terminalHandle: 'term_folder',
title: 'Folder worker',
cwd: folderPath,
worktreeId: TEST_FOLDER_WORKSPACE_KEY,
wslDistro: null
}
]
hasPty: (candidate) => candidate === 'pty-folder-legacy',
listProcesses
})
const revealTerminalSession = vi.fn().mockResolvedValue({
tabId: 'legacy-folder-worker'
@@ -17022,6 +17617,8 @@ describe('OrcaRuntimeService', () => {
})
)
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
expect(listProcesses).toHaveBeenCalledOnce()
expect(listProcesses).toHaveBeenCalledWith(null)
expect(revealTerminalSession).toHaveBeenCalledWith(TEST_FOLDER_WORKSPACE_KEY, {
ptyId: 'pty-folder-legacy',
title: 'Folder worker',
@@ -17029,7 +17626,11 @@ describe('OrcaRuntimeService', () => {
presentation: 'background',
tabId: 'legacy-folder-worker',
leafId: HEADLESS_LEAF_ID,
focus: false
focus: false,
expectedProcessIdentity: {
terminalHandle: 'term_folder',
incarnationId
}
})
})
@@ -17101,21 +17702,23 @@ describe('OrcaRuntimeService', () => {
}
]
} as unknown as OrchestrationDb)
const listProcesses = vi.fn(async () => [
{
id: ptyId,
incarnationId,
terminalHandle: 'term_ssh_folder',
title: 'SSH folder worker',
cwd: folderPath,
worktreeId: TEST_FOLDER_WORKSPACE_KEY,
wslDistro: null
}
])
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
listProcesses: async () => [
{
id: ptyId,
incarnationId,
terminalHandle: 'term_ssh_folder',
title: 'SSH folder worker',
cwd: folderPath,
worktreeId: TEST_FOLDER_WORKSPACE_KEY,
wslDistro: null
}
]
hasPty: (candidate) => candidate === ptyId,
listProcesses
})
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-ssh-folder-worker' })
runtime.setNotifier({ revealTerminalSession } as never)
@@ -17149,6 +17752,8 @@ describe('OrcaRuntimeService', () => {
expect(localSession.sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
expect(getWorkspaceSession).toHaveBeenCalledWith(`ssh:${connectionId}`)
expect(setWorkspaceSession).toHaveBeenCalledWith(expect.any(Object), `ssh:${connectionId}`)
expect(listProcesses).toHaveBeenCalledOnce()
expect(listProcesses).toHaveBeenCalledWith(connectionId)
expect(sshSession.tabsByWorktree[TEST_FOLDER_WORKSPACE_KEY]).toContainEqual(
expect.objectContaining({
id: 'legacy-ssh-folder-worker',
@@ -17163,7 +17768,11 @@ describe('OrcaRuntimeService', () => {
presentation: 'background',
tabId: 'legacy-ssh-folder-worker',
leafId: HEADLESS_LEAF_ID,
focus: false
focus: false,
expectedProcessIdentity: {
terminalHandle: 'term_ssh_folder',
incarnationId
}
})
})
@@ -17321,6 +17930,7 @@ describe('OrcaRuntimeService', () => {
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
hasPty: (candidate) => candidate === ptyId,
listProcesses
})
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-ssh-worker' })
@@ -17365,12 +17975,29 @@ describe('OrcaRuntimeService', () => {
exitedDispatchIds: [],
deferredDispatchIds: []
})
await expect(
runtime.adoptTerminalOrphans({
worktree: `id:${TEST_WORKTREE_ID}`,
expectedTopologyRevision: 0,
claims: [
{
terminal: 'term_ssh_legacy',
ptyId,
incarnationId,
tabId: 'legacy-ssh-worker',
leafId: HEADLESS_LEAF_ID
}
]
})
).resolves.toMatchObject({ adopted: false })
expect(listProcesses).toHaveBeenLastCalledWith(connectionId)
} finally {
unregisterSshGitProvider(connectionId)
}
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
expect(setWorkspaceSession).toHaveBeenCalledWith(expect.any(Object), `ssh:${connectionId}`)
expect(listProcesses).toHaveBeenCalledTimes(2)
expect(revealTerminalSession).toHaveBeenCalledWith(TEST_WORKTREE_ID, {
ptyId,
title: 'SSH legacy worker',
@@ -17378,7 +18005,11 @@ describe('OrcaRuntimeService', () => {
presentation: 'background',
tabId: 'legacy-ssh-worker',
leafId: HEADLESS_LEAF_ID,
focus: false
focus: false,
expectedProcessIdentity: {
terminalHandle: 'term_ssh_legacy',
incarnationId
}
})
})
@@ -17445,21 +18076,23 @@ describe('OrcaRuntimeService', () => {
}
]
} as unknown as OrchestrationDb)
const listProcesses = vi.fn(async () => [
{
id: 'pty-wsl-legacy',
incarnationId,
terminalHandle: 'term_wsl_legacy',
title: 'WSL legacy worker',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID,
wslDistro: observedDistro
}
])
runtime.setPtyController({
write: vi.fn(() => true),
kill: vi.fn(() => true),
getForegroundProcess: async () => null,
listProcesses: async () => [
{
id: 'pty-wsl-legacy',
incarnationId,
terminalHandle: 'term_wsl_legacy',
title: 'WSL legacy worker',
cwd: TEST_WORKTREE_PATH,
worktreeId: TEST_WORKTREE_ID,
wslDistro: observedDistro
}
]
hasPty: (ptyId) => ptyId === 'pty-wsl-legacy',
listProcesses
})
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-wsl-worker' })
runtime.setNotifier({ revealTerminalSession } as never)
@@ -17486,6 +18119,8 @@ describe('OrcaRuntimeService', () => {
})
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
expect(revealTerminalSession).toHaveBeenCalledOnce()
expect(listProcesses).toHaveBeenCalledTimes(2)
expect(listProcesses.mock.calls).toEqual([[null], [null]])
})
it('restores orphan pane and group topology without replacing a newer host-owned tab', async () => {
+456 -176
View File
@@ -36,10 +36,7 @@ import {
type AgentStatusEntry
} from '../../shared/agent-status-types'
import { indexAgentStatusRowsByPaneKey } from '../agent-hooks/agent-status-pane-index'
import type {
AgentHookAuthorityAttestation,
AgentHookAuthorityEvidence
} from '../agent-hooks/server'
import type { AgentHookAuthorityAttestation } from '../agent-hooks/server'
import type {
AgentSessionClaimedSpawnResult,
AgentSessionExecutionClaim,
@@ -110,6 +107,7 @@ import {
createSetupCompletionScanner
} from './orchestration/setup-completion-signal'
import type { RuntimeOrchestrationEnvelope } from '../../shared/runtime-rpc-envelope'
import type { TerminalRevealIdentity } from '../../shared/terminal-reveal-identity'
import type {
OrchestrationCompatibilityEvidence,
OrchestrationCompatibilityHostStamp
@@ -1506,7 +1504,7 @@ type RuntimePtyController = {
resize?(ptyId: string, cols: number, rows: number): boolean
// Why: exact-id mobile polls should not enumerate every local and SSH PTY.
hasPty?(ptyId: string): boolean | null
listProcesses?(): Promise<PtyProcessInfo[]>
listProcesses?(connectionId?: string | null): Promise<PtyProcessInfo[]>
serializeBuffer?(
ptyId: string,
opts?: { scrollbackRows?: number; altScreenForcesZeroRows?: boolean }
@@ -1530,6 +1528,17 @@ type RuntimePtyController = {
getSize?(ptyId: string): { cols: number; rows: number } | null
}
type PtyControllerTerminalIdentity = Readonly<{
handle: string
incarnationId: string
wslDistro?: string | null
}>
type PtyControllerInventory = Readonly<{
livePtyIds: ReadonlySet<string>
terminalIdentityByPtyId: ReadonlyMap<string, PtyControllerTerminalIdentity>
}>
type WorktreeStartupDraftPaste = {
agent: TuiAgent
content: string
@@ -1645,10 +1654,14 @@ type RuntimeNotifier = {
splitDirection?: 'horizontal' | 'vertical'
splitTelemetrySource?: TerminalPaneSplitSource
focus?: boolean
expectedProcessIdentity?: {
terminalHandle: string
incarnationId: string
}
}
):
| Promise<{ tabId: string; title?: string | null }>
| { tabId: string; title?: string | null }
| Promise<{ tabId: string; title?: string | null; identity?: TerminalRevealIdentity }>
| { tabId: string; title?: string | null; identity?: TerminalRevealIdentity }
| void
resolveLegacyWorkerTerminalRecovery?(paneKey: string, resolution: 'adopted' | 'exited'): void
splitTerminal(
@@ -1733,11 +1746,6 @@ export type OrchestrationCompatibilityTerminalAuthority = {
| { kind: 'ssh'; targetId: string }
}
export type OrchestrationCompatibilityAuthorityInputs = {
hydratedCommitments: readonly AgentHookAuthorityEvidence[]
currentObservations: readonly AgentHookAuthorityEvidence[]
}
export type LegacyWorkerTerminalRecoveryResult = {
blockedPaneCount: number
adoptedDispatchIds: string[]
@@ -2615,10 +2623,6 @@ export class OrcaRuntimeService {
private handles = new Map<string, TerminalHandleRecord>()
private handleByLeafKey = new Map<string, string>()
private handleByPtyId = new Map<string, string>()
private controllerTerminalIdentityByPtyId = new Map<
string,
{ handle: string; incarnationId: string; wslDistro?: string | null }
>()
private detachedPreAllocatedLeaves = new Map<string, RuntimeLeafRecord>()
private graphSyncCallbacks: (() => void)[] = []
private waitersByHandle = new Map<string, Set<TerminalWaiter>>()
@@ -3011,12 +3015,6 @@ export class OrcaRuntimeService {
private readonly getAgentProviderSessionRowsForPaneFn:
| ((paneKey: string) => AgentStatusIpcPayload[])
| null
private readonly getHydratedAgentHookAuthorityFn:
| (() => readonly AgentHookAuthorityEvidence[])
| null
private readonly getCurrentAgentHookAuthorityFn:
| (() => readonly AgentHookAuthorityEvidence[])
| null
private readonly attestAgentHookCompatibilityAuthorityFn:
| ((candidate: {
paneKey: string
@@ -3038,7 +3036,17 @@ export class OrcaRuntimeService {
>()
private sshRelayRecoveryGenerationByTargetId = new Map<string, number>()
private legacyWorkerTerminalRecoveryQueue: Promise<void> = Promise.resolve()
private legacyWorkerTerminalRecoveryRetries = new Map<
string,
{
attempt: number
connectionId?: string
materializeRenderer: boolean
timer: ReturnType<typeof setTimeout> | null
}
>()
private legacyWorkerTerminalMaterializedPanes = new Set<string>()
private legacyWorkerTerminalIdentityReceiptPanes = new Set<string>()
private legacyWorkerRecoveredPtys = new Set<string>()
private accountServices: RuntimeAccountServices | null = null
private commitMessageAgentEnv: CommitMessageAgentEnvironmentResolvers | null = null
@@ -3074,8 +3082,6 @@ export class OrcaRuntimeService {
* only carrier of the provider session a transcript is addressed by. */
getAgentProviderSessionSnapshot?: () => AgentStatusIpcPayload[]
getAgentProviderSessionRowsForPane?: (paneKey: string) => AgentStatusIpcPayload[]
getHydratedAgentHookAuthority?: () => readonly AgentHookAuthorityEvidence[]
getCurrentAgentHookAuthority?: () => readonly AgentHookAuthorityEvidence[]
attestAgentHookCompatibilityAuthority?: (candidate: {
paneKey: string
launchTokenHash: string
@@ -3114,8 +3120,6 @@ export class OrcaRuntimeService {
this.getAgentProviderSessionSnapshotFn =
deps?.getAgentProviderSessionSnapshot ?? deps?.getAgentStatusSnapshot ?? null
this.getAgentProviderSessionRowsForPaneFn = deps?.getAgentProviderSessionRowsForPane ?? null
this.getHydratedAgentHookAuthorityFn = deps?.getHydratedAgentHookAuthority ?? null
this.getCurrentAgentHookAuthorityFn = deps?.getCurrentAgentHookAuthority ?? null
this.attestAgentHookCompatibilityAuthorityFn =
deps?.attestAgentHookCompatibilityAuthority ?? null
this.canRecoverPersistentLocalPtysFn = deps?.canRecoverPersistentLocalPtys ?? (() => true)
@@ -3659,6 +3663,70 @@ export class OrcaRuntimeService {
return result
}
private hasExactTerminalSurfaceIdentity(expected: {
worktreeId: string
tabId: string
leafId: string
ptyId: string
terminalHandle: string
incarnationId: string
}): boolean {
if (this.graphStatus !== 'ready') {
return false
}
const pty = this.ptysById.get(expected.ptyId)
if (
!pty?.connected ||
pty.incarnationId !== expected.incarnationId ||
pty.tabId !== expected.tabId ||
pty.paneKey !== makePaneKey(expected.tabId, expected.leafId) ||
!runtimeWorktreeIdsEqual(pty.worktreeId, expected.worktreeId) ||
this.handleByPtyId.get(expected.ptyId) !== expected.terminalHandle
) {
return false
}
const tab = this.tabs.get(expected.tabId)
const leaf = this.leaves.get(this.getLeafKey(expected.tabId, expected.leafId))
const ptyLeaves = this.getLeavesForPty(expected.ptyId)
return (
Boolean(tab && runtimeWorktreeIdsEqual(tab.worktreeId, expected.worktreeId)) &&
Boolean(
leaf &&
leaf.ptyId === expected.ptyId &&
runtimeWorktreeIdsEqual(leaf.worktreeId, expected.worktreeId)
) &&
ptyLeaves.length === 1 &&
ptyLeaves[0]?.tabId === expected.tabId &&
ptyLeaves[0]?.leafId === expected.leafId
)
}
private hasExactPersistedTerminalSurfaceIdentity(expected: {
worktreeId: string
tabId: string
leafId: string
ptyId: string
incarnationId: string
}): boolean {
const session = this.getWorkspaceSessionForWorktree(expected.worktreeId)
const sessionWorktreeId = session
? resolveTerminalSessionWorktreeId(session, expected.worktreeId)
: null
if (!session || !sessionWorktreeId) {
return false
}
const tab = session.tabsByWorktree[sessionWorktreeId]?.find(
(candidate) => candidate.id === expected.tabId
)
const paneKey = makePaneKey(expected.tabId, expected.leafId)
return Boolean(
tab &&
session.terminalLayoutsByTabId[expected.tabId]?.ptyIdsByLeafId?.[expected.leafId] ===
expected.ptyId &&
session.terminalPtyIncarnationsByPaneKey?.[paneKey] === expected.incarnationId
)
}
private persistLegacyWorkerTerminalRecoveryResolution(
candidate: LegacyWorkerTerminalRecoveryPlan['candidates'][number],
resolution: 'adopted' | 'exited'
@@ -3672,23 +3740,8 @@ export class OrcaRuntimeService {
if (!record || !runtimeWorktreeIdsEqual(record.worktreeId, candidate.worktreeId)) {
return true
}
if (
resolution === 'exited' &&
record.automaticResumeBlockedBy !== 'legacy-orchestration-worker'
) {
return true
}
const next = structuredClone(session)
if (resolution === 'adopted') {
delete next.sleepingAgentSessionsByPaneKey?.[candidate.paneKey]
} else {
const unblocked = { ...record }
delete unblocked.automaticResumeBlockedBy
next.sleepingAgentSessionsByPaneKey = {
...next.sleepingAgentSessionsByPaneKey,
[candidate.paneKey]: unblocked
}
}
delete next.sleepingAgentSessionsByPaneKey?.[candidate.paneKey]
try {
this.setWorkspaceSessionForWorktree(candidate.worktreeId, next)
store.flushOrThrow()
@@ -3704,6 +3757,85 @@ export class OrcaRuntimeService {
}
}
private updateLegacyWorkerTerminalRecoveryRetry(
plan: LegacyWorkerTerminalRecoveryPlan,
deferredDispatchIds: ReadonlySet<string>,
options: { connectionId?: string; materializeRenderer?: boolean }
): void {
const scopeKey = options.connectionId ? `ssh:${options.connectionId}` : 'local'
const hasDeferredWorker = plan.candidates.some((candidate) => {
const sshPty = parseAppSshPtyId(candidate.ptyId)
const inScope = options.connectionId
? sshPty?.connectionId === options.connectionId
: sshPty === null
return inScope && deferredDispatchIds.has(candidate.dispatchId)
})
if (!hasDeferredWorker) {
this.cancelLegacyWorkerTerminalRecoveryRetry(scopeKey)
return
}
const existing = this.legacyWorkerTerminalRecoveryRetries.get(scopeKey)
const retry = existing ?? {
attempt: 0,
...(options.connectionId ? { connectionId: options.connectionId } : {}),
materializeRenderer: options.materializeRenderer === true,
timer: null
}
retry.materializeRenderer ||= options.materializeRenderer === true
this.legacyWorkerTerminalRecoveryRetries.set(scopeKey, retry)
this.armLegacyWorkerTerminalRecoveryRetry(scopeKey, retry)
}
private cancelLegacyWorkerTerminalRecoveryRetry(scopeKey: string): void {
const retry = this.legacyWorkerTerminalRecoveryRetries.get(scopeKey)
if (retry?.timer) {
clearTimeout(retry.timer)
}
this.legacyWorkerTerminalRecoveryRetries.delete(scopeKey)
}
private getLegacyWorkerControllerPtyLiveness(
candidate: LegacyWorkerTerminalRecoveryPlan['candidates'][number]
): boolean | null {
try {
return this.ptyController?.hasPty?.(candidate.ptyId) ?? null
} catch {
return null
}
}
private armLegacyWorkerTerminalRecoveryRetry(
scopeKey: string,
retry: {
attempt: number
connectionId?: string
materializeRenderer: boolean
timer: ReturnType<typeof setTimeout> | null
}
): void {
if (retry.timer) {
return
}
const delayMs = Math.min(1_000 * 2 ** retry.attempt, 30_000)
retry.attempt += 1
retry.timer = setTimeout(() => {
retry.timer = null
void this.reconcileLegacyWorkerTerminals({
...(retry.connectionId ? { connectionId: retry.connectionId } : {}),
materializeRenderer: retry.materializeRenderer
}).catch((error) => {
console.warn('[orchestration] worker terminal recovery retry failed', {
scope: scopeKey,
error
})
if (this.legacyWorkerTerminalRecoveryRetries.get(scopeKey) === retry) {
this.armLegacyWorkerTerminalRecoveryRetry(scopeKey, retry)
}
})
}, delayMs)
retry.timer.unref?.()
}
private async reconcileLegacyWorkerTerminalsNow(options: {
connectionId?: string
materializeRenderer?: boolean
@@ -3712,142 +3844,251 @@ export class OrcaRuntimeService {
const adoptedDispatchIds: string[] = []
const exitedDispatchIds: string[] = []
const deferredDispatchIds = new Set(plan.ambiguousDispatchIds)
for (const candidate of plan.candidates) {
let workspace: TerminalWorkspaceLaunchScope
try {
workspace = await this.resolveTerminalWorkspaceLaunchScope(`id:${candidate.worktreeId}`)
} catch {
deferredDispatchIds.add(candidate.dispatchId)
continue
const recoveryCandidatesByProvider = new Map<
string,
{
connectionId: string | null
entries: {
candidate: (typeof plan.candidates)[number]
workspace: TerminalWorkspaceLaunchScope
resolvedWorkspace: ResolvedWorktree
}[]
}
const sshPty = parseAppSshPtyId(candidate.ptyId)
if (workspace.connectionId) {
if (
options.connectionId !== workspace.connectionId ||
sshPty?.connectionId !== workspace.connectionId
>()
for (const candidate of plan.candidates) {
try {
const workspace = await this.resolveTerminalWorkspaceLaunchScope(
`id:${candidate.worktreeId}`
)
const sshPty = parseAppSshPtyId(candidate.ptyId)
if (workspace.connectionId) {
if (
options.connectionId !== workspace.connectionId ||
sshPty?.connectionId !== workspace.connectionId
) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
} else if (
options.connectionId !== undefined ||
sshPty !== null ||
!this.canRecoverPersistentLocalPtysFn()
) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
} else if (
options.connectionId !== undefined ||
sshPty !== null ||
!this.canRecoverPersistentLocalPtysFn()
) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
const resolvedWorkspace = workspace.folderWorkspace
? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace)
: await this.resolveWorktreeSelector(`id:${workspace.id}`)
const livePtyIds = await this.refreshPtyWorktreeRecordsFromController([resolvedWorkspace])
if (!livePtyIds) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
if (!livePtyIds.has(candidate.ptyId)) {
if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) {
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited')
exitedDispatchIds.push(candidate.dispatchId)
} else {
deferredDispatchIds.add(candidate.dispatchId)
const resolvedWorkspace = workspace.folderWorkspace
? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace)
: await this.resolveWorktreeSelector(`id:${workspace.id}`)
const connectionId = workspace.connectionId ?? null
const providerKey = connectionId === null ? 'local' : `ssh:${connectionId}`
const provider = recoveryCandidatesByProvider.get(providerKey) ?? {
connectionId,
entries: []
}
continue
}
const controllerIdentity = this.controllerTerminalIdentityByPtyId.get(candidate.ptyId)
if (
controllerIdentity?.handle !== candidate.terminalHandle ||
controllerIdentity.incarnationId !== candidate.incarnationId
) {
provider.entries.push({ candidate, workspace, resolvedWorkspace })
recoveryCandidatesByProvider.set(providerKey, provider)
} catch {
deferredDispatchIds.add(candidate.dispatchId)
}
}
for (const provider of recoveryCandidatesByProvider.values()) {
const resolvedWorktrees = [
...new Map(
provider.entries.map(({ resolvedWorkspace }) => [resolvedWorkspace.id, resolvedWorkspace])
).values()
]
const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory(
resolvedWorktrees,
null,
undefined,
provider.connectionId
)
if (!inventory) {
provider.entries.forEach(({ candidate }) => deferredDispatchIds.add(candidate.dispatchId))
continue
}
const session = this.getWorkspaceSessionForWorktree(candidate.worktreeId)
const sessionWorktreeId = session
? resolveTerminalSessionWorktreeId(session, candidate.worktreeId)
: null
const activeTabId = sessionWorktreeId
? session?.activeTabIdByWorktree?.[sessionWorktreeId]
: undefined
const activeGroupId = sessionWorktreeId
? session?.activeGroupIdByWorktree?.[sessionWorktreeId]
: undefined
try {
await this.adoptTerminalOrphans({
worktree: `id:${candidate.worktreeId}`,
expectedTopologyRevision: this.getTerminalTopologyRevision(candidate.worktreeId),
...(activeTabId ? { activeTabId } : {}),
...(activeGroupId ? { activeGroupId } : {}),
claims: [
{
terminal: candidate.terminalHandle,
ptyId: candidate.ptyId,
incarnationId: candidate.incarnationId,
tabId: candidate.tabId,
leafId: candidate.leafId
}
]
})
} catch (error) {
console.warn('[orchestration] legacy worker terminal adoption deferred', {
dispatchId: candidate.dispatchId,
error
})
deferredDispatchIds.add(candidate.dispatchId)
continue
}
let rendererMaterialized =
options.materializeRenderer !== true ||
this.legacyWorkerTerminalMaterializedPanes.has(candidate.paneKey)
const pty = this.ptysById.get(candidate.ptyId)
if (
options.materializeRenderer &&
!rendererMaterialized &&
pty &&
this.notifier?.revealTerminalSession
) {
for (let attempt = 0; attempt < 2 && !rendererMaterialized; attempt += 1) {
for (const { candidate, workspace } of provider.entries) {
if (!inventory.livePtyIds.has(candidate.ptyId)) {
if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) {
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited')
exitedDispatchIds.push(candidate.dispatchId)
} else {
deferredDispatchIds.add(candidate.dispatchId)
}
continue
}
const controllerIdentity = inventory.terminalIdentityByPtyId.get(candidate.ptyId)
if (
controllerIdentity?.handle !== candidate.terminalHandle ||
controllerIdentity.incarnationId !== candidate.incarnationId
) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
const controllerLiveness = this.getLegacyWorkerControllerPtyLiveness(candidate)
if (controllerLiveness === false) {
this.onPtyExit(candidate.ptyId, 0, candidate.incarnationId)
if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) {
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited')
exitedDispatchIds.push(candidate.dispatchId)
} else {
deferredDispatchIds.add(candidate.dispatchId)
}
continue
}
if (controllerLiveness === null) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
const session = this.getWorkspaceSessionForWorktree(candidate.worktreeId)
const sessionWorktreeId = session
? resolveTerminalSessionWorktreeId(session, candidate.worktreeId)
: null
const activeTabId = sessionWorktreeId
? session?.activeTabIdByWorktree?.[sessionWorktreeId]
: undefined
const activeGroupId = sessionWorktreeId
? session?.activeGroupIdByWorktree?.[sessionWorktreeId]
: undefined
const exactSurfaceAlreadyPublished =
this.hasExactPersistedTerminalSurfaceIdentity(candidate) &&
this.hasExactTerminalSurfaceIdentity(candidate)
if (!exactSurfaceAlreadyPublished) {
try {
await this.notifier.revealTerminalSession(candidate.worktreeId, {
ptyId: candidate.ptyId,
title: getLatestPtyTitle(pty) ?? pty.controllerTitle,
activate: false,
presentation: 'background',
tabId: candidate.tabId,
leafId: candidate.leafId,
focus: false
})
rendererMaterialized = true
this.legacyWorkerTerminalMaterializedPanes.add(candidate.paneKey)
await this.adoptTerminalOrphansFromInventory(
{
worktree: `id:${candidate.worktreeId}`,
expectedTopologyRevision: this.getTerminalTopologyRevision(candidate.worktreeId),
...(activeTabId ? { activeTabId } : {}),
...(activeGroupId ? { activeGroupId } : {}),
claims: [
{
terminal: candidate.terminalHandle,
ptyId: candidate.ptyId,
incarnationId: candidate.incarnationId,
tabId: candidate.tabId,
leafId: candidate.leafId
}
]
},
workspace,
inventory
)
} catch (error) {
if (attempt === 0) {
await new Promise<void>((resolve) => setTimeout(resolve, 100))
continue
}
console.warn('[orchestration] adopted legacy worker was not revealed', {
console.warn('[orchestration] legacy worker terminal adoption deferred', {
dispatchId: candidate.dispatchId,
error
})
deferredDispatchIds.add(candidate.dispatchId)
continue
}
}
let rendererMaterialized =
options.materializeRenderer !== true ||
this.legacyWorkerTerminalMaterializedPanes.has(candidate.paneKey)
const pty = this.ptysById.get(candidate.ptyId)
if (
options.materializeRenderer &&
!rendererMaterialized &&
pty &&
this.notifier?.revealTerminalSession
) {
for (let attempt = 0; attempt < 2 && !rendererMaterialized; attempt += 1) {
try {
const reveal = await this.notifier.revealTerminalSession(candidate.worktreeId, {
ptyId: candidate.ptyId,
title: getLatestPtyTitle(pty) ?? pty.controllerTitle,
activate: false,
presentation: 'background',
tabId: candidate.tabId,
leafId: candidate.leafId,
focus: false,
expectedProcessIdentity: {
terminalHandle: candidate.terminalHandle,
incarnationId: candidate.incarnationId
}
})
if (reveal?.identity) {
const identity = reveal.identity
if (
!runtimeWorktreeIdsEqual(identity.worktreeId, candidate.worktreeId) ||
identity.tabId !== candidate.tabId ||
identity.leafId !== candidate.leafId ||
identity.ptyId !== candidate.ptyId
) {
throw new Error('terminal_reveal_identity_mismatch')
}
this.legacyWorkerTerminalIdentityReceiptPanes.add(candidate.paneKey)
}
rendererMaterialized = true
this.legacyWorkerTerminalMaterializedPanes.add(candidate.paneKey)
} catch (error) {
if (attempt === 0) {
await new Promise<void>((resolve) => setTimeout(resolve, 100))
continue
}
console.warn('[orchestration] adopted legacy worker was not revealed', {
dispatchId: candidate.dispatchId,
error
})
}
}
}
if (!rendererMaterialized) {
this.legacyWorkerTerminalMaterializedPanes.delete(candidate.paneKey)
this.legacyWorkerTerminalIdentityReceiptPanes.delete(candidate.paneKey)
deferredDispatchIds.add(candidate.dispatchId)
continue
}
if (
this.legacyWorkerTerminalIdentityReceiptPanes.has(candidate.paneKey) &&
!this.hasExactTerminalSurfaceIdentity({
worktreeId: candidate.worktreeId,
tabId: candidate.tabId,
leafId: candidate.leafId,
ptyId: candidate.ptyId,
terminalHandle: candidate.terminalHandle,
incarnationId: candidate.incarnationId
})
) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
const postRevealLiveness = this.getLegacyWorkerControllerPtyLiveness(candidate)
if (postRevealLiveness === false) {
this.legacyWorkerTerminalMaterializedPanes.delete(candidate.paneKey)
this.legacyWorkerTerminalIdentityReceiptPanes.delete(candidate.paneKey)
this.onPtyExit(candidate.ptyId, 0, candidate.incarnationId)
if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) {
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited')
exitedDispatchIds.push(candidate.dispatchId)
} else {
deferredDispatchIds.add(candidate.dispatchId)
}
continue
}
if (postRevealLiveness === null) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
if (!this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'adopted')) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
this.legacyWorkerRecoveredPtys.add(candidate.ptyId)
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'adopted')
adoptedDispatchIds.push(candidate.dispatchId)
}
if (
!rendererMaterialized ||
!this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'adopted')
) {
deferredDispatchIds.add(candidate.dispatchId)
continue
}
this.legacyWorkerRecoveredPtys.add(candidate.ptyId)
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'adopted')
adoptedDispatchIds.push(candidate.dispatchId)
}
return {
const result = {
blockedPaneCount: plan.blockedPanes.length,
adoptedDispatchIds,
exitedDispatchIds,
deferredDispatchIds: [...deferredDispatchIds]
}
this.updateLegacyWorkerTerminalRecoveryRetry(plan, deferredDispatchIds, options)
return result
}
setAutomationService(service: AutomationService): void {
@@ -4213,6 +4454,9 @@ export class OrcaRuntimeService {
notifySshStateChanged(targetId: string, state: SshConnectionState): void {
this.bumpSshRelayRecoveryGeneration(targetId)
this.invalidateSshWorktreeScanCache(targetId)
if (state.status !== 'connected') {
this.cancelLegacyWorkerTerminalRecoveryRetry(`ssh:${targetId}`)
}
this.emitClientEvent({ type: 'sshStateChanged', targetId, state: getPublicSshState(state)! })
}
@@ -10345,13 +10589,6 @@ export class OrcaRuntimeService {
return 'local'
}
getOrchestrationCompatibilityAuthorityInputs(): OrchestrationCompatibilityAuthorityInputs {
return {
hydratedCommitments: this.getHydratedAgentHookAuthorityFn?.() ?? [],
currentObservations: this.getCurrentAgentHookAuthorityFn?.() ?? []
}
}
registerOrchestrationCompatibilitySshAttachment(
targetId: string,
connectionIncarnation: string
@@ -13554,13 +13791,24 @@ export class OrcaRuntimeService {
const resolvedWorkspace = workspace.folderWorkspace
? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace)
: await this.resolveWorktreeSelector(`id:${workspace.id}`)
const livePtyIds = await this.refreshPtyWorktreeRecordsFromController(
const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory(
[resolvedWorkspace],
workspace.id
workspace.id,
undefined,
workspace.connectionId ?? null
)
if (!livePtyIds) {
if (!inventory) {
throw new Error('terminal_liveness_unavailable')
}
return this.adoptTerminalOrphansFromInventory(request, workspace, inventory)
}
private async adoptTerminalOrphansFromInventory(
request: RuntimeTerminalOrphanAdoptionRequest,
workspace: TerminalWorkspaceLaunchScope,
inventory: PtyControllerInventory
): Promise<RuntimeTerminalOrphanAdoptionResult> {
const { livePtyIds, terminalIdentityByPtyId } = inventory
const store = this.store
const session = this.getWorkspaceSessionForWorktree(workspace.id)
if (!store?.setWorkspaceSession || !store.flushOrThrow || !session) {
@@ -13593,7 +13841,7 @@ export class OrcaRuntimeService {
seenPaneKeys.add(paneKey)
const live = this.getLivePtyForHandle(claim.terminal)
const pty = live?.pty
const controllerIdentity = this.controllerTerminalIdentityByPtyId.get(claim.ptyId)
const controllerIdentity = terminalIdentityByPtyId.get(claim.ptyId)
if (
!pty ||
pty.ptyId !== claim.ptyId ||
@@ -13681,10 +13929,14 @@ export class OrcaRuntimeService {
)
})
if (isExactPersisted && sessionWorktreeId === workspace.id) {
for (const { claim, pty, paneKey } of validated) {
pty.tabId = claim.tabId
pty.paneKey = paneKey
}
return {
adopted: false,
topologyRevision: currentRevision,
snapshot: await this.listMobileSessionTabs(`id:${workspace.id}`)
snapshot: this.getTerminalOrphanAdoptionSnapshot(workspace.id)
}
}
if (currentRevision !== request.expectedTopologyRevision) {
@@ -13999,10 +14251,19 @@ export class OrcaRuntimeService {
return {
adopted: true,
topologyRevision: persisted.terminalTopologyRevisionByRepoId?.[repoId] ?? currentRevision + 1,
snapshot: await this.listMobileSessionTabs(`id:${workspace.id}`)
snapshot: this.getTerminalOrphanAdoptionSnapshot(workspace.id)
}
}
private getTerminalOrphanAdoptionSnapshot(worktreeId: string): RuntimeMobileSessionTabsResult {
this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId, {
allowAttachedWindow: true,
onlyRuntimeOwnedTerminals: true
})
this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId)
return this.getMobileSessionTabsForWorktree(worktreeId)
}
private buildTerminalVisualLayouts(
terminals: RuntimeTerminalSummary[],
worktreesById: Map<string, ResolvedWorktree>,
@@ -26302,17 +26563,34 @@ export class OrcaRuntimeService {
targetWorktreeId: string | null = null,
deadline?: number
): Promise<Set<string> | null> {
const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory(
resolvedWorktrees,
targetWorktreeId,
deadline
)
return inventory ? new Set(inventory.livePtyIds) : null
}
private async refreshPtyWorktreeRecordsWithControllerInventory(
resolvedWorktrees: ResolvedWorktree[],
targetWorktreeId: string | null = null,
deadline?: number,
connectionId?: string | null
): Promise<PtyControllerInventory | null> {
if (targetWorktreeId === FLOATING_TERMINAL_WORKTREE_ID) {
const targetedLiveness = this.refreshFloatingWorkspacePtyLiveness()
if (targetedLiveness !== null) {
return targetedLiveness
return {
livePtyIds: targetedLiveness,
terminalIdentityByPtyId: new Map()
}
}
}
if (!this.ptyController?.listProcesses) {
return null
}
const sessionsResult = await withTimeoutResult(
this.ptyController.listProcesses(),
this.ptyController.listProcesses(connectionId),
deadline === undefined
? PTY_CONTROLLER_LIST_TIMEOUT_MS
: Math.max(1, Math.min(PTY_CONTROLLER_LIST_TIMEOUT_MS, deadline - Date.now()))
@@ -26322,10 +26600,7 @@ export class OrcaRuntimeService {
return null
}
const sessions = sessionsResult.value
const controllerIdentityByPtyId = new Map<
string,
{ handle: string; incarnationId: string; wslDistro?: string | null }
>()
const controllerIdentityByPtyId = new Map<string, PtyControllerTerminalIdentity>()
const ptyIdByControllerHandle = new Map<string, string>()
const ambiguousControllerPtyIds = new Set<string>()
for (const session of sessions) {
@@ -26356,7 +26631,6 @@ export class OrcaRuntimeService {
for (const ptyId of ambiguousControllerPtyIds) {
controllerIdentityByPtyId.delete(ptyId)
}
this.controllerTerminalIdentityByPtyId = controllerIdentityByPtyId
const persistedWorktreeIdByPtyId = indexPersistedPtyWorktreeBindings(
this.store?.getWorkspaceSession?.()
)
@@ -26432,6 +26706,9 @@ export class OrcaRuntimeService {
this.refreshPtyForegroundAgent(session.id)
}
for (const pty of this.ptysById.values()) {
if (connectionId !== undefined && pty.connectionId !== connectionId) {
continue
}
if (!allLivePtyIds.has(pty.ptyId) && !this.leafExistsForPty(pty.ptyId)) {
if (this.ptyController.hasPty?.(pty.ptyId) === true) {
// Why: an SSH spawn can become addressable before an overlapping relay list includes it.
@@ -26451,7 +26728,10 @@ export class OrcaRuntimeService {
}
}
this.pruneDisconnectedPtyRecords()
return targetWorktreeId ? selectedLivePtyIds : allLivePtyIds
return {
livePtyIds: targetWorktreeId ? selectedLivePtyIds : allLivePtyIds,
terminalIdentityByPtyId: controllerIdentityByPtyId
}
}
private refreshFloatingWorkspacePtyLiveness(): Set<string> | null {
+261 -52
View File
@@ -171,10 +171,10 @@ function legacyMessageMatchesQuestion(
message: MessageRow,
question: string,
options: string[],
recipientHandle: string
recipientHandles: readonly string[]
): boolean {
if (
message.to_handle !== recipientHandle ||
!recipientHandles.includes(message.to_handle) ||
normalizeLegacyQuestionText(message.body) !== normalizeLegacyQuestionText(question)
) {
return false
@@ -1212,6 +1212,13 @@ export class OrchestrationDb {
`The ${params.role} compatibility principal is already committed to different proof.`
)
}
if (existing.status === 'revoked') {
throw new OrchestrationError(
'legacy_read_only',
`The ${params.role} compatibility principal has been revoked. No effects were applied.`,
{ effectsApplied: false }
)
}
this.db.exec('COMMIT')
return { principal: existing, duplicate: true }
}
@@ -1263,6 +1270,15 @@ export class OrchestrationDb {
.all(runId) as LegacyCompatibilityPrincipalRow[]
}
getLegacyCoordinatorPrincipal(runId: string): LegacyCompatibilityPrincipalRow | undefined {
return this.db
.prepare(
`SELECT * FROM legacy_compatibility_principals
WHERE run_id = ? AND role = 'coordinator'`
)
.get(runId) as LegacyCompatibilityPrincipalRow | undefined
}
resolveLegacyCompatibilityPrincipalByIdentity(params: {
runId: string
role: LegacyPrincipalRole
@@ -1362,6 +1378,10 @@ export class OrchestrationDb {
}
isLegacyCoordinatorHandle(runId: string, terminalHandle: string): boolean {
const principal = this.getLegacyCoordinatorPrincipal(runId)
if (principal?.terminal_handle === terminalHandle) {
return true
}
return Boolean(
this.db
.prepare(
@@ -1403,18 +1423,24 @@ export class OrchestrationDb {
if (!principal || principal.role !== 'worker' || !principal.dispatch_id) {
throw new OrchestrationError('request_mismatch', 'Legacy worker principal was not found.')
}
const runAddress = `run:${principal.run_id}`
const rows = this.db
.prepare(
`SELECT * FROM messages
WHERE run_id = ? AND delivery_contract = 'legacy_direct'
AND from_handle = ? AND to_handle = ? AND type = 'worker_done'
WHERE run_id = ?
AND (
(delivery_contract = 'legacy_direct' AND to_handle = ?) OR
(delivery_contract = 'current_delivery' AND to_handle = ?)
)
AND from_handle = ? AND type = 'worker_done'
AND subject = ? AND body = ? AND payload IS ?
ORDER BY sequence`
)
.all(
principal.run_id,
principal.terminal_handle,
params.recipientHandle,
runAddress,
principal.terminal_handle,
params.subject,
params.body,
params.payload
@@ -1444,10 +1470,11 @@ export class OrchestrationDb {
this.db
.prepare(
`SELECT 1 FROM messages
WHERE run_id = ? AND delivery_contract = 'current_delivery' AND read = 0
WHERE run_id = ? AND to_handle = ?
AND delivery_contract = 'current_delivery' AND read = 0
LIMIT 1`
)
.get(runId)
.get(runId, `run:${runId}`)
)
}
@@ -1832,7 +1859,14 @@ export class OrchestrationDb {
runId: string
coordinatorHandle: string
coordinatorPaneKey: string
allowLegacyCompatibility?: boolean
takeoverLegacy?: boolean
legacyCoordinatorAuthority?: {
runId: string
principalId: string | null
terminalHandle: string
paneKey: string
consumerGeneration: number
}
}): RunRow | undefined {
this.db.exec('BEGIN IMMEDIATE')
try {
@@ -1845,30 +1879,95 @@ export class OrchestrationDb {
run.coordinator_pane_key !== null &&
isEquivalentPaneKey(run.coordinator_pane_key, params.coordinatorPaneKey)
const adoption = this.getLegacyAdoption()
const adoptedRun = adoption?.adopted_run_id === params.runId
const legacyAuthority = params.legacyCoordinatorAuthority
const legacyPrincipalId = legacyAuthority?.principalId
const legacyPrincipal = legacyPrincipalId
? this.getLegacyCompatibilityPrincipal(legacyPrincipalId)
: undefined
const provenLegacyBinding = Boolean(
adoptedRun &&
legacyAuthority &&
legacyAuthority.principalId !== null &&
legacyAuthority.runId === params.runId &&
legacyAuthority.consumerGeneration === run.consumer_generation &&
legacyPrincipal?.run_id === params.runId &&
legacyPrincipal.role === 'coordinator' &&
legacyPrincipal.status === 'committed' &&
legacyPrincipal.terminal_handle === legacyAuthority.terminalHandle &&
isEquivalentPaneKey(legacyPrincipal.pane_key, legacyAuthority.paneKey) &&
params.coordinatorHandle === legacyAuthority.terminalHandle &&
isEquivalentPaneKey(params.coordinatorPaneKey, legacyAuthority.paneKey)
)
if (legacyAuthority && !provenLegacyBinding) {
throw new OrchestrationError(
'legacy_read_only',
'This retained legacy coordinator no longer has lifecycle authority. No effects were applied.',
{ effectsApplied: false }
)
}
const activeLegacyAssignment =
adoption?.adopted_run_id === params.runId &&
adoptedRun &&
Boolean(
this.db
.prepare(
`SELECT 1 FROM dispatch_contexts
WHERE run_id = ? AND contract_version = ?
AND status IN ('pending', 'dispatched')
UNION ALL
SELECT 1 FROM legacy_compatibility_principals
WHERE run_id = ? AND status = 'committed'
LIMIT 1`
)
.get(params.runId, LEGACY_CONTRACT_VERSION, params.runId)
.get(params.runId, LEGACY_CONTRACT_VERSION)
)
if (activeLegacyAssignment && !sameBinding && !params.allowLegacyCompatibility) {
const coordinatorPrincipal = adoptedRun
? this.getLegacyCoordinatorPrincipal(params.runId)
: undefined
const retainedCoordinatorHandle =
coordinatorPrincipal?.terminal_handle ??
run.coordinator_handle ??
this.getUniqueLegacyTaskCreatorHandle(params.runId)
const takeoverAlreadyApplied = Boolean(
params.takeoverLegacy &&
sameBinding &&
run.coordinator_handle === params.coordinatorHandle &&
coordinatorPrincipal?.status !== 'committed'
)
if (params.takeoverLegacy && !adoptedRun) {
throw new OrchestrationError(
'invalid_argument',
'Legacy takeover is only available for the automatically adopted Run.'
)
}
if (
activeLegacyAssignment &&
!sameBinding &&
!provenLegacyBinding &&
!params.takeoverLegacy
) {
throw new OrchestrationError(
'consumer_fenced',
'This adopted Run still has a live legacy principal; only its attested coordinator may rebind it.',
{ effectsApplied: false }
'This adopted Run still has live legacy work. Its attested coordinator may rebind it, or a current coordinator may explicitly use run-use --takeover-legacy.',
{
effectsApplied: false,
recoveryCommand: `orca orchestration run-use --id ${params.runId} --takeover-legacy`
}
)
}
this.unbindOtherRunsForPane(params.coordinatorPaneKey, params.runId)
if (!sameBinding || run.coordinator_handle !== params.coordinatorHandle) {
if (
(params.takeoverLegacy && !takeoverAlreadyApplied) ||
!sameBinding ||
run.coordinator_handle !== params.coordinatorHandle
) {
if (adoptedRun && (params.takeoverLegacy || !activeLegacyAssignment)) {
if (
coordinatorPrincipal?.status === 'committed' &&
(params.takeoverLegacy ||
coordinatorPrincipal.terminal_handle !== params.coordinatorHandle ||
!isEquivalentPaneKey(coordinatorPrincipal.pane_key, params.coordinatorPaneKey))
) {
this.setLegacyCompatibilityPrincipalStatus(coordinatorPrincipal.id, 'revoked')
}
}
this.db
.prepare(
`UPDATE runs
@@ -1879,6 +1978,9 @@ export class OrchestrationDb {
)
.run(params.coordinatorHandle, params.coordinatorPaneKey, params.runId)
this.fenceOutstandingDelivery(params.runId)
if (params.takeoverLegacy) {
this.promoteLegacyCoordinatorMailForTakeover(params.runId, retainedCoordinatorHandle)
}
}
this.db.exec('COMMIT')
} catch (error) {
@@ -1953,6 +2055,51 @@ export class OrchestrationDb {
.run(runId)
}
private promoteLegacyCoordinatorMailForTakeover(
runId: string,
retainedCoordinatorHandle: string | null
): void {
if (!retainedCoordinatorHandle) {
return
}
this.db
.prepare(
`UPDATE messages
SET to_handle = ?, delivery_contract = 'current_delivery',
read = 0, delivered_at = NULL
WHERE run_id = ? AND delivery_contract = 'legacy_direct'
AND to_handle = ?
AND EXISTS(
SELECT 1 FROM dispatch_contexts d
WHERE d.run_id = messages.run_id
AND d.contract_version = ?
AND (
messages.from_handle = d.assignee_handle OR
messages.from_handle = 'dispatch:' || d.id
)
)
AND (
read = 0 OR EXISTS(
SELECT 1 FROM question_threads q
WHERE q.message_id = messages.id AND q.status = 'pending'
)
)`
)
.run(`run:${runId}`, runId, retainedCoordinatorHandle, LEGACY_CONTRACT_VERSION)
}
private getUniqueLegacyTaskCreatorHandle(runId: string): string | null {
const rows = this.db
.prepare(
`SELECT DISTINCT created_by_terminal_handle AS handle
FROM tasks
WHERE run_id = ? AND created_by_terminal_handle IS NOT NULL
LIMIT 2`
)
.all(runId) as { handle: string }[]
return rows.length === 1 ? (rows[0]?.handle ?? null) : null
}
private requireCurrentConsumer(runId: string, consumerGeneration: number): RunRow {
const run = this.getRunRaw(runId)
if (!run || run.legacy === 1 || run.consumer_generation !== consumerGeneration) {
@@ -2269,12 +2416,20 @@ export class OrchestrationDb {
let message = params.message.existingId
? this.getMessageById(params.message.existingId)
: undefined
const delivery = this.resolveLegacyWorkerCoordinatorDelivery(
principal.run_id,
params.message.to
)
if (params.message.existingId) {
const matchesOriginalLegacyRoute =
message?.delivery_contract === 'legacy_direct' && message.to_handle === params.message.to
const matchesCurrentRoute =
message?.delivery_contract === delivery.contract && message.to_handle === delivery.to
if (
!message ||
message.run_id !== principal.run_id ||
message.delivery_contract !== 'legacy_direct' ||
message.to_handle !== params.message.to
message.from_handle !== principal.terminal_handle ||
(!matchesOriginalLegacyRoute && !matchesCurrentRoute)
) {
throw new OrchestrationError(
'request_mismatch',
@@ -2284,7 +2439,7 @@ export class OrchestrationDb {
} else {
message = this.insertMessage({
from: principal.terminal_handle,
to: params.message.to,
to: delivery.to,
subject: params.message.subject,
body: params.message.body,
type: params.message.type,
@@ -2292,7 +2447,7 @@ export class OrchestrationDb {
payload: params.message.payload,
senderPaneKey: principal.pane_key,
runId: principal.run_id,
deliveryContract: 'legacy_direct'
deliveryContract: delivery.contract
})
}
@@ -2300,12 +2455,24 @@ export class OrchestrationDb {
if (params.lifecycle.kind === 'heartbeat') {
this.recordHeartbeat(dispatchId, params.lifecycle.at)
} else if (params.lifecycle.kind === 'worker_report') {
settlement = this.settleWorkerReportInTransaction({
taskId: params.lifecycle.taskId,
dispatchId,
outcome: params.lifecycle.outcome,
result: params.lifecycle.result
})
const persistedOutcome =
params.message.existingId &&
dispatch.task_id === params.lifecycle.taskId &&
dispatch.status === 'completed'
? 'succeeded'
: params.message.existingId &&
dispatch.task_id === params.lifecycle.taskId &&
dispatch.status === 'failed'
? 'failed'
: undefined
settlement = persistedOutcome
? { action: 'settled', outcome: persistedOutcome, duplicate: true }
: this.settleWorkerReportInTransaction({
taskId: params.lifecycle.taskId,
dispatchId,
outcome: params.lifecycle.outcome,
result: params.lifecycle.result
})
if (settlement.action === 'rejected') {
throw new OrchestrationError(settlement.code, settlement.reason)
}
@@ -2380,24 +2547,36 @@ export class OrchestrationDb {
)
}
const existingQuestionId =
params.existingQuestionId &&
!this.db
.prepare(
`SELECT 1 FROM legacy_operation_receipts
WHERE principal_id = ? AND method = 'orchestration.ask' AND effect_id = ?
LIMIT 1`
)
.get(principal.id, params.existingQuestionId)
? params.existingQuestionId
: undefined
let question: QuestionRow
let message: MessageRow
if (params.existingQuestionId) {
const existingQuestion = this.getQuestion(params.existingQuestionId)
const existingMessage = this.getMessageById(params.existingQuestionId)
const delivery = this.resolveLegacyWorkerCoordinatorDelivery(
principal.run_id,
params.recipientHandle
)
if (existingQuestionId) {
const existingQuestion = this.getQuestion(existingQuestionId)
const existingMessage = this.getMessageById(existingQuestionId)
if (
!existingQuestion ||
!existingMessage ||
existingQuestion.run_id !== principal.run_id ||
existingQuestion.dispatch_id !== dispatchId ||
existingQuestion.status !== 'pending' ||
existingMessage.delivery_contract !== 'legacy_direct' ||
!legacyMessageMatchesQuestion(
existingMessage,
params.question,
params.options ?? [],
params.recipientHandle
)
existingMessage.delivery_contract !== delivery.contract ||
!legacyMessageMatchesQuestion(existingMessage, params.question, params.options ?? [], [
delivery.to
])
) {
throw new OrchestrationError(
'request_mismatch',
@@ -2409,7 +2588,7 @@ export class OrchestrationDb {
} else {
message = this.insertMessage({
from: principal.terminal_handle,
to: params.recipientHandle,
to: delivery.to,
subject: 'Question',
body: params.question,
type: 'question',
@@ -2421,7 +2600,7 @@ export class OrchestrationDb {
}),
senderPaneKey: principal.pane_key,
runId: principal.run_id,
deliveryContract: 'legacy_direct'
deliveryContract: delivery.contract
})
this.db
.prepare('UPDATE messages SET thread_id = ? WHERE id = ?')
@@ -2473,21 +2652,37 @@ export class OrchestrationDb {
question: QuestionRow
message: MessageRow
answerAcknowledged: boolean
claimedByOperation: boolean
}[] {
const principal = this.requireCommittedLegacyPrincipal(params.principalId, 'worker')
const runAddress = `run:${principal.run_id}`
const rows = this.db
.prepare(
`SELECT q.*, m.id AS source_message_id
`SELECT q.*, m.id AS source_message_id,
EXISTS(
SELECT 1 FROM legacy_operation_receipts lor
WHERE lor.principal_id = ? AND lor.method = 'orchestration.ask'
AND lor.effect_id = q.message_id
) AS claimed_by_operation
FROM question_threads q
INNER JOIN messages m ON m.id = q.message_id
WHERE q.run_id = ? AND q.dispatch_id = ?
AND m.delivery_contract = 'legacy_direct'
AND m.to_handle = ?
AND (
(m.delivery_contract = 'legacy_direct' AND m.to_handle = ?) OR
(m.delivery_contract = 'current_delivery' AND m.to_handle = ?)
)
ORDER BY m.sequence
LIMIT 501`
)
.all(principal.run_id, principal.dispatch_id, params.recipientHandle) as (QuestionRow & {
.all(
principal.id,
principal.run_id,
principal.dispatch_id,
params.recipientHandle,
runAddress
) as (QuestionRow & {
source_message_id: string
claimed_by_operation: number
})[]
if (rows.length > 500) {
throw new OrchestrationError(
@@ -2500,17 +2695,16 @@ export class OrchestrationDb {
const message = this.getMessageById(row.source_message_id)
return Boolean(
message &&
legacyMessageMatchesQuestion(
message,
params.question,
params.options ?? [],
params.recipientHandle
)
legacyMessageMatchesQuestion(message, params.question, params.options ?? [], [
params.recipientHandle,
runAddress
])
)
})
.map((row) => ({
question: exposeQuestionTimestamps(row),
message: this.getMessageById(row.message_id) as MessageRow,
claimedByOperation: row.claimed_by_operation === 1,
answerAcknowledged: row.answer_message_id
? Boolean(
this.db
@@ -2525,6 +2719,22 @@ export class OrchestrationDb {
}))
}
private resolveLegacyWorkerCoordinatorDelivery(
runId: string,
retainedCoordinatorHandle: string
): { to: string; contract: MessageDeliveryContract } {
const run = this.getRunRaw(runId)
const principal = this.getLegacyCoordinatorPrincipal(runId)
const takenOver =
run?.coordinator_handle !== null &&
(principal?.status === 'revoked' ||
(run?.coordinator_handle !== undefined &&
run.coordinator_handle !== retainedCoordinatorHandle))
return takenOver
? { to: `run:${runId}`, contract: 'current_delivery' }
: { to: retainedCoordinatorHandle, contract: 'legacy_direct' }
}
commitLegacyReplyOperation(params: {
principalId: string
operationKey: string
@@ -3658,11 +3868,10 @@ export class OrchestrationDb {
wd.agent_terminal_handle
FROM dispatch_contexts dc
INNER JOIN worker_dispatches wd ON wd.dispatch_id = dc.id
WHERE dc.contract_version = ?
AND wd.state IN ('starting', 'ready', 'start_unknown', 'stopping', 'stop_unknown')
WHERE wd.state IN ('starting', 'ready', 'start_unknown', 'stopping', 'stop_unknown')
ORDER BY dc.rowid`
)
.all(LEGACY_CONTRACT_VERSION) as LegacyWorkerTerminalRecoveryRow[]
.all() as LegacyWorkerTerminalRecoveryRow[]
}
getFederatedDispatch(dispatchId: string): FederatedDispatchRow | undefined {
@@ -77,6 +77,14 @@ describe('formatMessageBanner', () => {
)
})
it('lets the CLI resolve the live sender for Run and Dispatch addresses', () => {
for (const to_handle of ['run:run_test', 'dispatch:dispatch_test']) {
const banner = formatMessageBanner(makeMessage({ to_handle }))
expect(banner).toContain('[Reply: orca orchestration reply --id msg_test1 --body "..."]')
expect(banner).not.toContain(`--from ${to_handle}`)
}
})
it('marks legacy messages read-only without reply or acknowledgment affordances', () => {
const banner = formatMessageBanner(
makeMessage({ id: 'msg_legacy', run_id: 'run_legacy_local' })
+5 -4
View File
@@ -86,10 +86,11 @@ export function formatMessageBanner(
}
if (authority === 'current') {
// Why: older shells can lack Orca's terminal identity environment.
lines.push(
`[Reply: orca orchestration reply --id ${msg.id} --from ${msg.to_handle} --body "..."]`
)
const explicitFrom =
msg.to_handle.startsWith('run:') || msg.to_handle.startsWith('dispatch:')
? ''
: ` --from ${msg.to_handle}`
lines.push(`[Reply: orca orchestration reply --id ${msg.id}${explicitFrom} --body "..."]`)
}
lines.push(SEPARATOR)
@@ -1,6 +1,4 @@
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { rmSync } from 'node:fs'
import { afterEach, describe, expect, it } from 'vitest'
import Database from '../../sqlite/sync-database'
import {
@@ -9,21 +7,10 @@ import {
LEGACY_RUN_ID,
OrchestrationDb
} from './db'
type CutoverFixture = {
dbPath: string
currentRunId: string
unrelatedRunId: string
currentDispatchId: string
legacyTaskId: string
legacyDispatchId: string
legacyGateId: string
legacyMessageIds: string[]
legacyQuestionId: string
legacyDeliveryId: string
rejectionMessageId: string
lookalikeMessageId: string
}
import {
createLegacyStorageCutoverFixture,
type LegacyStorageCutoverFixture
} from './orchestration-legacy-storage-test-fixture'
describe('OrchestrationDb legacy contract storage', () => {
let db: OrchestrationDb | undefined
@@ -38,129 +25,14 @@ describe('OrchestrationDb legacy contract storage', () => {
}
})
function createCutoverFixture(): CutoverFixture {
tempDir = mkdtempSync(join(tmpdir(), 'orca-legacy-storage-'))
const dbPath = join(tempDir, 'orchestration.db')
const first = new OrchestrationDb(dbPath)
const currentRun = first.createRun({
objective: 'Current work',
coordinatorHandle: 'term_current_coord',
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111'
})
const currentTask = first.createTask({ spec: 'current', runId: currentRun.id })
const unrelatedRun = first.createRun({
objective: 'Unrelated current work',
coordinatorHandle: 'term_unrelated_coord',
coordinatorPaneKey: 'tab_unrelated:55555555-5555-4555-8555-555555555555'
})
const currentDispatch = first.createDispatchContext(
currentTask.id,
'term_current_worker',
'tab_current:22222222-2222-4222-9222-222222222222',
'current_launch_hash'
)
first.insertMessage({
runId: currentRun.id,
from: 'term_current_worker',
to: `run:${currentRun.id}`,
subject: 'current mail'
})
const legacyTask = first.createTask({
spec: 'legacy',
createdByTerminalHandle: 'term_legacy_coord'
})
first.createDispatchContext(
legacyTask.id,
'term_legacy_worker',
'tab_legacy:33333333-3333-4333-8333-333333333333'
)
const legacyGate = first.createGate({
taskId: legacyTask.id,
question: 'Retained gate?'
})
first.resolveGate(legacyGate.id, 'continue')
const retryDispatch = first.createDispatchContext(
legacyTask.id,
'term_legacy_worker',
'tab_legacy:33333333-3333-4333-8333-333333333333'
)
const legacyMessages = [
first.insertMessage({
from: 'term_legacy_coord',
to: 'term_legacy_worker',
subject: 'read worker mail'
}),
first.insertMessage({
from: 'term_legacy_worker',
to: 'term_legacy_coord',
subject: 'read coordinator mail'
}),
first.insertMessage({
from: 'term_legacy_coord',
to: 'term_legacy_worker',
subject: 'second worker page'
})
]
first.markAsRead(legacyMessages.map((message) => message.id))
const question = first.createQuestion({
runId: LEGACY_RUN_ID,
dispatchId: retryDispatch.id,
askerHandle: 'term_legacy_worker',
question: 'Retained question?'
})
const rejection = first.insertMessage({
from: 'term_legacy_worker',
to: 'term_legacy_coord',
subject: 'Rejected heartbeat',
type: 'heartbeat',
payload: JSON.stringify({ _orcaLifecycleRejection: { code: 'migration', reason: 'cutover' } })
})
const lookalike = first.insertMessage({
from: 'term_legacy_worker',
to: 'term_legacy_coord',
subject: 'Ordinary legacy mail',
payload: JSON.stringify({ '.orcaLifecycleRejection': 'not an audit marker' })
})
first.close()
const raw = new Database(dbPath)
const legacyDeliveryId = 'delivery_legacy_outstanding'
raw
.prepare(
`INSERT INTO deliveries (
id, run_id, consumer_generation, message_ids, status
) VALUES (?, ?, 0, ?, 'outstanding')`
)
.run(legacyDeliveryId, LEGACY_RUN_ID, JSON.stringify([legacyMessages[0].id]))
raw.exec(`
DROP INDEX IF EXISTS idx_messages_delivery_contract;
DROP TABLE legacy_mail_receipts;
DROP TABLE legacy_operation_receipts;
DROP TABLE legacy_compatibility_principals;
DROP TABLE legacy_adoptions;
`)
raw.pragma('user_version = 18')
raw.close()
return {
dbPath,
currentRunId: currentRun.id,
unrelatedRunId: unrelatedRun.id,
currentDispatchId: currentDispatch.id,
legacyTaskId: legacyTask.id,
legacyDispatchId: retryDispatch.id,
legacyGateId: legacyGate.id,
legacyMessageIds: legacyMessages.map((message) => message.id),
legacyQuestionId: question.message.id,
legacyDeliveryId,
rejectionMessageId: rejection.id,
lookalikeMessageId: lookalike.id
}
function createCutoverFixture(): LegacyStorageCutoverFixture {
const created = createLegacyStorageCutoverFixture()
tempDir = created.tempDir
return created.fixture
}
function openAdoptedFixture(): {
fixture: CutoverFixture
fixture: LegacyStorageCutoverFixture
adoptedRunId: string
workerPrincipalId: string
coordinatorPrincipalId: string
@@ -411,13 +283,19 @@ describe('OrchestrationDb legacy contract storage', () => {
coordinatorHandle: 'term_other',
coordinatorPaneKey: 'tab_other:55555555-5555-4555-8555-555555555555'
})
).toThrow(/only its attested coordinator/)
).toThrow(/attested coordinator may rebind/)
const bound = db!.bindRun({
runId: state.adoptedRunId,
coordinatorHandle: 'term_legacy_coord',
coordinatorPaneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
allowLegacyCompatibility: true
legacyCoordinatorAuthority: {
runId: state.adoptedRunId,
principalId: state.coordinatorPrincipalId,
terminalHandle: 'term_legacy_coord',
paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
consumerGeneration: 0
}
})
expect(bound).toMatchObject({ coordinator_handle: 'term_legacy_coord' })
expect(
@@ -429,6 +307,99 @@ describe('OrchestrationDb legacy contract storage', () => {
).toMatchObject({ coordinator_handle: 'term_legacy_coord' })
})
it.each([
['generation', { consumerGeneration: 1 }],
['principal', { principalId: 'legacy_principal_wrong' }],
['handle', { terminalHandle: 'term_wrong' }],
['pane', { paneKey: 'tab_wrong:99999999-9999-4999-8999-999999999999' }]
] as const)('rejects stale legacy coordinator %s proof inside Run binding', (_label, patch) => {
const state = openAdoptedFixture()
const before = db!.getRun(state.adoptedRunId)
expect(() =>
db!.bindRun({
runId: state.adoptedRunId,
coordinatorHandle: 'term_legacy_coord',
coordinatorPaneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
legacyCoordinatorAuthority: {
runId: state.adoptedRunId,
principalId: state.coordinatorPrincipalId,
terminalHandle: 'term_legacy_coord',
paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
consumerGeneration: 0,
...patch
}
})
).toThrow(/no longer has lifecycle authority/)
expect(db!.getRun(state.adoptedRunId)).toEqual(before)
})
it('revokes the legacy coordinator when a current coordinator takes over after settlement', () => {
const state = openAdoptedFixture()
db!.settleWorkerReport({
taskId: state.fixture.legacyTaskId,
dispatchId: state.fixture.legacyDispatchId,
outcome: 'succeeded',
result: 'done'
})
expect(
db!.bindRun({
runId: state.adoptedRunId,
coordinatorHandle: 'term_current_coord',
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111'
})
).toMatchObject({ coordinator_handle: 'term_current_coord' })
expect(db!.getLegacyCompatibilityPrincipal(state.coordinatorPrincipalId)?.status).toBe(
'revoked'
)
expect(() =>
db!.commitLegacyCompatibilityPrincipal({
runId: state.adoptedRunId,
role: 'coordinator',
hostScope: 'local:runtime_1',
terminalHandle: 'term_legacy_coord',
paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
launchTokenHash: 'coord_launch_hash',
processIncarnation: 'process_coord'
})
).toThrow(/revoked/)
})
it('promotes only mail addressed to the replaced legacy coordinator', () => {
const state = openAdoptedFixture()
const coordinatorMail = db!.insertMessage({
runId: state.adoptedRunId,
deliveryContract: 'legacy_direct',
from: 'term_legacy_worker',
to: 'term_legacy_coord',
subject: 'coordinator outcome'
})
const workerMail = db!.insertMessage({
runId: state.adoptedRunId,
deliveryContract: 'legacy_direct',
from: 'term_legacy_worker',
to: 'term_other_worker',
subject: 'worker-only guidance'
})
db!.bindRun({
runId: state.adoptedRunId,
coordinatorHandle: 'term_current_coord',
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111',
takeoverLegacy: true
})
expect(db!.getMessageById(coordinatorMail.id)).toMatchObject({
to_handle: `run:${state.adoptedRunId}`,
delivery_contract: 'current_delivery'
})
expect(db!.getMessageById(workerMail.id)).toMatchObject({
to_handle: 'term_other_worker',
delivery_contract: 'legacy_direct'
})
})
it('commits legacy messages, lifecycle effects, and invocation receipts exactly once', () => {
const state = openAdoptedFixture()
const params = {
@@ -508,6 +479,65 @@ describe('OrchestrationDb legacy contract storage', () => {
expect(db!.getLegacyCompatibilityPrincipal(state.workerPrincipalId)?.status).toBe('settled')
})
it('reconstructs a read pre-takeover completion through its original legacy route', () => {
const state = openAdoptedFixture()
const taskId = db!.getDispatchContextById(state.fixture.legacyDispatchId)!.task_id
const accepted = db!.insertMessage({
runId: state.adoptedRunId,
deliveryContract: 'legacy_direct',
from: 'term_legacy_worker',
to: 'term_legacy_coord',
subject: 'Completed',
body: 'accepted before takeover',
type: 'worker_done'
})
db!.markAsRead([accepted.id])
db!.settleWorkerReport({
taskId,
dispatchId: state.fixture.legacyDispatchId,
outcome: 'succeeded',
result: 'accepted before takeover'
})
db!.bindRun({
runId: state.adoptedRunId,
coordinatorHandle: 'term_current_coord',
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111',
takeoverLegacy: true
})
const beforeIds = db!.getInbox(100).map((message) => message.id)
const reconstructed = db!.commitLegacyLifecycleOperation({
principalId: state.workerPrincipalId,
operationKey: 'read_completion_after_takeover',
method: 'orchestration.send',
payloadHash: 'read_completion_after_takeover_payload',
message: {
existingId: accepted.id,
to: 'term_legacy_coord',
subject: 'Completed',
body: 'accepted before takeover',
type: 'worker_done'
},
lifecycle: {
kind: 'worker_report',
taskId,
outcome: 'succeeded',
result: 'accepted before takeover'
}
})
expect(reconstructed).toMatchObject({
duplicate: false,
message: {
id: accepted.id,
to_handle: 'term_legacy_coord',
delivery_contract: 'legacy_direct'
},
settlement: { action: 'settled', outcome: 'succeeded', duplicate: true }
})
expect(db!.getInbox(100).map((message) => message.id)).toEqual(beforeIds)
})
it('rejects cross-cutover completion reconstruction for another recipient', () => {
const state = openAdoptedFixture()
const taskId = db!.getDispatchContextById(state.fixture.legacyDispatchId)!.task_id
@@ -625,6 +655,38 @@ describe('OrchestrationDb legacy contract storage', () => {
})
).toHaveLength(2)
const inherited = db!.createQuestion({
runId: state.adoptedRunId,
dispatchId: state.fixture.legacyDispatchId,
askerHandle: 'term_legacy_worker',
question: 'Inherited?'
})
const sqlite = (db as unknown as { db: Database.Database }).db
sqlite
.prepare(
`UPDATE messages
SET from_handle = 'term_legacy_worker', to_handle = 'term_legacy_coord',
delivery_contract = 'legacy_direct'
WHERE id = ?`
)
.run(inherited.message.id)
const inheritedAsk = {
...ask,
operationKey: 'ask_inherited_1',
payloadHash: 'ask_inherited_payload_1',
question: 'Inherited?',
options: [],
existingQuestionId: inherited.message.id
}
const adopted = db!.commitLegacyAskOperation(inheritedAsk)
const distinct = db!.commitLegacyAskOperation({
...inheritedAsk,
operationKey: 'ask_inherited_2',
payloadHash: 'ask_inherited_payload_2'
})
expect(adopted.message.id).toBe(inherited.message.id)
expect(distinct.message.id).not.toBe(inherited.message.id)
const reply = {
principalId: state.coordinatorPrincipalId,
operationKey: 'reply_invocation_1',
@@ -0,0 +1,147 @@
import { mkdtempSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import Database from '../../sqlite/sync-database'
import { LEGACY_RUN_ID, OrchestrationDb } from './db'
export type LegacyStorageCutoverFixture = {
dbPath: string
currentRunId: string
unrelatedRunId: string
currentDispatchId: string
legacyTaskId: string
legacyDispatchId: string
legacyGateId: string
legacyMessageIds: string[]
legacyQuestionId: string
legacyDeliveryId: string
rejectionMessageId: string
lookalikeMessageId: string
}
export function createLegacyStorageCutoverFixture(): {
fixture: LegacyStorageCutoverFixture
tempDir: string
} {
const tempDir = mkdtempSync(join(tmpdir(), 'orca-legacy-storage-'))
const dbPath = join(tempDir, 'orchestration.db')
const first = new OrchestrationDb(dbPath)
const currentRun = first.createRun({
objective: 'Current work',
coordinatorHandle: 'term_current_coord',
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111'
})
const currentTask = first.createTask({ spec: 'current', runId: currentRun.id })
const unrelatedRun = first.createRun({
objective: 'Unrelated current work',
coordinatorHandle: 'term_unrelated_coord',
coordinatorPaneKey: 'tab_unrelated:55555555-5555-4555-8555-555555555555'
})
const currentDispatch = first.createDispatchContext(
currentTask.id,
'term_current_worker',
'tab_current:22222222-2222-4222-9222-222222222222',
'current_launch_hash'
)
first.insertMessage({
runId: currentRun.id,
from: 'term_current_worker',
to: `run:${currentRun.id}`,
subject: 'current mail'
})
const legacyTask = first.createTask({
spec: 'legacy',
createdByTerminalHandle: 'term_legacy_coord'
})
first.createDispatchContext(
legacyTask.id,
'term_legacy_worker',
'tab_legacy:33333333-3333-4333-8333-333333333333'
)
const legacyGate = first.createGate({
taskId: legacyTask.id,
question: 'Retained gate?'
})
first.resolveGate(legacyGate.id, 'continue')
const retryDispatch = first.createDispatchContext(
legacyTask.id,
'term_legacy_worker',
'tab_legacy:33333333-3333-4333-8333-333333333333'
)
const legacyMessages = [
first.insertMessage({
from: 'term_legacy_coord',
to: 'term_legacy_worker',
subject: 'read worker mail'
}),
first.insertMessage({
from: 'term_legacy_worker',
to: 'term_legacy_coord',
subject: 'read coordinator mail'
}),
first.insertMessage({
from: 'term_legacy_coord',
to: 'term_legacy_worker',
subject: 'second worker page'
})
]
first.markAsRead(legacyMessages.map((message) => message.id))
const question = first.createQuestion({
runId: LEGACY_RUN_ID,
dispatchId: retryDispatch.id,
askerHandle: 'term_legacy_worker',
question: 'Retained question?'
})
const rejection = first.insertMessage({
from: 'term_legacy_worker',
to: 'term_legacy_coord',
subject: 'Rejected heartbeat',
type: 'heartbeat',
payload: JSON.stringify({ _orcaLifecycleRejection: { code: 'migration', reason: 'cutover' } })
})
const lookalike = first.insertMessage({
from: 'term_legacy_worker',
to: 'term_legacy_coord',
subject: 'Ordinary legacy mail',
payload: JSON.stringify({ '.orcaLifecycleRejection': 'not an audit marker' })
})
first.close()
const raw = new Database(dbPath)
const legacyDeliveryId = 'delivery_legacy_outstanding'
raw
.prepare(
`INSERT INTO deliveries (
id, run_id, consumer_generation, message_ids, status
) VALUES (?, ?, 0, ?, 'outstanding')`
)
.run(legacyDeliveryId, LEGACY_RUN_ID, JSON.stringify([legacyMessages[0].id]))
raw.exec(`
DROP INDEX IF EXISTS idx_messages_delivery_contract;
DROP TABLE legacy_mail_receipts;
DROP TABLE legacy_operation_receipts;
DROP TABLE legacy_compatibility_principals;
DROP TABLE legacy_adoptions;
`)
raw.pragma('user_version = 18')
raw.close()
return {
tempDir,
fixture: {
dbPath,
currentRunId: currentRun.id,
unrelatedRunId: unrelatedRun.id,
currentDispatchId: currentDispatch.id,
legacyTaskId: legacyTask.id,
legacyDispatchId: retryDispatch.id,
legacyGateId: legacyGate.id,
legacyMessageIds: legacyMessages.map((message) => message.id),
legacyQuestionId: question.message.id,
legacyDeliveryId,
rejectionMessageId: rejection.id,
lookalikeMessageId: lookalike.id
}
}
}
@@ -29,7 +29,8 @@ describe('legacy worker terminal recovery planning', () => {
blockedPanes: [
{
worktreeId: 'repo::/workspace',
paneKey: `tab-worker:${LEAF_ID}`
paneKey: `tab-worker:${LEAF_ID}`,
contractVersion: 0
}
],
candidates: [
@@ -50,7 +51,8 @@ describe('legacy worker terminal recovery planning', () => {
blockedPanes: [
{
worktreeId: 'repo::/workspace',
paneKey: `tab-worker:${LEAF_ID}`
paneKey: `tab-worker:${LEAF_ID}`,
contractVersion: 0
}
],
candidates: [],
@@ -4,6 +4,7 @@ import type { LegacyWorkerTerminalRecoveryRow } from './types'
export type LegacyWorkerTerminalRecoveryCandidate = {
dispatchId: string
contractVersion: number
taskId: string
worktreeId: string
terminalHandle: string
@@ -16,7 +17,7 @@ export type LegacyWorkerTerminalRecoveryCandidate = {
}
export type LegacyWorkerTerminalRecoveryPlan = {
blockedPanes: { worktreeId: string; paneKey: string }[]
blockedPanes: { worktreeId: string; paneKey: string; contractVersion: number }[]
candidates: LegacyWorkerTerminalRecoveryCandidate[]
ambiguousDispatchIds: string[]
}
@@ -48,14 +49,21 @@ function countCandidateKeys(
export function planLegacyWorkerTerminalRecovery(
rows: readonly LegacyWorkerTerminalRecoveryRow[]
): LegacyWorkerTerminalRecoveryPlan {
const blockedPanes = new Map<string, { worktreeId: string; paneKey: string }>()
const blockedPanes = new Map<
string,
{ worktreeId: string; paneKey: string; contractVersion: number }
>()
const parsedCandidates: LegacyWorkerTerminalRecoveryCandidate[] = []
for (const row of rows) {
const worktreeId = row.worktree_id?.trim()
const paneKey = row.assignee_pane_key?.trim()
const pane = paneKey ? parsePaneKey(paneKey) : null
if (worktreeId && paneKey && pane) {
blockedPanes.set(`${worktreeId}\0${paneKey}`, { worktreeId, paneKey })
blockedPanes.set(`${worktreeId}\0${paneKey}`, {
worktreeId,
paneKey,
contractVersion: row.contract_version
})
}
const terminalHandle = row.assignee_handle?.trim()
const workerHandle = row.agent_terminal_handle?.trim()
@@ -74,6 +82,7 @@ export function planLegacyWorkerTerminalRecovery(
}
parsedCandidates.push({
dispatchId: row.dispatch_id,
contractVersion: row.contract_version,
taskId: row.task_id,
worktreeId,
terminalHandle,
@@ -1,5 +1,5 @@
import { afterEach, describe, expect, it } from 'vitest'
import { OrchestrationDb } from './db'
import { CURRENT_CONTRACT_VERSION, OrchestrationDb } from './db'
describe('OrchestrationDb worker Dispatch state', () => {
let db: OrchestrationDb | undefined
@@ -44,6 +44,14 @@ describe('OrchestrationDb worker Dispatch state', () => {
status: 'dispatched',
assignee_handle: 'term_worker'
})
expect(d.listLegacyWorkerTerminalRecoveryRows()).toEqual([
expect.objectContaining({
dispatch_id: started.dispatch.id,
contract_version: CURRENT_CONTRACT_VERSION,
worker_state: 'ready',
agent_terminal_handle: 'term_worker'
})
])
})
it('commits worker-start mutation acceptance with the starting Dispatch', () => {
+10
View File
@@ -53,6 +53,14 @@ export type RpcRequest = {
orchestrationCompatibilityEvidence?: OrchestrationCompatibilityEvidence
}
export type LegacyCoordinatorAuthorityProof = Readonly<{
runId: string
principalId: string | null
terminalHandle: string
paneKey: string
consumerGeneration: number
}>
export type RpcContext = {
runtime: OrcaRuntimeService
// Why: lets long-poll handlers release immediately on client disconnect instead of running down timeoutMs. See design doc §3.1.
@@ -82,6 +90,8 @@ export type RpcContext = {
}
// Why: only the compatibility authority router can set this trusted scope; user params cannot bypass Run consumer binding.
legacyCoordinatorRunId?: string
legacyCoordinatorAuthority?: LegacyCoordinatorAuthorityProof
revalidateLegacyCoordinator?: () => string
// Why: federation pins the authenticated saved-environment caller without exposing its token to handlers or storage.
authenticatedCallerFingerprint?: string
pairing?: PairingRpcContext
@@ -46,6 +46,10 @@ function makeRuntime(ui: PersistedUIState = getDefaultUIState()): OrcaRuntimeSer
updateUIState: vi.fn((updates: Partial<PersistedUIState>) => {
currentUI = { ...currentUI, ...updates }
return currentUI
}),
getOrchestrationDb: () => ({
getLegacyAdoption: () => undefined,
resolveLegacyWorkerCandidate: () => undefined
})
} as unknown as OrcaRuntimeService
}
@@ -0,0 +1,18 @@
import type { RuntimeCapability } from '../../../shared/protocol-version'
import type { TerminalStreamFrame } from '../../../shared/terminal-stream-protocol'
import type { PairingRpcContext } from './core'
export type RpcDispatchStreamingOptions = {
connectionId?: string
signal?: AbortSignal
clientId?: string
pairedDeviceId?: string
clientKind?: 'mobile' | 'runtime'
clientCapabilities?: readonly RuntimeCapability[]
pairing?: PairingRpcContext
sendBinary?: (bytes: Uint8Array<ArrayBufferLike>) => boolean | void
registerBinaryStreamHandler?: (
streamId: number,
handler: (frame: TerminalStreamFrame) => void
) => () => void
}
+49 -58
View File
@@ -10,13 +10,11 @@ import {
isStreamingMethod,
type RpcAnyMethod,
type RpcEnvelopeMeta,
type PairingRpcContext,
type RpcRegistry,
type RpcRequest,
type RpcResponse
} from './core'
import type { TerminalStreamFrame } from '../../../shared/terminal-stream-protocol'
import type { FeatureInteractionId } from '../../../shared/feature-interactions'
import {
computerErrorData,
@@ -29,20 +27,18 @@ import {
import { ALL_RPC_METHODS } from './methods'
import { emulatorProbe, emulatorProbeError } from '../../emulator/emulator-probe'
import type { OrcaRuntimeService } from '../orca-runtime'
import type { RuntimeCapability } from '../../../shared/protocol-version'
import {
OrchestrationMutationExecutor,
authenticatedCallerFingerprint,
getOrchestrationMutationExecutor,
type OrchestrationMutationExecutor,
type DurableMutationInvocation
} from './orchestration-mutation-executor'
import { orchestrationMigrationFence } from './orchestration-contract-fence'
import { getRuntimeFeatureInteractionId } from './runtime-feature-interaction'
import { recordRuntimeFeatureInteraction } from './runtime-feature-interaction'
import { OrchestrationLegacyCompatibility } from './orchestration-legacy-compatibility'
import type { RpcDispatchStreamingOptions } from './dispatcher-stream-options'
export type DispatcherOptions = {
runtime: OrcaRuntimeService
methods?: readonly RpcAnyMethod[]
}
export type DispatcherOptions = { runtime: OrcaRuntimeService; methods?: readonly RpcAnyMethod[] }
export class RpcDispatcher {
private readonly runtime: OrcaRuntimeService
@@ -53,7 +49,7 @@ export class RpcDispatcher {
constructor({ runtime, methods = ALL_RPC_METHODS }: DispatcherOptions) {
this.runtime = runtime
this.registry = buildRegistry(methods)
this.orchestrationMutations = new OrchestrationMutationExecutor(runtime)
this.orchestrationMutations = getOrchestrationMutationExecutor(runtime)
this.legacyOrchestration = new OrchestrationLegacyCompatibility(runtime)
}
@@ -91,8 +87,7 @@ export class RpcDispatcher {
)
}
const isEmulator = request.method.startsWith('emulator.')
if (isEmulator) {
if (request.method.startsWith('emulator.')) {
emulatorProbe(`rpc ${request.method}`, request.params)
}
try {
@@ -105,8 +100,13 @@ export class RpcDispatcher {
return successResponse(request.id, meta, compatibility.result)
}
const effectiveParams = compatibility.params ?? parsedParams.value
const invoke = (mutation?: DurableMutationInvocation) =>
method.handler(effectiveParams, {
const legacyCoordinator = this.legacyOrchestration.createCoordinatorInvocation(
request,
compatibility.legacyCoordinatorAuthority
)
const invoke = (mutation?: DurableMutationInvocation) => {
const legacyCoordinatorRunId = legacyCoordinator?.revalidate()
return method.handler(effectiveParams, {
runtime: this.runtime,
signal: options?.signal,
requestId: request.id,
@@ -114,13 +114,22 @@ export class RpcDispatcher {
authenticatedCallerFingerprint: authenticatedCallerFingerprint(request),
recordMutationReceipt: mutation?.recordReceipt,
orchestrationMutation: mutation?.identity,
legacyCoordinatorRunId: compatibility.legacyCoordinatorRunId
legacyCoordinatorRunId,
legacyCoordinatorAuthority: legacyCoordinator?.authority,
revalidateLegacyCoordinator: legacyCoordinator?.revalidate
})
}
const result = await this.orchestrationMutations.run(request, effectiveParams, invoke)
this.recordRuntimeFeatureInteraction(request.method, result, undefined, request.params)
recordRuntimeFeatureInteraction(
this.runtime,
request.method,
result,
undefined,
request.params
)
return successResponse(request.id, meta, result)
} catch (error) {
if (isEmulator) {
if (request.method.startsWith('emulator.')) {
emulatorProbeError(`rpc ${request.method}`, error, { params: request.params })
}
return this.mapError(request, meta, error)
@@ -133,20 +142,7 @@ export class RpcDispatcher {
async dispatchStreaming(
request: RpcRequest,
reply: (response: string) => void,
options?: {
connectionId?: string
signal?: AbortSignal
clientId?: string
pairedDeviceId?: string
clientKind?: 'mobile' | 'runtime'
clientCapabilities?: readonly RuntimeCapability[]
pairing?: PairingRpcContext
sendBinary?: (bytes: Uint8Array<ArrayBufferLike>) => boolean | void
registerBinaryStreamHandler?: (
streamId: number,
handler: (frame: TerminalStreamFrame) => void
) => () => void
}
options?: RpcDispatchStreamingOptions
): Promise<void> {
const meta = this.meta()
const method = this.registry.get(request.method)
@@ -183,8 +179,13 @@ export class RpcDispatcher {
return
}
const effectiveParams = compatibility.params ?? parsedParams.value
const invoke = (mutation?: DurableMutationInvocation) =>
method.handler(effectiveParams, {
const legacyCoordinator = this.legacyOrchestration.createCoordinatorInvocation(
request,
compatibility.legacyCoordinatorAuthority
)
const invoke = (mutation?: DurableMutationInvocation) => {
const legacyCoordinatorRunId = legacyCoordinator?.revalidate()
return method.handler(effectiveParams, {
runtime: this.runtime,
signal: options?.signal,
requestId: request.id,
@@ -200,10 +201,19 @@ export class RpcDispatcher {
pairing: options?.pairing,
sendBinary: options?.sendBinary,
registerBinaryStreamHandler: options?.registerBinaryStreamHandler,
legacyCoordinatorRunId: compatibility.legacyCoordinatorRunId
legacyCoordinatorRunId,
legacyCoordinatorAuthority: legacyCoordinator?.authority,
revalidateLegacyCoordinator: legacyCoordinator?.revalidate
})
}
const result = await this.orchestrationMutations.run(request, effectiveParams, invoke)
this.recordRuntimeFeatureInteraction(request.method, result, undefined, request.params)
recordRuntimeFeatureInteraction(
this.runtime,
request.method,
result,
undefined,
request.params
)
reply(JSON.stringify(successResponse(request.id, meta, result)))
} catch (error) {
reply(JSON.stringify(this.mapError(request, meta, error)))
@@ -213,7 +223,8 @@ export class RpcDispatcher {
const recordedStreamingFeatureInteractions = new Set<FeatureInteractionId>()
const emit = (result: unknown): void => {
this.recordRuntimeFeatureInteraction(
recordRuntimeFeatureInteraction(
this.runtime,
request.method,
result,
recordedStreamingFeatureInteractions,
@@ -243,7 +254,8 @@ export class RpcDispatcher {
},
emit
)
this.recordRuntimeFeatureInteraction(
recordRuntimeFeatureInteraction(
this.runtime,
request.method,
result,
recordedStreamingFeatureInteractions,
@@ -310,25 +322,4 @@ export class RpcDispatcher {
private meta(): RpcEnvelopeMeta {
return { runtimeId: this.runtime.getRuntimeId() }
}
private recordRuntimeFeatureInteraction(
method: string,
result: unknown,
alreadyRecorded?: Set<FeatureInteractionId>,
rawParams?: unknown
): void {
const id = getRuntimeFeatureInteractionId(method, result, rawParams)
if (!id) {
return
}
if (alreadyRecorded?.has(id)) {
return
}
try {
this.runtime.recordFeatureInteraction(id)
alreadyRecorded?.add(id)
} catch {
// Best-effort education state must not break runtime tools.
}
}
}
@@ -1,6 +1,6 @@
import { z } from 'zod'
import { defineMethod, type RpcMethod } from '../core'
import { OptionalString, requiredString } from '../schemas'
import { OptionalBoolean, OptionalString, requiredString } from '../schemas'
import type { OrcaRuntimeService } from '../../orca-runtime'
import { OrchestrationError } from '../../orchestration/orchestration-error'
@@ -11,7 +11,8 @@ const RunCreateParams = z.object({
const RunUseParams = z.object({
id: requiredString('Missing --id'),
from: requiredString('Missing coordinator terminal')
from: requiredString('Missing coordinator terminal'),
takeoverLegacy: OptionalBoolean
})
const RunCurrentParams = z.object({ from: requiredString('Missing coordinator terminal') })
@@ -51,7 +52,7 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [
defineMethod({
name: 'orchestration.runUse',
params: RunUseParams,
handler: (params, { runtime, legacyCoordinatorRunId }) => {
handler: (params, { runtime, legacyCoordinatorAuthority }) => {
const paneKey = requireCallerPane(runtime, params.from)
const db = runtime.getOrchestrationDb()
const priorRun = db.getCurrentRunForPane(paneKey)
@@ -59,7 +60,8 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [
runId: params.id,
coordinatorHandle: params.from,
coordinatorPaneKey: paneKey,
allowLegacyCompatibility: legacyCoordinatorRunId === params.id
takeoverLegacy: params.takeoverLegacy,
legacyCoordinatorAuthority
})
if (!run) {
throw new OrchestrationError(
+99 -6
View File
@@ -2,7 +2,12 @@
import { z } from 'zod'
import { defineMethod, type RpcMethod } from '../core'
import { OptionalFiniteNumber, OptionalString, OptionalBoolean, requiredString } from '../schemas'
import type { MessageType, MessagePriority, TaskStatus } from '../../orchestration/db'
import {
LEGACY_CONTRACT_VERSION,
type MessageType,
type MessagePriority,
type TaskStatus
} from '../../orchestration/db'
import { MESSAGE_TYPES } from '../../orchestration/types'
import { buildDispatchPreamble } from '../../orchestration/preamble'
import { formatMessageBanner } from '../../orchestration/formatter'
@@ -379,6 +384,49 @@ function resolveMessageRun(
return { run, dispatchId: dispatch?.id ?? dispatchId }
}
function legacyWorkerDeliveryContract(
runtime: OrcaRuntimeService,
runId: string | undefined,
recipient: string
): 'legacy_direct' | undefined {
if (!runId) {
return undefined
}
if (!recipient.startsWith('dispatch:')) {
return runtime
.getOrchestrationDb()
.resolveLegacyWorkerCandidate({ runId, terminalHandle: recipient })
? 'legacy_direct'
: undefined
}
const dispatch = runtime
.getOrchestrationDb()
.getDispatchContextById(recipient.slice('dispatch:'.length))
return dispatch?.run_id === runId &&
dispatch.contract_version === LEGACY_CONTRACT_VERSION &&
(dispatch.status === 'pending' || dispatch.status === 'dispatched')
? 'legacy_direct'
: undefined
}
function interruptedAcknowledgedCheck(
runId: string,
acknowledged: string,
reason: 'consumer_fenced' | 'outcome_unknown' | 'waiter_exists'
): Record<string, unknown> {
return {
runId,
deliveryId: null,
messages: [],
count: 0,
acknowledged,
timedOut: false,
cancelled: false,
connectionLost: false,
waitInterrupted: reason
}
}
function rejectFederatedExplicitTarget(params: { to?: string; run?: string }): void {
if (params.to || params.run) {
throw new OrchestrationError(
@@ -395,7 +443,10 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
defineMethod({
name: 'orchestration.send',
params: SendParams,
handler: async (params, { runtime, orchestrationCapability }) => {
handler: async (
params,
{ runtime, orchestrationCapability, legacyCoordinatorRunId, revalidateLegacyCoordinator }
) => {
const db = runtime.getOrchestrationDb()
const from = params.from ?? 'unknown'
// Why: caller-supplied pane fields are only compatibility metadata; lifecycle authority uses the runtime-observed pane plus capability.
@@ -536,6 +587,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
'Coordinator-to-worker control mail cannot report worker lifecycle.'
)
}
revalidateLegacyCoordinator?.()
const relay = db.enqueueFederationRelay({
dispatchId,
direction: 'to_worker',
@@ -562,6 +614,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
}
}
// Point-to-point — existing single-recipient behavior
revalidateLegacyCoordinator?.()
const msg = db.insertMessage({
from,
to,
@@ -572,7 +625,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
threadId: params.threadId,
payload: params.payload,
senderPaneKey,
runId: routing.run?.id
runId: routing.run?.id,
deliveryContract: legacyWorkerDeliveryContract(
runtime,
routing.run?.id ?? legacyCoordinatorRunId,
to
)
})
const dispatch = routing.dispatchId
? db.getDispatchContextById(routing.dispatchId)
@@ -629,6 +687,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
throw new Error(`No recipients resolved for group address: ${to}`)
}
revalidateLegacyCoordinator?.()
const threadId = params.threadId ?? `thread_${Date.now()}`
const messages = handles.map((handle) =>
db.insertMessage({
@@ -641,7 +700,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
threadId,
payload: params.payload,
senderPaneKey,
runId: routing.run?.id
runId: routing.run?.id,
deliveryContract: legacyWorkerDeliveryContract(
runtime,
routing.run?.id ?? legacyCoordinatorRunId,
handle
)
})
)
for (const message of messages) {
@@ -655,7 +719,16 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
defineMethod({
name: 'orchestration.check',
params: CheckParams,
handler: async (params, { runtime, signal, legacyCoordinatorRunId }) => {
handler: async (
params,
{
runtime,
signal,
legacyCoordinatorRunId,
revalidateLegacyCoordinator,
recordMutationReceipt
}
) => {
const db = runtime.getOrchestrationDb()
const handle = params.terminal ?? 'unknown'
const typeFilter = parseMessageTypes(params.types)
@@ -682,6 +755,11 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
deliveryId: params.ack
})
: undefined
if (acknowledged) {
recordMutationReceipt?.(
interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'outcome_unknown')
)
}
if (params.peek || params.all || params.unread === false) {
const history = db.getRunMailboxHistory(run.id, 100, typeFilter)
const messages =
@@ -745,14 +823,28 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
signal,
exclusive: true
})
try {
revalidateLegacyCoordinator?.()
} catch (error) {
if (!acknowledged) {
throw error
}
return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'consumer_fenced')
}
const latestRun = db.getRun(run.id)
if (!latestRun || latestRun.consumer_generation !== generation) {
if (acknowledged) {
return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'consumer_fenced')
}
throw new OrchestrationError(
'consumer_fenced',
'This mailbox consumer was replaced while waiting.'
)
}
if (waitResult === 'waiter_exists') {
if (acknowledged) {
return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'waiter_exists')
}
throw new OrchestrationError(
'waiter_exists',
`Run ${run.id} already has an active actionable waiter.`
@@ -1119,7 +1211,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
defineMethod({
name: 'orchestration.dispatch',
params: DispatchParams,
handler: async (params, { runtime, legacyCoordinatorRunId }) => {
handler: async (params, { runtime, legacyCoordinatorRunId, revalidateLegacyCoordinator }) => {
const db = runtime.getOrchestrationDb()
const task = db.getTask(params.task)
if (!task) {
@@ -1184,6 +1276,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
)
}
revalidateLegacyCoordinator?.()
const ctx = db.createDispatchContext(params.task, to, assigneePaneKey)
const dispatchCapability = params.inject
? db.mintDispatchCapability({
@@ -1,4 +1,3 @@
import { parsePaneKey } from '../../../shared/stable-pane-id'
import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence'
import type {
DispatchContextRow,
@@ -6,9 +5,14 @@ import type {
LegacyPrincipalRole
} from '../orchestration/types'
import { OrchestrationError } from '../orchestration/orchestration-error'
import type { OrcaRuntimeService } from '../orca-runtime'
import type { OrcaRuntimeService, OrchestrationCompatibilityCallerAuthority } from '../orca-runtime'
import { LEGACY_CONTRACT_VERSION } from '../orchestration/db'
import type { RpcRequest } from './core'
import {
equivalentLegacyPaneKey,
legacyCoordinatorReadOnly,
legacyReadOnlyError
} from './orchestration-legacy-process-identity'
export type LegacyPrincipalCandidate = {
runId: string
@@ -22,8 +26,9 @@ export function resolveAttestedLegacyPrincipal(args: {
runtime: OrcaRuntimeService
evidence?: OrchestrationCompatibilityEvidence
candidate: LegacyPrincipalCandidate
authority?: OrchestrationCompatibilityCallerAuthority
}): LegacyCompatibilityPrincipalRow {
const authority = verifyAttestedLegacyCandidate(args)
const authority = args.authority ?? verifyAttestedLegacyCandidate(args)
return args.runtime.getOrchestrationDb().commitLegacyCompatibilityPrincipal({
runId: args.candidate.runId,
dispatchId: args.candidate.dispatchId,
@@ -36,7 +41,7 @@ export function resolveAttestedLegacyPrincipal(args: {
}).principal
}
function verifyAttestedLegacyCandidate(args: {
export function verifyAttestedLegacyCandidate(args: {
runtime: OrcaRuntimeService
evidence?: OrchestrationCompatibilityEvidence
candidate: LegacyPrincipalCandidate
@@ -44,7 +49,7 @@ function verifyAttestedLegacyCandidate(args: {
const authority = args.runtime.verifyOrchestrationCompatibilityCaller(args.evidence)
if (
!authority ||
!equivalentPaneKey(args.candidate.paneKey, authority.paneKey) ||
!equivalentLegacyPaneKey(args.candidate.paneKey, authority.paneKey) ||
args.candidate.terminalHandle !== authority.terminalHandle
) {
throw legacyReadOnlyError()
@@ -221,37 +226,6 @@ export class LegacyCompatibilityAuthority {
})
return principal
}
resolveProvenCoordinatorScope(request: RpcRequest, requestedRunId?: string): string | undefined {
const db = this.runtime.getOrchestrationDb()
const adoption = db.getLegacyAdoption()
const requestedRun = requestedRunId ?? adoption?.adopted_run_id
if (!adoption || requestedRun !== adoption.adopted_run_id) {
return undefined
}
const candidate = db.resolveLegacyCoordinatorCandidate({
runId: adoption.adopted_run_id,
terminalHandle: request.orchestrationCompatibilityEvidence?.terminalHandle,
paneKey: request.orchestrationCompatibilityEvidence?.paneKey
})
if (!candidate) {
if (request.orchestrationCompatibilityEvidence) {
throw legacyCoordinatorReadOnly()
}
return undefined
}
verifyAttestedLegacyCandidate({
runtime: this.runtime,
evidence: request.orchestrationCompatibilityEvidence,
candidate: {
runId: adoption.adopted_run_id,
role: 'coordinator',
terminalHandle: candidate.terminalHandle,
paneKey: candidate.paneKey
}
})
return adoption.adopted_run_id
}
}
function candidateFromDispatch(dispatch: DispatchContextRow): LegacyPrincipalCandidate {
@@ -281,31 +255,3 @@ function candidateFromPrincipal(
paneKey: principal.pane_key
}
}
function equivalentPaneKey(a: string | null | undefined, b: string | null | undefined): boolean {
if (!a || !b) {
return false
}
if (a === b) {
return true
}
const aLeaf = parsePaneKey(a)?.leafId
const bLeaf = parsePaneKey(b)?.leafId
return Boolean(aLeaf && bLeaf && aLeaf === bLeaf)
}
function legacyReadOnlyError(): OrchestrationError {
return new OrchestrationError(
'legacy_read_only',
'This retained legacy assignment could not prove authority from its original live process. No effects were applied.',
{ effectsApplied: false }
)
}
function legacyCoordinatorReadOnly(): OrchestrationError {
return new OrchestrationError(
'legacy_read_only',
'This retained legacy coordinator could not prove its original process identity. No effects were applied.',
{ effectsApplied: false }
)
}
@@ -300,6 +300,82 @@ describe('legacy compatibility through RpcDispatcher', () => {
expect(harness.notify).toHaveBeenCalledOnce()
})
it('replays an A-era completion without touching a newer current attempt', async () => {
const harness = createHarness()
const payload = JSON.stringify({
taskId: harness.taskId,
dispatchId: harness.dispatchId
})
const completion = harness.db.insertMessage({
runId: harness.adoptedRunId,
from: WORKER_HANDLE,
to: COORDINATOR_HANDLE,
subject: 'Failed: pinned A treated this as complete',
body: 'persisted result',
type: 'worker_done',
payload,
senderPaneKey: WORKER_PANE,
deliveryContract: 'legacy_direct'
})
harness.db.settleWorkerReport({
taskId: harness.taskId,
dispatchId: harness.dispatchId,
outcome: 'succeeded',
result: 'persisted result'
})
harness.db.commitLegacyCompatibilityPrincipal({
runId: harness.adoptedRunId,
dispatchId: harness.dispatchId,
role: 'worker',
hostScope: JSON.stringify({ kind: 'local', hostId: 'local' }),
terminalHandle: WORKER_HANDLE,
paneKey: WORKER_PANE,
launchTokenHash: createHash('sha256').update('worker-token').digest('hex'),
processIncarnation: 'process-1'
})
harness.db.updateTaskStatus(harness.taskId, 'ready')
const currentDispatch = harness.db.createDispatchContext(
harness.taskId,
'term_current_worker',
'tab_current_worker:77777777-7777-4777-8777-777777777777',
'current-launch-hash'
)
const currentTaskBefore = harness.db.getTask(harness.taskId)
const currentDispatchBefore = harness.db.getDispatchContextById(currentDispatch.id)
const before = counts(harness.db)
const response = await harness.dispatcher.dispatch(
request(
'orchestration.send',
{
from: WORKER_HANDLE,
to: COORDINATOR_HANDLE,
subject: completion.subject,
body: completion.body,
type: 'worker_done',
payload
},
evidence('worker'),
'reconstruct-pinned-a-completion'
)
)
expect(response).toMatchObject({
ok: true,
result: {
message: { id: completion.id },
lifecycle: { action: 'settled', outcome: 'succeeded', duplicate: true }
}
})
expect(harness.db.getTask(harness.taskId)).toEqual(currentTaskBefore)
expect(harness.db.getDispatchContextById(harness.dispatchId)?.status).toBe('completed')
expect(harness.db.getDispatchContextById(currentDispatch.id)).toEqual(currentDispatchBefore)
expect(counts(harness.db)).toEqual({
...before,
legacy_operation_receipts: before.legacy_operation_receipts + 1
})
})
it('rejects a legacy lifecycle recipient outside the adopted Run with zero effects', async () => {
const harness = createHarness()
const before = counts(harness.db)
@@ -367,6 +443,35 @@ describe('legacy compatibility through RpcDispatcher', () => {
expect(harness.notify).not.toHaveBeenCalled()
})
it('keeps distinct compatibility ask invocations on distinct questions', async () => {
const harness = createHarness()
const ask = {
from: WORKER_HANDLE,
to: COORDINATOR_HANDLE,
question: 'Same question?',
options: ['yes', 'no'],
timeoutMs: 1
}
const first = await harness.dispatcher.dispatch(
request('orchestration.ask', ask, evidence('worker'), 'ask-one')
)
const replay = await harness.dispatcher.dispatch(
request('orchestration.ask', ask, evidence('worker'), 'ask-one')
)
const second = await harness.dispatcher.dispatch(
request('orchestration.ask', ask, evidence('worker'), 'ask-two')
)
expect(first).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: false } } })
expect(replay).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: true } } })
expect(second).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: false } } })
const firstId = (first as { result: { messageId: string } }).result.messageId
const replayId = (replay as { result: { messageId: string } }).result.messageId
const secondId = (second as { result: { messageId: string } }).result.messageId
expect(replayId).toBe(firstId)
expect(secondId).not.toBe(firstId)
})
it('does not infer an outcome for a current Dispatch when legacy adoption exists', async () => {
const harness = createHarness()
const run = harness.db.createRun({
@@ -524,6 +629,56 @@ describe('legacy compatibility through RpcDispatcher', () => {
expect(harness.notify).not.toHaveBeenCalled()
})
it('advertises only Run-addressed current delivery to a legacy coordinator', async () => {
const harness = createHarness()
harness.db.insertMessage({
runId: harness.adoptedRunId,
from: 'term_current_worker',
to: WORKER_HANDLE,
subject: 'worker-only current mail',
deliveryContract: 'current_delivery'
})
const workerOnly = await harness.dispatcher.dispatch(
request(
'orchestration.check',
{ terminal: COORDINATOR_HANDLE },
evidence('coordinator'),
'worker-current-mail'
)
)
expect(workerOnly).toMatchObject({
ok: true,
result: { legacyCompatibility: { currentDelivery: undefined } }
})
harness.db.insertMessage({
runId: harness.adoptedRunId,
from: 'term_current_worker',
to: `run:${harness.adoptedRunId}`,
subject: 'Run current mail',
deliveryContract: 'current_delivery'
})
const runMail = await harness.dispatcher.dispatch(
request(
'orchestration.check',
{ terminal: COORDINATOR_HANDLE },
evidence('coordinator'),
'run-current-mail'
)
)
expect(runMail).toMatchObject({
ok: true,
result: {
legacyCompatibility: {
currentDelivery: {
runId: harness.adoptedRunId,
checkCommand: expect.stringContaining(`--run ${harness.adoptedRunId}`)
}
}
}
})
})
it.each(['dispatch', 'websocket'] as const)(
'%s binds an attested coordinator once and rejects contradictory proof before binding',
async (transport) => {
@@ -588,4 +743,42 @@ describe('legacy compatibility through RpcDispatcher', () => {
expect(harness.db.getRun(harness.adoptedRunId)?.consumer_generation).toBe(0)
expect(counts(harness.db)).toEqual(before)
})
it('rejects stale legacy coordinator proof after current takeover', async () => {
const harness = createHarness()
const principal = harness.db.commitLegacyCompatibilityPrincipal({
runId: harness.adoptedRunId,
role: 'coordinator',
hostScope: JSON.stringify({ kind: 'local', hostId: 'local' }),
terminalHandle: COORDINATOR_HANDLE,
paneKey: COORDINATOR_PANE,
launchTokenHash: createHash('sha256').update('coordinator-token').digest('hex'),
processIncarnation: 'process-1'
}).principal
harness.db.settleWorkerReport({
taskId: harness.taskId,
dispatchId: harness.dispatchId,
outcome: 'succeeded',
result: 'done'
})
harness.db.bindRun({
runId: harness.adoptedRunId,
coordinatorHandle: 'term_current_coord',
coordinatorPaneKey: 'tab_current:55555555-5555-4555-8555-555555555555'
})
expect(harness.db.getLegacyCompatibilityPrincipal(principal.id)?.status).toBe('revoked')
const before = counts(harness.db)
const response = await harness.dispatcher.dispatch(
request(
'orchestration.taskList',
{},
evidence('coordinator'),
'stale-coordinator-after-takeover'
)
)
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
expect(counts(harness.db)).toEqual(before)
})
})
@@ -1,4 +1,4 @@
import type { RpcRequest } from './core'
import type { LegacyCoordinatorAuthorityProof, RpcRequest } from './core'
import type { OrcaRuntimeService } from '../orca-runtime'
import { LegacyCompatibilityAuthority } from './orchestration-legacy-authority'
import { handleLegacyLifecycleSend } from './orchestration-legacy-lifecycle'
@@ -10,6 +10,7 @@ import type {
LegacyReplyParams,
LegacySendParams
} from './orchestration-legacy-operation'
import { LegacyCoordinatorAuthority } from './orchestration-legacy-coordinator-authority'
const COORDINATOR_PREFLIGHT_METHODS = new Set([
'orchestration.taskCreate',
@@ -19,19 +20,31 @@ const COORDINATOR_PREFLIGHT_METHODS = new Set([
'orchestration.gateCreate',
'orchestration.gateResolve',
'orchestration.runUse',
'orchestration.send',
'orchestration.check',
'orchestration.reply'
])
export type LegacyCompatibilityRoute =
| { handled: true; result: unknown }
| { handled: false; params?: unknown; legacyCoordinatorRunId?: string }
| {
handled: false
params?: unknown
legacyCoordinatorAuthority?: LegacyCoordinatorAuthorityProof
}
export type LegacyCoordinatorInvocation = Readonly<{
authority: LegacyCoordinatorAuthorityProof
revalidate: () => string
}>
export class OrchestrationLegacyCompatibility {
private readonly authority: LegacyCompatibilityAuthority
private readonly coordinatorAuthority: LegacyCoordinatorAuthority
constructor(private readonly runtime: OrcaRuntimeService) {
this.authority = new LegacyCompatibilityAuthority(runtime)
this.coordinatorAuthority = new LegacyCoordinatorAuthority(runtime)
}
async tryHandle(
@@ -50,18 +63,40 @@ export class OrchestrationLegacyCompatibility {
return { handled: false }
}
const values = params as Record<string, unknown>
if (request.method === 'orchestration.runUse' && values.takeoverLegacy === true) {
return { handled: false }
}
const requestedRunId =
request.method === 'orchestration.runUse' ? stringValue(values.id) : stringValue(values.run)
const runId = this.authority.resolveProvenCoordinatorScope(request, requestedRunId)
return runId
const authority = this.coordinatorAuthority.resolve(request, requestedRunId)
return authority
? {
handled: false,
params: { ...values, run: runId },
legacyCoordinatorRunId: runId
params: { ...values, run: authority.runId },
legacyCoordinatorAuthority: authority
}
: { handled: false }
}
revalidateCoordinatorAuthority(
request: RpcRequest,
proof: LegacyCoordinatorAuthorityProof
): string {
return this.coordinatorAuthority.revalidate(request, proof)
}
createCoordinatorInvocation(
request: RpcRequest,
authority?: LegacyCoordinatorAuthorityProof
): LegacyCoordinatorInvocation | undefined {
return authority
? {
authority,
revalidate: () => this.revalidateCoordinatorAuthority(request, authority)
}
: undefined
}
private async route(
request: RpcRequest,
params: unknown,
@@ -0,0 +1,158 @@
import type { OrcaRuntimeService, OrchestrationCompatibilityCallerAuthority } from '../orca-runtime'
import type { LegacyCompatibilityPrincipalRow } from '../orchestration/types'
import type { LegacyCoordinatorAuthorityProof, RpcRequest } from './core'
import {
resolveAttestedLegacyPrincipal,
verifyAttestedLegacyCandidate,
type LegacyPrincipalCandidate
} from './orchestration-legacy-authority'
import {
equivalentLegacyPaneKey,
legacyCoordinatorReadOnly
} from './orchestration-legacy-process-identity'
export class LegacyCoordinatorAuthority {
constructor(private readonly runtime: OrcaRuntimeService) {}
resolve(
request: RpcRequest,
requestedRunId?: string
): LegacyCoordinatorAuthorityProof | undefined {
const db = this.runtime.getOrchestrationDb()
const adoption = db.getLegacyAdoption()
const requestedRun = requestedRunId ?? adoption?.adopted_run_id
if (!adoption || requestedRun !== adoption.adopted_run_id) {
return undefined
}
const candidate = db.resolveLegacyCoordinatorCandidate({
runId: adoption.adopted_run_id,
terminalHandle: request.orchestrationCompatibilityEvidence?.terminalHandle,
paneKey: request.orchestrationCompatibilityEvidence?.paneKey
})
if (!candidate) {
if (request.orchestrationCompatibilityEvidence) {
throw legacyCoordinatorReadOnly()
}
return undefined
}
const existing = db.getLegacyCoordinatorPrincipal(adoption.adopted_run_id)
const proofCandidate = existing
? this.candidate(adoption.adopted_run_id, existing)
: this.candidate(adoption.adopted_run_id, candidate)
const attestation = verifyAttestedLegacyCandidate({
runtime: this.runtime,
evidence: request.orchestrationCompatibilityEvidence,
candidate: proofCandidate
})
const run = db.getRun(adoption.adopted_run_id)
if (!run || !this.bindingMatches(run, proofCandidate)) {
throw legacyCoordinatorReadOnly()
}
if (existing && !this.principalMatchesAttestation(existing, attestation)) {
throw legacyCoordinatorReadOnly()
}
const principal = existing
? existing
: request.method === 'orchestration.runUse'
? resolveAttestedLegacyPrincipal({
runtime: this.runtime,
evidence: request.orchestrationCompatibilityEvidence,
candidate: proofCandidate,
authority: attestation
})
: undefined
return {
runId: run.id,
principalId: principal?.id ?? null,
terminalHandle: proofCandidate.terminalHandle,
paneKey: proofCandidate.paneKey,
consumerGeneration: run.consumer_generation
}
}
revalidate(request: RpcRequest, proof: LegacyCoordinatorAuthorityProof): string {
const db = this.runtime.getOrchestrationDb()
const adoption = db.getLegacyAdoption()
const run = db.getRun(proof.runId)
if (
adoption?.adopted_run_id !== proof.runId ||
!run ||
run.consumer_generation !== proof.consumerGeneration
) {
throw legacyCoordinatorReadOnly()
}
const candidate = this.candidate(proof.runId, proof)
const principal = db.getLegacyCoordinatorPrincipal(proof.runId)
if (proof.principalId) {
if (
principal?.id !== proof.principalId ||
principal.status !== 'committed' ||
principal.terminal_handle !== proof.terminalHandle ||
!equivalentLegacyPaneKey(principal.pane_key, proof.paneKey)
) {
throw legacyCoordinatorReadOnly()
}
} else if (
principal &&
(principal.status !== 'committed' ||
principal.terminal_handle !== proof.terminalHandle ||
!equivalentLegacyPaneKey(principal.pane_key, proof.paneKey))
) {
throw legacyCoordinatorReadOnly()
}
const attestation = verifyAttestedLegacyCandidate({
runtime: this.runtime,
evidence: request.orchestrationCompatibilityEvidence,
candidate
})
if (principal && !this.principalMatchesAttestation(principal, attestation)) {
throw legacyCoordinatorReadOnly()
}
if (!this.bindingMatches(run, candidate)) {
throw legacyCoordinatorReadOnly()
}
return run.id
}
private candidate(
runId: string,
identity: {
terminalHandle?: string
terminal_handle?: string
paneKey?: string
pane_key?: string
}
): LegacyPrincipalCandidate {
return {
runId,
role: 'coordinator',
terminalHandle: identity.terminalHandle ?? (identity.terminal_handle as string),
paneKey: identity.paneKey ?? (identity.pane_key as string)
}
}
private bindingMatches(
run: { coordinator_handle: string | null; coordinator_pane_key: string | null },
candidate: LegacyPrincipalCandidate
): boolean {
return (
!run.coordinator_pane_key ||
(run.coordinator_handle === candidate.terminalHandle &&
equivalentLegacyPaneKey(run.coordinator_pane_key, candidate.paneKey))
)
}
private principalMatchesAttestation(
principal: LegacyCompatibilityPrincipalRow,
attestation: OrchestrationCompatibilityCallerAuthority
): boolean {
return (
principal.status === 'committed' &&
principal.host_scope === JSON.stringify(attestation.hostScope) &&
principal.terminal_handle === attestation.terminalHandle &&
equivalentLegacyPaneKey(principal.pane_key, attestation.paneKey) &&
principal.launch_token_hash === attestation.launchTokenHash &&
principal.process_incarnation === attestation.processIncarnation
)
}
}
@@ -0,0 +1,849 @@
import { createHash } from 'node:crypto'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { ORCHESTRATION_CONTRACT_VERSION } from '../../../shared/protocol-version'
import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence'
import Database from '../../sqlite/sync-database'
import { OrcaRuntimeService } from '../orca-runtime'
import { OrchestrationDb } from '../orchestration/db'
import type { RpcRequest, RpcResponse } from './core'
import { RpcDispatcher } from './dispatcher'
import { ORCHESTRATION_METHODS } from './methods/orchestration'
const COORDINATOR_HANDLE = 'term_legacy_coord'
const COORDINATOR_PANE = 'tab_coord:44444444-4444-4444-8444-444444444444'
const WORKER_HANDLE = 'term_legacy_worker'
const WORKER_PANE = 'tab_worker:33333333-3333-4333-8333-333333333333'
type Transport = 'dispatch' | 'websocket'
type Harness = {
db: OrchestrationDb
dispatcher: RpcDispatcher
runtime: OrcaRuntimeService
adoptedRunId: string
taskId: string
dispatchId: string
}
const tempDirs: string[] = []
const databases: OrchestrationDb[] = []
afterEach(() => {
databases.splice(0).forEach((database) => database.close())
tempDirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }))
})
function createHarness(): Harness {
const dir = mkdtempSync(join(tmpdir(), 'orca-legacy-coordinator-race-'))
tempDirs.push(dir)
const dbPath = join(dir, 'orchestration.db')
const before = new OrchestrationDb(dbPath)
const task = before.createTask({
spec: 'legacy assignment',
createdByTerminalHandle: COORDINATOR_HANDLE
})
const dispatch = before.createDispatchContext(task.id, WORKER_HANDLE, WORKER_PANE)
before.close()
const raw = new Database(dbPath)
raw.exec(`
UPDATE dispatch_contexts SET process_incarnation = 'process-1';
DROP INDEX IF EXISTS idx_messages_delivery_contract;
DROP TABLE legacy_mail_receipts;
DROP TABLE legacy_operation_receipts;
DROP TABLE legacy_compatibility_principals;
DROP TABLE legacy_adoptions;
`)
raw.pragma('user_version = 18')
raw.close()
const db = new OrchestrationDb(dbPath)
databases.push(db)
const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string
const runtime = new OrcaRuntimeService()
runtime.setOrchestrationDb(db)
vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) =>
handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : handle === WORKER_HANDLE ? WORKER_PANE : null
)
vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation(
(compatibilityEvidence) => {
const coordinator =
compatibilityEvidence?.terminalHandle === COORDINATOR_HANDLE &&
compatibilityEvidence.paneKey === COORDINATOR_PANE &&
compatibilityEvidence.launchToken === 'coordinator-token'
const worker =
compatibilityEvidence?.terminalHandle === WORKER_HANDLE &&
compatibilityEvidence.paneKey === WORKER_PANE &&
compatibilityEvidence.launchToken === 'worker-token'
return coordinator || worker
? {
hostScope: { kind: 'local', hostId: 'local' },
terminalHandle: coordinator ? COORDINATOR_HANDLE : WORKER_HANDLE,
paneKey: coordinator ? COORDINATOR_PANE : WORKER_PANE,
processIncarnation: 'process-1',
launchTokenHash: createHash('sha256')
.update(coordinator ? 'coordinator-token' : 'worker-token')
.digest('hex')
}
: null
}
)
return {
db,
dispatcher: new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }),
runtime,
adoptedRunId,
taskId: task.id,
dispatchId: dispatch.id
}
}
function evidence(
role: 'coordinator' | 'worker' = 'coordinator'
): OrchestrationCompatibilityEvidence {
return {
terminalHandle: role === 'coordinator' ? COORDINATOR_HANDLE : WORKER_HANDLE,
paneKey: role === 'coordinator' ? COORDINATOR_PANE : WORKER_PANE,
launchToken: role === 'coordinator' ? 'coordinator-token' : 'worker-token'
}
}
function request(
method: string,
params: unknown,
invocationId: string,
callerEvidence = evidence()
): RpcRequest {
return {
id: `rpc_${invocationId}`,
authToken: 'caller-token',
method,
params,
orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION,
orchestrationRequestId: invocationId,
compatibilityInvocationId: invocationId,
orchestrationCompatibilityEvidence: callerEvidence
}
}
async function invoke(
dispatcher: RpcDispatcher,
rpcRequest: RpcRequest,
transport: Transport
): Promise<RpcResponse> {
if (transport === 'dispatch') {
return await dispatcher.dispatch(rpcRequest)
}
const replies: string[] = []
await dispatcher.dispatchStreaming(rpcRequest, (reply) => replies.push(reply))
expect(replies).toHaveLength(1)
return JSON.parse(replies[0]) as RpcResponse
}
function installTakeoverDuringMutationPreflight(harness: Harness): void {
const originalBegin = harness.db.beginMutationReceipt.bind(harness.db)
vi.spyOn(harness.db, 'beginMutationReceipt').mockImplementation((identity) => {
const begun = originalBegin(identity)
settleLegacyWorkerAndTakeOver(harness, 'settled during takeover')
return begun
})
}
function settleLegacyWorkerAndTakeOver(harness: Harness, result: string): void {
harness.db.settleWorkerReport({
taskId: harness.taskId,
dispatchId: harness.dispatchId,
outcome: 'succeeded',
result
})
harness.db.bindRun({
runId: harness.adoptedRunId,
coordinatorHandle: 'term_current_coord',
coordinatorPaneKey: 'tab_current:55555555-5555-4555-8555-555555555555'
})
}
function mutationReceiptCount(db: OrchestrationDb): number {
const sqlite = (db as unknown as { db: Database.Database }).db
return (
sqlite.prepare('SELECT COUNT(*) AS count FROM mutation_receipts').get() as { count: number }
).count
}
function messageCount(db: OrchestrationDb): number {
const sqlite = (db as unknown as { db: Database.Database }).db
return (sqlite.prepare('SELECT COUNT(*) AS count FROM messages').get() as { count: number }).count
}
describe('legacy coordinator takeover races', () => {
it.each(['dispatch', 'websocket'] as const)(
'%s routes a pre-bind coordinator send to the retained worker mailbox',
async (transport) => {
const harness = createHarness()
const response = await invoke(
harness.dispatcher,
request(
'orchestration.send',
{ from: COORDINATOR_HANDLE, to: WORKER_HANDLE, subject: 'continue safely' },
`send-direct-${transport}`
),
transport
)
expect(response).toMatchObject({
ok: true,
result: {
message: {
run_id: harness.adoptedRunId,
to_handle: WORKER_HANDLE,
delivery_contract: 'legacy_direct'
}
}
})
const workerCheck = await harness.dispatcher.dispatch(
request(
'orchestration.check',
{ terminal: WORKER_HANDLE },
`worker-check-${transport}`,
evidence('worker')
)
)
expect(workerCheck).toMatchObject({
ok: true,
result: {
messages: [
{
subject: 'continue safely',
run_id: harness.adoptedRunId,
delivery_contract: 'legacy_direct'
}
],
legacyCompatibility: { ackMessageIds: [expect.any(String)] }
}
})
expect(
harness.db.getInbox(100).some((message) => message.run_id === 'run_legacy_local')
).toBe(false)
}
)
it('routes an exact legacy Dispatch recipient to compatibility delivery', async () => {
const harness = createHarness()
const response = await harness.dispatcher.dispatch(
request(
'orchestration.send',
{
from: COORDINATOR_HANDLE,
to: `dispatch:${harness.dispatchId}`,
subject: 'dispatch guidance'
},
'send-dispatch'
)
)
expect(response).toMatchObject({
ok: true,
result: {
message: {
run_id: harness.adoptedRunId,
to_handle: `dispatch:${harness.dispatchId}`,
delivery_contract: 'legacy_direct'
}
}
})
const workerCheck = await harness.dispatcher.dispatch(
request(
'orchestration.check',
{ terminal: WORKER_HANDLE },
'worker-check-dispatch',
evidence('worker')
)
)
expect(workerCheck).toMatchObject({
ok: true,
result: { messages: [{ subject: 'dispatch guidance' }] }
})
})
it('partitions a coordinator group send by legacy recipient contract', async () => {
const harness = createHarness()
vi.spyOn(harness.runtime, 'listTerminals').mockResolvedValue({
terminals: [
{ handle: COORDINATOR_HANDLE },
{ handle: WORKER_HANDLE },
{ handle: 'term_current_worker' }
],
totalCount: 3,
truncated: false
} as never)
const response = await harness.dispatcher.dispatch(
request(
'orchestration.send',
{ from: COORDINATOR_HANDLE, to: '@all', subject: 'group guidance' },
'send-group'
)
)
expect(response).toMatchObject({ ok: true, result: { recipients: 2 } })
const messages = harness.db
.getInbox(100)
.filter((message) => message.subject === 'group guidance')
expect(messages).toEqual(
expect.arrayContaining([
expect.objectContaining({
run_id: harness.adoptedRunId,
to_handle: WORKER_HANDLE,
delivery_contract: 'legacy_direct'
}),
expect.objectContaining({
run_id: harness.adoptedRunId,
to_handle: 'term_current_worker',
delivery_contract: 'current_delivery'
})
])
)
})
it.each(['dispatch', 'websocket'] as const)(
'%s rejects a coordinator send after takeover during durable preflight',
async (transport) => {
const harness = createHarness()
installTakeoverDuringMutationPreflight(harness)
const before = messageCount(harness.db)
const response = await invoke(
harness.dispatcher,
request(
'orchestration.send',
{ from: COORDINATOR_HANDLE, to: WORKER_HANDLE, subject: 'must not send' },
`send-takeover-${transport}`
),
transport
)
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
expect(messageCount(harness.db)).toBe(before)
expect(mutationReceiptCount(harness.db)).toBe(0)
}
)
it('revalidates a group send after asynchronous terminal discovery', async () => {
const harness = createHarness()
let resolveTerminals:
| ((value: {
terminals: { handle: string }[]
totalCount: number
truncated: boolean
}) => void)
| undefined
let signalListingStarted: (() => void) | undefined
const listingStarted = new Promise<void>((resolve) => {
signalListingStarted = resolve
})
vi.spyOn(harness.runtime, 'listTerminals').mockImplementation(
() =>
new Promise((resolve) => {
resolveTerminals = resolve as typeof resolveTerminals
signalListingStarted?.()
}) as never
)
const before = messageCount(harness.db)
const pending = harness.dispatcher.dispatch(
request(
'orchestration.send',
{ from: COORDINATOR_HANDLE, to: '@all', subject: 'must remain unsent' },
'send-group-takeover'
)
)
await listingStarted
settleLegacyWorkerAndTakeOver(harness, 'settled during terminal discovery')
resolveTerminals?.({
terminals: [{ handle: COORDINATOR_HANDLE }, { handle: WORKER_HANDLE }],
totalCount: 2,
truncated: false
})
await expect(pending).resolves.toMatchObject({
ok: false,
error: { code: 'legacy_read_only' }
})
expect(messageCount(harness.db)).toBe(before)
expect(mutationReceiptCount(harness.db)).toBe(0)
})
it('reports a committed ACK when coordinator takeover interrupts its wait', async () => {
const harness = createHarness()
const incoming = harness.db.insertMessage({
runId: harness.adoptedRunId,
from: WORKER_HANDLE,
to: `run:${harness.adoptedRunId}`,
subject: 'ready to acknowledge'
})
const first = await harness.dispatcher.dispatch(
request(
'orchestration.check',
{ terminal: COORDINATOR_HANDLE, run: harness.adoptedRunId },
'check-delivery-before-takeover'
)
)
expect(first).toMatchObject({
ok: true,
result: { deliveryId: expect.any(String), messages: [{ id: incoming.id }] }
})
const deliveryId = (first as { result: { deliveryId: string } }).result.deliveryId
let resolveWait: (() => void) | undefined
let signalWaitStarted: (() => void) | undefined
const waitStarted = new Promise<void>((resolve) => {
signalWaitStarted = resolve
})
vi.spyOn(harness.runtime, 'waitForMessage').mockImplementation(
() =>
new Promise((resolve) => {
resolveWait = () => resolve('notified')
signalWaitStarted?.()
})
)
const pending = harness.dispatcher.dispatch(
request(
'orchestration.check',
{
terminal: COORDINATOR_HANDLE,
run: harness.adoptedRunId,
ack: deliveryId,
wait: true,
timeoutMs: 10_000
},
'check-ack-takeover'
)
)
await waitStarted
expect(harness.db.getMessageById(incoming.id)?.read).toBe(1)
settleLegacyWorkerAndTakeOver(harness, 'settled during acknowledged wait')
resolveWait?.()
await expect(pending).resolves.toMatchObject({
ok: true,
result: {
acknowledged: deliveryId,
count: 0,
waitInterrupted: 'consumer_fenced',
mutation: { requestId: 'check-ack-takeover', replayed: false }
}
})
const sqlite = (harness.db as unknown as { db: Database.Database }).db
expect(sqlite.prepare('SELECT state FROM mutation_receipts').get()).toEqual({
state: 'completed'
})
})
it('keeps concurrent duplicates on the live ACK-and-wait result', async () => {
const harness = createHarness()
harness.db.insertMessage({
runId: harness.adoptedRunId,
from: WORKER_HANDLE,
to: `run:${harness.adoptedRunId}`,
subject: 'first delivery'
})
const firstCheck = await harness.dispatcher.dispatch(
request(
'orchestration.check',
{ terminal: COORDINATOR_HANDLE, run: harness.adoptedRunId },
'check-first-delivery'
)
)
const deliveryId = (firstCheck as { result: { deliveryId: string } }).result.deliveryId
let resolveWait: (() => void) | undefined
let signalWaitStarted: (() => void) | undefined
const waitStarted = new Promise<void>((resolve) => {
signalWaitStarted = resolve
})
vi.spyOn(harness.runtime, 'waitForMessage').mockImplementation(
() =>
new Promise((resolve) => {
resolveWait = () => resolve('notified')
signalWaitStarted?.()
})
)
const ackRequest = request(
'orchestration.check',
{
terminal: COORDINATOR_HANDLE,
run: harness.adoptedRunId,
ack: deliveryId,
wait: true,
timeoutMs: 10_000
},
'check-ack-concurrent'
)
const original = harness.dispatcher.dispatch(ackRequest)
await waitStarted
const secondLiveDispatcher = new RpcDispatcher({
runtime: harness.runtime,
methods: ORCHESTRATION_METHODS
})
const duplicate = secondLiveDispatcher.dispatch({
...ackRequest,
id: 'rpc_check-ack-concurrent-duplicate'
})
let duplicateSettled = false
void duplicate.then(() => {
duplicateSettled = true
})
await Promise.resolve()
expect(duplicateSettled).toBe(false)
const arrived = harness.db.insertMessage({
runId: harness.adoptedRunId,
from: WORKER_HANDLE,
to: `run:${harness.adoptedRunId}`,
subject: 'arrived while waiting'
})
resolveWait?.()
const [originalResult, duplicateResult] = await Promise.all([original, duplicate])
expect(originalResult).toMatchObject({
ok: true,
result: {
messages: [{ id: arrived.id }],
mutation: { requestId: 'check-ack-concurrent', replayed: false }
}
})
expect(duplicateResult).toMatchObject({
ok: true,
result: {
messages: [{ id: arrived.id }],
mutation: { requestId: 'check-ack-concurrent', replayed: true }
}
})
})
it.each(['dispatch', 'websocket'] as const)(
'%s rejects run-use after takeover during durable preflight',
async (transport) => {
const harness = createHarness()
installTakeoverDuringMutationPreflight(harness)
const response = await invoke(
harness.dispatcher,
request(
'orchestration.runUse',
{ id: harness.adoptedRunId, from: COORDINATOR_HANDLE },
`run-use-takeover-${transport}`
),
transport
)
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
expect(harness.db.getRun(harness.adoptedRunId)).toMatchObject({
coordinator_handle: 'term_current_coord',
consumer_generation: 1
})
expect(mutationReceiptCount(harness.db)).toBe(0)
expect(harness.db.getLegacyCoordinatorPrincipal(harness.adoptedRunId)?.status).toBe('revoked')
}
)
it.each(['dispatch', 'websocket'] as const)(
'%s rejects a mutation after takeover during durable preflight',
async (transport) => {
const harness = createHarness()
const target = harness.db.createTask({
spec: 'must remain ready',
runId: harness.adoptedRunId
})
installTakeoverDuringMutationPreflight(harness)
const response = await invoke(
harness.dispatcher,
request(
'orchestration.taskUpdate',
{
id: target.id,
status: 'completed',
callerTerminalHandle: COORDINATOR_HANDLE
},
`task-update-takeover-${transport}`
),
transport
)
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
expect(harness.db.getTask(target.id)?.status).toBe('ready')
expect(mutationReceiptCount(harness.db)).toBe(0)
}
)
it('revalidates after asynchronous agent detection', async () => {
const harness = createHarness()
const target = harness.db.createTask({
spec: 'must not dispatch after takeover',
runId: harness.adoptedRunId
})
const targetHandle = 'term_current_worker'
const targetPane = 'tab_current_worker:77777777-7777-4777-8777-777777777777'
vi.spyOn(harness.runtime, 'getTerminalPaneKey').mockImplementation((handle: string) =>
handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : handle === targetHandle ? targetPane : null
)
vi.spyOn(harness.runtime, 'getTerminalProcessIncarnation').mockReturnValue('current-process')
let resolveDetection: ((detected: boolean) => void) | undefined
let signalDetectionStarted: (() => void) | undefined
const detectionStarted = new Promise<void>((resolve) => {
signalDetectionStarted = resolve
})
vi.spyOn(harness.runtime, 'isTerminalRunningAgent').mockImplementation(
() =>
new Promise<boolean>((resolve) => {
resolveDetection = resolve
signalDetectionStarted?.()
})
)
const sendPrompt = vi
.spyOn(harness.runtime, 'sendTerminalAgentPrompt')
.mockResolvedValue({ handle: targetHandle, accepted: true, bytesWritten: 1 })
const pending = harness.dispatcher.dispatch(
request(
'orchestration.dispatch',
{ task: target.id, to: targetHandle, from: COORDINATOR_HANDLE, inject: true },
'dispatch-detection-takeover'
)
)
await detectionStarted
settleLegacyWorkerAndTakeOver(harness, 'settled during detection')
resolveDetection?.(true)
const response = await pending
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
expect(harness.db.getTask(target.id)?.status).toBe('ready')
expect(harness.db.getDispatchContext(target.id)).toBeUndefined()
expect(sendPrompt).not.toHaveBeenCalled()
})
it('explicitly takes over an active adopted Run without stranding its legacy worker', async () => {
const harness = createHarness()
const currentHandle = 'term_current_coord'
const currentPane = 'tab_current:55555555-5555-4555-8555-555555555555'
vi.mocked(harness.runtime.getTerminalPaneKey).mockImplementation((handle) =>
handle === COORDINATOR_HANDLE
? COORDINATOR_PANE
: handle === WORKER_HANDLE
? WORKER_PANE
: handle === currentHandle
? currentPane
: null
)
await expect(
harness.dispatcher.dispatch(
request(
'orchestration.runUse',
{ id: harness.adoptedRunId, from: COORDINATOR_HANDLE },
'bind-original-coordinator'
)
)
).resolves.toMatchObject({ ok: true })
const pendingAsk = await harness.dispatcher.dispatch(
request(
'orchestration.ask',
{
from: WORKER_HANDLE,
to: COORDINATOR_HANDLE,
question: 'Continue with the migration?',
timeoutMs: 0
},
'pending-before-takeover',
evidence('worker')
)
)
const questionId = (pendingAsk as { result: { messageId: string } }).result.messageId
const unrelated = harness.db.insertMessage({
runId: harness.adoptedRunId,
from: WORKER_HANDLE,
to: 'term_peer_worker',
subject: 'peer-only',
deliveryContract: 'legacy_direct'
})
const currentRequest = (method: string, params: unknown, invocationId: string): RpcRequest => ({
...request(method, params, invocationId),
orchestrationCompatibilityEvidence: undefined
})
const takeoverParams = {
id: harness.adoptedRunId,
from: currentHandle,
takeoverLegacy: true
}
const takeover = await harness.dispatcher.dispatch(
currentRequest('orchestration.runUse', takeoverParams, 'explicit-takeover')
)
const repeated = await harness.dispatcher.dispatch(
currentRequest('orchestration.runUse', takeoverParams, 'explicit-takeover-repeat')
)
expect(takeover).toMatchObject({
ok: true,
result: { binding: { consumerGeneration: 2 } }
})
expect(repeated).toMatchObject({
ok: true,
result: { binding: { consumerGeneration: 2 } }
})
expect(harness.db.getDispatchContextById(harness.dispatchId)?.status).toBe('dispatched')
expect(harness.db.getLegacyCoordinatorPrincipal(harness.adoptedRunId)?.status).toBe('revoked')
expect(harness.db.getMessageById(unrelated.id)).toMatchObject({
to_handle: 'term_peer_worker',
delivery_contract: 'legacy_direct'
})
const promoted = await harness.dispatcher.dispatch(
currentRequest(
'orchestration.check',
{ terminal: currentHandle, run: harness.adoptedRunId, format: true },
'read-promoted-question'
)
)
expect(promoted).toMatchObject({
ok: true,
result: {
messages: [
{
id: questionId,
to_handle: `run:${harness.adoptedRunId}`,
delivery_contract: 'current_delivery'
}
],
formatted: expect.not.stringContaining(`--from run:${harness.adoptedRunId}`)
}
})
const deliveryId = (promoted as { result: { deliveryId: string } }).result.deliveryId
await expect(
harness.dispatcher.dispatch(
currentRequest(
'orchestration.reply',
{
id: questionId,
body: 'Yes, continue.',
from: currentHandle,
run: harness.adoptedRunId
},
'reply-after-takeover'
)
)
).resolves.toMatchObject({ ok: true, result: { question: { status: 'answered' } } })
await expect(
harness.dispatcher.dispatch(
request(
'orchestration.ask',
{ from: WORKER_HANDLE, resume: questionId, timeoutMs: 100 },
'resume-after-takeover',
evidence('worker')
)
)
).resolves.toMatchObject({ ok: true, result: { answer: 'Yes, continue.' } })
const followUp = await harness.dispatcher.dispatch(
currentRequest(
'orchestration.send',
{
from: currentHandle,
to: `dispatch:${harness.dispatchId}`,
subject: 'Continue from the new coordinator'
},
'current-coordinator-to-legacy-worker'
)
)
expect(followUp).toMatchObject({
ok: true,
result: {
message: {
run_id: harness.adoptedRunId,
to_handle: `dispatch:${harness.dispatchId}`,
delivery_contract: 'legacy_direct'
}
}
})
const workerFollowUp = await harness.dispatcher.dispatch(
request(
'orchestration.check',
{ terminal: WORKER_HANDLE },
'legacy-worker-check-after-takeover',
evidence('worker')
)
)
expect(workerFollowUp).toMatchObject({
ok: true,
result: {
messages: [{ subject: 'Continue from the new coordinator' }],
legacyCompatibility: { ackMessageIds: [expect.any(String)] }
}
})
const [followUpMessageId] = (
workerFollowUp as { result: { legacyCompatibility: { ackMessageIds: string[] } } }
).result.legacyCompatibility.ackMessageIds
await expect(
harness.dispatcher.dispatch(
request(
'orchestration.check',
{
terminal: WORKER_HANDLE,
compatibilityAck: JSON.stringify({ messageIds: [followUpMessageId] })
},
'legacy-worker-ack-after-takeover',
evidence('worker')
)
)
).resolves.toMatchObject({
ok: true,
result: {
acknowledged: [followUpMessageId],
legacyCompatibility: { acknowledged: true }
}
})
await harness.dispatcher.dispatch(
currentRequest(
'orchestration.check',
{ terminal: currentHandle, run: harness.adoptedRunId, ack: deliveryId },
'ack-promoted-question'
)
)
await expect(
harness.dispatcher.dispatch(
request(
'orchestration.send',
{
from: WORKER_HANDLE,
to: COORDINATOR_HANDLE,
subject: 'Need current coordinator',
type: 'escalation',
payload: JSON.stringify({ taskId: harness.taskId })
},
'escalation-after-takeover',
evidence('worker')
)
)
).resolves.toMatchObject({
ok: true,
result: {
message: {
to_handle: `run:${harness.adoptedRunId}`,
delivery_contract: 'current_delivery'
}
}
})
await expect(
harness.dispatcher.dispatch(
request(
'orchestration.taskList',
{ callerTerminalHandle: COORDINATOR_HANDLE },
'old-coordinator-fenced'
)
)
).resolves.toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
})
})
@@ -104,7 +104,7 @@ export async function handleLegacyLifecycleSend(args: {
: { kind: 'message_only' }
})
if (!committed.duplicate) {
runtime.notifyMessageArrived(params.to, committed.message.type)
runtime.notifyMessageArrived(committed.message.to_handle, committed.message.type)
}
return {
message: committed.message,
@@ -0,0 +1,33 @@
import { parsePaneKey } from '../../../shared/stable-pane-id'
import { OrchestrationError } from '../orchestration/orchestration-error'
export function equivalentLegacyPaneKey(
a: string | null | undefined,
b: string | null | undefined
): boolean {
if (!a || !b) {
return false
}
if (a === b) {
return true
}
const aLeaf = parsePaneKey(a)?.leafId
const bLeaf = parsePaneKey(b)?.leafId
return Boolean(aLeaf && bLeaf && aLeaf === bLeaf)
}
export function legacyReadOnlyError(): OrchestrationError {
return new OrchestrationError(
'legacy_read_only',
'This retained legacy assignment could not prove authority from its original live process. No effects were applied.',
{ effectsApplied: false }
)
}
export function legacyCoordinatorReadOnly(): OrchestrationError {
return new OrchestrationError(
'legacy_read_only',
'This retained legacy coordinator could not prove its original process identity. No effects were applied.',
{ effectsApplied: false }
)
}
@@ -65,8 +65,9 @@ export async function handleLegacyAsk(args: {
options,
recipientHandle: recipient
})
const pending = matches.filter((match) => match.question.status === 'pending')
const lostAnswer = matches.find(
const unclaimed = matches.filter((match) => !match.claimedByOperation)
const pending = unclaimed.filter((match) => match.question.status === 'pending')
const lostAnswer = unclaimed.find(
(match) => match.question.status === 'answered' && !match.answerAcknowledged
)
if (lostAnswer) {
@@ -98,7 +99,7 @@ export async function handleLegacyAsk(args: {
questionId = committed.question.message_id
duplicate = committed.duplicate || Boolean(existingQuestionId)
if (!duplicate) {
runtime.notifyMessageArrived(recipient, committed.message.type)
runtime.notifyMessageArrived(committed.message.to_handle, committed.message.type)
}
}
@@ -55,6 +55,10 @@ export class OrchestrationMutationExecutor {
: db.beginMutationReceipt(identity)
if (begun.disposition === 'completed') {
const active = this.inFlight.get(key)
if (active) {
return attachMutationReceipt(await active, requestId, true)
}
return attachMutationReceipt(JSON.parse(begun.row.receipt ?? 'null'), requestId, true)
}
if (begun.disposition === 'pending') {
@@ -102,6 +106,20 @@ export class OrchestrationMutationExecutor {
}
}
const executorsByRuntime = new WeakMap<OrcaRuntimeService, OrchestrationMutationExecutor>()
export function getOrchestrationMutationExecutor(
runtime: OrcaRuntimeService
): OrchestrationMutationExecutor {
const existing = executorsByRuntime.get(runtime)
if (existing) {
return existing
}
const executor = new OrchestrationMutationExecutor(runtime)
executorsByRuntime.set(runtime, executor)
return executor
}
export function authenticatedCallerFingerprint(request: RpcRequest): string {
const callerToken =
request.authToken ||
@@ -252,7 +252,10 @@ describe('durable orchestration mutation ledger', () => {
})
it('recovers a lost ask acceptance without creating a second question', async () => {
const db = new OrchestrationDb(':memory:')
const dir = mkdtempSync(join(tmpdir(), 'orca-mutation-ask-recovery-'))
paths.push(dir)
const dbPath = join(dir, 'orchestration.db')
const db = new OrchestrationDb(dbPath)
const runtime = new OrcaRuntimeService()
runtime.setOrchestrationDb(db)
vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_worker:leaf_worker')
@@ -290,7 +293,16 @@ describe('durable orchestration mutation ledger', () => {
const first = firstDispatcher.dispatch(askRequest, { signal: controller.signal })
await vi.waitFor(() => expect(db.getInbox(10)).toHaveLength(1))
const restartedDispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS })
const restartedDb = new OrchestrationDb(dbPath)
const restartedRuntime = new OrcaRuntimeService()
restartedRuntime.setOrchestrationDb(restartedDb)
vi.spyOn(restartedRuntime, 'getTerminalPaneKey').mockReturnValue('tab_worker:leaf_worker')
vi.spyOn(restartedRuntime, 'getTerminalProcessIncarnation').mockReturnValue('runtime:pty:1')
vi.spyOn(restartedRuntime, 'notifyMessageArrived').mockImplementation(() => {})
const restartedDispatcher = new RpcDispatcher({
runtime: restartedRuntime,
methods: ORCHESTRATION_METHODS
})
const recovered = await restartedDispatcher.dispatch({ ...askRequest, id: 'rpc_ask_2' })
expect(recovered).toMatchObject({
ok: true,
@@ -304,6 +316,7 @@ describe('durable orchestration mutation ledger', () => {
controller.abort()
await first
restartedDb.close()
db.close()
})
})
@@ -1,5 +1,6 @@
import type { FeatureInteractionId } from '../../../shared/feature-interactions'
import { isBrowserPaneUiRuntimeRpcParams } from '../../../shared/runtime-rpc-feature-interaction-source'
import type { OrcaRuntimeService } from '../orca-runtime'
export function getRuntimeFeatureInteractionId(
method: string,
@@ -37,6 +38,25 @@ export function getRuntimeFeatureInteractionId(
return method.startsWith('orchestration.') ? 'agent-orchestration' : null
}
export function recordRuntimeFeatureInteraction(
runtime: OrcaRuntimeService,
method: string,
result: unknown,
alreadyRecorded?: Set<FeatureInteractionId>,
rawParams?: unknown
): void {
const id = getRuntimeFeatureInteractionId(method, result, rawParams)
if (!id || alreadyRecorded?.has(id)) {
return
}
try {
runtime.recordFeatureInteraction(id)
alreadyRecorded?.add(id)
} catch {
// Best-effort education state must not break runtime tools.
}
}
function hasBooleanResult(value: unknown, key: string): boolean {
return (
value !== null && typeof value === 'object' && (value as Record<string, unknown>)[key] === true
+35
View File
@@ -66,6 +66,10 @@ import {
import type { Store } from '../persistence'
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
import { runRemoteOrcaCli } from './ssh-remote-orca-cli'
import {
acknowledgeRemoteOrcaCliPostOutput,
parseRemoteOrcaCliPostOutput
} from './ssh-remote-orchestration-post-output'
import { toSshExecutionHostId, type ExecutionHostId } from '../../shared/execution-host'
import { isTerminalLeafId, makePaneKey } from '../../shared/stable-pane-id'
import { isValidTerminalTabId } from '../../shared/terminal-tab-id'
@@ -716,6 +720,37 @@ export class SshRelaySession {
this.runtime.releaseOrchestrationCompatibilitySshAttachment(runtimeAuthority.attachmentId)
}
})
mux.onRequest('orca.cli.postOutput', async (params) => {
if (!this.runtime) {
throw new Error('Orca runtime is unavailable')
}
const rawEnv = params.env
const env =
rawEnv && typeof rawEnv === 'object' && !Array.isArray(rawEnv)
? Object.fromEntries(
Object.entries(rawEnv).filter(
(entry): entry is [string, string] =>
typeof entry[0] === 'string' && typeof entry[1] === 'string'
)
)
: {}
const runtimeAuthority = this.runtime.registerOrchestrationCompatibilitySshAttachment(
this.targetId,
connectionIncarnation
)
this.activeCompatibilityAttachmentIds.add(runtimeAuthority.attachmentId)
try {
await acknowledgeRemoteOrcaCliPostOutput(this.runtime, {
postOutput: parseRemoteOrcaCliPostOutput(params.postOutput),
env,
runtimeAuthority
})
return { acknowledged: true }
} finally {
this.activeCompatibilityAttachmentIds.delete(runtimeAuthority.attachmentId)
this.runtime.releaseOrchestrationCompatibilitySshAttachment(runtimeAuthority.attachmentId)
}
})
}
// Why: ship plugin/extension source from Orca so agent-event changes don't force a relay redeploy (agent-status-over-ssh.md §4/§8). Best-effort.
@@ -43,8 +43,23 @@ export type RemoteOrcaCliResult = {
stdout: string
stderr: string
exitCode: number
postOutput?: RemoteOrcaCliPostOutput
}
export type RemoteOrcaCliPostOutput =
| {
kind: 'legacy_check_ack'
terminal: string
messageIds: string[]
types?: string[]
}
| {
kind: 'legacy_question_ack'
terminal: string
questionId: string
answerMessageId: string
}
export type HostCliPassthroughOptions = {
execPath?: string
cliEntryPath?: string
@@ -0,0 +1,40 @@
import type { RpcResponse } from '../runtime/rpc/core'
import type { ParsedRemoteCli } from './ssh-remote-cli-argument-error'
import type { RemoteOrcaCliResult } from './ssh-remote-cli-host-passthrough'
import { formatRemoteCli } from './ssh-remote-cli-format'
import {
formatRemoteOrchestrationAsk,
getRemoteCliExitCode
} from './ssh-remote-orchestration-ask-output'
import { formatRemoteOrchestrationCheck } from './ssh-remote-orchestration-check-output'
import { getRemoteCliPostOutput } from './ssh-remote-orchestration-post-output'
import { resolveRemoteCliHandle } from './ssh-remote-cli-args'
export function formatInProcessRemoteCliResult(
parsed: ParsedRemoteCli,
env: Record<string, string>,
response: RpcResponse,
json: boolean
): RemoteOrcaCliResult {
const command = parsed.commandPath.join(' ')
const formatted =
command === 'orchestration check'
? formatRemoteOrchestrationCheck(
response,
json,
resolveRemoteCliHandle(parsed.flags, env, 'terminal'),
parsed.flags.has('format')
)
: command === 'orchestration ask'
? formatRemoteOrchestrationAsk(response, json)
: json
? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' }
: formatRemoteCli(response)
const postOutput = getRemoteCliPostOutput(parsed, env, response)
return {
stdout: formatted.stdout,
stderr: formatted.stderr,
exitCode: getRemoteCliExitCode(command, response),
...(postOutput ? { postOutput } : {})
}
}
-1
View File
@@ -24,7 +24,6 @@ const LEGACY_FALLBACK_OPTIONS: HostCliPassthroughOptions = {
userDataPath: '/host/user-data',
entryExists: () => false
}
type FakeChild = EventEmitter & {
stdout: EventEmitter
stderr: EventEmitter
+48 -18
View File
@@ -6,7 +6,6 @@ import { ORCHESTRATION_CONTRACT_VERSION } from '../../shared/protocol-version'
import { RpcDispatcher } from '../runtime/rpc/dispatcher'
import type { RpcResponse } from '../runtime/rpc/core'
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
import { formatRemoteCli } from './ssh-remote-cli-format'
import {
HostCliUnavailableError,
runHostOrcaCliPassthrough,
@@ -26,9 +25,9 @@ import { buildRemoteCliError } from './ssh-remote-cli-error-response'
import { getRemoteLinearHelp, tryDispatchRemoteLinearCli } from './ssh-remote-linear-cli'
import {
getRemoteOrchestrationPayload,
hasRemoteLifecycleRejection,
resolveRemoteOrchestrationSender
} from './ssh-remote-orchestration-send'
import { formatInProcessRemoteCliResult } from './ssh-remote-cli-in-process-result'
export type { RemoteOrcaCliRequest, RemoteOrcaCliResult } from './ssh-remote-cli-host-passthrough'
@@ -51,6 +50,7 @@ export async function runRemoteOrcaCli(
): Promise<RemoteOrcaCliResult> {
const parsed = parseRemoteCliArgs(request.argv)
const json = parsed.flags.has('json')
const command = parsed.commandPath.join(' ')
const interactiveMessage = HOST_INTERACTIVE_COMMANDS[parsed.commandPath[0] ?? '']
if (interactiveMessage) {
@@ -64,6 +64,17 @@ export async function runRemoteOrcaCli(
return { stdout: '', stderr: `${interactiveMessage}\n`, exitCode: 1 }
}
if (command === 'orchestration check' || command === 'orchestration ask') {
// Why: compatibility ACKs must wait until relay stdout is observable; a host CLI child can only flush into main's capture pipe.
return await runLegacyRemoteOrcaCli(
runtime,
request,
parsed,
json,
new HostCliUnavailableError('output-ordered orchestration bridge required')
)
}
let passthroughFailure: HostCliUnavailableError | null = null
try {
return await runHostOrcaCliPassthrough(request, passthroughOptions)
@@ -101,16 +112,7 @@ async function runLegacyRemoteOrcaCli(
passthroughFailure.message,
request.runtimeAuthority
)
const formatted = json
? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' }
: formatRemoteCli(response)
return {
stdout: formatted.stdout,
stderr: formatted.stderr,
// Why: the legacy SSH bridge bypasses the local CLI handler that turns
// a persisted lifecycle rejection into an unsuccessful command.
exitCode: response.ok && !hasRemoteLifecycleRejection(response.result) ? 0 : 1
}
return formatInProcessRemoteCliResult(parsed, request.env, response, json)
} catch (err) {
const message = err instanceof Error ? err.message : String(err)
const code =
@@ -141,13 +143,17 @@ async function dispatchRemoteCli(
runtimeAuthority: RemoteOrcaCliRequest['runtimeAuthority']
): Promise<RpcResponse> {
const command = parsed.commandPath.join(' ')
const compatibilityInvocationId = randomUUID()
const inheritedEvidence = readOrchestrationCompatibilityEvidence(env)
const orchestrationCompatibilityEvidence = runtimeAuthority
? { ...inheritedEvidence, host: runtimeAuthority }
: inheritedEvidence
const compatibilityEnvelope: RuntimeOrchestrationEnvelope = {
compatibilityInvocationId,
compatibilityInvocationId: randomUUID(),
orchestrationRequestId:
optionalRemoteCliString(parsed.flags, 'retry-request') ??
(command === 'orchestration check' || command === 'orchestration ask'
? randomUUID()
: undefined),
orchestrationCompatibilityEvidence
}
const linearResponse = await tryDispatchRemoteLinearCli(dispatcher, parsed, env, stdin)
@@ -201,8 +207,7 @@ async function dispatchRemoteCli(
},
{
...compatibilityEnvelope,
orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability'),
orchestrationRequestId: optionalRemoteCliString(parsed.flags, 'retry-request')
orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability')
}
)
}
@@ -215,15 +220,39 @@ async function dispatchRemoteCli(
terminalPaneKey: parsed.flags.has('terminal')
? undefined
: env.ORCA_PANE_KEY || undefined,
unread: parsed.flags.has('unread') ? true : undefined,
unread: parsed.flags.has('unread') ? true : parsed.flags.has('peek') ? false : undefined,
peek: parsed.flags.has('peek') ? true : undefined,
all: parsed.flags.has('all') ? true : undefined,
types: optionalRemoteCliString(parsed.flags, 'types'),
format: parsed.flags.has('format') ? true : undefined,
inject: parsed.flags.has('inject') ? true : undefined,
compatibilityCliCommand: 'orca',
run: optionalRemoteCliString(parsed.flags, 'run'),
ack: optionalRemoteCliString(parsed.flags, 'ack'),
wait: parsed.flags.has('wait') ? true : undefined,
timeoutMs: optionalRemoteCliNumber(parsed.flags, 'timeout-ms')
},
compatibilityEnvelope
)
case 'orchestration ask':
return await call(
dispatcher,
'orchestration.ask',
{
to: optionalRemoteCliString(parsed.flags, 'to'),
question: optionalRemoteCliString(parsed.flags, 'question'),
resume: optionalRemoteCliString(parsed.flags, 'resume'),
options: optionalRemoteCliString(parsed.flags, 'options'),
timeoutMs: optionalRemoteCliNumber(parsed.flags, 'timeout-ms'),
from: resolveRemoteCliHandle(parsed.flags, env, 'from'),
run: optionalRemoteCliString(parsed.flags, 'run'),
compatibilityCliCommand: 'orca'
},
{
...compatibilityEnvelope,
orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability')
}
)
case 'orchestration reply':
return await call(
dispatcher,
@@ -271,7 +300,8 @@ async function call(
? ORCHESTRATION_CONTRACT_VERSION
: undefined,
orchestrationRequestId: envelope?.orchestrationRequestId,
compatibilityInvocationId: envelope?.compatibilityInvocationId,
compatibilityInvocationId:
envelope?.orchestrationRequestId ?? envelope?.compatibilityInvocationId,
orchestrationCompatibilityEvidence: envelope?.orchestrationCompatibilityEvidence
})
}
@@ -0,0 +1,66 @@
import type { RpcResponse } from '../runtime/rpc/core'
import { formatRemoteCli } from './ssh-remote-cli-format'
import { hasRemoteLifecycleRejection } from './ssh-remote-orchestration-send'
export function formatRemoteOrchestrationAsk(
response: RpcResponse,
json: boolean
): { stdout: string; stderr: string } {
if (!response.ok || !isRecord(response.result)) {
return json
? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' }
: formatRemoteCli(response)
}
if (json) {
return { stdout: `${JSON.stringify(response.result)}\n`, stderr: '' }
}
if (isRecord(response.result.legacyCompatibility)) {
const compatibility = response.result.legacyCompatibility
if (compatibility.resumeRequired === true && typeof compatibility.resumeCommand === 'string') {
return {
stdout:
`Question ${String(response.result.messageId ?? 'unknown')} committed.\n` +
`Resume with: ${compatibility.resumeCommand}\n`,
stderr: ''
}
}
}
const answer = typeof response.result.answer === 'string' ? response.result.answer : ''
const thread = typeof response.result.threadId === 'string' ? response.result.threadId : 'unknown'
const timeoutMs =
typeof response.result.timeoutMs === 'number' ? response.result.timeoutMs : undefined
const stderr = response.result.timedOut
? `ask timeout after ${timeoutMs ?? 0}ms (thread ${thread})\n`
: response.result.cancelled
? response.result.connectionLost
? `ask connection closed (question ${String(response.result.messageId ?? 'unknown')})\n`
: `ask cancelled (question ${String(response.result.messageId ?? 'unknown')})\n`
: ''
return { stdout: answer ? `${answer}\n` : '', stderr }
}
export function getRemoteCliExitCode(command: string, response: RpcResponse): number {
if (!response.ok || hasRemoteLifecycleRejection(response.result)) {
return 1
}
if (
command === 'orchestration ask' &&
isRecord(response.result) &&
isRecord(response.result.legacyCompatibility) &&
response.result.legacyCompatibility.resumeRequired === true
) {
return 75
}
if (
command === 'orchestration ask' &&
isRecord(response.result) &&
(response.result.timedOut === true || response.result.cancelled === true)
) {
return 1
}
return 0
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
}
@@ -0,0 +1,48 @@
import {
formatOrchestrationCheckText,
prepareOrchestrationCheckOutput,
type OrchestrationCheckOutput,
type OrchestrationMessageSummary
} from '../../shared/orchestration-check-output'
import type { RpcResponse } from '../runtime/rpc/core'
import { formatRemoteCli } from './ssh-remote-cli-format'
export function formatRemoteOrchestrationCheck(
response: RpcResponse,
json: boolean,
terminal: string,
formattedRequested: boolean
): { stdout: string; stderr: string } {
if (!response.ok || !isRecord(response.result)) {
return json
? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' }
: formatRemoteCli(response)
}
const messages = Array.isArray(response.result.messages)
? response.result.messages.filter(isMessageSummary)
: []
const count = typeof response.result.count === 'number' ? response.result.count : messages.length
const result = prepareOrchestrationCheckOutput(
{ ...(response.result as OrchestrationCheckOutput), messages, count },
terminal,
formattedRequested
)
if (json) {
return {
stdout: `${JSON.stringify({ ...response, result }, null, 2)}\n`,
stderr: ''
}
}
return {
stdout: `${formatOrchestrationCheckText(result, terminal)}\n`,
stderr: ''
}
}
function isMessageSummary(value: unknown): value is OrchestrationMessageSummary {
return isRecord(value) && typeof value.id === 'string' && typeof value.from_handle === 'string'
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
}
@@ -0,0 +1,394 @@
import { createHash } from 'node:crypto'
import { describe, expect, it, vi } from 'vitest'
vi.mock('electron', () => ({
app: {
isPackaged: false,
getAppPath: () => '/host/app'
}
}))
vi.mock('../persistence', () => ({
getCanonicalUserDataPath: () => '/host/user-data'
}))
import { OrchestrationDb } from '../runtime/orchestration/db'
import { OrcaRuntimeService } from '../runtime/orca-runtime'
import type Database from '../sqlite/sync-database'
import type { HostCliPassthroughOptions } from './ssh-remote-cli-host-passthrough'
import { runRemoteOrcaCli } from './ssh-remote-orca-cli'
import { acknowledgeRemoteOrcaCliPostOutput } from './ssh-remote-orchestration-post-output'
const LEGACY_FALLBACK_OPTIONS: HostCliPassthroughOptions = {
execPath: '/host/electron',
cliEntryPath: '/host/app/out/cli/index.js',
userDataPath: '/host/user-data',
entryExists: () => false
}
const WORKER_HANDLE = 'term_legacy_ssh_worker'
const WORKER_PANE = 'tab_legacy_ssh:33333333-3333-4333-8333-333333333333'
const COORDINATOR_HANDLE = 'term_legacy_ssh_coord'
const COORDINATOR_PANE = 'tab_legacy_coord:44444444-4444-4444-8444-444444444444'
const WORKER_ENV = {
ORCA_TERMINAL_HANDLE: WORKER_HANDLE,
ORCA_PANE_KEY: WORKER_PANE,
ORCA_AGENT_LAUNCH_TOKEN: 'legacy-ssh-token'
}
const COORDINATOR_ENV = {
ORCA_TERMINAL_HANDLE: COORDINATOR_HANDLE,
ORCA_PANE_KEY: COORDINATOR_PANE,
ORCA_AGENT_LAUNCH_TOKEN: 'legacy-ssh-coordinator-token'
}
const RUNTIME_AUTHORITY = {
kind: 'ssh' as const,
targetId: 'saved-target',
connectionIncarnation: 'connection-1',
attachmentId: 'attachment-1'
}
function createLegacyRuntime() {
const db = new OrchestrationDb(':memory:')
const run = db.createRun({
objective: 'Adopted legacy SSH work',
coordinatorHandle: COORDINATOR_HANDLE,
coordinatorPaneKey: COORDINATOR_PANE
})
const task = db.createTask({
spec: 'legacy SSH assignment',
runId: run.id,
createdByTerminalHandle: COORDINATOR_HANDLE
})
const dispatch = db.createDispatchContext(task.id, WORKER_HANDLE, WORKER_PANE)
const sqlite = (db as unknown as { db: Database.Database }).db
sqlite
.prepare(
`UPDATE dispatch_contexts
SET contract_version = 0, process_incarnation = 'process-1'
WHERE id = ?`
)
.run(dispatch.id)
sqlite
.prepare(
`INSERT INTO legacy_adoptions (source_run_id, adopted_run_id, scheduler_state_lost)
VALUES ('run_legacy_local', ?, 1)`
)
.run(run.id)
const runtime = new OrcaRuntimeService()
runtime.setOrchestrationDb(db)
vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) =>
handle === WORKER_HANDLE ? WORKER_PANE : handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : null
)
vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation((evidence) => {
const worker =
evidence?.terminalHandle === WORKER_HANDLE &&
evidence.paneKey === WORKER_PANE &&
evidence.launchToken === WORKER_ENV.ORCA_AGENT_LAUNCH_TOKEN
const coordinator =
evidence?.terminalHandle === COORDINATOR_HANDLE &&
evidence.paneKey === COORDINATOR_PANE &&
evidence.launchToken === COORDINATOR_ENV.ORCA_AGENT_LAUNCH_TOKEN
if (
(!worker && !coordinator) ||
evidence.host?.kind !== 'ssh' ||
evidence.host.targetId !== RUNTIME_AUTHORITY.targetId ||
evidence.host.connectionIncarnation !== RUNTIME_AUTHORITY.connectionIncarnation
) {
return null
}
const identity = worker
? {
terminalHandle: WORKER_HANDLE,
paneKey: WORKER_PANE,
launchToken: WORKER_ENV.ORCA_AGENT_LAUNCH_TOKEN
}
: {
terminalHandle: COORDINATOR_HANDLE,
paneKey: COORDINATOR_PANE,
launchToken: COORDINATOR_ENV.ORCA_AGENT_LAUNCH_TOKEN
}
return {
hostScope: { kind: 'ssh', targetId: RUNTIME_AUTHORITY.targetId },
terminalHandle: identity.terminalHandle,
paneKey: identity.paneKey,
processIncarnation: 'process-1',
launchTokenHash: createHash('sha256').update(identity.launchToken).digest('hex')
}
})
vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {})
return { db, runtime, run, dispatch }
}
describe('legacy SSH orchestration fallback', () => {
it('acknowledges a consuming check only after remote output', async () => {
const { db, runtime, run } = createLegacyRuntime()
const message = db.insertMessage({
runId: run.id,
deliveryContract: 'legacy_direct',
from: COORDINATOR_HANDLE,
to: WORKER_HANDLE,
subject: 'retained SSH mail'
})
const request = {
argv: ['orchestration', 'check', '--unread', '--inject', '--json'],
cwd: '/home/alice/repo',
env: WORKER_ENV,
runtimeAuthority: RUNTIME_AUTHORITY
}
try {
const first = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
const replay = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
expect(JSON.parse(first.stdout)).toMatchObject({
result: {
messages: [{ id: message.id }],
formatted: expect.stringContaining('retained SSH mail')
}
})
expect(first.postOutput).toEqual({
kind: 'legacy_check_ack',
terminal: WORKER_HANDLE,
messageIds: [message.id]
})
expect(JSON.parse(replay.stdout)).toMatchObject({
result: { messages: [{ id: message.id }] }
})
expect(db.getMessageById(message.id)?.read).toBe(0)
await acknowledgeRemoteOrcaCliPostOutput(runtime, {
postOutput: first.postOutput!,
env: WORKER_ENV,
runtimeAuthority: RUNTIME_AUTHORITY
})
const afterAck = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
expect(JSON.parse(afterAck.stdout)).toMatchObject({ result: { messages: [], count: 0 } })
expect(afterAck.postOutput).toBeUndefined()
expect(db.getMessageById(message.id)?.read).toBe(1)
} finally {
db.close()
}
})
it('keeps peek formatted and non-consuming', async () => {
const { db, runtime, run } = createLegacyRuntime()
const message = db.insertMessage({
runId: run.id,
deliveryContract: 'legacy_direct',
from: COORDINATOR_HANDLE,
to: WORKER_HANDLE,
subject: 'inspect retained SSH mail'
})
try {
const peek = await runRemoteOrcaCli(
runtime,
{
argv: ['orchestration', 'check', '--peek', '--format', '--json'],
cwd: '/home/alice/repo',
env: WORKER_ENV,
runtimeAuthority: RUNTIME_AUTHORITY
},
LEGACY_FALLBACK_OPTIONS
)
const output = JSON.parse(peek.stdout) as {
result: { formatted: string }
}
expect(output).toMatchObject({
result: {
messages: [{ id: message.id }],
formatted: expect.stringContaining('inspect retained SSH mail'),
legacyCompatibility: { readOnly: true }
}
})
expect(output.result.formatted).toContain(`${message.id} [legacy, read-only]`)
expect(output.result.formatted).toContain(
'Inspection only: reply and acknowledgment are unavailable.'
)
expect(output.result.formatted).not.toContain('orchestration reply')
expect(peek.postOutput).toBeUndefined()
expect(db.getMessageById(message.id)?.read).toBe(0)
} finally {
db.close()
}
})
it('reads and acknowledges current Run delivery', async () => {
const { db, runtime, run } = createLegacyRuntime()
const message = db.insertMessage({
runId: run.id,
from: WORKER_HANDLE,
to: `run:${run.id}`,
subject: 'current Run mail'
})
const baseRequest = {
cwd: '/home/alice/repo',
env: COORDINATOR_ENV,
runtimeAuthority: RUNTIME_AUTHORITY
}
const spawn = vi.fn()
const hostCliAvailable = {
...LEGACY_FALLBACK_OPTIONS,
entryExists: () => true,
spawn: spawn as HostCliPassthroughOptions['spawn']
}
try {
const checked = await runRemoteOrcaCli(
runtime,
{
...baseRequest,
argv: ['orchestration', 'check', '--run', run.id]
},
hostCliAvailable
)
expect(checked.stdout).toContain('Delivery ')
expect(checked.stdout).toContain(`${message.id} [status] from=${WORKER_HANDLE}`)
expect(spawn).not.toHaveBeenCalled()
const checkedJson = await runRemoteOrcaCli(
runtime,
{
...baseRequest,
argv: ['orchestration', 'check', '--run', run.id, '--json']
},
hostCliAvailable
)
const result = JSON.parse(checkedJson.stdout) as {
result: { deliveryId: string; messages: { id: string }[] }
}
expect(result.result.messages).toEqual([expect.objectContaining({ id: message.id })])
const acknowledged = await runRemoteOrcaCli(
runtime,
{
...baseRequest,
argv: [
'orchestration',
'check',
'--run',
run.id,
'--ack',
result.result.deliveryId,
'--json'
]
},
hostCliAvailable
)
expect(JSON.parse(acknowledged.stdout)).toMatchObject({
result: { acknowledged: result.result.deliveryId }
})
expect(db.getMessageById(message.id)?.read).toBe(1)
} finally {
db.close()
}
})
it('replays an SSH legacy ask by retry request without creating another question', async () => {
const { db, runtime } = createLegacyRuntime()
const argv = [
'orchestration',
'ask',
'--to',
COORDINATOR_HANDLE,
'--question',
'Continue?',
'--timeout-ms',
'1',
'--retry-request',
'ssh-question-1',
'--json'
]
const request = {
argv,
cwd: '/home/alice/repo',
env: WORKER_ENV,
runtimeAuthority: RUNTIME_AUTHORITY
}
try {
const first = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
const replay = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
const sqlite = (db as unknown as { db: Database.Database }).db
const firstResult = JSON.parse(first.stdout) as { messageId: string; timedOut: boolean }
const replayResult = JSON.parse(replay.stdout) as { messageId: string; timedOut: boolean }
expect(first).toMatchObject({ exitCode: 1 })
expect(firstResult).toMatchObject({ messageId: expect.any(String), timedOut: true })
expect(replay).toMatchObject({ exitCode: 1 })
expect(replayResult).toMatchObject({ messageId: firstResult.messageId, timedOut: true })
expect(
(
sqlite.prepare('SELECT COUNT(*) AS count FROM question_threads').get() as {
count: number
}
).count
).toBe(1)
} finally {
db.close()
}
})
it('resumes and acknowledges a question answer', async () => {
const { db, runtime, run, dispatch } = createLegacyRuntime()
const pending = db.createQuestion({
runId: run.id,
dispatchId: dispatch.id,
askerHandle: WORKER_HANDLE,
question: 'Proceed?'
})
const answer = db.answerQuestion({
messageId: pending.question.message_id,
runId: run.id,
consumerGeneration: run.consumer_generation,
body: 'yes'
})
const sqlite = (db as unknown as { db: Database.Database }).db
sqlite
.prepare(
`UPDATE messages
SET delivery_contract = 'legacy_direct', read = 0
WHERE id = ?`
)
.run(answer.message.id)
try {
const result = await runRemoteOrcaCli(
runtime,
{
argv: ['orchestration', 'ask', '--resume', pending.question.message_id, '--json'],
cwd: '/home/alice/repo',
env: WORKER_ENV,
runtimeAuthority: RUNTIME_AUTHORITY
},
LEGACY_FALLBACK_OPTIONS
)
expect(result.exitCode).toBe(0)
expect(JSON.parse(result.stdout)).toMatchObject({
answer: 'yes',
messageId: pending.question.message_id,
timedOut: false
})
expect(result.postOutput).toEqual({
kind: 'legacy_question_ack',
terminal: WORKER_HANDLE,
questionId: pending.question.message_id,
answerMessageId: answer.message.id
})
expect(db.getMessageById(answer.message.id)?.read).toBe(0)
await acknowledgeRemoteOrcaCliPostOutput(runtime, {
postOutput: result.postOutput!,
env: WORKER_ENV,
runtimeAuthority: RUNTIME_AUTHORITY
})
expect(db.getMessageById(answer.message.id)?.read).toBe(1)
} finally {
db.close()
}
})
})
@@ -0,0 +1,143 @@
import { randomUUID } from 'node:crypto'
import { readOrchestrationCompatibilityEvidence } from '../../shared/orchestration-compatibility-evidence'
import { ORCHESTRATION_CONTRACT_VERSION } from '../../shared/protocol-version'
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
import type { RpcResponse } from '../runtime/rpc/core'
import { RpcDispatcher } from '../runtime/rpc/dispatcher'
import type {
RemoteOrcaCliPostOutput,
RemoteOrcaCliRequest
} from './ssh-remote-cli-host-passthrough'
import { RemoteCliArgumentError, type ParsedRemoteCli } from './ssh-remote-cli-argument-error'
import { optionalRemoteCliString, resolveRemoteCliHandle } from './ssh-remote-cli-args'
export async function acknowledgeRemoteOrcaCliPostOutput(
runtime: OrcaRuntimeService,
args: {
postOutput: RemoteOrcaCliPostOutput
env: Record<string, string>
runtimeAuthority?: RemoteOrcaCliRequest['runtimeAuthority']
}
): Promise<void> {
const inheritedEvidence = readOrchestrationCompatibilityEvidence(args.env)
const orchestrationCompatibilityEvidence = args.runtimeAuthority
? { ...inheritedEvidence, host: args.runtimeAuthority }
: inheritedEvidence
const params =
args.postOutput.kind === 'legacy_check_ack'
? {
terminal: args.postOutput.terminal,
compatibilityAck: JSON.stringify({
messageIds: args.postOutput.messageIds,
types: args.postOutput.types
})
}
: {
terminal: args.postOutput.terminal,
compatibilityQuestionAck: JSON.stringify({
questionId: args.postOutput.questionId,
answerMessageId: args.postOutput.answerMessageId
})
}
const response = await new RpcDispatcher({ runtime }).dispatch({
id: `remote-cli-post-output-${randomUUID()}`,
authToken: 'remote-cli',
method: 'orchestration.check',
params,
orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION,
compatibilityInvocationId: randomUUID(),
orchestrationCompatibilityEvidence
})
if (!response.ok) {
throw new Error(response.error.message)
}
}
export function parseRemoteOrcaCliPostOutput(value: unknown): RemoteOrcaCliPostOutput {
if (!isRecord(value) || typeof value.kind !== 'string' || typeof value.terminal !== 'string') {
throw invalidPostOutput()
}
if (
value.kind === 'legacy_check_ack' &&
Array.isArray(value.messageIds) &&
value.messageIds.every((id) => typeof id === 'string') &&
(value.types === undefined ||
(Array.isArray(value.types) && value.types.every((type) => typeof type === 'string')))
) {
return {
kind: value.kind,
terminal: value.terminal,
messageIds: value.messageIds,
...(value.types === undefined ? {} : { types: value.types })
}
}
if (
value.kind === 'legacy_question_ack' &&
typeof value.questionId === 'string' &&
typeof value.answerMessageId === 'string'
) {
return {
kind: value.kind,
terminal: value.terminal,
questionId: value.questionId,
answerMessageId: value.answerMessageId
}
}
throw invalidPostOutput()
}
export function getRemoteCliPostOutput(
parsed: ParsedRemoteCli,
env: Record<string, string>,
response: RpcResponse
): RemoteOrcaCliPostOutput | undefined {
if (!response.ok || !isRecord(response.result)) {
return undefined
}
const compatibility = response.result.legacyCompatibility
if (!isRecord(compatibility)) {
return undefined
}
const command = parsed.commandPath.join(' ')
if (
command === 'orchestration check' &&
Array.isArray(compatibility.ackMessageIds) &&
compatibility.ackMessageIds.length > 0 &&
compatibility.ackMessageIds.every((id) => typeof id === 'string')
) {
const types = optionalRemoteCliString(parsed.flags, 'types')
?.split(',')
.map((type) => type.trim())
.filter(Boolean)
return {
kind: 'legacy_check_ack',
terminal: resolveRemoteCliHandle(parsed.flags, env, 'terminal'),
messageIds: compatibility.ackMessageIds,
...(types ? { types } : {})
}
}
const acknowledgement = compatibility.answerAcknowledgement
if (
command === 'orchestration ask' &&
response.result.answer !== null &&
isRecord(acknowledgement) &&
typeof acknowledgement.questionId === 'string' &&
typeof acknowledgement.answerMessageId === 'string'
) {
return {
kind: 'legacy_question_ack',
terminal: resolveRemoteCliHandle(parsed.flags, env, 'from'),
questionId: acknowledgement.questionId,
answerMessageId: acknowledgement.answerMessageId
}
}
return undefined
}
function invalidPostOutput(): RemoteCliArgumentError {
return new RemoteCliArgumentError('invalid_argument', 'Invalid SSH CLI post-output action.')
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
}
@@ -829,4 +829,73 @@ describe('attachMainWindowServices', () => {
await expect(revealPromise).resolves.toEqual({ tabId: 'tab-1', title: 'SSH tmux' })
expect(removeListenerMock).toHaveBeenCalledWith('terminal:tabCreateReply', handler)
})
it('requires an exact renderer identity receipt for recovered worker reveals', async () => {
const sendMock = vi.fn()
const mainWindow = createMainWindow({ send: sendMock })
const runtime = createRuntime()
attachMainWindowServices(mainWindow as never, createStore(), runtime as never)
const notifier = runtime.setNotifier.mock.calls[0][0] as {
revealTerminalSession: (
worktreeId: string,
opts: {
ptyId: string
tabId: string
leafId: string
expectedProcessIdentity: { terminalHandle: string; incarnationId: string }
}
) => Promise<unknown>
}
const opts = {
ptyId: 'pty-worker',
tabId: 'tab-worker',
leafId: 'leaf-worker',
expectedProcessIdentity: {
terminalHandle: 'term_worker',
incarnationId: 'inc-worker'
}
}
const mismatch = notifier.revealTerminalSession('worktree-1', opts)
const mismatchPayload = sendMock.mock.calls.at(-1)?.[1]
const mismatchHandler = onMock.mock.calls.findLast(
([channel]) => channel === 'terminal:tabCreateReply'
)?.[1]
mismatchHandler?.(
{ sender: mainWindow.webContents },
{
requestId: mismatchPayload.requestId,
tabId: 'tab-worker',
identity: {
worktreeId: 'worktree-1',
tabId: 'tab-worker',
leafId: 'leaf-worker',
ptyId: 'pty-replacement'
}
}
)
await expect(mismatch).rejects.toThrow('terminal_reveal_identity_mismatch')
const exact = notifier.revealTerminalSession('worktree-1', opts)
const exactPayload = sendMock.mock.calls.at(-1)?.[1]
const exactHandler = onMock.mock.calls.findLast(
([channel]) => channel === 'terminal:tabCreateReply'
)?.[1]
const identity = {
worktreeId: 'worktree-1',
tabId: 'tab-worker',
leafId: 'leaf-worker',
ptyId: 'pty-worker'
}
exactHandler?.(
{ sender: mainWindow.webContents },
{ requestId: exactPayload.requestId, tabId: 'tab-worker', identity }
)
await expect(exact).resolves.toEqual({
tabId: 'tab-worker',
title: undefined,
identity
})
})
})
+27 -5
View File
@@ -49,6 +49,7 @@ import type {
RuntimeMarkdownSaveTabResult
} from '../../shared/mobile-markdown-document'
import type { RuntimeMobileSessionTabMove } from '../../shared/runtime-types'
import type { TerminalTabCreateReply } from '../../shared/terminal-reveal-identity'
import { isNativeFileDropPayload, type NativeFileDropPayload } from '../../shared/native-file-drop'
import { requestMobileMarkdownFromRenderer } from './mobile-markdown-request-relay'
import { requestTerminalTabCloseFromRenderer } from './terminal-tab-close-request-relay'
@@ -347,14 +348,20 @@ function registerRuntimeWindowLifecycle(
revealTerminalSession: (worktreeId, opts) =>
new Promise((resolve, reject) => {
const requestId = randomUUID()
const expectedIdentity = opts.expectedProcessIdentity
? opts.tabId && opts.leafId
? { worktreeId, tabId: opts.tabId, leafId: opts.leafId, ptyId: opts.ptyId }
: null
: undefined
if (expectedIdentity === null) {
reject(new Error('terminal_reveal_identity_required'))
return
}
const timer = setTimeout(() => {
ipcMain.removeListener('terminal:tabCreateReply', handler)
reject(new Error('Terminal reveal timed out'))
}, 10_000)
const handler = (
event: Electron.IpcMainEvent,
reply: { requestId: string; tabId?: string; title?: string; error?: string }
): void => {
const handler = (event: Electron.IpcMainEvent, reply: TerminalTabCreateReply): void => {
// Why: requestId is renderer-supplied, so only the targeted main window may satisfy the reveal.
if (event.sender !== mainWindow.webContents || reply.requestId !== requestId) {
return
@@ -365,7 +372,22 @@ function registerRuntimeWindowLifecycle(
reject(new Error(reply.error))
return
}
resolve({ tabId: reply.tabId!, title: reply.title })
if (
expectedIdentity &&
(!reply.identity ||
reply.identity.worktreeId !== expectedIdentity.worktreeId ||
reply.identity.tabId !== expectedIdentity.tabId ||
reply.identity.leafId !== expectedIdentity.leafId ||
reply.identity.ptyId !== expectedIdentity.ptyId)
) {
reject(new Error('terminal_reveal_identity_mismatch'))
return
}
resolve({
tabId: reply.tabId!,
title: reply.title,
...(reply.identity ? { identity: reply.identity } : {})
})
}
ipcMain.on('terminal:tabCreateReply', handler)
send('ui:createTerminal', {
+2 -6
View File
@@ -18,6 +18,7 @@ import type {
TerminalTabCloseRequest,
TerminalTabCloseResponse
} from '../shared/terminal-tab-close'
import type { TerminalTabCreateReply } from '../shared/terminal-reveal-identity'
import type {
LocalLogTailChangedPayload,
LocalLogTailReadArgs,
@@ -3149,12 +3150,7 @@ export type PreloadApi = {
onRequestTerminalTabMount: (
callback: (data: { worktreeId: string; tabId?: string; ptyId?: string }) => void
) => () => void
replyTerminalCreate: (reply: {
requestId: string
tabId?: string
title?: string
error?: string
}) => void
replyTerminalCreate: (reply: TerminalTabCreateReply) => void
onSplitTerminal: (
callback: (data: {
tabId: string
+2 -6
View File
@@ -13,6 +13,7 @@ import type { CliInstallStatus } from '../shared/cli-install-types'
import type { AgentHookInstallStatus } from '../shared/agent-hook-types'
import type { CodexConfigSyncStatus } from '../shared/codex-config-sync-types'
import type { TerminalPaneSplitSource } from '../shared/feature-education-telemetry'
import type { TerminalTabCreateReply } from '../shared/terminal-reveal-identity'
import type { ProjectExecutionRuntimeResolution } from '../shared/project-execution-runtime'
import type { StartupCommandDelivery } from '../shared/codex-startup-delivery'
import type {
@@ -3752,12 +3753,7 @@ const api = {
ipcRenderer.on('terminal:requestTabMount', listener)
return () => ipcRenderer.removeListener('terminal:requestTabMount', listener)
},
replyTerminalCreate: (reply: {
requestId: string
tabId?: string
title?: string
error?: string
}): void => {
replyTerminalCreate: (reply: TerminalTabCreateReply): void => {
ipcRenderer.send('terminal:tabCreateReply', reply)
},
onSplitTerminal: (
+85 -11
View File
@@ -198,6 +198,8 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise<void> {
let nextSeq = 1
let highestReceivedSeq = 0
const requestId = 1
const postOutputRequestId = 2
let initialExitCode = 0
const sendRequest = (): void => {
const env = pickRemoteCliEnv(process.env)
@@ -219,6 +221,59 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise<void> {
sock.write(frame)
}
const finish = (exitCode: number): void => {
sock.destroy()
process.exit(exitCode)
}
const sendPostOutput = (postOutput: unknown): void => {
sock.write(
encodeJsonRpcFrame(
{
jsonrpc: '2.0',
id: postOutputRequestId,
method: 'orca.cli.postOutput',
params: { postOutput, env: pickRemoteCliEnv(process.env) }
},
nextSeq++,
highestReceivedSeq
)
)
}
const writeOutput = (
result: { stdout?: unknown; stderr?: unknown },
onFlushed: (error?: Error) => void
): void => {
const writes = [
[process.stdout, result.stdout],
[process.stderr, result.stderr]
].filter((entry): entry is [NodeJS.WriteStream, string] => typeof entry[1] === 'string')
if (writes.length === 0) {
onFlushed()
return
}
let pending = writes.length
let completed = false
for (const [stream, output] of writes) {
stream.write(output, (error) => {
if (completed) {
return
}
if (error) {
completed = true
onFlushed(error)
return
}
pending -= 1
if (pending === 0) {
completed = true
onFlushed()
}
})
}
}
const decoder = new FrameDecoder((frame: DecodedFrame) => {
if (frame.id > highestReceivedSeq) {
highestReceivedSeq = frame.id
@@ -227,28 +282,41 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise<void> {
return
}
const msg = parseJsonRpcMessage(frame.payload)
if (!('id' in msg) || msg.id !== requestId || !('result' in msg || 'error' in msg)) {
if (
!('id' in msg) ||
(msg.id !== requestId && msg.id !== postOutputRequestId) ||
!('result' in msg || 'error' in msg)
) {
return
}
const response = msg as JsonRpcResponse
if (response.error) {
process.stderr.write(`${response.error.message}\n`)
sock.destroy()
process.exit(1)
finish(1)
return
}
if (response.id === postOutputRequestId) {
finish(initialExitCode)
return
}
const result = (response.result ?? {}) as {
stdout?: unknown
stderr?: unknown
exitCode?: unknown
postOutput?: unknown
}
if (typeof result.stdout === 'string' && result.stdout.length > 0) {
process.stdout.write(result.stdout)
}
if (typeof result.stderr === 'string' && result.stderr.length > 0) {
process.stderr.write(result.stderr)
}
sock.destroy()
process.exit(typeof result.exitCode === 'number' ? result.exitCode : 0)
initialExitCode = typeof result.exitCode === 'number' ? result.exitCode : 0
writeOutput(result, (error) => {
if (error) {
finish(1)
return
}
if (result.postOutput === undefined) {
finish(initialExitCode)
return
}
sendPostOutput(result.postOutput)
})
})
const connectTimeout = setTimeout(() => {
@@ -443,6 +511,12 @@ async function main(): Promise<void> {
timeoutMs: remoteCliRequestTimeoutMs(params)
})
})
dispatcher.onRequest('orca.cli.postOutput', async (params, context) => {
return await dispatcher.requestAnyClient('orca.cli.postOutput', params, {
excludeClientId: context.clientId,
timeoutMs: remoteCliRequestTimeoutMs(params)
})
})
function configureRelayGraceTime(params: Record<string, unknown>): { graceTimeMs: number } {
const seconds = Number(params.graceTimeSeconds)
+60 -10
View File
@@ -1504,7 +1504,6 @@ describe('useIpcEvents updater integration', () => {
openAgentTabsInChatByDefault: false
}
}
vi.doMock('react', async () => {
const actual = await vi.importActual<typeof ReactModule>('react')
return {
@@ -1789,7 +1788,11 @@ describe('useIpcEvents updater integration', () => {
})
it('surfaces terminal creates without stealing focus unless requested', async () => {
const createTab = vi.fn(() => ({ id: 'tab-new' }))
const createTab = vi.fn(
(_worktreeId: string, _groupId?: string, _tabType?: string, options?: { id?: string }) => ({
id: options?.id ?? 'tab-new'
})
)
const setActiveView = vi.fn()
const setActiveWorktree = vi.fn()
const markWorktreeVisited = vi.fn()
@@ -1876,6 +1879,20 @@ describe('useIpcEvents updater integration', () => {
activeRuntimeEnvironmentId: undefined as string | undefined
}
}
updateTabPtyId.mockImplementation((tabId: string, ptyId: string) => {
storeState.ptyIdsByTabId[tabId] = [
...new Set([...(storeState.ptyIdsByTabId[tabId] ?? []), ptyId])
]
for (const tabs of Object.values(storeState.tabsByWorktree)) {
const tab = tabs.find((candidate) => candidate.id === tabId)
if (tab) {
tab.ptyId = ptyId
}
}
})
setTabLayout.mockImplementation((tabId: string, layout: unknown) => {
storeState.terminalLayoutsByTabId[tabId] = layout
})
const createTerminalListenerRef: {
current:
| ((data: {
@@ -2280,6 +2297,7 @@ describe('useIpcEvents updater integration', () => {
setActiveTabType.mockClear()
setActiveTab.mockClear()
revealWorktreeInSidebar.mockClear()
dispatchEvent.mockClear()
createTerminalListenerRef.current({
worktreeId: 'wt-2',
title: 'Runner',
@@ -2588,6 +2606,7 @@ describe('useIpcEvents updater integration', () => {
setActiveTab.mockClear()
revealWorktreeInSidebar.mockClear()
replyTerminalCreate.mockClear()
dispatchEvent.mockClear()
requestTerminalCreateListenerRef.current({
requestId: 'req-renderer-backed-background',
worktreeId: 'wt-2',
@@ -2605,6 +2624,12 @@ describe('useIpcEvents updater integration', () => {
expect(setActiveTabType).not.toHaveBeenCalled()
expect(setActiveTab).not.toHaveBeenCalled()
expect(revealWorktreeInSidebar).not.toHaveBeenCalled()
expect(dispatchEvent).toHaveBeenCalledWith(
expect.objectContaining({
type: 'orca-background-mount-terminal-worktree',
detail: { worktreeId: 'wt-2', tabIds: ['tab-new'] }
})
)
expect(replyTerminalCreate).toHaveBeenCalledWith({
requestId: 'req-renderer-backed-background',
tabId: 'tab-new',
@@ -2708,6 +2733,7 @@ describe('useIpcEvents updater integration', () => {
setActiveTabType.mockClear()
setActiveTab.mockClear()
revealWorktreeInSidebar.mockClear()
dispatchEvent.mockClear()
createTerminalListenerRef.current({
worktreeId: 'wt-2',
ptyId: 'pty-bg-3',
@@ -2725,6 +2751,13 @@ describe('useIpcEvents updater integration', () => {
expect(setActiveTabType).not.toHaveBeenCalled()
expect(setActiveTab).not.toHaveBeenCalled()
expect(revealWorktreeInSidebar).not.toHaveBeenCalled()
expect(dispatchEvent).toHaveBeenCalledOnce()
expect(dispatchEvent).toHaveBeenCalledWith(
expect.objectContaining({
type: 'orca-background-mount-terminal-worktree',
detail: { worktreeId: 'wt-2', tabIds: ['tab-cli-bg-reveal'] }
})
)
createTab.mockClear()
setActiveView.mockClear()
@@ -2734,6 +2767,7 @@ describe('useIpcEvents updater integration', () => {
revealWorktreeInSidebar.mockClear()
focusRuntimeTerminalSurface.mockClear()
focusTerminalTabSurface.mockClear()
dispatchEvent.mockClear()
createTerminalListenerRef.current({
worktreeId: 'wt-2',
ptyId: 'pty-recovery-bg',
@@ -2754,6 +2788,13 @@ describe('useIpcEvents updater integration', () => {
expect(revealWorktreeInSidebar).not.toHaveBeenCalled()
expect(focusRuntimeTerminalSurface).not.toHaveBeenCalled()
expect(focusTerminalTabSurface).not.toHaveBeenCalled()
expect(dispatchEvent).toHaveBeenCalledOnce()
expect(dispatchEvent).toHaveBeenCalledWith(
expect.objectContaining({
type: 'orca-background-mount-terminal-worktree',
detail: { worktreeId: 'wt-2', tabIds: ['tab-recovery-bg'] }
})
)
storeState.tabsByWorktree = {
'wt-2': [{ id: 'tab-existing', ptyId: 'pty-bg', title: 'Terminal 1' }]
@@ -2836,7 +2877,13 @@ describe('useIpcEvents updater integration', () => {
expect(replyTerminalCreate).toHaveBeenCalledWith({
requestId: 'req-adopt-pending',
tabId: pendingTabId,
title: 'Terminal 3'
title: 'Terminal 3',
identity: {
worktreeId: 'wt-2',
tabId: pendingTabId,
leafId: pendingLeafId,
ptyId: 'serve-cf39bedb-a33a-417c-9ab6-f304dc27a6c0'
}
})
storeState.tabsByWorktree = {
@@ -2902,7 +2949,13 @@ describe('useIpcEvents updater integration', () => {
expect(replyTerminalCreate).toHaveBeenCalledWith({
requestId: 'req-split',
tabId: 'tab-existing',
title: 'Terminal 1'
title: 'Terminal 1',
identity: {
worktreeId: 'wt-2',
tabId: 'tab-existing',
leafId: 'leaf-split',
ptyId: 'pty-split'
}
})
storeState.terminalLayoutsByTabId = {
@@ -5234,7 +5287,7 @@ describe('useIpcEvents agent status snapshot integration', () => {
vi.unstubAllGlobals()
})
it('reconciles adopted and exited legacy worker recovery without broad record cleanup', async () => {
it('retires the exact sleeping record after adopted or exited legacy worker recovery', async () => {
const clearSleepingAgentSession = vi.fn()
const setSleepingAgentAutomaticResumeBlocked = vi.fn()
let listener:
@@ -5275,11 +5328,8 @@ describe('useIpcEvents agent status snapshot integration', () => {
clearSleepingAgentSession.mockClear()
listener?.({ paneKey: 'tab-exited:leaf-exited', resolution: 'exited' })
expect(clearSleepingAgentSession).not.toHaveBeenCalled()
expect(setSleepingAgentAutomaticResumeBlocked).toHaveBeenCalledWith(
'tab-exited:leaf-exited',
false
)
expect(clearSleepingAgentSession).toHaveBeenCalledWith('tab-exited:leaf-exited')
expect(setSleepingAgentAutomaticResumeBlocked).not.toHaveBeenCalled()
})
it.each([
+17 -8
View File
@@ -101,6 +101,7 @@ import { track } from '@/lib/telemetry'
import { singlePaneLayoutSnapshot } from '@/store/slices/terminal-helpers'
import { buildWorkspaceSessionPayload } from '@/lib/workspace-session'
import { persistWorkspaceSessionByHost } from '@/lib/workspace-session-host-persistence'
import { verifyTerminalRevealIdentity } from '@/lib/terminal-reveal-identity'
import { getLinearIssueWorkspaceName } from '../../../shared/workspace-name'
import type { RuntimeClientEvent } from '../../../shared/runtime-client-events'
import { applyHostWorktreeTerminalSleepState } from '@/components/terminal-pane/pty-shutdown-exit-deferral'
@@ -1576,11 +1577,24 @@ export function useIpcEvents(): void {
...(launchAgent ? { launchAgent } : {})
})
}
if (ptyId && terminalPresentation === 'background') {
requestBackgroundTerminalWorktreeMount({ worktreeId, tabIds: [tab.id] })
}
if (requestId) {
const identity =
ptyId && tabId && leafId
? verifyTerminalRevealIdentity(useAppStore.getState(), {
worktreeId,
tabId,
leafId,
ptyId
})
: undefined
window.api.ui.replyTerminalCreate({
requestId,
tabId: tab.id,
title: title ?? tab.title
title: title ?? tab.title,
...(identity ? { identity } : {})
})
}
} catch (err) {
@@ -3348,13 +3362,8 @@ export function useIpcEvents(): void {
unsubs.push(unsubscribeMigrationUnsupportedClear)
}
const unsubscribeLegacyWorkerTerminalRecovery =
window.api.agentStatus.onLegacyWorkerTerminalRecovery?.(({ paneKey, resolution }) => {
const store = useAppStore.getState()
if (resolution === 'adopted') {
store.clearSleepingAgentSession(paneKey)
return
}
store.setSleepingAgentAutomaticResumeBlocked(paneKey, false)
window.api.agentStatus.onLegacyWorkerTerminalRecovery?.(({ paneKey }) => {
useAppStore.getState().clearSleepingAgentSession(paneKey)
})
if (unsubscribeLegacyWorkerTerminalRecovery) {
unsubs.push(unsubscribeLegacyWorkerTerminalRecovery)
@@ -11,7 +11,7 @@ afterEach(() => {
})
describe('legacy worker sleeping-session recovery', () => {
it('keeps the session dormant until exited recovery removes only its resume block', () => {
it('never resumes a proven-exited legacy worker on workspace activation', () => {
const record: SleepingAgentSessionRecord = {
paneKey: 'tab-legacy:leaf-legacy',
tabId: 'tab-legacy',
@@ -46,16 +46,8 @@ describe('legacy worker sleeping-session recovery', () => {
expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(0)
expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey]).toBe(record)
useAppStore.getState().setSleepingAgentAutomaticResumeBlocked(record.paneKey, false)
const unblocked = useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey]
expect(unblocked).toMatchObject({
paneKey: record.paneKey,
providerSession: record.providerSession,
origin: 'live'
})
expect(unblocked?.automaticResumeBlockedBy).toBeUndefined()
expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(1)
useAppStore.getState().clearSleepingAgentSession(record.paneKey)
expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(0)
expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey]).toBeUndefined()
})
})
@@ -0,0 +1,65 @@
import { describe, expect, it } from 'vitest'
import { verifyTerminalRevealIdentity } from './terminal-reveal-identity'
const expected = {
worktreeId: 'worktree-1',
tabId: 'tab-1',
leafId: 'leaf-1',
ptyId: 'pty-1'
}
const mismatchCases: {
name: string
state: Parameters<typeof verifyTerminalRevealIdentity>[0]
}[] = [
{
name: 'worktree',
state: {
tabsByWorktree: { 'worktree-2': [{ id: 'tab-1' }] },
terminalLayoutsByTabId: {
'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-1' } }
}
}
},
{
name: 'leaf',
state: {
tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] },
terminalLayoutsByTabId: {
'tab-1': { ptyIdsByLeafId: { 'leaf-2': 'pty-1' } }
}
}
},
{
name: 'pty',
state: {
tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] },
terminalLayoutsByTabId: {
'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-2' } }
}
}
}
]
describe('verifyTerminalRevealIdentity', () => {
it('returns the exact fresh renderer binding', () => {
expect(
verifyTerminalRevealIdentity(
{
tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] },
terminalLayoutsByTabId: {
'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-1' } }
}
},
expected
)
).toEqual(expected)
})
it.each(mismatchCases)('rejects a mismatched $name owner', ({ state }) => {
expect(() => verifyTerminalRevealIdentity(state, expected)).toThrow(
'terminal_reveal_identity_mismatch'
)
})
})
@@ -0,0 +1,23 @@
import type { TerminalRevealIdentity } from '../../../shared/terminal-reveal-identity'
type TerminalRevealState = {
tabsByWorktree: Record<string, readonly { id: string }[]>
terminalLayoutsByTabId: Record<
string,
{ ptyIdsByLeafId?: Record<string, string | undefined> } | undefined
>
}
export function verifyTerminalRevealIdentity(
state: TerminalRevealState,
expected: TerminalRevealIdentity
): TerminalRevealIdentity {
const ownsTab = state.tabsByWorktree[expected.worktreeId]?.some(
(tab) => tab.id === expected.tabId
)
const boundPtyId = state.terminalLayoutsByTabId[expected.tabId]?.ptyIdsByLeafId?.[expected.leafId]
if (!ownsTab || boundPtyId !== expected.ptyId) {
throw new Error('terminal_reveal_identity_mismatch')
}
return expected
}
@@ -0,0 +1,42 @@
import { describe, expect, it } from 'vitest'
import { prepareOrchestrationCheckOutput } from './orchestration-check-output'
describe('prepareOrchestrationCheckOutput', () => {
it('keeps mixed read-only mail safe and current Run replies executable', () => {
const prepared = prepareOrchestrationCheckOutput(
{
count: 2,
messages: [
{
id: 'msg_current',
run_id: 'run_adopted',
delivery_contract: 'current_delivery',
from_handle: 'term_worker',
to_handle: 'run:run_adopted',
subject: 'Question'
},
{
id: 'msg_legacy',
run_id: 'run_legacy_local',
delivery_contract: 'audit_only',
from_handle: 'term_legacy',
to_handle: 'term_coord',
subject: 'Old reply'
}
],
formatted: '[Reply: unsafe stale formatter output]'
},
'term_current_coord',
true
)
expect(prepared.formatted).toContain(
'[Reply: orca orchestration reply --id msg_current --body "..."]'
)
expect(prepared.formatted).not.toContain('--from run:run_adopted')
expect(prepared.formatted).toContain(
'[Inspection only: reply and acknowledgment are unavailable.]'
)
expect(prepared.formatted).not.toContain('unsafe stale formatter output')
})
})
+199
View File
@@ -0,0 +1,199 @@
import { ORCHESTRATION_LEGACY_RUN_ID } from './orchestration-rpc-contract'
export type OrchestrationMessageSummary = {
id: string
run_id?: string
delivery_contract?: 'legacy_direct' | 'current_delivery' | 'audit_only'
from_handle: string
to_handle?: string
subject?: string
type?: string
body?: string
payload?: string | null
priority?: string
read?: number
}
export type LegacyCompatibilityResult = {
recovery?: boolean
readOnly?: boolean
ackMessageIds?: string[]
answerAcknowledgement?: {
questionId: string
answerMessageId: string
}
currentDelivery?: {
runId: string
checkCommand: string
ackCommand: string
}
resumeRequired?: boolean
resumeCommand?: string
}
export type OrchestrationCheckOutput = {
messages: OrchestrationMessageSummary[]
count: number
formatted?: string
deliveryId?: string | null
timedOut?: boolean
cancelled?: boolean
connectionLost?: boolean
legacyCompatibility?: LegacyCompatibilityResult
}
export function formatMessageReadOnlyTag(
message: OrchestrationMessageSummary,
legacyCompatibilityActive = false
): string {
return isLegacyReadOnlyMessage(message, legacyCompatibilityActive) ? ' [legacy, read-only]' : ''
}
export function isLegacyReadOnlyMessage(
message: OrchestrationMessageSummary,
legacyCompatibilityActive = false
): boolean {
return (
message.run_id === ORCHESTRATION_LEGACY_RUN_ID ||
(message.delivery_contract === 'legacy_direct' && !legacyCompatibilityActive) ||
message.delivery_contract === 'audit_only'
)
}
export function formatOrchestrationCheckText(
result: OrchestrationCheckOutput,
checkedTerminal: string
): string {
const prepared = prepareOrchestrationCheckOutput(
result,
checkedTerminal,
result.formatted !== undefined
)
const compatibilityActive = Boolean(
prepared.legacyCompatibility && !prepared.legacyCompatibility.readOnly
)
const legacyHeader = prepared.legacyCompatibility
? prepared.legacyCompatibility.readOnly
? '[LEGACY READ-ONLY]\n'
: prepared.legacyCompatibility.recovery
? '[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]\n'
: '[LEGACY COMPATIBILITY]\n'
: ''
const deliveryNotice = formatCurrentDeliveryNotice(prepared.legacyCompatibility?.currentDelivery)
if (prepared.formatted) {
return `${legacyHeader}${prepared.formatted}${deliveryNotice}`
}
if (prepared.count === 0) {
if (prepared.timedOut) {
return `${legacyHeader}Wait timed out; no messages were consumed.${deliveryNotice}`
}
if (prepared.cancelled) {
const cancelled = prepared.connectionLost
? 'Wait cancelled because the connection closed; no messages were consumed.'
: 'Wait cancelled; no messages were consumed.'
return `${legacyHeader}${cancelled}${deliveryNotice}`
}
return `${legacyHeader}No messages.${deliveryNotice}`
}
const rendered = prepared.messages
.map(
(message) =>
`${message.id}${formatMessageReadOnlyTag(
message,
compatibilityActive
)} [${message.type ?? 'status'}] from=${message.from_handle} "${message.subject}"`
)
.join('\n')
const output = prepared.deliveryId ? `Delivery ${prepared.deliveryId}\n${rendered}` : rendered
return `${legacyHeader}${output}${deliveryNotice}`
}
export function prepareOrchestrationCheckOutput<T extends OrchestrationCheckOutput>(
result: T,
checkedTerminal: string,
formattedRequested: boolean
): T {
const compatibilityActive = Boolean(
result.legacyCompatibility && !result.legacyCompatibility.readOnly
)
if (
!formattedRequested ||
!result.messages.some((message) => isLegacyReadOnlyMessage(message, compatibilityActive))
) {
return result
}
return {
...result,
formatted: formatLegacyAwareCheckMessages(result.messages, checkedTerminal, compatibilityActive)
}
}
function formatMessagePriorityTag(message: OrchestrationMessageSummary): string {
return message.priority === 'urgent' ? ' [URGENT]' : message.priority === 'high' ? ' [HIGH]' : ''
}
function escapeTerminalControlCharacters(value: string): string {
return [...value]
.map((character) => {
const code = character.charCodeAt(0)
if (character === '\n' || (code >= 0x20 && code < 0x7f) || code > 0x9f) {
return character
}
return `\\x${code.toString(16).padStart(2, '0')}`
})
.join('')
}
function formatQuotedMessageField(label: string, value?: string): string {
return `[${label}]\n${escapeTerminalControlCharacters(value ?? '')
.split('\n')
.map((line) => ` ${line}`)
.join('\n')}`
}
function formatLegacyAwareCheckMessages(
messages: OrchestrationMessageSummary[],
checkedTerminal: string,
legacyCompatibilityActive = false
): string {
return messages
.map((message) => {
const legacyReadOnly = isLegacyReadOnlyMessage(message, legacyCompatibilityActive)
const lines = [
`${message.id}${formatMessageReadOnlyTag(message, legacyCompatibilityActive)}${formatMessagePriorityTag(message)} [${message.type ?? 'status'}] from=${message.from_handle}`,
formatQuotedMessageField('subject', message.subject)
]
if (legacyReadOnly) {
lines.push('[Inspection only: reply and acknowledgment are unavailable.]')
}
if (message.body) {
lines.push(formatQuotedMessageField('body', message.body))
}
if (message.payload) {
lines.push(formatQuotedMessageField('payload', message.payload))
}
if (!legacyReadOnly) {
const replyTarget = message.to_handle ?? checkedTerminal
const replyFrom =
replyTarget.startsWith('run:') || replyTarget.startsWith('dispatch:')
? ''
: ` --from ${replyTarget}`
lines.push(`[Reply: orca orchestration reply --id ${message.id}${replyFrom} --body "..."]`)
}
return lines.join('\n')
})
.join('\n\n')
}
function formatCurrentDeliveryNotice(
delivery: LegacyCompatibilityResult['currentDelivery']
): string {
if (!delivery) {
return ''
}
return (
`\n[CURRENT RUN MAIL WAITING]\n` +
`Read: ${delivery.checkCommand}\n` +
`Then acknowledge the delivery ID it prints: ${delivery.ackCommand}`
)
}
+14
View File
@@ -0,0 +1,14 @@
export type TerminalRevealIdentity = {
worktreeId: string
tabId: string
leafId: string
ptyId: string
}
export type TerminalTabCreateReply = {
requestId: string
tabId?: string
title?: string
identity?: TerminalRevealIdentity
error?: string
}