mirror of
https://github.com/stablyai/orca.git
synced 2026-10-20 00:03:45 +00:00
fix(orchestration): complete legacy worker cutover recovery
This commit is contained in:
@@ -71,8 +71,15 @@ describe('orchestration skill guidance', () => {
|
||||
expect(migration).toContain(
|
||||
'It must not spawn, write, signal, stop, switch, focus, split, or inject a terminal.'
|
||||
)
|
||||
expect(migration).toContain('task-list --run run_legacy_local')
|
||||
expect(migration).not.toContain('task-list --run run_legacy_local')
|
||||
expect(migration).toContain('run_legacy_local is an empty audit tombstone')
|
||||
expect(migration).toContain('Recovered orchestration work from a contract update')
|
||||
expect(migration).toContain('run-show --id <adopted_run_id>')
|
||||
expect(migration).toContain('task-list --run <adopted_run_id>')
|
||||
expect(migration).toContain('Legacy inspection remains available without consuming mail')
|
||||
expect(migration).toContain('run-use --id <adopted_run_id> --takeover-legacy')
|
||||
expect(migration).toContain('Takeover fences only the old coordinator')
|
||||
expect(migration).toContain('Live legacy workers keep their original Tasks, Dispatches')
|
||||
expect(migration).toContain(
|
||||
'keep the original worker as the only editor until it reaches a stable handoff point'
|
||||
)
|
||||
|
||||
@@ -74,14 +74,27 @@ Legacy inspection remains available without consuming mail:
|
||||
|
||||
```bash
|
||||
orca orchestration run-list --json
|
||||
# run_legacy_local is an empty audit tombstone after adoption.
|
||||
orca orchestration run-show --id run_legacy_local --json
|
||||
orca orchestration task-list --run run_legacy_local --json
|
||||
# In run-list, find the ordinary Run whose objective is:
|
||||
# "Recovered orchestration work from a contract update"
|
||||
orca orchestration run-show --id <adopted_run_id> --json
|
||||
orca orchestration task-list --run <adopted_run_id> --json
|
||||
orca orchestration inbox --full --json
|
||||
orca orchestration check --terminal <legacy_handle> --peek --format --json
|
||||
orca terminal read --terminal <legacy_handle> --json
|
||||
orca terminal wait --terminal <legacy_handle> --for tui-idle --timeout-ms 60000 --json
|
||||
```
|
||||
|
||||
If the original coordinator is unavailable or cannot prove its retained authority, a current coordinator may explicitly take over the adopted Run:
|
||||
|
||||
```bash
|
||||
orca orchestration run-use --id <adopted_run_id> --takeover-legacy --json
|
||||
orca orchestration check --run <adopted_run_id> --json
|
||||
```
|
||||
|
||||
Takeover fences only the old coordinator, binds the current one, and moves pending worker mail into current Run Delivery. Live legacy workers keep their original Tasks, Dispatches, processes, filesystems, and old prompt commands; their later questions, escalations, and completion reports route to the current coordinator. Do not use takeover while the original coordinator is still actively coordinating, because its later lifecycle mutations are rejected.
|
||||
|
||||
Do not launch a replacement editor merely because the desktop app or runtime was updated. If adoption cannot prove continuing authority, keep the original worker as the only editor until it reaches a stable handoff point, then use a new current Dispatch in a conflict-free placement for any remaining work.
|
||||
|
||||
## Ownership
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -71,4 +71,24 @@ describe('orchestration check identity', () => {
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('preserves the pinned legacy --inject check signature', async () => {
|
||||
process.env.ORCA_TERMINAL_HANDLE = 'term_legacy_worker'
|
||||
|
||||
await invokeCheck(
|
||||
new Map<string, string | boolean>([
|
||||
['unread', true],
|
||||
['inject', true]
|
||||
])
|
||||
)
|
||||
|
||||
expect(callMock).toHaveBeenCalledWith(
|
||||
'orchestration.check',
|
||||
expect.objectContaining({
|
||||
terminal: 'term_legacy_worker',
|
||||
unread: true,
|
||||
inject: true
|
||||
})
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -179,6 +179,7 @@ describe('legacy orchestration CLI inspection', () => {
|
||||
id: 'msg_current',
|
||||
run_id: 'run_current',
|
||||
from_handle: 'term_current',
|
||||
to_handle: 'run:run_current',
|
||||
subject: 'current question',
|
||||
type: 'question',
|
||||
body: 'May I continue?',
|
||||
@@ -217,7 +218,8 @@ describe('legacy orchestration CLI inspection', () => {
|
||||
expect(response.result.formatted).not.toContain('RUNTIME_SENTINEL')
|
||||
expect(
|
||||
response.result.formatted.split('\n').filter((line) => line.startsWith('[Reply:'))
|
||||
).toEqual(['[Reply: orca orchestration reply --id msg_current --from term_coord --body "..."]'])
|
||||
).toEqual(['[Reply: orca orchestration reply --id msg_current --body "..."]'])
|
||||
expect(response.result.formatted).not.toContain('--from run:run_current')
|
||||
})
|
||||
|
||||
it('preserves runtime formatting when every message belongs to a current Run', async () => {
|
||||
|
||||
@@ -62,6 +62,28 @@ describe('lightweight Run CLI handlers', () => {
|
||||
from: 'term_coord'
|
||||
})
|
||||
})
|
||||
|
||||
it('passes explicit legacy takeover only when requested', async () => {
|
||||
callMock.mockResolvedValue({
|
||||
result: { run: { id: 'run_adopted', objective: 'Recovered work' } }
|
||||
})
|
||||
await ORCHESTRATION_HANDLERS['orchestration run-use']({
|
||||
flags: new Map<string, string | boolean>([
|
||||
['id', 'run_adopted'],
|
||||
['from', 'term_current'],
|
||||
['takeover-legacy', true]
|
||||
]),
|
||||
client: { call: callMock },
|
||||
cwd: '/tmp/repo',
|
||||
json: true
|
||||
} as never)
|
||||
|
||||
expect(callMock).toHaveBeenCalledWith('orchestration.runUse', {
|
||||
id: 'run_adopted',
|
||||
from: 'term_current',
|
||||
takeoverLegacy: true
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('orchestration reset CLI handler', () => {
|
||||
|
||||
@@ -21,10 +21,14 @@ import type {
|
||||
} from '../../shared/orchestration-worker-output'
|
||||
import type { NativeChatMessage } from '../../shared/native-chat-types'
|
||||
import type { RuntimeTerminalRead } from '../../shared/runtime-types'
|
||||
import { orchestrationMigrationData } from '../../shared/orchestration-rpc-contract'
|
||||
import {
|
||||
ORCHESTRATION_LEGACY_RUN_ID,
|
||||
orchestrationMigrationData
|
||||
} from '../../shared/orchestration-rpc-contract'
|
||||
formatMessageReadOnlyTag,
|
||||
formatOrchestrationCheckText,
|
||||
prepareOrchestrationCheckOutput,
|
||||
type LegacyCompatibilityResult,
|
||||
type OrchestrationMessageSummary as MessageSummary
|
||||
} from '../../shared/orchestration-check-output'
|
||||
|
||||
// Why: 15 s is well under Claude Code's ~2 min Bash-tool silence budget while keeping log volume low. See design doc §3.4.
|
||||
const DEFAULT_KEEPALIVE_INTERVAL_MS = 15_000
|
||||
@@ -74,93 +78,6 @@ const TASK_STATUS_VALUES = [
|
||||
'blocked'
|
||||
] as const
|
||||
|
||||
type MessageSummary = {
|
||||
id: string
|
||||
run_id?: string
|
||||
delivery_contract?: 'legacy_direct' | 'current_delivery' | 'audit_only'
|
||||
from_handle: string
|
||||
to_handle?: string
|
||||
subject?: string
|
||||
type?: string
|
||||
body?: string
|
||||
payload?: string | null
|
||||
priority?: string
|
||||
read?: number
|
||||
}
|
||||
|
||||
function formatMessageReadOnlyTag(
|
||||
message: MessageSummary,
|
||||
legacyCompatibilityActive = false
|
||||
): string {
|
||||
return isLegacyReadOnlyMessage(message, legacyCompatibilityActive) ? ' [legacy, read-only]' : ''
|
||||
}
|
||||
|
||||
function isLegacyReadOnlyMessage(
|
||||
message: MessageSummary,
|
||||
legacyCompatibilityActive = false
|
||||
): boolean {
|
||||
return (
|
||||
message.run_id === ORCHESTRATION_LEGACY_RUN_ID ||
|
||||
(message.delivery_contract === 'legacy_direct' && !legacyCompatibilityActive) ||
|
||||
message.delivery_contract === 'audit_only'
|
||||
)
|
||||
}
|
||||
|
||||
function formatMessagePriorityTag(message: MessageSummary): string {
|
||||
return message.priority === 'urgent' ? ' [URGENT]' : message.priority === 'high' ? ' [HIGH]' : ''
|
||||
}
|
||||
|
||||
function escapeTerminalControlCharacters(value: string): string {
|
||||
return [...value]
|
||||
.map((character) => {
|
||||
const code = character.charCodeAt(0)
|
||||
if (character === '\n' || (code >= 0x20 && code < 0x7f) || code > 0x9f) {
|
||||
return character
|
||||
}
|
||||
return `\\x${code.toString(16).padStart(2, '0')}`
|
||||
})
|
||||
.join('')
|
||||
}
|
||||
|
||||
function formatQuotedMessageField(label: string, value?: string): string {
|
||||
return `[${label}]\n${escapeTerminalControlCharacters(value ?? '')
|
||||
.split('\n')
|
||||
.map((line) => ` ${line}`)
|
||||
.join('\n')}`
|
||||
}
|
||||
|
||||
function formatLegacyAwareCheckMessages(
|
||||
messages: MessageSummary[],
|
||||
checkedTerminal: string,
|
||||
legacyCompatibilityActive = false
|
||||
): string {
|
||||
return messages
|
||||
.map((message) => {
|
||||
const legacyReadOnly = isLegacyReadOnlyMessage(message, legacyCompatibilityActive)
|
||||
const lines = [
|
||||
`${message.id}${formatMessageReadOnlyTag(message, legacyCompatibilityActive)}${formatMessagePriorityTag(message)} [${message.type ?? 'status'}] from=${message.from_handle}`,
|
||||
formatQuotedMessageField('subject', message.subject)
|
||||
]
|
||||
if (legacyReadOnly) {
|
||||
lines.push('[Inspection only: reply and acknowledgment are unavailable.]')
|
||||
}
|
||||
if (message.body) {
|
||||
lines.push(formatQuotedMessageField('body', message.body))
|
||||
}
|
||||
if (message.payload) {
|
||||
lines.push(formatQuotedMessageField('payload', message.payload))
|
||||
}
|
||||
if (!legacyReadOnly) {
|
||||
const replyFrom = message.to_handle ?? checkedTerminal
|
||||
lines.push(
|
||||
`[Reply: orca orchestration reply --id ${message.id} --from ${replyFrom} --body "..."]`
|
||||
)
|
||||
}
|
||||
return lines.join('\n')
|
||||
})
|
||||
.join('\n\n')
|
||||
}
|
||||
|
||||
type LifecycleSendRejection = {
|
||||
action: 'rejected'
|
||||
code: string
|
||||
@@ -181,23 +98,6 @@ type OrchestrationSendResult =
|
||||
lifecycle?: { action: 'completed' | 'failed' }
|
||||
}
|
||||
|
||||
type LegacyCompatibilityResult = {
|
||||
recovery?: boolean
|
||||
readOnly?: boolean
|
||||
ackMessageIds?: string[]
|
||||
answerAcknowledgement?: {
|
||||
questionId: string
|
||||
answerMessageId: string
|
||||
}
|
||||
currentDelivery?: {
|
||||
runId: string
|
||||
checkCommand: string
|
||||
ackCommand: string
|
||||
}
|
||||
resumeRequired?: boolean
|
||||
resumeCommand?: string
|
||||
}
|
||||
|
||||
function resolveCompatibilityCliCommand(): 'orca' | 'orca-ide' | 'orca-dev' {
|
||||
const configured = process.env.ORCA_CLI_COMMAND
|
||||
if (configured === 'orca' || configured === 'orca-ide' || configured === 'orca-dev') {
|
||||
@@ -206,19 +106,6 @@ function resolveCompatibilityCliCommand(): 'orca' | 'orca-ide' | 'orca-dev' {
|
||||
return process.platform === 'linux' ? 'orca-ide' : 'orca'
|
||||
}
|
||||
|
||||
function formatCurrentDeliveryNotice(
|
||||
delivery: LegacyCompatibilityResult['currentDelivery']
|
||||
): string {
|
||||
if (!delivery) {
|
||||
return ''
|
||||
}
|
||||
return (
|
||||
`\n[CURRENT RUN MAIL WAITING]\n` +
|
||||
`Read: ${delivery.checkCommand}\n` +
|
||||
`Then acknowledge the delivery ID it prints: ${delivery.ackCommand}`
|
||||
)
|
||||
}
|
||||
|
||||
function resolvePackagedWindowsCompatibilityCommand(): 'orca' | 'orca-ide' | undefined {
|
||||
if (process.env.ORCA_WINDOWS_PACKAGED_CLI_LAUNCHER !== '1') {
|
||||
return undefined
|
||||
@@ -560,7 +447,8 @@ export const ORCHESTRATION_HANDLERS: Record<string, CommandHandler> = {
|
||||
run: { id: string; objective: string; consumer_generation: number }
|
||||
}>(client, flags, 'orchestration.runUse', {
|
||||
id: getRequiredStringFlag(flags, 'id'),
|
||||
from
|
||||
from,
|
||||
...(flags.has('takeover-legacy') ? { takeoverLegacy: true } : {})
|
||||
})
|
||||
printResult(result, json, (r) => `Using Run ${r.run.id}: ${r.run.objective}`)
|
||||
},
|
||||
@@ -715,6 +603,7 @@ export const ORCHESTRATION_HANDLERS: Record<string, CommandHandler> = {
|
||||
all: flags.has('all') ? true : undefined,
|
||||
types: getOptionalStringFlag(flags, 'types'),
|
||||
format: flags.has('format') ? true : undefined,
|
||||
inject: flags.has('inject') ? true : undefined,
|
||||
compatibilityCliCommand: resolveCompatibilityCliCommand(),
|
||||
run: getOptionalStringFlag(flags, 'run'),
|
||||
ack: getOptionalStringFlag(flags, 'ack'),
|
||||
@@ -752,64 +641,11 @@ export const ORCHESTRATION_HANDLERS: Record<string, CommandHandler> = {
|
||||
}
|
||||
}
|
||||
}
|
||||
const compatibilityActive = Boolean(
|
||||
result.result.legacyCompatibility && !result.result.legacyCompatibility.readOnly
|
||||
)
|
||||
if (
|
||||
flags.has('format') &&
|
||||
result.result.messages.some((message) =>
|
||||
isLegacyReadOnlyMessage(message, compatibilityActive)
|
||||
)
|
||||
) {
|
||||
// Why: formatted is one opaque batch with untrusted bodies, so selective banner parsing cannot safely remove legacy actions.
|
||||
result = {
|
||||
...result,
|
||||
result: {
|
||||
...result.result,
|
||||
formatted: formatLegacyAwareCheckMessages(
|
||||
result.result.messages,
|
||||
terminal,
|
||||
compatibilityActive
|
||||
)
|
||||
}
|
||||
}
|
||||
result = {
|
||||
...result,
|
||||
result: prepareOrchestrationCheckOutput(result.result, terminal, flags.has('format'))
|
||||
}
|
||||
printResult(result, json, (r) => {
|
||||
const legacyHeader = r.legacyCompatibility
|
||||
? r.legacyCompatibility.readOnly
|
||||
? '[LEGACY READ-ONLY]\n'
|
||||
: r.legacyCompatibility.recovery
|
||||
? '[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]\n'
|
||||
: '[LEGACY COMPATIBILITY]\n'
|
||||
: ''
|
||||
const deliveryNotice = formatCurrentDeliveryNotice(r.legacyCompatibility?.currentDelivery)
|
||||
if (r.formatted) {
|
||||
return `${legacyHeader}${r.formatted}${deliveryNotice}`
|
||||
}
|
||||
if (r.count === 0) {
|
||||
if (r.timedOut) {
|
||||
return `${legacyHeader}Wait timed out; no messages were consumed.${deliveryNotice}`
|
||||
}
|
||||
if (r.cancelled) {
|
||||
const cancelled = r.connectionLost
|
||||
? 'Wait cancelled because the connection closed; no messages were consumed.'
|
||||
: 'Wait cancelled; no messages were consumed.'
|
||||
return `${legacyHeader}${cancelled}${deliveryNotice}`
|
||||
}
|
||||
return `${legacyHeader}No messages.${deliveryNotice}`
|
||||
}
|
||||
const rendered = r.messages
|
||||
.map(
|
||||
(m) =>
|
||||
`${m.id}${formatMessageReadOnlyTag(
|
||||
m,
|
||||
Boolean(r.legacyCompatibility && !r.legacyCompatibility.readOnly)
|
||||
)} [${m.type ?? 'status'}] from=${m.from_handle} "${m.subject}"`
|
||||
)
|
||||
.join('\n')
|
||||
const output = r.deliveryId ? `Delivery ${r.deliveryId}\n${rendered}` : rendered
|
||||
return `${legacyHeader}${output}${deliveryNotice}`
|
||||
})
|
||||
printResult(result, json, (r) => formatOrchestrationCheckText(r, terminal))
|
||||
const compatibilityAck = result.result.legacyCompatibility?.ackMessageIds
|
||||
if (compatibilityAck && compatibilityAck.length > 0) {
|
||||
await flushStdout()
|
||||
|
||||
@@ -16,8 +16,12 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
{
|
||||
path: ['orchestration', 'run-use'],
|
||||
summary: 'Bind this coordinator terminal to an existing Run',
|
||||
usage: 'orca orchestration run-use --id <run_id> [--from <handle>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'id', 'from', 'retry-request']
|
||||
usage:
|
||||
'orca orchestration run-use --id <run_id> [--from <handle>] [--takeover-legacy] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'id', 'from', 'takeover-legacy', 'retry-request'],
|
||||
notes: [
|
||||
'--takeover-legacy explicitly replaces an unavailable legacy coordinator while preserving live worker assignments.'
|
||||
]
|
||||
},
|
||||
{
|
||||
path: ['orchestration', 'run-current'],
|
||||
|
||||
@@ -2220,8 +2220,6 @@ void app.whenReady().then(async () => {
|
||||
getAgentProviderSessionSnapshot: () => agentHookServer.getStatusSnapshot(),
|
||||
getAgentProviderSessionRowsForPane: (paneKey) =>
|
||||
agentHookServer.getStatusSnapshotForPane(paneKey),
|
||||
getHydratedAgentHookAuthority: () => agentHookServer.getHydratedAuthorityCommitments(),
|
||||
getCurrentAgentHookAuthority: () => agentHookServer.getCurrentAuthorityObservations(),
|
||||
attestAgentHookCompatibilityAuthority: (candidate) =>
|
||||
agentHookServer.attestCompatibilityAuthority(candidate),
|
||||
canRecoverPersistentLocalPtys: () => getDaemonProvider() !== null,
|
||||
|
||||
@@ -5374,6 +5374,37 @@ describe('registerPtyHandlers', () => {
|
||||
expect(listProcesses).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('scopes runtime inventories to the requested provider', async () => {
|
||||
const localList = vi
|
||||
.spyOn(getLocalPtyProvider(), 'listProcesses')
|
||||
.mockResolvedValue([{ id: 'local-pty', title: 'Local', cwd: '/local' }])
|
||||
const sshAList = vi.fn(async () => [{ id: 'ssh-a-pty' }])
|
||||
const sshBList = vi.fn(async () => {
|
||||
throw new Error('ssh-b unavailable')
|
||||
})
|
||||
registerSshPtyProvider('ssh-a', { listProcesses: sshAList } as never)
|
||||
registerSshPtyProvider('ssh-b', { listProcesses: sshBList } as never)
|
||||
const runtime = { setPtyController: vi.fn() }
|
||||
handlers.clear()
|
||||
registerPtyHandlers(mainWindow as never, runtime as never)
|
||||
const controller = runtime.setPtyController.mock.calls[0]?.[0] as {
|
||||
listProcesses(connectionId?: string | null): Promise<{ id: string }[]>
|
||||
}
|
||||
|
||||
await expect(controller.listProcesses(null)).resolves.toEqual([
|
||||
{ id: 'local-pty', title: 'Local', cwd: '/local' }
|
||||
])
|
||||
expect(localList).toHaveBeenCalledOnce()
|
||||
expect(sshAList).not.toHaveBeenCalled()
|
||||
expect(sshBList).not.toHaveBeenCalled()
|
||||
|
||||
await expect(controller.listProcesses('ssh-a')).resolves.toEqual([{ id: 'ssh-a-pty' }])
|
||||
expect(sshAList).toHaveBeenCalledOnce()
|
||||
expect(sshBList).not.toHaveBeenCalled()
|
||||
|
||||
await expect(controller.listProcesses()).rejects.toThrow('ssh-b unavailable')
|
||||
})
|
||||
|
||||
it('returns unavailable runtime confirmation for unsupported or missing providers', async () => {
|
||||
registerSshPtyProvider('ssh-1', {} as never)
|
||||
setPtyOwnership('unsupported-pty', 'ssh-1')
|
||||
|
||||
+7
-1
@@ -4253,7 +4253,13 @@ export function registerPtyHandlers(
|
||||
return null
|
||||
}
|
||||
},
|
||||
listProcesses: async () => {
|
||||
listProcesses: async (connectionId) => {
|
||||
if (connectionId === null) {
|
||||
return localProvider.listProcesses()
|
||||
}
|
||||
if (connectionId !== undefined) {
|
||||
return getProvider(connectionId).listProcesses()
|
||||
}
|
||||
const providerSessions = await Promise.all([
|
||||
localProvider.listProcesses(),
|
||||
...Array.from(sshProviders.values(), (provider) => provider.listProcesses())
|
||||
|
||||
@@ -16337,6 +16337,17 @@ describe('OrcaRuntimeService', () => {
|
||||
['pty-shell', 'inc-shell', 'term_shell', 'Shell']
|
||||
] as const
|
||||
const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow: vi.fn() } as never)
|
||||
const listProcesses = vi.fn(async () =>
|
||||
processes.map(([id, incarnationId, terminalHandle, title]) => ({
|
||||
id,
|
||||
incarnationId,
|
||||
terminalHandle,
|
||||
title,
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
wslDistro: null
|
||||
}))
|
||||
)
|
||||
runtime.setPtyController({
|
||||
write: (ptyId, data) => {
|
||||
writes.push([ptyId, data])
|
||||
@@ -16345,16 +16356,7 @@ describe('OrcaRuntimeService', () => {
|
||||
resize,
|
||||
kill: () => true,
|
||||
getForegroundProcess: async () => null,
|
||||
listProcesses: async () =>
|
||||
processes.map(([id, incarnationId, terminalHandle, title]) => ({
|
||||
id,
|
||||
incarnationId,
|
||||
terminalHandle,
|
||||
title,
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
wslDistro: null
|
||||
}))
|
||||
listProcesses
|
||||
})
|
||||
const before = await runtime.listTerminals(`id:${TEST_WORKTREE_ID}`)
|
||||
expect(before.terminals.map((terminal) => terminal.tabId)).toEqual(
|
||||
@@ -16442,6 +16444,14 @@ describe('OrcaRuntimeService', () => {
|
||||
tail: ['legacy output']
|
||||
})
|
||||
|
||||
const inventoryCount = listProcesses.mock.calls.length
|
||||
const agentPty = (
|
||||
runtime as unknown as {
|
||||
ptysById: Map<string, { tabId: string | null; paneKey: string | null }>
|
||||
}
|
||||
).ptysById.get('pty-agent')!
|
||||
agentPty.tabId = null
|
||||
agentPty.paneKey = null
|
||||
const secondClient = await runtime.adoptTerminalOrphans({
|
||||
worktree: `id:${TEST_WORKTREE_ID}`,
|
||||
expectedTopologyRevision: 0,
|
||||
@@ -16454,6 +16464,22 @@ describe('OrcaRuntimeService', () => {
|
||||
}))
|
||||
})
|
||||
expect(secondClient).toMatchObject({ adopted: false, topologyRevision: 1 })
|
||||
expect(agentPty).toMatchObject({
|
||||
tabId: 'tab-agent',
|
||||
paneKey: makePaneKey('tab-agent', HEADLESS_LEAF_ID)
|
||||
})
|
||||
expect(listProcesses).toHaveBeenCalledTimes(inventoryCount + 1)
|
||||
expect(listProcesses).toHaveBeenLastCalledWith(null)
|
||||
expect(
|
||||
(await runtime.listTerminals()).terminals.find((terminal) => terminal.ptyId === 'pty-agent')
|
||||
).toMatchObject({
|
||||
handle: 'term_agent',
|
||||
orphaned: false,
|
||||
tabId: 'tab-agent',
|
||||
leafId: HEADLESS_LEAF_ID
|
||||
})
|
||||
expect(listProcesses).toHaveBeenCalledTimes(inventoryCount + 2)
|
||||
expect(listProcesses).toHaveBeenLastCalledWith(undefined)
|
||||
await expect(
|
||||
runtime.adoptTerminalOrphans({
|
||||
worktree: `id:${TEST_WORKTREE_ID}`,
|
||||
@@ -16560,6 +16586,7 @@ describe('OrcaRuntimeService', () => {
|
||||
getForegroundProcess: async () => null,
|
||||
hasRendererSerializer: () => true,
|
||||
serializeBuffer,
|
||||
hasPty: (ptyId) => ptyId === 'pty-legacy',
|
||||
listProcesses: async () => [
|
||||
{
|
||||
id: 'pty-legacy',
|
||||
@@ -16606,7 +16633,11 @@ describe('OrcaRuntimeService', () => {
|
||||
presentation: 'background',
|
||||
tabId: 'legacy-worker',
|
||||
leafId: workerLeafId,
|
||||
focus: false
|
||||
focus: false,
|
||||
expectedProcessIdentity: {
|
||||
terminalHandle: 'term_legacy',
|
||||
incarnationId
|
||||
}
|
||||
})
|
||||
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted')
|
||||
expect(write).not.toHaveBeenCalled()
|
||||
@@ -16683,6 +16714,7 @@ describe('OrcaRuntimeService', () => {
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
hasPty: (ptyId) => ptyId === 'pty-reveal-retry',
|
||||
listProcesses: async () => [
|
||||
{
|
||||
id: 'pty-reveal-retry',
|
||||
@@ -16716,6 +16748,269 @@ describe('OrcaRuntimeService', () => {
|
||||
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted')
|
||||
})
|
||||
|
||||
function makePostRevealWorkerRecoveryHarness(hasPty: (ptyId: string) => boolean | null): {
|
||||
runtime: OrcaRuntimeService
|
||||
getSession: () => WorkspaceSessionState
|
||||
workerPaneKey: string
|
||||
ptyId: string
|
||||
incarnationId: string
|
||||
terminalHandle: string
|
||||
revealTerminalSession: ReturnType<typeof vi.fn>
|
||||
resolveLegacyWorkerTerminalRecovery: ReturnType<typeof vi.fn>
|
||||
} {
|
||||
const workerPaneKey = `legacy-post-reveal:${HEADLESS_LEAF_ID}`
|
||||
const ptyId = 'pty-post-reveal'
|
||||
const incarnationId = '45454545-4545-4545-8545-454545454545'
|
||||
const terminalHandle = 'term_post_reveal'
|
||||
const session: WorkspaceSessionState = {
|
||||
...getDefaultWorkspaceSession(),
|
||||
tabsByWorktree: { [TEST_WORKTREE_ID]: [] },
|
||||
sleepingAgentSessionsByPaneKey: {
|
||||
[workerPaneKey]: {
|
||||
paneKey: workerPaneKey,
|
||||
tabId: 'legacy-post-reveal',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
agent: 'codex',
|
||||
providerSession: { key: 'session_id', id: 'legacy-post-reveal-session' },
|
||||
prompt: 'continue',
|
||||
state: 'working',
|
||||
capturedAt: 1,
|
||||
updatedAt: 1,
|
||||
origin: 'live'
|
||||
}
|
||||
}
|
||||
}
|
||||
const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session)
|
||||
const runtime = new OrcaRuntimeService(
|
||||
{ ...runtimeStore, flushOrThrow: vi.fn() } as never,
|
||||
undefined,
|
||||
{ canRecoverPersistentLocalPtys: () => true }
|
||||
)
|
||||
runtime.setOrchestrationDb({
|
||||
listLegacyWorkerTerminalRecoveryRows: () => [
|
||||
{
|
||||
dispatch_id: 'dispatch-post-reveal',
|
||||
task_id: 'task-post-reveal',
|
||||
dispatch_status: 'completed',
|
||||
contract_version: 0,
|
||||
assignee_handle: 'term_post_reveal',
|
||||
assignee_pane_key: workerPaneKey,
|
||||
process_incarnation: `${ptyId}:${incarnationId}`,
|
||||
worker_state: 'ready',
|
||||
worktree_id: TEST_WORKTREE_ID,
|
||||
agent_terminal_handle: 'term_post_reveal'
|
||||
}
|
||||
]
|
||||
} as unknown as OrchestrationDb)
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
hasPty,
|
||||
listProcesses: async () => [
|
||||
{
|
||||
id: ptyId,
|
||||
incarnationId,
|
||||
terminalHandle: 'term_post_reveal',
|
||||
title: 'Post-reveal worker',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
wslDistro: null
|
||||
}
|
||||
]
|
||||
})
|
||||
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-post-reveal' })
|
||||
const resolveLegacyWorkerTerminalRecovery = vi.fn()
|
||||
runtime.setNotifier({
|
||||
revealTerminalSession,
|
||||
resolveLegacyWorkerTerminalRecovery
|
||||
} as never)
|
||||
return {
|
||||
runtime,
|
||||
getSession,
|
||||
workerPaneKey,
|
||||
ptyId,
|
||||
incarnationId,
|
||||
terminalHandle,
|
||||
revealTerminalSession,
|
||||
resolveLegacyWorkerTerminalRecovery
|
||||
}
|
||||
}
|
||||
|
||||
it('keeps a revealed worker fenced until its exact renderer graph is published', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const harness = makePostRevealWorkerRecoveryHarness(() => true)
|
||||
const identity = {
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
tabId: 'legacy-post-reveal',
|
||||
leafId: HEADLESS_LEAF_ID,
|
||||
ptyId: harness.ptyId
|
||||
}
|
||||
harness.revealTerminalSession.mockResolvedValue({
|
||||
tabId: identity.tabId,
|
||||
identity
|
||||
})
|
||||
|
||||
await expect(
|
||||
harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true })
|
||||
).resolves.toMatchObject({
|
||||
adoptedDispatchIds: [],
|
||||
exitedDispatchIds: [],
|
||||
deferredDispatchIds: ['dispatch-post-reveal']
|
||||
})
|
||||
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
|
||||
expect(
|
||||
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
|
||||
).toBeDefined()
|
||||
expect(harness.resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled()
|
||||
|
||||
harness.runtime.attachWindow(1)
|
||||
harness.runtime.syncWindowGraph(1, {
|
||||
tabs: [
|
||||
{
|
||||
tabId: identity.tabId,
|
||||
worktreeId: identity.worktreeId,
|
||||
title: 'Post-reveal worker',
|
||||
activeLeafId: identity.leafId,
|
||||
layout: null
|
||||
}
|
||||
],
|
||||
leaves: [
|
||||
{
|
||||
tabId: identity.tabId,
|
||||
worktreeId: identity.worktreeId,
|
||||
leafId: identity.leafId,
|
||||
paneRuntimeId: 1,
|
||||
ptyId: identity.ptyId
|
||||
}
|
||||
]
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
|
||||
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
|
||||
expect(
|
||||
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
|
||||
).toBeUndefined()
|
||||
expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(
|
||||
harness.workerPaneKey,
|
||||
'adopted'
|
||||
)
|
||||
expect(
|
||||
(await harness.runtime.listTerminals()).terminals.filter(
|
||||
(terminal) => terminal.ptyId === identity.ptyId
|
||||
)
|
||||
).toEqual([
|
||||
expect.objectContaining({
|
||||
handle: harness.terminalHandle,
|
||||
incarnationId: harness.incarnationId,
|
||||
orphaned: false,
|
||||
worktreeId: identity.worktreeId,
|
||||
tabId: identity.tabId,
|
||||
leafId: identity.leafId
|
||||
})
|
||||
])
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('retires the exact worker when it exits after renderer reveal', async () => {
|
||||
const hasPty = vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(false)
|
||||
const harness = makePostRevealWorkerRecoveryHarness(hasPty)
|
||||
|
||||
await expect(
|
||||
harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true })
|
||||
).resolves.toMatchObject({
|
||||
adoptedDispatchIds: [],
|
||||
exitedDispatchIds: ['dispatch-post-reveal'],
|
||||
deferredDispatchIds: []
|
||||
})
|
||||
|
||||
expect(hasPty).toHaveBeenCalledTimes(2)
|
||||
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
|
||||
expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([])
|
||||
expect(
|
||||
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
|
||||
).toBeUndefined()
|
||||
expect(
|
||||
(
|
||||
harness.runtime as unknown as {
|
||||
ptysById: Map<string, { connected: boolean; incarnationId?: string }>
|
||||
}
|
||||
).ptysById.get(harness.ptyId)
|
||||
).toMatchObject({ connected: false, incarnationId: '45454545-4545-4545-8545-454545454545' })
|
||||
expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(
|
||||
harness.workerPaneKey,
|
||||
'exited'
|
||||
)
|
||||
})
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: 'null',
|
||||
makeHasPty: () =>
|
||||
vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(null).mockReturnValue(true)
|
||||
},
|
||||
{
|
||||
name: 'throw',
|
||||
makeHasPty: () =>
|
||||
vi
|
||||
.fn()
|
||||
.mockReturnValueOnce(true)
|
||||
.mockImplementationOnce(() => {
|
||||
throw new Error('provider unavailable')
|
||||
})
|
||||
.mockReturnValue(true)
|
||||
}
|
||||
])(
|
||||
'retries post-reveal $name liveness without duplicating the worker surface',
|
||||
async ({ makeHasPty }) => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const hasPty = makeHasPty()
|
||||
const harness = makePostRevealWorkerRecoveryHarness(hasPty)
|
||||
|
||||
await expect(
|
||||
harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true })
|
||||
).resolves.toMatchObject({
|
||||
adoptedDispatchIds: [],
|
||||
exitedDispatchIds: [],
|
||||
deferredDispatchIds: ['dispatch-post-reveal']
|
||||
})
|
||||
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
|
||||
expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([
|
||||
expect.objectContaining({ id: 'legacy-post-reveal', ptyId: harness.ptyId })
|
||||
])
|
||||
expect(
|
||||
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
|
||||
?.automaticResumeBlockedBy
|
||||
).toBe('legacy-orchestration-worker')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
|
||||
expect(hasPty).toHaveBeenCalledTimes(4)
|
||||
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
|
||||
expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([
|
||||
expect.objectContaining({ id: 'legacy-post-reveal', ptyId: harness.ptyId })
|
||||
])
|
||||
expect(
|
||||
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
|
||||
).toBeUndefined()
|
||||
expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(
|
||||
harness.workerPaneKey,
|
||||
'adopted'
|
||||
)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
expect(hasPty).toHaveBeenCalledTimes(4)
|
||||
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
it('keeps the legacy worker resume fence in memory when persistence fails', async () => {
|
||||
const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}`
|
||||
const incarnationId = '99999999-9999-4999-8999-999999999999'
|
||||
@@ -16768,6 +17063,7 @@ describe('OrcaRuntimeService', () => {
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
hasPty: (ptyId) => ptyId === 'pty-persistence-failure',
|
||||
listProcesses: async () => [
|
||||
{
|
||||
id: 'pty-persistence-failure',
|
||||
@@ -16804,8 +17100,9 @@ describe('OrcaRuntimeService', () => {
|
||||
warn.mockRestore()
|
||||
})
|
||||
|
||||
it('unblocks provider resume only after authoritative inventory proves the legacy PTY exited', async () => {
|
||||
it('retires provider resume only after authoritative inventory proves the legacy PTY exited', async () => {
|
||||
const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}`
|
||||
const secondWorkerPaneKey = `legacy-worker-two:${HEADLESS_SECOND_LEAF_ID}`
|
||||
const incarnationId = '33333333-3333-4333-8333-333333333333'
|
||||
const session: WorkspaceSessionState = {
|
||||
...getDefaultWorkspaceSession(),
|
||||
@@ -16822,6 +17119,18 @@ describe('OrcaRuntimeService', () => {
|
||||
capturedAt: 1,
|
||||
updatedAt: 1,
|
||||
origin: 'live'
|
||||
},
|
||||
[secondWorkerPaneKey]: {
|
||||
paneKey: secondWorkerPaneKey,
|
||||
tabId: 'legacy-worker-two',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
agent: 'codex',
|
||||
providerSession: { key: 'session_id', id: 'legacy-codex-session-two' },
|
||||
prompt: 'continue',
|
||||
state: 'working',
|
||||
capturedAt: 1,
|
||||
updatedAt: 1,
|
||||
origin: 'live'
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16844,17 +17153,50 @@ describe('OrcaRuntimeService', () => {
|
||||
worker_state: 'ready',
|
||||
worktree_id: TEST_WORKTREE_ID,
|
||||
agent_terminal_handle: 'term_exited'
|
||||
},
|
||||
{
|
||||
dispatch_id: 'dispatch-exited-two',
|
||||
task_id: 'task-exited-two',
|
||||
dispatch_status: 'completed',
|
||||
contract_version: 0,
|
||||
assignee_handle: 'term_exited_two',
|
||||
assignee_pane_key: secondWorkerPaneKey,
|
||||
process_incarnation: `pty-exited-two:${incarnationId}`,
|
||||
worker_state: 'ready',
|
||||
worktree_id: TEST_WORKTREE_ID,
|
||||
agent_terminal_handle: 'term_exited_two'
|
||||
}
|
||||
]
|
||||
} as unknown as OrchestrationDb)
|
||||
const listProcesses = vi.fn(async (connectionId?: string | null) => {
|
||||
if (connectionId !== null) {
|
||||
throw new Error('unrelated SSH inventory must not run')
|
||||
}
|
||||
return [
|
||||
{
|
||||
id: 'pty-exited-two',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_exited_two',
|
||||
title: 'Exited worker',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
wslDistro: null
|
||||
}
|
||||
]
|
||||
})
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
listProcesses: async () => []
|
||||
hasPty: () => false,
|
||||
listProcesses
|
||||
})
|
||||
const revealTerminalSession = vi.fn()
|
||||
const resolveLegacyWorkerTerminalRecovery = vi.fn()
|
||||
runtime.setNotifier({ resolveLegacyWorkerTerminalRecovery } as never)
|
||||
runtime.setNotifier({
|
||||
revealTerminalSession,
|
||||
resolveLegacyWorkerTerminalRecovery
|
||||
} as never)
|
||||
|
||||
runtime.prepareLegacyWorkerTerminalRecovery()
|
||||
expect(
|
||||
@@ -16863,16 +17205,27 @@ describe('OrcaRuntimeService', () => {
|
||||
|
||||
await expect(runtime.reconcileLegacyWorkerTerminals()).resolves.toMatchObject({
|
||||
adoptedDispatchIds: [],
|
||||
exitedDispatchIds: ['dispatch-exited'],
|
||||
exitedDispatchIds: ['dispatch-exited', 'dispatch-exited-two'],
|
||||
deferredDispatchIds: []
|
||||
})
|
||||
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
|
||||
expect(getSession().sleepingAgentSessionsByPaneKey?.[secondWorkerPaneKey]).toBeUndefined()
|
||||
expect(listProcesses).toHaveBeenCalledOnce()
|
||||
expect(listProcesses).toHaveBeenCalledWith(null)
|
||||
expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([])
|
||||
expect(revealTerminalSession).not.toHaveBeenCalled()
|
||||
expect(
|
||||
getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy
|
||||
).toBeUndefined()
|
||||
(
|
||||
runtime as unknown as {
|
||||
ptysById: Map<string, { connected: boolean }>
|
||||
}
|
||||
).ptysById.get('pty-exited-two')
|
||||
).toMatchObject({ connected: false })
|
||||
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'exited')
|
||||
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(secondWorkerPaneKey, 'exited')
|
||||
})
|
||||
|
||||
it('keeps provider resume fenced when authoritative PTY inventory is unavailable', async () => {
|
||||
it('retries inventory and unknown liveness without revealing a ghost worker', async () => {
|
||||
const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}`
|
||||
const incarnationId = '55555555-5555-4555-8555-555555555555'
|
||||
const session: WorkspaceSessionState = {
|
||||
@@ -16915,23 +17268,263 @@ describe('OrcaRuntimeService', () => {
|
||||
}
|
||||
]
|
||||
} as unknown as OrchestrationDb)
|
||||
const listProcesses = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('local provider unavailable'))
|
||||
.mockResolvedValue([
|
||||
{
|
||||
id: 'pty-inventory-unavailable',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_inventory_unavailable',
|
||||
title: 'Recovered legacy worker',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
wslDistro: null
|
||||
}
|
||||
])
|
||||
const hasPty = vi.fn().mockReturnValueOnce(null).mockReturnValue(true)
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null
|
||||
getForegroundProcess: async () => null,
|
||||
hasPty,
|
||||
listProcesses
|
||||
})
|
||||
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-worker' })
|
||||
const resolveLegacyWorkerTerminalRecovery = vi.fn()
|
||||
runtime.setNotifier({ resolveLegacyWorkerTerminalRecovery } as never)
|
||||
runtime.setNotifier({
|
||||
revealTerminalSession,
|
||||
resolveLegacyWorkerTerminalRecovery
|
||||
} as never)
|
||||
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
await expect(
|
||||
runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true })
|
||||
).resolves.toMatchObject({
|
||||
adoptedDispatchIds: [],
|
||||
exitedDispatchIds: [],
|
||||
deferredDispatchIds: ['dispatch-inventory-unavailable']
|
||||
})
|
||||
expect(
|
||||
getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy
|
||||
).toBe('legacy-orchestration-worker')
|
||||
expect(resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled()
|
||||
expect(listProcesses).toHaveBeenCalledOnce()
|
||||
expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([])
|
||||
expect(revealTerminalSession).not.toHaveBeenCalled()
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
|
||||
expect(listProcesses).toHaveBeenCalledTimes(2)
|
||||
expect(listProcesses.mock.calls).toEqual([[null], [null]])
|
||||
expect(hasPty).toHaveBeenCalledOnce()
|
||||
expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([])
|
||||
expect(revealTerminalSession).not.toHaveBeenCalled()
|
||||
expect(
|
||||
getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy
|
||||
).toBe('legacy-orchestration-worker')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(2_000)
|
||||
|
||||
expect(listProcesses).toHaveBeenCalledTimes(3)
|
||||
expect(hasPty).toHaveBeenCalledTimes(3)
|
||||
expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([
|
||||
expect.objectContaining({ id: 'legacy-worker', ptyId: 'pty-inventory-unavailable' })
|
||||
])
|
||||
expect(revealTerminalSession).toHaveBeenCalledOnce()
|
||||
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
|
||||
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted')
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
expect(listProcesses).toHaveBeenCalledTimes(3)
|
||||
expect(revealTerminalSession).toHaveBeenCalledOnce()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('cancels a coalesced SSH worker recovery retry when its provider disconnects', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const runtime = new OrcaRuntimeService(store)
|
||||
const reconcile = vi.spyOn(runtime, 'reconcileLegacyWorkerTerminals').mockResolvedValue({
|
||||
blockedPaneCount: 1,
|
||||
adoptedDispatchIds: [],
|
||||
exitedDispatchIds: [],
|
||||
deferredDispatchIds: []
|
||||
})
|
||||
const retryInternals = runtime as unknown as {
|
||||
updateLegacyWorkerTerminalRecoveryRetry: (
|
||||
plan: {
|
||||
candidates: { dispatchId: string; ptyId: string }[]
|
||||
},
|
||||
deferredDispatchIds: ReadonlySet<string>,
|
||||
options: { connectionId?: string; materializeRenderer?: boolean }
|
||||
) => void
|
||||
}
|
||||
const plan = {
|
||||
candidates: [
|
||||
{
|
||||
dispatchId: 'dispatch-ssh-retry',
|
||||
ptyId: 'ssh:ssh-retry@@pty-worker'
|
||||
}
|
||||
]
|
||||
}
|
||||
const deferred = new Set(['dispatch-ssh-retry'])
|
||||
|
||||
retryInternals.updateLegacyWorkerTerminalRecoveryRetry(plan, deferred, {
|
||||
connectionId: 'ssh-retry',
|
||||
materializeRenderer: true
|
||||
})
|
||||
retryInternals.updateLegacyWorkerTerminalRecoveryRetry(plan, deferred, {
|
||||
connectionId: 'ssh-retry',
|
||||
materializeRenderer: true
|
||||
})
|
||||
runtime.notifySshStateChanged('ssh-retry', {
|
||||
targetId: 'ssh-retry',
|
||||
status: 'disconnected',
|
||||
error: null,
|
||||
reconnectAttempt: 0
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
|
||||
expect(reconcile).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps live workers fenced without exact controller identity evidence', async () => {
|
||||
const incarnationId = '56565656-5656-4656-8656-565656565656'
|
||||
const cases = [
|
||||
{
|
||||
name: 'missing',
|
||||
leafId: '11111111-1111-4111-8111-111111111111',
|
||||
terminalHandle: 'term_missing'
|
||||
},
|
||||
{
|
||||
name: 'ambiguous',
|
||||
leafId: '22222222-2222-4222-8222-222222222222',
|
||||
terminalHandle: 'term_ambiguous'
|
||||
},
|
||||
{
|
||||
name: 'wrong-handle',
|
||||
leafId: '33333333-3333-4333-8333-333333333333',
|
||||
terminalHandle: 'term_wrong_handle'
|
||||
},
|
||||
{
|
||||
name: 'wrong-incarnation',
|
||||
leafId: '44444444-4444-4444-8444-444444444444',
|
||||
terminalHandle: 'term_wrong_incarnation'
|
||||
}
|
||||
] as const
|
||||
const sleepingAgentSessionsByPaneKey = Object.fromEntries(
|
||||
cases.map(({ name, leafId }) => {
|
||||
const paneKey = `legacy-${name}:${leafId}`
|
||||
return [
|
||||
paneKey,
|
||||
{
|
||||
paneKey,
|
||||
tabId: `legacy-${name}`,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
agent: 'codex',
|
||||
providerSession: { key: 'session_id', id: `session-${name}` },
|
||||
prompt: 'continue',
|
||||
state: 'working',
|
||||
capturedAt: 1,
|
||||
updatedAt: 1,
|
||||
origin: 'live'
|
||||
}
|
||||
]
|
||||
})
|
||||
) as WorkspaceSessionState['sleepingAgentSessionsByPaneKey']
|
||||
const session: WorkspaceSessionState = {
|
||||
...getDefaultWorkspaceSession(),
|
||||
tabsByWorktree: { [TEST_WORKTREE_ID]: [] },
|
||||
sleepingAgentSessionsByPaneKey
|
||||
}
|
||||
const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session)
|
||||
const runtime = new OrcaRuntimeService(
|
||||
{ ...runtimeStore, flushOrThrow: vi.fn() } as never,
|
||||
undefined,
|
||||
{ canRecoverPersistentLocalPtys: () => true }
|
||||
)
|
||||
runtime.setOrchestrationDb({
|
||||
listLegacyWorkerTerminalRecoveryRows: () =>
|
||||
cases.map(({ name, leafId, terminalHandle }) => ({
|
||||
dispatch_id: `dispatch-${name}`,
|
||||
task_id: `task-${name}`,
|
||||
dispatch_status: 'completed',
|
||||
contract_version: 0,
|
||||
assignee_handle: terminalHandle,
|
||||
assignee_pane_key: `legacy-${name}:${leafId}`,
|
||||
process_incarnation: `pty-${name}:${incarnationId}`,
|
||||
worker_state: 'ready',
|
||||
worktree_id: TEST_WORKTREE_ID,
|
||||
agent_terminal_handle: terminalHandle
|
||||
}))
|
||||
} as unknown as OrchestrationDb)
|
||||
const listProcesses = vi.fn(async () => [
|
||||
{
|
||||
id: 'pty-missing',
|
||||
title: 'Missing identity',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID
|
||||
},
|
||||
{
|
||||
id: 'pty-ambiguous',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_ambiguous',
|
||||
title: 'Ambiguous identity',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID
|
||||
},
|
||||
{
|
||||
id: 'pty-ambiguous-other',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_ambiguous',
|
||||
title: 'Ambiguous identity duplicate',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID
|
||||
},
|
||||
{
|
||||
id: 'pty-wrong-handle',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_other',
|
||||
title: 'Wrong handle',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID
|
||||
},
|
||||
{
|
||||
id: 'pty-wrong-incarnation',
|
||||
incarnationId: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
|
||||
terminalHandle: 'term_wrong_incarnation',
|
||||
title: 'Wrong incarnation',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID
|
||||
}
|
||||
])
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
hasPty: (candidate) => candidate === 'pty-folder-legacy',
|
||||
listProcesses
|
||||
})
|
||||
|
||||
await expect(runtime.reconcileLegacyWorkerTerminals()).resolves.toMatchObject({
|
||||
adoptedDispatchIds: [],
|
||||
exitedDispatchIds: [],
|
||||
deferredDispatchIds: ['dispatch-inventory-unavailable']
|
||||
deferredDispatchIds: cases.map(({ name }) => `dispatch-${name}`)
|
||||
})
|
||||
expect(
|
||||
getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy
|
||||
).toBe('legacy-orchestration-worker')
|
||||
expect(resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled()
|
||||
expect(listProcesses).toHaveBeenCalledOnce()
|
||||
expect(listProcesses).toHaveBeenCalledWith(null)
|
||||
for (const { name, leafId } of cases) {
|
||||
expect(
|
||||
getSession().sleepingAgentSessionsByPaneKey?.[`legacy-${name}:${leafId}`]
|
||||
?.automaticResumeBlockedBy
|
||||
).toBe('legacy-orchestration-worker')
|
||||
}
|
||||
})
|
||||
|
||||
it('adopts an exact live legacy worker in a folder workspace', async () => {
|
||||
@@ -16986,21 +17579,23 @@ describe('OrcaRuntimeService', () => {
|
||||
}
|
||||
]
|
||||
} as unknown as OrchestrationDb)
|
||||
const listProcesses = vi.fn(async () => [
|
||||
{
|
||||
id: 'pty-folder-legacy',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_folder',
|
||||
title: 'Folder worker',
|
||||
cwd: folderPath,
|
||||
worktreeId: TEST_FOLDER_WORKSPACE_KEY,
|
||||
wslDistro: null
|
||||
}
|
||||
])
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
listProcesses: async () => [
|
||||
{
|
||||
id: 'pty-folder-legacy',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_folder',
|
||||
title: 'Folder worker',
|
||||
cwd: folderPath,
|
||||
worktreeId: TEST_FOLDER_WORKSPACE_KEY,
|
||||
wslDistro: null
|
||||
}
|
||||
]
|
||||
hasPty: (candidate) => candidate === 'pty-folder-legacy',
|
||||
listProcesses
|
||||
})
|
||||
const revealTerminalSession = vi.fn().mockResolvedValue({
|
||||
tabId: 'legacy-folder-worker'
|
||||
@@ -17022,6 +17617,8 @@ describe('OrcaRuntimeService', () => {
|
||||
})
|
||||
)
|
||||
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
|
||||
expect(listProcesses).toHaveBeenCalledOnce()
|
||||
expect(listProcesses).toHaveBeenCalledWith(null)
|
||||
expect(revealTerminalSession).toHaveBeenCalledWith(TEST_FOLDER_WORKSPACE_KEY, {
|
||||
ptyId: 'pty-folder-legacy',
|
||||
title: 'Folder worker',
|
||||
@@ -17029,7 +17626,11 @@ describe('OrcaRuntimeService', () => {
|
||||
presentation: 'background',
|
||||
tabId: 'legacy-folder-worker',
|
||||
leafId: HEADLESS_LEAF_ID,
|
||||
focus: false
|
||||
focus: false,
|
||||
expectedProcessIdentity: {
|
||||
terminalHandle: 'term_folder',
|
||||
incarnationId
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -17101,21 +17702,23 @@ describe('OrcaRuntimeService', () => {
|
||||
}
|
||||
]
|
||||
} as unknown as OrchestrationDb)
|
||||
const listProcesses = vi.fn(async () => [
|
||||
{
|
||||
id: ptyId,
|
||||
incarnationId,
|
||||
terminalHandle: 'term_ssh_folder',
|
||||
title: 'SSH folder worker',
|
||||
cwd: folderPath,
|
||||
worktreeId: TEST_FOLDER_WORKSPACE_KEY,
|
||||
wslDistro: null
|
||||
}
|
||||
])
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
listProcesses: async () => [
|
||||
{
|
||||
id: ptyId,
|
||||
incarnationId,
|
||||
terminalHandle: 'term_ssh_folder',
|
||||
title: 'SSH folder worker',
|
||||
cwd: folderPath,
|
||||
worktreeId: TEST_FOLDER_WORKSPACE_KEY,
|
||||
wslDistro: null
|
||||
}
|
||||
]
|
||||
hasPty: (candidate) => candidate === ptyId,
|
||||
listProcesses
|
||||
})
|
||||
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-ssh-folder-worker' })
|
||||
runtime.setNotifier({ revealTerminalSession } as never)
|
||||
@@ -17149,6 +17752,8 @@ describe('OrcaRuntimeService', () => {
|
||||
expect(localSession.sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
|
||||
expect(getWorkspaceSession).toHaveBeenCalledWith(`ssh:${connectionId}`)
|
||||
expect(setWorkspaceSession).toHaveBeenCalledWith(expect.any(Object), `ssh:${connectionId}`)
|
||||
expect(listProcesses).toHaveBeenCalledOnce()
|
||||
expect(listProcesses).toHaveBeenCalledWith(connectionId)
|
||||
expect(sshSession.tabsByWorktree[TEST_FOLDER_WORKSPACE_KEY]).toContainEqual(
|
||||
expect.objectContaining({
|
||||
id: 'legacy-ssh-folder-worker',
|
||||
@@ -17163,7 +17768,11 @@ describe('OrcaRuntimeService', () => {
|
||||
presentation: 'background',
|
||||
tabId: 'legacy-ssh-folder-worker',
|
||||
leafId: HEADLESS_LEAF_ID,
|
||||
focus: false
|
||||
focus: false,
|
||||
expectedProcessIdentity: {
|
||||
terminalHandle: 'term_ssh_folder',
|
||||
incarnationId
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -17321,6 +17930,7 @@ describe('OrcaRuntimeService', () => {
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
hasPty: (candidate) => candidate === ptyId,
|
||||
listProcesses
|
||||
})
|
||||
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-ssh-worker' })
|
||||
@@ -17365,12 +17975,29 @@ describe('OrcaRuntimeService', () => {
|
||||
exitedDispatchIds: [],
|
||||
deferredDispatchIds: []
|
||||
})
|
||||
await expect(
|
||||
runtime.adoptTerminalOrphans({
|
||||
worktree: `id:${TEST_WORKTREE_ID}`,
|
||||
expectedTopologyRevision: 0,
|
||||
claims: [
|
||||
{
|
||||
terminal: 'term_ssh_legacy',
|
||||
ptyId,
|
||||
incarnationId,
|
||||
tabId: 'legacy-ssh-worker',
|
||||
leafId: HEADLESS_LEAF_ID
|
||||
}
|
||||
]
|
||||
})
|
||||
).resolves.toMatchObject({ adopted: false })
|
||||
expect(listProcesses).toHaveBeenLastCalledWith(connectionId)
|
||||
} finally {
|
||||
unregisterSshGitProvider(connectionId)
|
||||
}
|
||||
|
||||
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
|
||||
expect(setWorkspaceSession).toHaveBeenCalledWith(expect.any(Object), `ssh:${connectionId}`)
|
||||
expect(listProcesses).toHaveBeenCalledTimes(2)
|
||||
expect(revealTerminalSession).toHaveBeenCalledWith(TEST_WORKTREE_ID, {
|
||||
ptyId,
|
||||
title: 'SSH legacy worker',
|
||||
@@ -17378,7 +18005,11 @@ describe('OrcaRuntimeService', () => {
|
||||
presentation: 'background',
|
||||
tabId: 'legacy-ssh-worker',
|
||||
leafId: HEADLESS_LEAF_ID,
|
||||
focus: false
|
||||
focus: false,
|
||||
expectedProcessIdentity: {
|
||||
terminalHandle: 'term_ssh_legacy',
|
||||
incarnationId
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -17445,21 +18076,23 @@ describe('OrcaRuntimeService', () => {
|
||||
}
|
||||
]
|
||||
} as unknown as OrchestrationDb)
|
||||
const listProcesses = vi.fn(async () => [
|
||||
{
|
||||
id: 'pty-wsl-legacy',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_wsl_legacy',
|
||||
title: 'WSL legacy worker',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
wslDistro: observedDistro
|
||||
}
|
||||
])
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
listProcesses: async () => [
|
||||
{
|
||||
id: 'pty-wsl-legacy',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_wsl_legacy',
|
||||
title: 'WSL legacy worker',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
wslDistro: observedDistro
|
||||
}
|
||||
]
|
||||
hasPty: (ptyId) => ptyId === 'pty-wsl-legacy',
|
||||
listProcesses
|
||||
})
|
||||
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-wsl-worker' })
|
||||
runtime.setNotifier({ revealTerminalSession } as never)
|
||||
@@ -17486,6 +18119,8 @@ describe('OrcaRuntimeService', () => {
|
||||
})
|
||||
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
|
||||
expect(revealTerminalSession).toHaveBeenCalledOnce()
|
||||
expect(listProcesses).toHaveBeenCalledTimes(2)
|
||||
expect(listProcesses.mock.calls).toEqual([[null], [null]])
|
||||
})
|
||||
|
||||
it('restores orphan pane and group topology without replacing a newer host-owned tab', async () => {
|
||||
|
||||
+456
-176
@@ -36,10 +36,7 @@ import {
|
||||
type AgentStatusEntry
|
||||
} from '../../shared/agent-status-types'
|
||||
import { indexAgentStatusRowsByPaneKey } from '../agent-hooks/agent-status-pane-index'
|
||||
import type {
|
||||
AgentHookAuthorityAttestation,
|
||||
AgentHookAuthorityEvidence
|
||||
} from '../agent-hooks/server'
|
||||
import type { AgentHookAuthorityAttestation } from '../agent-hooks/server'
|
||||
import type {
|
||||
AgentSessionClaimedSpawnResult,
|
||||
AgentSessionExecutionClaim,
|
||||
@@ -110,6 +107,7 @@ import {
|
||||
createSetupCompletionScanner
|
||||
} from './orchestration/setup-completion-signal'
|
||||
import type { RuntimeOrchestrationEnvelope } from '../../shared/runtime-rpc-envelope'
|
||||
import type { TerminalRevealIdentity } from '../../shared/terminal-reveal-identity'
|
||||
import type {
|
||||
OrchestrationCompatibilityEvidence,
|
||||
OrchestrationCompatibilityHostStamp
|
||||
@@ -1506,7 +1504,7 @@ type RuntimePtyController = {
|
||||
resize?(ptyId: string, cols: number, rows: number): boolean
|
||||
// Why: exact-id mobile polls should not enumerate every local and SSH PTY.
|
||||
hasPty?(ptyId: string): boolean | null
|
||||
listProcesses?(): Promise<PtyProcessInfo[]>
|
||||
listProcesses?(connectionId?: string | null): Promise<PtyProcessInfo[]>
|
||||
serializeBuffer?(
|
||||
ptyId: string,
|
||||
opts?: { scrollbackRows?: number; altScreenForcesZeroRows?: boolean }
|
||||
@@ -1530,6 +1528,17 @@ type RuntimePtyController = {
|
||||
getSize?(ptyId: string): { cols: number; rows: number } | null
|
||||
}
|
||||
|
||||
type PtyControllerTerminalIdentity = Readonly<{
|
||||
handle: string
|
||||
incarnationId: string
|
||||
wslDistro?: string | null
|
||||
}>
|
||||
|
||||
type PtyControllerInventory = Readonly<{
|
||||
livePtyIds: ReadonlySet<string>
|
||||
terminalIdentityByPtyId: ReadonlyMap<string, PtyControllerTerminalIdentity>
|
||||
}>
|
||||
|
||||
type WorktreeStartupDraftPaste = {
|
||||
agent: TuiAgent
|
||||
content: string
|
||||
@@ -1645,10 +1654,14 @@ type RuntimeNotifier = {
|
||||
splitDirection?: 'horizontal' | 'vertical'
|
||||
splitTelemetrySource?: TerminalPaneSplitSource
|
||||
focus?: boolean
|
||||
expectedProcessIdentity?: {
|
||||
terminalHandle: string
|
||||
incarnationId: string
|
||||
}
|
||||
}
|
||||
):
|
||||
| Promise<{ tabId: string; title?: string | null }>
|
||||
| { tabId: string; title?: string | null }
|
||||
| Promise<{ tabId: string; title?: string | null; identity?: TerminalRevealIdentity }>
|
||||
| { tabId: string; title?: string | null; identity?: TerminalRevealIdentity }
|
||||
| void
|
||||
resolveLegacyWorkerTerminalRecovery?(paneKey: string, resolution: 'adopted' | 'exited'): void
|
||||
splitTerminal(
|
||||
@@ -1733,11 +1746,6 @@ export type OrchestrationCompatibilityTerminalAuthority = {
|
||||
| { kind: 'ssh'; targetId: string }
|
||||
}
|
||||
|
||||
export type OrchestrationCompatibilityAuthorityInputs = {
|
||||
hydratedCommitments: readonly AgentHookAuthorityEvidence[]
|
||||
currentObservations: readonly AgentHookAuthorityEvidence[]
|
||||
}
|
||||
|
||||
export type LegacyWorkerTerminalRecoveryResult = {
|
||||
blockedPaneCount: number
|
||||
adoptedDispatchIds: string[]
|
||||
@@ -2615,10 +2623,6 @@ export class OrcaRuntimeService {
|
||||
private handles = new Map<string, TerminalHandleRecord>()
|
||||
private handleByLeafKey = new Map<string, string>()
|
||||
private handleByPtyId = new Map<string, string>()
|
||||
private controllerTerminalIdentityByPtyId = new Map<
|
||||
string,
|
||||
{ handle: string; incarnationId: string; wslDistro?: string | null }
|
||||
>()
|
||||
private detachedPreAllocatedLeaves = new Map<string, RuntimeLeafRecord>()
|
||||
private graphSyncCallbacks: (() => void)[] = []
|
||||
private waitersByHandle = new Map<string, Set<TerminalWaiter>>()
|
||||
@@ -3011,12 +3015,6 @@ export class OrcaRuntimeService {
|
||||
private readonly getAgentProviderSessionRowsForPaneFn:
|
||||
| ((paneKey: string) => AgentStatusIpcPayload[])
|
||||
| null
|
||||
private readonly getHydratedAgentHookAuthorityFn:
|
||||
| (() => readonly AgentHookAuthorityEvidence[])
|
||||
| null
|
||||
private readonly getCurrentAgentHookAuthorityFn:
|
||||
| (() => readonly AgentHookAuthorityEvidence[])
|
||||
| null
|
||||
private readonly attestAgentHookCompatibilityAuthorityFn:
|
||||
| ((candidate: {
|
||||
paneKey: string
|
||||
@@ -3038,7 +3036,17 @@ export class OrcaRuntimeService {
|
||||
>()
|
||||
private sshRelayRecoveryGenerationByTargetId = new Map<string, number>()
|
||||
private legacyWorkerTerminalRecoveryQueue: Promise<void> = Promise.resolve()
|
||||
private legacyWorkerTerminalRecoveryRetries = new Map<
|
||||
string,
|
||||
{
|
||||
attempt: number
|
||||
connectionId?: string
|
||||
materializeRenderer: boolean
|
||||
timer: ReturnType<typeof setTimeout> | null
|
||||
}
|
||||
>()
|
||||
private legacyWorkerTerminalMaterializedPanes = new Set<string>()
|
||||
private legacyWorkerTerminalIdentityReceiptPanes = new Set<string>()
|
||||
private legacyWorkerRecoveredPtys = new Set<string>()
|
||||
private accountServices: RuntimeAccountServices | null = null
|
||||
private commitMessageAgentEnv: CommitMessageAgentEnvironmentResolvers | null = null
|
||||
@@ -3074,8 +3082,6 @@ export class OrcaRuntimeService {
|
||||
* only carrier of the provider session a transcript is addressed by. */
|
||||
getAgentProviderSessionSnapshot?: () => AgentStatusIpcPayload[]
|
||||
getAgentProviderSessionRowsForPane?: (paneKey: string) => AgentStatusIpcPayload[]
|
||||
getHydratedAgentHookAuthority?: () => readonly AgentHookAuthorityEvidence[]
|
||||
getCurrentAgentHookAuthority?: () => readonly AgentHookAuthorityEvidence[]
|
||||
attestAgentHookCompatibilityAuthority?: (candidate: {
|
||||
paneKey: string
|
||||
launchTokenHash: string
|
||||
@@ -3114,8 +3120,6 @@ export class OrcaRuntimeService {
|
||||
this.getAgentProviderSessionSnapshotFn =
|
||||
deps?.getAgentProviderSessionSnapshot ?? deps?.getAgentStatusSnapshot ?? null
|
||||
this.getAgentProviderSessionRowsForPaneFn = deps?.getAgentProviderSessionRowsForPane ?? null
|
||||
this.getHydratedAgentHookAuthorityFn = deps?.getHydratedAgentHookAuthority ?? null
|
||||
this.getCurrentAgentHookAuthorityFn = deps?.getCurrentAgentHookAuthority ?? null
|
||||
this.attestAgentHookCompatibilityAuthorityFn =
|
||||
deps?.attestAgentHookCompatibilityAuthority ?? null
|
||||
this.canRecoverPersistentLocalPtysFn = deps?.canRecoverPersistentLocalPtys ?? (() => true)
|
||||
@@ -3659,6 +3663,70 @@ export class OrcaRuntimeService {
|
||||
return result
|
||||
}
|
||||
|
||||
private hasExactTerminalSurfaceIdentity(expected: {
|
||||
worktreeId: string
|
||||
tabId: string
|
||||
leafId: string
|
||||
ptyId: string
|
||||
terminalHandle: string
|
||||
incarnationId: string
|
||||
}): boolean {
|
||||
if (this.graphStatus !== 'ready') {
|
||||
return false
|
||||
}
|
||||
const pty = this.ptysById.get(expected.ptyId)
|
||||
if (
|
||||
!pty?.connected ||
|
||||
pty.incarnationId !== expected.incarnationId ||
|
||||
pty.tabId !== expected.tabId ||
|
||||
pty.paneKey !== makePaneKey(expected.tabId, expected.leafId) ||
|
||||
!runtimeWorktreeIdsEqual(pty.worktreeId, expected.worktreeId) ||
|
||||
this.handleByPtyId.get(expected.ptyId) !== expected.terminalHandle
|
||||
) {
|
||||
return false
|
||||
}
|
||||
const tab = this.tabs.get(expected.tabId)
|
||||
const leaf = this.leaves.get(this.getLeafKey(expected.tabId, expected.leafId))
|
||||
const ptyLeaves = this.getLeavesForPty(expected.ptyId)
|
||||
return (
|
||||
Boolean(tab && runtimeWorktreeIdsEqual(tab.worktreeId, expected.worktreeId)) &&
|
||||
Boolean(
|
||||
leaf &&
|
||||
leaf.ptyId === expected.ptyId &&
|
||||
runtimeWorktreeIdsEqual(leaf.worktreeId, expected.worktreeId)
|
||||
) &&
|
||||
ptyLeaves.length === 1 &&
|
||||
ptyLeaves[0]?.tabId === expected.tabId &&
|
||||
ptyLeaves[0]?.leafId === expected.leafId
|
||||
)
|
||||
}
|
||||
|
||||
private hasExactPersistedTerminalSurfaceIdentity(expected: {
|
||||
worktreeId: string
|
||||
tabId: string
|
||||
leafId: string
|
||||
ptyId: string
|
||||
incarnationId: string
|
||||
}): boolean {
|
||||
const session = this.getWorkspaceSessionForWorktree(expected.worktreeId)
|
||||
const sessionWorktreeId = session
|
||||
? resolveTerminalSessionWorktreeId(session, expected.worktreeId)
|
||||
: null
|
||||
if (!session || !sessionWorktreeId) {
|
||||
return false
|
||||
}
|
||||
const tab = session.tabsByWorktree[sessionWorktreeId]?.find(
|
||||
(candidate) => candidate.id === expected.tabId
|
||||
)
|
||||
const paneKey = makePaneKey(expected.tabId, expected.leafId)
|
||||
return Boolean(
|
||||
tab &&
|
||||
session.terminalLayoutsByTabId[expected.tabId]?.ptyIdsByLeafId?.[expected.leafId] ===
|
||||
expected.ptyId &&
|
||||
session.terminalPtyIncarnationsByPaneKey?.[paneKey] === expected.incarnationId
|
||||
)
|
||||
}
|
||||
|
||||
private persistLegacyWorkerTerminalRecoveryResolution(
|
||||
candidate: LegacyWorkerTerminalRecoveryPlan['candidates'][number],
|
||||
resolution: 'adopted' | 'exited'
|
||||
@@ -3672,23 +3740,8 @@ export class OrcaRuntimeService {
|
||||
if (!record || !runtimeWorktreeIdsEqual(record.worktreeId, candidate.worktreeId)) {
|
||||
return true
|
||||
}
|
||||
if (
|
||||
resolution === 'exited' &&
|
||||
record.automaticResumeBlockedBy !== 'legacy-orchestration-worker'
|
||||
) {
|
||||
return true
|
||||
}
|
||||
const next = structuredClone(session)
|
||||
if (resolution === 'adopted') {
|
||||
delete next.sleepingAgentSessionsByPaneKey?.[candidate.paneKey]
|
||||
} else {
|
||||
const unblocked = { ...record }
|
||||
delete unblocked.automaticResumeBlockedBy
|
||||
next.sleepingAgentSessionsByPaneKey = {
|
||||
...next.sleepingAgentSessionsByPaneKey,
|
||||
[candidate.paneKey]: unblocked
|
||||
}
|
||||
}
|
||||
delete next.sleepingAgentSessionsByPaneKey?.[candidate.paneKey]
|
||||
try {
|
||||
this.setWorkspaceSessionForWorktree(candidate.worktreeId, next)
|
||||
store.flushOrThrow()
|
||||
@@ -3704,6 +3757,85 @@ export class OrcaRuntimeService {
|
||||
}
|
||||
}
|
||||
|
||||
private updateLegacyWorkerTerminalRecoveryRetry(
|
||||
plan: LegacyWorkerTerminalRecoveryPlan,
|
||||
deferredDispatchIds: ReadonlySet<string>,
|
||||
options: { connectionId?: string; materializeRenderer?: boolean }
|
||||
): void {
|
||||
const scopeKey = options.connectionId ? `ssh:${options.connectionId}` : 'local'
|
||||
const hasDeferredWorker = plan.candidates.some((candidate) => {
|
||||
const sshPty = parseAppSshPtyId(candidate.ptyId)
|
||||
const inScope = options.connectionId
|
||||
? sshPty?.connectionId === options.connectionId
|
||||
: sshPty === null
|
||||
return inScope && deferredDispatchIds.has(candidate.dispatchId)
|
||||
})
|
||||
if (!hasDeferredWorker) {
|
||||
this.cancelLegacyWorkerTerminalRecoveryRetry(scopeKey)
|
||||
return
|
||||
}
|
||||
const existing = this.legacyWorkerTerminalRecoveryRetries.get(scopeKey)
|
||||
const retry = existing ?? {
|
||||
attempt: 0,
|
||||
...(options.connectionId ? { connectionId: options.connectionId } : {}),
|
||||
materializeRenderer: options.materializeRenderer === true,
|
||||
timer: null
|
||||
}
|
||||
retry.materializeRenderer ||= options.materializeRenderer === true
|
||||
this.legacyWorkerTerminalRecoveryRetries.set(scopeKey, retry)
|
||||
this.armLegacyWorkerTerminalRecoveryRetry(scopeKey, retry)
|
||||
}
|
||||
|
||||
private cancelLegacyWorkerTerminalRecoveryRetry(scopeKey: string): void {
|
||||
const retry = this.legacyWorkerTerminalRecoveryRetries.get(scopeKey)
|
||||
if (retry?.timer) {
|
||||
clearTimeout(retry.timer)
|
||||
}
|
||||
this.legacyWorkerTerminalRecoveryRetries.delete(scopeKey)
|
||||
}
|
||||
|
||||
private getLegacyWorkerControllerPtyLiveness(
|
||||
candidate: LegacyWorkerTerminalRecoveryPlan['candidates'][number]
|
||||
): boolean | null {
|
||||
try {
|
||||
return this.ptyController?.hasPty?.(candidate.ptyId) ?? null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
private armLegacyWorkerTerminalRecoveryRetry(
|
||||
scopeKey: string,
|
||||
retry: {
|
||||
attempt: number
|
||||
connectionId?: string
|
||||
materializeRenderer: boolean
|
||||
timer: ReturnType<typeof setTimeout> | null
|
||||
}
|
||||
): void {
|
||||
if (retry.timer) {
|
||||
return
|
||||
}
|
||||
const delayMs = Math.min(1_000 * 2 ** retry.attempt, 30_000)
|
||||
retry.attempt += 1
|
||||
retry.timer = setTimeout(() => {
|
||||
retry.timer = null
|
||||
void this.reconcileLegacyWorkerTerminals({
|
||||
...(retry.connectionId ? { connectionId: retry.connectionId } : {}),
|
||||
materializeRenderer: retry.materializeRenderer
|
||||
}).catch((error) => {
|
||||
console.warn('[orchestration] worker terminal recovery retry failed', {
|
||||
scope: scopeKey,
|
||||
error
|
||||
})
|
||||
if (this.legacyWorkerTerminalRecoveryRetries.get(scopeKey) === retry) {
|
||||
this.armLegacyWorkerTerminalRecoveryRetry(scopeKey, retry)
|
||||
}
|
||||
})
|
||||
}, delayMs)
|
||||
retry.timer.unref?.()
|
||||
}
|
||||
|
||||
private async reconcileLegacyWorkerTerminalsNow(options: {
|
||||
connectionId?: string
|
||||
materializeRenderer?: boolean
|
||||
@@ -3712,142 +3844,251 @@ export class OrcaRuntimeService {
|
||||
const adoptedDispatchIds: string[] = []
|
||||
const exitedDispatchIds: string[] = []
|
||||
const deferredDispatchIds = new Set(plan.ambiguousDispatchIds)
|
||||
for (const candidate of plan.candidates) {
|
||||
let workspace: TerminalWorkspaceLaunchScope
|
||||
try {
|
||||
workspace = await this.resolveTerminalWorkspaceLaunchScope(`id:${candidate.worktreeId}`)
|
||||
} catch {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
const recoveryCandidatesByProvider = new Map<
|
||||
string,
|
||||
{
|
||||
connectionId: string | null
|
||||
entries: {
|
||||
candidate: (typeof plan.candidates)[number]
|
||||
workspace: TerminalWorkspaceLaunchScope
|
||||
resolvedWorkspace: ResolvedWorktree
|
||||
}[]
|
||||
}
|
||||
const sshPty = parseAppSshPtyId(candidate.ptyId)
|
||||
if (workspace.connectionId) {
|
||||
if (
|
||||
options.connectionId !== workspace.connectionId ||
|
||||
sshPty?.connectionId !== workspace.connectionId
|
||||
>()
|
||||
for (const candidate of plan.candidates) {
|
||||
try {
|
||||
const workspace = await this.resolveTerminalWorkspaceLaunchScope(
|
||||
`id:${candidate.worktreeId}`
|
||||
)
|
||||
const sshPty = parseAppSshPtyId(candidate.ptyId)
|
||||
if (workspace.connectionId) {
|
||||
if (
|
||||
options.connectionId !== workspace.connectionId ||
|
||||
sshPty?.connectionId !== workspace.connectionId
|
||||
) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
} else if (
|
||||
options.connectionId !== undefined ||
|
||||
sshPty !== null ||
|
||||
!this.canRecoverPersistentLocalPtysFn()
|
||||
) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
} else if (
|
||||
options.connectionId !== undefined ||
|
||||
sshPty !== null ||
|
||||
!this.canRecoverPersistentLocalPtysFn()
|
||||
) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
const resolvedWorkspace = workspace.folderWorkspace
|
||||
? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace)
|
||||
: await this.resolveWorktreeSelector(`id:${workspace.id}`)
|
||||
const livePtyIds = await this.refreshPtyWorktreeRecordsFromController([resolvedWorkspace])
|
||||
if (!livePtyIds) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
if (!livePtyIds.has(candidate.ptyId)) {
|
||||
if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) {
|
||||
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited')
|
||||
exitedDispatchIds.push(candidate.dispatchId)
|
||||
} else {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
const resolvedWorkspace = workspace.folderWorkspace
|
||||
? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace)
|
||||
: await this.resolveWorktreeSelector(`id:${workspace.id}`)
|
||||
const connectionId = workspace.connectionId ?? null
|
||||
const providerKey = connectionId === null ? 'local' : `ssh:${connectionId}`
|
||||
const provider = recoveryCandidatesByProvider.get(providerKey) ?? {
|
||||
connectionId,
|
||||
entries: []
|
||||
}
|
||||
continue
|
||||
}
|
||||
const controllerIdentity = this.controllerTerminalIdentityByPtyId.get(candidate.ptyId)
|
||||
if (
|
||||
controllerIdentity?.handle !== candidate.terminalHandle ||
|
||||
controllerIdentity.incarnationId !== candidate.incarnationId
|
||||
) {
|
||||
provider.entries.push({ candidate, workspace, resolvedWorkspace })
|
||||
recoveryCandidatesByProvider.set(providerKey, provider)
|
||||
} catch {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
}
|
||||
}
|
||||
for (const provider of recoveryCandidatesByProvider.values()) {
|
||||
const resolvedWorktrees = [
|
||||
...new Map(
|
||||
provider.entries.map(({ resolvedWorkspace }) => [resolvedWorkspace.id, resolvedWorkspace])
|
||||
).values()
|
||||
]
|
||||
const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory(
|
||||
resolvedWorktrees,
|
||||
null,
|
||||
undefined,
|
||||
provider.connectionId
|
||||
)
|
||||
if (!inventory) {
|
||||
provider.entries.forEach(({ candidate }) => deferredDispatchIds.add(candidate.dispatchId))
|
||||
continue
|
||||
}
|
||||
const session = this.getWorkspaceSessionForWorktree(candidate.worktreeId)
|
||||
const sessionWorktreeId = session
|
||||
? resolveTerminalSessionWorktreeId(session, candidate.worktreeId)
|
||||
: null
|
||||
const activeTabId = sessionWorktreeId
|
||||
? session?.activeTabIdByWorktree?.[sessionWorktreeId]
|
||||
: undefined
|
||||
const activeGroupId = sessionWorktreeId
|
||||
? session?.activeGroupIdByWorktree?.[sessionWorktreeId]
|
||||
: undefined
|
||||
try {
|
||||
await this.adoptTerminalOrphans({
|
||||
worktree: `id:${candidate.worktreeId}`,
|
||||
expectedTopologyRevision: this.getTerminalTopologyRevision(candidate.worktreeId),
|
||||
...(activeTabId ? { activeTabId } : {}),
|
||||
...(activeGroupId ? { activeGroupId } : {}),
|
||||
claims: [
|
||||
{
|
||||
terminal: candidate.terminalHandle,
|
||||
ptyId: candidate.ptyId,
|
||||
incarnationId: candidate.incarnationId,
|
||||
tabId: candidate.tabId,
|
||||
leafId: candidate.leafId
|
||||
}
|
||||
]
|
||||
})
|
||||
} catch (error) {
|
||||
console.warn('[orchestration] legacy worker terminal adoption deferred', {
|
||||
dispatchId: candidate.dispatchId,
|
||||
error
|
||||
})
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
let rendererMaterialized =
|
||||
options.materializeRenderer !== true ||
|
||||
this.legacyWorkerTerminalMaterializedPanes.has(candidate.paneKey)
|
||||
const pty = this.ptysById.get(candidate.ptyId)
|
||||
if (
|
||||
options.materializeRenderer &&
|
||||
!rendererMaterialized &&
|
||||
pty &&
|
||||
this.notifier?.revealTerminalSession
|
||||
) {
|
||||
for (let attempt = 0; attempt < 2 && !rendererMaterialized; attempt += 1) {
|
||||
for (const { candidate, workspace } of provider.entries) {
|
||||
if (!inventory.livePtyIds.has(candidate.ptyId)) {
|
||||
if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) {
|
||||
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited')
|
||||
exitedDispatchIds.push(candidate.dispatchId)
|
||||
} else {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
}
|
||||
continue
|
||||
}
|
||||
const controllerIdentity = inventory.terminalIdentityByPtyId.get(candidate.ptyId)
|
||||
if (
|
||||
controllerIdentity?.handle !== candidate.terminalHandle ||
|
||||
controllerIdentity.incarnationId !== candidate.incarnationId
|
||||
) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
const controllerLiveness = this.getLegacyWorkerControllerPtyLiveness(candidate)
|
||||
if (controllerLiveness === false) {
|
||||
this.onPtyExit(candidate.ptyId, 0, candidate.incarnationId)
|
||||
if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) {
|
||||
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited')
|
||||
exitedDispatchIds.push(candidate.dispatchId)
|
||||
} else {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (controllerLiveness === null) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
const session = this.getWorkspaceSessionForWorktree(candidate.worktreeId)
|
||||
const sessionWorktreeId = session
|
||||
? resolveTerminalSessionWorktreeId(session, candidate.worktreeId)
|
||||
: null
|
||||
const activeTabId = sessionWorktreeId
|
||||
? session?.activeTabIdByWorktree?.[sessionWorktreeId]
|
||||
: undefined
|
||||
const activeGroupId = sessionWorktreeId
|
||||
? session?.activeGroupIdByWorktree?.[sessionWorktreeId]
|
||||
: undefined
|
||||
const exactSurfaceAlreadyPublished =
|
||||
this.hasExactPersistedTerminalSurfaceIdentity(candidate) &&
|
||||
this.hasExactTerminalSurfaceIdentity(candidate)
|
||||
if (!exactSurfaceAlreadyPublished) {
|
||||
try {
|
||||
await this.notifier.revealTerminalSession(candidate.worktreeId, {
|
||||
ptyId: candidate.ptyId,
|
||||
title: getLatestPtyTitle(pty) ?? pty.controllerTitle,
|
||||
activate: false,
|
||||
presentation: 'background',
|
||||
tabId: candidate.tabId,
|
||||
leafId: candidate.leafId,
|
||||
focus: false
|
||||
})
|
||||
rendererMaterialized = true
|
||||
this.legacyWorkerTerminalMaterializedPanes.add(candidate.paneKey)
|
||||
await this.adoptTerminalOrphansFromInventory(
|
||||
{
|
||||
worktree: `id:${candidate.worktreeId}`,
|
||||
expectedTopologyRevision: this.getTerminalTopologyRevision(candidate.worktreeId),
|
||||
...(activeTabId ? { activeTabId } : {}),
|
||||
...(activeGroupId ? { activeGroupId } : {}),
|
||||
claims: [
|
||||
{
|
||||
terminal: candidate.terminalHandle,
|
||||
ptyId: candidate.ptyId,
|
||||
incarnationId: candidate.incarnationId,
|
||||
tabId: candidate.tabId,
|
||||
leafId: candidate.leafId
|
||||
}
|
||||
]
|
||||
},
|
||||
workspace,
|
||||
inventory
|
||||
)
|
||||
} catch (error) {
|
||||
if (attempt === 0) {
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 100))
|
||||
continue
|
||||
}
|
||||
console.warn('[orchestration] adopted legacy worker was not revealed', {
|
||||
console.warn('[orchestration] legacy worker terminal adoption deferred', {
|
||||
dispatchId: candidate.dispatchId,
|
||||
error
|
||||
})
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
}
|
||||
let rendererMaterialized =
|
||||
options.materializeRenderer !== true ||
|
||||
this.legacyWorkerTerminalMaterializedPanes.has(candidate.paneKey)
|
||||
const pty = this.ptysById.get(candidate.ptyId)
|
||||
if (
|
||||
options.materializeRenderer &&
|
||||
!rendererMaterialized &&
|
||||
pty &&
|
||||
this.notifier?.revealTerminalSession
|
||||
) {
|
||||
for (let attempt = 0; attempt < 2 && !rendererMaterialized; attempt += 1) {
|
||||
try {
|
||||
const reveal = await this.notifier.revealTerminalSession(candidate.worktreeId, {
|
||||
ptyId: candidate.ptyId,
|
||||
title: getLatestPtyTitle(pty) ?? pty.controllerTitle,
|
||||
activate: false,
|
||||
presentation: 'background',
|
||||
tabId: candidate.tabId,
|
||||
leafId: candidate.leafId,
|
||||
focus: false,
|
||||
expectedProcessIdentity: {
|
||||
terminalHandle: candidate.terminalHandle,
|
||||
incarnationId: candidate.incarnationId
|
||||
}
|
||||
})
|
||||
if (reveal?.identity) {
|
||||
const identity = reveal.identity
|
||||
if (
|
||||
!runtimeWorktreeIdsEqual(identity.worktreeId, candidate.worktreeId) ||
|
||||
identity.tabId !== candidate.tabId ||
|
||||
identity.leafId !== candidate.leafId ||
|
||||
identity.ptyId !== candidate.ptyId
|
||||
) {
|
||||
throw new Error('terminal_reveal_identity_mismatch')
|
||||
}
|
||||
this.legacyWorkerTerminalIdentityReceiptPanes.add(candidate.paneKey)
|
||||
}
|
||||
rendererMaterialized = true
|
||||
this.legacyWorkerTerminalMaterializedPanes.add(candidate.paneKey)
|
||||
} catch (error) {
|
||||
if (attempt === 0) {
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 100))
|
||||
continue
|
||||
}
|
||||
console.warn('[orchestration] adopted legacy worker was not revealed', {
|
||||
dispatchId: candidate.dispatchId,
|
||||
error
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!rendererMaterialized) {
|
||||
this.legacyWorkerTerminalMaterializedPanes.delete(candidate.paneKey)
|
||||
this.legacyWorkerTerminalIdentityReceiptPanes.delete(candidate.paneKey)
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
if (
|
||||
this.legacyWorkerTerminalIdentityReceiptPanes.has(candidate.paneKey) &&
|
||||
!this.hasExactTerminalSurfaceIdentity({
|
||||
worktreeId: candidate.worktreeId,
|
||||
tabId: candidate.tabId,
|
||||
leafId: candidate.leafId,
|
||||
ptyId: candidate.ptyId,
|
||||
terminalHandle: candidate.terminalHandle,
|
||||
incarnationId: candidate.incarnationId
|
||||
})
|
||||
) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
const postRevealLiveness = this.getLegacyWorkerControllerPtyLiveness(candidate)
|
||||
if (postRevealLiveness === false) {
|
||||
this.legacyWorkerTerminalMaterializedPanes.delete(candidate.paneKey)
|
||||
this.legacyWorkerTerminalIdentityReceiptPanes.delete(candidate.paneKey)
|
||||
this.onPtyExit(candidate.ptyId, 0, candidate.incarnationId)
|
||||
if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) {
|
||||
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited')
|
||||
exitedDispatchIds.push(candidate.dispatchId)
|
||||
} else {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (postRevealLiveness === null) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
if (!this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'adopted')) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
this.legacyWorkerRecoveredPtys.add(candidate.ptyId)
|
||||
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'adopted')
|
||||
adoptedDispatchIds.push(candidate.dispatchId)
|
||||
}
|
||||
if (
|
||||
!rendererMaterialized ||
|
||||
!this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'adopted')
|
||||
) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
this.legacyWorkerRecoveredPtys.add(candidate.ptyId)
|
||||
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'adopted')
|
||||
adoptedDispatchIds.push(candidate.dispatchId)
|
||||
}
|
||||
return {
|
||||
const result = {
|
||||
blockedPaneCount: plan.blockedPanes.length,
|
||||
adoptedDispatchIds,
|
||||
exitedDispatchIds,
|
||||
deferredDispatchIds: [...deferredDispatchIds]
|
||||
}
|
||||
this.updateLegacyWorkerTerminalRecoveryRetry(plan, deferredDispatchIds, options)
|
||||
return result
|
||||
}
|
||||
|
||||
setAutomationService(service: AutomationService): void {
|
||||
@@ -4213,6 +4454,9 @@ export class OrcaRuntimeService {
|
||||
notifySshStateChanged(targetId: string, state: SshConnectionState): void {
|
||||
this.bumpSshRelayRecoveryGeneration(targetId)
|
||||
this.invalidateSshWorktreeScanCache(targetId)
|
||||
if (state.status !== 'connected') {
|
||||
this.cancelLegacyWorkerTerminalRecoveryRetry(`ssh:${targetId}`)
|
||||
}
|
||||
this.emitClientEvent({ type: 'sshStateChanged', targetId, state: getPublicSshState(state)! })
|
||||
}
|
||||
|
||||
@@ -10345,13 +10589,6 @@ export class OrcaRuntimeService {
|
||||
return 'local'
|
||||
}
|
||||
|
||||
getOrchestrationCompatibilityAuthorityInputs(): OrchestrationCompatibilityAuthorityInputs {
|
||||
return {
|
||||
hydratedCommitments: this.getHydratedAgentHookAuthorityFn?.() ?? [],
|
||||
currentObservations: this.getCurrentAgentHookAuthorityFn?.() ?? []
|
||||
}
|
||||
}
|
||||
|
||||
registerOrchestrationCompatibilitySshAttachment(
|
||||
targetId: string,
|
||||
connectionIncarnation: string
|
||||
@@ -13554,13 +13791,24 @@ export class OrcaRuntimeService {
|
||||
const resolvedWorkspace = workspace.folderWorkspace
|
||||
? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace)
|
||||
: await this.resolveWorktreeSelector(`id:${workspace.id}`)
|
||||
const livePtyIds = await this.refreshPtyWorktreeRecordsFromController(
|
||||
const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory(
|
||||
[resolvedWorkspace],
|
||||
workspace.id
|
||||
workspace.id,
|
||||
undefined,
|
||||
workspace.connectionId ?? null
|
||||
)
|
||||
if (!livePtyIds) {
|
||||
if (!inventory) {
|
||||
throw new Error('terminal_liveness_unavailable')
|
||||
}
|
||||
return this.adoptTerminalOrphansFromInventory(request, workspace, inventory)
|
||||
}
|
||||
|
||||
private async adoptTerminalOrphansFromInventory(
|
||||
request: RuntimeTerminalOrphanAdoptionRequest,
|
||||
workspace: TerminalWorkspaceLaunchScope,
|
||||
inventory: PtyControllerInventory
|
||||
): Promise<RuntimeTerminalOrphanAdoptionResult> {
|
||||
const { livePtyIds, terminalIdentityByPtyId } = inventory
|
||||
const store = this.store
|
||||
const session = this.getWorkspaceSessionForWorktree(workspace.id)
|
||||
if (!store?.setWorkspaceSession || !store.flushOrThrow || !session) {
|
||||
@@ -13593,7 +13841,7 @@ export class OrcaRuntimeService {
|
||||
seenPaneKeys.add(paneKey)
|
||||
const live = this.getLivePtyForHandle(claim.terminal)
|
||||
const pty = live?.pty
|
||||
const controllerIdentity = this.controllerTerminalIdentityByPtyId.get(claim.ptyId)
|
||||
const controllerIdentity = terminalIdentityByPtyId.get(claim.ptyId)
|
||||
if (
|
||||
!pty ||
|
||||
pty.ptyId !== claim.ptyId ||
|
||||
@@ -13681,10 +13929,14 @@ export class OrcaRuntimeService {
|
||||
)
|
||||
})
|
||||
if (isExactPersisted && sessionWorktreeId === workspace.id) {
|
||||
for (const { claim, pty, paneKey } of validated) {
|
||||
pty.tabId = claim.tabId
|
||||
pty.paneKey = paneKey
|
||||
}
|
||||
return {
|
||||
adopted: false,
|
||||
topologyRevision: currentRevision,
|
||||
snapshot: await this.listMobileSessionTabs(`id:${workspace.id}`)
|
||||
snapshot: this.getTerminalOrphanAdoptionSnapshot(workspace.id)
|
||||
}
|
||||
}
|
||||
if (currentRevision !== request.expectedTopologyRevision) {
|
||||
@@ -13999,10 +14251,19 @@ export class OrcaRuntimeService {
|
||||
return {
|
||||
adopted: true,
|
||||
topologyRevision: persisted.terminalTopologyRevisionByRepoId?.[repoId] ?? currentRevision + 1,
|
||||
snapshot: await this.listMobileSessionTabs(`id:${workspace.id}`)
|
||||
snapshot: this.getTerminalOrphanAdoptionSnapshot(workspace.id)
|
||||
}
|
||||
}
|
||||
|
||||
private getTerminalOrphanAdoptionSnapshot(worktreeId: string): RuntimeMobileSessionTabsResult {
|
||||
this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId, {
|
||||
allowAttachedWindow: true,
|
||||
onlyRuntimeOwnedTerminals: true
|
||||
})
|
||||
this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId)
|
||||
return this.getMobileSessionTabsForWorktree(worktreeId)
|
||||
}
|
||||
|
||||
private buildTerminalVisualLayouts(
|
||||
terminals: RuntimeTerminalSummary[],
|
||||
worktreesById: Map<string, ResolvedWorktree>,
|
||||
@@ -26302,17 +26563,34 @@ export class OrcaRuntimeService {
|
||||
targetWorktreeId: string | null = null,
|
||||
deadline?: number
|
||||
): Promise<Set<string> | null> {
|
||||
const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory(
|
||||
resolvedWorktrees,
|
||||
targetWorktreeId,
|
||||
deadline
|
||||
)
|
||||
return inventory ? new Set(inventory.livePtyIds) : null
|
||||
}
|
||||
|
||||
private async refreshPtyWorktreeRecordsWithControllerInventory(
|
||||
resolvedWorktrees: ResolvedWorktree[],
|
||||
targetWorktreeId: string | null = null,
|
||||
deadline?: number,
|
||||
connectionId?: string | null
|
||||
): Promise<PtyControllerInventory | null> {
|
||||
if (targetWorktreeId === FLOATING_TERMINAL_WORKTREE_ID) {
|
||||
const targetedLiveness = this.refreshFloatingWorkspacePtyLiveness()
|
||||
if (targetedLiveness !== null) {
|
||||
return targetedLiveness
|
||||
return {
|
||||
livePtyIds: targetedLiveness,
|
||||
terminalIdentityByPtyId: new Map()
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!this.ptyController?.listProcesses) {
|
||||
return null
|
||||
}
|
||||
const sessionsResult = await withTimeoutResult(
|
||||
this.ptyController.listProcesses(),
|
||||
this.ptyController.listProcesses(connectionId),
|
||||
deadline === undefined
|
||||
? PTY_CONTROLLER_LIST_TIMEOUT_MS
|
||||
: Math.max(1, Math.min(PTY_CONTROLLER_LIST_TIMEOUT_MS, deadline - Date.now()))
|
||||
@@ -26322,10 +26600,7 @@ export class OrcaRuntimeService {
|
||||
return null
|
||||
}
|
||||
const sessions = sessionsResult.value
|
||||
const controllerIdentityByPtyId = new Map<
|
||||
string,
|
||||
{ handle: string; incarnationId: string; wslDistro?: string | null }
|
||||
>()
|
||||
const controllerIdentityByPtyId = new Map<string, PtyControllerTerminalIdentity>()
|
||||
const ptyIdByControllerHandle = new Map<string, string>()
|
||||
const ambiguousControllerPtyIds = new Set<string>()
|
||||
for (const session of sessions) {
|
||||
@@ -26356,7 +26631,6 @@ export class OrcaRuntimeService {
|
||||
for (const ptyId of ambiguousControllerPtyIds) {
|
||||
controllerIdentityByPtyId.delete(ptyId)
|
||||
}
|
||||
this.controllerTerminalIdentityByPtyId = controllerIdentityByPtyId
|
||||
const persistedWorktreeIdByPtyId = indexPersistedPtyWorktreeBindings(
|
||||
this.store?.getWorkspaceSession?.()
|
||||
)
|
||||
@@ -26432,6 +26706,9 @@ export class OrcaRuntimeService {
|
||||
this.refreshPtyForegroundAgent(session.id)
|
||||
}
|
||||
for (const pty of this.ptysById.values()) {
|
||||
if (connectionId !== undefined && pty.connectionId !== connectionId) {
|
||||
continue
|
||||
}
|
||||
if (!allLivePtyIds.has(pty.ptyId) && !this.leafExistsForPty(pty.ptyId)) {
|
||||
if (this.ptyController.hasPty?.(pty.ptyId) === true) {
|
||||
// Why: an SSH spawn can become addressable before an overlapping relay list includes it.
|
||||
@@ -26451,7 +26728,10 @@ export class OrcaRuntimeService {
|
||||
}
|
||||
}
|
||||
this.pruneDisconnectedPtyRecords()
|
||||
return targetWorktreeId ? selectedLivePtyIds : allLivePtyIds
|
||||
return {
|
||||
livePtyIds: targetWorktreeId ? selectedLivePtyIds : allLivePtyIds,
|
||||
terminalIdentityByPtyId: controllerIdentityByPtyId
|
||||
}
|
||||
}
|
||||
|
||||
private refreshFloatingWorkspacePtyLiveness(): Set<string> | null {
|
||||
|
||||
@@ -171,10 +171,10 @@ function legacyMessageMatchesQuestion(
|
||||
message: MessageRow,
|
||||
question: string,
|
||||
options: string[],
|
||||
recipientHandle: string
|
||||
recipientHandles: readonly string[]
|
||||
): boolean {
|
||||
if (
|
||||
message.to_handle !== recipientHandle ||
|
||||
!recipientHandles.includes(message.to_handle) ||
|
||||
normalizeLegacyQuestionText(message.body) !== normalizeLegacyQuestionText(question)
|
||||
) {
|
||||
return false
|
||||
@@ -1212,6 +1212,13 @@ export class OrchestrationDb {
|
||||
`The ${params.role} compatibility principal is already committed to different proof.`
|
||||
)
|
||||
}
|
||||
if (existing.status === 'revoked') {
|
||||
throw new OrchestrationError(
|
||||
'legacy_read_only',
|
||||
`The ${params.role} compatibility principal has been revoked. No effects were applied.`,
|
||||
{ effectsApplied: false }
|
||||
)
|
||||
}
|
||||
this.db.exec('COMMIT')
|
||||
return { principal: existing, duplicate: true }
|
||||
}
|
||||
@@ -1263,6 +1270,15 @@ export class OrchestrationDb {
|
||||
.all(runId) as LegacyCompatibilityPrincipalRow[]
|
||||
}
|
||||
|
||||
getLegacyCoordinatorPrincipal(runId: string): LegacyCompatibilityPrincipalRow | undefined {
|
||||
return this.db
|
||||
.prepare(
|
||||
`SELECT * FROM legacy_compatibility_principals
|
||||
WHERE run_id = ? AND role = 'coordinator'`
|
||||
)
|
||||
.get(runId) as LegacyCompatibilityPrincipalRow | undefined
|
||||
}
|
||||
|
||||
resolveLegacyCompatibilityPrincipalByIdentity(params: {
|
||||
runId: string
|
||||
role: LegacyPrincipalRole
|
||||
@@ -1362,6 +1378,10 @@ export class OrchestrationDb {
|
||||
}
|
||||
|
||||
isLegacyCoordinatorHandle(runId: string, terminalHandle: string): boolean {
|
||||
const principal = this.getLegacyCoordinatorPrincipal(runId)
|
||||
if (principal?.terminal_handle === terminalHandle) {
|
||||
return true
|
||||
}
|
||||
return Boolean(
|
||||
this.db
|
||||
.prepare(
|
||||
@@ -1403,18 +1423,24 @@ export class OrchestrationDb {
|
||||
if (!principal || principal.role !== 'worker' || !principal.dispatch_id) {
|
||||
throw new OrchestrationError('request_mismatch', 'Legacy worker principal was not found.')
|
||||
}
|
||||
const runAddress = `run:${principal.run_id}`
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`SELECT * FROM messages
|
||||
WHERE run_id = ? AND delivery_contract = 'legacy_direct'
|
||||
AND from_handle = ? AND to_handle = ? AND type = 'worker_done'
|
||||
WHERE run_id = ?
|
||||
AND (
|
||||
(delivery_contract = 'legacy_direct' AND to_handle = ?) OR
|
||||
(delivery_contract = 'current_delivery' AND to_handle = ?)
|
||||
)
|
||||
AND from_handle = ? AND type = 'worker_done'
|
||||
AND subject = ? AND body = ? AND payload IS ?
|
||||
ORDER BY sequence`
|
||||
)
|
||||
.all(
|
||||
principal.run_id,
|
||||
principal.terminal_handle,
|
||||
params.recipientHandle,
|
||||
runAddress,
|
||||
principal.terminal_handle,
|
||||
params.subject,
|
||||
params.body,
|
||||
params.payload
|
||||
@@ -1444,10 +1470,11 @@ export class OrchestrationDb {
|
||||
this.db
|
||||
.prepare(
|
||||
`SELECT 1 FROM messages
|
||||
WHERE run_id = ? AND delivery_contract = 'current_delivery' AND read = 0
|
||||
WHERE run_id = ? AND to_handle = ?
|
||||
AND delivery_contract = 'current_delivery' AND read = 0
|
||||
LIMIT 1`
|
||||
)
|
||||
.get(runId)
|
||||
.get(runId, `run:${runId}`)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1832,7 +1859,14 @@ export class OrchestrationDb {
|
||||
runId: string
|
||||
coordinatorHandle: string
|
||||
coordinatorPaneKey: string
|
||||
allowLegacyCompatibility?: boolean
|
||||
takeoverLegacy?: boolean
|
||||
legacyCoordinatorAuthority?: {
|
||||
runId: string
|
||||
principalId: string | null
|
||||
terminalHandle: string
|
||||
paneKey: string
|
||||
consumerGeneration: number
|
||||
}
|
||||
}): RunRow | undefined {
|
||||
this.db.exec('BEGIN IMMEDIATE')
|
||||
try {
|
||||
@@ -1845,30 +1879,95 @@ export class OrchestrationDb {
|
||||
run.coordinator_pane_key !== null &&
|
||||
isEquivalentPaneKey(run.coordinator_pane_key, params.coordinatorPaneKey)
|
||||
const adoption = this.getLegacyAdoption()
|
||||
const adoptedRun = adoption?.adopted_run_id === params.runId
|
||||
const legacyAuthority = params.legacyCoordinatorAuthority
|
||||
const legacyPrincipalId = legacyAuthority?.principalId
|
||||
const legacyPrincipal = legacyPrincipalId
|
||||
? this.getLegacyCompatibilityPrincipal(legacyPrincipalId)
|
||||
: undefined
|
||||
const provenLegacyBinding = Boolean(
|
||||
adoptedRun &&
|
||||
legacyAuthority &&
|
||||
legacyAuthority.principalId !== null &&
|
||||
legacyAuthority.runId === params.runId &&
|
||||
legacyAuthority.consumerGeneration === run.consumer_generation &&
|
||||
legacyPrincipal?.run_id === params.runId &&
|
||||
legacyPrincipal.role === 'coordinator' &&
|
||||
legacyPrincipal.status === 'committed' &&
|
||||
legacyPrincipal.terminal_handle === legacyAuthority.terminalHandle &&
|
||||
isEquivalentPaneKey(legacyPrincipal.pane_key, legacyAuthority.paneKey) &&
|
||||
params.coordinatorHandle === legacyAuthority.terminalHandle &&
|
||||
isEquivalentPaneKey(params.coordinatorPaneKey, legacyAuthority.paneKey)
|
||||
)
|
||||
if (legacyAuthority && !provenLegacyBinding) {
|
||||
throw new OrchestrationError(
|
||||
'legacy_read_only',
|
||||
'This retained legacy coordinator no longer has lifecycle authority. No effects were applied.',
|
||||
{ effectsApplied: false }
|
||||
)
|
||||
}
|
||||
const activeLegacyAssignment =
|
||||
adoption?.adopted_run_id === params.runId &&
|
||||
adoptedRun &&
|
||||
Boolean(
|
||||
this.db
|
||||
.prepare(
|
||||
`SELECT 1 FROM dispatch_contexts
|
||||
WHERE run_id = ? AND contract_version = ?
|
||||
AND status IN ('pending', 'dispatched')
|
||||
UNION ALL
|
||||
SELECT 1 FROM legacy_compatibility_principals
|
||||
WHERE run_id = ? AND status = 'committed'
|
||||
LIMIT 1`
|
||||
)
|
||||
.get(params.runId, LEGACY_CONTRACT_VERSION, params.runId)
|
||||
.get(params.runId, LEGACY_CONTRACT_VERSION)
|
||||
)
|
||||
if (activeLegacyAssignment && !sameBinding && !params.allowLegacyCompatibility) {
|
||||
const coordinatorPrincipal = adoptedRun
|
||||
? this.getLegacyCoordinatorPrincipal(params.runId)
|
||||
: undefined
|
||||
const retainedCoordinatorHandle =
|
||||
coordinatorPrincipal?.terminal_handle ??
|
||||
run.coordinator_handle ??
|
||||
this.getUniqueLegacyTaskCreatorHandle(params.runId)
|
||||
const takeoverAlreadyApplied = Boolean(
|
||||
params.takeoverLegacy &&
|
||||
sameBinding &&
|
||||
run.coordinator_handle === params.coordinatorHandle &&
|
||||
coordinatorPrincipal?.status !== 'committed'
|
||||
)
|
||||
if (params.takeoverLegacy && !adoptedRun) {
|
||||
throw new OrchestrationError(
|
||||
'invalid_argument',
|
||||
'Legacy takeover is only available for the automatically adopted Run.'
|
||||
)
|
||||
}
|
||||
if (
|
||||
activeLegacyAssignment &&
|
||||
!sameBinding &&
|
||||
!provenLegacyBinding &&
|
||||
!params.takeoverLegacy
|
||||
) {
|
||||
throw new OrchestrationError(
|
||||
'consumer_fenced',
|
||||
'This adopted Run still has a live legacy principal; only its attested coordinator may rebind it.',
|
||||
{ effectsApplied: false }
|
||||
'This adopted Run still has live legacy work. Its attested coordinator may rebind it, or a current coordinator may explicitly use run-use --takeover-legacy.',
|
||||
{
|
||||
effectsApplied: false,
|
||||
recoveryCommand: `orca orchestration run-use --id ${params.runId} --takeover-legacy`
|
||||
}
|
||||
)
|
||||
}
|
||||
this.unbindOtherRunsForPane(params.coordinatorPaneKey, params.runId)
|
||||
if (!sameBinding || run.coordinator_handle !== params.coordinatorHandle) {
|
||||
if (
|
||||
(params.takeoverLegacy && !takeoverAlreadyApplied) ||
|
||||
!sameBinding ||
|
||||
run.coordinator_handle !== params.coordinatorHandle
|
||||
) {
|
||||
if (adoptedRun && (params.takeoverLegacy || !activeLegacyAssignment)) {
|
||||
if (
|
||||
coordinatorPrincipal?.status === 'committed' &&
|
||||
(params.takeoverLegacy ||
|
||||
coordinatorPrincipal.terminal_handle !== params.coordinatorHandle ||
|
||||
!isEquivalentPaneKey(coordinatorPrincipal.pane_key, params.coordinatorPaneKey))
|
||||
) {
|
||||
this.setLegacyCompatibilityPrincipalStatus(coordinatorPrincipal.id, 'revoked')
|
||||
}
|
||||
}
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE runs
|
||||
@@ -1879,6 +1978,9 @@ export class OrchestrationDb {
|
||||
)
|
||||
.run(params.coordinatorHandle, params.coordinatorPaneKey, params.runId)
|
||||
this.fenceOutstandingDelivery(params.runId)
|
||||
if (params.takeoverLegacy) {
|
||||
this.promoteLegacyCoordinatorMailForTakeover(params.runId, retainedCoordinatorHandle)
|
||||
}
|
||||
}
|
||||
this.db.exec('COMMIT')
|
||||
} catch (error) {
|
||||
@@ -1953,6 +2055,51 @@ export class OrchestrationDb {
|
||||
.run(runId)
|
||||
}
|
||||
|
||||
private promoteLegacyCoordinatorMailForTakeover(
|
||||
runId: string,
|
||||
retainedCoordinatorHandle: string | null
|
||||
): void {
|
||||
if (!retainedCoordinatorHandle) {
|
||||
return
|
||||
}
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE messages
|
||||
SET to_handle = ?, delivery_contract = 'current_delivery',
|
||||
read = 0, delivered_at = NULL
|
||||
WHERE run_id = ? AND delivery_contract = 'legacy_direct'
|
||||
AND to_handle = ?
|
||||
AND EXISTS(
|
||||
SELECT 1 FROM dispatch_contexts d
|
||||
WHERE d.run_id = messages.run_id
|
||||
AND d.contract_version = ?
|
||||
AND (
|
||||
messages.from_handle = d.assignee_handle OR
|
||||
messages.from_handle = 'dispatch:' || d.id
|
||||
)
|
||||
)
|
||||
AND (
|
||||
read = 0 OR EXISTS(
|
||||
SELECT 1 FROM question_threads q
|
||||
WHERE q.message_id = messages.id AND q.status = 'pending'
|
||||
)
|
||||
)`
|
||||
)
|
||||
.run(`run:${runId}`, runId, retainedCoordinatorHandle, LEGACY_CONTRACT_VERSION)
|
||||
}
|
||||
|
||||
private getUniqueLegacyTaskCreatorHandle(runId: string): string | null {
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`SELECT DISTINCT created_by_terminal_handle AS handle
|
||||
FROM tasks
|
||||
WHERE run_id = ? AND created_by_terminal_handle IS NOT NULL
|
||||
LIMIT 2`
|
||||
)
|
||||
.all(runId) as { handle: string }[]
|
||||
return rows.length === 1 ? (rows[0]?.handle ?? null) : null
|
||||
}
|
||||
|
||||
private requireCurrentConsumer(runId: string, consumerGeneration: number): RunRow {
|
||||
const run = this.getRunRaw(runId)
|
||||
if (!run || run.legacy === 1 || run.consumer_generation !== consumerGeneration) {
|
||||
@@ -2269,12 +2416,20 @@ export class OrchestrationDb {
|
||||
let message = params.message.existingId
|
||||
? this.getMessageById(params.message.existingId)
|
||||
: undefined
|
||||
const delivery = this.resolveLegacyWorkerCoordinatorDelivery(
|
||||
principal.run_id,
|
||||
params.message.to
|
||||
)
|
||||
if (params.message.existingId) {
|
||||
const matchesOriginalLegacyRoute =
|
||||
message?.delivery_contract === 'legacy_direct' && message.to_handle === params.message.to
|
||||
const matchesCurrentRoute =
|
||||
message?.delivery_contract === delivery.contract && message.to_handle === delivery.to
|
||||
if (
|
||||
!message ||
|
||||
message.run_id !== principal.run_id ||
|
||||
message.delivery_contract !== 'legacy_direct' ||
|
||||
message.to_handle !== params.message.to
|
||||
message.from_handle !== principal.terminal_handle ||
|
||||
(!matchesOriginalLegacyRoute && !matchesCurrentRoute)
|
||||
) {
|
||||
throw new OrchestrationError(
|
||||
'request_mismatch',
|
||||
@@ -2284,7 +2439,7 @@ export class OrchestrationDb {
|
||||
} else {
|
||||
message = this.insertMessage({
|
||||
from: principal.terminal_handle,
|
||||
to: params.message.to,
|
||||
to: delivery.to,
|
||||
subject: params.message.subject,
|
||||
body: params.message.body,
|
||||
type: params.message.type,
|
||||
@@ -2292,7 +2447,7 @@ export class OrchestrationDb {
|
||||
payload: params.message.payload,
|
||||
senderPaneKey: principal.pane_key,
|
||||
runId: principal.run_id,
|
||||
deliveryContract: 'legacy_direct'
|
||||
deliveryContract: delivery.contract
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2300,12 +2455,24 @@ export class OrchestrationDb {
|
||||
if (params.lifecycle.kind === 'heartbeat') {
|
||||
this.recordHeartbeat(dispatchId, params.lifecycle.at)
|
||||
} else if (params.lifecycle.kind === 'worker_report') {
|
||||
settlement = this.settleWorkerReportInTransaction({
|
||||
taskId: params.lifecycle.taskId,
|
||||
dispatchId,
|
||||
outcome: params.lifecycle.outcome,
|
||||
result: params.lifecycle.result
|
||||
})
|
||||
const persistedOutcome =
|
||||
params.message.existingId &&
|
||||
dispatch.task_id === params.lifecycle.taskId &&
|
||||
dispatch.status === 'completed'
|
||||
? 'succeeded'
|
||||
: params.message.existingId &&
|
||||
dispatch.task_id === params.lifecycle.taskId &&
|
||||
dispatch.status === 'failed'
|
||||
? 'failed'
|
||||
: undefined
|
||||
settlement = persistedOutcome
|
||||
? { action: 'settled', outcome: persistedOutcome, duplicate: true }
|
||||
: this.settleWorkerReportInTransaction({
|
||||
taskId: params.lifecycle.taskId,
|
||||
dispatchId,
|
||||
outcome: params.lifecycle.outcome,
|
||||
result: params.lifecycle.result
|
||||
})
|
||||
if (settlement.action === 'rejected') {
|
||||
throw new OrchestrationError(settlement.code, settlement.reason)
|
||||
}
|
||||
@@ -2380,24 +2547,36 @@ export class OrchestrationDb {
|
||||
)
|
||||
}
|
||||
|
||||
const existingQuestionId =
|
||||
params.existingQuestionId &&
|
||||
!this.db
|
||||
.prepare(
|
||||
`SELECT 1 FROM legacy_operation_receipts
|
||||
WHERE principal_id = ? AND method = 'orchestration.ask' AND effect_id = ?
|
||||
LIMIT 1`
|
||||
)
|
||||
.get(principal.id, params.existingQuestionId)
|
||||
? params.existingQuestionId
|
||||
: undefined
|
||||
let question: QuestionRow
|
||||
let message: MessageRow
|
||||
if (params.existingQuestionId) {
|
||||
const existingQuestion = this.getQuestion(params.existingQuestionId)
|
||||
const existingMessage = this.getMessageById(params.existingQuestionId)
|
||||
const delivery = this.resolveLegacyWorkerCoordinatorDelivery(
|
||||
principal.run_id,
|
||||
params.recipientHandle
|
||||
)
|
||||
if (existingQuestionId) {
|
||||
const existingQuestion = this.getQuestion(existingQuestionId)
|
||||
const existingMessage = this.getMessageById(existingQuestionId)
|
||||
if (
|
||||
!existingQuestion ||
|
||||
!existingMessage ||
|
||||
existingQuestion.run_id !== principal.run_id ||
|
||||
existingQuestion.dispatch_id !== dispatchId ||
|
||||
existingQuestion.status !== 'pending' ||
|
||||
existingMessage.delivery_contract !== 'legacy_direct' ||
|
||||
!legacyMessageMatchesQuestion(
|
||||
existingMessage,
|
||||
params.question,
|
||||
params.options ?? [],
|
||||
params.recipientHandle
|
||||
)
|
||||
existingMessage.delivery_contract !== delivery.contract ||
|
||||
!legacyMessageMatchesQuestion(existingMessage, params.question, params.options ?? [], [
|
||||
delivery.to
|
||||
])
|
||||
) {
|
||||
throw new OrchestrationError(
|
||||
'request_mismatch',
|
||||
@@ -2409,7 +2588,7 @@ export class OrchestrationDb {
|
||||
} else {
|
||||
message = this.insertMessage({
|
||||
from: principal.terminal_handle,
|
||||
to: params.recipientHandle,
|
||||
to: delivery.to,
|
||||
subject: 'Question',
|
||||
body: params.question,
|
||||
type: 'question',
|
||||
@@ -2421,7 +2600,7 @@ export class OrchestrationDb {
|
||||
}),
|
||||
senderPaneKey: principal.pane_key,
|
||||
runId: principal.run_id,
|
||||
deliveryContract: 'legacy_direct'
|
||||
deliveryContract: delivery.contract
|
||||
})
|
||||
this.db
|
||||
.prepare('UPDATE messages SET thread_id = ? WHERE id = ?')
|
||||
@@ -2473,21 +2652,37 @@ export class OrchestrationDb {
|
||||
question: QuestionRow
|
||||
message: MessageRow
|
||||
answerAcknowledged: boolean
|
||||
claimedByOperation: boolean
|
||||
}[] {
|
||||
const principal = this.requireCommittedLegacyPrincipal(params.principalId, 'worker')
|
||||
const runAddress = `run:${principal.run_id}`
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`SELECT q.*, m.id AS source_message_id
|
||||
`SELECT q.*, m.id AS source_message_id,
|
||||
EXISTS(
|
||||
SELECT 1 FROM legacy_operation_receipts lor
|
||||
WHERE lor.principal_id = ? AND lor.method = 'orchestration.ask'
|
||||
AND lor.effect_id = q.message_id
|
||||
) AS claimed_by_operation
|
||||
FROM question_threads q
|
||||
INNER JOIN messages m ON m.id = q.message_id
|
||||
WHERE q.run_id = ? AND q.dispatch_id = ?
|
||||
AND m.delivery_contract = 'legacy_direct'
|
||||
AND m.to_handle = ?
|
||||
AND (
|
||||
(m.delivery_contract = 'legacy_direct' AND m.to_handle = ?) OR
|
||||
(m.delivery_contract = 'current_delivery' AND m.to_handle = ?)
|
||||
)
|
||||
ORDER BY m.sequence
|
||||
LIMIT 501`
|
||||
)
|
||||
.all(principal.run_id, principal.dispatch_id, params.recipientHandle) as (QuestionRow & {
|
||||
.all(
|
||||
principal.id,
|
||||
principal.run_id,
|
||||
principal.dispatch_id,
|
||||
params.recipientHandle,
|
||||
runAddress
|
||||
) as (QuestionRow & {
|
||||
source_message_id: string
|
||||
claimed_by_operation: number
|
||||
})[]
|
||||
if (rows.length > 500) {
|
||||
throw new OrchestrationError(
|
||||
@@ -2500,17 +2695,16 @@ export class OrchestrationDb {
|
||||
const message = this.getMessageById(row.source_message_id)
|
||||
return Boolean(
|
||||
message &&
|
||||
legacyMessageMatchesQuestion(
|
||||
message,
|
||||
params.question,
|
||||
params.options ?? [],
|
||||
params.recipientHandle
|
||||
)
|
||||
legacyMessageMatchesQuestion(message, params.question, params.options ?? [], [
|
||||
params.recipientHandle,
|
||||
runAddress
|
||||
])
|
||||
)
|
||||
})
|
||||
.map((row) => ({
|
||||
question: exposeQuestionTimestamps(row),
|
||||
message: this.getMessageById(row.message_id) as MessageRow,
|
||||
claimedByOperation: row.claimed_by_operation === 1,
|
||||
answerAcknowledged: row.answer_message_id
|
||||
? Boolean(
|
||||
this.db
|
||||
@@ -2525,6 +2719,22 @@ export class OrchestrationDb {
|
||||
}))
|
||||
}
|
||||
|
||||
private resolveLegacyWorkerCoordinatorDelivery(
|
||||
runId: string,
|
||||
retainedCoordinatorHandle: string
|
||||
): { to: string; contract: MessageDeliveryContract } {
|
||||
const run = this.getRunRaw(runId)
|
||||
const principal = this.getLegacyCoordinatorPrincipal(runId)
|
||||
const takenOver =
|
||||
run?.coordinator_handle !== null &&
|
||||
(principal?.status === 'revoked' ||
|
||||
(run?.coordinator_handle !== undefined &&
|
||||
run.coordinator_handle !== retainedCoordinatorHandle))
|
||||
return takenOver
|
||||
? { to: `run:${runId}`, contract: 'current_delivery' }
|
||||
: { to: retainedCoordinatorHandle, contract: 'legacy_direct' }
|
||||
}
|
||||
|
||||
commitLegacyReplyOperation(params: {
|
||||
principalId: string
|
||||
operationKey: string
|
||||
@@ -3658,11 +3868,10 @@ export class OrchestrationDb {
|
||||
wd.agent_terminal_handle
|
||||
FROM dispatch_contexts dc
|
||||
INNER JOIN worker_dispatches wd ON wd.dispatch_id = dc.id
|
||||
WHERE dc.contract_version = ?
|
||||
AND wd.state IN ('starting', 'ready', 'start_unknown', 'stopping', 'stop_unknown')
|
||||
WHERE wd.state IN ('starting', 'ready', 'start_unknown', 'stopping', 'stop_unknown')
|
||||
ORDER BY dc.rowid`
|
||||
)
|
||||
.all(LEGACY_CONTRACT_VERSION) as LegacyWorkerTerminalRecoveryRow[]
|
||||
.all() as LegacyWorkerTerminalRecoveryRow[]
|
||||
}
|
||||
|
||||
getFederatedDispatch(dispatchId: string): FederatedDispatchRow | undefined {
|
||||
|
||||
@@ -77,6 +77,14 @@ describe('formatMessageBanner', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('lets the CLI resolve the live sender for Run and Dispatch addresses', () => {
|
||||
for (const to_handle of ['run:run_test', 'dispatch:dispatch_test']) {
|
||||
const banner = formatMessageBanner(makeMessage({ to_handle }))
|
||||
expect(banner).toContain('[Reply: orca orchestration reply --id msg_test1 --body "..."]')
|
||||
expect(banner).not.toContain(`--from ${to_handle}`)
|
||||
}
|
||||
})
|
||||
|
||||
it('marks legacy messages read-only without reply or acknowledgment affordances', () => {
|
||||
const banner = formatMessageBanner(
|
||||
makeMessage({ id: 'msg_legacy', run_id: 'run_legacy_local' })
|
||||
|
||||
@@ -86,10 +86,11 @@ export function formatMessageBanner(
|
||||
}
|
||||
|
||||
if (authority === 'current') {
|
||||
// Why: older shells can lack Orca's terminal identity environment.
|
||||
lines.push(
|
||||
`[Reply: orca orchestration reply --id ${msg.id} --from ${msg.to_handle} --body "..."]`
|
||||
)
|
||||
const explicitFrom =
|
||||
msg.to_handle.startsWith('run:') || msg.to_handle.startsWith('dispatch:')
|
||||
? ''
|
||||
: ` --from ${msg.to_handle}`
|
||||
lines.push(`[Reply: orca orchestration reply --id ${msg.id}${explicitFrom} --body "..."]`)
|
||||
}
|
||||
lines.push(SEPARATOR)
|
||||
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { rmSync } from 'node:fs'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import Database from '../../sqlite/sync-database'
|
||||
import {
|
||||
@@ -9,21 +7,10 @@ import {
|
||||
LEGACY_RUN_ID,
|
||||
OrchestrationDb
|
||||
} from './db'
|
||||
|
||||
type CutoverFixture = {
|
||||
dbPath: string
|
||||
currentRunId: string
|
||||
unrelatedRunId: string
|
||||
currentDispatchId: string
|
||||
legacyTaskId: string
|
||||
legacyDispatchId: string
|
||||
legacyGateId: string
|
||||
legacyMessageIds: string[]
|
||||
legacyQuestionId: string
|
||||
legacyDeliveryId: string
|
||||
rejectionMessageId: string
|
||||
lookalikeMessageId: string
|
||||
}
|
||||
import {
|
||||
createLegacyStorageCutoverFixture,
|
||||
type LegacyStorageCutoverFixture
|
||||
} from './orchestration-legacy-storage-test-fixture'
|
||||
|
||||
describe('OrchestrationDb legacy contract storage', () => {
|
||||
let db: OrchestrationDb | undefined
|
||||
@@ -38,129 +25,14 @@ describe('OrchestrationDb legacy contract storage', () => {
|
||||
}
|
||||
})
|
||||
|
||||
function createCutoverFixture(): CutoverFixture {
|
||||
tempDir = mkdtempSync(join(tmpdir(), 'orca-legacy-storage-'))
|
||||
const dbPath = join(tempDir, 'orchestration.db')
|
||||
const first = new OrchestrationDb(dbPath)
|
||||
const currentRun = first.createRun({
|
||||
objective: 'Current work',
|
||||
coordinatorHandle: 'term_current_coord',
|
||||
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111'
|
||||
})
|
||||
const currentTask = first.createTask({ spec: 'current', runId: currentRun.id })
|
||||
const unrelatedRun = first.createRun({
|
||||
objective: 'Unrelated current work',
|
||||
coordinatorHandle: 'term_unrelated_coord',
|
||||
coordinatorPaneKey: 'tab_unrelated:55555555-5555-4555-8555-555555555555'
|
||||
})
|
||||
const currentDispatch = first.createDispatchContext(
|
||||
currentTask.id,
|
||||
'term_current_worker',
|
||||
'tab_current:22222222-2222-4222-9222-222222222222',
|
||||
'current_launch_hash'
|
||||
)
|
||||
first.insertMessage({
|
||||
runId: currentRun.id,
|
||||
from: 'term_current_worker',
|
||||
to: `run:${currentRun.id}`,
|
||||
subject: 'current mail'
|
||||
})
|
||||
|
||||
const legacyTask = first.createTask({
|
||||
spec: 'legacy',
|
||||
createdByTerminalHandle: 'term_legacy_coord'
|
||||
})
|
||||
first.createDispatchContext(
|
||||
legacyTask.id,
|
||||
'term_legacy_worker',
|
||||
'tab_legacy:33333333-3333-4333-8333-333333333333'
|
||||
)
|
||||
const legacyGate = first.createGate({
|
||||
taskId: legacyTask.id,
|
||||
question: 'Retained gate?'
|
||||
})
|
||||
first.resolveGate(legacyGate.id, 'continue')
|
||||
const retryDispatch = first.createDispatchContext(
|
||||
legacyTask.id,
|
||||
'term_legacy_worker',
|
||||
'tab_legacy:33333333-3333-4333-8333-333333333333'
|
||||
)
|
||||
const legacyMessages = [
|
||||
first.insertMessage({
|
||||
from: 'term_legacy_coord',
|
||||
to: 'term_legacy_worker',
|
||||
subject: 'read worker mail'
|
||||
}),
|
||||
first.insertMessage({
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'read coordinator mail'
|
||||
}),
|
||||
first.insertMessage({
|
||||
from: 'term_legacy_coord',
|
||||
to: 'term_legacy_worker',
|
||||
subject: 'second worker page'
|
||||
})
|
||||
]
|
||||
first.markAsRead(legacyMessages.map((message) => message.id))
|
||||
const question = first.createQuestion({
|
||||
runId: LEGACY_RUN_ID,
|
||||
dispatchId: retryDispatch.id,
|
||||
askerHandle: 'term_legacy_worker',
|
||||
question: 'Retained question?'
|
||||
})
|
||||
const rejection = first.insertMessage({
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'Rejected heartbeat',
|
||||
type: 'heartbeat',
|
||||
payload: JSON.stringify({ _orcaLifecycleRejection: { code: 'migration', reason: 'cutover' } })
|
||||
})
|
||||
const lookalike = first.insertMessage({
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'Ordinary legacy mail',
|
||||
payload: JSON.stringify({ '.orcaLifecycleRejection': 'not an audit marker' })
|
||||
})
|
||||
first.close()
|
||||
|
||||
const raw = new Database(dbPath)
|
||||
const legacyDeliveryId = 'delivery_legacy_outstanding'
|
||||
raw
|
||||
.prepare(
|
||||
`INSERT INTO deliveries (
|
||||
id, run_id, consumer_generation, message_ids, status
|
||||
) VALUES (?, ?, 0, ?, 'outstanding')`
|
||||
)
|
||||
.run(legacyDeliveryId, LEGACY_RUN_ID, JSON.stringify([legacyMessages[0].id]))
|
||||
raw.exec(`
|
||||
DROP INDEX IF EXISTS idx_messages_delivery_contract;
|
||||
DROP TABLE legacy_mail_receipts;
|
||||
DROP TABLE legacy_operation_receipts;
|
||||
DROP TABLE legacy_compatibility_principals;
|
||||
DROP TABLE legacy_adoptions;
|
||||
`)
|
||||
raw.pragma('user_version = 18')
|
||||
raw.close()
|
||||
|
||||
return {
|
||||
dbPath,
|
||||
currentRunId: currentRun.id,
|
||||
unrelatedRunId: unrelatedRun.id,
|
||||
currentDispatchId: currentDispatch.id,
|
||||
legacyTaskId: legacyTask.id,
|
||||
legacyDispatchId: retryDispatch.id,
|
||||
legacyGateId: legacyGate.id,
|
||||
legacyMessageIds: legacyMessages.map((message) => message.id),
|
||||
legacyQuestionId: question.message.id,
|
||||
legacyDeliveryId,
|
||||
rejectionMessageId: rejection.id,
|
||||
lookalikeMessageId: lookalike.id
|
||||
}
|
||||
function createCutoverFixture(): LegacyStorageCutoverFixture {
|
||||
const created = createLegacyStorageCutoverFixture()
|
||||
tempDir = created.tempDir
|
||||
return created.fixture
|
||||
}
|
||||
|
||||
function openAdoptedFixture(): {
|
||||
fixture: CutoverFixture
|
||||
fixture: LegacyStorageCutoverFixture
|
||||
adoptedRunId: string
|
||||
workerPrincipalId: string
|
||||
coordinatorPrincipalId: string
|
||||
@@ -411,13 +283,19 @@ describe('OrchestrationDb legacy contract storage', () => {
|
||||
coordinatorHandle: 'term_other',
|
||||
coordinatorPaneKey: 'tab_other:55555555-5555-4555-8555-555555555555'
|
||||
})
|
||||
).toThrow(/only its attested coordinator/)
|
||||
).toThrow(/attested coordinator may rebind/)
|
||||
|
||||
const bound = db!.bindRun({
|
||||
runId: state.adoptedRunId,
|
||||
coordinatorHandle: 'term_legacy_coord',
|
||||
coordinatorPaneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
|
||||
allowLegacyCompatibility: true
|
||||
legacyCoordinatorAuthority: {
|
||||
runId: state.adoptedRunId,
|
||||
principalId: state.coordinatorPrincipalId,
|
||||
terminalHandle: 'term_legacy_coord',
|
||||
paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
|
||||
consumerGeneration: 0
|
||||
}
|
||||
})
|
||||
expect(bound).toMatchObject({ coordinator_handle: 'term_legacy_coord' })
|
||||
expect(
|
||||
@@ -429,6 +307,99 @@ describe('OrchestrationDb legacy contract storage', () => {
|
||||
).toMatchObject({ coordinator_handle: 'term_legacy_coord' })
|
||||
})
|
||||
|
||||
it.each([
|
||||
['generation', { consumerGeneration: 1 }],
|
||||
['principal', { principalId: 'legacy_principal_wrong' }],
|
||||
['handle', { terminalHandle: 'term_wrong' }],
|
||||
['pane', { paneKey: 'tab_wrong:99999999-9999-4999-8999-999999999999' }]
|
||||
] as const)('rejects stale legacy coordinator %s proof inside Run binding', (_label, patch) => {
|
||||
const state = openAdoptedFixture()
|
||||
const before = db!.getRun(state.adoptedRunId)
|
||||
|
||||
expect(() =>
|
||||
db!.bindRun({
|
||||
runId: state.adoptedRunId,
|
||||
coordinatorHandle: 'term_legacy_coord',
|
||||
coordinatorPaneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
|
||||
legacyCoordinatorAuthority: {
|
||||
runId: state.adoptedRunId,
|
||||
principalId: state.coordinatorPrincipalId,
|
||||
terminalHandle: 'term_legacy_coord',
|
||||
paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
|
||||
consumerGeneration: 0,
|
||||
...patch
|
||||
}
|
||||
})
|
||||
).toThrow(/no longer has lifecycle authority/)
|
||||
expect(db!.getRun(state.adoptedRunId)).toEqual(before)
|
||||
})
|
||||
|
||||
it('revokes the legacy coordinator when a current coordinator takes over after settlement', () => {
|
||||
const state = openAdoptedFixture()
|
||||
db!.settleWorkerReport({
|
||||
taskId: state.fixture.legacyTaskId,
|
||||
dispatchId: state.fixture.legacyDispatchId,
|
||||
outcome: 'succeeded',
|
||||
result: 'done'
|
||||
})
|
||||
|
||||
expect(
|
||||
db!.bindRun({
|
||||
runId: state.adoptedRunId,
|
||||
coordinatorHandle: 'term_current_coord',
|
||||
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111'
|
||||
})
|
||||
).toMatchObject({ coordinator_handle: 'term_current_coord' })
|
||||
expect(db!.getLegacyCompatibilityPrincipal(state.coordinatorPrincipalId)?.status).toBe(
|
||||
'revoked'
|
||||
)
|
||||
expect(() =>
|
||||
db!.commitLegacyCompatibilityPrincipal({
|
||||
runId: state.adoptedRunId,
|
||||
role: 'coordinator',
|
||||
hostScope: 'local:runtime_1',
|
||||
terminalHandle: 'term_legacy_coord',
|
||||
paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
|
||||
launchTokenHash: 'coord_launch_hash',
|
||||
processIncarnation: 'process_coord'
|
||||
})
|
||||
).toThrow(/revoked/)
|
||||
})
|
||||
|
||||
it('promotes only mail addressed to the replaced legacy coordinator', () => {
|
||||
const state = openAdoptedFixture()
|
||||
const coordinatorMail = db!.insertMessage({
|
||||
runId: state.adoptedRunId,
|
||||
deliveryContract: 'legacy_direct',
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'coordinator outcome'
|
||||
})
|
||||
const workerMail = db!.insertMessage({
|
||||
runId: state.adoptedRunId,
|
||||
deliveryContract: 'legacy_direct',
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_other_worker',
|
||||
subject: 'worker-only guidance'
|
||||
})
|
||||
|
||||
db!.bindRun({
|
||||
runId: state.adoptedRunId,
|
||||
coordinatorHandle: 'term_current_coord',
|
||||
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111',
|
||||
takeoverLegacy: true
|
||||
})
|
||||
|
||||
expect(db!.getMessageById(coordinatorMail.id)).toMatchObject({
|
||||
to_handle: `run:${state.adoptedRunId}`,
|
||||
delivery_contract: 'current_delivery'
|
||||
})
|
||||
expect(db!.getMessageById(workerMail.id)).toMatchObject({
|
||||
to_handle: 'term_other_worker',
|
||||
delivery_contract: 'legacy_direct'
|
||||
})
|
||||
})
|
||||
|
||||
it('commits legacy messages, lifecycle effects, and invocation receipts exactly once', () => {
|
||||
const state = openAdoptedFixture()
|
||||
const params = {
|
||||
@@ -508,6 +479,65 @@ describe('OrchestrationDb legacy contract storage', () => {
|
||||
expect(db!.getLegacyCompatibilityPrincipal(state.workerPrincipalId)?.status).toBe('settled')
|
||||
})
|
||||
|
||||
it('reconstructs a read pre-takeover completion through its original legacy route', () => {
|
||||
const state = openAdoptedFixture()
|
||||
const taskId = db!.getDispatchContextById(state.fixture.legacyDispatchId)!.task_id
|
||||
const accepted = db!.insertMessage({
|
||||
runId: state.adoptedRunId,
|
||||
deliveryContract: 'legacy_direct',
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'Completed',
|
||||
body: 'accepted before takeover',
|
||||
type: 'worker_done'
|
||||
})
|
||||
db!.markAsRead([accepted.id])
|
||||
db!.settleWorkerReport({
|
||||
taskId,
|
||||
dispatchId: state.fixture.legacyDispatchId,
|
||||
outcome: 'succeeded',
|
||||
result: 'accepted before takeover'
|
||||
})
|
||||
db!.bindRun({
|
||||
runId: state.adoptedRunId,
|
||||
coordinatorHandle: 'term_current_coord',
|
||||
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111',
|
||||
takeoverLegacy: true
|
||||
})
|
||||
const beforeIds = db!.getInbox(100).map((message) => message.id)
|
||||
|
||||
const reconstructed = db!.commitLegacyLifecycleOperation({
|
||||
principalId: state.workerPrincipalId,
|
||||
operationKey: 'read_completion_after_takeover',
|
||||
method: 'orchestration.send',
|
||||
payloadHash: 'read_completion_after_takeover_payload',
|
||||
message: {
|
||||
existingId: accepted.id,
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'Completed',
|
||||
body: 'accepted before takeover',
|
||||
type: 'worker_done'
|
||||
},
|
||||
lifecycle: {
|
||||
kind: 'worker_report',
|
||||
taskId,
|
||||
outcome: 'succeeded',
|
||||
result: 'accepted before takeover'
|
||||
}
|
||||
})
|
||||
|
||||
expect(reconstructed).toMatchObject({
|
||||
duplicate: false,
|
||||
message: {
|
||||
id: accepted.id,
|
||||
to_handle: 'term_legacy_coord',
|
||||
delivery_contract: 'legacy_direct'
|
||||
},
|
||||
settlement: { action: 'settled', outcome: 'succeeded', duplicate: true }
|
||||
})
|
||||
expect(db!.getInbox(100).map((message) => message.id)).toEqual(beforeIds)
|
||||
})
|
||||
|
||||
it('rejects cross-cutover completion reconstruction for another recipient', () => {
|
||||
const state = openAdoptedFixture()
|
||||
const taskId = db!.getDispatchContextById(state.fixture.legacyDispatchId)!.task_id
|
||||
@@ -625,6 +655,38 @@ describe('OrchestrationDb legacy contract storage', () => {
|
||||
})
|
||||
).toHaveLength(2)
|
||||
|
||||
const inherited = db!.createQuestion({
|
||||
runId: state.adoptedRunId,
|
||||
dispatchId: state.fixture.legacyDispatchId,
|
||||
askerHandle: 'term_legacy_worker',
|
||||
question: 'Inherited?'
|
||||
})
|
||||
const sqlite = (db as unknown as { db: Database.Database }).db
|
||||
sqlite
|
||||
.prepare(
|
||||
`UPDATE messages
|
||||
SET from_handle = 'term_legacy_worker', to_handle = 'term_legacy_coord',
|
||||
delivery_contract = 'legacy_direct'
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run(inherited.message.id)
|
||||
const inheritedAsk = {
|
||||
...ask,
|
||||
operationKey: 'ask_inherited_1',
|
||||
payloadHash: 'ask_inherited_payload_1',
|
||||
question: 'Inherited?',
|
||||
options: [],
|
||||
existingQuestionId: inherited.message.id
|
||||
}
|
||||
const adopted = db!.commitLegacyAskOperation(inheritedAsk)
|
||||
const distinct = db!.commitLegacyAskOperation({
|
||||
...inheritedAsk,
|
||||
operationKey: 'ask_inherited_2',
|
||||
payloadHash: 'ask_inherited_payload_2'
|
||||
})
|
||||
expect(adopted.message.id).toBe(inherited.message.id)
|
||||
expect(distinct.message.id).not.toBe(inherited.message.id)
|
||||
|
||||
const reply = {
|
||||
principalId: state.coordinatorPrincipalId,
|
||||
operationKey: 'reply_invocation_1',
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
import { mkdtempSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import Database from '../../sqlite/sync-database'
|
||||
import { LEGACY_RUN_ID, OrchestrationDb } from './db'
|
||||
|
||||
export type LegacyStorageCutoverFixture = {
|
||||
dbPath: string
|
||||
currentRunId: string
|
||||
unrelatedRunId: string
|
||||
currentDispatchId: string
|
||||
legacyTaskId: string
|
||||
legacyDispatchId: string
|
||||
legacyGateId: string
|
||||
legacyMessageIds: string[]
|
||||
legacyQuestionId: string
|
||||
legacyDeliveryId: string
|
||||
rejectionMessageId: string
|
||||
lookalikeMessageId: string
|
||||
}
|
||||
|
||||
export function createLegacyStorageCutoverFixture(): {
|
||||
fixture: LegacyStorageCutoverFixture
|
||||
tempDir: string
|
||||
} {
|
||||
const tempDir = mkdtempSync(join(tmpdir(), 'orca-legacy-storage-'))
|
||||
const dbPath = join(tempDir, 'orchestration.db')
|
||||
const first = new OrchestrationDb(dbPath)
|
||||
const currentRun = first.createRun({
|
||||
objective: 'Current work',
|
||||
coordinatorHandle: 'term_current_coord',
|
||||
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111'
|
||||
})
|
||||
const currentTask = first.createTask({ spec: 'current', runId: currentRun.id })
|
||||
const unrelatedRun = first.createRun({
|
||||
objective: 'Unrelated current work',
|
||||
coordinatorHandle: 'term_unrelated_coord',
|
||||
coordinatorPaneKey: 'tab_unrelated:55555555-5555-4555-8555-555555555555'
|
||||
})
|
||||
const currentDispatch = first.createDispatchContext(
|
||||
currentTask.id,
|
||||
'term_current_worker',
|
||||
'tab_current:22222222-2222-4222-9222-222222222222',
|
||||
'current_launch_hash'
|
||||
)
|
||||
first.insertMessage({
|
||||
runId: currentRun.id,
|
||||
from: 'term_current_worker',
|
||||
to: `run:${currentRun.id}`,
|
||||
subject: 'current mail'
|
||||
})
|
||||
|
||||
const legacyTask = first.createTask({
|
||||
spec: 'legacy',
|
||||
createdByTerminalHandle: 'term_legacy_coord'
|
||||
})
|
||||
first.createDispatchContext(
|
||||
legacyTask.id,
|
||||
'term_legacy_worker',
|
||||
'tab_legacy:33333333-3333-4333-8333-333333333333'
|
||||
)
|
||||
const legacyGate = first.createGate({
|
||||
taskId: legacyTask.id,
|
||||
question: 'Retained gate?'
|
||||
})
|
||||
first.resolveGate(legacyGate.id, 'continue')
|
||||
const retryDispatch = first.createDispatchContext(
|
||||
legacyTask.id,
|
||||
'term_legacy_worker',
|
||||
'tab_legacy:33333333-3333-4333-8333-333333333333'
|
||||
)
|
||||
const legacyMessages = [
|
||||
first.insertMessage({
|
||||
from: 'term_legacy_coord',
|
||||
to: 'term_legacy_worker',
|
||||
subject: 'read worker mail'
|
||||
}),
|
||||
first.insertMessage({
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'read coordinator mail'
|
||||
}),
|
||||
first.insertMessage({
|
||||
from: 'term_legacy_coord',
|
||||
to: 'term_legacy_worker',
|
||||
subject: 'second worker page'
|
||||
})
|
||||
]
|
||||
first.markAsRead(legacyMessages.map((message) => message.id))
|
||||
const question = first.createQuestion({
|
||||
runId: LEGACY_RUN_ID,
|
||||
dispatchId: retryDispatch.id,
|
||||
askerHandle: 'term_legacy_worker',
|
||||
question: 'Retained question?'
|
||||
})
|
||||
const rejection = first.insertMessage({
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'Rejected heartbeat',
|
||||
type: 'heartbeat',
|
||||
payload: JSON.stringify({ _orcaLifecycleRejection: { code: 'migration', reason: 'cutover' } })
|
||||
})
|
||||
const lookalike = first.insertMessage({
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'Ordinary legacy mail',
|
||||
payload: JSON.stringify({ '.orcaLifecycleRejection': 'not an audit marker' })
|
||||
})
|
||||
first.close()
|
||||
|
||||
const raw = new Database(dbPath)
|
||||
const legacyDeliveryId = 'delivery_legacy_outstanding'
|
||||
raw
|
||||
.prepare(
|
||||
`INSERT INTO deliveries (
|
||||
id, run_id, consumer_generation, message_ids, status
|
||||
) VALUES (?, ?, 0, ?, 'outstanding')`
|
||||
)
|
||||
.run(legacyDeliveryId, LEGACY_RUN_ID, JSON.stringify([legacyMessages[0].id]))
|
||||
raw.exec(`
|
||||
DROP INDEX IF EXISTS idx_messages_delivery_contract;
|
||||
DROP TABLE legacy_mail_receipts;
|
||||
DROP TABLE legacy_operation_receipts;
|
||||
DROP TABLE legacy_compatibility_principals;
|
||||
DROP TABLE legacy_adoptions;
|
||||
`)
|
||||
raw.pragma('user_version = 18')
|
||||
raw.close()
|
||||
|
||||
return {
|
||||
tempDir,
|
||||
fixture: {
|
||||
dbPath,
|
||||
currentRunId: currentRun.id,
|
||||
unrelatedRunId: unrelatedRun.id,
|
||||
currentDispatchId: currentDispatch.id,
|
||||
legacyTaskId: legacyTask.id,
|
||||
legacyDispatchId: retryDispatch.id,
|
||||
legacyGateId: legacyGate.id,
|
||||
legacyMessageIds: legacyMessages.map((message) => message.id),
|
||||
legacyQuestionId: question.message.id,
|
||||
legacyDeliveryId,
|
||||
rejectionMessageId: rejection.id,
|
||||
lookalikeMessageId: lookalike.id
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
-2
@@ -29,7 +29,8 @@ describe('legacy worker terminal recovery planning', () => {
|
||||
blockedPanes: [
|
||||
{
|
||||
worktreeId: 'repo::/workspace',
|
||||
paneKey: `tab-worker:${LEAF_ID}`
|
||||
paneKey: `tab-worker:${LEAF_ID}`,
|
||||
contractVersion: 0
|
||||
}
|
||||
],
|
||||
candidates: [
|
||||
@@ -50,7 +51,8 @@ describe('legacy worker terminal recovery planning', () => {
|
||||
blockedPanes: [
|
||||
{
|
||||
worktreeId: 'repo::/workspace',
|
||||
paneKey: `tab-worker:${LEAF_ID}`
|
||||
paneKey: `tab-worker:${LEAF_ID}`,
|
||||
contractVersion: 0
|
||||
}
|
||||
],
|
||||
candidates: [],
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { LegacyWorkerTerminalRecoveryRow } from './types'
|
||||
|
||||
export type LegacyWorkerTerminalRecoveryCandidate = {
|
||||
dispatchId: string
|
||||
contractVersion: number
|
||||
taskId: string
|
||||
worktreeId: string
|
||||
terminalHandle: string
|
||||
@@ -16,7 +17,7 @@ export type LegacyWorkerTerminalRecoveryCandidate = {
|
||||
}
|
||||
|
||||
export type LegacyWorkerTerminalRecoveryPlan = {
|
||||
blockedPanes: { worktreeId: string; paneKey: string }[]
|
||||
blockedPanes: { worktreeId: string; paneKey: string; contractVersion: number }[]
|
||||
candidates: LegacyWorkerTerminalRecoveryCandidate[]
|
||||
ambiguousDispatchIds: string[]
|
||||
}
|
||||
@@ -48,14 +49,21 @@ function countCandidateKeys(
|
||||
export function planLegacyWorkerTerminalRecovery(
|
||||
rows: readonly LegacyWorkerTerminalRecoveryRow[]
|
||||
): LegacyWorkerTerminalRecoveryPlan {
|
||||
const blockedPanes = new Map<string, { worktreeId: string; paneKey: string }>()
|
||||
const blockedPanes = new Map<
|
||||
string,
|
||||
{ worktreeId: string; paneKey: string; contractVersion: number }
|
||||
>()
|
||||
const parsedCandidates: LegacyWorkerTerminalRecoveryCandidate[] = []
|
||||
for (const row of rows) {
|
||||
const worktreeId = row.worktree_id?.trim()
|
||||
const paneKey = row.assignee_pane_key?.trim()
|
||||
const pane = paneKey ? parsePaneKey(paneKey) : null
|
||||
if (worktreeId && paneKey && pane) {
|
||||
blockedPanes.set(`${worktreeId}\0${paneKey}`, { worktreeId, paneKey })
|
||||
blockedPanes.set(`${worktreeId}\0${paneKey}`, {
|
||||
worktreeId,
|
||||
paneKey,
|
||||
contractVersion: row.contract_version
|
||||
})
|
||||
}
|
||||
const terminalHandle = row.assignee_handle?.trim()
|
||||
const workerHandle = row.agent_terminal_handle?.trim()
|
||||
@@ -74,6 +82,7 @@ export function planLegacyWorkerTerminalRecovery(
|
||||
}
|
||||
parsedCandidates.push({
|
||||
dispatchId: row.dispatch_id,
|
||||
contractVersion: row.contract_version,
|
||||
taskId: row.task_id,
|
||||
worktreeId,
|
||||
terminalHandle,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { OrchestrationDb } from './db'
|
||||
import { CURRENT_CONTRACT_VERSION, OrchestrationDb } from './db'
|
||||
|
||||
describe('OrchestrationDb worker Dispatch state', () => {
|
||||
let db: OrchestrationDb | undefined
|
||||
@@ -44,6 +44,14 @@ describe('OrchestrationDb worker Dispatch state', () => {
|
||||
status: 'dispatched',
|
||||
assignee_handle: 'term_worker'
|
||||
})
|
||||
expect(d.listLegacyWorkerTerminalRecoveryRows()).toEqual([
|
||||
expect.objectContaining({
|
||||
dispatch_id: started.dispatch.id,
|
||||
contract_version: CURRENT_CONTRACT_VERSION,
|
||||
worker_state: 'ready',
|
||||
agent_terminal_handle: 'term_worker'
|
||||
})
|
||||
])
|
||||
})
|
||||
|
||||
it('commits worker-start mutation acceptance with the starting Dispatch', () => {
|
||||
|
||||
@@ -53,6 +53,14 @@ export type RpcRequest = {
|
||||
orchestrationCompatibilityEvidence?: OrchestrationCompatibilityEvidence
|
||||
}
|
||||
|
||||
export type LegacyCoordinatorAuthorityProof = Readonly<{
|
||||
runId: string
|
||||
principalId: string | null
|
||||
terminalHandle: string
|
||||
paneKey: string
|
||||
consumerGeneration: number
|
||||
}>
|
||||
|
||||
export type RpcContext = {
|
||||
runtime: OrcaRuntimeService
|
||||
// Why: lets long-poll handlers release immediately on client disconnect instead of running down timeoutMs. See design doc §3.1.
|
||||
@@ -82,6 +90,8 @@ export type RpcContext = {
|
||||
}
|
||||
// Why: only the compatibility authority router can set this trusted scope; user params cannot bypass Run consumer binding.
|
||||
legacyCoordinatorRunId?: string
|
||||
legacyCoordinatorAuthority?: LegacyCoordinatorAuthorityProof
|
||||
revalidateLegacyCoordinator?: () => string
|
||||
// Why: federation pins the authenticated saved-environment caller without exposing its token to handlers or storage.
|
||||
authenticatedCallerFingerprint?: string
|
||||
pairing?: PairingRpcContext
|
||||
|
||||
@@ -46,6 +46,10 @@ function makeRuntime(ui: PersistedUIState = getDefaultUIState()): OrcaRuntimeSer
|
||||
updateUIState: vi.fn((updates: Partial<PersistedUIState>) => {
|
||||
currentUI = { ...currentUI, ...updates }
|
||||
return currentUI
|
||||
}),
|
||||
getOrchestrationDb: () => ({
|
||||
getLegacyAdoption: () => undefined,
|
||||
resolveLegacyWorkerCandidate: () => undefined
|
||||
})
|
||||
} as unknown as OrcaRuntimeService
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { RuntimeCapability } from '../../../shared/protocol-version'
|
||||
import type { TerminalStreamFrame } from '../../../shared/terminal-stream-protocol'
|
||||
import type { PairingRpcContext } from './core'
|
||||
|
||||
export type RpcDispatchStreamingOptions = {
|
||||
connectionId?: string
|
||||
signal?: AbortSignal
|
||||
clientId?: string
|
||||
pairedDeviceId?: string
|
||||
clientKind?: 'mobile' | 'runtime'
|
||||
clientCapabilities?: readonly RuntimeCapability[]
|
||||
pairing?: PairingRpcContext
|
||||
sendBinary?: (bytes: Uint8Array<ArrayBufferLike>) => boolean | void
|
||||
registerBinaryStreamHandler?: (
|
||||
streamId: number,
|
||||
handler: (frame: TerminalStreamFrame) => void
|
||||
) => () => void
|
||||
}
|
||||
@@ -10,13 +10,11 @@ import {
|
||||
isStreamingMethod,
|
||||
type RpcAnyMethod,
|
||||
type RpcEnvelopeMeta,
|
||||
type PairingRpcContext,
|
||||
type RpcRegistry,
|
||||
type RpcRequest,
|
||||
type RpcResponse
|
||||
} from './core'
|
||||
|
||||
import type { TerminalStreamFrame } from '../../../shared/terminal-stream-protocol'
|
||||
import type { FeatureInteractionId } from '../../../shared/feature-interactions'
|
||||
import {
|
||||
computerErrorData,
|
||||
@@ -29,20 +27,18 @@ import {
|
||||
import { ALL_RPC_METHODS } from './methods'
|
||||
import { emulatorProbe, emulatorProbeError } from '../../emulator/emulator-probe'
|
||||
import type { OrcaRuntimeService } from '../orca-runtime'
|
||||
import type { RuntimeCapability } from '../../../shared/protocol-version'
|
||||
import {
|
||||
OrchestrationMutationExecutor,
|
||||
authenticatedCallerFingerprint,
|
||||
getOrchestrationMutationExecutor,
|
||||
type OrchestrationMutationExecutor,
|
||||
type DurableMutationInvocation
|
||||
} from './orchestration-mutation-executor'
|
||||
import { orchestrationMigrationFence } from './orchestration-contract-fence'
|
||||
import { getRuntimeFeatureInteractionId } from './runtime-feature-interaction'
|
||||
import { recordRuntimeFeatureInteraction } from './runtime-feature-interaction'
|
||||
import { OrchestrationLegacyCompatibility } from './orchestration-legacy-compatibility'
|
||||
import type { RpcDispatchStreamingOptions } from './dispatcher-stream-options'
|
||||
|
||||
export type DispatcherOptions = {
|
||||
runtime: OrcaRuntimeService
|
||||
methods?: readonly RpcAnyMethod[]
|
||||
}
|
||||
export type DispatcherOptions = { runtime: OrcaRuntimeService; methods?: readonly RpcAnyMethod[] }
|
||||
|
||||
export class RpcDispatcher {
|
||||
private readonly runtime: OrcaRuntimeService
|
||||
@@ -53,7 +49,7 @@ export class RpcDispatcher {
|
||||
constructor({ runtime, methods = ALL_RPC_METHODS }: DispatcherOptions) {
|
||||
this.runtime = runtime
|
||||
this.registry = buildRegistry(methods)
|
||||
this.orchestrationMutations = new OrchestrationMutationExecutor(runtime)
|
||||
this.orchestrationMutations = getOrchestrationMutationExecutor(runtime)
|
||||
this.legacyOrchestration = new OrchestrationLegacyCompatibility(runtime)
|
||||
}
|
||||
|
||||
@@ -91,8 +87,7 @@ export class RpcDispatcher {
|
||||
)
|
||||
}
|
||||
|
||||
const isEmulator = request.method.startsWith('emulator.')
|
||||
if (isEmulator) {
|
||||
if (request.method.startsWith('emulator.')) {
|
||||
emulatorProbe(`rpc ${request.method}`, request.params)
|
||||
}
|
||||
try {
|
||||
@@ -105,8 +100,13 @@ export class RpcDispatcher {
|
||||
return successResponse(request.id, meta, compatibility.result)
|
||||
}
|
||||
const effectiveParams = compatibility.params ?? parsedParams.value
|
||||
const invoke = (mutation?: DurableMutationInvocation) =>
|
||||
method.handler(effectiveParams, {
|
||||
const legacyCoordinator = this.legacyOrchestration.createCoordinatorInvocation(
|
||||
request,
|
||||
compatibility.legacyCoordinatorAuthority
|
||||
)
|
||||
const invoke = (mutation?: DurableMutationInvocation) => {
|
||||
const legacyCoordinatorRunId = legacyCoordinator?.revalidate()
|
||||
return method.handler(effectiveParams, {
|
||||
runtime: this.runtime,
|
||||
signal: options?.signal,
|
||||
requestId: request.id,
|
||||
@@ -114,13 +114,22 @@ export class RpcDispatcher {
|
||||
authenticatedCallerFingerprint: authenticatedCallerFingerprint(request),
|
||||
recordMutationReceipt: mutation?.recordReceipt,
|
||||
orchestrationMutation: mutation?.identity,
|
||||
legacyCoordinatorRunId: compatibility.legacyCoordinatorRunId
|
||||
legacyCoordinatorRunId,
|
||||
legacyCoordinatorAuthority: legacyCoordinator?.authority,
|
||||
revalidateLegacyCoordinator: legacyCoordinator?.revalidate
|
||||
})
|
||||
}
|
||||
const result = await this.orchestrationMutations.run(request, effectiveParams, invoke)
|
||||
this.recordRuntimeFeatureInteraction(request.method, result, undefined, request.params)
|
||||
recordRuntimeFeatureInteraction(
|
||||
this.runtime,
|
||||
request.method,
|
||||
result,
|
||||
undefined,
|
||||
request.params
|
||||
)
|
||||
return successResponse(request.id, meta, result)
|
||||
} catch (error) {
|
||||
if (isEmulator) {
|
||||
if (request.method.startsWith('emulator.')) {
|
||||
emulatorProbeError(`rpc ${request.method}`, error, { params: request.params })
|
||||
}
|
||||
return this.mapError(request, meta, error)
|
||||
@@ -133,20 +142,7 @@ export class RpcDispatcher {
|
||||
async dispatchStreaming(
|
||||
request: RpcRequest,
|
||||
reply: (response: string) => void,
|
||||
options?: {
|
||||
connectionId?: string
|
||||
signal?: AbortSignal
|
||||
clientId?: string
|
||||
pairedDeviceId?: string
|
||||
clientKind?: 'mobile' | 'runtime'
|
||||
clientCapabilities?: readonly RuntimeCapability[]
|
||||
pairing?: PairingRpcContext
|
||||
sendBinary?: (bytes: Uint8Array<ArrayBufferLike>) => boolean | void
|
||||
registerBinaryStreamHandler?: (
|
||||
streamId: number,
|
||||
handler: (frame: TerminalStreamFrame) => void
|
||||
) => () => void
|
||||
}
|
||||
options?: RpcDispatchStreamingOptions
|
||||
): Promise<void> {
|
||||
const meta = this.meta()
|
||||
const method = this.registry.get(request.method)
|
||||
@@ -183,8 +179,13 @@ export class RpcDispatcher {
|
||||
return
|
||||
}
|
||||
const effectiveParams = compatibility.params ?? parsedParams.value
|
||||
const invoke = (mutation?: DurableMutationInvocation) =>
|
||||
method.handler(effectiveParams, {
|
||||
const legacyCoordinator = this.legacyOrchestration.createCoordinatorInvocation(
|
||||
request,
|
||||
compatibility.legacyCoordinatorAuthority
|
||||
)
|
||||
const invoke = (mutation?: DurableMutationInvocation) => {
|
||||
const legacyCoordinatorRunId = legacyCoordinator?.revalidate()
|
||||
return method.handler(effectiveParams, {
|
||||
runtime: this.runtime,
|
||||
signal: options?.signal,
|
||||
requestId: request.id,
|
||||
@@ -200,10 +201,19 @@ export class RpcDispatcher {
|
||||
pairing: options?.pairing,
|
||||
sendBinary: options?.sendBinary,
|
||||
registerBinaryStreamHandler: options?.registerBinaryStreamHandler,
|
||||
legacyCoordinatorRunId: compatibility.legacyCoordinatorRunId
|
||||
legacyCoordinatorRunId,
|
||||
legacyCoordinatorAuthority: legacyCoordinator?.authority,
|
||||
revalidateLegacyCoordinator: legacyCoordinator?.revalidate
|
||||
})
|
||||
}
|
||||
const result = await this.orchestrationMutations.run(request, effectiveParams, invoke)
|
||||
this.recordRuntimeFeatureInteraction(request.method, result, undefined, request.params)
|
||||
recordRuntimeFeatureInteraction(
|
||||
this.runtime,
|
||||
request.method,
|
||||
result,
|
||||
undefined,
|
||||
request.params
|
||||
)
|
||||
reply(JSON.stringify(successResponse(request.id, meta, result)))
|
||||
} catch (error) {
|
||||
reply(JSON.stringify(this.mapError(request, meta, error)))
|
||||
@@ -213,7 +223,8 @@ export class RpcDispatcher {
|
||||
|
||||
const recordedStreamingFeatureInteractions = new Set<FeatureInteractionId>()
|
||||
const emit = (result: unknown): void => {
|
||||
this.recordRuntimeFeatureInteraction(
|
||||
recordRuntimeFeatureInteraction(
|
||||
this.runtime,
|
||||
request.method,
|
||||
result,
|
||||
recordedStreamingFeatureInteractions,
|
||||
@@ -243,7 +254,8 @@ export class RpcDispatcher {
|
||||
},
|
||||
emit
|
||||
)
|
||||
this.recordRuntimeFeatureInteraction(
|
||||
recordRuntimeFeatureInteraction(
|
||||
this.runtime,
|
||||
request.method,
|
||||
result,
|
||||
recordedStreamingFeatureInteractions,
|
||||
@@ -310,25 +322,4 @@ export class RpcDispatcher {
|
||||
private meta(): RpcEnvelopeMeta {
|
||||
return { runtimeId: this.runtime.getRuntimeId() }
|
||||
}
|
||||
|
||||
private recordRuntimeFeatureInteraction(
|
||||
method: string,
|
||||
result: unknown,
|
||||
alreadyRecorded?: Set<FeatureInteractionId>,
|
||||
rawParams?: unknown
|
||||
): void {
|
||||
const id = getRuntimeFeatureInteractionId(method, result, rawParams)
|
||||
if (!id) {
|
||||
return
|
||||
}
|
||||
if (alreadyRecorded?.has(id)) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
this.runtime.recordFeatureInteraction(id)
|
||||
alreadyRecorded?.add(id)
|
||||
} catch {
|
||||
// Best-effort education state must not break runtime tools.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { z } from 'zod'
|
||||
import { defineMethod, type RpcMethod } from '../core'
|
||||
import { OptionalString, requiredString } from '../schemas'
|
||||
import { OptionalBoolean, OptionalString, requiredString } from '../schemas'
|
||||
import type { OrcaRuntimeService } from '../../orca-runtime'
|
||||
import { OrchestrationError } from '../../orchestration/orchestration-error'
|
||||
|
||||
@@ -11,7 +11,8 @@ const RunCreateParams = z.object({
|
||||
|
||||
const RunUseParams = z.object({
|
||||
id: requiredString('Missing --id'),
|
||||
from: requiredString('Missing coordinator terminal')
|
||||
from: requiredString('Missing coordinator terminal'),
|
||||
takeoverLegacy: OptionalBoolean
|
||||
})
|
||||
|
||||
const RunCurrentParams = z.object({ from: requiredString('Missing coordinator terminal') })
|
||||
@@ -51,7 +52,7 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [
|
||||
defineMethod({
|
||||
name: 'orchestration.runUse',
|
||||
params: RunUseParams,
|
||||
handler: (params, { runtime, legacyCoordinatorRunId }) => {
|
||||
handler: (params, { runtime, legacyCoordinatorAuthority }) => {
|
||||
const paneKey = requireCallerPane(runtime, params.from)
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const priorRun = db.getCurrentRunForPane(paneKey)
|
||||
@@ -59,7 +60,8 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [
|
||||
runId: params.id,
|
||||
coordinatorHandle: params.from,
|
||||
coordinatorPaneKey: paneKey,
|
||||
allowLegacyCompatibility: legacyCoordinatorRunId === params.id
|
||||
takeoverLegacy: params.takeoverLegacy,
|
||||
legacyCoordinatorAuthority
|
||||
})
|
||||
if (!run) {
|
||||
throw new OrchestrationError(
|
||||
|
||||
@@ -2,7 +2,12 @@
|
||||
import { z } from 'zod'
|
||||
import { defineMethod, type RpcMethod } from '../core'
|
||||
import { OptionalFiniteNumber, OptionalString, OptionalBoolean, requiredString } from '../schemas'
|
||||
import type { MessageType, MessagePriority, TaskStatus } from '../../orchestration/db'
|
||||
import {
|
||||
LEGACY_CONTRACT_VERSION,
|
||||
type MessageType,
|
||||
type MessagePriority,
|
||||
type TaskStatus
|
||||
} from '../../orchestration/db'
|
||||
import { MESSAGE_TYPES } from '../../orchestration/types'
|
||||
import { buildDispatchPreamble } from '../../orchestration/preamble'
|
||||
import { formatMessageBanner } from '../../orchestration/formatter'
|
||||
@@ -379,6 +384,49 @@ function resolveMessageRun(
|
||||
return { run, dispatchId: dispatch?.id ?? dispatchId }
|
||||
}
|
||||
|
||||
function legacyWorkerDeliveryContract(
|
||||
runtime: OrcaRuntimeService,
|
||||
runId: string | undefined,
|
||||
recipient: string
|
||||
): 'legacy_direct' | undefined {
|
||||
if (!runId) {
|
||||
return undefined
|
||||
}
|
||||
if (!recipient.startsWith('dispatch:')) {
|
||||
return runtime
|
||||
.getOrchestrationDb()
|
||||
.resolveLegacyWorkerCandidate({ runId, terminalHandle: recipient })
|
||||
? 'legacy_direct'
|
||||
: undefined
|
||||
}
|
||||
const dispatch = runtime
|
||||
.getOrchestrationDb()
|
||||
.getDispatchContextById(recipient.slice('dispatch:'.length))
|
||||
return dispatch?.run_id === runId &&
|
||||
dispatch.contract_version === LEGACY_CONTRACT_VERSION &&
|
||||
(dispatch.status === 'pending' || dispatch.status === 'dispatched')
|
||||
? 'legacy_direct'
|
||||
: undefined
|
||||
}
|
||||
|
||||
function interruptedAcknowledgedCheck(
|
||||
runId: string,
|
||||
acknowledged: string,
|
||||
reason: 'consumer_fenced' | 'outcome_unknown' | 'waiter_exists'
|
||||
): Record<string, unknown> {
|
||||
return {
|
||||
runId,
|
||||
deliveryId: null,
|
||||
messages: [],
|
||||
count: 0,
|
||||
acknowledged,
|
||||
timedOut: false,
|
||||
cancelled: false,
|
||||
connectionLost: false,
|
||||
waitInterrupted: reason
|
||||
}
|
||||
}
|
||||
|
||||
function rejectFederatedExplicitTarget(params: { to?: string; run?: string }): void {
|
||||
if (params.to || params.run) {
|
||||
throw new OrchestrationError(
|
||||
@@ -395,7 +443,10 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
defineMethod({
|
||||
name: 'orchestration.send',
|
||||
params: SendParams,
|
||||
handler: async (params, { runtime, orchestrationCapability }) => {
|
||||
handler: async (
|
||||
params,
|
||||
{ runtime, orchestrationCapability, legacyCoordinatorRunId, revalidateLegacyCoordinator }
|
||||
) => {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const from = params.from ?? 'unknown'
|
||||
// Why: caller-supplied pane fields are only compatibility metadata; lifecycle authority uses the runtime-observed pane plus capability.
|
||||
@@ -536,6 +587,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
'Coordinator-to-worker control mail cannot report worker lifecycle.'
|
||||
)
|
||||
}
|
||||
revalidateLegacyCoordinator?.()
|
||||
const relay = db.enqueueFederationRelay({
|
||||
dispatchId,
|
||||
direction: 'to_worker',
|
||||
@@ -562,6 +614,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
}
|
||||
}
|
||||
// Point-to-point — existing single-recipient behavior
|
||||
revalidateLegacyCoordinator?.()
|
||||
const msg = db.insertMessage({
|
||||
from,
|
||||
to,
|
||||
@@ -572,7 +625,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
threadId: params.threadId,
|
||||
payload: params.payload,
|
||||
senderPaneKey,
|
||||
runId: routing.run?.id
|
||||
runId: routing.run?.id,
|
||||
deliveryContract: legacyWorkerDeliveryContract(
|
||||
runtime,
|
||||
routing.run?.id ?? legacyCoordinatorRunId,
|
||||
to
|
||||
)
|
||||
})
|
||||
const dispatch = routing.dispatchId
|
||||
? db.getDispatchContextById(routing.dispatchId)
|
||||
@@ -629,6 +687,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
throw new Error(`No recipients resolved for group address: ${to}`)
|
||||
}
|
||||
|
||||
revalidateLegacyCoordinator?.()
|
||||
const threadId = params.threadId ?? `thread_${Date.now()}`
|
||||
const messages = handles.map((handle) =>
|
||||
db.insertMessage({
|
||||
@@ -641,7 +700,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
threadId,
|
||||
payload: params.payload,
|
||||
senderPaneKey,
|
||||
runId: routing.run?.id
|
||||
runId: routing.run?.id,
|
||||
deliveryContract: legacyWorkerDeliveryContract(
|
||||
runtime,
|
||||
routing.run?.id ?? legacyCoordinatorRunId,
|
||||
handle
|
||||
)
|
||||
})
|
||||
)
|
||||
for (const message of messages) {
|
||||
@@ -655,7 +719,16 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
defineMethod({
|
||||
name: 'orchestration.check',
|
||||
params: CheckParams,
|
||||
handler: async (params, { runtime, signal, legacyCoordinatorRunId }) => {
|
||||
handler: async (
|
||||
params,
|
||||
{
|
||||
runtime,
|
||||
signal,
|
||||
legacyCoordinatorRunId,
|
||||
revalidateLegacyCoordinator,
|
||||
recordMutationReceipt
|
||||
}
|
||||
) => {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const handle = params.terminal ?? 'unknown'
|
||||
const typeFilter = parseMessageTypes(params.types)
|
||||
@@ -682,6 +755,11 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
deliveryId: params.ack
|
||||
})
|
||||
: undefined
|
||||
if (acknowledged) {
|
||||
recordMutationReceipt?.(
|
||||
interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'outcome_unknown')
|
||||
)
|
||||
}
|
||||
if (params.peek || params.all || params.unread === false) {
|
||||
const history = db.getRunMailboxHistory(run.id, 100, typeFilter)
|
||||
const messages =
|
||||
@@ -745,14 +823,28 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
signal,
|
||||
exclusive: true
|
||||
})
|
||||
try {
|
||||
revalidateLegacyCoordinator?.()
|
||||
} catch (error) {
|
||||
if (!acknowledged) {
|
||||
throw error
|
||||
}
|
||||
return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'consumer_fenced')
|
||||
}
|
||||
const latestRun = db.getRun(run.id)
|
||||
if (!latestRun || latestRun.consumer_generation !== generation) {
|
||||
if (acknowledged) {
|
||||
return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'consumer_fenced')
|
||||
}
|
||||
throw new OrchestrationError(
|
||||
'consumer_fenced',
|
||||
'This mailbox consumer was replaced while waiting.'
|
||||
)
|
||||
}
|
||||
if (waitResult === 'waiter_exists') {
|
||||
if (acknowledged) {
|
||||
return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'waiter_exists')
|
||||
}
|
||||
throw new OrchestrationError(
|
||||
'waiter_exists',
|
||||
`Run ${run.id} already has an active actionable waiter.`
|
||||
@@ -1119,7 +1211,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
defineMethod({
|
||||
name: 'orchestration.dispatch',
|
||||
params: DispatchParams,
|
||||
handler: async (params, { runtime, legacyCoordinatorRunId }) => {
|
||||
handler: async (params, { runtime, legacyCoordinatorRunId, revalidateLegacyCoordinator }) => {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const task = db.getTask(params.task)
|
||||
if (!task) {
|
||||
@@ -1184,6 +1276,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
)
|
||||
}
|
||||
|
||||
revalidateLegacyCoordinator?.()
|
||||
const ctx = db.createDispatchContext(params.task, to, assigneePaneKey)
|
||||
const dispatchCapability = params.inject
|
||||
? db.mintDispatchCapability({
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { parsePaneKey } from '../../../shared/stable-pane-id'
|
||||
import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence'
|
||||
import type {
|
||||
DispatchContextRow,
|
||||
@@ -6,9 +5,14 @@ import type {
|
||||
LegacyPrincipalRole
|
||||
} from '../orchestration/types'
|
||||
import { OrchestrationError } from '../orchestration/orchestration-error'
|
||||
import type { OrcaRuntimeService } from '../orca-runtime'
|
||||
import type { OrcaRuntimeService, OrchestrationCompatibilityCallerAuthority } from '../orca-runtime'
|
||||
import { LEGACY_CONTRACT_VERSION } from '../orchestration/db'
|
||||
import type { RpcRequest } from './core'
|
||||
import {
|
||||
equivalentLegacyPaneKey,
|
||||
legacyCoordinatorReadOnly,
|
||||
legacyReadOnlyError
|
||||
} from './orchestration-legacy-process-identity'
|
||||
|
||||
export type LegacyPrincipalCandidate = {
|
||||
runId: string
|
||||
@@ -22,8 +26,9 @@ export function resolveAttestedLegacyPrincipal(args: {
|
||||
runtime: OrcaRuntimeService
|
||||
evidence?: OrchestrationCompatibilityEvidence
|
||||
candidate: LegacyPrincipalCandidate
|
||||
authority?: OrchestrationCompatibilityCallerAuthority
|
||||
}): LegacyCompatibilityPrincipalRow {
|
||||
const authority = verifyAttestedLegacyCandidate(args)
|
||||
const authority = args.authority ?? verifyAttestedLegacyCandidate(args)
|
||||
return args.runtime.getOrchestrationDb().commitLegacyCompatibilityPrincipal({
|
||||
runId: args.candidate.runId,
|
||||
dispatchId: args.candidate.dispatchId,
|
||||
@@ -36,7 +41,7 @@ export function resolveAttestedLegacyPrincipal(args: {
|
||||
}).principal
|
||||
}
|
||||
|
||||
function verifyAttestedLegacyCandidate(args: {
|
||||
export function verifyAttestedLegacyCandidate(args: {
|
||||
runtime: OrcaRuntimeService
|
||||
evidence?: OrchestrationCompatibilityEvidence
|
||||
candidate: LegacyPrincipalCandidate
|
||||
@@ -44,7 +49,7 @@ function verifyAttestedLegacyCandidate(args: {
|
||||
const authority = args.runtime.verifyOrchestrationCompatibilityCaller(args.evidence)
|
||||
if (
|
||||
!authority ||
|
||||
!equivalentPaneKey(args.candidate.paneKey, authority.paneKey) ||
|
||||
!equivalentLegacyPaneKey(args.candidate.paneKey, authority.paneKey) ||
|
||||
args.candidate.terminalHandle !== authority.terminalHandle
|
||||
) {
|
||||
throw legacyReadOnlyError()
|
||||
@@ -221,37 +226,6 @@ export class LegacyCompatibilityAuthority {
|
||||
})
|
||||
return principal
|
||||
}
|
||||
|
||||
resolveProvenCoordinatorScope(request: RpcRequest, requestedRunId?: string): string | undefined {
|
||||
const db = this.runtime.getOrchestrationDb()
|
||||
const adoption = db.getLegacyAdoption()
|
||||
const requestedRun = requestedRunId ?? adoption?.adopted_run_id
|
||||
if (!adoption || requestedRun !== adoption.adopted_run_id) {
|
||||
return undefined
|
||||
}
|
||||
const candidate = db.resolveLegacyCoordinatorCandidate({
|
||||
runId: adoption.adopted_run_id,
|
||||
terminalHandle: request.orchestrationCompatibilityEvidence?.terminalHandle,
|
||||
paneKey: request.orchestrationCompatibilityEvidence?.paneKey
|
||||
})
|
||||
if (!candidate) {
|
||||
if (request.orchestrationCompatibilityEvidence) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
verifyAttestedLegacyCandidate({
|
||||
runtime: this.runtime,
|
||||
evidence: request.orchestrationCompatibilityEvidence,
|
||||
candidate: {
|
||||
runId: adoption.adopted_run_id,
|
||||
role: 'coordinator',
|
||||
terminalHandle: candidate.terminalHandle,
|
||||
paneKey: candidate.paneKey
|
||||
}
|
||||
})
|
||||
return adoption.adopted_run_id
|
||||
}
|
||||
}
|
||||
|
||||
function candidateFromDispatch(dispatch: DispatchContextRow): LegacyPrincipalCandidate {
|
||||
@@ -281,31 +255,3 @@ function candidateFromPrincipal(
|
||||
paneKey: principal.pane_key
|
||||
}
|
||||
}
|
||||
|
||||
function equivalentPaneKey(a: string | null | undefined, b: string | null | undefined): boolean {
|
||||
if (!a || !b) {
|
||||
return false
|
||||
}
|
||||
if (a === b) {
|
||||
return true
|
||||
}
|
||||
const aLeaf = parsePaneKey(a)?.leafId
|
||||
const bLeaf = parsePaneKey(b)?.leafId
|
||||
return Boolean(aLeaf && bLeaf && aLeaf === bLeaf)
|
||||
}
|
||||
|
||||
function legacyReadOnlyError(): OrchestrationError {
|
||||
return new OrchestrationError(
|
||||
'legacy_read_only',
|
||||
'This retained legacy assignment could not prove authority from its original live process. No effects were applied.',
|
||||
{ effectsApplied: false }
|
||||
)
|
||||
}
|
||||
|
||||
function legacyCoordinatorReadOnly(): OrchestrationError {
|
||||
return new OrchestrationError(
|
||||
'legacy_read_only',
|
||||
'This retained legacy coordinator could not prove its original process identity. No effects were applied.',
|
||||
{ effectsApplied: false }
|
||||
)
|
||||
}
|
||||
|
||||
@@ -300,6 +300,82 @@ describe('legacy compatibility through RpcDispatcher', () => {
|
||||
expect(harness.notify).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('replays an A-era completion without touching a newer current attempt', async () => {
|
||||
const harness = createHarness()
|
||||
const payload = JSON.stringify({
|
||||
taskId: harness.taskId,
|
||||
dispatchId: harness.dispatchId
|
||||
})
|
||||
const completion = harness.db.insertMessage({
|
||||
runId: harness.adoptedRunId,
|
||||
from: WORKER_HANDLE,
|
||||
to: COORDINATOR_HANDLE,
|
||||
subject: 'Failed: pinned A treated this as complete',
|
||||
body: 'persisted result',
|
||||
type: 'worker_done',
|
||||
payload,
|
||||
senderPaneKey: WORKER_PANE,
|
||||
deliveryContract: 'legacy_direct'
|
||||
})
|
||||
harness.db.settleWorkerReport({
|
||||
taskId: harness.taskId,
|
||||
dispatchId: harness.dispatchId,
|
||||
outcome: 'succeeded',
|
||||
result: 'persisted result'
|
||||
})
|
||||
harness.db.commitLegacyCompatibilityPrincipal({
|
||||
runId: harness.adoptedRunId,
|
||||
dispatchId: harness.dispatchId,
|
||||
role: 'worker',
|
||||
hostScope: JSON.stringify({ kind: 'local', hostId: 'local' }),
|
||||
terminalHandle: WORKER_HANDLE,
|
||||
paneKey: WORKER_PANE,
|
||||
launchTokenHash: createHash('sha256').update('worker-token').digest('hex'),
|
||||
processIncarnation: 'process-1'
|
||||
})
|
||||
harness.db.updateTaskStatus(harness.taskId, 'ready')
|
||||
const currentDispatch = harness.db.createDispatchContext(
|
||||
harness.taskId,
|
||||
'term_current_worker',
|
||||
'tab_current_worker:77777777-7777-4777-8777-777777777777',
|
||||
'current-launch-hash'
|
||||
)
|
||||
const currentTaskBefore = harness.db.getTask(harness.taskId)
|
||||
const currentDispatchBefore = harness.db.getDispatchContextById(currentDispatch.id)
|
||||
const before = counts(harness.db)
|
||||
|
||||
const response = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.send',
|
||||
{
|
||||
from: WORKER_HANDLE,
|
||||
to: COORDINATOR_HANDLE,
|
||||
subject: completion.subject,
|
||||
body: completion.body,
|
||||
type: 'worker_done',
|
||||
payload
|
||||
},
|
||||
evidence('worker'),
|
||||
'reconstruct-pinned-a-completion'
|
||||
)
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
message: { id: completion.id },
|
||||
lifecycle: { action: 'settled', outcome: 'succeeded', duplicate: true }
|
||||
}
|
||||
})
|
||||
expect(harness.db.getTask(harness.taskId)).toEqual(currentTaskBefore)
|
||||
expect(harness.db.getDispatchContextById(harness.dispatchId)?.status).toBe('completed')
|
||||
expect(harness.db.getDispatchContextById(currentDispatch.id)).toEqual(currentDispatchBefore)
|
||||
expect(counts(harness.db)).toEqual({
|
||||
...before,
|
||||
legacy_operation_receipts: before.legacy_operation_receipts + 1
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects a legacy lifecycle recipient outside the adopted Run with zero effects', async () => {
|
||||
const harness = createHarness()
|
||||
const before = counts(harness.db)
|
||||
@@ -367,6 +443,35 @@ describe('legacy compatibility through RpcDispatcher', () => {
|
||||
expect(harness.notify).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps distinct compatibility ask invocations on distinct questions', async () => {
|
||||
const harness = createHarness()
|
||||
const ask = {
|
||||
from: WORKER_HANDLE,
|
||||
to: COORDINATOR_HANDLE,
|
||||
question: 'Same question?',
|
||||
options: ['yes', 'no'],
|
||||
timeoutMs: 1
|
||||
}
|
||||
const first = await harness.dispatcher.dispatch(
|
||||
request('orchestration.ask', ask, evidence('worker'), 'ask-one')
|
||||
)
|
||||
const replay = await harness.dispatcher.dispatch(
|
||||
request('orchestration.ask', ask, evidence('worker'), 'ask-one')
|
||||
)
|
||||
const second = await harness.dispatcher.dispatch(
|
||||
request('orchestration.ask', ask, evidence('worker'), 'ask-two')
|
||||
)
|
||||
|
||||
expect(first).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: false } } })
|
||||
expect(replay).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: true } } })
|
||||
expect(second).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: false } } })
|
||||
const firstId = (first as { result: { messageId: string } }).result.messageId
|
||||
const replayId = (replay as { result: { messageId: string } }).result.messageId
|
||||
const secondId = (second as { result: { messageId: string } }).result.messageId
|
||||
expect(replayId).toBe(firstId)
|
||||
expect(secondId).not.toBe(firstId)
|
||||
})
|
||||
|
||||
it('does not infer an outcome for a current Dispatch when legacy adoption exists', async () => {
|
||||
const harness = createHarness()
|
||||
const run = harness.db.createRun({
|
||||
@@ -524,6 +629,56 @@ describe('legacy compatibility through RpcDispatcher', () => {
|
||||
expect(harness.notify).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('advertises only Run-addressed current delivery to a legacy coordinator', async () => {
|
||||
const harness = createHarness()
|
||||
harness.db.insertMessage({
|
||||
runId: harness.adoptedRunId,
|
||||
from: 'term_current_worker',
|
||||
to: WORKER_HANDLE,
|
||||
subject: 'worker-only current mail',
|
||||
deliveryContract: 'current_delivery'
|
||||
})
|
||||
const workerOnly = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{ terminal: COORDINATOR_HANDLE },
|
||||
evidence('coordinator'),
|
||||
'worker-current-mail'
|
||||
)
|
||||
)
|
||||
expect(workerOnly).toMatchObject({
|
||||
ok: true,
|
||||
result: { legacyCompatibility: { currentDelivery: undefined } }
|
||||
})
|
||||
|
||||
harness.db.insertMessage({
|
||||
runId: harness.adoptedRunId,
|
||||
from: 'term_current_worker',
|
||||
to: `run:${harness.adoptedRunId}`,
|
||||
subject: 'Run current mail',
|
||||
deliveryContract: 'current_delivery'
|
||||
})
|
||||
const runMail = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{ terminal: COORDINATOR_HANDLE },
|
||||
evidence('coordinator'),
|
||||
'run-current-mail'
|
||||
)
|
||||
)
|
||||
expect(runMail).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
legacyCompatibility: {
|
||||
currentDelivery: {
|
||||
runId: harness.adoptedRunId,
|
||||
checkCommand: expect.stringContaining(`--run ${harness.adoptedRunId}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it.each(['dispatch', 'websocket'] as const)(
|
||||
'%s binds an attested coordinator once and rejects contradictory proof before binding',
|
||||
async (transport) => {
|
||||
@@ -588,4 +743,42 @@ describe('legacy compatibility through RpcDispatcher', () => {
|
||||
expect(harness.db.getRun(harness.adoptedRunId)?.consumer_generation).toBe(0)
|
||||
expect(counts(harness.db)).toEqual(before)
|
||||
})
|
||||
|
||||
it('rejects stale legacy coordinator proof after current takeover', async () => {
|
||||
const harness = createHarness()
|
||||
const principal = harness.db.commitLegacyCompatibilityPrincipal({
|
||||
runId: harness.adoptedRunId,
|
||||
role: 'coordinator',
|
||||
hostScope: JSON.stringify({ kind: 'local', hostId: 'local' }),
|
||||
terminalHandle: COORDINATOR_HANDLE,
|
||||
paneKey: COORDINATOR_PANE,
|
||||
launchTokenHash: createHash('sha256').update('coordinator-token').digest('hex'),
|
||||
processIncarnation: 'process-1'
|
||||
}).principal
|
||||
harness.db.settleWorkerReport({
|
||||
taskId: harness.taskId,
|
||||
dispatchId: harness.dispatchId,
|
||||
outcome: 'succeeded',
|
||||
result: 'done'
|
||||
})
|
||||
harness.db.bindRun({
|
||||
runId: harness.adoptedRunId,
|
||||
coordinatorHandle: 'term_current_coord',
|
||||
coordinatorPaneKey: 'tab_current:55555555-5555-4555-8555-555555555555'
|
||||
})
|
||||
expect(harness.db.getLegacyCompatibilityPrincipal(principal.id)?.status).toBe('revoked')
|
||||
const before = counts(harness.db)
|
||||
|
||||
const response = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.taskList',
|
||||
{},
|
||||
evidence('coordinator'),
|
||||
'stale-coordinator-after-takeover'
|
||||
)
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
|
||||
expect(counts(harness.db)).toEqual(before)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { RpcRequest } from './core'
|
||||
import type { LegacyCoordinatorAuthorityProof, RpcRequest } from './core'
|
||||
import type { OrcaRuntimeService } from '../orca-runtime'
|
||||
import { LegacyCompatibilityAuthority } from './orchestration-legacy-authority'
|
||||
import { handleLegacyLifecycleSend } from './orchestration-legacy-lifecycle'
|
||||
@@ -10,6 +10,7 @@ import type {
|
||||
LegacyReplyParams,
|
||||
LegacySendParams
|
||||
} from './orchestration-legacy-operation'
|
||||
import { LegacyCoordinatorAuthority } from './orchestration-legacy-coordinator-authority'
|
||||
|
||||
const COORDINATOR_PREFLIGHT_METHODS = new Set([
|
||||
'orchestration.taskCreate',
|
||||
@@ -19,19 +20,31 @@ const COORDINATOR_PREFLIGHT_METHODS = new Set([
|
||||
'orchestration.gateCreate',
|
||||
'orchestration.gateResolve',
|
||||
'orchestration.runUse',
|
||||
'orchestration.send',
|
||||
'orchestration.check',
|
||||
'orchestration.reply'
|
||||
])
|
||||
|
||||
export type LegacyCompatibilityRoute =
|
||||
| { handled: true; result: unknown }
|
||||
| { handled: false; params?: unknown; legacyCoordinatorRunId?: string }
|
||||
| {
|
||||
handled: false
|
||||
params?: unknown
|
||||
legacyCoordinatorAuthority?: LegacyCoordinatorAuthorityProof
|
||||
}
|
||||
|
||||
export type LegacyCoordinatorInvocation = Readonly<{
|
||||
authority: LegacyCoordinatorAuthorityProof
|
||||
revalidate: () => string
|
||||
}>
|
||||
|
||||
export class OrchestrationLegacyCompatibility {
|
||||
private readonly authority: LegacyCompatibilityAuthority
|
||||
private readonly coordinatorAuthority: LegacyCoordinatorAuthority
|
||||
|
||||
constructor(private readonly runtime: OrcaRuntimeService) {
|
||||
this.authority = new LegacyCompatibilityAuthority(runtime)
|
||||
this.coordinatorAuthority = new LegacyCoordinatorAuthority(runtime)
|
||||
}
|
||||
|
||||
async tryHandle(
|
||||
@@ -50,18 +63,40 @@ export class OrchestrationLegacyCompatibility {
|
||||
return { handled: false }
|
||||
}
|
||||
const values = params as Record<string, unknown>
|
||||
if (request.method === 'orchestration.runUse' && values.takeoverLegacy === true) {
|
||||
return { handled: false }
|
||||
}
|
||||
const requestedRunId =
|
||||
request.method === 'orchestration.runUse' ? stringValue(values.id) : stringValue(values.run)
|
||||
const runId = this.authority.resolveProvenCoordinatorScope(request, requestedRunId)
|
||||
return runId
|
||||
const authority = this.coordinatorAuthority.resolve(request, requestedRunId)
|
||||
return authority
|
||||
? {
|
||||
handled: false,
|
||||
params: { ...values, run: runId },
|
||||
legacyCoordinatorRunId: runId
|
||||
params: { ...values, run: authority.runId },
|
||||
legacyCoordinatorAuthority: authority
|
||||
}
|
||||
: { handled: false }
|
||||
}
|
||||
|
||||
revalidateCoordinatorAuthority(
|
||||
request: RpcRequest,
|
||||
proof: LegacyCoordinatorAuthorityProof
|
||||
): string {
|
||||
return this.coordinatorAuthority.revalidate(request, proof)
|
||||
}
|
||||
|
||||
createCoordinatorInvocation(
|
||||
request: RpcRequest,
|
||||
authority?: LegacyCoordinatorAuthorityProof
|
||||
): LegacyCoordinatorInvocation | undefined {
|
||||
return authority
|
||||
? {
|
||||
authority,
|
||||
revalidate: () => this.revalidateCoordinatorAuthority(request, authority)
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
|
||||
private async route(
|
||||
request: RpcRequest,
|
||||
params: unknown,
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
import type { OrcaRuntimeService, OrchestrationCompatibilityCallerAuthority } from '../orca-runtime'
|
||||
import type { LegacyCompatibilityPrincipalRow } from '../orchestration/types'
|
||||
import type { LegacyCoordinatorAuthorityProof, RpcRequest } from './core'
|
||||
import {
|
||||
resolveAttestedLegacyPrincipal,
|
||||
verifyAttestedLegacyCandidate,
|
||||
type LegacyPrincipalCandidate
|
||||
} from './orchestration-legacy-authority'
|
||||
import {
|
||||
equivalentLegacyPaneKey,
|
||||
legacyCoordinatorReadOnly
|
||||
} from './orchestration-legacy-process-identity'
|
||||
|
||||
export class LegacyCoordinatorAuthority {
|
||||
constructor(private readonly runtime: OrcaRuntimeService) {}
|
||||
|
||||
resolve(
|
||||
request: RpcRequest,
|
||||
requestedRunId?: string
|
||||
): LegacyCoordinatorAuthorityProof | undefined {
|
||||
const db = this.runtime.getOrchestrationDb()
|
||||
const adoption = db.getLegacyAdoption()
|
||||
const requestedRun = requestedRunId ?? adoption?.adopted_run_id
|
||||
if (!adoption || requestedRun !== adoption.adopted_run_id) {
|
||||
return undefined
|
||||
}
|
||||
const candidate = db.resolveLegacyCoordinatorCandidate({
|
||||
runId: adoption.adopted_run_id,
|
||||
terminalHandle: request.orchestrationCompatibilityEvidence?.terminalHandle,
|
||||
paneKey: request.orchestrationCompatibilityEvidence?.paneKey
|
||||
})
|
||||
if (!candidate) {
|
||||
if (request.orchestrationCompatibilityEvidence) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
const existing = db.getLegacyCoordinatorPrincipal(adoption.adopted_run_id)
|
||||
const proofCandidate = existing
|
||||
? this.candidate(adoption.adopted_run_id, existing)
|
||||
: this.candidate(adoption.adopted_run_id, candidate)
|
||||
const attestation = verifyAttestedLegacyCandidate({
|
||||
runtime: this.runtime,
|
||||
evidence: request.orchestrationCompatibilityEvidence,
|
||||
candidate: proofCandidate
|
||||
})
|
||||
const run = db.getRun(adoption.adopted_run_id)
|
||||
if (!run || !this.bindingMatches(run, proofCandidate)) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
if (existing && !this.principalMatchesAttestation(existing, attestation)) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
const principal = existing
|
||||
? existing
|
||||
: request.method === 'orchestration.runUse'
|
||||
? resolveAttestedLegacyPrincipal({
|
||||
runtime: this.runtime,
|
||||
evidence: request.orchestrationCompatibilityEvidence,
|
||||
candidate: proofCandidate,
|
||||
authority: attestation
|
||||
})
|
||||
: undefined
|
||||
return {
|
||||
runId: run.id,
|
||||
principalId: principal?.id ?? null,
|
||||
terminalHandle: proofCandidate.terminalHandle,
|
||||
paneKey: proofCandidate.paneKey,
|
||||
consumerGeneration: run.consumer_generation
|
||||
}
|
||||
}
|
||||
|
||||
revalidate(request: RpcRequest, proof: LegacyCoordinatorAuthorityProof): string {
|
||||
const db = this.runtime.getOrchestrationDb()
|
||||
const adoption = db.getLegacyAdoption()
|
||||
const run = db.getRun(proof.runId)
|
||||
if (
|
||||
adoption?.adopted_run_id !== proof.runId ||
|
||||
!run ||
|
||||
run.consumer_generation !== proof.consumerGeneration
|
||||
) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
const candidate = this.candidate(proof.runId, proof)
|
||||
const principal = db.getLegacyCoordinatorPrincipal(proof.runId)
|
||||
if (proof.principalId) {
|
||||
if (
|
||||
principal?.id !== proof.principalId ||
|
||||
principal.status !== 'committed' ||
|
||||
principal.terminal_handle !== proof.terminalHandle ||
|
||||
!equivalentLegacyPaneKey(principal.pane_key, proof.paneKey)
|
||||
) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
} else if (
|
||||
principal &&
|
||||
(principal.status !== 'committed' ||
|
||||
principal.terminal_handle !== proof.terminalHandle ||
|
||||
!equivalentLegacyPaneKey(principal.pane_key, proof.paneKey))
|
||||
) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
const attestation = verifyAttestedLegacyCandidate({
|
||||
runtime: this.runtime,
|
||||
evidence: request.orchestrationCompatibilityEvidence,
|
||||
candidate
|
||||
})
|
||||
if (principal && !this.principalMatchesAttestation(principal, attestation)) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
if (!this.bindingMatches(run, candidate)) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
return run.id
|
||||
}
|
||||
|
||||
private candidate(
|
||||
runId: string,
|
||||
identity: {
|
||||
terminalHandle?: string
|
||||
terminal_handle?: string
|
||||
paneKey?: string
|
||||
pane_key?: string
|
||||
}
|
||||
): LegacyPrincipalCandidate {
|
||||
return {
|
||||
runId,
|
||||
role: 'coordinator',
|
||||
terminalHandle: identity.terminalHandle ?? (identity.terminal_handle as string),
|
||||
paneKey: identity.paneKey ?? (identity.pane_key as string)
|
||||
}
|
||||
}
|
||||
|
||||
private bindingMatches(
|
||||
run: { coordinator_handle: string | null; coordinator_pane_key: string | null },
|
||||
candidate: LegacyPrincipalCandidate
|
||||
): boolean {
|
||||
return (
|
||||
!run.coordinator_pane_key ||
|
||||
(run.coordinator_handle === candidate.terminalHandle &&
|
||||
equivalentLegacyPaneKey(run.coordinator_pane_key, candidate.paneKey))
|
||||
)
|
||||
}
|
||||
|
||||
private principalMatchesAttestation(
|
||||
principal: LegacyCompatibilityPrincipalRow,
|
||||
attestation: OrchestrationCompatibilityCallerAuthority
|
||||
): boolean {
|
||||
return (
|
||||
principal.status === 'committed' &&
|
||||
principal.host_scope === JSON.stringify(attestation.hostScope) &&
|
||||
principal.terminal_handle === attestation.terminalHandle &&
|
||||
equivalentLegacyPaneKey(principal.pane_key, attestation.paneKey) &&
|
||||
principal.launch_token_hash === attestation.launchTokenHash &&
|
||||
principal.process_incarnation === attestation.processIncarnation
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,849 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { ORCHESTRATION_CONTRACT_VERSION } from '../../../shared/protocol-version'
|
||||
import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence'
|
||||
import Database from '../../sqlite/sync-database'
|
||||
import { OrcaRuntimeService } from '../orca-runtime'
|
||||
import { OrchestrationDb } from '../orchestration/db'
|
||||
import type { RpcRequest, RpcResponse } from './core'
|
||||
import { RpcDispatcher } from './dispatcher'
|
||||
import { ORCHESTRATION_METHODS } from './methods/orchestration'
|
||||
|
||||
const COORDINATOR_HANDLE = 'term_legacy_coord'
|
||||
const COORDINATOR_PANE = 'tab_coord:44444444-4444-4444-8444-444444444444'
|
||||
const WORKER_HANDLE = 'term_legacy_worker'
|
||||
const WORKER_PANE = 'tab_worker:33333333-3333-4333-8333-333333333333'
|
||||
|
||||
type Transport = 'dispatch' | 'websocket'
|
||||
|
||||
type Harness = {
|
||||
db: OrchestrationDb
|
||||
dispatcher: RpcDispatcher
|
||||
runtime: OrcaRuntimeService
|
||||
adoptedRunId: string
|
||||
taskId: string
|
||||
dispatchId: string
|
||||
}
|
||||
|
||||
const tempDirs: string[] = []
|
||||
const databases: OrchestrationDb[] = []
|
||||
|
||||
afterEach(() => {
|
||||
databases.splice(0).forEach((database) => database.close())
|
||||
tempDirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }))
|
||||
})
|
||||
|
||||
function createHarness(): Harness {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-legacy-coordinator-race-'))
|
||||
tempDirs.push(dir)
|
||||
const dbPath = join(dir, 'orchestration.db')
|
||||
const before = new OrchestrationDb(dbPath)
|
||||
const task = before.createTask({
|
||||
spec: 'legacy assignment',
|
||||
createdByTerminalHandle: COORDINATOR_HANDLE
|
||||
})
|
||||
const dispatch = before.createDispatchContext(task.id, WORKER_HANDLE, WORKER_PANE)
|
||||
before.close()
|
||||
|
||||
const raw = new Database(dbPath)
|
||||
raw.exec(`
|
||||
UPDATE dispatch_contexts SET process_incarnation = 'process-1';
|
||||
DROP INDEX IF EXISTS idx_messages_delivery_contract;
|
||||
DROP TABLE legacy_mail_receipts;
|
||||
DROP TABLE legacy_operation_receipts;
|
||||
DROP TABLE legacy_compatibility_principals;
|
||||
DROP TABLE legacy_adoptions;
|
||||
`)
|
||||
raw.pragma('user_version = 18')
|
||||
raw.close()
|
||||
|
||||
const db = new OrchestrationDb(dbPath)
|
||||
databases.push(db)
|
||||
const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string
|
||||
const runtime = new OrcaRuntimeService()
|
||||
runtime.setOrchestrationDb(db)
|
||||
vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) =>
|
||||
handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : handle === WORKER_HANDLE ? WORKER_PANE : null
|
||||
)
|
||||
vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation(
|
||||
(compatibilityEvidence) => {
|
||||
const coordinator =
|
||||
compatibilityEvidence?.terminalHandle === COORDINATOR_HANDLE &&
|
||||
compatibilityEvidence.paneKey === COORDINATOR_PANE &&
|
||||
compatibilityEvidence.launchToken === 'coordinator-token'
|
||||
const worker =
|
||||
compatibilityEvidence?.terminalHandle === WORKER_HANDLE &&
|
||||
compatibilityEvidence.paneKey === WORKER_PANE &&
|
||||
compatibilityEvidence.launchToken === 'worker-token'
|
||||
return coordinator || worker
|
||||
? {
|
||||
hostScope: { kind: 'local', hostId: 'local' },
|
||||
terminalHandle: coordinator ? COORDINATOR_HANDLE : WORKER_HANDLE,
|
||||
paneKey: coordinator ? COORDINATOR_PANE : WORKER_PANE,
|
||||
processIncarnation: 'process-1',
|
||||
launchTokenHash: createHash('sha256')
|
||||
.update(coordinator ? 'coordinator-token' : 'worker-token')
|
||||
.digest('hex')
|
||||
}
|
||||
: null
|
||||
}
|
||||
)
|
||||
return {
|
||||
db,
|
||||
dispatcher: new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }),
|
||||
runtime,
|
||||
adoptedRunId,
|
||||
taskId: task.id,
|
||||
dispatchId: dispatch.id
|
||||
}
|
||||
}
|
||||
|
||||
function evidence(
|
||||
role: 'coordinator' | 'worker' = 'coordinator'
|
||||
): OrchestrationCompatibilityEvidence {
|
||||
return {
|
||||
terminalHandle: role === 'coordinator' ? COORDINATOR_HANDLE : WORKER_HANDLE,
|
||||
paneKey: role === 'coordinator' ? COORDINATOR_PANE : WORKER_PANE,
|
||||
launchToken: role === 'coordinator' ? 'coordinator-token' : 'worker-token'
|
||||
}
|
||||
}
|
||||
|
||||
function request(
|
||||
method: string,
|
||||
params: unknown,
|
||||
invocationId: string,
|
||||
callerEvidence = evidence()
|
||||
): RpcRequest {
|
||||
return {
|
||||
id: `rpc_${invocationId}`,
|
||||
authToken: 'caller-token',
|
||||
method,
|
||||
params,
|
||||
orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION,
|
||||
orchestrationRequestId: invocationId,
|
||||
compatibilityInvocationId: invocationId,
|
||||
orchestrationCompatibilityEvidence: callerEvidence
|
||||
}
|
||||
}
|
||||
|
||||
async function invoke(
|
||||
dispatcher: RpcDispatcher,
|
||||
rpcRequest: RpcRequest,
|
||||
transport: Transport
|
||||
): Promise<RpcResponse> {
|
||||
if (transport === 'dispatch') {
|
||||
return await dispatcher.dispatch(rpcRequest)
|
||||
}
|
||||
const replies: string[] = []
|
||||
await dispatcher.dispatchStreaming(rpcRequest, (reply) => replies.push(reply))
|
||||
expect(replies).toHaveLength(1)
|
||||
return JSON.parse(replies[0]) as RpcResponse
|
||||
}
|
||||
|
||||
function installTakeoverDuringMutationPreflight(harness: Harness): void {
|
||||
const originalBegin = harness.db.beginMutationReceipt.bind(harness.db)
|
||||
vi.spyOn(harness.db, 'beginMutationReceipt').mockImplementation((identity) => {
|
||||
const begun = originalBegin(identity)
|
||||
settleLegacyWorkerAndTakeOver(harness, 'settled during takeover')
|
||||
return begun
|
||||
})
|
||||
}
|
||||
|
||||
function settleLegacyWorkerAndTakeOver(harness: Harness, result: string): void {
|
||||
harness.db.settleWorkerReport({
|
||||
taskId: harness.taskId,
|
||||
dispatchId: harness.dispatchId,
|
||||
outcome: 'succeeded',
|
||||
result
|
||||
})
|
||||
harness.db.bindRun({
|
||||
runId: harness.adoptedRunId,
|
||||
coordinatorHandle: 'term_current_coord',
|
||||
coordinatorPaneKey: 'tab_current:55555555-5555-4555-8555-555555555555'
|
||||
})
|
||||
}
|
||||
|
||||
function mutationReceiptCount(db: OrchestrationDb): number {
|
||||
const sqlite = (db as unknown as { db: Database.Database }).db
|
||||
return (
|
||||
sqlite.prepare('SELECT COUNT(*) AS count FROM mutation_receipts').get() as { count: number }
|
||||
).count
|
||||
}
|
||||
|
||||
function messageCount(db: OrchestrationDb): number {
|
||||
const sqlite = (db as unknown as { db: Database.Database }).db
|
||||
return (sqlite.prepare('SELECT COUNT(*) AS count FROM messages').get() as { count: number }).count
|
||||
}
|
||||
|
||||
describe('legacy coordinator takeover races', () => {
|
||||
it.each(['dispatch', 'websocket'] as const)(
|
||||
'%s routes a pre-bind coordinator send to the retained worker mailbox',
|
||||
async (transport) => {
|
||||
const harness = createHarness()
|
||||
|
||||
const response = await invoke(
|
||||
harness.dispatcher,
|
||||
request(
|
||||
'orchestration.send',
|
||||
{ from: COORDINATOR_HANDLE, to: WORKER_HANDLE, subject: 'continue safely' },
|
||||
`send-direct-${transport}`
|
||||
),
|
||||
transport
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
message: {
|
||||
run_id: harness.adoptedRunId,
|
||||
to_handle: WORKER_HANDLE,
|
||||
delivery_contract: 'legacy_direct'
|
||||
}
|
||||
}
|
||||
})
|
||||
const workerCheck = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{ terminal: WORKER_HANDLE },
|
||||
`worker-check-${transport}`,
|
||||
evidence('worker')
|
||||
)
|
||||
)
|
||||
expect(workerCheck).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
messages: [
|
||||
{
|
||||
subject: 'continue safely',
|
||||
run_id: harness.adoptedRunId,
|
||||
delivery_contract: 'legacy_direct'
|
||||
}
|
||||
],
|
||||
legacyCompatibility: { ackMessageIds: [expect.any(String)] }
|
||||
}
|
||||
})
|
||||
expect(
|
||||
harness.db.getInbox(100).some((message) => message.run_id === 'run_legacy_local')
|
||||
).toBe(false)
|
||||
}
|
||||
)
|
||||
|
||||
it('routes an exact legacy Dispatch recipient to compatibility delivery', async () => {
|
||||
const harness = createHarness()
|
||||
|
||||
const response = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.send',
|
||||
{
|
||||
from: COORDINATOR_HANDLE,
|
||||
to: `dispatch:${harness.dispatchId}`,
|
||||
subject: 'dispatch guidance'
|
||||
},
|
||||
'send-dispatch'
|
||||
)
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
message: {
|
||||
run_id: harness.adoptedRunId,
|
||||
to_handle: `dispatch:${harness.dispatchId}`,
|
||||
delivery_contract: 'legacy_direct'
|
||||
}
|
||||
}
|
||||
})
|
||||
const workerCheck = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{ terminal: WORKER_HANDLE },
|
||||
'worker-check-dispatch',
|
||||
evidence('worker')
|
||||
)
|
||||
)
|
||||
expect(workerCheck).toMatchObject({
|
||||
ok: true,
|
||||
result: { messages: [{ subject: 'dispatch guidance' }] }
|
||||
})
|
||||
})
|
||||
|
||||
it('partitions a coordinator group send by legacy recipient contract', async () => {
|
||||
const harness = createHarness()
|
||||
vi.spyOn(harness.runtime, 'listTerminals').mockResolvedValue({
|
||||
terminals: [
|
||||
{ handle: COORDINATOR_HANDLE },
|
||||
{ handle: WORKER_HANDLE },
|
||||
{ handle: 'term_current_worker' }
|
||||
],
|
||||
totalCount: 3,
|
||||
truncated: false
|
||||
} as never)
|
||||
|
||||
const response = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.send',
|
||||
{ from: COORDINATOR_HANDLE, to: '@all', subject: 'group guidance' },
|
||||
'send-group'
|
||||
)
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({ ok: true, result: { recipients: 2 } })
|
||||
const messages = harness.db
|
||||
.getInbox(100)
|
||||
.filter((message) => message.subject === 'group guidance')
|
||||
expect(messages).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
run_id: harness.adoptedRunId,
|
||||
to_handle: WORKER_HANDLE,
|
||||
delivery_contract: 'legacy_direct'
|
||||
}),
|
||||
expect.objectContaining({
|
||||
run_id: harness.adoptedRunId,
|
||||
to_handle: 'term_current_worker',
|
||||
delivery_contract: 'current_delivery'
|
||||
})
|
||||
])
|
||||
)
|
||||
})
|
||||
|
||||
it.each(['dispatch', 'websocket'] as const)(
|
||||
'%s rejects a coordinator send after takeover during durable preflight',
|
||||
async (transport) => {
|
||||
const harness = createHarness()
|
||||
installTakeoverDuringMutationPreflight(harness)
|
||||
const before = messageCount(harness.db)
|
||||
|
||||
const response = await invoke(
|
||||
harness.dispatcher,
|
||||
request(
|
||||
'orchestration.send',
|
||||
{ from: COORDINATOR_HANDLE, to: WORKER_HANDLE, subject: 'must not send' },
|
||||
`send-takeover-${transport}`
|
||||
),
|
||||
transport
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
|
||||
expect(messageCount(harness.db)).toBe(before)
|
||||
expect(mutationReceiptCount(harness.db)).toBe(0)
|
||||
}
|
||||
)
|
||||
|
||||
it('revalidates a group send after asynchronous terminal discovery', async () => {
|
||||
const harness = createHarness()
|
||||
let resolveTerminals:
|
||||
| ((value: {
|
||||
terminals: { handle: string }[]
|
||||
totalCount: number
|
||||
truncated: boolean
|
||||
}) => void)
|
||||
| undefined
|
||||
let signalListingStarted: (() => void) | undefined
|
||||
const listingStarted = new Promise<void>((resolve) => {
|
||||
signalListingStarted = resolve
|
||||
})
|
||||
vi.spyOn(harness.runtime, 'listTerminals').mockImplementation(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
resolveTerminals = resolve as typeof resolveTerminals
|
||||
signalListingStarted?.()
|
||||
}) as never
|
||||
)
|
||||
const before = messageCount(harness.db)
|
||||
const pending = harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.send',
|
||||
{ from: COORDINATOR_HANDLE, to: '@all', subject: 'must remain unsent' },
|
||||
'send-group-takeover'
|
||||
)
|
||||
)
|
||||
await listingStarted
|
||||
settleLegacyWorkerAndTakeOver(harness, 'settled during terminal discovery')
|
||||
resolveTerminals?.({
|
||||
terminals: [{ handle: COORDINATOR_HANDLE }, { handle: WORKER_HANDLE }],
|
||||
totalCount: 2,
|
||||
truncated: false
|
||||
})
|
||||
|
||||
await expect(pending).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { code: 'legacy_read_only' }
|
||||
})
|
||||
expect(messageCount(harness.db)).toBe(before)
|
||||
expect(mutationReceiptCount(harness.db)).toBe(0)
|
||||
})
|
||||
|
||||
it('reports a committed ACK when coordinator takeover interrupts its wait', async () => {
|
||||
const harness = createHarness()
|
||||
const incoming = harness.db.insertMessage({
|
||||
runId: harness.adoptedRunId,
|
||||
from: WORKER_HANDLE,
|
||||
to: `run:${harness.adoptedRunId}`,
|
||||
subject: 'ready to acknowledge'
|
||||
})
|
||||
const first = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{ terminal: COORDINATOR_HANDLE, run: harness.adoptedRunId },
|
||||
'check-delivery-before-takeover'
|
||||
)
|
||||
)
|
||||
expect(first).toMatchObject({
|
||||
ok: true,
|
||||
result: { deliveryId: expect.any(String), messages: [{ id: incoming.id }] }
|
||||
})
|
||||
const deliveryId = (first as { result: { deliveryId: string } }).result.deliveryId
|
||||
let resolveWait: (() => void) | undefined
|
||||
let signalWaitStarted: (() => void) | undefined
|
||||
const waitStarted = new Promise<void>((resolve) => {
|
||||
signalWaitStarted = resolve
|
||||
})
|
||||
vi.spyOn(harness.runtime, 'waitForMessage').mockImplementation(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
resolveWait = () => resolve('notified')
|
||||
signalWaitStarted?.()
|
||||
})
|
||||
)
|
||||
const pending = harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{
|
||||
terminal: COORDINATOR_HANDLE,
|
||||
run: harness.adoptedRunId,
|
||||
ack: deliveryId,
|
||||
wait: true,
|
||||
timeoutMs: 10_000
|
||||
},
|
||||
'check-ack-takeover'
|
||||
)
|
||||
)
|
||||
await waitStarted
|
||||
expect(harness.db.getMessageById(incoming.id)?.read).toBe(1)
|
||||
settleLegacyWorkerAndTakeOver(harness, 'settled during acknowledged wait')
|
||||
resolveWait?.()
|
||||
|
||||
await expect(pending).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
acknowledged: deliveryId,
|
||||
count: 0,
|
||||
waitInterrupted: 'consumer_fenced',
|
||||
mutation: { requestId: 'check-ack-takeover', replayed: false }
|
||||
}
|
||||
})
|
||||
const sqlite = (harness.db as unknown as { db: Database.Database }).db
|
||||
expect(sqlite.prepare('SELECT state FROM mutation_receipts').get()).toEqual({
|
||||
state: 'completed'
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps concurrent duplicates on the live ACK-and-wait result', async () => {
|
||||
const harness = createHarness()
|
||||
harness.db.insertMessage({
|
||||
runId: harness.adoptedRunId,
|
||||
from: WORKER_HANDLE,
|
||||
to: `run:${harness.adoptedRunId}`,
|
||||
subject: 'first delivery'
|
||||
})
|
||||
const firstCheck = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{ terminal: COORDINATOR_HANDLE, run: harness.adoptedRunId },
|
||||
'check-first-delivery'
|
||||
)
|
||||
)
|
||||
const deliveryId = (firstCheck as { result: { deliveryId: string } }).result.deliveryId
|
||||
let resolveWait: (() => void) | undefined
|
||||
let signalWaitStarted: (() => void) | undefined
|
||||
const waitStarted = new Promise<void>((resolve) => {
|
||||
signalWaitStarted = resolve
|
||||
})
|
||||
vi.spyOn(harness.runtime, 'waitForMessage').mockImplementation(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
resolveWait = () => resolve('notified')
|
||||
signalWaitStarted?.()
|
||||
})
|
||||
)
|
||||
const ackRequest = request(
|
||||
'orchestration.check',
|
||||
{
|
||||
terminal: COORDINATOR_HANDLE,
|
||||
run: harness.adoptedRunId,
|
||||
ack: deliveryId,
|
||||
wait: true,
|
||||
timeoutMs: 10_000
|
||||
},
|
||||
'check-ack-concurrent'
|
||||
)
|
||||
const original = harness.dispatcher.dispatch(ackRequest)
|
||||
await waitStarted
|
||||
const secondLiveDispatcher = new RpcDispatcher({
|
||||
runtime: harness.runtime,
|
||||
methods: ORCHESTRATION_METHODS
|
||||
})
|
||||
const duplicate = secondLiveDispatcher.dispatch({
|
||||
...ackRequest,
|
||||
id: 'rpc_check-ack-concurrent-duplicate'
|
||||
})
|
||||
let duplicateSettled = false
|
||||
void duplicate.then(() => {
|
||||
duplicateSettled = true
|
||||
})
|
||||
await Promise.resolve()
|
||||
expect(duplicateSettled).toBe(false)
|
||||
|
||||
const arrived = harness.db.insertMessage({
|
||||
runId: harness.adoptedRunId,
|
||||
from: WORKER_HANDLE,
|
||||
to: `run:${harness.adoptedRunId}`,
|
||||
subject: 'arrived while waiting'
|
||||
})
|
||||
resolveWait?.()
|
||||
const [originalResult, duplicateResult] = await Promise.all([original, duplicate])
|
||||
expect(originalResult).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
messages: [{ id: arrived.id }],
|
||||
mutation: { requestId: 'check-ack-concurrent', replayed: false }
|
||||
}
|
||||
})
|
||||
expect(duplicateResult).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
messages: [{ id: arrived.id }],
|
||||
mutation: { requestId: 'check-ack-concurrent', replayed: true }
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it.each(['dispatch', 'websocket'] as const)(
|
||||
'%s rejects run-use after takeover during durable preflight',
|
||||
async (transport) => {
|
||||
const harness = createHarness()
|
||||
installTakeoverDuringMutationPreflight(harness)
|
||||
|
||||
const response = await invoke(
|
||||
harness.dispatcher,
|
||||
request(
|
||||
'orchestration.runUse',
|
||||
{ id: harness.adoptedRunId, from: COORDINATOR_HANDLE },
|
||||
`run-use-takeover-${transport}`
|
||||
),
|
||||
transport
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
|
||||
expect(harness.db.getRun(harness.adoptedRunId)).toMatchObject({
|
||||
coordinator_handle: 'term_current_coord',
|
||||
consumer_generation: 1
|
||||
})
|
||||
expect(mutationReceiptCount(harness.db)).toBe(0)
|
||||
expect(harness.db.getLegacyCoordinatorPrincipal(harness.adoptedRunId)?.status).toBe('revoked')
|
||||
}
|
||||
)
|
||||
|
||||
it.each(['dispatch', 'websocket'] as const)(
|
||||
'%s rejects a mutation after takeover during durable preflight',
|
||||
async (transport) => {
|
||||
const harness = createHarness()
|
||||
const target = harness.db.createTask({
|
||||
spec: 'must remain ready',
|
||||
runId: harness.adoptedRunId
|
||||
})
|
||||
installTakeoverDuringMutationPreflight(harness)
|
||||
|
||||
const response = await invoke(
|
||||
harness.dispatcher,
|
||||
request(
|
||||
'orchestration.taskUpdate',
|
||||
{
|
||||
id: target.id,
|
||||
status: 'completed',
|
||||
callerTerminalHandle: COORDINATOR_HANDLE
|
||||
},
|
||||
`task-update-takeover-${transport}`
|
||||
),
|
||||
transport
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
|
||||
expect(harness.db.getTask(target.id)?.status).toBe('ready')
|
||||
expect(mutationReceiptCount(harness.db)).toBe(0)
|
||||
}
|
||||
)
|
||||
|
||||
it('revalidates after asynchronous agent detection', async () => {
|
||||
const harness = createHarness()
|
||||
const target = harness.db.createTask({
|
||||
spec: 'must not dispatch after takeover',
|
||||
runId: harness.adoptedRunId
|
||||
})
|
||||
const targetHandle = 'term_current_worker'
|
||||
const targetPane = 'tab_current_worker:77777777-7777-4777-8777-777777777777'
|
||||
vi.spyOn(harness.runtime, 'getTerminalPaneKey').mockImplementation((handle: string) =>
|
||||
handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : handle === targetHandle ? targetPane : null
|
||||
)
|
||||
vi.spyOn(harness.runtime, 'getTerminalProcessIncarnation').mockReturnValue('current-process')
|
||||
let resolveDetection: ((detected: boolean) => void) | undefined
|
||||
let signalDetectionStarted: (() => void) | undefined
|
||||
const detectionStarted = new Promise<void>((resolve) => {
|
||||
signalDetectionStarted = resolve
|
||||
})
|
||||
vi.spyOn(harness.runtime, 'isTerminalRunningAgent').mockImplementation(
|
||||
() =>
|
||||
new Promise<boolean>((resolve) => {
|
||||
resolveDetection = resolve
|
||||
signalDetectionStarted?.()
|
||||
})
|
||||
)
|
||||
const sendPrompt = vi
|
||||
.spyOn(harness.runtime, 'sendTerminalAgentPrompt')
|
||||
.mockResolvedValue({ handle: targetHandle, accepted: true, bytesWritten: 1 })
|
||||
|
||||
const pending = harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.dispatch',
|
||||
{ task: target.id, to: targetHandle, from: COORDINATOR_HANDLE, inject: true },
|
||||
'dispatch-detection-takeover'
|
||||
)
|
||||
)
|
||||
await detectionStarted
|
||||
settleLegacyWorkerAndTakeOver(harness, 'settled during detection')
|
||||
resolveDetection?.(true)
|
||||
const response = await pending
|
||||
|
||||
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
|
||||
expect(harness.db.getTask(target.id)?.status).toBe('ready')
|
||||
expect(harness.db.getDispatchContext(target.id)).toBeUndefined()
|
||||
expect(sendPrompt).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('explicitly takes over an active adopted Run without stranding its legacy worker', async () => {
|
||||
const harness = createHarness()
|
||||
const currentHandle = 'term_current_coord'
|
||||
const currentPane = 'tab_current:55555555-5555-4555-8555-555555555555'
|
||||
vi.mocked(harness.runtime.getTerminalPaneKey).mockImplementation((handle) =>
|
||||
handle === COORDINATOR_HANDLE
|
||||
? COORDINATOR_PANE
|
||||
: handle === WORKER_HANDLE
|
||||
? WORKER_PANE
|
||||
: handle === currentHandle
|
||||
? currentPane
|
||||
: null
|
||||
)
|
||||
|
||||
await expect(
|
||||
harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.runUse',
|
||||
{ id: harness.adoptedRunId, from: COORDINATOR_HANDLE },
|
||||
'bind-original-coordinator'
|
||||
)
|
||||
)
|
||||
).resolves.toMatchObject({ ok: true })
|
||||
const pendingAsk = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.ask',
|
||||
{
|
||||
from: WORKER_HANDLE,
|
||||
to: COORDINATOR_HANDLE,
|
||||
question: 'Continue with the migration?',
|
||||
timeoutMs: 0
|
||||
},
|
||||
'pending-before-takeover',
|
||||
evidence('worker')
|
||||
)
|
||||
)
|
||||
const questionId = (pendingAsk as { result: { messageId: string } }).result.messageId
|
||||
const unrelated = harness.db.insertMessage({
|
||||
runId: harness.adoptedRunId,
|
||||
from: WORKER_HANDLE,
|
||||
to: 'term_peer_worker',
|
||||
subject: 'peer-only',
|
||||
deliveryContract: 'legacy_direct'
|
||||
})
|
||||
|
||||
const currentRequest = (method: string, params: unknown, invocationId: string): RpcRequest => ({
|
||||
...request(method, params, invocationId),
|
||||
orchestrationCompatibilityEvidence: undefined
|
||||
})
|
||||
const takeoverParams = {
|
||||
id: harness.adoptedRunId,
|
||||
from: currentHandle,
|
||||
takeoverLegacy: true
|
||||
}
|
||||
const takeover = await harness.dispatcher.dispatch(
|
||||
currentRequest('orchestration.runUse', takeoverParams, 'explicit-takeover')
|
||||
)
|
||||
const repeated = await harness.dispatcher.dispatch(
|
||||
currentRequest('orchestration.runUse', takeoverParams, 'explicit-takeover-repeat')
|
||||
)
|
||||
|
||||
expect(takeover).toMatchObject({
|
||||
ok: true,
|
||||
result: { binding: { consumerGeneration: 2 } }
|
||||
})
|
||||
expect(repeated).toMatchObject({
|
||||
ok: true,
|
||||
result: { binding: { consumerGeneration: 2 } }
|
||||
})
|
||||
expect(harness.db.getDispatchContextById(harness.dispatchId)?.status).toBe('dispatched')
|
||||
expect(harness.db.getLegacyCoordinatorPrincipal(harness.adoptedRunId)?.status).toBe('revoked')
|
||||
expect(harness.db.getMessageById(unrelated.id)).toMatchObject({
|
||||
to_handle: 'term_peer_worker',
|
||||
delivery_contract: 'legacy_direct'
|
||||
})
|
||||
|
||||
const promoted = await harness.dispatcher.dispatch(
|
||||
currentRequest(
|
||||
'orchestration.check',
|
||||
{ terminal: currentHandle, run: harness.adoptedRunId, format: true },
|
||||
'read-promoted-question'
|
||||
)
|
||||
)
|
||||
expect(promoted).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
messages: [
|
||||
{
|
||||
id: questionId,
|
||||
to_handle: `run:${harness.adoptedRunId}`,
|
||||
delivery_contract: 'current_delivery'
|
||||
}
|
||||
],
|
||||
formatted: expect.not.stringContaining(`--from run:${harness.adoptedRunId}`)
|
||||
}
|
||||
})
|
||||
const deliveryId = (promoted as { result: { deliveryId: string } }).result.deliveryId
|
||||
await expect(
|
||||
harness.dispatcher.dispatch(
|
||||
currentRequest(
|
||||
'orchestration.reply',
|
||||
{
|
||||
id: questionId,
|
||||
body: 'Yes, continue.',
|
||||
from: currentHandle,
|
||||
run: harness.adoptedRunId
|
||||
},
|
||||
'reply-after-takeover'
|
||||
)
|
||||
)
|
||||
).resolves.toMatchObject({ ok: true, result: { question: { status: 'answered' } } })
|
||||
await expect(
|
||||
harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.ask',
|
||||
{ from: WORKER_HANDLE, resume: questionId, timeoutMs: 100 },
|
||||
'resume-after-takeover',
|
||||
evidence('worker')
|
||||
)
|
||||
)
|
||||
).resolves.toMatchObject({ ok: true, result: { answer: 'Yes, continue.' } })
|
||||
|
||||
const followUp = await harness.dispatcher.dispatch(
|
||||
currentRequest(
|
||||
'orchestration.send',
|
||||
{
|
||||
from: currentHandle,
|
||||
to: `dispatch:${harness.dispatchId}`,
|
||||
subject: 'Continue from the new coordinator'
|
||||
},
|
||||
'current-coordinator-to-legacy-worker'
|
||||
)
|
||||
)
|
||||
expect(followUp).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
message: {
|
||||
run_id: harness.adoptedRunId,
|
||||
to_handle: `dispatch:${harness.dispatchId}`,
|
||||
delivery_contract: 'legacy_direct'
|
||||
}
|
||||
}
|
||||
})
|
||||
const workerFollowUp = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{ terminal: WORKER_HANDLE },
|
||||
'legacy-worker-check-after-takeover',
|
||||
evidence('worker')
|
||||
)
|
||||
)
|
||||
expect(workerFollowUp).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
messages: [{ subject: 'Continue from the new coordinator' }],
|
||||
legacyCompatibility: { ackMessageIds: [expect.any(String)] }
|
||||
}
|
||||
})
|
||||
const [followUpMessageId] = (
|
||||
workerFollowUp as { result: { legacyCompatibility: { ackMessageIds: string[] } } }
|
||||
).result.legacyCompatibility.ackMessageIds
|
||||
await expect(
|
||||
harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{
|
||||
terminal: WORKER_HANDLE,
|
||||
compatibilityAck: JSON.stringify({ messageIds: [followUpMessageId] })
|
||||
},
|
||||
'legacy-worker-ack-after-takeover',
|
||||
evidence('worker')
|
||||
)
|
||||
)
|
||||
).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
acknowledged: [followUpMessageId],
|
||||
legacyCompatibility: { acknowledged: true }
|
||||
}
|
||||
})
|
||||
|
||||
await harness.dispatcher.dispatch(
|
||||
currentRequest(
|
||||
'orchestration.check',
|
||||
{ terminal: currentHandle, run: harness.adoptedRunId, ack: deliveryId },
|
||||
'ack-promoted-question'
|
||||
)
|
||||
)
|
||||
await expect(
|
||||
harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.send',
|
||||
{
|
||||
from: WORKER_HANDLE,
|
||||
to: COORDINATOR_HANDLE,
|
||||
subject: 'Need current coordinator',
|
||||
type: 'escalation',
|
||||
payload: JSON.stringify({ taskId: harness.taskId })
|
||||
},
|
||||
'escalation-after-takeover',
|
||||
evidence('worker')
|
||||
)
|
||||
)
|
||||
).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
message: {
|
||||
to_handle: `run:${harness.adoptedRunId}`,
|
||||
delivery_contract: 'current_delivery'
|
||||
}
|
||||
}
|
||||
})
|
||||
await expect(
|
||||
harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.taskList',
|
||||
{ callerTerminalHandle: COORDINATOR_HANDLE },
|
||||
'old-coordinator-fenced'
|
||||
)
|
||||
)
|
||||
).resolves.toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
|
||||
})
|
||||
})
|
||||
@@ -104,7 +104,7 @@ export async function handleLegacyLifecycleSend(args: {
|
||||
: { kind: 'message_only' }
|
||||
})
|
||||
if (!committed.duplicate) {
|
||||
runtime.notifyMessageArrived(params.to, committed.message.type)
|
||||
runtime.notifyMessageArrived(committed.message.to_handle, committed.message.type)
|
||||
}
|
||||
return {
|
||||
message: committed.message,
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import { parsePaneKey } from '../../../shared/stable-pane-id'
|
||||
import { OrchestrationError } from '../orchestration/orchestration-error'
|
||||
|
||||
export function equivalentLegacyPaneKey(
|
||||
a: string | null | undefined,
|
||||
b: string | null | undefined
|
||||
): boolean {
|
||||
if (!a || !b) {
|
||||
return false
|
||||
}
|
||||
if (a === b) {
|
||||
return true
|
||||
}
|
||||
const aLeaf = parsePaneKey(a)?.leafId
|
||||
const bLeaf = parsePaneKey(b)?.leafId
|
||||
return Boolean(aLeaf && bLeaf && aLeaf === bLeaf)
|
||||
}
|
||||
|
||||
export function legacyReadOnlyError(): OrchestrationError {
|
||||
return new OrchestrationError(
|
||||
'legacy_read_only',
|
||||
'This retained legacy assignment could not prove authority from its original live process. No effects were applied.',
|
||||
{ effectsApplied: false }
|
||||
)
|
||||
}
|
||||
|
||||
export function legacyCoordinatorReadOnly(): OrchestrationError {
|
||||
return new OrchestrationError(
|
||||
'legacy_read_only',
|
||||
'This retained legacy coordinator could not prove its original process identity. No effects were applied.',
|
||||
{ effectsApplied: false }
|
||||
)
|
||||
}
|
||||
@@ -65,8 +65,9 @@ export async function handleLegacyAsk(args: {
|
||||
options,
|
||||
recipientHandle: recipient
|
||||
})
|
||||
const pending = matches.filter((match) => match.question.status === 'pending')
|
||||
const lostAnswer = matches.find(
|
||||
const unclaimed = matches.filter((match) => !match.claimedByOperation)
|
||||
const pending = unclaimed.filter((match) => match.question.status === 'pending')
|
||||
const lostAnswer = unclaimed.find(
|
||||
(match) => match.question.status === 'answered' && !match.answerAcknowledged
|
||||
)
|
||||
if (lostAnswer) {
|
||||
@@ -98,7 +99,7 @@ export async function handleLegacyAsk(args: {
|
||||
questionId = committed.question.message_id
|
||||
duplicate = committed.duplicate || Boolean(existingQuestionId)
|
||||
if (!duplicate) {
|
||||
runtime.notifyMessageArrived(recipient, committed.message.type)
|
||||
runtime.notifyMessageArrived(committed.message.to_handle, committed.message.type)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -55,6 +55,10 @@ export class OrchestrationMutationExecutor {
|
||||
: db.beginMutationReceipt(identity)
|
||||
|
||||
if (begun.disposition === 'completed') {
|
||||
const active = this.inFlight.get(key)
|
||||
if (active) {
|
||||
return attachMutationReceipt(await active, requestId, true)
|
||||
}
|
||||
return attachMutationReceipt(JSON.parse(begun.row.receipt ?? 'null'), requestId, true)
|
||||
}
|
||||
if (begun.disposition === 'pending') {
|
||||
@@ -102,6 +106,20 @@ export class OrchestrationMutationExecutor {
|
||||
}
|
||||
}
|
||||
|
||||
const executorsByRuntime = new WeakMap<OrcaRuntimeService, OrchestrationMutationExecutor>()
|
||||
|
||||
export function getOrchestrationMutationExecutor(
|
||||
runtime: OrcaRuntimeService
|
||||
): OrchestrationMutationExecutor {
|
||||
const existing = executorsByRuntime.get(runtime)
|
||||
if (existing) {
|
||||
return existing
|
||||
}
|
||||
const executor = new OrchestrationMutationExecutor(runtime)
|
||||
executorsByRuntime.set(runtime, executor)
|
||||
return executor
|
||||
}
|
||||
|
||||
export function authenticatedCallerFingerprint(request: RpcRequest): string {
|
||||
const callerToken =
|
||||
request.authToken ||
|
||||
|
||||
@@ -252,7 +252,10 @@ describe('durable orchestration mutation ledger', () => {
|
||||
})
|
||||
|
||||
it('recovers a lost ask acceptance without creating a second question', async () => {
|
||||
const db = new OrchestrationDb(':memory:')
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-mutation-ask-recovery-'))
|
||||
paths.push(dir)
|
||||
const dbPath = join(dir, 'orchestration.db')
|
||||
const db = new OrchestrationDb(dbPath)
|
||||
const runtime = new OrcaRuntimeService()
|
||||
runtime.setOrchestrationDb(db)
|
||||
vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_worker:leaf_worker')
|
||||
@@ -290,7 +293,16 @@ describe('durable orchestration mutation ledger', () => {
|
||||
const first = firstDispatcher.dispatch(askRequest, { signal: controller.signal })
|
||||
await vi.waitFor(() => expect(db.getInbox(10)).toHaveLength(1))
|
||||
|
||||
const restartedDispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS })
|
||||
const restartedDb = new OrchestrationDb(dbPath)
|
||||
const restartedRuntime = new OrcaRuntimeService()
|
||||
restartedRuntime.setOrchestrationDb(restartedDb)
|
||||
vi.spyOn(restartedRuntime, 'getTerminalPaneKey').mockReturnValue('tab_worker:leaf_worker')
|
||||
vi.spyOn(restartedRuntime, 'getTerminalProcessIncarnation').mockReturnValue('runtime:pty:1')
|
||||
vi.spyOn(restartedRuntime, 'notifyMessageArrived').mockImplementation(() => {})
|
||||
const restartedDispatcher = new RpcDispatcher({
|
||||
runtime: restartedRuntime,
|
||||
methods: ORCHESTRATION_METHODS
|
||||
})
|
||||
const recovered = await restartedDispatcher.dispatch({ ...askRequest, id: 'rpc_ask_2' })
|
||||
expect(recovered).toMatchObject({
|
||||
ok: true,
|
||||
@@ -304,6 +316,7 @@ describe('durable orchestration mutation ledger', () => {
|
||||
|
||||
controller.abort()
|
||||
await first
|
||||
restartedDb.close()
|
||||
db.close()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { FeatureInteractionId } from '../../../shared/feature-interactions'
|
||||
import { isBrowserPaneUiRuntimeRpcParams } from '../../../shared/runtime-rpc-feature-interaction-source'
|
||||
import type { OrcaRuntimeService } from '../orca-runtime'
|
||||
|
||||
export function getRuntimeFeatureInteractionId(
|
||||
method: string,
|
||||
@@ -37,6 +38,25 @@ export function getRuntimeFeatureInteractionId(
|
||||
return method.startsWith('orchestration.') ? 'agent-orchestration' : null
|
||||
}
|
||||
|
||||
export function recordRuntimeFeatureInteraction(
|
||||
runtime: OrcaRuntimeService,
|
||||
method: string,
|
||||
result: unknown,
|
||||
alreadyRecorded?: Set<FeatureInteractionId>,
|
||||
rawParams?: unknown
|
||||
): void {
|
||||
const id = getRuntimeFeatureInteractionId(method, result, rawParams)
|
||||
if (!id || alreadyRecorded?.has(id)) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
runtime.recordFeatureInteraction(id)
|
||||
alreadyRecorded?.add(id)
|
||||
} catch {
|
||||
// Best-effort education state must not break runtime tools.
|
||||
}
|
||||
}
|
||||
|
||||
function hasBooleanResult(value: unknown, key: string): boolean {
|
||||
return (
|
||||
value !== null && typeof value === 'object' && (value as Record<string, unknown>)[key] === true
|
||||
|
||||
@@ -66,6 +66,10 @@ import {
|
||||
import type { Store } from '../persistence'
|
||||
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
|
||||
import { runRemoteOrcaCli } from './ssh-remote-orca-cli'
|
||||
import {
|
||||
acknowledgeRemoteOrcaCliPostOutput,
|
||||
parseRemoteOrcaCliPostOutput
|
||||
} from './ssh-remote-orchestration-post-output'
|
||||
import { toSshExecutionHostId, type ExecutionHostId } from '../../shared/execution-host'
|
||||
import { isTerminalLeafId, makePaneKey } from '../../shared/stable-pane-id'
|
||||
import { isValidTerminalTabId } from '../../shared/terminal-tab-id'
|
||||
@@ -716,6 +720,37 @@ export class SshRelaySession {
|
||||
this.runtime.releaseOrchestrationCompatibilitySshAttachment(runtimeAuthority.attachmentId)
|
||||
}
|
||||
})
|
||||
mux.onRequest('orca.cli.postOutput', async (params) => {
|
||||
if (!this.runtime) {
|
||||
throw new Error('Orca runtime is unavailable')
|
||||
}
|
||||
const rawEnv = params.env
|
||||
const env =
|
||||
rawEnv && typeof rawEnv === 'object' && !Array.isArray(rawEnv)
|
||||
? Object.fromEntries(
|
||||
Object.entries(rawEnv).filter(
|
||||
(entry): entry is [string, string] =>
|
||||
typeof entry[0] === 'string' && typeof entry[1] === 'string'
|
||||
)
|
||||
)
|
||||
: {}
|
||||
const runtimeAuthority = this.runtime.registerOrchestrationCompatibilitySshAttachment(
|
||||
this.targetId,
|
||||
connectionIncarnation
|
||||
)
|
||||
this.activeCompatibilityAttachmentIds.add(runtimeAuthority.attachmentId)
|
||||
try {
|
||||
await acknowledgeRemoteOrcaCliPostOutput(this.runtime, {
|
||||
postOutput: parseRemoteOrcaCliPostOutput(params.postOutput),
|
||||
env,
|
||||
runtimeAuthority
|
||||
})
|
||||
return { acknowledged: true }
|
||||
} finally {
|
||||
this.activeCompatibilityAttachmentIds.delete(runtimeAuthority.attachmentId)
|
||||
this.runtime.releaseOrchestrationCompatibilitySshAttachment(runtimeAuthority.attachmentId)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Why: ship plugin/extension source from Orca so agent-event changes don't force a relay redeploy (agent-status-over-ssh.md §4/§8). Best-effort.
|
||||
|
||||
@@ -43,8 +43,23 @@ export type RemoteOrcaCliResult = {
|
||||
stdout: string
|
||||
stderr: string
|
||||
exitCode: number
|
||||
postOutput?: RemoteOrcaCliPostOutput
|
||||
}
|
||||
|
||||
export type RemoteOrcaCliPostOutput =
|
||||
| {
|
||||
kind: 'legacy_check_ack'
|
||||
terminal: string
|
||||
messageIds: string[]
|
||||
types?: string[]
|
||||
}
|
||||
| {
|
||||
kind: 'legacy_question_ack'
|
||||
terminal: string
|
||||
questionId: string
|
||||
answerMessageId: string
|
||||
}
|
||||
|
||||
export type HostCliPassthroughOptions = {
|
||||
execPath?: string
|
||||
cliEntryPath?: string
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import type { RpcResponse } from '../runtime/rpc/core'
|
||||
import type { ParsedRemoteCli } from './ssh-remote-cli-argument-error'
|
||||
import type { RemoteOrcaCliResult } from './ssh-remote-cli-host-passthrough'
|
||||
import { formatRemoteCli } from './ssh-remote-cli-format'
|
||||
import {
|
||||
formatRemoteOrchestrationAsk,
|
||||
getRemoteCliExitCode
|
||||
} from './ssh-remote-orchestration-ask-output'
|
||||
import { formatRemoteOrchestrationCheck } from './ssh-remote-orchestration-check-output'
|
||||
import { getRemoteCliPostOutput } from './ssh-remote-orchestration-post-output'
|
||||
import { resolveRemoteCliHandle } from './ssh-remote-cli-args'
|
||||
|
||||
export function formatInProcessRemoteCliResult(
|
||||
parsed: ParsedRemoteCli,
|
||||
env: Record<string, string>,
|
||||
response: RpcResponse,
|
||||
json: boolean
|
||||
): RemoteOrcaCliResult {
|
||||
const command = parsed.commandPath.join(' ')
|
||||
const formatted =
|
||||
command === 'orchestration check'
|
||||
? formatRemoteOrchestrationCheck(
|
||||
response,
|
||||
json,
|
||||
resolveRemoteCliHandle(parsed.flags, env, 'terminal'),
|
||||
parsed.flags.has('format')
|
||||
)
|
||||
: command === 'orchestration ask'
|
||||
? formatRemoteOrchestrationAsk(response, json)
|
||||
: json
|
||||
? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' }
|
||||
: formatRemoteCli(response)
|
||||
const postOutput = getRemoteCliPostOutput(parsed, env, response)
|
||||
return {
|
||||
stdout: formatted.stdout,
|
||||
stderr: formatted.stderr,
|
||||
exitCode: getRemoteCliExitCode(command, response),
|
||||
...(postOutput ? { postOutput } : {})
|
||||
}
|
||||
}
|
||||
@@ -24,7 +24,6 @@ const LEGACY_FALLBACK_OPTIONS: HostCliPassthroughOptions = {
|
||||
userDataPath: '/host/user-data',
|
||||
entryExists: () => false
|
||||
}
|
||||
|
||||
type FakeChild = EventEmitter & {
|
||||
stdout: EventEmitter
|
||||
stderr: EventEmitter
|
||||
|
||||
@@ -6,7 +6,6 @@ import { ORCHESTRATION_CONTRACT_VERSION } from '../../shared/protocol-version'
|
||||
import { RpcDispatcher } from '../runtime/rpc/dispatcher'
|
||||
import type { RpcResponse } from '../runtime/rpc/core'
|
||||
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
|
||||
import { formatRemoteCli } from './ssh-remote-cli-format'
|
||||
import {
|
||||
HostCliUnavailableError,
|
||||
runHostOrcaCliPassthrough,
|
||||
@@ -26,9 +25,9 @@ import { buildRemoteCliError } from './ssh-remote-cli-error-response'
|
||||
import { getRemoteLinearHelp, tryDispatchRemoteLinearCli } from './ssh-remote-linear-cli'
|
||||
import {
|
||||
getRemoteOrchestrationPayload,
|
||||
hasRemoteLifecycleRejection,
|
||||
resolveRemoteOrchestrationSender
|
||||
} from './ssh-remote-orchestration-send'
|
||||
import { formatInProcessRemoteCliResult } from './ssh-remote-cli-in-process-result'
|
||||
|
||||
export type { RemoteOrcaCliRequest, RemoteOrcaCliResult } from './ssh-remote-cli-host-passthrough'
|
||||
|
||||
@@ -51,6 +50,7 @@ export async function runRemoteOrcaCli(
|
||||
): Promise<RemoteOrcaCliResult> {
|
||||
const parsed = parseRemoteCliArgs(request.argv)
|
||||
const json = parsed.flags.has('json')
|
||||
const command = parsed.commandPath.join(' ')
|
||||
|
||||
const interactiveMessage = HOST_INTERACTIVE_COMMANDS[parsed.commandPath[0] ?? '']
|
||||
if (interactiveMessage) {
|
||||
@@ -64,6 +64,17 @@ export async function runRemoteOrcaCli(
|
||||
return { stdout: '', stderr: `${interactiveMessage}\n`, exitCode: 1 }
|
||||
}
|
||||
|
||||
if (command === 'orchestration check' || command === 'orchestration ask') {
|
||||
// Why: compatibility ACKs must wait until relay stdout is observable; a host CLI child can only flush into main's capture pipe.
|
||||
return await runLegacyRemoteOrcaCli(
|
||||
runtime,
|
||||
request,
|
||||
parsed,
|
||||
json,
|
||||
new HostCliUnavailableError('output-ordered orchestration bridge required')
|
||||
)
|
||||
}
|
||||
|
||||
let passthroughFailure: HostCliUnavailableError | null = null
|
||||
try {
|
||||
return await runHostOrcaCliPassthrough(request, passthroughOptions)
|
||||
@@ -101,16 +112,7 @@ async function runLegacyRemoteOrcaCli(
|
||||
passthroughFailure.message,
|
||||
request.runtimeAuthority
|
||||
)
|
||||
const formatted = json
|
||||
? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' }
|
||||
: formatRemoteCli(response)
|
||||
return {
|
||||
stdout: formatted.stdout,
|
||||
stderr: formatted.stderr,
|
||||
// Why: the legacy SSH bridge bypasses the local CLI handler that turns
|
||||
// a persisted lifecycle rejection into an unsuccessful command.
|
||||
exitCode: response.ok && !hasRemoteLifecycleRejection(response.result) ? 0 : 1
|
||||
}
|
||||
return formatInProcessRemoteCliResult(parsed, request.env, response, json)
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err)
|
||||
const code =
|
||||
@@ -141,13 +143,17 @@ async function dispatchRemoteCli(
|
||||
runtimeAuthority: RemoteOrcaCliRequest['runtimeAuthority']
|
||||
): Promise<RpcResponse> {
|
||||
const command = parsed.commandPath.join(' ')
|
||||
const compatibilityInvocationId = randomUUID()
|
||||
const inheritedEvidence = readOrchestrationCompatibilityEvidence(env)
|
||||
const orchestrationCompatibilityEvidence = runtimeAuthority
|
||||
? { ...inheritedEvidence, host: runtimeAuthority }
|
||||
: inheritedEvidence
|
||||
const compatibilityEnvelope: RuntimeOrchestrationEnvelope = {
|
||||
compatibilityInvocationId,
|
||||
compatibilityInvocationId: randomUUID(),
|
||||
orchestrationRequestId:
|
||||
optionalRemoteCliString(parsed.flags, 'retry-request') ??
|
||||
(command === 'orchestration check' || command === 'orchestration ask'
|
||||
? randomUUID()
|
||||
: undefined),
|
||||
orchestrationCompatibilityEvidence
|
||||
}
|
||||
const linearResponse = await tryDispatchRemoteLinearCli(dispatcher, parsed, env, stdin)
|
||||
@@ -201,8 +207,7 @@ async function dispatchRemoteCli(
|
||||
},
|
||||
{
|
||||
...compatibilityEnvelope,
|
||||
orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability'),
|
||||
orchestrationRequestId: optionalRemoteCliString(parsed.flags, 'retry-request')
|
||||
orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability')
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -215,15 +220,39 @@ async function dispatchRemoteCli(
|
||||
terminalPaneKey: parsed.flags.has('terminal')
|
||||
? undefined
|
||||
: env.ORCA_PANE_KEY || undefined,
|
||||
unread: parsed.flags.has('unread') ? true : undefined,
|
||||
unread: parsed.flags.has('unread') ? true : parsed.flags.has('peek') ? false : undefined,
|
||||
peek: parsed.flags.has('peek') ? true : undefined,
|
||||
all: parsed.flags.has('all') ? true : undefined,
|
||||
types: optionalRemoteCliString(parsed.flags, 'types'),
|
||||
format: parsed.flags.has('format') ? true : undefined,
|
||||
inject: parsed.flags.has('inject') ? true : undefined,
|
||||
compatibilityCliCommand: 'orca',
|
||||
run: optionalRemoteCliString(parsed.flags, 'run'),
|
||||
ack: optionalRemoteCliString(parsed.flags, 'ack'),
|
||||
wait: parsed.flags.has('wait') ? true : undefined,
|
||||
timeoutMs: optionalRemoteCliNumber(parsed.flags, 'timeout-ms')
|
||||
},
|
||||
compatibilityEnvelope
|
||||
)
|
||||
case 'orchestration ask':
|
||||
return await call(
|
||||
dispatcher,
|
||||
'orchestration.ask',
|
||||
{
|
||||
to: optionalRemoteCliString(parsed.flags, 'to'),
|
||||
question: optionalRemoteCliString(parsed.flags, 'question'),
|
||||
resume: optionalRemoteCliString(parsed.flags, 'resume'),
|
||||
options: optionalRemoteCliString(parsed.flags, 'options'),
|
||||
timeoutMs: optionalRemoteCliNumber(parsed.flags, 'timeout-ms'),
|
||||
from: resolveRemoteCliHandle(parsed.flags, env, 'from'),
|
||||
run: optionalRemoteCliString(parsed.flags, 'run'),
|
||||
compatibilityCliCommand: 'orca'
|
||||
},
|
||||
{
|
||||
...compatibilityEnvelope,
|
||||
orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability')
|
||||
}
|
||||
)
|
||||
case 'orchestration reply':
|
||||
return await call(
|
||||
dispatcher,
|
||||
@@ -271,7 +300,8 @@ async function call(
|
||||
? ORCHESTRATION_CONTRACT_VERSION
|
||||
: undefined,
|
||||
orchestrationRequestId: envelope?.orchestrationRequestId,
|
||||
compatibilityInvocationId: envelope?.compatibilityInvocationId,
|
||||
compatibilityInvocationId:
|
||||
envelope?.orchestrationRequestId ?? envelope?.compatibilityInvocationId,
|
||||
orchestrationCompatibilityEvidence: envelope?.orchestrationCompatibilityEvidence
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import type { RpcResponse } from '../runtime/rpc/core'
|
||||
import { formatRemoteCli } from './ssh-remote-cli-format'
|
||||
import { hasRemoteLifecycleRejection } from './ssh-remote-orchestration-send'
|
||||
|
||||
export function formatRemoteOrchestrationAsk(
|
||||
response: RpcResponse,
|
||||
json: boolean
|
||||
): { stdout: string; stderr: string } {
|
||||
if (!response.ok || !isRecord(response.result)) {
|
||||
return json
|
||||
? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' }
|
||||
: formatRemoteCli(response)
|
||||
}
|
||||
if (json) {
|
||||
return { stdout: `${JSON.stringify(response.result)}\n`, stderr: '' }
|
||||
}
|
||||
if (isRecord(response.result.legacyCompatibility)) {
|
||||
const compatibility = response.result.legacyCompatibility
|
||||
if (compatibility.resumeRequired === true && typeof compatibility.resumeCommand === 'string') {
|
||||
return {
|
||||
stdout:
|
||||
`Question ${String(response.result.messageId ?? 'unknown')} committed.\n` +
|
||||
`Resume with: ${compatibility.resumeCommand}\n`,
|
||||
stderr: ''
|
||||
}
|
||||
}
|
||||
}
|
||||
const answer = typeof response.result.answer === 'string' ? response.result.answer : ''
|
||||
const thread = typeof response.result.threadId === 'string' ? response.result.threadId : 'unknown'
|
||||
const timeoutMs =
|
||||
typeof response.result.timeoutMs === 'number' ? response.result.timeoutMs : undefined
|
||||
const stderr = response.result.timedOut
|
||||
? `ask timeout after ${timeoutMs ?? 0}ms (thread ${thread})\n`
|
||||
: response.result.cancelled
|
||||
? response.result.connectionLost
|
||||
? `ask connection closed (question ${String(response.result.messageId ?? 'unknown')})\n`
|
||||
: `ask cancelled (question ${String(response.result.messageId ?? 'unknown')})\n`
|
||||
: ''
|
||||
return { stdout: answer ? `${answer}\n` : '', stderr }
|
||||
}
|
||||
|
||||
export function getRemoteCliExitCode(command: string, response: RpcResponse): number {
|
||||
if (!response.ok || hasRemoteLifecycleRejection(response.result)) {
|
||||
return 1
|
||||
}
|
||||
if (
|
||||
command === 'orchestration ask' &&
|
||||
isRecord(response.result) &&
|
||||
isRecord(response.result.legacyCompatibility) &&
|
||||
response.result.legacyCompatibility.resumeRequired === true
|
||||
) {
|
||||
return 75
|
||||
}
|
||||
if (
|
||||
command === 'orchestration ask' &&
|
||||
isRecord(response.result) &&
|
||||
(response.result.timedOut === true || response.result.cancelled === true)
|
||||
) {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import {
|
||||
formatOrchestrationCheckText,
|
||||
prepareOrchestrationCheckOutput,
|
||||
type OrchestrationCheckOutput,
|
||||
type OrchestrationMessageSummary
|
||||
} from '../../shared/orchestration-check-output'
|
||||
import type { RpcResponse } from '../runtime/rpc/core'
|
||||
import { formatRemoteCli } from './ssh-remote-cli-format'
|
||||
|
||||
export function formatRemoteOrchestrationCheck(
|
||||
response: RpcResponse,
|
||||
json: boolean,
|
||||
terminal: string,
|
||||
formattedRequested: boolean
|
||||
): { stdout: string; stderr: string } {
|
||||
if (!response.ok || !isRecord(response.result)) {
|
||||
return json
|
||||
? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' }
|
||||
: formatRemoteCli(response)
|
||||
}
|
||||
const messages = Array.isArray(response.result.messages)
|
||||
? response.result.messages.filter(isMessageSummary)
|
||||
: []
|
||||
const count = typeof response.result.count === 'number' ? response.result.count : messages.length
|
||||
const result = prepareOrchestrationCheckOutput(
|
||||
{ ...(response.result as OrchestrationCheckOutput), messages, count },
|
||||
terminal,
|
||||
formattedRequested
|
||||
)
|
||||
if (json) {
|
||||
return {
|
||||
stdout: `${JSON.stringify({ ...response, result }, null, 2)}\n`,
|
||||
stderr: ''
|
||||
}
|
||||
}
|
||||
return {
|
||||
stdout: `${formatOrchestrationCheckText(result, terminal)}\n`,
|
||||
stderr: ''
|
||||
}
|
||||
}
|
||||
|
||||
function isMessageSummary(value: unknown): value is OrchestrationMessageSummary {
|
||||
return isRecord(value) && typeof value.id === 'string' && typeof value.from_handle === 'string'
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
|
||||
}
|
||||
@@ -0,0 +1,394 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
app: {
|
||||
isPackaged: false,
|
||||
getAppPath: () => '/host/app'
|
||||
}
|
||||
}))
|
||||
vi.mock('../persistence', () => ({
|
||||
getCanonicalUserDataPath: () => '/host/user-data'
|
||||
}))
|
||||
|
||||
import { OrchestrationDb } from '../runtime/orchestration/db'
|
||||
import { OrcaRuntimeService } from '../runtime/orca-runtime'
|
||||
import type Database from '../sqlite/sync-database'
|
||||
import type { HostCliPassthroughOptions } from './ssh-remote-cli-host-passthrough'
|
||||
import { runRemoteOrcaCli } from './ssh-remote-orca-cli'
|
||||
import { acknowledgeRemoteOrcaCliPostOutput } from './ssh-remote-orchestration-post-output'
|
||||
|
||||
const LEGACY_FALLBACK_OPTIONS: HostCliPassthroughOptions = {
|
||||
execPath: '/host/electron',
|
||||
cliEntryPath: '/host/app/out/cli/index.js',
|
||||
userDataPath: '/host/user-data',
|
||||
entryExists: () => false
|
||||
}
|
||||
const WORKER_HANDLE = 'term_legacy_ssh_worker'
|
||||
const WORKER_PANE = 'tab_legacy_ssh:33333333-3333-4333-8333-333333333333'
|
||||
const COORDINATOR_HANDLE = 'term_legacy_ssh_coord'
|
||||
const COORDINATOR_PANE = 'tab_legacy_coord:44444444-4444-4444-8444-444444444444'
|
||||
const WORKER_ENV = {
|
||||
ORCA_TERMINAL_HANDLE: WORKER_HANDLE,
|
||||
ORCA_PANE_KEY: WORKER_PANE,
|
||||
ORCA_AGENT_LAUNCH_TOKEN: 'legacy-ssh-token'
|
||||
}
|
||||
const COORDINATOR_ENV = {
|
||||
ORCA_TERMINAL_HANDLE: COORDINATOR_HANDLE,
|
||||
ORCA_PANE_KEY: COORDINATOR_PANE,
|
||||
ORCA_AGENT_LAUNCH_TOKEN: 'legacy-ssh-coordinator-token'
|
||||
}
|
||||
const RUNTIME_AUTHORITY = {
|
||||
kind: 'ssh' as const,
|
||||
targetId: 'saved-target',
|
||||
connectionIncarnation: 'connection-1',
|
||||
attachmentId: 'attachment-1'
|
||||
}
|
||||
|
||||
function createLegacyRuntime() {
|
||||
const db = new OrchestrationDb(':memory:')
|
||||
const run = db.createRun({
|
||||
objective: 'Adopted legacy SSH work',
|
||||
coordinatorHandle: COORDINATOR_HANDLE,
|
||||
coordinatorPaneKey: COORDINATOR_PANE
|
||||
})
|
||||
const task = db.createTask({
|
||||
spec: 'legacy SSH assignment',
|
||||
runId: run.id,
|
||||
createdByTerminalHandle: COORDINATOR_HANDLE
|
||||
})
|
||||
const dispatch = db.createDispatchContext(task.id, WORKER_HANDLE, WORKER_PANE)
|
||||
const sqlite = (db as unknown as { db: Database.Database }).db
|
||||
sqlite
|
||||
.prepare(
|
||||
`UPDATE dispatch_contexts
|
||||
SET contract_version = 0, process_incarnation = 'process-1'
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run(dispatch.id)
|
||||
sqlite
|
||||
.prepare(
|
||||
`INSERT INTO legacy_adoptions (source_run_id, adopted_run_id, scheduler_state_lost)
|
||||
VALUES ('run_legacy_local', ?, 1)`
|
||||
)
|
||||
.run(run.id)
|
||||
|
||||
const runtime = new OrcaRuntimeService()
|
||||
runtime.setOrchestrationDb(db)
|
||||
vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) =>
|
||||
handle === WORKER_HANDLE ? WORKER_PANE : handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : null
|
||||
)
|
||||
vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation((evidence) => {
|
||||
const worker =
|
||||
evidence?.terminalHandle === WORKER_HANDLE &&
|
||||
evidence.paneKey === WORKER_PANE &&
|
||||
evidence.launchToken === WORKER_ENV.ORCA_AGENT_LAUNCH_TOKEN
|
||||
const coordinator =
|
||||
evidence?.terminalHandle === COORDINATOR_HANDLE &&
|
||||
evidence.paneKey === COORDINATOR_PANE &&
|
||||
evidence.launchToken === COORDINATOR_ENV.ORCA_AGENT_LAUNCH_TOKEN
|
||||
if (
|
||||
(!worker && !coordinator) ||
|
||||
evidence.host?.kind !== 'ssh' ||
|
||||
evidence.host.targetId !== RUNTIME_AUTHORITY.targetId ||
|
||||
evidence.host.connectionIncarnation !== RUNTIME_AUTHORITY.connectionIncarnation
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const identity = worker
|
||||
? {
|
||||
terminalHandle: WORKER_HANDLE,
|
||||
paneKey: WORKER_PANE,
|
||||
launchToken: WORKER_ENV.ORCA_AGENT_LAUNCH_TOKEN
|
||||
}
|
||||
: {
|
||||
terminalHandle: COORDINATOR_HANDLE,
|
||||
paneKey: COORDINATOR_PANE,
|
||||
launchToken: COORDINATOR_ENV.ORCA_AGENT_LAUNCH_TOKEN
|
||||
}
|
||||
return {
|
||||
hostScope: { kind: 'ssh', targetId: RUNTIME_AUTHORITY.targetId },
|
||||
terminalHandle: identity.terminalHandle,
|
||||
paneKey: identity.paneKey,
|
||||
processIncarnation: 'process-1',
|
||||
launchTokenHash: createHash('sha256').update(identity.launchToken).digest('hex')
|
||||
}
|
||||
})
|
||||
vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {})
|
||||
return { db, runtime, run, dispatch }
|
||||
}
|
||||
|
||||
describe('legacy SSH orchestration fallback', () => {
|
||||
it('acknowledges a consuming check only after remote output', async () => {
|
||||
const { db, runtime, run } = createLegacyRuntime()
|
||||
const message = db.insertMessage({
|
||||
runId: run.id,
|
||||
deliveryContract: 'legacy_direct',
|
||||
from: COORDINATOR_HANDLE,
|
||||
to: WORKER_HANDLE,
|
||||
subject: 'retained SSH mail'
|
||||
})
|
||||
const request = {
|
||||
argv: ['orchestration', 'check', '--unread', '--inject', '--json'],
|
||||
cwd: '/home/alice/repo',
|
||||
env: WORKER_ENV,
|
||||
runtimeAuthority: RUNTIME_AUTHORITY
|
||||
}
|
||||
|
||||
try {
|
||||
const first = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
|
||||
const replay = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
|
||||
|
||||
expect(JSON.parse(first.stdout)).toMatchObject({
|
||||
result: {
|
||||
messages: [{ id: message.id }],
|
||||
formatted: expect.stringContaining('retained SSH mail')
|
||||
}
|
||||
})
|
||||
expect(first.postOutput).toEqual({
|
||||
kind: 'legacy_check_ack',
|
||||
terminal: WORKER_HANDLE,
|
||||
messageIds: [message.id]
|
||||
})
|
||||
expect(JSON.parse(replay.stdout)).toMatchObject({
|
||||
result: { messages: [{ id: message.id }] }
|
||||
})
|
||||
expect(db.getMessageById(message.id)?.read).toBe(0)
|
||||
|
||||
await acknowledgeRemoteOrcaCliPostOutput(runtime, {
|
||||
postOutput: first.postOutput!,
|
||||
env: WORKER_ENV,
|
||||
runtimeAuthority: RUNTIME_AUTHORITY
|
||||
})
|
||||
const afterAck = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
|
||||
|
||||
expect(JSON.parse(afterAck.stdout)).toMatchObject({ result: { messages: [], count: 0 } })
|
||||
expect(afterAck.postOutput).toBeUndefined()
|
||||
expect(db.getMessageById(message.id)?.read).toBe(1)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps peek formatted and non-consuming', async () => {
|
||||
const { db, runtime, run } = createLegacyRuntime()
|
||||
const message = db.insertMessage({
|
||||
runId: run.id,
|
||||
deliveryContract: 'legacy_direct',
|
||||
from: COORDINATOR_HANDLE,
|
||||
to: WORKER_HANDLE,
|
||||
subject: 'inspect retained SSH mail'
|
||||
})
|
||||
|
||||
try {
|
||||
const peek = await runRemoteOrcaCli(
|
||||
runtime,
|
||||
{
|
||||
argv: ['orchestration', 'check', '--peek', '--format', '--json'],
|
||||
cwd: '/home/alice/repo',
|
||||
env: WORKER_ENV,
|
||||
runtimeAuthority: RUNTIME_AUTHORITY
|
||||
},
|
||||
LEGACY_FALLBACK_OPTIONS
|
||||
)
|
||||
|
||||
const output = JSON.parse(peek.stdout) as {
|
||||
result: { formatted: string }
|
||||
}
|
||||
expect(output).toMatchObject({
|
||||
result: {
|
||||
messages: [{ id: message.id }],
|
||||
formatted: expect.stringContaining('inspect retained SSH mail'),
|
||||
legacyCompatibility: { readOnly: true }
|
||||
}
|
||||
})
|
||||
expect(output.result.formatted).toContain(`${message.id} [legacy, read-only]`)
|
||||
expect(output.result.formatted).toContain(
|
||||
'Inspection only: reply and acknowledgment are unavailable.'
|
||||
)
|
||||
expect(output.result.formatted).not.toContain('orchestration reply')
|
||||
expect(peek.postOutput).toBeUndefined()
|
||||
expect(db.getMessageById(message.id)?.read).toBe(0)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('reads and acknowledges current Run delivery', async () => {
|
||||
const { db, runtime, run } = createLegacyRuntime()
|
||||
const message = db.insertMessage({
|
||||
runId: run.id,
|
||||
from: WORKER_HANDLE,
|
||||
to: `run:${run.id}`,
|
||||
subject: 'current Run mail'
|
||||
})
|
||||
const baseRequest = {
|
||||
cwd: '/home/alice/repo',
|
||||
env: COORDINATOR_ENV,
|
||||
runtimeAuthority: RUNTIME_AUTHORITY
|
||||
}
|
||||
const spawn = vi.fn()
|
||||
const hostCliAvailable = {
|
||||
...LEGACY_FALLBACK_OPTIONS,
|
||||
entryExists: () => true,
|
||||
spawn: spawn as HostCliPassthroughOptions['spawn']
|
||||
}
|
||||
|
||||
try {
|
||||
const checked = await runRemoteOrcaCli(
|
||||
runtime,
|
||||
{
|
||||
...baseRequest,
|
||||
argv: ['orchestration', 'check', '--run', run.id]
|
||||
},
|
||||
hostCliAvailable
|
||||
)
|
||||
expect(checked.stdout).toContain('Delivery ')
|
||||
expect(checked.stdout).toContain(`${message.id} [status] from=${WORKER_HANDLE}`)
|
||||
expect(spawn).not.toHaveBeenCalled()
|
||||
|
||||
const checkedJson = await runRemoteOrcaCli(
|
||||
runtime,
|
||||
{
|
||||
...baseRequest,
|
||||
argv: ['orchestration', 'check', '--run', run.id, '--json']
|
||||
},
|
||||
hostCliAvailable
|
||||
)
|
||||
const result = JSON.parse(checkedJson.stdout) as {
|
||||
result: { deliveryId: string; messages: { id: string }[] }
|
||||
}
|
||||
expect(result.result.messages).toEqual([expect.objectContaining({ id: message.id })])
|
||||
|
||||
const acknowledged = await runRemoteOrcaCli(
|
||||
runtime,
|
||||
{
|
||||
...baseRequest,
|
||||
argv: [
|
||||
'orchestration',
|
||||
'check',
|
||||
'--run',
|
||||
run.id,
|
||||
'--ack',
|
||||
result.result.deliveryId,
|
||||
'--json'
|
||||
]
|
||||
},
|
||||
hostCliAvailable
|
||||
)
|
||||
|
||||
expect(JSON.parse(acknowledged.stdout)).toMatchObject({
|
||||
result: { acknowledged: result.result.deliveryId }
|
||||
})
|
||||
expect(db.getMessageById(message.id)?.read).toBe(1)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('replays an SSH legacy ask by retry request without creating another question', async () => {
|
||||
const { db, runtime } = createLegacyRuntime()
|
||||
const argv = [
|
||||
'orchestration',
|
||||
'ask',
|
||||
'--to',
|
||||
COORDINATOR_HANDLE,
|
||||
'--question',
|
||||
'Continue?',
|
||||
'--timeout-ms',
|
||||
'1',
|
||||
'--retry-request',
|
||||
'ssh-question-1',
|
||||
'--json'
|
||||
]
|
||||
const request = {
|
||||
argv,
|
||||
cwd: '/home/alice/repo',
|
||||
env: WORKER_ENV,
|
||||
runtimeAuthority: RUNTIME_AUTHORITY
|
||||
}
|
||||
|
||||
try {
|
||||
const first = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
|
||||
const replay = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
|
||||
const sqlite = (db as unknown as { db: Database.Database }).db
|
||||
|
||||
const firstResult = JSON.parse(first.stdout) as { messageId: string; timedOut: boolean }
|
||||
const replayResult = JSON.parse(replay.stdout) as { messageId: string; timedOut: boolean }
|
||||
expect(first).toMatchObject({ exitCode: 1 })
|
||||
expect(firstResult).toMatchObject({ messageId: expect.any(String), timedOut: true })
|
||||
expect(replay).toMatchObject({ exitCode: 1 })
|
||||
expect(replayResult).toMatchObject({ messageId: firstResult.messageId, timedOut: true })
|
||||
expect(
|
||||
(
|
||||
sqlite.prepare('SELECT COUNT(*) AS count FROM question_threads').get() as {
|
||||
count: number
|
||||
}
|
||||
).count
|
||||
).toBe(1)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('resumes and acknowledges a question answer', async () => {
|
||||
const { db, runtime, run, dispatch } = createLegacyRuntime()
|
||||
const pending = db.createQuestion({
|
||||
runId: run.id,
|
||||
dispatchId: dispatch.id,
|
||||
askerHandle: WORKER_HANDLE,
|
||||
question: 'Proceed?'
|
||||
})
|
||||
const answer = db.answerQuestion({
|
||||
messageId: pending.question.message_id,
|
||||
runId: run.id,
|
||||
consumerGeneration: run.consumer_generation,
|
||||
body: 'yes'
|
||||
})
|
||||
const sqlite = (db as unknown as { db: Database.Database }).db
|
||||
sqlite
|
||||
.prepare(
|
||||
`UPDATE messages
|
||||
SET delivery_contract = 'legacy_direct', read = 0
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run(answer.message.id)
|
||||
|
||||
try {
|
||||
const result = await runRemoteOrcaCli(
|
||||
runtime,
|
||||
{
|
||||
argv: ['orchestration', 'ask', '--resume', pending.question.message_id, '--json'],
|
||||
cwd: '/home/alice/repo',
|
||||
env: WORKER_ENV,
|
||||
runtimeAuthority: RUNTIME_AUTHORITY
|
||||
},
|
||||
LEGACY_FALLBACK_OPTIONS
|
||||
)
|
||||
|
||||
expect(result.exitCode).toBe(0)
|
||||
expect(JSON.parse(result.stdout)).toMatchObject({
|
||||
answer: 'yes',
|
||||
messageId: pending.question.message_id,
|
||||
timedOut: false
|
||||
})
|
||||
expect(result.postOutput).toEqual({
|
||||
kind: 'legacy_question_ack',
|
||||
terminal: WORKER_HANDLE,
|
||||
questionId: pending.question.message_id,
|
||||
answerMessageId: answer.message.id
|
||||
})
|
||||
expect(db.getMessageById(answer.message.id)?.read).toBe(0)
|
||||
|
||||
await acknowledgeRemoteOrcaCliPostOutput(runtime, {
|
||||
postOutput: result.postOutput!,
|
||||
env: WORKER_ENV,
|
||||
runtimeAuthority: RUNTIME_AUTHORITY
|
||||
})
|
||||
|
||||
expect(db.getMessageById(answer.message.id)?.read).toBe(1)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,143 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { readOrchestrationCompatibilityEvidence } from '../../shared/orchestration-compatibility-evidence'
|
||||
import { ORCHESTRATION_CONTRACT_VERSION } from '../../shared/protocol-version'
|
||||
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
|
||||
import type { RpcResponse } from '../runtime/rpc/core'
|
||||
import { RpcDispatcher } from '../runtime/rpc/dispatcher'
|
||||
import type {
|
||||
RemoteOrcaCliPostOutput,
|
||||
RemoteOrcaCliRequest
|
||||
} from './ssh-remote-cli-host-passthrough'
|
||||
import { RemoteCliArgumentError, type ParsedRemoteCli } from './ssh-remote-cli-argument-error'
|
||||
import { optionalRemoteCliString, resolveRemoteCliHandle } from './ssh-remote-cli-args'
|
||||
|
||||
export async function acknowledgeRemoteOrcaCliPostOutput(
|
||||
runtime: OrcaRuntimeService,
|
||||
args: {
|
||||
postOutput: RemoteOrcaCliPostOutput
|
||||
env: Record<string, string>
|
||||
runtimeAuthority?: RemoteOrcaCliRequest['runtimeAuthority']
|
||||
}
|
||||
): Promise<void> {
|
||||
const inheritedEvidence = readOrchestrationCompatibilityEvidence(args.env)
|
||||
const orchestrationCompatibilityEvidence = args.runtimeAuthority
|
||||
? { ...inheritedEvidence, host: args.runtimeAuthority }
|
||||
: inheritedEvidence
|
||||
const params =
|
||||
args.postOutput.kind === 'legacy_check_ack'
|
||||
? {
|
||||
terminal: args.postOutput.terminal,
|
||||
compatibilityAck: JSON.stringify({
|
||||
messageIds: args.postOutput.messageIds,
|
||||
types: args.postOutput.types
|
||||
})
|
||||
}
|
||||
: {
|
||||
terminal: args.postOutput.terminal,
|
||||
compatibilityQuestionAck: JSON.stringify({
|
||||
questionId: args.postOutput.questionId,
|
||||
answerMessageId: args.postOutput.answerMessageId
|
||||
})
|
||||
}
|
||||
const response = await new RpcDispatcher({ runtime }).dispatch({
|
||||
id: `remote-cli-post-output-${randomUUID()}`,
|
||||
authToken: 'remote-cli',
|
||||
method: 'orchestration.check',
|
||||
params,
|
||||
orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION,
|
||||
compatibilityInvocationId: randomUUID(),
|
||||
orchestrationCompatibilityEvidence
|
||||
})
|
||||
if (!response.ok) {
|
||||
throw new Error(response.error.message)
|
||||
}
|
||||
}
|
||||
|
||||
export function parseRemoteOrcaCliPostOutput(value: unknown): RemoteOrcaCliPostOutput {
|
||||
if (!isRecord(value) || typeof value.kind !== 'string' || typeof value.terminal !== 'string') {
|
||||
throw invalidPostOutput()
|
||||
}
|
||||
if (
|
||||
value.kind === 'legacy_check_ack' &&
|
||||
Array.isArray(value.messageIds) &&
|
||||
value.messageIds.every((id) => typeof id === 'string') &&
|
||||
(value.types === undefined ||
|
||||
(Array.isArray(value.types) && value.types.every((type) => typeof type === 'string')))
|
||||
) {
|
||||
return {
|
||||
kind: value.kind,
|
||||
terminal: value.terminal,
|
||||
messageIds: value.messageIds,
|
||||
...(value.types === undefined ? {} : { types: value.types })
|
||||
}
|
||||
}
|
||||
if (
|
||||
value.kind === 'legacy_question_ack' &&
|
||||
typeof value.questionId === 'string' &&
|
||||
typeof value.answerMessageId === 'string'
|
||||
) {
|
||||
return {
|
||||
kind: value.kind,
|
||||
terminal: value.terminal,
|
||||
questionId: value.questionId,
|
||||
answerMessageId: value.answerMessageId
|
||||
}
|
||||
}
|
||||
throw invalidPostOutput()
|
||||
}
|
||||
|
||||
export function getRemoteCliPostOutput(
|
||||
parsed: ParsedRemoteCli,
|
||||
env: Record<string, string>,
|
||||
response: RpcResponse
|
||||
): RemoteOrcaCliPostOutput | undefined {
|
||||
if (!response.ok || !isRecord(response.result)) {
|
||||
return undefined
|
||||
}
|
||||
const compatibility = response.result.legacyCompatibility
|
||||
if (!isRecord(compatibility)) {
|
||||
return undefined
|
||||
}
|
||||
const command = parsed.commandPath.join(' ')
|
||||
if (
|
||||
command === 'orchestration check' &&
|
||||
Array.isArray(compatibility.ackMessageIds) &&
|
||||
compatibility.ackMessageIds.length > 0 &&
|
||||
compatibility.ackMessageIds.every((id) => typeof id === 'string')
|
||||
) {
|
||||
const types = optionalRemoteCliString(parsed.flags, 'types')
|
||||
?.split(',')
|
||||
.map((type) => type.trim())
|
||||
.filter(Boolean)
|
||||
return {
|
||||
kind: 'legacy_check_ack',
|
||||
terminal: resolveRemoteCliHandle(parsed.flags, env, 'terminal'),
|
||||
messageIds: compatibility.ackMessageIds,
|
||||
...(types ? { types } : {})
|
||||
}
|
||||
}
|
||||
const acknowledgement = compatibility.answerAcknowledgement
|
||||
if (
|
||||
command === 'orchestration ask' &&
|
||||
response.result.answer !== null &&
|
||||
isRecord(acknowledgement) &&
|
||||
typeof acknowledgement.questionId === 'string' &&
|
||||
typeof acknowledgement.answerMessageId === 'string'
|
||||
) {
|
||||
return {
|
||||
kind: 'legacy_question_ack',
|
||||
terminal: resolveRemoteCliHandle(parsed.flags, env, 'from'),
|
||||
questionId: acknowledgement.questionId,
|
||||
answerMessageId: acknowledgement.answerMessageId
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
function invalidPostOutput(): RemoteCliArgumentError {
|
||||
return new RemoteCliArgumentError('invalid_argument', 'Invalid SSH CLI post-output action.')
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
|
||||
}
|
||||
@@ -829,4 +829,73 @@ describe('attachMainWindowServices', () => {
|
||||
await expect(revealPromise).resolves.toEqual({ tabId: 'tab-1', title: 'SSH tmux' })
|
||||
expect(removeListenerMock).toHaveBeenCalledWith('terminal:tabCreateReply', handler)
|
||||
})
|
||||
|
||||
it('requires an exact renderer identity receipt for recovered worker reveals', async () => {
|
||||
const sendMock = vi.fn()
|
||||
const mainWindow = createMainWindow({ send: sendMock })
|
||||
const runtime = createRuntime()
|
||||
|
||||
attachMainWindowServices(mainWindow as never, createStore(), runtime as never)
|
||||
|
||||
const notifier = runtime.setNotifier.mock.calls[0][0] as {
|
||||
revealTerminalSession: (
|
||||
worktreeId: string,
|
||||
opts: {
|
||||
ptyId: string
|
||||
tabId: string
|
||||
leafId: string
|
||||
expectedProcessIdentity: { terminalHandle: string; incarnationId: string }
|
||||
}
|
||||
) => Promise<unknown>
|
||||
}
|
||||
const opts = {
|
||||
ptyId: 'pty-worker',
|
||||
tabId: 'tab-worker',
|
||||
leafId: 'leaf-worker',
|
||||
expectedProcessIdentity: {
|
||||
terminalHandle: 'term_worker',
|
||||
incarnationId: 'inc-worker'
|
||||
}
|
||||
}
|
||||
const mismatch = notifier.revealTerminalSession('worktree-1', opts)
|
||||
const mismatchPayload = sendMock.mock.calls.at(-1)?.[1]
|
||||
const mismatchHandler = onMock.mock.calls.findLast(
|
||||
([channel]) => channel === 'terminal:tabCreateReply'
|
||||
)?.[1]
|
||||
mismatchHandler?.(
|
||||
{ sender: mainWindow.webContents },
|
||||
{
|
||||
requestId: mismatchPayload.requestId,
|
||||
tabId: 'tab-worker',
|
||||
identity: {
|
||||
worktreeId: 'worktree-1',
|
||||
tabId: 'tab-worker',
|
||||
leafId: 'leaf-worker',
|
||||
ptyId: 'pty-replacement'
|
||||
}
|
||||
}
|
||||
)
|
||||
await expect(mismatch).rejects.toThrow('terminal_reveal_identity_mismatch')
|
||||
|
||||
const exact = notifier.revealTerminalSession('worktree-1', opts)
|
||||
const exactPayload = sendMock.mock.calls.at(-1)?.[1]
|
||||
const exactHandler = onMock.mock.calls.findLast(
|
||||
([channel]) => channel === 'terminal:tabCreateReply'
|
||||
)?.[1]
|
||||
const identity = {
|
||||
worktreeId: 'worktree-1',
|
||||
tabId: 'tab-worker',
|
||||
leafId: 'leaf-worker',
|
||||
ptyId: 'pty-worker'
|
||||
}
|
||||
exactHandler?.(
|
||||
{ sender: mainWindow.webContents },
|
||||
{ requestId: exactPayload.requestId, tabId: 'tab-worker', identity }
|
||||
)
|
||||
await expect(exact).resolves.toEqual({
|
||||
tabId: 'tab-worker',
|
||||
title: undefined,
|
||||
identity
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -49,6 +49,7 @@ import type {
|
||||
RuntimeMarkdownSaveTabResult
|
||||
} from '../../shared/mobile-markdown-document'
|
||||
import type { RuntimeMobileSessionTabMove } from '../../shared/runtime-types'
|
||||
import type { TerminalTabCreateReply } from '../../shared/terminal-reveal-identity'
|
||||
import { isNativeFileDropPayload, type NativeFileDropPayload } from '../../shared/native-file-drop'
|
||||
import { requestMobileMarkdownFromRenderer } from './mobile-markdown-request-relay'
|
||||
import { requestTerminalTabCloseFromRenderer } from './terminal-tab-close-request-relay'
|
||||
@@ -347,14 +348,20 @@ function registerRuntimeWindowLifecycle(
|
||||
revealTerminalSession: (worktreeId, opts) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const requestId = randomUUID()
|
||||
const expectedIdentity = opts.expectedProcessIdentity
|
||||
? opts.tabId && opts.leafId
|
||||
? { worktreeId, tabId: opts.tabId, leafId: opts.leafId, ptyId: opts.ptyId }
|
||||
: null
|
||||
: undefined
|
||||
if (expectedIdentity === null) {
|
||||
reject(new Error('terminal_reveal_identity_required'))
|
||||
return
|
||||
}
|
||||
const timer = setTimeout(() => {
|
||||
ipcMain.removeListener('terminal:tabCreateReply', handler)
|
||||
reject(new Error('Terminal reveal timed out'))
|
||||
}, 10_000)
|
||||
const handler = (
|
||||
event: Electron.IpcMainEvent,
|
||||
reply: { requestId: string; tabId?: string; title?: string; error?: string }
|
||||
): void => {
|
||||
const handler = (event: Electron.IpcMainEvent, reply: TerminalTabCreateReply): void => {
|
||||
// Why: requestId is renderer-supplied, so only the targeted main window may satisfy the reveal.
|
||||
if (event.sender !== mainWindow.webContents || reply.requestId !== requestId) {
|
||||
return
|
||||
@@ -365,7 +372,22 @@ function registerRuntimeWindowLifecycle(
|
||||
reject(new Error(reply.error))
|
||||
return
|
||||
}
|
||||
resolve({ tabId: reply.tabId!, title: reply.title })
|
||||
if (
|
||||
expectedIdentity &&
|
||||
(!reply.identity ||
|
||||
reply.identity.worktreeId !== expectedIdentity.worktreeId ||
|
||||
reply.identity.tabId !== expectedIdentity.tabId ||
|
||||
reply.identity.leafId !== expectedIdentity.leafId ||
|
||||
reply.identity.ptyId !== expectedIdentity.ptyId)
|
||||
) {
|
||||
reject(new Error('terminal_reveal_identity_mismatch'))
|
||||
return
|
||||
}
|
||||
resolve({
|
||||
tabId: reply.tabId!,
|
||||
title: reply.title,
|
||||
...(reply.identity ? { identity: reply.identity } : {})
|
||||
})
|
||||
}
|
||||
ipcMain.on('terminal:tabCreateReply', handler)
|
||||
send('ui:createTerminal', {
|
||||
|
||||
@@ -18,6 +18,7 @@ import type {
|
||||
TerminalTabCloseRequest,
|
||||
TerminalTabCloseResponse
|
||||
} from '../shared/terminal-tab-close'
|
||||
import type { TerminalTabCreateReply } from '../shared/terminal-reveal-identity'
|
||||
import type {
|
||||
LocalLogTailChangedPayload,
|
||||
LocalLogTailReadArgs,
|
||||
@@ -3149,12 +3150,7 @@ export type PreloadApi = {
|
||||
onRequestTerminalTabMount: (
|
||||
callback: (data: { worktreeId: string; tabId?: string; ptyId?: string }) => void
|
||||
) => () => void
|
||||
replyTerminalCreate: (reply: {
|
||||
requestId: string
|
||||
tabId?: string
|
||||
title?: string
|
||||
error?: string
|
||||
}) => void
|
||||
replyTerminalCreate: (reply: TerminalTabCreateReply) => void
|
||||
onSplitTerminal: (
|
||||
callback: (data: {
|
||||
tabId: string
|
||||
|
||||
@@ -13,6 +13,7 @@ import type { CliInstallStatus } from '../shared/cli-install-types'
|
||||
import type { AgentHookInstallStatus } from '../shared/agent-hook-types'
|
||||
import type { CodexConfigSyncStatus } from '../shared/codex-config-sync-types'
|
||||
import type { TerminalPaneSplitSource } from '../shared/feature-education-telemetry'
|
||||
import type { TerminalTabCreateReply } from '../shared/terminal-reveal-identity'
|
||||
import type { ProjectExecutionRuntimeResolution } from '../shared/project-execution-runtime'
|
||||
import type { StartupCommandDelivery } from '../shared/codex-startup-delivery'
|
||||
import type {
|
||||
@@ -3752,12 +3753,7 @@ const api = {
|
||||
ipcRenderer.on('terminal:requestTabMount', listener)
|
||||
return () => ipcRenderer.removeListener('terminal:requestTabMount', listener)
|
||||
},
|
||||
replyTerminalCreate: (reply: {
|
||||
requestId: string
|
||||
tabId?: string
|
||||
title?: string
|
||||
error?: string
|
||||
}): void => {
|
||||
replyTerminalCreate: (reply: TerminalTabCreateReply): void => {
|
||||
ipcRenderer.send('terminal:tabCreateReply', reply)
|
||||
},
|
||||
onSplitTerminal: (
|
||||
|
||||
+85
-11
@@ -198,6 +198,8 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise<void> {
|
||||
let nextSeq = 1
|
||||
let highestReceivedSeq = 0
|
||||
const requestId = 1
|
||||
const postOutputRequestId = 2
|
||||
let initialExitCode = 0
|
||||
|
||||
const sendRequest = (): void => {
|
||||
const env = pickRemoteCliEnv(process.env)
|
||||
@@ -219,6 +221,59 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise<void> {
|
||||
sock.write(frame)
|
||||
}
|
||||
|
||||
const finish = (exitCode: number): void => {
|
||||
sock.destroy()
|
||||
process.exit(exitCode)
|
||||
}
|
||||
|
||||
const sendPostOutput = (postOutput: unknown): void => {
|
||||
sock.write(
|
||||
encodeJsonRpcFrame(
|
||||
{
|
||||
jsonrpc: '2.0',
|
||||
id: postOutputRequestId,
|
||||
method: 'orca.cli.postOutput',
|
||||
params: { postOutput, env: pickRemoteCliEnv(process.env) }
|
||||
},
|
||||
nextSeq++,
|
||||
highestReceivedSeq
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
const writeOutput = (
|
||||
result: { stdout?: unknown; stderr?: unknown },
|
||||
onFlushed: (error?: Error) => void
|
||||
): void => {
|
||||
const writes = [
|
||||
[process.stdout, result.stdout],
|
||||
[process.stderr, result.stderr]
|
||||
].filter((entry): entry is [NodeJS.WriteStream, string] => typeof entry[1] === 'string')
|
||||
if (writes.length === 0) {
|
||||
onFlushed()
|
||||
return
|
||||
}
|
||||
let pending = writes.length
|
||||
let completed = false
|
||||
for (const [stream, output] of writes) {
|
||||
stream.write(output, (error) => {
|
||||
if (completed) {
|
||||
return
|
||||
}
|
||||
if (error) {
|
||||
completed = true
|
||||
onFlushed(error)
|
||||
return
|
||||
}
|
||||
pending -= 1
|
||||
if (pending === 0) {
|
||||
completed = true
|
||||
onFlushed()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const decoder = new FrameDecoder((frame: DecodedFrame) => {
|
||||
if (frame.id > highestReceivedSeq) {
|
||||
highestReceivedSeq = frame.id
|
||||
@@ -227,28 +282,41 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise<void> {
|
||||
return
|
||||
}
|
||||
const msg = parseJsonRpcMessage(frame.payload)
|
||||
if (!('id' in msg) || msg.id !== requestId || !('result' in msg || 'error' in msg)) {
|
||||
if (
|
||||
!('id' in msg) ||
|
||||
(msg.id !== requestId && msg.id !== postOutputRequestId) ||
|
||||
!('result' in msg || 'error' in msg)
|
||||
) {
|
||||
return
|
||||
}
|
||||
const response = msg as JsonRpcResponse
|
||||
if (response.error) {
|
||||
process.stderr.write(`${response.error.message}\n`)
|
||||
sock.destroy()
|
||||
process.exit(1)
|
||||
finish(1)
|
||||
return
|
||||
}
|
||||
if (response.id === postOutputRequestId) {
|
||||
finish(initialExitCode)
|
||||
return
|
||||
}
|
||||
const result = (response.result ?? {}) as {
|
||||
stdout?: unknown
|
||||
stderr?: unknown
|
||||
exitCode?: unknown
|
||||
postOutput?: unknown
|
||||
}
|
||||
if (typeof result.stdout === 'string' && result.stdout.length > 0) {
|
||||
process.stdout.write(result.stdout)
|
||||
}
|
||||
if (typeof result.stderr === 'string' && result.stderr.length > 0) {
|
||||
process.stderr.write(result.stderr)
|
||||
}
|
||||
sock.destroy()
|
||||
process.exit(typeof result.exitCode === 'number' ? result.exitCode : 0)
|
||||
initialExitCode = typeof result.exitCode === 'number' ? result.exitCode : 0
|
||||
writeOutput(result, (error) => {
|
||||
if (error) {
|
||||
finish(1)
|
||||
return
|
||||
}
|
||||
if (result.postOutput === undefined) {
|
||||
finish(initialExitCode)
|
||||
return
|
||||
}
|
||||
sendPostOutput(result.postOutput)
|
||||
})
|
||||
})
|
||||
|
||||
const connectTimeout = setTimeout(() => {
|
||||
@@ -443,6 +511,12 @@ async function main(): Promise<void> {
|
||||
timeoutMs: remoteCliRequestTimeoutMs(params)
|
||||
})
|
||||
})
|
||||
dispatcher.onRequest('orca.cli.postOutput', async (params, context) => {
|
||||
return await dispatcher.requestAnyClient('orca.cli.postOutput', params, {
|
||||
excludeClientId: context.clientId,
|
||||
timeoutMs: remoteCliRequestTimeoutMs(params)
|
||||
})
|
||||
})
|
||||
|
||||
function configureRelayGraceTime(params: Record<string, unknown>): { graceTimeMs: number } {
|
||||
const seconds = Number(params.graceTimeSeconds)
|
||||
|
||||
@@ -1504,7 +1504,6 @@ describe('useIpcEvents updater integration', () => {
|
||||
openAgentTabsInChatByDefault: false
|
||||
}
|
||||
}
|
||||
|
||||
vi.doMock('react', async () => {
|
||||
const actual = await vi.importActual<typeof ReactModule>('react')
|
||||
return {
|
||||
@@ -1789,7 +1788,11 @@ describe('useIpcEvents updater integration', () => {
|
||||
})
|
||||
|
||||
it('surfaces terminal creates without stealing focus unless requested', async () => {
|
||||
const createTab = vi.fn(() => ({ id: 'tab-new' }))
|
||||
const createTab = vi.fn(
|
||||
(_worktreeId: string, _groupId?: string, _tabType?: string, options?: { id?: string }) => ({
|
||||
id: options?.id ?? 'tab-new'
|
||||
})
|
||||
)
|
||||
const setActiveView = vi.fn()
|
||||
const setActiveWorktree = vi.fn()
|
||||
const markWorktreeVisited = vi.fn()
|
||||
@@ -1876,6 +1879,20 @@ describe('useIpcEvents updater integration', () => {
|
||||
activeRuntimeEnvironmentId: undefined as string | undefined
|
||||
}
|
||||
}
|
||||
updateTabPtyId.mockImplementation((tabId: string, ptyId: string) => {
|
||||
storeState.ptyIdsByTabId[tabId] = [
|
||||
...new Set([...(storeState.ptyIdsByTabId[tabId] ?? []), ptyId])
|
||||
]
|
||||
for (const tabs of Object.values(storeState.tabsByWorktree)) {
|
||||
const tab = tabs.find((candidate) => candidate.id === tabId)
|
||||
if (tab) {
|
||||
tab.ptyId = ptyId
|
||||
}
|
||||
}
|
||||
})
|
||||
setTabLayout.mockImplementation((tabId: string, layout: unknown) => {
|
||||
storeState.terminalLayoutsByTabId[tabId] = layout
|
||||
})
|
||||
const createTerminalListenerRef: {
|
||||
current:
|
||||
| ((data: {
|
||||
@@ -2280,6 +2297,7 @@ describe('useIpcEvents updater integration', () => {
|
||||
setActiveTabType.mockClear()
|
||||
setActiveTab.mockClear()
|
||||
revealWorktreeInSidebar.mockClear()
|
||||
dispatchEvent.mockClear()
|
||||
createTerminalListenerRef.current({
|
||||
worktreeId: 'wt-2',
|
||||
title: 'Runner',
|
||||
@@ -2588,6 +2606,7 @@ describe('useIpcEvents updater integration', () => {
|
||||
setActiveTab.mockClear()
|
||||
revealWorktreeInSidebar.mockClear()
|
||||
replyTerminalCreate.mockClear()
|
||||
dispatchEvent.mockClear()
|
||||
requestTerminalCreateListenerRef.current({
|
||||
requestId: 'req-renderer-backed-background',
|
||||
worktreeId: 'wt-2',
|
||||
@@ -2605,6 +2624,12 @@ describe('useIpcEvents updater integration', () => {
|
||||
expect(setActiveTabType).not.toHaveBeenCalled()
|
||||
expect(setActiveTab).not.toHaveBeenCalled()
|
||||
expect(revealWorktreeInSidebar).not.toHaveBeenCalled()
|
||||
expect(dispatchEvent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: 'orca-background-mount-terminal-worktree',
|
||||
detail: { worktreeId: 'wt-2', tabIds: ['tab-new'] }
|
||||
})
|
||||
)
|
||||
expect(replyTerminalCreate).toHaveBeenCalledWith({
|
||||
requestId: 'req-renderer-backed-background',
|
||||
tabId: 'tab-new',
|
||||
@@ -2708,6 +2733,7 @@ describe('useIpcEvents updater integration', () => {
|
||||
setActiveTabType.mockClear()
|
||||
setActiveTab.mockClear()
|
||||
revealWorktreeInSidebar.mockClear()
|
||||
dispatchEvent.mockClear()
|
||||
createTerminalListenerRef.current({
|
||||
worktreeId: 'wt-2',
|
||||
ptyId: 'pty-bg-3',
|
||||
@@ -2725,6 +2751,13 @@ describe('useIpcEvents updater integration', () => {
|
||||
expect(setActiveTabType).not.toHaveBeenCalled()
|
||||
expect(setActiveTab).not.toHaveBeenCalled()
|
||||
expect(revealWorktreeInSidebar).not.toHaveBeenCalled()
|
||||
expect(dispatchEvent).toHaveBeenCalledOnce()
|
||||
expect(dispatchEvent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: 'orca-background-mount-terminal-worktree',
|
||||
detail: { worktreeId: 'wt-2', tabIds: ['tab-cli-bg-reveal'] }
|
||||
})
|
||||
)
|
||||
|
||||
createTab.mockClear()
|
||||
setActiveView.mockClear()
|
||||
@@ -2734,6 +2767,7 @@ describe('useIpcEvents updater integration', () => {
|
||||
revealWorktreeInSidebar.mockClear()
|
||||
focusRuntimeTerminalSurface.mockClear()
|
||||
focusTerminalTabSurface.mockClear()
|
||||
dispatchEvent.mockClear()
|
||||
createTerminalListenerRef.current({
|
||||
worktreeId: 'wt-2',
|
||||
ptyId: 'pty-recovery-bg',
|
||||
@@ -2754,6 +2788,13 @@ describe('useIpcEvents updater integration', () => {
|
||||
expect(revealWorktreeInSidebar).not.toHaveBeenCalled()
|
||||
expect(focusRuntimeTerminalSurface).not.toHaveBeenCalled()
|
||||
expect(focusTerminalTabSurface).not.toHaveBeenCalled()
|
||||
expect(dispatchEvent).toHaveBeenCalledOnce()
|
||||
expect(dispatchEvent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: 'orca-background-mount-terminal-worktree',
|
||||
detail: { worktreeId: 'wt-2', tabIds: ['tab-recovery-bg'] }
|
||||
})
|
||||
)
|
||||
|
||||
storeState.tabsByWorktree = {
|
||||
'wt-2': [{ id: 'tab-existing', ptyId: 'pty-bg', title: 'Terminal 1' }]
|
||||
@@ -2836,7 +2877,13 @@ describe('useIpcEvents updater integration', () => {
|
||||
expect(replyTerminalCreate).toHaveBeenCalledWith({
|
||||
requestId: 'req-adopt-pending',
|
||||
tabId: pendingTabId,
|
||||
title: 'Terminal 3'
|
||||
title: 'Terminal 3',
|
||||
identity: {
|
||||
worktreeId: 'wt-2',
|
||||
tabId: pendingTabId,
|
||||
leafId: pendingLeafId,
|
||||
ptyId: 'serve-cf39bedb-a33a-417c-9ab6-f304dc27a6c0'
|
||||
}
|
||||
})
|
||||
|
||||
storeState.tabsByWorktree = {
|
||||
@@ -2902,7 +2949,13 @@ describe('useIpcEvents updater integration', () => {
|
||||
expect(replyTerminalCreate).toHaveBeenCalledWith({
|
||||
requestId: 'req-split',
|
||||
tabId: 'tab-existing',
|
||||
title: 'Terminal 1'
|
||||
title: 'Terminal 1',
|
||||
identity: {
|
||||
worktreeId: 'wt-2',
|
||||
tabId: 'tab-existing',
|
||||
leafId: 'leaf-split',
|
||||
ptyId: 'pty-split'
|
||||
}
|
||||
})
|
||||
|
||||
storeState.terminalLayoutsByTabId = {
|
||||
@@ -5234,7 +5287,7 @@ describe('useIpcEvents agent status snapshot integration', () => {
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
it('reconciles adopted and exited legacy worker recovery without broad record cleanup', async () => {
|
||||
it('retires the exact sleeping record after adopted or exited legacy worker recovery', async () => {
|
||||
const clearSleepingAgentSession = vi.fn()
|
||||
const setSleepingAgentAutomaticResumeBlocked = vi.fn()
|
||||
let listener:
|
||||
@@ -5275,11 +5328,8 @@ describe('useIpcEvents agent status snapshot integration', () => {
|
||||
|
||||
clearSleepingAgentSession.mockClear()
|
||||
listener?.({ paneKey: 'tab-exited:leaf-exited', resolution: 'exited' })
|
||||
expect(clearSleepingAgentSession).not.toHaveBeenCalled()
|
||||
expect(setSleepingAgentAutomaticResumeBlocked).toHaveBeenCalledWith(
|
||||
'tab-exited:leaf-exited',
|
||||
false
|
||||
)
|
||||
expect(clearSleepingAgentSession).toHaveBeenCalledWith('tab-exited:leaf-exited')
|
||||
expect(setSleepingAgentAutomaticResumeBlocked).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each([
|
||||
|
||||
@@ -101,6 +101,7 @@ import { track } from '@/lib/telemetry'
|
||||
import { singlePaneLayoutSnapshot } from '@/store/slices/terminal-helpers'
|
||||
import { buildWorkspaceSessionPayload } from '@/lib/workspace-session'
|
||||
import { persistWorkspaceSessionByHost } from '@/lib/workspace-session-host-persistence'
|
||||
import { verifyTerminalRevealIdentity } from '@/lib/terminal-reveal-identity'
|
||||
import { getLinearIssueWorkspaceName } from '../../../shared/workspace-name'
|
||||
import type { RuntimeClientEvent } from '../../../shared/runtime-client-events'
|
||||
import { applyHostWorktreeTerminalSleepState } from '@/components/terminal-pane/pty-shutdown-exit-deferral'
|
||||
@@ -1576,11 +1577,24 @@ export function useIpcEvents(): void {
|
||||
...(launchAgent ? { launchAgent } : {})
|
||||
})
|
||||
}
|
||||
if (ptyId && terminalPresentation === 'background') {
|
||||
requestBackgroundTerminalWorktreeMount({ worktreeId, tabIds: [tab.id] })
|
||||
}
|
||||
if (requestId) {
|
||||
const identity =
|
||||
ptyId && tabId && leafId
|
||||
? verifyTerminalRevealIdentity(useAppStore.getState(), {
|
||||
worktreeId,
|
||||
tabId,
|
||||
leafId,
|
||||
ptyId
|
||||
})
|
||||
: undefined
|
||||
window.api.ui.replyTerminalCreate({
|
||||
requestId,
|
||||
tabId: tab.id,
|
||||
title: title ?? tab.title
|
||||
title: title ?? tab.title,
|
||||
...(identity ? { identity } : {})
|
||||
})
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -3348,13 +3362,8 @@ export function useIpcEvents(): void {
|
||||
unsubs.push(unsubscribeMigrationUnsupportedClear)
|
||||
}
|
||||
const unsubscribeLegacyWorkerTerminalRecovery =
|
||||
window.api.agentStatus.onLegacyWorkerTerminalRecovery?.(({ paneKey, resolution }) => {
|
||||
const store = useAppStore.getState()
|
||||
if (resolution === 'adopted') {
|
||||
store.clearSleepingAgentSession(paneKey)
|
||||
return
|
||||
}
|
||||
store.setSleepingAgentAutomaticResumeBlocked(paneKey, false)
|
||||
window.api.agentStatus.onLegacyWorkerTerminalRecovery?.(({ paneKey }) => {
|
||||
useAppStore.getState().clearSleepingAgentSession(paneKey)
|
||||
})
|
||||
if (unsubscribeLegacyWorkerTerminalRecovery) {
|
||||
unsubs.push(unsubscribeLegacyWorkerTerminalRecovery)
|
||||
|
||||
@@ -11,7 +11,7 @@ afterEach(() => {
|
||||
})
|
||||
|
||||
describe('legacy worker sleeping-session recovery', () => {
|
||||
it('keeps the session dormant until exited recovery removes only its resume block', () => {
|
||||
it('never resumes a proven-exited legacy worker on workspace activation', () => {
|
||||
const record: SleepingAgentSessionRecord = {
|
||||
paneKey: 'tab-legacy:leaf-legacy',
|
||||
tabId: 'tab-legacy',
|
||||
@@ -46,16 +46,8 @@ describe('legacy worker sleeping-session recovery', () => {
|
||||
expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(0)
|
||||
expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey]).toBe(record)
|
||||
|
||||
useAppStore.getState().setSleepingAgentAutomaticResumeBlocked(record.paneKey, false)
|
||||
const unblocked = useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey]
|
||||
expect(unblocked).toMatchObject({
|
||||
paneKey: record.paneKey,
|
||||
providerSession: record.providerSession,
|
||||
origin: 'live'
|
||||
})
|
||||
expect(unblocked?.automaticResumeBlockedBy).toBeUndefined()
|
||||
|
||||
expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(1)
|
||||
useAppStore.getState().clearSleepingAgentSession(record.paneKey)
|
||||
expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(0)
|
||||
expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey]).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { verifyTerminalRevealIdentity } from './terminal-reveal-identity'
|
||||
|
||||
const expected = {
|
||||
worktreeId: 'worktree-1',
|
||||
tabId: 'tab-1',
|
||||
leafId: 'leaf-1',
|
||||
ptyId: 'pty-1'
|
||||
}
|
||||
|
||||
const mismatchCases: {
|
||||
name: string
|
||||
state: Parameters<typeof verifyTerminalRevealIdentity>[0]
|
||||
}[] = [
|
||||
{
|
||||
name: 'worktree',
|
||||
state: {
|
||||
tabsByWorktree: { 'worktree-2': [{ id: 'tab-1' }] },
|
||||
terminalLayoutsByTabId: {
|
||||
'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-1' } }
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'leaf',
|
||||
state: {
|
||||
tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] },
|
||||
terminalLayoutsByTabId: {
|
||||
'tab-1': { ptyIdsByLeafId: { 'leaf-2': 'pty-1' } }
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'pty',
|
||||
state: {
|
||||
tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] },
|
||||
terminalLayoutsByTabId: {
|
||||
'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-2' } }
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
describe('verifyTerminalRevealIdentity', () => {
|
||||
it('returns the exact fresh renderer binding', () => {
|
||||
expect(
|
||||
verifyTerminalRevealIdentity(
|
||||
{
|
||||
tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] },
|
||||
terminalLayoutsByTabId: {
|
||||
'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-1' } }
|
||||
}
|
||||
},
|
||||
expected
|
||||
)
|
||||
).toEqual(expected)
|
||||
})
|
||||
|
||||
it.each(mismatchCases)('rejects a mismatched $name owner', ({ state }) => {
|
||||
expect(() => verifyTerminalRevealIdentity(state, expected)).toThrow(
|
||||
'terminal_reveal_identity_mismatch'
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,23 @@
|
||||
import type { TerminalRevealIdentity } from '../../../shared/terminal-reveal-identity'
|
||||
|
||||
type TerminalRevealState = {
|
||||
tabsByWorktree: Record<string, readonly { id: string }[]>
|
||||
terminalLayoutsByTabId: Record<
|
||||
string,
|
||||
{ ptyIdsByLeafId?: Record<string, string | undefined> } | undefined
|
||||
>
|
||||
}
|
||||
|
||||
export function verifyTerminalRevealIdentity(
|
||||
state: TerminalRevealState,
|
||||
expected: TerminalRevealIdentity
|
||||
): TerminalRevealIdentity {
|
||||
const ownsTab = state.tabsByWorktree[expected.worktreeId]?.some(
|
||||
(tab) => tab.id === expected.tabId
|
||||
)
|
||||
const boundPtyId = state.terminalLayoutsByTabId[expected.tabId]?.ptyIdsByLeafId?.[expected.leafId]
|
||||
if (!ownsTab || boundPtyId !== expected.ptyId) {
|
||||
throw new Error('terminal_reveal_identity_mismatch')
|
||||
}
|
||||
return expected
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { prepareOrchestrationCheckOutput } from './orchestration-check-output'
|
||||
|
||||
describe('prepareOrchestrationCheckOutput', () => {
|
||||
it('keeps mixed read-only mail safe and current Run replies executable', () => {
|
||||
const prepared = prepareOrchestrationCheckOutput(
|
||||
{
|
||||
count: 2,
|
||||
messages: [
|
||||
{
|
||||
id: 'msg_current',
|
||||
run_id: 'run_adopted',
|
||||
delivery_contract: 'current_delivery',
|
||||
from_handle: 'term_worker',
|
||||
to_handle: 'run:run_adopted',
|
||||
subject: 'Question'
|
||||
},
|
||||
{
|
||||
id: 'msg_legacy',
|
||||
run_id: 'run_legacy_local',
|
||||
delivery_contract: 'audit_only',
|
||||
from_handle: 'term_legacy',
|
||||
to_handle: 'term_coord',
|
||||
subject: 'Old reply'
|
||||
}
|
||||
],
|
||||
formatted: '[Reply: unsafe stale formatter output]'
|
||||
},
|
||||
'term_current_coord',
|
||||
true
|
||||
)
|
||||
|
||||
expect(prepared.formatted).toContain(
|
||||
'[Reply: orca orchestration reply --id msg_current --body "..."]'
|
||||
)
|
||||
expect(prepared.formatted).not.toContain('--from run:run_adopted')
|
||||
expect(prepared.formatted).toContain(
|
||||
'[Inspection only: reply and acknowledgment are unavailable.]'
|
||||
)
|
||||
expect(prepared.formatted).not.toContain('unsafe stale formatter output')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,199 @@
|
||||
import { ORCHESTRATION_LEGACY_RUN_ID } from './orchestration-rpc-contract'
|
||||
|
||||
export type OrchestrationMessageSummary = {
|
||||
id: string
|
||||
run_id?: string
|
||||
delivery_contract?: 'legacy_direct' | 'current_delivery' | 'audit_only'
|
||||
from_handle: string
|
||||
to_handle?: string
|
||||
subject?: string
|
||||
type?: string
|
||||
body?: string
|
||||
payload?: string | null
|
||||
priority?: string
|
||||
read?: number
|
||||
}
|
||||
|
||||
export type LegacyCompatibilityResult = {
|
||||
recovery?: boolean
|
||||
readOnly?: boolean
|
||||
ackMessageIds?: string[]
|
||||
answerAcknowledgement?: {
|
||||
questionId: string
|
||||
answerMessageId: string
|
||||
}
|
||||
currentDelivery?: {
|
||||
runId: string
|
||||
checkCommand: string
|
||||
ackCommand: string
|
||||
}
|
||||
resumeRequired?: boolean
|
||||
resumeCommand?: string
|
||||
}
|
||||
|
||||
export type OrchestrationCheckOutput = {
|
||||
messages: OrchestrationMessageSummary[]
|
||||
count: number
|
||||
formatted?: string
|
||||
deliveryId?: string | null
|
||||
timedOut?: boolean
|
||||
cancelled?: boolean
|
||||
connectionLost?: boolean
|
||||
legacyCompatibility?: LegacyCompatibilityResult
|
||||
}
|
||||
|
||||
export function formatMessageReadOnlyTag(
|
||||
message: OrchestrationMessageSummary,
|
||||
legacyCompatibilityActive = false
|
||||
): string {
|
||||
return isLegacyReadOnlyMessage(message, legacyCompatibilityActive) ? ' [legacy, read-only]' : ''
|
||||
}
|
||||
|
||||
export function isLegacyReadOnlyMessage(
|
||||
message: OrchestrationMessageSummary,
|
||||
legacyCompatibilityActive = false
|
||||
): boolean {
|
||||
return (
|
||||
message.run_id === ORCHESTRATION_LEGACY_RUN_ID ||
|
||||
(message.delivery_contract === 'legacy_direct' && !legacyCompatibilityActive) ||
|
||||
message.delivery_contract === 'audit_only'
|
||||
)
|
||||
}
|
||||
|
||||
export function formatOrchestrationCheckText(
|
||||
result: OrchestrationCheckOutput,
|
||||
checkedTerminal: string
|
||||
): string {
|
||||
const prepared = prepareOrchestrationCheckOutput(
|
||||
result,
|
||||
checkedTerminal,
|
||||
result.formatted !== undefined
|
||||
)
|
||||
const compatibilityActive = Boolean(
|
||||
prepared.legacyCompatibility && !prepared.legacyCompatibility.readOnly
|
||||
)
|
||||
const legacyHeader = prepared.legacyCompatibility
|
||||
? prepared.legacyCompatibility.readOnly
|
||||
? '[LEGACY READ-ONLY]\n'
|
||||
: prepared.legacyCompatibility.recovery
|
||||
? '[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]\n'
|
||||
: '[LEGACY COMPATIBILITY]\n'
|
||||
: ''
|
||||
const deliveryNotice = formatCurrentDeliveryNotice(prepared.legacyCompatibility?.currentDelivery)
|
||||
if (prepared.formatted) {
|
||||
return `${legacyHeader}${prepared.formatted}${deliveryNotice}`
|
||||
}
|
||||
if (prepared.count === 0) {
|
||||
if (prepared.timedOut) {
|
||||
return `${legacyHeader}Wait timed out; no messages were consumed.${deliveryNotice}`
|
||||
}
|
||||
if (prepared.cancelled) {
|
||||
const cancelled = prepared.connectionLost
|
||||
? 'Wait cancelled because the connection closed; no messages were consumed.'
|
||||
: 'Wait cancelled; no messages were consumed.'
|
||||
return `${legacyHeader}${cancelled}${deliveryNotice}`
|
||||
}
|
||||
return `${legacyHeader}No messages.${deliveryNotice}`
|
||||
}
|
||||
const rendered = prepared.messages
|
||||
.map(
|
||||
(message) =>
|
||||
`${message.id}${formatMessageReadOnlyTag(
|
||||
message,
|
||||
compatibilityActive
|
||||
)} [${message.type ?? 'status'}] from=${message.from_handle} "${message.subject}"`
|
||||
)
|
||||
.join('\n')
|
||||
const output = prepared.deliveryId ? `Delivery ${prepared.deliveryId}\n${rendered}` : rendered
|
||||
return `${legacyHeader}${output}${deliveryNotice}`
|
||||
}
|
||||
|
||||
export function prepareOrchestrationCheckOutput<T extends OrchestrationCheckOutput>(
|
||||
result: T,
|
||||
checkedTerminal: string,
|
||||
formattedRequested: boolean
|
||||
): T {
|
||||
const compatibilityActive = Boolean(
|
||||
result.legacyCompatibility && !result.legacyCompatibility.readOnly
|
||||
)
|
||||
if (
|
||||
!formattedRequested ||
|
||||
!result.messages.some((message) => isLegacyReadOnlyMessage(message, compatibilityActive))
|
||||
) {
|
||||
return result
|
||||
}
|
||||
return {
|
||||
...result,
|
||||
formatted: formatLegacyAwareCheckMessages(result.messages, checkedTerminal, compatibilityActive)
|
||||
}
|
||||
}
|
||||
|
||||
function formatMessagePriorityTag(message: OrchestrationMessageSummary): string {
|
||||
return message.priority === 'urgent' ? ' [URGENT]' : message.priority === 'high' ? ' [HIGH]' : ''
|
||||
}
|
||||
|
||||
function escapeTerminalControlCharacters(value: string): string {
|
||||
return [...value]
|
||||
.map((character) => {
|
||||
const code = character.charCodeAt(0)
|
||||
if (character === '\n' || (code >= 0x20 && code < 0x7f) || code > 0x9f) {
|
||||
return character
|
||||
}
|
||||
return `\\x${code.toString(16).padStart(2, '0')}`
|
||||
})
|
||||
.join('')
|
||||
}
|
||||
|
||||
function formatQuotedMessageField(label: string, value?: string): string {
|
||||
return `[${label}]\n${escapeTerminalControlCharacters(value ?? '')
|
||||
.split('\n')
|
||||
.map((line) => ` ${line}`)
|
||||
.join('\n')}`
|
||||
}
|
||||
|
||||
function formatLegacyAwareCheckMessages(
|
||||
messages: OrchestrationMessageSummary[],
|
||||
checkedTerminal: string,
|
||||
legacyCompatibilityActive = false
|
||||
): string {
|
||||
return messages
|
||||
.map((message) => {
|
||||
const legacyReadOnly = isLegacyReadOnlyMessage(message, legacyCompatibilityActive)
|
||||
const lines = [
|
||||
`${message.id}${formatMessageReadOnlyTag(message, legacyCompatibilityActive)}${formatMessagePriorityTag(message)} [${message.type ?? 'status'}] from=${message.from_handle}`,
|
||||
formatQuotedMessageField('subject', message.subject)
|
||||
]
|
||||
if (legacyReadOnly) {
|
||||
lines.push('[Inspection only: reply and acknowledgment are unavailable.]')
|
||||
}
|
||||
if (message.body) {
|
||||
lines.push(formatQuotedMessageField('body', message.body))
|
||||
}
|
||||
if (message.payload) {
|
||||
lines.push(formatQuotedMessageField('payload', message.payload))
|
||||
}
|
||||
if (!legacyReadOnly) {
|
||||
const replyTarget = message.to_handle ?? checkedTerminal
|
||||
const replyFrom =
|
||||
replyTarget.startsWith('run:') || replyTarget.startsWith('dispatch:')
|
||||
? ''
|
||||
: ` --from ${replyTarget}`
|
||||
lines.push(`[Reply: orca orchestration reply --id ${message.id}${replyFrom} --body "..."]`)
|
||||
}
|
||||
return lines.join('\n')
|
||||
})
|
||||
.join('\n\n')
|
||||
}
|
||||
|
||||
function formatCurrentDeliveryNotice(
|
||||
delivery: LegacyCompatibilityResult['currentDelivery']
|
||||
): string {
|
||||
if (!delivery) {
|
||||
return ''
|
||||
}
|
||||
return (
|
||||
`\n[CURRENT RUN MAIL WAITING]\n` +
|
||||
`Read: ${delivery.checkCommand}\n` +
|
||||
`Then acknowledge the delivery ID it prints: ${delivery.ackCommand}`
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
export type TerminalRevealIdentity = {
|
||||
worktreeId: string
|
||||
tabId: string
|
||||
leafId: string
|
||||
ptyId: string
|
||||
}
|
||||
|
||||
export type TerminalTabCreateReply = {
|
||||
requestId: string
|
||||
tabId?: string
|
||||
title?: string
|
||||
identity?: TerminalRevealIdentity
|
||||
error?: string
|
||||
}
|
||||
@@ -16,7 +16,10 @@ import {
|
||||
import { waitForActivePaneHookDescriptor, waitForActivePanePtyId } from './helpers/terminal'
|
||||
import { RuntimeClient } from '../../src/cli/runtime-client'
|
||||
import Database from '../../src/main/sqlite/sync-database'
|
||||
import { LEGACY_CONTRACT_VERSION } from '../../src/main/runtime/orchestration/db'
|
||||
import {
|
||||
CURRENT_CONTRACT_VERSION,
|
||||
LEGACY_CONTRACT_VERSION
|
||||
} from '../../src/main/runtime/orchestration/db'
|
||||
import { DEFAULT_LOCAL_ORCA_PROFILE_ID } from '../../src/shared/orca-profiles'
|
||||
import type { RuntimeTerminalListResult, RuntimeTerminalRead } from '../../src/shared/runtime-types'
|
||||
|
||||
@@ -200,6 +203,44 @@ function stripLegacyWorkerRendererBinding(
|
||||
writeFileSync(persistedDataPath(userDataDir), `${JSON.stringify(data, null, 2)}\n`, 'utf8')
|
||||
}
|
||||
|
||||
function assertDispatchRemainsCurrent(
|
||||
userDataDir: string,
|
||||
input: {
|
||||
dispatchId: string
|
||||
terminalHandle: string
|
||||
paneKey: string
|
||||
processIncarnation: string
|
||||
worktreeId: string
|
||||
}
|
||||
): void {
|
||||
const db = new Database(path.join(userDataDir, 'orchestration.db'))
|
||||
try {
|
||||
const authority = db
|
||||
.prepare(
|
||||
`SELECT dc.status AS dispatch_status, dc.assignee_handle, dc.assignee_pane_key,
|
||||
dc.process_incarnation, dc.contract_version, dc.capability_hash,
|
||||
wd.state AS worker_state, wd.worktree_id, wd.agent_terminal_handle
|
||||
FROM dispatch_contexts dc
|
||||
INNER JOIN worker_dispatches wd ON wd.dispatch_id = dc.id
|
||||
WHERE dc.id = ?`
|
||||
)
|
||||
.get(input.dispatchId)
|
||||
expect(authority).toEqual({
|
||||
dispatch_status: 'dispatched',
|
||||
assignee_handle: input.terminalHandle,
|
||||
assignee_pane_key: input.paneKey,
|
||||
process_incarnation: input.processIncarnation,
|
||||
contract_version: CURRENT_CONTRACT_VERSION,
|
||||
capability_hash: expect.any(String),
|
||||
worker_state: 'ready',
|
||||
worktree_id: input.worktreeId,
|
||||
agent_terminal_handle: input.terminalHandle
|
||||
})
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
}
|
||||
|
||||
function markDispatchAsPreUpdateLegacy(
|
||||
userDataDir: string,
|
||||
input: {
|
||||
@@ -233,9 +274,9 @@ function markDispatchAsPreUpdateLegacy(
|
||||
})
|
||||
db.prepare(
|
||||
`UPDATE dispatch_contexts
|
||||
SET contract_version = ?, capability_hash = NULL, capability_revoked_at = NULL,
|
||||
launch_token_hash = NULL
|
||||
WHERE id = ?`
|
||||
SET contract_version = ?, capability_hash = NULL, capability_revoked_at = NULL,
|
||||
launch_token_hash = NULL
|
||||
WHERE id = ?`
|
||||
).run(LEGACY_CONTRACT_VERSION, input.dispatchId)
|
||||
} finally {
|
||||
db.close()
|
||||
@@ -248,285 +289,307 @@ test.afterAll(() => {
|
||||
rmSync(fakeCliDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
test('adopts one live legacy worker after restart without replaying resume', async (// oxlint-disable-next-line no-empty-pattern -- This lifecycle test owns both Electron launches and intentionally opts out of the default app fixture.
|
||||
{}, testInfo) => {
|
||||
test.setTimeout(300_000)
|
||||
const repoPath = existsSync(TEST_REPO_PATH_FILE)
|
||||
? readFileSync(TEST_REPO_PATH_FILE, 'utf8').trim()
|
||||
: ''
|
||||
test.skip(!repoPath || !existsSync(repoPath), 'Global setup did not produce a seeded test repo')
|
||||
for (const contractVersion of [LEGACY_CONTRACT_VERSION, CURRENT_CONTRACT_VERSION]) {
|
||||
const contractLabel = contractVersion === LEGACY_CONTRACT_VERSION ? 'legacy' : 'current'
|
||||
test(`adopts one live ${contractLabel} worker after restart without replaying resume`, async (// oxlint-disable-next-line no-empty-pattern -- This lifecycle test owns both Electron launches and intentionally opts out of the default app fixture.
|
||||
{}, testInfo) => {
|
||||
test.setTimeout(300_000)
|
||||
rmSync(spawnLedgerPath, { force: true })
|
||||
rmSync(interruptionLedgerPath, { force: true })
|
||||
const repoPath = existsSync(TEST_REPO_PATH_FILE)
|
||||
? readFileSync(TEST_REPO_PATH_FILE, 'utf8').trim()
|
||||
: ''
|
||||
test.skip(!repoPath || !existsSync(repoPath), 'Global setup did not produce a seeded test repo')
|
||||
|
||||
const session = createRestartSession(testInfo, {
|
||||
PATH: `${fakeCliDir}${path.delimiter}${process.env.PATH ?? ''}`,
|
||||
ORCA_E2E_SPAWN_LEDGER: spawnLedgerPath,
|
||||
ORCA_E2E_INTERRUPTION_LEDGER: interruptionLedgerPath
|
||||
})
|
||||
let firstApp: ElectronApplication | null = null
|
||||
let secondApp: ElectronApplication | null = null
|
||||
const session = createRestartSession(testInfo, {
|
||||
PATH: `${fakeCliDir}${path.delimiter}${process.env.PATH ?? ''}`,
|
||||
ORCA_E2E_SPAWN_LEDGER: spawnLedgerPath,
|
||||
ORCA_E2E_INTERRUPTION_LEDGER: interruptionLedgerPath
|
||||
})
|
||||
let firstApp: ElectronApplication | null = null
|
||||
let secondApp: ElectronApplication | null = null
|
||||
|
||||
try {
|
||||
const first = await session.launch()
|
||||
firstApp = first.app
|
||||
const worktreeId = await attachRepoAndOpenTerminal(first.page, repoPath)
|
||||
await waitForSessionReady(first.page)
|
||||
await ensureTerminalVisible(first.page)
|
||||
const coordinatorTabId = await getActiveTabId(first.page)
|
||||
expect(coordinatorTabId).toBeTruthy()
|
||||
await waitForActivePanePtyId(first.page)
|
||||
const coordinatorPane = await waitForActivePaneHookDescriptor(first.page)
|
||||
const firstClient = new RuntimeClient(session.userDataDir, 30_000, null, null)
|
||||
const coordinator = await firstClient.call<{ terminal: { handle: string } }>(
|
||||
'terminal.resolvePane',
|
||||
{ paneKey: coordinatorPane.paneKey }
|
||||
)
|
||||
const coordinatorTerminal = await firstClient.call<{
|
||||
terminal: { worktreeId: string }
|
||||
}>('terminal.show', { terminal: coordinator.result.terminal.handle })
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const listed = await firstClient.call<{ worktrees: { id: string }[] }>('worktree.list', {})
|
||||
return listed.result.worktrees.some(
|
||||
(candidate) => candidate.id === coordinatorTerminal.result.terminal.worktreeId
|
||||
)
|
||||
})
|
||||
.toBe(true)
|
||||
const run = await firstClient.call<{ run: { id: string } }>('orchestration.runCreate', {
|
||||
objective: 'Legacy worker restart recovery',
|
||||
from: coordinator.result.terminal.handle
|
||||
})
|
||||
const task = await firstClient.call<{ task: { id: string } }>('orchestration.taskCreate', {
|
||||
spec: 'Respond ACK and remain idle',
|
||||
run: run.result.run.id,
|
||||
callerTerminalHandle: coordinator.result.terminal.handle
|
||||
})
|
||||
const started = await firstClient.call<{
|
||||
effects: { kind: string; role?: string; id?: string }[]
|
||||
}>('orchestration.workerStart', {
|
||||
task: task.result.task.id,
|
||||
from: coordinator.result.terminal.handle,
|
||||
agent: 'codex',
|
||||
timeoutMs: 15_000
|
||||
})
|
||||
const workerHandle = started.result.effects.find(
|
||||
(effect) => effect.kind === 'terminal' && effect.role === 'agent'
|
||||
)?.id
|
||||
expect(workerHandle).toBeTruthy()
|
||||
|
||||
let worker = (
|
||||
await firstClient.call<RuntimeTerminalListResult>('terminal.list')
|
||||
).result.terminals.find((terminal) => terminal.title === 'Codex Ready')
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const listed = await firstClient.call<RuntimeTerminalListResult>('terminal.list')
|
||||
worker = listed.result.terminals.find((terminal) => terminal.title === 'Codex Ready')
|
||||
return worker?.ptyId ?? null
|
||||
})
|
||||
.toBeTruthy()
|
||||
expect(worker?.incarnationId).toBeTruthy()
|
||||
const workerPaneKey = `${worker!.tabId}:${worker!.leafId}`
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const read = await firstClient.call<{ terminal: RuntimeTerminalRead }>('terminal.read', {
|
||||
terminal: worker!.handle,
|
||||
limit: 200
|
||||
try {
|
||||
const first = await session.launch()
|
||||
firstApp = first.app
|
||||
const worktreeId = await attachRepoAndOpenTerminal(first.page, repoPath)
|
||||
await waitForSessionReady(first.page)
|
||||
await ensureTerminalVisible(first.page)
|
||||
const coordinatorTabId = await getActiveTabId(first.page)
|
||||
expect(coordinatorTabId).toBeTruthy()
|
||||
await waitForActivePanePtyId(first.page)
|
||||
const coordinatorPane = await waitForActivePaneHookDescriptor(first.page)
|
||||
const firstClient = new RuntimeClient(session.userDataDir, 30_000, null, null)
|
||||
const coordinator = await firstClient.call<{ terminal: { handle: string } }>(
|
||||
'terminal.resolvePane',
|
||||
{ paneKey: coordinatorPane.paneKey }
|
||||
)
|
||||
const coordinatorTerminal = await firstClient.call<{
|
||||
terminal: { worktreeId: string }
|
||||
}>('terminal.show', { terminal: coordinator.result.terminal.handle })
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const listed = await firstClient.call<{ worktrees: { id: string }[] }>(
|
||||
'worktree.list',
|
||||
{}
|
||||
)
|
||||
return listed.result.worktrees.some(
|
||||
(candidate) => candidate.id === coordinatorTerminal.result.terminal.worktreeId
|
||||
)
|
||||
})
|
||||
return read.result.terminal.tail.join('\n')
|
||||
.toBe(true)
|
||||
const run = await firstClient.call<{ run: { id: string } }>('orchestration.runCreate', {
|
||||
objective: 'Legacy worker restart recovery',
|
||||
from: coordinator.result.terminal.handle
|
||||
})
|
||||
.toContain('ACK')
|
||||
const initialWorker = {
|
||||
ptyId: worker!.ptyId,
|
||||
incarnationId: worker!.incarnationId,
|
||||
worktreeId: worker!.worktreeId,
|
||||
tabId: worker!.tabId,
|
||||
leafId: worker!.leafId
|
||||
}
|
||||
const initialDispatch = await firstClient.call<{
|
||||
dispatch: {
|
||||
id: string
|
||||
task_id: string
|
||||
assignee_handle: string
|
||||
assignee_pane_key: string
|
||||
process_incarnation: string
|
||||
} | null
|
||||
}>('orchestration.dispatchShow', { task: task.result.task.id })
|
||||
expect(initialDispatch.result.dispatch).toEqual(
|
||||
expect.objectContaining({
|
||||
task_id: task.result.task.id,
|
||||
assignee_pane_key: workerPaneKey,
|
||||
process_incarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`
|
||||
const task = await firstClient.call<{ task: { id: string } }>('orchestration.taskCreate', {
|
||||
spec: 'Respond ACK and remain idle',
|
||||
run: run.result.run.id,
|
||||
callerTerminalHandle: coordinator.result.terminal.handle
|
||||
})
|
||||
)
|
||||
const dispatchHandle = initialDispatch.result.dispatch!.assignee_handle
|
||||
await expect.poll(() => readLedger(spawnLedgerPath)).toHaveLength(1)
|
||||
const [initialSpawn] = readLedger(spawnLedgerPath)
|
||||
expect(isProcessAlive(initialSpawn.pid)).toBe(true)
|
||||
expect(readLedger(interruptionLedgerPath)).toEqual([])
|
||||
const started = await firstClient.call<{
|
||||
effects: { kind: string; role?: string; id?: string }[]
|
||||
}>('orchestration.workerStart', {
|
||||
task: task.result.task.id,
|
||||
from: coordinator.result.terminal.handle,
|
||||
agent: 'codex',
|
||||
timeoutMs: 15_000
|
||||
})
|
||||
const workerHandle = started.result.effects.find(
|
||||
(effect) => effect.kind === 'terminal' && effect.role === 'agent'
|
||||
)?.id
|
||||
expect(workerHandle).toBeTruthy()
|
||||
|
||||
const transcriptPath = session.seedCodexResumeRollout(PROVIDER_SESSION_ID, repoPath)
|
||||
await first.page.evaluate(
|
||||
({ paneKey, tabId, worktreeId: workerWorktreeId, terminalHandle, transcript }) => {
|
||||
window.__store?.getState().setAgentStatus(
|
||||
paneKey,
|
||||
{ state: 'working', prompt: 'Respond ACK and remain idle', agentType: 'codex' },
|
||||
'Codex Ready',
|
||||
undefined,
|
||||
{ tabId, worktreeId: workerWorktreeId, terminalHandle },
|
||||
{
|
||||
providerSession: {
|
||||
key: 'session_id',
|
||||
id: 'e2e-legacy-orchestration-worker',
|
||||
transcriptPath: transcript
|
||||
},
|
||||
launchConfig: {
|
||||
agentCommand: 'codex',
|
||||
agentArgs: '--dangerously-bypass-approvals-and-sandbox',
|
||||
agentEnv: {}
|
||||
}
|
||||
}
|
||||
)
|
||||
window.__store?.getState().captureAllSleepingAgentSessions('quit')
|
||||
},
|
||||
{
|
||||
paneKey: workerPaneKey,
|
||||
tabId: worker!.tabId,
|
||||
let worker = (
|
||||
await firstClient.call<RuntimeTerminalListResult>('terminal.list')
|
||||
).result.terminals.find((terminal) => terminal.title === 'Codex Ready')
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const listed = await firstClient.call<RuntimeTerminalListResult>('terminal.list')
|
||||
worker = listed.result.terminals.find((terminal) => terminal.title === 'Codex Ready')
|
||||
return worker?.ptyId ?? null
|
||||
})
|
||||
.toBeTruthy()
|
||||
expect(worker?.incarnationId).toBeTruthy()
|
||||
const workerPaneKey = `${worker!.tabId}:${worker!.leafId}`
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const read = await firstClient.call<{ terminal: RuntimeTerminalRead }>('terminal.read', {
|
||||
terminal: worker!.handle,
|
||||
limit: 200
|
||||
})
|
||||
return read.result.terminal.tail.join('\n')
|
||||
})
|
||||
.toContain('ACK')
|
||||
const initialWorker = {
|
||||
ptyId: worker!.ptyId,
|
||||
incarnationId: worker!.incarnationId,
|
||||
worktreeId: worker!.worktreeId,
|
||||
terminalHandle: worker!.handle,
|
||||
transcript: transcriptPath
|
||||
tabId: worker!.tabId,
|
||||
leafId: worker!.leafId
|
||||
}
|
||||
)
|
||||
await expect
|
||||
.poll(() => hasPersistedResumeRecord(session.userDataDir, workerPaneKey), {
|
||||
timeout: 30_000
|
||||
})
|
||||
.toBe(true)
|
||||
|
||||
await session.close(firstApp)
|
||||
firstApp = null
|
||||
expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn])
|
||||
expect(readLedger(interruptionLedgerPath)).toEqual([])
|
||||
expect(isProcessAlive(initialSpawn.pid)).toBe(true)
|
||||
|
||||
stripLegacyWorkerRendererBinding(session.userDataDir, {
|
||||
worktreeId,
|
||||
coordinatorTabId: coordinatorTabId!,
|
||||
workerTabId: worker!.tabId,
|
||||
workerPaneKey
|
||||
})
|
||||
markDispatchAsPreUpdateLegacy(session.userDataDir, {
|
||||
dispatchId: initialDispatch.result.dispatch!.id,
|
||||
terminalHandle: dispatchHandle,
|
||||
paneKey: workerPaneKey,
|
||||
processIncarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`,
|
||||
worktreeId: initialWorker.worktreeId
|
||||
})
|
||||
|
||||
const second = await session.launch()
|
||||
secondApp = second.app
|
||||
await waitForSessionReady(second.page)
|
||||
expect(await waitForActiveWorktree(second.page)).toBe(worktreeId)
|
||||
const secondClient = new RuntimeClient(session.userDataDir, 30_000, null, null)
|
||||
let recovered = (
|
||||
await secondClient.call<RuntimeTerminalListResult>('terminal.list')
|
||||
).result.terminals.find((terminal) => terminal.ptyId === initialWorker.ptyId)
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const listed = await secondClient.call<RuntimeTerminalListResult>('terminal.list')
|
||||
const matches = listed.result.terminals.filter(
|
||||
(terminal) => terminal.ptyId === initialWorker.ptyId
|
||||
)
|
||||
recovered = matches[0]
|
||||
return matches
|
||||
})
|
||||
.toEqual([
|
||||
const initialDispatch = await firstClient.call<{
|
||||
dispatch: {
|
||||
id: string
|
||||
task_id: string
|
||||
assignee_handle: string
|
||||
assignee_pane_key: string
|
||||
process_incarnation: string
|
||||
} | null
|
||||
}>('orchestration.dispatchShow', { task: task.result.task.id })
|
||||
expect(initialDispatch.result.dispatch).toEqual(
|
||||
expect.objectContaining({
|
||||
...initialWorker,
|
||||
connected: true,
|
||||
writable: true
|
||||
task_id: task.result.task.id,
|
||||
assignee_pane_key: workerPaneKey,
|
||||
process_incarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`
|
||||
})
|
||||
])
|
||||
)
|
||||
const dispatchHandle = initialDispatch.result.dispatch!.assignee_handle
|
||||
await expect.poll(() => readLedger(spawnLedgerPath)).toHaveLength(1)
|
||||
const [initialSpawn] = readLedger(spawnLedgerPath)
|
||||
expect(isProcessAlive(initialSpawn.pid)).toBe(true)
|
||||
expect(readLedger(interruptionLedgerPath)).toEqual([])
|
||||
|
||||
const recoveredTab = second.page.locator(
|
||||
`[data-testid="sortable-tab"][data-tab-id="${initialWorker.tabId}"]`
|
||||
)
|
||||
await expect(recoveredTab).toBeVisible()
|
||||
await expect(recoveredTab).toHaveCount(1)
|
||||
await expect(recoveredTab).toHaveAttribute('data-active', 'false')
|
||||
await expect(
|
||||
second.page.locator(`[data-testid="sortable-tab"][data-tab-id="${coordinatorTabId!}"]`)
|
||||
).toHaveAttribute('data-active', 'true')
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const read = await secondClient.call<{ terminal: RuntimeTerminalRead }>('terminal.read', {
|
||||
terminal: recovered!.handle,
|
||||
limit: 200
|
||||
const transcriptPath = session.seedCodexResumeRollout(PROVIDER_SESSION_ID, repoPath)
|
||||
await first.page.evaluate(
|
||||
({ paneKey, tabId, worktreeId: workerWorktreeId, terminalHandle, transcript }) => {
|
||||
window.__store?.getState().setAgentStatus(
|
||||
paneKey,
|
||||
{ state: 'working', prompt: 'Respond ACK and remain idle', agentType: 'codex' },
|
||||
'Codex Ready',
|
||||
undefined,
|
||||
{ tabId, worktreeId: workerWorktreeId, terminalHandle },
|
||||
{
|
||||
providerSession: {
|
||||
key: 'session_id',
|
||||
id: 'e2e-legacy-orchestration-worker',
|
||||
transcriptPath: transcript
|
||||
},
|
||||
launchConfig: {
|
||||
agentCommand: 'codex',
|
||||
agentArgs: '--dangerously-bypass-approvals-and-sandbox',
|
||||
agentEnv: {}
|
||||
}
|
||||
}
|
||||
)
|
||||
window.__store?.getState().captureAllSleepingAgentSessions('quit')
|
||||
},
|
||||
{
|
||||
paneKey: workerPaneKey,
|
||||
tabId: worker!.tabId,
|
||||
worktreeId: worker!.worktreeId,
|
||||
terminalHandle: worker!.handle,
|
||||
transcript: transcriptPath
|
||||
}
|
||||
)
|
||||
await expect
|
||||
.poll(() => hasPersistedResumeRecord(session.userDataDir, workerPaneKey), {
|
||||
timeout: 30_000
|
||||
})
|
||||
return read.result.terminal.tail.join('\n')
|
||||
})
|
||||
.toContain('ACK')
|
||||
.toBe(true)
|
||||
|
||||
const restoredRun = await secondClient.call<{ run: { id: string } }>('orchestration.runShow', {
|
||||
id: run.result.run.id
|
||||
})
|
||||
expect(restoredRun.result.run.id).toBe(run.result.run.id)
|
||||
const tasks = await secondClient.call<{ tasks: { id: string }[] }>('orchestration.taskList', {
|
||||
run: run.result.run.id
|
||||
})
|
||||
expect(tasks.result.tasks).toEqual(
|
||||
expect.arrayContaining([expect.objectContaining({ id: task.result.task.id })])
|
||||
)
|
||||
const recoveredDispatch = await secondClient.call<{
|
||||
dispatch: {
|
||||
id: string
|
||||
task_id: string
|
||||
assignee_handle: string
|
||||
assignee_pane_key: string
|
||||
process_incarnation: string
|
||||
contract_version: number
|
||||
} | null
|
||||
}>('orchestration.dispatchShow', { task: task.result.task.id })
|
||||
expect(recoveredDispatch.result.dispatch).toEqual(
|
||||
expect.objectContaining({
|
||||
id: initialDispatch.result.dispatch!.id,
|
||||
task_id: task.result.task.id,
|
||||
assignee_handle: dispatchHandle,
|
||||
assignee_pane_key: workerPaneKey,
|
||||
process_incarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`,
|
||||
contract_version: LEGACY_CONTRACT_VERSION
|
||||
})
|
||||
)
|
||||
await expect
|
||||
.poll(async () => ({
|
||||
renderer: await readRendererRecoveryState(second.page, workerPaneKey, initialWorker.tabId),
|
||||
persisted: hasPersistedResumeRecord(session.userDataDir, workerPaneKey)
|
||||
}))
|
||||
.toEqual({
|
||||
renderer: { sleeping: false, resumeClaim: false, pendingStartup: false },
|
||||
persisted: false
|
||||
})
|
||||
expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn])
|
||||
expect(readLedger(interruptionLedgerPath)).toEqual([])
|
||||
expect(isProcessAlive(initialSpawn.pid)).toBe(true)
|
||||
await session.close(firstApp)
|
||||
firstApp = null
|
||||
expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn])
|
||||
expect(readLedger(interruptionLedgerPath)).toEqual([])
|
||||
expect(isProcessAlive(initialSpawn.pid)).toBe(true)
|
||||
|
||||
const otherWorktreeId = await switchToOtherWorktree(second.page, worktreeId)
|
||||
expect(otherWorktreeId).toBeTruthy()
|
||||
await switchToWorktree(second.page, worktreeId)
|
||||
await expect(recoveredTab).toBeVisible()
|
||||
await expect(recoveredTab).toHaveCount(1)
|
||||
await expect(recoveredTab).toHaveAttribute('data-active', 'false')
|
||||
await expect
|
||||
.poll(async () => readRendererRecoveryState(second.page, workerPaneKey, initialWorker.tabId))
|
||||
.toEqual({ sleeping: false, resumeClaim: false, pendingStartup: false })
|
||||
expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn])
|
||||
expect(readLedger(interruptionLedgerPath)).toEqual([])
|
||||
expect(isProcessAlive(initialSpawn.pid)).toBe(true)
|
||||
await expect(second.page.locator('body')).not.toContainText('Conversation interrupted')
|
||||
} finally {
|
||||
if (secondApp) {
|
||||
await session.close(secondApp).catch(() => undefined)
|
||||
stripLegacyWorkerRendererBinding(session.userDataDir, {
|
||||
worktreeId,
|
||||
coordinatorTabId: coordinatorTabId!,
|
||||
workerTabId: worker!.tabId,
|
||||
workerPaneKey
|
||||
})
|
||||
const dispatchIdentity = {
|
||||
dispatchId: initialDispatch.result.dispatch!.id,
|
||||
terminalHandle: dispatchHandle,
|
||||
paneKey: workerPaneKey,
|
||||
processIncarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`,
|
||||
worktreeId: initialWorker.worktreeId
|
||||
}
|
||||
if (contractVersion === LEGACY_CONTRACT_VERSION) {
|
||||
markDispatchAsPreUpdateLegacy(session.userDataDir, dispatchIdentity)
|
||||
} else {
|
||||
assertDispatchRemainsCurrent(session.userDataDir, dispatchIdentity)
|
||||
}
|
||||
|
||||
const second = await session.launch()
|
||||
secondApp = second.app
|
||||
await waitForSessionReady(second.page)
|
||||
expect(await waitForActiveWorktree(second.page)).toBe(worktreeId)
|
||||
const secondClient = new RuntimeClient(session.userDataDir, 30_000, null, null)
|
||||
let recovered = (
|
||||
await secondClient.call<RuntimeTerminalListResult>('terminal.list')
|
||||
).result.terminals.find((terminal) => terminal.ptyId === initialWorker.ptyId)
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const listed = await secondClient.call<RuntimeTerminalListResult>('terminal.list')
|
||||
const matches = listed.result.terminals.filter(
|
||||
(terminal) => terminal.ptyId === initialWorker.ptyId
|
||||
)
|
||||
recovered = matches[0]
|
||||
return matches
|
||||
})
|
||||
.toEqual([
|
||||
expect.objectContaining({
|
||||
...initialWorker,
|
||||
connected: true,
|
||||
writable: true
|
||||
})
|
||||
])
|
||||
|
||||
const recoveredTab = second.page.locator(
|
||||
`[data-testid="sortable-tab"][data-tab-id="${initialWorker.tabId}"]`
|
||||
)
|
||||
await expect(recoveredTab).toBeVisible()
|
||||
await expect(recoveredTab).toHaveCount(1)
|
||||
await expect(recoveredTab).toHaveAttribute('data-active', 'false')
|
||||
await expect(
|
||||
second.page.locator(`[data-testid="sortable-tab"][data-tab-id="${coordinatorTabId!}"]`)
|
||||
).toHaveAttribute('data-active', 'true')
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const read = await secondClient.call<{ terminal: RuntimeTerminalRead }>('terminal.read', {
|
||||
terminal: recovered!.handle,
|
||||
limit: 200
|
||||
})
|
||||
return read.result.terminal.tail.join('\n')
|
||||
})
|
||||
.toContain('ACK')
|
||||
|
||||
const restoredRun = await secondClient.call<{ run: { id: string } }>(
|
||||
'orchestration.runShow',
|
||||
{
|
||||
id: run.result.run.id
|
||||
}
|
||||
)
|
||||
expect(restoredRun.result.run.id).toBe(run.result.run.id)
|
||||
const tasks = await secondClient.call<{ tasks: { id: string }[] }>('orchestration.taskList', {
|
||||
run: run.result.run.id
|
||||
})
|
||||
expect(tasks.result.tasks).toEqual(
|
||||
expect.arrayContaining([expect.objectContaining({ id: task.result.task.id })])
|
||||
)
|
||||
const recoveredDispatch = await secondClient.call<{
|
||||
dispatch: {
|
||||
id: string
|
||||
task_id: string
|
||||
assignee_handle: string
|
||||
assignee_pane_key: string
|
||||
process_incarnation: string
|
||||
contract_version: number
|
||||
} | null
|
||||
}>('orchestration.dispatchShow', { task: task.result.task.id })
|
||||
expect(recoveredDispatch.result.dispatch).toEqual(
|
||||
expect.objectContaining({
|
||||
id: initialDispatch.result.dispatch!.id,
|
||||
task_id: task.result.task.id,
|
||||
assignee_handle: dispatchHandle,
|
||||
assignee_pane_key: workerPaneKey,
|
||||
process_incarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`,
|
||||
contract_version: contractVersion
|
||||
})
|
||||
)
|
||||
await expect
|
||||
.poll(async () => ({
|
||||
renderer: await readRendererRecoveryState(
|
||||
second.page,
|
||||
workerPaneKey,
|
||||
initialWorker.tabId
|
||||
),
|
||||
persisted: hasPersistedResumeRecord(session.userDataDir, workerPaneKey)
|
||||
}))
|
||||
.toEqual({
|
||||
renderer: { sleeping: false, resumeClaim: false, pendingStartup: false },
|
||||
persisted: false
|
||||
})
|
||||
expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn])
|
||||
expect(readLedger(interruptionLedgerPath)).toEqual([])
|
||||
expect(isProcessAlive(initialSpawn.pid)).toBe(true)
|
||||
|
||||
const otherWorktreeId = await switchToOtherWorktree(second.page, worktreeId)
|
||||
expect(otherWorktreeId).toBeTruthy()
|
||||
await switchToWorktree(second.page, worktreeId)
|
||||
await expect(recoveredTab).toBeVisible()
|
||||
await expect(recoveredTab).toHaveCount(1)
|
||||
await expect(recoveredTab).toHaveAttribute('data-active', 'false')
|
||||
await expect
|
||||
.poll(async () =>
|
||||
readRendererRecoveryState(second.page, workerPaneKey, initialWorker.tabId)
|
||||
)
|
||||
.toEqual({ sleeping: false, resumeClaim: false, pendingStartup: false })
|
||||
expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn])
|
||||
expect(readLedger(interruptionLedgerPath)).toEqual([])
|
||||
expect(isProcessAlive(initialSpawn.pid)).toBe(true)
|
||||
await expect(second.page.locator('body')).not.toContainText('Conversation interrupted')
|
||||
} finally {
|
||||
if (secondApp) {
|
||||
await session.close(secondApp).catch(() => undefined)
|
||||
}
|
||||
if (firstApp) {
|
||||
await session.close(firstApp).catch(() => undefined)
|
||||
}
|
||||
await session.dispose()
|
||||
}
|
||||
if (firstApp) {
|
||||
await session.close(firstApp).catch(() => undefined)
|
||||
}
|
||||
await session.dispose()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user