mirror of
https://github.com/stablyai/orca.git
synced 2026-10-28 00:02:54 +00:00
fix(orchestration): complete legacy worker cutover recovery
This commit is contained in:
File diff suppressed because one or more lines are too long
@@ -71,4 +71,24 @@ describe('orchestration check identity', () => {
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('preserves the pinned legacy --inject check signature', async () => {
|
||||
process.env.ORCA_TERMINAL_HANDLE = 'term_legacy_worker'
|
||||
|
||||
await invokeCheck(
|
||||
new Map<string, string | boolean>([
|
||||
['unread', true],
|
||||
['inject', true]
|
||||
])
|
||||
)
|
||||
|
||||
expect(callMock).toHaveBeenCalledWith(
|
||||
'orchestration.check',
|
||||
expect.objectContaining({
|
||||
terminal: 'term_legacy_worker',
|
||||
unread: true,
|
||||
inject: true
|
||||
})
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -179,6 +179,7 @@ describe('legacy orchestration CLI inspection', () => {
|
||||
id: 'msg_current',
|
||||
run_id: 'run_current',
|
||||
from_handle: 'term_current',
|
||||
to_handle: 'run:run_current',
|
||||
subject: 'current question',
|
||||
type: 'question',
|
||||
body: 'May I continue?',
|
||||
@@ -217,7 +218,8 @@ describe('legacy orchestration CLI inspection', () => {
|
||||
expect(response.result.formatted).not.toContain('RUNTIME_SENTINEL')
|
||||
expect(
|
||||
response.result.formatted.split('\n').filter((line) => line.startsWith('[Reply:'))
|
||||
).toEqual(['[Reply: orca orchestration reply --id msg_current --from term_coord --body "..."]'])
|
||||
).toEqual(['[Reply: orca orchestration reply --id msg_current --body "..."]'])
|
||||
expect(response.result.formatted).not.toContain('--from run:run_current')
|
||||
})
|
||||
|
||||
it('preserves runtime formatting when every message belongs to a current Run', async () => {
|
||||
|
||||
@@ -62,6 +62,28 @@ describe('lightweight Run CLI handlers', () => {
|
||||
from: 'term_coord'
|
||||
})
|
||||
})
|
||||
|
||||
it('passes explicit legacy takeover only when requested', async () => {
|
||||
callMock.mockResolvedValue({
|
||||
result: { run: { id: 'run_adopted', objective: 'Recovered work' } }
|
||||
})
|
||||
await ORCHESTRATION_HANDLERS['orchestration run-use']({
|
||||
flags: new Map<string, string | boolean>([
|
||||
['id', 'run_adopted'],
|
||||
['from', 'term_current'],
|
||||
['takeover-legacy', true]
|
||||
]),
|
||||
client: { call: callMock },
|
||||
cwd: '/tmp/repo',
|
||||
json: true
|
||||
} as never)
|
||||
|
||||
expect(callMock).toHaveBeenCalledWith('orchestration.runUse', {
|
||||
id: 'run_adopted',
|
||||
from: 'term_current',
|
||||
takeoverLegacy: true
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('orchestration reset CLI handler', () => {
|
||||
|
||||
@@ -21,10 +21,14 @@ import type {
|
||||
} from '../../shared/orchestration-worker-output'
|
||||
import type { NativeChatMessage } from '../../shared/native-chat-types'
|
||||
import type { RuntimeTerminalRead } from '../../shared/runtime-types'
|
||||
import { orchestrationMigrationData } from '../../shared/orchestration-rpc-contract'
|
||||
import {
|
||||
ORCHESTRATION_LEGACY_RUN_ID,
|
||||
orchestrationMigrationData
|
||||
} from '../../shared/orchestration-rpc-contract'
|
||||
formatMessageReadOnlyTag,
|
||||
formatOrchestrationCheckText,
|
||||
prepareOrchestrationCheckOutput,
|
||||
type LegacyCompatibilityResult,
|
||||
type OrchestrationMessageSummary as MessageSummary
|
||||
} from '../../shared/orchestration-check-output'
|
||||
|
||||
// Why: 15 s is well under Claude Code's ~2 min Bash-tool silence budget while keeping log volume low. See design doc §3.4.
|
||||
const DEFAULT_KEEPALIVE_INTERVAL_MS = 15_000
|
||||
@@ -74,93 +78,6 @@ const TASK_STATUS_VALUES = [
|
||||
'blocked'
|
||||
] as const
|
||||
|
||||
type MessageSummary = {
|
||||
id: string
|
||||
run_id?: string
|
||||
delivery_contract?: 'legacy_direct' | 'current_delivery' | 'audit_only'
|
||||
from_handle: string
|
||||
to_handle?: string
|
||||
subject?: string
|
||||
type?: string
|
||||
body?: string
|
||||
payload?: string | null
|
||||
priority?: string
|
||||
read?: number
|
||||
}
|
||||
|
||||
function formatMessageReadOnlyTag(
|
||||
message: MessageSummary,
|
||||
legacyCompatibilityActive = false
|
||||
): string {
|
||||
return isLegacyReadOnlyMessage(message, legacyCompatibilityActive) ? ' [legacy, read-only]' : ''
|
||||
}
|
||||
|
||||
function isLegacyReadOnlyMessage(
|
||||
message: MessageSummary,
|
||||
legacyCompatibilityActive = false
|
||||
): boolean {
|
||||
return (
|
||||
message.run_id === ORCHESTRATION_LEGACY_RUN_ID ||
|
||||
(message.delivery_contract === 'legacy_direct' && !legacyCompatibilityActive) ||
|
||||
message.delivery_contract === 'audit_only'
|
||||
)
|
||||
}
|
||||
|
||||
function formatMessagePriorityTag(message: MessageSummary): string {
|
||||
return message.priority === 'urgent' ? ' [URGENT]' : message.priority === 'high' ? ' [HIGH]' : ''
|
||||
}
|
||||
|
||||
function escapeTerminalControlCharacters(value: string): string {
|
||||
return [...value]
|
||||
.map((character) => {
|
||||
const code = character.charCodeAt(0)
|
||||
if (character === '\n' || (code >= 0x20 && code < 0x7f) || code > 0x9f) {
|
||||
return character
|
||||
}
|
||||
return `\\x${code.toString(16).padStart(2, '0')}`
|
||||
})
|
||||
.join('')
|
||||
}
|
||||
|
||||
function formatQuotedMessageField(label: string, value?: string): string {
|
||||
return `[${label}]\n${escapeTerminalControlCharacters(value ?? '')
|
||||
.split('\n')
|
||||
.map((line) => ` ${line}`)
|
||||
.join('\n')}`
|
||||
}
|
||||
|
||||
function formatLegacyAwareCheckMessages(
|
||||
messages: MessageSummary[],
|
||||
checkedTerminal: string,
|
||||
legacyCompatibilityActive = false
|
||||
): string {
|
||||
return messages
|
||||
.map((message) => {
|
||||
const legacyReadOnly = isLegacyReadOnlyMessage(message, legacyCompatibilityActive)
|
||||
const lines = [
|
||||
`${message.id}${formatMessageReadOnlyTag(message, legacyCompatibilityActive)}${formatMessagePriorityTag(message)} [${message.type ?? 'status'}] from=${message.from_handle}`,
|
||||
formatQuotedMessageField('subject', message.subject)
|
||||
]
|
||||
if (legacyReadOnly) {
|
||||
lines.push('[Inspection only: reply and acknowledgment are unavailable.]')
|
||||
}
|
||||
if (message.body) {
|
||||
lines.push(formatQuotedMessageField('body', message.body))
|
||||
}
|
||||
if (message.payload) {
|
||||
lines.push(formatQuotedMessageField('payload', message.payload))
|
||||
}
|
||||
if (!legacyReadOnly) {
|
||||
const replyFrom = message.to_handle ?? checkedTerminal
|
||||
lines.push(
|
||||
`[Reply: orca orchestration reply --id ${message.id} --from ${replyFrom} --body "..."]`
|
||||
)
|
||||
}
|
||||
return lines.join('\n')
|
||||
})
|
||||
.join('\n\n')
|
||||
}
|
||||
|
||||
type LifecycleSendRejection = {
|
||||
action: 'rejected'
|
||||
code: string
|
||||
@@ -181,23 +98,6 @@ type OrchestrationSendResult =
|
||||
lifecycle?: { action: 'completed' | 'failed' }
|
||||
}
|
||||
|
||||
type LegacyCompatibilityResult = {
|
||||
recovery?: boolean
|
||||
readOnly?: boolean
|
||||
ackMessageIds?: string[]
|
||||
answerAcknowledgement?: {
|
||||
questionId: string
|
||||
answerMessageId: string
|
||||
}
|
||||
currentDelivery?: {
|
||||
runId: string
|
||||
checkCommand: string
|
||||
ackCommand: string
|
||||
}
|
||||
resumeRequired?: boolean
|
||||
resumeCommand?: string
|
||||
}
|
||||
|
||||
function resolveCompatibilityCliCommand(): 'orca' | 'orca-ide' | 'orca-dev' {
|
||||
const configured = process.env.ORCA_CLI_COMMAND
|
||||
if (configured === 'orca' || configured === 'orca-ide' || configured === 'orca-dev') {
|
||||
@@ -206,19 +106,6 @@ function resolveCompatibilityCliCommand(): 'orca' | 'orca-ide' | 'orca-dev' {
|
||||
return process.platform === 'linux' ? 'orca-ide' : 'orca'
|
||||
}
|
||||
|
||||
function formatCurrentDeliveryNotice(
|
||||
delivery: LegacyCompatibilityResult['currentDelivery']
|
||||
): string {
|
||||
if (!delivery) {
|
||||
return ''
|
||||
}
|
||||
return (
|
||||
`\n[CURRENT RUN MAIL WAITING]\n` +
|
||||
`Read: ${delivery.checkCommand}\n` +
|
||||
`Then acknowledge the delivery ID it prints: ${delivery.ackCommand}`
|
||||
)
|
||||
}
|
||||
|
||||
function resolvePackagedWindowsCompatibilityCommand(): 'orca' | 'orca-ide' | undefined {
|
||||
if (process.env.ORCA_WINDOWS_PACKAGED_CLI_LAUNCHER !== '1') {
|
||||
return undefined
|
||||
@@ -560,7 +447,8 @@ export const ORCHESTRATION_HANDLERS: Record<string, CommandHandler> = {
|
||||
run: { id: string; objective: string; consumer_generation: number }
|
||||
}>(client, flags, 'orchestration.runUse', {
|
||||
id: getRequiredStringFlag(flags, 'id'),
|
||||
from
|
||||
from,
|
||||
...(flags.has('takeover-legacy') ? { takeoverLegacy: true } : {})
|
||||
})
|
||||
printResult(result, json, (r) => `Using Run ${r.run.id}: ${r.run.objective}`)
|
||||
},
|
||||
@@ -715,6 +603,7 @@ export const ORCHESTRATION_HANDLERS: Record<string, CommandHandler> = {
|
||||
all: flags.has('all') ? true : undefined,
|
||||
types: getOptionalStringFlag(flags, 'types'),
|
||||
format: flags.has('format') ? true : undefined,
|
||||
inject: flags.has('inject') ? true : undefined,
|
||||
compatibilityCliCommand: resolveCompatibilityCliCommand(),
|
||||
run: getOptionalStringFlag(flags, 'run'),
|
||||
ack: getOptionalStringFlag(flags, 'ack'),
|
||||
@@ -752,64 +641,11 @@ export const ORCHESTRATION_HANDLERS: Record<string, CommandHandler> = {
|
||||
}
|
||||
}
|
||||
}
|
||||
const compatibilityActive = Boolean(
|
||||
result.result.legacyCompatibility && !result.result.legacyCompatibility.readOnly
|
||||
)
|
||||
if (
|
||||
flags.has('format') &&
|
||||
result.result.messages.some((message) =>
|
||||
isLegacyReadOnlyMessage(message, compatibilityActive)
|
||||
)
|
||||
) {
|
||||
// Why: formatted is one opaque batch with untrusted bodies, so selective banner parsing cannot safely remove legacy actions.
|
||||
result = {
|
||||
...result,
|
||||
result: {
|
||||
...result.result,
|
||||
formatted: formatLegacyAwareCheckMessages(
|
||||
result.result.messages,
|
||||
terminal,
|
||||
compatibilityActive
|
||||
)
|
||||
}
|
||||
}
|
||||
result = {
|
||||
...result,
|
||||
result: prepareOrchestrationCheckOutput(result.result, terminal, flags.has('format'))
|
||||
}
|
||||
printResult(result, json, (r) => {
|
||||
const legacyHeader = r.legacyCompatibility
|
||||
? r.legacyCompatibility.readOnly
|
||||
? '[LEGACY READ-ONLY]\n'
|
||||
: r.legacyCompatibility.recovery
|
||||
? '[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]\n'
|
||||
: '[LEGACY COMPATIBILITY]\n'
|
||||
: ''
|
||||
const deliveryNotice = formatCurrentDeliveryNotice(r.legacyCompatibility?.currentDelivery)
|
||||
if (r.formatted) {
|
||||
return `${legacyHeader}${r.formatted}${deliveryNotice}`
|
||||
}
|
||||
if (r.count === 0) {
|
||||
if (r.timedOut) {
|
||||
return `${legacyHeader}Wait timed out; no messages were consumed.${deliveryNotice}`
|
||||
}
|
||||
if (r.cancelled) {
|
||||
const cancelled = r.connectionLost
|
||||
? 'Wait cancelled because the connection closed; no messages were consumed.'
|
||||
: 'Wait cancelled; no messages were consumed.'
|
||||
return `${legacyHeader}${cancelled}${deliveryNotice}`
|
||||
}
|
||||
return `${legacyHeader}No messages.${deliveryNotice}`
|
||||
}
|
||||
const rendered = r.messages
|
||||
.map(
|
||||
(m) =>
|
||||
`${m.id}${formatMessageReadOnlyTag(
|
||||
m,
|
||||
Boolean(r.legacyCompatibility && !r.legacyCompatibility.readOnly)
|
||||
)} [${m.type ?? 'status'}] from=${m.from_handle} "${m.subject}"`
|
||||
)
|
||||
.join('\n')
|
||||
const output = r.deliveryId ? `Delivery ${r.deliveryId}\n${rendered}` : rendered
|
||||
return `${legacyHeader}${output}${deliveryNotice}`
|
||||
})
|
||||
printResult(result, json, (r) => formatOrchestrationCheckText(r, terminal))
|
||||
const compatibilityAck = result.result.legacyCompatibility?.ackMessageIds
|
||||
if (compatibilityAck && compatibilityAck.length > 0) {
|
||||
await flushStdout()
|
||||
|
||||
@@ -16,8 +16,12 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
{
|
||||
path: ['orchestration', 'run-use'],
|
||||
summary: 'Bind this coordinator terminal to an existing Run',
|
||||
usage: 'orca orchestration run-use --id <run_id> [--from <handle>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'id', 'from', 'retry-request']
|
||||
usage:
|
||||
'orca orchestration run-use --id <run_id> [--from <handle>] [--takeover-legacy] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'id', 'from', 'takeover-legacy', 'retry-request'],
|
||||
notes: [
|
||||
'--takeover-legacy explicitly replaces an unavailable legacy coordinator while preserving live worker assignments.'
|
||||
]
|
||||
},
|
||||
{
|
||||
path: ['orchestration', 'run-current'],
|
||||
|
||||
@@ -2220,8 +2220,6 @@ void app.whenReady().then(async () => {
|
||||
getAgentProviderSessionSnapshot: () => agentHookServer.getStatusSnapshot(),
|
||||
getAgentProviderSessionRowsForPane: (paneKey) =>
|
||||
agentHookServer.getStatusSnapshotForPane(paneKey),
|
||||
getHydratedAgentHookAuthority: () => agentHookServer.getHydratedAuthorityCommitments(),
|
||||
getCurrentAgentHookAuthority: () => agentHookServer.getCurrentAuthorityObservations(),
|
||||
attestAgentHookCompatibilityAuthority: (candidate) =>
|
||||
agentHookServer.attestCompatibilityAuthority(candidate),
|
||||
canRecoverPersistentLocalPtys: () => getDaemonProvider() !== null,
|
||||
|
||||
@@ -5374,6 +5374,37 @@ describe('registerPtyHandlers', () => {
|
||||
expect(listProcesses).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('scopes runtime inventories to the requested provider', async () => {
|
||||
const localList = vi
|
||||
.spyOn(getLocalPtyProvider(), 'listProcesses')
|
||||
.mockResolvedValue([{ id: 'local-pty', title: 'Local', cwd: '/local' }])
|
||||
const sshAList = vi.fn(async () => [{ id: 'ssh-a-pty' }])
|
||||
const sshBList = vi.fn(async () => {
|
||||
throw new Error('ssh-b unavailable')
|
||||
})
|
||||
registerSshPtyProvider('ssh-a', { listProcesses: sshAList } as never)
|
||||
registerSshPtyProvider('ssh-b', { listProcesses: sshBList } as never)
|
||||
const runtime = { setPtyController: vi.fn() }
|
||||
handlers.clear()
|
||||
registerPtyHandlers(mainWindow as never, runtime as never)
|
||||
const controller = runtime.setPtyController.mock.calls[0]?.[0] as {
|
||||
listProcesses(connectionId?: string | null): Promise<{ id: string }[]>
|
||||
}
|
||||
|
||||
await expect(controller.listProcesses(null)).resolves.toEqual([
|
||||
{ id: 'local-pty', title: 'Local', cwd: '/local' }
|
||||
])
|
||||
expect(localList).toHaveBeenCalledOnce()
|
||||
expect(sshAList).not.toHaveBeenCalled()
|
||||
expect(sshBList).not.toHaveBeenCalled()
|
||||
|
||||
await expect(controller.listProcesses('ssh-a')).resolves.toEqual([{ id: 'ssh-a-pty' }])
|
||||
expect(sshAList).toHaveBeenCalledOnce()
|
||||
expect(sshBList).not.toHaveBeenCalled()
|
||||
|
||||
await expect(controller.listProcesses()).rejects.toThrow('ssh-b unavailable')
|
||||
})
|
||||
|
||||
it('returns unavailable runtime confirmation for unsupported or missing providers', async () => {
|
||||
registerSshPtyProvider('ssh-1', {} as never)
|
||||
setPtyOwnership('unsupported-pty', 'ssh-1')
|
||||
|
||||
+7
-1
@@ -4253,7 +4253,13 @@ export function registerPtyHandlers(
|
||||
return null
|
||||
}
|
||||
},
|
||||
listProcesses: async () => {
|
||||
listProcesses: async (connectionId) => {
|
||||
if (connectionId === null) {
|
||||
return localProvider.listProcesses()
|
||||
}
|
||||
if (connectionId !== undefined) {
|
||||
return getProvider(connectionId).listProcesses()
|
||||
}
|
||||
const providerSessions = await Promise.all([
|
||||
localProvider.listProcesses(),
|
||||
...Array.from(sshProviders.values(), (provider) => provider.listProcesses())
|
||||
|
||||
@@ -16337,6 +16337,17 @@ describe('OrcaRuntimeService', () => {
|
||||
['pty-shell', 'inc-shell', 'term_shell', 'Shell']
|
||||
] as const
|
||||
const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow: vi.fn() } as never)
|
||||
const listProcesses = vi.fn(async () =>
|
||||
processes.map(([id, incarnationId, terminalHandle, title]) => ({
|
||||
id,
|
||||
incarnationId,
|
||||
terminalHandle,
|
||||
title,
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
wslDistro: null
|
||||
}))
|
||||
)
|
||||
runtime.setPtyController({
|
||||
write: (ptyId, data) => {
|
||||
writes.push([ptyId, data])
|
||||
@@ -16345,16 +16356,7 @@ describe('OrcaRuntimeService', () => {
|
||||
resize,
|
||||
kill: () => true,
|
||||
getForegroundProcess: async () => null,
|
||||
listProcesses: async () =>
|
||||
processes.map(([id, incarnationId, terminalHandle, title]) => ({
|
||||
id,
|
||||
incarnationId,
|
||||
terminalHandle,
|
||||
title,
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
wslDistro: null
|
||||
}))
|
||||
listProcesses
|
||||
})
|
||||
const before = await runtime.listTerminals(`id:${TEST_WORKTREE_ID}`)
|
||||
expect(before.terminals.map((terminal) => terminal.tabId)).toEqual(
|
||||
@@ -16442,6 +16444,14 @@ describe('OrcaRuntimeService', () => {
|
||||
tail: ['legacy output']
|
||||
})
|
||||
|
||||
const inventoryCount = listProcesses.mock.calls.length
|
||||
const agentPty = (
|
||||
runtime as unknown as {
|
||||
ptysById: Map<string, { tabId: string | null; paneKey: string | null }>
|
||||
}
|
||||
).ptysById.get('pty-agent')!
|
||||
agentPty.tabId = null
|
||||
agentPty.paneKey = null
|
||||
const secondClient = await runtime.adoptTerminalOrphans({
|
||||
worktree: `id:${TEST_WORKTREE_ID}`,
|
||||
expectedTopologyRevision: 0,
|
||||
@@ -16454,6 +16464,22 @@ describe('OrcaRuntimeService', () => {
|
||||
}))
|
||||
})
|
||||
expect(secondClient).toMatchObject({ adopted: false, topologyRevision: 1 })
|
||||
expect(agentPty).toMatchObject({
|
||||
tabId: 'tab-agent',
|
||||
paneKey: makePaneKey('tab-agent', HEADLESS_LEAF_ID)
|
||||
})
|
||||
expect(listProcesses).toHaveBeenCalledTimes(inventoryCount + 1)
|
||||
expect(listProcesses).toHaveBeenLastCalledWith(null)
|
||||
expect(
|
||||
(await runtime.listTerminals()).terminals.find((terminal) => terminal.ptyId === 'pty-agent')
|
||||
).toMatchObject({
|
||||
handle: 'term_agent',
|
||||
orphaned: false,
|
||||
tabId: 'tab-agent',
|
||||
leafId: HEADLESS_LEAF_ID
|
||||
})
|
||||
expect(listProcesses).toHaveBeenCalledTimes(inventoryCount + 2)
|
||||
expect(listProcesses).toHaveBeenLastCalledWith(undefined)
|
||||
await expect(
|
||||
runtime.adoptTerminalOrphans({
|
||||
worktree: `id:${TEST_WORKTREE_ID}`,
|
||||
@@ -16560,6 +16586,7 @@ describe('OrcaRuntimeService', () => {
|
||||
getForegroundProcess: async () => null,
|
||||
hasRendererSerializer: () => true,
|
||||
serializeBuffer,
|
||||
hasPty: (ptyId) => ptyId === 'pty-legacy',
|
||||
listProcesses: async () => [
|
||||
{
|
||||
id: 'pty-legacy',
|
||||
@@ -16606,7 +16633,11 @@ describe('OrcaRuntimeService', () => {
|
||||
presentation: 'background',
|
||||
tabId: 'legacy-worker',
|
||||
leafId: workerLeafId,
|
||||
focus: false
|
||||
focus: false,
|
||||
expectedProcessIdentity: {
|
||||
terminalHandle: 'term_legacy',
|
||||
incarnationId
|
||||
}
|
||||
})
|
||||
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted')
|
||||
expect(write).not.toHaveBeenCalled()
|
||||
@@ -16683,6 +16714,7 @@ describe('OrcaRuntimeService', () => {
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
hasPty: (ptyId) => ptyId === 'pty-reveal-retry',
|
||||
listProcesses: async () => [
|
||||
{
|
||||
id: 'pty-reveal-retry',
|
||||
@@ -16716,6 +16748,269 @@ describe('OrcaRuntimeService', () => {
|
||||
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted')
|
||||
})
|
||||
|
||||
function makePostRevealWorkerRecoveryHarness(hasPty: (ptyId: string) => boolean | null): {
|
||||
runtime: OrcaRuntimeService
|
||||
getSession: () => WorkspaceSessionState
|
||||
workerPaneKey: string
|
||||
ptyId: string
|
||||
incarnationId: string
|
||||
terminalHandle: string
|
||||
revealTerminalSession: ReturnType<typeof vi.fn>
|
||||
resolveLegacyWorkerTerminalRecovery: ReturnType<typeof vi.fn>
|
||||
} {
|
||||
const workerPaneKey = `legacy-post-reveal:${HEADLESS_LEAF_ID}`
|
||||
const ptyId = 'pty-post-reveal'
|
||||
const incarnationId = '45454545-4545-4545-8545-454545454545'
|
||||
const terminalHandle = 'term_post_reveal'
|
||||
const session: WorkspaceSessionState = {
|
||||
...getDefaultWorkspaceSession(),
|
||||
tabsByWorktree: { [TEST_WORKTREE_ID]: [] },
|
||||
sleepingAgentSessionsByPaneKey: {
|
||||
[workerPaneKey]: {
|
||||
paneKey: workerPaneKey,
|
||||
tabId: 'legacy-post-reveal',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
agent: 'codex',
|
||||
providerSession: { key: 'session_id', id: 'legacy-post-reveal-session' },
|
||||
prompt: 'continue',
|
||||
state: 'working',
|
||||
capturedAt: 1,
|
||||
updatedAt: 1,
|
||||
origin: 'live'
|
||||
}
|
||||
}
|
||||
}
|
||||
const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session)
|
||||
const runtime = new OrcaRuntimeService(
|
||||
{ ...runtimeStore, flushOrThrow: vi.fn() } as never,
|
||||
undefined,
|
||||
{ canRecoverPersistentLocalPtys: () => true }
|
||||
)
|
||||
runtime.setOrchestrationDb({
|
||||
listLegacyWorkerTerminalRecoveryRows: () => [
|
||||
{
|
||||
dispatch_id: 'dispatch-post-reveal',
|
||||
task_id: 'task-post-reveal',
|
||||
dispatch_status: 'completed',
|
||||
contract_version: 0,
|
||||
assignee_handle: 'term_post_reveal',
|
||||
assignee_pane_key: workerPaneKey,
|
||||
process_incarnation: `${ptyId}:${incarnationId}`,
|
||||
worker_state: 'ready',
|
||||
worktree_id: TEST_WORKTREE_ID,
|
||||
agent_terminal_handle: 'term_post_reveal'
|
||||
}
|
||||
]
|
||||
} as unknown as OrchestrationDb)
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
hasPty,
|
||||
listProcesses: async () => [
|
||||
{
|
||||
id: ptyId,
|
||||
incarnationId,
|
||||
terminalHandle: 'term_post_reveal',
|
||||
title: 'Post-reveal worker',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
wslDistro: null
|
||||
}
|
||||
]
|
||||
})
|
||||
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-post-reveal' })
|
||||
const resolveLegacyWorkerTerminalRecovery = vi.fn()
|
||||
runtime.setNotifier({
|
||||
revealTerminalSession,
|
||||
resolveLegacyWorkerTerminalRecovery
|
||||
} as never)
|
||||
return {
|
||||
runtime,
|
||||
getSession,
|
||||
workerPaneKey,
|
||||
ptyId,
|
||||
incarnationId,
|
||||
terminalHandle,
|
||||
revealTerminalSession,
|
||||
resolveLegacyWorkerTerminalRecovery
|
||||
}
|
||||
}
|
||||
|
||||
it('keeps a revealed worker fenced until its exact renderer graph is published', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const harness = makePostRevealWorkerRecoveryHarness(() => true)
|
||||
const identity = {
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
tabId: 'legacy-post-reveal',
|
||||
leafId: HEADLESS_LEAF_ID,
|
||||
ptyId: harness.ptyId
|
||||
}
|
||||
harness.revealTerminalSession.mockResolvedValue({
|
||||
tabId: identity.tabId,
|
||||
identity
|
||||
})
|
||||
|
||||
await expect(
|
||||
harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true })
|
||||
).resolves.toMatchObject({
|
||||
adoptedDispatchIds: [],
|
||||
exitedDispatchIds: [],
|
||||
deferredDispatchIds: ['dispatch-post-reveal']
|
||||
})
|
||||
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
|
||||
expect(
|
||||
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
|
||||
).toBeDefined()
|
||||
expect(harness.resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled()
|
||||
|
||||
harness.runtime.attachWindow(1)
|
||||
harness.runtime.syncWindowGraph(1, {
|
||||
tabs: [
|
||||
{
|
||||
tabId: identity.tabId,
|
||||
worktreeId: identity.worktreeId,
|
||||
title: 'Post-reveal worker',
|
||||
activeLeafId: identity.leafId,
|
||||
layout: null
|
||||
}
|
||||
],
|
||||
leaves: [
|
||||
{
|
||||
tabId: identity.tabId,
|
||||
worktreeId: identity.worktreeId,
|
||||
leafId: identity.leafId,
|
||||
paneRuntimeId: 1,
|
||||
ptyId: identity.ptyId
|
||||
}
|
||||
]
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
|
||||
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
|
||||
expect(
|
||||
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
|
||||
).toBeUndefined()
|
||||
expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(
|
||||
harness.workerPaneKey,
|
||||
'adopted'
|
||||
)
|
||||
expect(
|
||||
(await harness.runtime.listTerminals()).terminals.filter(
|
||||
(terminal) => terminal.ptyId === identity.ptyId
|
||||
)
|
||||
).toEqual([
|
||||
expect.objectContaining({
|
||||
handle: harness.terminalHandle,
|
||||
incarnationId: harness.incarnationId,
|
||||
orphaned: false,
|
||||
worktreeId: identity.worktreeId,
|
||||
tabId: identity.tabId,
|
||||
leafId: identity.leafId
|
||||
})
|
||||
])
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('retires the exact worker when it exits after renderer reveal', async () => {
|
||||
const hasPty = vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(false)
|
||||
const harness = makePostRevealWorkerRecoveryHarness(hasPty)
|
||||
|
||||
await expect(
|
||||
harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true })
|
||||
).resolves.toMatchObject({
|
||||
adoptedDispatchIds: [],
|
||||
exitedDispatchIds: ['dispatch-post-reveal'],
|
||||
deferredDispatchIds: []
|
||||
})
|
||||
|
||||
expect(hasPty).toHaveBeenCalledTimes(2)
|
||||
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
|
||||
expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([])
|
||||
expect(
|
||||
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
|
||||
).toBeUndefined()
|
||||
expect(
|
||||
(
|
||||
harness.runtime as unknown as {
|
||||
ptysById: Map<string, { connected: boolean; incarnationId?: string }>
|
||||
}
|
||||
).ptysById.get(harness.ptyId)
|
||||
).toMatchObject({ connected: false, incarnationId: '45454545-4545-4545-8545-454545454545' })
|
||||
expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(
|
||||
harness.workerPaneKey,
|
||||
'exited'
|
||||
)
|
||||
})
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: 'null',
|
||||
makeHasPty: () =>
|
||||
vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(null).mockReturnValue(true)
|
||||
},
|
||||
{
|
||||
name: 'throw',
|
||||
makeHasPty: () =>
|
||||
vi
|
||||
.fn()
|
||||
.mockReturnValueOnce(true)
|
||||
.mockImplementationOnce(() => {
|
||||
throw new Error('provider unavailable')
|
||||
})
|
||||
.mockReturnValue(true)
|
||||
}
|
||||
])(
|
||||
'retries post-reveal $name liveness without duplicating the worker surface',
|
||||
async ({ makeHasPty }) => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const hasPty = makeHasPty()
|
||||
const harness = makePostRevealWorkerRecoveryHarness(hasPty)
|
||||
|
||||
await expect(
|
||||
harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true })
|
||||
).resolves.toMatchObject({
|
||||
adoptedDispatchIds: [],
|
||||
exitedDispatchIds: [],
|
||||
deferredDispatchIds: ['dispatch-post-reveal']
|
||||
})
|
||||
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
|
||||
expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([
|
||||
expect.objectContaining({ id: 'legacy-post-reveal', ptyId: harness.ptyId })
|
||||
])
|
||||
expect(
|
||||
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
|
||||
?.automaticResumeBlockedBy
|
||||
).toBe('legacy-orchestration-worker')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
|
||||
expect(hasPty).toHaveBeenCalledTimes(4)
|
||||
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
|
||||
expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([
|
||||
expect.objectContaining({ id: 'legacy-post-reveal', ptyId: harness.ptyId })
|
||||
])
|
||||
expect(
|
||||
harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey]
|
||||
).toBeUndefined()
|
||||
expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(
|
||||
harness.workerPaneKey,
|
||||
'adopted'
|
||||
)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
expect(hasPty).toHaveBeenCalledTimes(4)
|
||||
expect(harness.revealTerminalSession).toHaveBeenCalledOnce()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
it('keeps the legacy worker resume fence in memory when persistence fails', async () => {
|
||||
const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}`
|
||||
const incarnationId = '99999999-9999-4999-8999-999999999999'
|
||||
@@ -16768,6 +17063,7 @@ describe('OrcaRuntimeService', () => {
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
hasPty: (ptyId) => ptyId === 'pty-persistence-failure',
|
||||
listProcesses: async () => [
|
||||
{
|
||||
id: 'pty-persistence-failure',
|
||||
@@ -16804,8 +17100,9 @@ describe('OrcaRuntimeService', () => {
|
||||
warn.mockRestore()
|
||||
})
|
||||
|
||||
it('unblocks provider resume only after authoritative inventory proves the legacy PTY exited', async () => {
|
||||
it('retires provider resume only after authoritative inventory proves the legacy PTY exited', async () => {
|
||||
const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}`
|
||||
const secondWorkerPaneKey = `legacy-worker-two:${HEADLESS_SECOND_LEAF_ID}`
|
||||
const incarnationId = '33333333-3333-4333-8333-333333333333'
|
||||
const session: WorkspaceSessionState = {
|
||||
...getDefaultWorkspaceSession(),
|
||||
@@ -16822,6 +17119,18 @@ describe('OrcaRuntimeService', () => {
|
||||
capturedAt: 1,
|
||||
updatedAt: 1,
|
||||
origin: 'live'
|
||||
},
|
||||
[secondWorkerPaneKey]: {
|
||||
paneKey: secondWorkerPaneKey,
|
||||
tabId: 'legacy-worker-two',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
agent: 'codex',
|
||||
providerSession: { key: 'session_id', id: 'legacy-codex-session-two' },
|
||||
prompt: 'continue',
|
||||
state: 'working',
|
||||
capturedAt: 1,
|
||||
updatedAt: 1,
|
||||
origin: 'live'
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16844,17 +17153,50 @@ describe('OrcaRuntimeService', () => {
|
||||
worker_state: 'ready',
|
||||
worktree_id: TEST_WORKTREE_ID,
|
||||
agent_terminal_handle: 'term_exited'
|
||||
},
|
||||
{
|
||||
dispatch_id: 'dispatch-exited-two',
|
||||
task_id: 'task-exited-two',
|
||||
dispatch_status: 'completed',
|
||||
contract_version: 0,
|
||||
assignee_handle: 'term_exited_two',
|
||||
assignee_pane_key: secondWorkerPaneKey,
|
||||
process_incarnation: `pty-exited-two:${incarnationId}`,
|
||||
worker_state: 'ready',
|
||||
worktree_id: TEST_WORKTREE_ID,
|
||||
agent_terminal_handle: 'term_exited_two'
|
||||
}
|
||||
]
|
||||
} as unknown as OrchestrationDb)
|
||||
const listProcesses = vi.fn(async (connectionId?: string | null) => {
|
||||
if (connectionId !== null) {
|
||||
throw new Error('unrelated SSH inventory must not run')
|
||||
}
|
||||
return [
|
||||
{
|
||||
id: 'pty-exited-two',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_exited_two',
|
||||
title: 'Exited worker',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
wslDistro: null
|
||||
}
|
||||
]
|
||||
})
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
listProcesses: async () => []
|
||||
hasPty: () => false,
|
||||
listProcesses
|
||||
})
|
||||
const revealTerminalSession = vi.fn()
|
||||
const resolveLegacyWorkerTerminalRecovery = vi.fn()
|
||||
runtime.setNotifier({ resolveLegacyWorkerTerminalRecovery } as never)
|
||||
runtime.setNotifier({
|
||||
revealTerminalSession,
|
||||
resolveLegacyWorkerTerminalRecovery
|
||||
} as never)
|
||||
|
||||
runtime.prepareLegacyWorkerTerminalRecovery()
|
||||
expect(
|
||||
@@ -16863,16 +17205,27 @@ describe('OrcaRuntimeService', () => {
|
||||
|
||||
await expect(runtime.reconcileLegacyWorkerTerminals()).resolves.toMatchObject({
|
||||
adoptedDispatchIds: [],
|
||||
exitedDispatchIds: ['dispatch-exited'],
|
||||
exitedDispatchIds: ['dispatch-exited', 'dispatch-exited-two'],
|
||||
deferredDispatchIds: []
|
||||
})
|
||||
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
|
||||
expect(getSession().sleepingAgentSessionsByPaneKey?.[secondWorkerPaneKey]).toBeUndefined()
|
||||
expect(listProcesses).toHaveBeenCalledOnce()
|
||||
expect(listProcesses).toHaveBeenCalledWith(null)
|
||||
expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([])
|
||||
expect(revealTerminalSession).not.toHaveBeenCalled()
|
||||
expect(
|
||||
getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy
|
||||
).toBeUndefined()
|
||||
(
|
||||
runtime as unknown as {
|
||||
ptysById: Map<string, { connected: boolean }>
|
||||
}
|
||||
).ptysById.get('pty-exited-two')
|
||||
).toMatchObject({ connected: false })
|
||||
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'exited')
|
||||
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(secondWorkerPaneKey, 'exited')
|
||||
})
|
||||
|
||||
it('keeps provider resume fenced when authoritative PTY inventory is unavailable', async () => {
|
||||
it('retries inventory and unknown liveness without revealing a ghost worker', async () => {
|
||||
const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}`
|
||||
const incarnationId = '55555555-5555-4555-8555-555555555555'
|
||||
const session: WorkspaceSessionState = {
|
||||
@@ -16915,23 +17268,263 @@ describe('OrcaRuntimeService', () => {
|
||||
}
|
||||
]
|
||||
} as unknown as OrchestrationDb)
|
||||
const listProcesses = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('local provider unavailable'))
|
||||
.mockResolvedValue([
|
||||
{
|
||||
id: 'pty-inventory-unavailable',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_inventory_unavailable',
|
||||
title: 'Recovered legacy worker',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
wslDistro: null
|
||||
}
|
||||
])
|
||||
const hasPty = vi.fn().mockReturnValueOnce(null).mockReturnValue(true)
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null
|
||||
getForegroundProcess: async () => null,
|
||||
hasPty,
|
||||
listProcesses
|
||||
})
|
||||
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-worker' })
|
||||
const resolveLegacyWorkerTerminalRecovery = vi.fn()
|
||||
runtime.setNotifier({ resolveLegacyWorkerTerminalRecovery } as never)
|
||||
runtime.setNotifier({
|
||||
revealTerminalSession,
|
||||
resolveLegacyWorkerTerminalRecovery
|
||||
} as never)
|
||||
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
await expect(
|
||||
runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true })
|
||||
).resolves.toMatchObject({
|
||||
adoptedDispatchIds: [],
|
||||
exitedDispatchIds: [],
|
||||
deferredDispatchIds: ['dispatch-inventory-unavailable']
|
||||
})
|
||||
expect(
|
||||
getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy
|
||||
).toBe('legacy-orchestration-worker')
|
||||
expect(resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled()
|
||||
expect(listProcesses).toHaveBeenCalledOnce()
|
||||
expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([])
|
||||
expect(revealTerminalSession).not.toHaveBeenCalled()
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
|
||||
expect(listProcesses).toHaveBeenCalledTimes(2)
|
||||
expect(listProcesses.mock.calls).toEqual([[null], [null]])
|
||||
expect(hasPty).toHaveBeenCalledOnce()
|
||||
expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([])
|
||||
expect(revealTerminalSession).not.toHaveBeenCalled()
|
||||
expect(
|
||||
getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy
|
||||
).toBe('legacy-orchestration-worker')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(2_000)
|
||||
|
||||
expect(listProcesses).toHaveBeenCalledTimes(3)
|
||||
expect(hasPty).toHaveBeenCalledTimes(3)
|
||||
expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([
|
||||
expect.objectContaining({ id: 'legacy-worker', ptyId: 'pty-inventory-unavailable' })
|
||||
])
|
||||
expect(revealTerminalSession).toHaveBeenCalledOnce()
|
||||
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
|
||||
expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted')
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
expect(listProcesses).toHaveBeenCalledTimes(3)
|
||||
expect(revealTerminalSession).toHaveBeenCalledOnce()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('cancels a coalesced SSH worker recovery retry when its provider disconnects', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const runtime = new OrcaRuntimeService(store)
|
||||
const reconcile = vi.spyOn(runtime, 'reconcileLegacyWorkerTerminals').mockResolvedValue({
|
||||
blockedPaneCount: 1,
|
||||
adoptedDispatchIds: [],
|
||||
exitedDispatchIds: [],
|
||||
deferredDispatchIds: []
|
||||
})
|
||||
const retryInternals = runtime as unknown as {
|
||||
updateLegacyWorkerTerminalRecoveryRetry: (
|
||||
plan: {
|
||||
candidates: { dispatchId: string; ptyId: string }[]
|
||||
},
|
||||
deferredDispatchIds: ReadonlySet<string>,
|
||||
options: { connectionId?: string; materializeRenderer?: boolean }
|
||||
) => void
|
||||
}
|
||||
const plan = {
|
||||
candidates: [
|
||||
{
|
||||
dispatchId: 'dispatch-ssh-retry',
|
||||
ptyId: 'ssh:ssh-retry@@pty-worker'
|
||||
}
|
||||
]
|
||||
}
|
||||
const deferred = new Set(['dispatch-ssh-retry'])
|
||||
|
||||
retryInternals.updateLegacyWorkerTerminalRecoveryRetry(plan, deferred, {
|
||||
connectionId: 'ssh-retry',
|
||||
materializeRenderer: true
|
||||
})
|
||||
retryInternals.updateLegacyWorkerTerminalRecoveryRetry(plan, deferred, {
|
||||
connectionId: 'ssh-retry',
|
||||
materializeRenderer: true
|
||||
})
|
||||
runtime.notifySshStateChanged('ssh-retry', {
|
||||
targetId: 'ssh-retry',
|
||||
status: 'disconnected',
|
||||
error: null,
|
||||
reconnectAttempt: 0
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
|
||||
expect(reconcile).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps live workers fenced without exact controller identity evidence', async () => {
|
||||
const incarnationId = '56565656-5656-4656-8656-565656565656'
|
||||
const cases = [
|
||||
{
|
||||
name: 'missing',
|
||||
leafId: '11111111-1111-4111-8111-111111111111',
|
||||
terminalHandle: 'term_missing'
|
||||
},
|
||||
{
|
||||
name: 'ambiguous',
|
||||
leafId: '22222222-2222-4222-8222-222222222222',
|
||||
terminalHandle: 'term_ambiguous'
|
||||
},
|
||||
{
|
||||
name: 'wrong-handle',
|
||||
leafId: '33333333-3333-4333-8333-333333333333',
|
||||
terminalHandle: 'term_wrong_handle'
|
||||
},
|
||||
{
|
||||
name: 'wrong-incarnation',
|
||||
leafId: '44444444-4444-4444-8444-444444444444',
|
||||
terminalHandle: 'term_wrong_incarnation'
|
||||
}
|
||||
] as const
|
||||
const sleepingAgentSessionsByPaneKey = Object.fromEntries(
|
||||
cases.map(({ name, leafId }) => {
|
||||
const paneKey = `legacy-${name}:${leafId}`
|
||||
return [
|
||||
paneKey,
|
||||
{
|
||||
paneKey,
|
||||
tabId: `legacy-${name}`,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
agent: 'codex',
|
||||
providerSession: { key: 'session_id', id: `session-${name}` },
|
||||
prompt: 'continue',
|
||||
state: 'working',
|
||||
capturedAt: 1,
|
||||
updatedAt: 1,
|
||||
origin: 'live'
|
||||
}
|
||||
]
|
||||
})
|
||||
) as WorkspaceSessionState['sleepingAgentSessionsByPaneKey']
|
||||
const session: WorkspaceSessionState = {
|
||||
...getDefaultWorkspaceSession(),
|
||||
tabsByWorktree: { [TEST_WORKTREE_ID]: [] },
|
||||
sleepingAgentSessionsByPaneKey
|
||||
}
|
||||
const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session)
|
||||
const runtime = new OrcaRuntimeService(
|
||||
{ ...runtimeStore, flushOrThrow: vi.fn() } as never,
|
||||
undefined,
|
||||
{ canRecoverPersistentLocalPtys: () => true }
|
||||
)
|
||||
runtime.setOrchestrationDb({
|
||||
listLegacyWorkerTerminalRecoveryRows: () =>
|
||||
cases.map(({ name, leafId, terminalHandle }) => ({
|
||||
dispatch_id: `dispatch-${name}`,
|
||||
task_id: `task-${name}`,
|
||||
dispatch_status: 'completed',
|
||||
contract_version: 0,
|
||||
assignee_handle: terminalHandle,
|
||||
assignee_pane_key: `legacy-${name}:${leafId}`,
|
||||
process_incarnation: `pty-${name}:${incarnationId}`,
|
||||
worker_state: 'ready',
|
||||
worktree_id: TEST_WORKTREE_ID,
|
||||
agent_terminal_handle: terminalHandle
|
||||
}))
|
||||
} as unknown as OrchestrationDb)
|
||||
const listProcesses = vi.fn(async () => [
|
||||
{
|
||||
id: 'pty-missing',
|
||||
title: 'Missing identity',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID
|
||||
},
|
||||
{
|
||||
id: 'pty-ambiguous',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_ambiguous',
|
||||
title: 'Ambiguous identity',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID
|
||||
},
|
||||
{
|
||||
id: 'pty-ambiguous-other',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_ambiguous',
|
||||
title: 'Ambiguous identity duplicate',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID
|
||||
},
|
||||
{
|
||||
id: 'pty-wrong-handle',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_other',
|
||||
title: 'Wrong handle',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID
|
||||
},
|
||||
{
|
||||
id: 'pty-wrong-incarnation',
|
||||
incarnationId: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
|
||||
terminalHandle: 'term_wrong_incarnation',
|
||||
title: 'Wrong incarnation',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID
|
||||
}
|
||||
])
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
hasPty: (candidate) => candidate === 'pty-folder-legacy',
|
||||
listProcesses
|
||||
})
|
||||
|
||||
await expect(runtime.reconcileLegacyWorkerTerminals()).resolves.toMatchObject({
|
||||
adoptedDispatchIds: [],
|
||||
exitedDispatchIds: [],
|
||||
deferredDispatchIds: ['dispatch-inventory-unavailable']
|
||||
deferredDispatchIds: cases.map(({ name }) => `dispatch-${name}`)
|
||||
})
|
||||
expect(
|
||||
getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy
|
||||
).toBe('legacy-orchestration-worker')
|
||||
expect(resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled()
|
||||
expect(listProcesses).toHaveBeenCalledOnce()
|
||||
expect(listProcesses).toHaveBeenCalledWith(null)
|
||||
for (const { name, leafId } of cases) {
|
||||
expect(
|
||||
getSession().sleepingAgentSessionsByPaneKey?.[`legacy-${name}:${leafId}`]
|
||||
?.automaticResumeBlockedBy
|
||||
).toBe('legacy-orchestration-worker')
|
||||
}
|
||||
})
|
||||
|
||||
it('adopts an exact live legacy worker in a folder workspace', async () => {
|
||||
@@ -16986,21 +17579,23 @@ describe('OrcaRuntimeService', () => {
|
||||
}
|
||||
]
|
||||
} as unknown as OrchestrationDb)
|
||||
const listProcesses = vi.fn(async () => [
|
||||
{
|
||||
id: 'pty-folder-legacy',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_folder',
|
||||
title: 'Folder worker',
|
||||
cwd: folderPath,
|
||||
worktreeId: TEST_FOLDER_WORKSPACE_KEY,
|
||||
wslDistro: null
|
||||
}
|
||||
])
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
listProcesses: async () => [
|
||||
{
|
||||
id: 'pty-folder-legacy',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_folder',
|
||||
title: 'Folder worker',
|
||||
cwd: folderPath,
|
||||
worktreeId: TEST_FOLDER_WORKSPACE_KEY,
|
||||
wslDistro: null
|
||||
}
|
||||
]
|
||||
hasPty: (candidate) => candidate === 'pty-folder-legacy',
|
||||
listProcesses
|
||||
})
|
||||
const revealTerminalSession = vi.fn().mockResolvedValue({
|
||||
tabId: 'legacy-folder-worker'
|
||||
@@ -17022,6 +17617,8 @@ describe('OrcaRuntimeService', () => {
|
||||
})
|
||||
)
|
||||
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
|
||||
expect(listProcesses).toHaveBeenCalledOnce()
|
||||
expect(listProcesses).toHaveBeenCalledWith(null)
|
||||
expect(revealTerminalSession).toHaveBeenCalledWith(TEST_FOLDER_WORKSPACE_KEY, {
|
||||
ptyId: 'pty-folder-legacy',
|
||||
title: 'Folder worker',
|
||||
@@ -17029,7 +17626,11 @@ describe('OrcaRuntimeService', () => {
|
||||
presentation: 'background',
|
||||
tabId: 'legacy-folder-worker',
|
||||
leafId: HEADLESS_LEAF_ID,
|
||||
focus: false
|
||||
focus: false,
|
||||
expectedProcessIdentity: {
|
||||
terminalHandle: 'term_folder',
|
||||
incarnationId
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -17101,21 +17702,23 @@ describe('OrcaRuntimeService', () => {
|
||||
}
|
||||
]
|
||||
} as unknown as OrchestrationDb)
|
||||
const listProcesses = vi.fn(async () => [
|
||||
{
|
||||
id: ptyId,
|
||||
incarnationId,
|
||||
terminalHandle: 'term_ssh_folder',
|
||||
title: 'SSH folder worker',
|
||||
cwd: folderPath,
|
||||
worktreeId: TEST_FOLDER_WORKSPACE_KEY,
|
||||
wslDistro: null
|
||||
}
|
||||
])
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
listProcesses: async () => [
|
||||
{
|
||||
id: ptyId,
|
||||
incarnationId,
|
||||
terminalHandle: 'term_ssh_folder',
|
||||
title: 'SSH folder worker',
|
||||
cwd: folderPath,
|
||||
worktreeId: TEST_FOLDER_WORKSPACE_KEY,
|
||||
wslDistro: null
|
||||
}
|
||||
]
|
||||
hasPty: (candidate) => candidate === ptyId,
|
||||
listProcesses
|
||||
})
|
||||
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-ssh-folder-worker' })
|
||||
runtime.setNotifier({ revealTerminalSession } as never)
|
||||
@@ -17149,6 +17752,8 @@ describe('OrcaRuntimeService', () => {
|
||||
expect(localSession.sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
|
||||
expect(getWorkspaceSession).toHaveBeenCalledWith(`ssh:${connectionId}`)
|
||||
expect(setWorkspaceSession).toHaveBeenCalledWith(expect.any(Object), `ssh:${connectionId}`)
|
||||
expect(listProcesses).toHaveBeenCalledOnce()
|
||||
expect(listProcesses).toHaveBeenCalledWith(connectionId)
|
||||
expect(sshSession.tabsByWorktree[TEST_FOLDER_WORKSPACE_KEY]).toContainEqual(
|
||||
expect.objectContaining({
|
||||
id: 'legacy-ssh-folder-worker',
|
||||
@@ -17163,7 +17768,11 @@ describe('OrcaRuntimeService', () => {
|
||||
presentation: 'background',
|
||||
tabId: 'legacy-ssh-folder-worker',
|
||||
leafId: HEADLESS_LEAF_ID,
|
||||
focus: false
|
||||
focus: false,
|
||||
expectedProcessIdentity: {
|
||||
terminalHandle: 'term_ssh_folder',
|
||||
incarnationId
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -17321,6 +17930,7 @@ describe('OrcaRuntimeService', () => {
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
hasPty: (candidate) => candidate === ptyId,
|
||||
listProcesses
|
||||
})
|
||||
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-ssh-worker' })
|
||||
@@ -17365,12 +17975,29 @@ describe('OrcaRuntimeService', () => {
|
||||
exitedDispatchIds: [],
|
||||
deferredDispatchIds: []
|
||||
})
|
||||
await expect(
|
||||
runtime.adoptTerminalOrphans({
|
||||
worktree: `id:${TEST_WORKTREE_ID}`,
|
||||
expectedTopologyRevision: 0,
|
||||
claims: [
|
||||
{
|
||||
terminal: 'term_ssh_legacy',
|
||||
ptyId,
|
||||
incarnationId,
|
||||
tabId: 'legacy-ssh-worker',
|
||||
leafId: HEADLESS_LEAF_ID
|
||||
}
|
||||
]
|
||||
})
|
||||
).resolves.toMatchObject({ adopted: false })
|
||||
expect(listProcesses).toHaveBeenLastCalledWith(connectionId)
|
||||
} finally {
|
||||
unregisterSshGitProvider(connectionId)
|
||||
}
|
||||
|
||||
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
|
||||
expect(setWorkspaceSession).toHaveBeenCalledWith(expect.any(Object), `ssh:${connectionId}`)
|
||||
expect(listProcesses).toHaveBeenCalledTimes(2)
|
||||
expect(revealTerminalSession).toHaveBeenCalledWith(TEST_WORKTREE_ID, {
|
||||
ptyId,
|
||||
title: 'SSH legacy worker',
|
||||
@@ -17378,7 +18005,11 @@ describe('OrcaRuntimeService', () => {
|
||||
presentation: 'background',
|
||||
tabId: 'legacy-ssh-worker',
|
||||
leafId: HEADLESS_LEAF_ID,
|
||||
focus: false
|
||||
focus: false,
|
||||
expectedProcessIdentity: {
|
||||
terminalHandle: 'term_ssh_legacy',
|
||||
incarnationId
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -17445,21 +18076,23 @@ describe('OrcaRuntimeService', () => {
|
||||
}
|
||||
]
|
||||
} as unknown as OrchestrationDb)
|
||||
const listProcesses = vi.fn(async () => [
|
||||
{
|
||||
id: 'pty-wsl-legacy',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_wsl_legacy',
|
||||
title: 'WSL legacy worker',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
wslDistro: observedDistro
|
||||
}
|
||||
])
|
||||
runtime.setPtyController({
|
||||
write: vi.fn(() => true),
|
||||
kill: vi.fn(() => true),
|
||||
getForegroundProcess: async () => null,
|
||||
listProcesses: async () => [
|
||||
{
|
||||
id: 'pty-wsl-legacy',
|
||||
incarnationId,
|
||||
terminalHandle: 'term_wsl_legacy',
|
||||
title: 'WSL legacy worker',
|
||||
cwd: TEST_WORKTREE_PATH,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
wslDistro: observedDistro
|
||||
}
|
||||
]
|
||||
hasPty: (ptyId) => ptyId === 'pty-wsl-legacy',
|
||||
listProcesses
|
||||
})
|
||||
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-wsl-worker' })
|
||||
runtime.setNotifier({ revealTerminalSession } as never)
|
||||
@@ -17486,6 +18119,8 @@ describe('OrcaRuntimeService', () => {
|
||||
})
|
||||
expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined()
|
||||
expect(revealTerminalSession).toHaveBeenCalledOnce()
|
||||
expect(listProcesses).toHaveBeenCalledTimes(2)
|
||||
expect(listProcesses.mock.calls).toEqual([[null], [null]])
|
||||
})
|
||||
|
||||
it('restores orphan pane and group topology without replacing a newer host-owned tab', async () => {
|
||||
|
||||
+456
-176
@@ -36,10 +36,7 @@ import {
|
||||
type AgentStatusEntry
|
||||
} from '../../shared/agent-status-types'
|
||||
import { indexAgentStatusRowsByPaneKey } from '../agent-hooks/agent-status-pane-index'
|
||||
import type {
|
||||
AgentHookAuthorityAttestation,
|
||||
AgentHookAuthorityEvidence
|
||||
} from '../agent-hooks/server'
|
||||
import type { AgentHookAuthorityAttestation } from '../agent-hooks/server'
|
||||
import type {
|
||||
AgentSessionClaimedSpawnResult,
|
||||
AgentSessionExecutionClaim,
|
||||
@@ -110,6 +107,7 @@ import {
|
||||
createSetupCompletionScanner
|
||||
} from './orchestration/setup-completion-signal'
|
||||
import type { RuntimeOrchestrationEnvelope } from '../../shared/runtime-rpc-envelope'
|
||||
import type { TerminalRevealIdentity } from '../../shared/terminal-reveal-identity'
|
||||
import type {
|
||||
OrchestrationCompatibilityEvidence,
|
||||
OrchestrationCompatibilityHostStamp
|
||||
@@ -1506,7 +1504,7 @@ type RuntimePtyController = {
|
||||
resize?(ptyId: string, cols: number, rows: number): boolean
|
||||
// Why: exact-id mobile polls should not enumerate every local and SSH PTY.
|
||||
hasPty?(ptyId: string): boolean | null
|
||||
listProcesses?(): Promise<PtyProcessInfo[]>
|
||||
listProcesses?(connectionId?: string | null): Promise<PtyProcessInfo[]>
|
||||
serializeBuffer?(
|
||||
ptyId: string,
|
||||
opts?: { scrollbackRows?: number; altScreenForcesZeroRows?: boolean }
|
||||
@@ -1530,6 +1528,17 @@ type RuntimePtyController = {
|
||||
getSize?(ptyId: string): { cols: number; rows: number } | null
|
||||
}
|
||||
|
||||
type PtyControllerTerminalIdentity = Readonly<{
|
||||
handle: string
|
||||
incarnationId: string
|
||||
wslDistro?: string | null
|
||||
}>
|
||||
|
||||
type PtyControllerInventory = Readonly<{
|
||||
livePtyIds: ReadonlySet<string>
|
||||
terminalIdentityByPtyId: ReadonlyMap<string, PtyControllerTerminalIdentity>
|
||||
}>
|
||||
|
||||
type WorktreeStartupDraftPaste = {
|
||||
agent: TuiAgent
|
||||
content: string
|
||||
@@ -1645,10 +1654,14 @@ type RuntimeNotifier = {
|
||||
splitDirection?: 'horizontal' | 'vertical'
|
||||
splitTelemetrySource?: TerminalPaneSplitSource
|
||||
focus?: boolean
|
||||
expectedProcessIdentity?: {
|
||||
terminalHandle: string
|
||||
incarnationId: string
|
||||
}
|
||||
}
|
||||
):
|
||||
| Promise<{ tabId: string; title?: string | null }>
|
||||
| { tabId: string; title?: string | null }
|
||||
| Promise<{ tabId: string; title?: string | null; identity?: TerminalRevealIdentity }>
|
||||
| { tabId: string; title?: string | null; identity?: TerminalRevealIdentity }
|
||||
| void
|
||||
resolveLegacyWorkerTerminalRecovery?(paneKey: string, resolution: 'adopted' | 'exited'): void
|
||||
splitTerminal(
|
||||
@@ -1733,11 +1746,6 @@ export type OrchestrationCompatibilityTerminalAuthority = {
|
||||
| { kind: 'ssh'; targetId: string }
|
||||
}
|
||||
|
||||
export type OrchestrationCompatibilityAuthorityInputs = {
|
||||
hydratedCommitments: readonly AgentHookAuthorityEvidence[]
|
||||
currentObservations: readonly AgentHookAuthorityEvidence[]
|
||||
}
|
||||
|
||||
export type LegacyWorkerTerminalRecoveryResult = {
|
||||
blockedPaneCount: number
|
||||
adoptedDispatchIds: string[]
|
||||
@@ -2615,10 +2623,6 @@ export class OrcaRuntimeService {
|
||||
private handles = new Map<string, TerminalHandleRecord>()
|
||||
private handleByLeafKey = new Map<string, string>()
|
||||
private handleByPtyId = new Map<string, string>()
|
||||
private controllerTerminalIdentityByPtyId = new Map<
|
||||
string,
|
||||
{ handle: string; incarnationId: string; wslDistro?: string | null }
|
||||
>()
|
||||
private detachedPreAllocatedLeaves = new Map<string, RuntimeLeafRecord>()
|
||||
private graphSyncCallbacks: (() => void)[] = []
|
||||
private waitersByHandle = new Map<string, Set<TerminalWaiter>>()
|
||||
@@ -3011,12 +3015,6 @@ export class OrcaRuntimeService {
|
||||
private readonly getAgentProviderSessionRowsForPaneFn:
|
||||
| ((paneKey: string) => AgentStatusIpcPayload[])
|
||||
| null
|
||||
private readonly getHydratedAgentHookAuthorityFn:
|
||||
| (() => readonly AgentHookAuthorityEvidence[])
|
||||
| null
|
||||
private readonly getCurrentAgentHookAuthorityFn:
|
||||
| (() => readonly AgentHookAuthorityEvidence[])
|
||||
| null
|
||||
private readonly attestAgentHookCompatibilityAuthorityFn:
|
||||
| ((candidate: {
|
||||
paneKey: string
|
||||
@@ -3038,7 +3036,17 @@ export class OrcaRuntimeService {
|
||||
>()
|
||||
private sshRelayRecoveryGenerationByTargetId = new Map<string, number>()
|
||||
private legacyWorkerTerminalRecoveryQueue: Promise<void> = Promise.resolve()
|
||||
private legacyWorkerTerminalRecoveryRetries = new Map<
|
||||
string,
|
||||
{
|
||||
attempt: number
|
||||
connectionId?: string
|
||||
materializeRenderer: boolean
|
||||
timer: ReturnType<typeof setTimeout> | null
|
||||
}
|
||||
>()
|
||||
private legacyWorkerTerminalMaterializedPanes = new Set<string>()
|
||||
private legacyWorkerTerminalIdentityReceiptPanes = new Set<string>()
|
||||
private legacyWorkerRecoveredPtys = new Set<string>()
|
||||
private accountServices: RuntimeAccountServices | null = null
|
||||
private commitMessageAgentEnv: CommitMessageAgentEnvironmentResolvers | null = null
|
||||
@@ -3074,8 +3082,6 @@ export class OrcaRuntimeService {
|
||||
* only carrier of the provider session a transcript is addressed by. */
|
||||
getAgentProviderSessionSnapshot?: () => AgentStatusIpcPayload[]
|
||||
getAgentProviderSessionRowsForPane?: (paneKey: string) => AgentStatusIpcPayload[]
|
||||
getHydratedAgentHookAuthority?: () => readonly AgentHookAuthorityEvidence[]
|
||||
getCurrentAgentHookAuthority?: () => readonly AgentHookAuthorityEvidence[]
|
||||
attestAgentHookCompatibilityAuthority?: (candidate: {
|
||||
paneKey: string
|
||||
launchTokenHash: string
|
||||
@@ -3114,8 +3120,6 @@ export class OrcaRuntimeService {
|
||||
this.getAgentProviderSessionSnapshotFn =
|
||||
deps?.getAgentProviderSessionSnapshot ?? deps?.getAgentStatusSnapshot ?? null
|
||||
this.getAgentProviderSessionRowsForPaneFn = deps?.getAgentProviderSessionRowsForPane ?? null
|
||||
this.getHydratedAgentHookAuthorityFn = deps?.getHydratedAgentHookAuthority ?? null
|
||||
this.getCurrentAgentHookAuthorityFn = deps?.getCurrentAgentHookAuthority ?? null
|
||||
this.attestAgentHookCompatibilityAuthorityFn =
|
||||
deps?.attestAgentHookCompatibilityAuthority ?? null
|
||||
this.canRecoverPersistentLocalPtysFn = deps?.canRecoverPersistentLocalPtys ?? (() => true)
|
||||
@@ -3659,6 +3663,70 @@ export class OrcaRuntimeService {
|
||||
return result
|
||||
}
|
||||
|
||||
private hasExactTerminalSurfaceIdentity(expected: {
|
||||
worktreeId: string
|
||||
tabId: string
|
||||
leafId: string
|
||||
ptyId: string
|
||||
terminalHandle: string
|
||||
incarnationId: string
|
||||
}): boolean {
|
||||
if (this.graphStatus !== 'ready') {
|
||||
return false
|
||||
}
|
||||
const pty = this.ptysById.get(expected.ptyId)
|
||||
if (
|
||||
!pty?.connected ||
|
||||
pty.incarnationId !== expected.incarnationId ||
|
||||
pty.tabId !== expected.tabId ||
|
||||
pty.paneKey !== makePaneKey(expected.tabId, expected.leafId) ||
|
||||
!runtimeWorktreeIdsEqual(pty.worktreeId, expected.worktreeId) ||
|
||||
this.handleByPtyId.get(expected.ptyId) !== expected.terminalHandle
|
||||
) {
|
||||
return false
|
||||
}
|
||||
const tab = this.tabs.get(expected.tabId)
|
||||
const leaf = this.leaves.get(this.getLeafKey(expected.tabId, expected.leafId))
|
||||
const ptyLeaves = this.getLeavesForPty(expected.ptyId)
|
||||
return (
|
||||
Boolean(tab && runtimeWorktreeIdsEqual(tab.worktreeId, expected.worktreeId)) &&
|
||||
Boolean(
|
||||
leaf &&
|
||||
leaf.ptyId === expected.ptyId &&
|
||||
runtimeWorktreeIdsEqual(leaf.worktreeId, expected.worktreeId)
|
||||
) &&
|
||||
ptyLeaves.length === 1 &&
|
||||
ptyLeaves[0]?.tabId === expected.tabId &&
|
||||
ptyLeaves[0]?.leafId === expected.leafId
|
||||
)
|
||||
}
|
||||
|
||||
private hasExactPersistedTerminalSurfaceIdentity(expected: {
|
||||
worktreeId: string
|
||||
tabId: string
|
||||
leafId: string
|
||||
ptyId: string
|
||||
incarnationId: string
|
||||
}): boolean {
|
||||
const session = this.getWorkspaceSessionForWorktree(expected.worktreeId)
|
||||
const sessionWorktreeId = session
|
||||
? resolveTerminalSessionWorktreeId(session, expected.worktreeId)
|
||||
: null
|
||||
if (!session || !sessionWorktreeId) {
|
||||
return false
|
||||
}
|
||||
const tab = session.tabsByWorktree[sessionWorktreeId]?.find(
|
||||
(candidate) => candidate.id === expected.tabId
|
||||
)
|
||||
const paneKey = makePaneKey(expected.tabId, expected.leafId)
|
||||
return Boolean(
|
||||
tab &&
|
||||
session.terminalLayoutsByTabId[expected.tabId]?.ptyIdsByLeafId?.[expected.leafId] ===
|
||||
expected.ptyId &&
|
||||
session.terminalPtyIncarnationsByPaneKey?.[paneKey] === expected.incarnationId
|
||||
)
|
||||
}
|
||||
|
||||
private persistLegacyWorkerTerminalRecoveryResolution(
|
||||
candidate: LegacyWorkerTerminalRecoveryPlan['candidates'][number],
|
||||
resolution: 'adopted' | 'exited'
|
||||
@@ -3672,23 +3740,8 @@ export class OrcaRuntimeService {
|
||||
if (!record || !runtimeWorktreeIdsEqual(record.worktreeId, candidate.worktreeId)) {
|
||||
return true
|
||||
}
|
||||
if (
|
||||
resolution === 'exited' &&
|
||||
record.automaticResumeBlockedBy !== 'legacy-orchestration-worker'
|
||||
) {
|
||||
return true
|
||||
}
|
||||
const next = structuredClone(session)
|
||||
if (resolution === 'adopted') {
|
||||
delete next.sleepingAgentSessionsByPaneKey?.[candidate.paneKey]
|
||||
} else {
|
||||
const unblocked = { ...record }
|
||||
delete unblocked.automaticResumeBlockedBy
|
||||
next.sleepingAgentSessionsByPaneKey = {
|
||||
...next.sleepingAgentSessionsByPaneKey,
|
||||
[candidate.paneKey]: unblocked
|
||||
}
|
||||
}
|
||||
delete next.sleepingAgentSessionsByPaneKey?.[candidate.paneKey]
|
||||
try {
|
||||
this.setWorkspaceSessionForWorktree(candidate.worktreeId, next)
|
||||
store.flushOrThrow()
|
||||
@@ -3704,6 +3757,85 @@ export class OrcaRuntimeService {
|
||||
}
|
||||
}
|
||||
|
||||
private updateLegacyWorkerTerminalRecoveryRetry(
|
||||
plan: LegacyWorkerTerminalRecoveryPlan,
|
||||
deferredDispatchIds: ReadonlySet<string>,
|
||||
options: { connectionId?: string; materializeRenderer?: boolean }
|
||||
): void {
|
||||
const scopeKey = options.connectionId ? `ssh:${options.connectionId}` : 'local'
|
||||
const hasDeferredWorker = plan.candidates.some((candidate) => {
|
||||
const sshPty = parseAppSshPtyId(candidate.ptyId)
|
||||
const inScope = options.connectionId
|
||||
? sshPty?.connectionId === options.connectionId
|
||||
: sshPty === null
|
||||
return inScope && deferredDispatchIds.has(candidate.dispatchId)
|
||||
})
|
||||
if (!hasDeferredWorker) {
|
||||
this.cancelLegacyWorkerTerminalRecoveryRetry(scopeKey)
|
||||
return
|
||||
}
|
||||
const existing = this.legacyWorkerTerminalRecoveryRetries.get(scopeKey)
|
||||
const retry = existing ?? {
|
||||
attempt: 0,
|
||||
...(options.connectionId ? { connectionId: options.connectionId } : {}),
|
||||
materializeRenderer: options.materializeRenderer === true,
|
||||
timer: null
|
||||
}
|
||||
retry.materializeRenderer ||= options.materializeRenderer === true
|
||||
this.legacyWorkerTerminalRecoveryRetries.set(scopeKey, retry)
|
||||
this.armLegacyWorkerTerminalRecoveryRetry(scopeKey, retry)
|
||||
}
|
||||
|
||||
private cancelLegacyWorkerTerminalRecoveryRetry(scopeKey: string): void {
|
||||
const retry = this.legacyWorkerTerminalRecoveryRetries.get(scopeKey)
|
||||
if (retry?.timer) {
|
||||
clearTimeout(retry.timer)
|
||||
}
|
||||
this.legacyWorkerTerminalRecoveryRetries.delete(scopeKey)
|
||||
}
|
||||
|
||||
private getLegacyWorkerControllerPtyLiveness(
|
||||
candidate: LegacyWorkerTerminalRecoveryPlan['candidates'][number]
|
||||
): boolean | null {
|
||||
try {
|
||||
return this.ptyController?.hasPty?.(candidate.ptyId) ?? null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
private armLegacyWorkerTerminalRecoveryRetry(
|
||||
scopeKey: string,
|
||||
retry: {
|
||||
attempt: number
|
||||
connectionId?: string
|
||||
materializeRenderer: boolean
|
||||
timer: ReturnType<typeof setTimeout> | null
|
||||
}
|
||||
): void {
|
||||
if (retry.timer) {
|
||||
return
|
||||
}
|
||||
const delayMs = Math.min(1_000 * 2 ** retry.attempt, 30_000)
|
||||
retry.attempt += 1
|
||||
retry.timer = setTimeout(() => {
|
||||
retry.timer = null
|
||||
void this.reconcileLegacyWorkerTerminals({
|
||||
...(retry.connectionId ? { connectionId: retry.connectionId } : {}),
|
||||
materializeRenderer: retry.materializeRenderer
|
||||
}).catch((error) => {
|
||||
console.warn('[orchestration] worker terminal recovery retry failed', {
|
||||
scope: scopeKey,
|
||||
error
|
||||
})
|
||||
if (this.legacyWorkerTerminalRecoveryRetries.get(scopeKey) === retry) {
|
||||
this.armLegacyWorkerTerminalRecoveryRetry(scopeKey, retry)
|
||||
}
|
||||
})
|
||||
}, delayMs)
|
||||
retry.timer.unref?.()
|
||||
}
|
||||
|
||||
private async reconcileLegacyWorkerTerminalsNow(options: {
|
||||
connectionId?: string
|
||||
materializeRenderer?: boolean
|
||||
@@ -3712,142 +3844,251 @@ export class OrcaRuntimeService {
|
||||
const adoptedDispatchIds: string[] = []
|
||||
const exitedDispatchIds: string[] = []
|
||||
const deferredDispatchIds = new Set(plan.ambiguousDispatchIds)
|
||||
for (const candidate of plan.candidates) {
|
||||
let workspace: TerminalWorkspaceLaunchScope
|
||||
try {
|
||||
workspace = await this.resolveTerminalWorkspaceLaunchScope(`id:${candidate.worktreeId}`)
|
||||
} catch {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
const recoveryCandidatesByProvider = new Map<
|
||||
string,
|
||||
{
|
||||
connectionId: string | null
|
||||
entries: {
|
||||
candidate: (typeof plan.candidates)[number]
|
||||
workspace: TerminalWorkspaceLaunchScope
|
||||
resolvedWorkspace: ResolvedWorktree
|
||||
}[]
|
||||
}
|
||||
const sshPty = parseAppSshPtyId(candidate.ptyId)
|
||||
if (workspace.connectionId) {
|
||||
if (
|
||||
options.connectionId !== workspace.connectionId ||
|
||||
sshPty?.connectionId !== workspace.connectionId
|
||||
>()
|
||||
for (const candidate of plan.candidates) {
|
||||
try {
|
||||
const workspace = await this.resolveTerminalWorkspaceLaunchScope(
|
||||
`id:${candidate.worktreeId}`
|
||||
)
|
||||
const sshPty = parseAppSshPtyId(candidate.ptyId)
|
||||
if (workspace.connectionId) {
|
||||
if (
|
||||
options.connectionId !== workspace.connectionId ||
|
||||
sshPty?.connectionId !== workspace.connectionId
|
||||
) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
} else if (
|
||||
options.connectionId !== undefined ||
|
||||
sshPty !== null ||
|
||||
!this.canRecoverPersistentLocalPtysFn()
|
||||
) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
} else if (
|
||||
options.connectionId !== undefined ||
|
||||
sshPty !== null ||
|
||||
!this.canRecoverPersistentLocalPtysFn()
|
||||
) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
const resolvedWorkspace = workspace.folderWorkspace
|
||||
? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace)
|
||||
: await this.resolveWorktreeSelector(`id:${workspace.id}`)
|
||||
const livePtyIds = await this.refreshPtyWorktreeRecordsFromController([resolvedWorkspace])
|
||||
if (!livePtyIds) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
if (!livePtyIds.has(candidate.ptyId)) {
|
||||
if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) {
|
||||
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited')
|
||||
exitedDispatchIds.push(candidate.dispatchId)
|
||||
} else {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
const resolvedWorkspace = workspace.folderWorkspace
|
||||
? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace)
|
||||
: await this.resolveWorktreeSelector(`id:${workspace.id}`)
|
||||
const connectionId = workspace.connectionId ?? null
|
||||
const providerKey = connectionId === null ? 'local' : `ssh:${connectionId}`
|
||||
const provider = recoveryCandidatesByProvider.get(providerKey) ?? {
|
||||
connectionId,
|
||||
entries: []
|
||||
}
|
||||
continue
|
||||
}
|
||||
const controllerIdentity = this.controllerTerminalIdentityByPtyId.get(candidate.ptyId)
|
||||
if (
|
||||
controllerIdentity?.handle !== candidate.terminalHandle ||
|
||||
controllerIdentity.incarnationId !== candidate.incarnationId
|
||||
) {
|
||||
provider.entries.push({ candidate, workspace, resolvedWorkspace })
|
||||
recoveryCandidatesByProvider.set(providerKey, provider)
|
||||
} catch {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
}
|
||||
}
|
||||
for (const provider of recoveryCandidatesByProvider.values()) {
|
||||
const resolvedWorktrees = [
|
||||
...new Map(
|
||||
provider.entries.map(({ resolvedWorkspace }) => [resolvedWorkspace.id, resolvedWorkspace])
|
||||
).values()
|
||||
]
|
||||
const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory(
|
||||
resolvedWorktrees,
|
||||
null,
|
||||
undefined,
|
||||
provider.connectionId
|
||||
)
|
||||
if (!inventory) {
|
||||
provider.entries.forEach(({ candidate }) => deferredDispatchIds.add(candidate.dispatchId))
|
||||
continue
|
||||
}
|
||||
const session = this.getWorkspaceSessionForWorktree(candidate.worktreeId)
|
||||
const sessionWorktreeId = session
|
||||
? resolveTerminalSessionWorktreeId(session, candidate.worktreeId)
|
||||
: null
|
||||
const activeTabId = sessionWorktreeId
|
||||
? session?.activeTabIdByWorktree?.[sessionWorktreeId]
|
||||
: undefined
|
||||
const activeGroupId = sessionWorktreeId
|
||||
? session?.activeGroupIdByWorktree?.[sessionWorktreeId]
|
||||
: undefined
|
||||
try {
|
||||
await this.adoptTerminalOrphans({
|
||||
worktree: `id:${candidate.worktreeId}`,
|
||||
expectedTopologyRevision: this.getTerminalTopologyRevision(candidate.worktreeId),
|
||||
...(activeTabId ? { activeTabId } : {}),
|
||||
...(activeGroupId ? { activeGroupId } : {}),
|
||||
claims: [
|
||||
{
|
||||
terminal: candidate.terminalHandle,
|
||||
ptyId: candidate.ptyId,
|
||||
incarnationId: candidate.incarnationId,
|
||||
tabId: candidate.tabId,
|
||||
leafId: candidate.leafId
|
||||
}
|
||||
]
|
||||
})
|
||||
} catch (error) {
|
||||
console.warn('[orchestration] legacy worker terminal adoption deferred', {
|
||||
dispatchId: candidate.dispatchId,
|
||||
error
|
||||
})
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
let rendererMaterialized =
|
||||
options.materializeRenderer !== true ||
|
||||
this.legacyWorkerTerminalMaterializedPanes.has(candidate.paneKey)
|
||||
const pty = this.ptysById.get(candidate.ptyId)
|
||||
if (
|
||||
options.materializeRenderer &&
|
||||
!rendererMaterialized &&
|
||||
pty &&
|
||||
this.notifier?.revealTerminalSession
|
||||
) {
|
||||
for (let attempt = 0; attempt < 2 && !rendererMaterialized; attempt += 1) {
|
||||
for (const { candidate, workspace } of provider.entries) {
|
||||
if (!inventory.livePtyIds.has(candidate.ptyId)) {
|
||||
if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) {
|
||||
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited')
|
||||
exitedDispatchIds.push(candidate.dispatchId)
|
||||
} else {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
}
|
||||
continue
|
||||
}
|
||||
const controllerIdentity = inventory.terminalIdentityByPtyId.get(candidate.ptyId)
|
||||
if (
|
||||
controllerIdentity?.handle !== candidate.terminalHandle ||
|
||||
controllerIdentity.incarnationId !== candidate.incarnationId
|
||||
) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
const controllerLiveness = this.getLegacyWorkerControllerPtyLiveness(candidate)
|
||||
if (controllerLiveness === false) {
|
||||
this.onPtyExit(candidate.ptyId, 0, candidate.incarnationId)
|
||||
if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) {
|
||||
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited')
|
||||
exitedDispatchIds.push(candidate.dispatchId)
|
||||
} else {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (controllerLiveness === null) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
const session = this.getWorkspaceSessionForWorktree(candidate.worktreeId)
|
||||
const sessionWorktreeId = session
|
||||
? resolveTerminalSessionWorktreeId(session, candidate.worktreeId)
|
||||
: null
|
||||
const activeTabId = sessionWorktreeId
|
||||
? session?.activeTabIdByWorktree?.[sessionWorktreeId]
|
||||
: undefined
|
||||
const activeGroupId = sessionWorktreeId
|
||||
? session?.activeGroupIdByWorktree?.[sessionWorktreeId]
|
||||
: undefined
|
||||
const exactSurfaceAlreadyPublished =
|
||||
this.hasExactPersistedTerminalSurfaceIdentity(candidate) &&
|
||||
this.hasExactTerminalSurfaceIdentity(candidate)
|
||||
if (!exactSurfaceAlreadyPublished) {
|
||||
try {
|
||||
await this.notifier.revealTerminalSession(candidate.worktreeId, {
|
||||
ptyId: candidate.ptyId,
|
||||
title: getLatestPtyTitle(pty) ?? pty.controllerTitle,
|
||||
activate: false,
|
||||
presentation: 'background',
|
||||
tabId: candidate.tabId,
|
||||
leafId: candidate.leafId,
|
||||
focus: false
|
||||
})
|
||||
rendererMaterialized = true
|
||||
this.legacyWorkerTerminalMaterializedPanes.add(candidate.paneKey)
|
||||
await this.adoptTerminalOrphansFromInventory(
|
||||
{
|
||||
worktree: `id:${candidate.worktreeId}`,
|
||||
expectedTopologyRevision: this.getTerminalTopologyRevision(candidate.worktreeId),
|
||||
...(activeTabId ? { activeTabId } : {}),
|
||||
...(activeGroupId ? { activeGroupId } : {}),
|
||||
claims: [
|
||||
{
|
||||
terminal: candidate.terminalHandle,
|
||||
ptyId: candidate.ptyId,
|
||||
incarnationId: candidate.incarnationId,
|
||||
tabId: candidate.tabId,
|
||||
leafId: candidate.leafId
|
||||
}
|
||||
]
|
||||
},
|
||||
workspace,
|
||||
inventory
|
||||
)
|
||||
} catch (error) {
|
||||
if (attempt === 0) {
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 100))
|
||||
continue
|
||||
}
|
||||
console.warn('[orchestration] adopted legacy worker was not revealed', {
|
||||
console.warn('[orchestration] legacy worker terminal adoption deferred', {
|
||||
dispatchId: candidate.dispatchId,
|
||||
error
|
||||
})
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
}
|
||||
let rendererMaterialized =
|
||||
options.materializeRenderer !== true ||
|
||||
this.legacyWorkerTerminalMaterializedPanes.has(candidate.paneKey)
|
||||
const pty = this.ptysById.get(candidate.ptyId)
|
||||
if (
|
||||
options.materializeRenderer &&
|
||||
!rendererMaterialized &&
|
||||
pty &&
|
||||
this.notifier?.revealTerminalSession
|
||||
) {
|
||||
for (let attempt = 0; attempt < 2 && !rendererMaterialized; attempt += 1) {
|
||||
try {
|
||||
const reveal = await this.notifier.revealTerminalSession(candidate.worktreeId, {
|
||||
ptyId: candidate.ptyId,
|
||||
title: getLatestPtyTitle(pty) ?? pty.controllerTitle,
|
||||
activate: false,
|
||||
presentation: 'background',
|
||||
tabId: candidate.tabId,
|
||||
leafId: candidate.leafId,
|
||||
focus: false,
|
||||
expectedProcessIdentity: {
|
||||
terminalHandle: candidate.terminalHandle,
|
||||
incarnationId: candidate.incarnationId
|
||||
}
|
||||
})
|
||||
if (reveal?.identity) {
|
||||
const identity = reveal.identity
|
||||
if (
|
||||
!runtimeWorktreeIdsEqual(identity.worktreeId, candidate.worktreeId) ||
|
||||
identity.tabId !== candidate.tabId ||
|
||||
identity.leafId !== candidate.leafId ||
|
||||
identity.ptyId !== candidate.ptyId
|
||||
) {
|
||||
throw new Error('terminal_reveal_identity_mismatch')
|
||||
}
|
||||
this.legacyWorkerTerminalIdentityReceiptPanes.add(candidate.paneKey)
|
||||
}
|
||||
rendererMaterialized = true
|
||||
this.legacyWorkerTerminalMaterializedPanes.add(candidate.paneKey)
|
||||
} catch (error) {
|
||||
if (attempt === 0) {
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 100))
|
||||
continue
|
||||
}
|
||||
console.warn('[orchestration] adopted legacy worker was not revealed', {
|
||||
dispatchId: candidate.dispatchId,
|
||||
error
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!rendererMaterialized) {
|
||||
this.legacyWorkerTerminalMaterializedPanes.delete(candidate.paneKey)
|
||||
this.legacyWorkerTerminalIdentityReceiptPanes.delete(candidate.paneKey)
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
if (
|
||||
this.legacyWorkerTerminalIdentityReceiptPanes.has(candidate.paneKey) &&
|
||||
!this.hasExactTerminalSurfaceIdentity({
|
||||
worktreeId: candidate.worktreeId,
|
||||
tabId: candidate.tabId,
|
||||
leafId: candidate.leafId,
|
||||
ptyId: candidate.ptyId,
|
||||
terminalHandle: candidate.terminalHandle,
|
||||
incarnationId: candidate.incarnationId
|
||||
})
|
||||
) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
const postRevealLiveness = this.getLegacyWorkerControllerPtyLiveness(candidate)
|
||||
if (postRevealLiveness === false) {
|
||||
this.legacyWorkerTerminalMaterializedPanes.delete(candidate.paneKey)
|
||||
this.legacyWorkerTerminalIdentityReceiptPanes.delete(candidate.paneKey)
|
||||
this.onPtyExit(candidate.ptyId, 0, candidate.incarnationId)
|
||||
if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) {
|
||||
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited')
|
||||
exitedDispatchIds.push(candidate.dispatchId)
|
||||
} else {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (postRevealLiveness === null) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
if (!this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'adopted')) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
this.legacyWorkerRecoveredPtys.add(candidate.ptyId)
|
||||
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'adopted')
|
||||
adoptedDispatchIds.push(candidate.dispatchId)
|
||||
}
|
||||
if (
|
||||
!rendererMaterialized ||
|
||||
!this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'adopted')
|
||||
) {
|
||||
deferredDispatchIds.add(candidate.dispatchId)
|
||||
continue
|
||||
}
|
||||
this.legacyWorkerRecoveredPtys.add(candidate.ptyId)
|
||||
this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'adopted')
|
||||
adoptedDispatchIds.push(candidate.dispatchId)
|
||||
}
|
||||
return {
|
||||
const result = {
|
||||
blockedPaneCount: plan.blockedPanes.length,
|
||||
adoptedDispatchIds,
|
||||
exitedDispatchIds,
|
||||
deferredDispatchIds: [...deferredDispatchIds]
|
||||
}
|
||||
this.updateLegacyWorkerTerminalRecoveryRetry(plan, deferredDispatchIds, options)
|
||||
return result
|
||||
}
|
||||
|
||||
setAutomationService(service: AutomationService): void {
|
||||
@@ -4213,6 +4454,9 @@ export class OrcaRuntimeService {
|
||||
notifySshStateChanged(targetId: string, state: SshConnectionState): void {
|
||||
this.bumpSshRelayRecoveryGeneration(targetId)
|
||||
this.invalidateSshWorktreeScanCache(targetId)
|
||||
if (state.status !== 'connected') {
|
||||
this.cancelLegacyWorkerTerminalRecoveryRetry(`ssh:${targetId}`)
|
||||
}
|
||||
this.emitClientEvent({ type: 'sshStateChanged', targetId, state: getPublicSshState(state)! })
|
||||
}
|
||||
|
||||
@@ -10345,13 +10589,6 @@ export class OrcaRuntimeService {
|
||||
return 'local'
|
||||
}
|
||||
|
||||
getOrchestrationCompatibilityAuthorityInputs(): OrchestrationCompatibilityAuthorityInputs {
|
||||
return {
|
||||
hydratedCommitments: this.getHydratedAgentHookAuthorityFn?.() ?? [],
|
||||
currentObservations: this.getCurrentAgentHookAuthorityFn?.() ?? []
|
||||
}
|
||||
}
|
||||
|
||||
registerOrchestrationCompatibilitySshAttachment(
|
||||
targetId: string,
|
||||
connectionIncarnation: string
|
||||
@@ -13554,13 +13791,24 @@ export class OrcaRuntimeService {
|
||||
const resolvedWorkspace = workspace.folderWorkspace
|
||||
? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace)
|
||||
: await this.resolveWorktreeSelector(`id:${workspace.id}`)
|
||||
const livePtyIds = await this.refreshPtyWorktreeRecordsFromController(
|
||||
const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory(
|
||||
[resolvedWorkspace],
|
||||
workspace.id
|
||||
workspace.id,
|
||||
undefined,
|
||||
workspace.connectionId ?? null
|
||||
)
|
||||
if (!livePtyIds) {
|
||||
if (!inventory) {
|
||||
throw new Error('terminal_liveness_unavailable')
|
||||
}
|
||||
return this.adoptTerminalOrphansFromInventory(request, workspace, inventory)
|
||||
}
|
||||
|
||||
private async adoptTerminalOrphansFromInventory(
|
||||
request: RuntimeTerminalOrphanAdoptionRequest,
|
||||
workspace: TerminalWorkspaceLaunchScope,
|
||||
inventory: PtyControllerInventory
|
||||
): Promise<RuntimeTerminalOrphanAdoptionResult> {
|
||||
const { livePtyIds, terminalIdentityByPtyId } = inventory
|
||||
const store = this.store
|
||||
const session = this.getWorkspaceSessionForWorktree(workspace.id)
|
||||
if (!store?.setWorkspaceSession || !store.flushOrThrow || !session) {
|
||||
@@ -13593,7 +13841,7 @@ export class OrcaRuntimeService {
|
||||
seenPaneKeys.add(paneKey)
|
||||
const live = this.getLivePtyForHandle(claim.terminal)
|
||||
const pty = live?.pty
|
||||
const controllerIdentity = this.controllerTerminalIdentityByPtyId.get(claim.ptyId)
|
||||
const controllerIdentity = terminalIdentityByPtyId.get(claim.ptyId)
|
||||
if (
|
||||
!pty ||
|
||||
pty.ptyId !== claim.ptyId ||
|
||||
@@ -13681,10 +13929,14 @@ export class OrcaRuntimeService {
|
||||
)
|
||||
})
|
||||
if (isExactPersisted && sessionWorktreeId === workspace.id) {
|
||||
for (const { claim, pty, paneKey } of validated) {
|
||||
pty.tabId = claim.tabId
|
||||
pty.paneKey = paneKey
|
||||
}
|
||||
return {
|
||||
adopted: false,
|
||||
topologyRevision: currentRevision,
|
||||
snapshot: await this.listMobileSessionTabs(`id:${workspace.id}`)
|
||||
snapshot: this.getTerminalOrphanAdoptionSnapshot(workspace.id)
|
||||
}
|
||||
}
|
||||
if (currentRevision !== request.expectedTopologyRevision) {
|
||||
@@ -13999,10 +14251,19 @@ export class OrcaRuntimeService {
|
||||
return {
|
||||
adopted: true,
|
||||
topologyRevision: persisted.terminalTopologyRevisionByRepoId?.[repoId] ?? currentRevision + 1,
|
||||
snapshot: await this.listMobileSessionTabs(`id:${workspace.id}`)
|
||||
snapshot: this.getTerminalOrphanAdoptionSnapshot(workspace.id)
|
||||
}
|
||||
}
|
||||
|
||||
private getTerminalOrphanAdoptionSnapshot(worktreeId: string): RuntimeMobileSessionTabsResult {
|
||||
this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId, {
|
||||
allowAttachedWindow: true,
|
||||
onlyRuntimeOwnedTerminals: true
|
||||
})
|
||||
this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId)
|
||||
return this.getMobileSessionTabsForWorktree(worktreeId)
|
||||
}
|
||||
|
||||
private buildTerminalVisualLayouts(
|
||||
terminals: RuntimeTerminalSummary[],
|
||||
worktreesById: Map<string, ResolvedWorktree>,
|
||||
@@ -26302,17 +26563,34 @@ export class OrcaRuntimeService {
|
||||
targetWorktreeId: string | null = null,
|
||||
deadline?: number
|
||||
): Promise<Set<string> | null> {
|
||||
const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory(
|
||||
resolvedWorktrees,
|
||||
targetWorktreeId,
|
||||
deadline
|
||||
)
|
||||
return inventory ? new Set(inventory.livePtyIds) : null
|
||||
}
|
||||
|
||||
private async refreshPtyWorktreeRecordsWithControllerInventory(
|
||||
resolvedWorktrees: ResolvedWorktree[],
|
||||
targetWorktreeId: string | null = null,
|
||||
deadline?: number,
|
||||
connectionId?: string | null
|
||||
): Promise<PtyControllerInventory | null> {
|
||||
if (targetWorktreeId === FLOATING_TERMINAL_WORKTREE_ID) {
|
||||
const targetedLiveness = this.refreshFloatingWorkspacePtyLiveness()
|
||||
if (targetedLiveness !== null) {
|
||||
return targetedLiveness
|
||||
return {
|
||||
livePtyIds: targetedLiveness,
|
||||
terminalIdentityByPtyId: new Map()
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!this.ptyController?.listProcesses) {
|
||||
return null
|
||||
}
|
||||
const sessionsResult = await withTimeoutResult(
|
||||
this.ptyController.listProcesses(),
|
||||
this.ptyController.listProcesses(connectionId),
|
||||
deadline === undefined
|
||||
? PTY_CONTROLLER_LIST_TIMEOUT_MS
|
||||
: Math.max(1, Math.min(PTY_CONTROLLER_LIST_TIMEOUT_MS, deadline - Date.now()))
|
||||
@@ -26322,10 +26600,7 @@ export class OrcaRuntimeService {
|
||||
return null
|
||||
}
|
||||
const sessions = sessionsResult.value
|
||||
const controllerIdentityByPtyId = new Map<
|
||||
string,
|
||||
{ handle: string; incarnationId: string; wslDistro?: string | null }
|
||||
>()
|
||||
const controllerIdentityByPtyId = new Map<string, PtyControllerTerminalIdentity>()
|
||||
const ptyIdByControllerHandle = new Map<string, string>()
|
||||
const ambiguousControllerPtyIds = new Set<string>()
|
||||
for (const session of sessions) {
|
||||
@@ -26356,7 +26631,6 @@ export class OrcaRuntimeService {
|
||||
for (const ptyId of ambiguousControllerPtyIds) {
|
||||
controllerIdentityByPtyId.delete(ptyId)
|
||||
}
|
||||
this.controllerTerminalIdentityByPtyId = controllerIdentityByPtyId
|
||||
const persistedWorktreeIdByPtyId = indexPersistedPtyWorktreeBindings(
|
||||
this.store?.getWorkspaceSession?.()
|
||||
)
|
||||
@@ -26432,6 +26706,9 @@ export class OrcaRuntimeService {
|
||||
this.refreshPtyForegroundAgent(session.id)
|
||||
}
|
||||
for (const pty of this.ptysById.values()) {
|
||||
if (connectionId !== undefined && pty.connectionId !== connectionId) {
|
||||
continue
|
||||
}
|
||||
if (!allLivePtyIds.has(pty.ptyId) && !this.leafExistsForPty(pty.ptyId)) {
|
||||
if (this.ptyController.hasPty?.(pty.ptyId) === true) {
|
||||
// Why: an SSH spawn can become addressable before an overlapping relay list includes it.
|
||||
@@ -26451,7 +26728,10 @@ export class OrcaRuntimeService {
|
||||
}
|
||||
}
|
||||
this.pruneDisconnectedPtyRecords()
|
||||
return targetWorktreeId ? selectedLivePtyIds : allLivePtyIds
|
||||
return {
|
||||
livePtyIds: targetWorktreeId ? selectedLivePtyIds : allLivePtyIds,
|
||||
terminalIdentityByPtyId: controllerIdentityByPtyId
|
||||
}
|
||||
}
|
||||
|
||||
private refreshFloatingWorkspacePtyLiveness(): Set<string> | null {
|
||||
|
||||
@@ -171,10 +171,10 @@ function legacyMessageMatchesQuestion(
|
||||
message: MessageRow,
|
||||
question: string,
|
||||
options: string[],
|
||||
recipientHandle: string
|
||||
recipientHandles: readonly string[]
|
||||
): boolean {
|
||||
if (
|
||||
message.to_handle !== recipientHandle ||
|
||||
!recipientHandles.includes(message.to_handle) ||
|
||||
normalizeLegacyQuestionText(message.body) !== normalizeLegacyQuestionText(question)
|
||||
) {
|
||||
return false
|
||||
@@ -1212,6 +1212,13 @@ export class OrchestrationDb {
|
||||
`The ${params.role} compatibility principal is already committed to different proof.`
|
||||
)
|
||||
}
|
||||
if (existing.status === 'revoked') {
|
||||
throw new OrchestrationError(
|
||||
'legacy_read_only',
|
||||
`The ${params.role} compatibility principal has been revoked. No effects were applied.`,
|
||||
{ effectsApplied: false }
|
||||
)
|
||||
}
|
||||
this.db.exec('COMMIT')
|
||||
return { principal: existing, duplicate: true }
|
||||
}
|
||||
@@ -1263,6 +1270,15 @@ export class OrchestrationDb {
|
||||
.all(runId) as LegacyCompatibilityPrincipalRow[]
|
||||
}
|
||||
|
||||
getLegacyCoordinatorPrincipal(runId: string): LegacyCompatibilityPrincipalRow | undefined {
|
||||
return this.db
|
||||
.prepare(
|
||||
`SELECT * FROM legacy_compatibility_principals
|
||||
WHERE run_id = ? AND role = 'coordinator'`
|
||||
)
|
||||
.get(runId) as LegacyCompatibilityPrincipalRow | undefined
|
||||
}
|
||||
|
||||
resolveLegacyCompatibilityPrincipalByIdentity(params: {
|
||||
runId: string
|
||||
role: LegacyPrincipalRole
|
||||
@@ -1362,6 +1378,10 @@ export class OrchestrationDb {
|
||||
}
|
||||
|
||||
isLegacyCoordinatorHandle(runId: string, terminalHandle: string): boolean {
|
||||
const principal = this.getLegacyCoordinatorPrincipal(runId)
|
||||
if (principal?.terminal_handle === terminalHandle) {
|
||||
return true
|
||||
}
|
||||
return Boolean(
|
||||
this.db
|
||||
.prepare(
|
||||
@@ -1403,18 +1423,24 @@ export class OrchestrationDb {
|
||||
if (!principal || principal.role !== 'worker' || !principal.dispatch_id) {
|
||||
throw new OrchestrationError('request_mismatch', 'Legacy worker principal was not found.')
|
||||
}
|
||||
const runAddress = `run:${principal.run_id}`
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`SELECT * FROM messages
|
||||
WHERE run_id = ? AND delivery_contract = 'legacy_direct'
|
||||
AND from_handle = ? AND to_handle = ? AND type = 'worker_done'
|
||||
WHERE run_id = ?
|
||||
AND (
|
||||
(delivery_contract = 'legacy_direct' AND to_handle = ?) OR
|
||||
(delivery_contract = 'current_delivery' AND to_handle = ?)
|
||||
)
|
||||
AND from_handle = ? AND type = 'worker_done'
|
||||
AND subject = ? AND body = ? AND payload IS ?
|
||||
ORDER BY sequence`
|
||||
)
|
||||
.all(
|
||||
principal.run_id,
|
||||
principal.terminal_handle,
|
||||
params.recipientHandle,
|
||||
runAddress,
|
||||
principal.terminal_handle,
|
||||
params.subject,
|
||||
params.body,
|
||||
params.payload
|
||||
@@ -1444,10 +1470,11 @@ export class OrchestrationDb {
|
||||
this.db
|
||||
.prepare(
|
||||
`SELECT 1 FROM messages
|
||||
WHERE run_id = ? AND delivery_contract = 'current_delivery' AND read = 0
|
||||
WHERE run_id = ? AND to_handle = ?
|
||||
AND delivery_contract = 'current_delivery' AND read = 0
|
||||
LIMIT 1`
|
||||
)
|
||||
.get(runId)
|
||||
.get(runId, `run:${runId}`)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1832,7 +1859,14 @@ export class OrchestrationDb {
|
||||
runId: string
|
||||
coordinatorHandle: string
|
||||
coordinatorPaneKey: string
|
||||
allowLegacyCompatibility?: boolean
|
||||
takeoverLegacy?: boolean
|
||||
legacyCoordinatorAuthority?: {
|
||||
runId: string
|
||||
principalId: string | null
|
||||
terminalHandle: string
|
||||
paneKey: string
|
||||
consumerGeneration: number
|
||||
}
|
||||
}): RunRow | undefined {
|
||||
this.db.exec('BEGIN IMMEDIATE')
|
||||
try {
|
||||
@@ -1845,30 +1879,95 @@ export class OrchestrationDb {
|
||||
run.coordinator_pane_key !== null &&
|
||||
isEquivalentPaneKey(run.coordinator_pane_key, params.coordinatorPaneKey)
|
||||
const adoption = this.getLegacyAdoption()
|
||||
const adoptedRun = adoption?.adopted_run_id === params.runId
|
||||
const legacyAuthority = params.legacyCoordinatorAuthority
|
||||
const legacyPrincipalId = legacyAuthority?.principalId
|
||||
const legacyPrincipal = legacyPrincipalId
|
||||
? this.getLegacyCompatibilityPrincipal(legacyPrincipalId)
|
||||
: undefined
|
||||
const provenLegacyBinding = Boolean(
|
||||
adoptedRun &&
|
||||
legacyAuthority &&
|
||||
legacyAuthority.principalId !== null &&
|
||||
legacyAuthority.runId === params.runId &&
|
||||
legacyAuthority.consumerGeneration === run.consumer_generation &&
|
||||
legacyPrincipal?.run_id === params.runId &&
|
||||
legacyPrincipal.role === 'coordinator' &&
|
||||
legacyPrincipal.status === 'committed' &&
|
||||
legacyPrincipal.terminal_handle === legacyAuthority.terminalHandle &&
|
||||
isEquivalentPaneKey(legacyPrincipal.pane_key, legacyAuthority.paneKey) &&
|
||||
params.coordinatorHandle === legacyAuthority.terminalHandle &&
|
||||
isEquivalentPaneKey(params.coordinatorPaneKey, legacyAuthority.paneKey)
|
||||
)
|
||||
if (legacyAuthority && !provenLegacyBinding) {
|
||||
throw new OrchestrationError(
|
||||
'legacy_read_only',
|
||||
'This retained legacy coordinator no longer has lifecycle authority. No effects were applied.',
|
||||
{ effectsApplied: false }
|
||||
)
|
||||
}
|
||||
const activeLegacyAssignment =
|
||||
adoption?.adopted_run_id === params.runId &&
|
||||
adoptedRun &&
|
||||
Boolean(
|
||||
this.db
|
||||
.prepare(
|
||||
`SELECT 1 FROM dispatch_contexts
|
||||
WHERE run_id = ? AND contract_version = ?
|
||||
AND status IN ('pending', 'dispatched')
|
||||
UNION ALL
|
||||
SELECT 1 FROM legacy_compatibility_principals
|
||||
WHERE run_id = ? AND status = 'committed'
|
||||
LIMIT 1`
|
||||
)
|
||||
.get(params.runId, LEGACY_CONTRACT_VERSION, params.runId)
|
||||
.get(params.runId, LEGACY_CONTRACT_VERSION)
|
||||
)
|
||||
if (activeLegacyAssignment && !sameBinding && !params.allowLegacyCompatibility) {
|
||||
const coordinatorPrincipal = adoptedRun
|
||||
? this.getLegacyCoordinatorPrincipal(params.runId)
|
||||
: undefined
|
||||
const retainedCoordinatorHandle =
|
||||
coordinatorPrincipal?.terminal_handle ??
|
||||
run.coordinator_handle ??
|
||||
this.getUniqueLegacyTaskCreatorHandle(params.runId)
|
||||
const takeoverAlreadyApplied = Boolean(
|
||||
params.takeoverLegacy &&
|
||||
sameBinding &&
|
||||
run.coordinator_handle === params.coordinatorHandle &&
|
||||
coordinatorPrincipal?.status !== 'committed'
|
||||
)
|
||||
if (params.takeoverLegacy && !adoptedRun) {
|
||||
throw new OrchestrationError(
|
||||
'invalid_argument',
|
||||
'Legacy takeover is only available for the automatically adopted Run.'
|
||||
)
|
||||
}
|
||||
if (
|
||||
activeLegacyAssignment &&
|
||||
!sameBinding &&
|
||||
!provenLegacyBinding &&
|
||||
!params.takeoverLegacy
|
||||
) {
|
||||
throw new OrchestrationError(
|
||||
'consumer_fenced',
|
||||
'This adopted Run still has a live legacy principal; only its attested coordinator may rebind it.',
|
||||
{ effectsApplied: false }
|
||||
'This adopted Run still has live legacy work. Its attested coordinator may rebind it, or a current coordinator may explicitly use run-use --takeover-legacy.',
|
||||
{
|
||||
effectsApplied: false,
|
||||
recoveryCommand: `orca orchestration run-use --id ${params.runId} --takeover-legacy`
|
||||
}
|
||||
)
|
||||
}
|
||||
this.unbindOtherRunsForPane(params.coordinatorPaneKey, params.runId)
|
||||
if (!sameBinding || run.coordinator_handle !== params.coordinatorHandle) {
|
||||
if (
|
||||
(params.takeoverLegacy && !takeoverAlreadyApplied) ||
|
||||
!sameBinding ||
|
||||
run.coordinator_handle !== params.coordinatorHandle
|
||||
) {
|
||||
if (adoptedRun && (params.takeoverLegacy || !activeLegacyAssignment)) {
|
||||
if (
|
||||
coordinatorPrincipal?.status === 'committed' &&
|
||||
(params.takeoverLegacy ||
|
||||
coordinatorPrincipal.terminal_handle !== params.coordinatorHandle ||
|
||||
!isEquivalentPaneKey(coordinatorPrincipal.pane_key, params.coordinatorPaneKey))
|
||||
) {
|
||||
this.setLegacyCompatibilityPrincipalStatus(coordinatorPrincipal.id, 'revoked')
|
||||
}
|
||||
}
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE runs
|
||||
@@ -1879,6 +1978,9 @@ export class OrchestrationDb {
|
||||
)
|
||||
.run(params.coordinatorHandle, params.coordinatorPaneKey, params.runId)
|
||||
this.fenceOutstandingDelivery(params.runId)
|
||||
if (params.takeoverLegacy) {
|
||||
this.promoteLegacyCoordinatorMailForTakeover(params.runId, retainedCoordinatorHandle)
|
||||
}
|
||||
}
|
||||
this.db.exec('COMMIT')
|
||||
} catch (error) {
|
||||
@@ -1953,6 +2055,51 @@ export class OrchestrationDb {
|
||||
.run(runId)
|
||||
}
|
||||
|
||||
private promoteLegacyCoordinatorMailForTakeover(
|
||||
runId: string,
|
||||
retainedCoordinatorHandle: string | null
|
||||
): void {
|
||||
if (!retainedCoordinatorHandle) {
|
||||
return
|
||||
}
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE messages
|
||||
SET to_handle = ?, delivery_contract = 'current_delivery',
|
||||
read = 0, delivered_at = NULL
|
||||
WHERE run_id = ? AND delivery_contract = 'legacy_direct'
|
||||
AND to_handle = ?
|
||||
AND EXISTS(
|
||||
SELECT 1 FROM dispatch_contexts d
|
||||
WHERE d.run_id = messages.run_id
|
||||
AND d.contract_version = ?
|
||||
AND (
|
||||
messages.from_handle = d.assignee_handle OR
|
||||
messages.from_handle = 'dispatch:' || d.id
|
||||
)
|
||||
)
|
||||
AND (
|
||||
read = 0 OR EXISTS(
|
||||
SELECT 1 FROM question_threads q
|
||||
WHERE q.message_id = messages.id AND q.status = 'pending'
|
||||
)
|
||||
)`
|
||||
)
|
||||
.run(`run:${runId}`, runId, retainedCoordinatorHandle, LEGACY_CONTRACT_VERSION)
|
||||
}
|
||||
|
||||
private getUniqueLegacyTaskCreatorHandle(runId: string): string | null {
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`SELECT DISTINCT created_by_terminal_handle AS handle
|
||||
FROM tasks
|
||||
WHERE run_id = ? AND created_by_terminal_handle IS NOT NULL
|
||||
LIMIT 2`
|
||||
)
|
||||
.all(runId) as { handle: string }[]
|
||||
return rows.length === 1 ? (rows[0]?.handle ?? null) : null
|
||||
}
|
||||
|
||||
private requireCurrentConsumer(runId: string, consumerGeneration: number): RunRow {
|
||||
const run = this.getRunRaw(runId)
|
||||
if (!run || run.legacy === 1 || run.consumer_generation !== consumerGeneration) {
|
||||
@@ -2269,12 +2416,20 @@ export class OrchestrationDb {
|
||||
let message = params.message.existingId
|
||||
? this.getMessageById(params.message.existingId)
|
||||
: undefined
|
||||
const delivery = this.resolveLegacyWorkerCoordinatorDelivery(
|
||||
principal.run_id,
|
||||
params.message.to
|
||||
)
|
||||
if (params.message.existingId) {
|
||||
const matchesOriginalLegacyRoute =
|
||||
message?.delivery_contract === 'legacy_direct' && message.to_handle === params.message.to
|
||||
const matchesCurrentRoute =
|
||||
message?.delivery_contract === delivery.contract && message.to_handle === delivery.to
|
||||
if (
|
||||
!message ||
|
||||
message.run_id !== principal.run_id ||
|
||||
message.delivery_contract !== 'legacy_direct' ||
|
||||
message.to_handle !== params.message.to
|
||||
message.from_handle !== principal.terminal_handle ||
|
||||
(!matchesOriginalLegacyRoute && !matchesCurrentRoute)
|
||||
) {
|
||||
throw new OrchestrationError(
|
||||
'request_mismatch',
|
||||
@@ -2284,7 +2439,7 @@ export class OrchestrationDb {
|
||||
} else {
|
||||
message = this.insertMessage({
|
||||
from: principal.terminal_handle,
|
||||
to: params.message.to,
|
||||
to: delivery.to,
|
||||
subject: params.message.subject,
|
||||
body: params.message.body,
|
||||
type: params.message.type,
|
||||
@@ -2292,7 +2447,7 @@ export class OrchestrationDb {
|
||||
payload: params.message.payload,
|
||||
senderPaneKey: principal.pane_key,
|
||||
runId: principal.run_id,
|
||||
deliveryContract: 'legacy_direct'
|
||||
deliveryContract: delivery.contract
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2300,12 +2455,24 @@ export class OrchestrationDb {
|
||||
if (params.lifecycle.kind === 'heartbeat') {
|
||||
this.recordHeartbeat(dispatchId, params.lifecycle.at)
|
||||
} else if (params.lifecycle.kind === 'worker_report') {
|
||||
settlement = this.settleWorkerReportInTransaction({
|
||||
taskId: params.lifecycle.taskId,
|
||||
dispatchId,
|
||||
outcome: params.lifecycle.outcome,
|
||||
result: params.lifecycle.result
|
||||
})
|
||||
const persistedOutcome =
|
||||
params.message.existingId &&
|
||||
dispatch.task_id === params.lifecycle.taskId &&
|
||||
dispatch.status === 'completed'
|
||||
? 'succeeded'
|
||||
: params.message.existingId &&
|
||||
dispatch.task_id === params.lifecycle.taskId &&
|
||||
dispatch.status === 'failed'
|
||||
? 'failed'
|
||||
: undefined
|
||||
settlement = persistedOutcome
|
||||
? { action: 'settled', outcome: persistedOutcome, duplicate: true }
|
||||
: this.settleWorkerReportInTransaction({
|
||||
taskId: params.lifecycle.taskId,
|
||||
dispatchId,
|
||||
outcome: params.lifecycle.outcome,
|
||||
result: params.lifecycle.result
|
||||
})
|
||||
if (settlement.action === 'rejected') {
|
||||
throw new OrchestrationError(settlement.code, settlement.reason)
|
||||
}
|
||||
@@ -2380,24 +2547,36 @@ export class OrchestrationDb {
|
||||
)
|
||||
}
|
||||
|
||||
const existingQuestionId =
|
||||
params.existingQuestionId &&
|
||||
!this.db
|
||||
.prepare(
|
||||
`SELECT 1 FROM legacy_operation_receipts
|
||||
WHERE principal_id = ? AND method = 'orchestration.ask' AND effect_id = ?
|
||||
LIMIT 1`
|
||||
)
|
||||
.get(principal.id, params.existingQuestionId)
|
||||
? params.existingQuestionId
|
||||
: undefined
|
||||
let question: QuestionRow
|
||||
let message: MessageRow
|
||||
if (params.existingQuestionId) {
|
||||
const existingQuestion = this.getQuestion(params.existingQuestionId)
|
||||
const existingMessage = this.getMessageById(params.existingQuestionId)
|
||||
const delivery = this.resolveLegacyWorkerCoordinatorDelivery(
|
||||
principal.run_id,
|
||||
params.recipientHandle
|
||||
)
|
||||
if (existingQuestionId) {
|
||||
const existingQuestion = this.getQuestion(existingQuestionId)
|
||||
const existingMessage = this.getMessageById(existingQuestionId)
|
||||
if (
|
||||
!existingQuestion ||
|
||||
!existingMessage ||
|
||||
existingQuestion.run_id !== principal.run_id ||
|
||||
existingQuestion.dispatch_id !== dispatchId ||
|
||||
existingQuestion.status !== 'pending' ||
|
||||
existingMessage.delivery_contract !== 'legacy_direct' ||
|
||||
!legacyMessageMatchesQuestion(
|
||||
existingMessage,
|
||||
params.question,
|
||||
params.options ?? [],
|
||||
params.recipientHandle
|
||||
)
|
||||
existingMessage.delivery_contract !== delivery.contract ||
|
||||
!legacyMessageMatchesQuestion(existingMessage, params.question, params.options ?? [], [
|
||||
delivery.to
|
||||
])
|
||||
) {
|
||||
throw new OrchestrationError(
|
||||
'request_mismatch',
|
||||
@@ -2409,7 +2588,7 @@ export class OrchestrationDb {
|
||||
} else {
|
||||
message = this.insertMessage({
|
||||
from: principal.terminal_handle,
|
||||
to: params.recipientHandle,
|
||||
to: delivery.to,
|
||||
subject: 'Question',
|
||||
body: params.question,
|
||||
type: 'question',
|
||||
@@ -2421,7 +2600,7 @@ export class OrchestrationDb {
|
||||
}),
|
||||
senderPaneKey: principal.pane_key,
|
||||
runId: principal.run_id,
|
||||
deliveryContract: 'legacy_direct'
|
||||
deliveryContract: delivery.contract
|
||||
})
|
||||
this.db
|
||||
.prepare('UPDATE messages SET thread_id = ? WHERE id = ?')
|
||||
@@ -2473,21 +2652,37 @@ export class OrchestrationDb {
|
||||
question: QuestionRow
|
||||
message: MessageRow
|
||||
answerAcknowledged: boolean
|
||||
claimedByOperation: boolean
|
||||
}[] {
|
||||
const principal = this.requireCommittedLegacyPrincipal(params.principalId, 'worker')
|
||||
const runAddress = `run:${principal.run_id}`
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`SELECT q.*, m.id AS source_message_id
|
||||
`SELECT q.*, m.id AS source_message_id,
|
||||
EXISTS(
|
||||
SELECT 1 FROM legacy_operation_receipts lor
|
||||
WHERE lor.principal_id = ? AND lor.method = 'orchestration.ask'
|
||||
AND lor.effect_id = q.message_id
|
||||
) AS claimed_by_operation
|
||||
FROM question_threads q
|
||||
INNER JOIN messages m ON m.id = q.message_id
|
||||
WHERE q.run_id = ? AND q.dispatch_id = ?
|
||||
AND m.delivery_contract = 'legacy_direct'
|
||||
AND m.to_handle = ?
|
||||
AND (
|
||||
(m.delivery_contract = 'legacy_direct' AND m.to_handle = ?) OR
|
||||
(m.delivery_contract = 'current_delivery' AND m.to_handle = ?)
|
||||
)
|
||||
ORDER BY m.sequence
|
||||
LIMIT 501`
|
||||
)
|
||||
.all(principal.run_id, principal.dispatch_id, params.recipientHandle) as (QuestionRow & {
|
||||
.all(
|
||||
principal.id,
|
||||
principal.run_id,
|
||||
principal.dispatch_id,
|
||||
params.recipientHandle,
|
||||
runAddress
|
||||
) as (QuestionRow & {
|
||||
source_message_id: string
|
||||
claimed_by_operation: number
|
||||
})[]
|
||||
if (rows.length > 500) {
|
||||
throw new OrchestrationError(
|
||||
@@ -2500,17 +2695,16 @@ export class OrchestrationDb {
|
||||
const message = this.getMessageById(row.source_message_id)
|
||||
return Boolean(
|
||||
message &&
|
||||
legacyMessageMatchesQuestion(
|
||||
message,
|
||||
params.question,
|
||||
params.options ?? [],
|
||||
params.recipientHandle
|
||||
)
|
||||
legacyMessageMatchesQuestion(message, params.question, params.options ?? [], [
|
||||
params.recipientHandle,
|
||||
runAddress
|
||||
])
|
||||
)
|
||||
})
|
||||
.map((row) => ({
|
||||
question: exposeQuestionTimestamps(row),
|
||||
message: this.getMessageById(row.message_id) as MessageRow,
|
||||
claimedByOperation: row.claimed_by_operation === 1,
|
||||
answerAcknowledged: row.answer_message_id
|
||||
? Boolean(
|
||||
this.db
|
||||
@@ -2525,6 +2719,22 @@ export class OrchestrationDb {
|
||||
}))
|
||||
}
|
||||
|
||||
private resolveLegacyWorkerCoordinatorDelivery(
|
||||
runId: string,
|
||||
retainedCoordinatorHandle: string
|
||||
): { to: string; contract: MessageDeliveryContract } {
|
||||
const run = this.getRunRaw(runId)
|
||||
const principal = this.getLegacyCoordinatorPrincipal(runId)
|
||||
const takenOver =
|
||||
run?.coordinator_handle !== null &&
|
||||
(principal?.status === 'revoked' ||
|
||||
(run?.coordinator_handle !== undefined &&
|
||||
run.coordinator_handle !== retainedCoordinatorHandle))
|
||||
return takenOver
|
||||
? { to: `run:${runId}`, contract: 'current_delivery' }
|
||||
: { to: retainedCoordinatorHandle, contract: 'legacy_direct' }
|
||||
}
|
||||
|
||||
commitLegacyReplyOperation(params: {
|
||||
principalId: string
|
||||
operationKey: string
|
||||
@@ -3658,11 +3868,10 @@ export class OrchestrationDb {
|
||||
wd.agent_terminal_handle
|
||||
FROM dispatch_contexts dc
|
||||
INNER JOIN worker_dispatches wd ON wd.dispatch_id = dc.id
|
||||
WHERE dc.contract_version = ?
|
||||
AND wd.state IN ('starting', 'ready', 'start_unknown', 'stopping', 'stop_unknown')
|
||||
WHERE wd.state IN ('starting', 'ready', 'start_unknown', 'stopping', 'stop_unknown')
|
||||
ORDER BY dc.rowid`
|
||||
)
|
||||
.all(LEGACY_CONTRACT_VERSION) as LegacyWorkerTerminalRecoveryRow[]
|
||||
.all() as LegacyWorkerTerminalRecoveryRow[]
|
||||
}
|
||||
|
||||
getFederatedDispatch(dispatchId: string): FederatedDispatchRow | undefined {
|
||||
|
||||
@@ -77,6 +77,14 @@ describe('formatMessageBanner', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('lets the CLI resolve the live sender for Run and Dispatch addresses', () => {
|
||||
for (const to_handle of ['run:run_test', 'dispatch:dispatch_test']) {
|
||||
const banner = formatMessageBanner(makeMessage({ to_handle }))
|
||||
expect(banner).toContain('[Reply: orca orchestration reply --id msg_test1 --body "..."]')
|
||||
expect(banner).not.toContain(`--from ${to_handle}`)
|
||||
}
|
||||
})
|
||||
|
||||
it('marks legacy messages read-only without reply or acknowledgment affordances', () => {
|
||||
const banner = formatMessageBanner(
|
||||
makeMessage({ id: 'msg_legacy', run_id: 'run_legacy_local' })
|
||||
|
||||
@@ -86,10 +86,11 @@ export function formatMessageBanner(
|
||||
}
|
||||
|
||||
if (authority === 'current') {
|
||||
// Why: older shells can lack Orca's terminal identity environment.
|
||||
lines.push(
|
||||
`[Reply: orca orchestration reply --id ${msg.id} --from ${msg.to_handle} --body "..."]`
|
||||
)
|
||||
const explicitFrom =
|
||||
msg.to_handle.startsWith('run:') || msg.to_handle.startsWith('dispatch:')
|
||||
? ''
|
||||
: ` --from ${msg.to_handle}`
|
||||
lines.push(`[Reply: orca orchestration reply --id ${msg.id}${explicitFrom} --body "..."]`)
|
||||
}
|
||||
lines.push(SEPARATOR)
|
||||
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { rmSync } from 'node:fs'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import Database from '../../sqlite/sync-database'
|
||||
import {
|
||||
@@ -9,21 +7,10 @@ import {
|
||||
LEGACY_RUN_ID,
|
||||
OrchestrationDb
|
||||
} from './db'
|
||||
|
||||
type CutoverFixture = {
|
||||
dbPath: string
|
||||
currentRunId: string
|
||||
unrelatedRunId: string
|
||||
currentDispatchId: string
|
||||
legacyTaskId: string
|
||||
legacyDispatchId: string
|
||||
legacyGateId: string
|
||||
legacyMessageIds: string[]
|
||||
legacyQuestionId: string
|
||||
legacyDeliveryId: string
|
||||
rejectionMessageId: string
|
||||
lookalikeMessageId: string
|
||||
}
|
||||
import {
|
||||
createLegacyStorageCutoverFixture,
|
||||
type LegacyStorageCutoverFixture
|
||||
} from './orchestration-legacy-storage-test-fixture'
|
||||
|
||||
describe('OrchestrationDb legacy contract storage', () => {
|
||||
let db: OrchestrationDb | undefined
|
||||
@@ -38,129 +25,14 @@ describe('OrchestrationDb legacy contract storage', () => {
|
||||
}
|
||||
})
|
||||
|
||||
function createCutoverFixture(): CutoverFixture {
|
||||
tempDir = mkdtempSync(join(tmpdir(), 'orca-legacy-storage-'))
|
||||
const dbPath = join(tempDir, 'orchestration.db')
|
||||
const first = new OrchestrationDb(dbPath)
|
||||
const currentRun = first.createRun({
|
||||
objective: 'Current work',
|
||||
coordinatorHandle: 'term_current_coord',
|
||||
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111'
|
||||
})
|
||||
const currentTask = first.createTask({ spec: 'current', runId: currentRun.id })
|
||||
const unrelatedRun = first.createRun({
|
||||
objective: 'Unrelated current work',
|
||||
coordinatorHandle: 'term_unrelated_coord',
|
||||
coordinatorPaneKey: 'tab_unrelated:55555555-5555-4555-8555-555555555555'
|
||||
})
|
||||
const currentDispatch = first.createDispatchContext(
|
||||
currentTask.id,
|
||||
'term_current_worker',
|
||||
'tab_current:22222222-2222-4222-9222-222222222222',
|
||||
'current_launch_hash'
|
||||
)
|
||||
first.insertMessage({
|
||||
runId: currentRun.id,
|
||||
from: 'term_current_worker',
|
||||
to: `run:${currentRun.id}`,
|
||||
subject: 'current mail'
|
||||
})
|
||||
|
||||
const legacyTask = first.createTask({
|
||||
spec: 'legacy',
|
||||
createdByTerminalHandle: 'term_legacy_coord'
|
||||
})
|
||||
first.createDispatchContext(
|
||||
legacyTask.id,
|
||||
'term_legacy_worker',
|
||||
'tab_legacy:33333333-3333-4333-8333-333333333333'
|
||||
)
|
||||
const legacyGate = first.createGate({
|
||||
taskId: legacyTask.id,
|
||||
question: 'Retained gate?'
|
||||
})
|
||||
first.resolveGate(legacyGate.id, 'continue')
|
||||
const retryDispatch = first.createDispatchContext(
|
||||
legacyTask.id,
|
||||
'term_legacy_worker',
|
||||
'tab_legacy:33333333-3333-4333-8333-333333333333'
|
||||
)
|
||||
const legacyMessages = [
|
||||
first.insertMessage({
|
||||
from: 'term_legacy_coord',
|
||||
to: 'term_legacy_worker',
|
||||
subject: 'read worker mail'
|
||||
}),
|
||||
first.insertMessage({
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'read coordinator mail'
|
||||
}),
|
||||
first.insertMessage({
|
||||
from: 'term_legacy_coord',
|
||||
to: 'term_legacy_worker',
|
||||
subject: 'second worker page'
|
||||
})
|
||||
]
|
||||
first.markAsRead(legacyMessages.map((message) => message.id))
|
||||
const question = first.createQuestion({
|
||||
runId: LEGACY_RUN_ID,
|
||||
dispatchId: retryDispatch.id,
|
||||
askerHandle: 'term_legacy_worker',
|
||||
question: 'Retained question?'
|
||||
})
|
||||
const rejection = first.insertMessage({
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'Rejected heartbeat',
|
||||
type: 'heartbeat',
|
||||
payload: JSON.stringify({ _orcaLifecycleRejection: { code: 'migration', reason: 'cutover' } })
|
||||
})
|
||||
const lookalike = first.insertMessage({
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'Ordinary legacy mail',
|
||||
payload: JSON.stringify({ '.orcaLifecycleRejection': 'not an audit marker' })
|
||||
})
|
||||
first.close()
|
||||
|
||||
const raw = new Database(dbPath)
|
||||
const legacyDeliveryId = 'delivery_legacy_outstanding'
|
||||
raw
|
||||
.prepare(
|
||||
`INSERT INTO deliveries (
|
||||
id, run_id, consumer_generation, message_ids, status
|
||||
) VALUES (?, ?, 0, ?, 'outstanding')`
|
||||
)
|
||||
.run(legacyDeliveryId, LEGACY_RUN_ID, JSON.stringify([legacyMessages[0].id]))
|
||||
raw.exec(`
|
||||
DROP INDEX IF EXISTS idx_messages_delivery_contract;
|
||||
DROP TABLE legacy_mail_receipts;
|
||||
DROP TABLE legacy_operation_receipts;
|
||||
DROP TABLE legacy_compatibility_principals;
|
||||
DROP TABLE legacy_adoptions;
|
||||
`)
|
||||
raw.pragma('user_version = 18')
|
||||
raw.close()
|
||||
|
||||
return {
|
||||
dbPath,
|
||||
currentRunId: currentRun.id,
|
||||
unrelatedRunId: unrelatedRun.id,
|
||||
currentDispatchId: currentDispatch.id,
|
||||
legacyTaskId: legacyTask.id,
|
||||
legacyDispatchId: retryDispatch.id,
|
||||
legacyGateId: legacyGate.id,
|
||||
legacyMessageIds: legacyMessages.map((message) => message.id),
|
||||
legacyQuestionId: question.message.id,
|
||||
legacyDeliveryId,
|
||||
rejectionMessageId: rejection.id,
|
||||
lookalikeMessageId: lookalike.id
|
||||
}
|
||||
function createCutoverFixture(): LegacyStorageCutoverFixture {
|
||||
const created = createLegacyStorageCutoverFixture()
|
||||
tempDir = created.tempDir
|
||||
return created.fixture
|
||||
}
|
||||
|
||||
function openAdoptedFixture(): {
|
||||
fixture: CutoverFixture
|
||||
fixture: LegacyStorageCutoverFixture
|
||||
adoptedRunId: string
|
||||
workerPrincipalId: string
|
||||
coordinatorPrincipalId: string
|
||||
@@ -411,13 +283,19 @@ describe('OrchestrationDb legacy contract storage', () => {
|
||||
coordinatorHandle: 'term_other',
|
||||
coordinatorPaneKey: 'tab_other:55555555-5555-4555-8555-555555555555'
|
||||
})
|
||||
).toThrow(/only its attested coordinator/)
|
||||
).toThrow(/attested coordinator may rebind/)
|
||||
|
||||
const bound = db!.bindRun({
|
||||
runId: state.adoptedRunId,
|
||||
coordinatorHandle: 'term_legacy_coord',
|
||||
coordinatorPaneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
|
||||
allowLegacyCompatibility: true
|
||||
legacyCoordinatorAuthority: {
|
||||
runId: state.adoptedRunId,
|
||||
principalId: state.coordinatorPrincipalId,
|
||||
terminalHandle: 'term_legacy_coord',
|
||||
paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
|
||||
consumerGeneration: 0
|
||||
}
|
||||
})
|
||||
expect(bound).toMatchObject({ coordinator_handle: 'term_legacy_coord' })
|
||||
expect(
|
||||
@@ -429,6 +307,99 @@ describe('OrchestrationDb legacy contract storage', () => {
|
||||
).toMatchObject({ coordinator_handle: 'term_legacy_coord' })
|
||||
})
|
||||
|
||||
it.each([
|
||||
['generation', { consumerGeneration: 1 }],
|
||||
['principal', { principalId: 'legacy_principal_wrong' }],
|
||||
['handle', { terminalHandle: 'term_wrong' }],
|
||||
['pane', { paneKey: 'tab_wrong:99999999-9999-4999-8999-999999999999' }]
|
||||
] as const)('rejects stale legacy coordinator %s proof inside Run binding', (_label, patch) => {
|
||||
const state = openAdoptedFixture()
|
||||
const before = db!.getRun(state.adoptedRunId)
|
||||
|
||||
expect(() =>
|
||||
db!.bindRun({
|
||||
runId: state.adoptedRunId,
|
||||
coordinatorHandle: 'term_legacy_coord',
|
||||
coordinatorPaneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
|
||||
legacyCoordinatorAuthority: {
|
||||
runId: state.adoptedRunId,
|
||||
principalId: state.coordinatorPrincipalId,
|
||||
terminalHandle: 'term_legacy_coord',
|
||||
paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
|
||||
consumerGeneration: 0,
|
||||
...patch
|
||||
}
|
||||
})
|
||||
).toThrow(/no longer has lifecycle authority/)
|
||||
expect(db!.getRun(state.adoptedRunId)).toEqual(before)
|
||||
})
|
||||
|
||||
it('revokes the legacy coordinator when a current coordinator takes over after settlement', () => {
|
||||
const state = openAdoptedFixture()
|
||||
db!.settleWorkerReport({
|
||||
taskId: state.fixture.legacyTaskId,
|
||||
dispatchId: state.fixture.legacyDispatchId,
|
||||
outcome: 'succeeded',
|
||||
result: 'done'
|
||||
})
|
||||
|
||||
expect(
|
||||
db!.bindRun({
|
||||
runId: state.adoptedRunId,
|
||||
coordinatorHandle: 'term_current_coord',
|
||||
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111'
|
||||
})
|
||||
).toMatchObject({ coordinator_handle: 'term_current_coord' })
|
||||
expect(db!.getLegacyCompatibilityPrincipal(state.coordinatorPrincipalId)?.status).toBe(
|
||||
'revoked'
|
||||
)
|
||||
expect(() =>
|
||||
db!.commitLegacyCompatibilityPrincipal({
|
||||
runId: state.adoptedRunId,
|
||||
role: 'coordinator',
|
||||
hostScope: 'local:runtime_1',
|
||||
terminalHandle: 'term_legacy_coord',
|
||||
paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444',
|
||||
launchTokenHash: 'coord_launch_hash',
|
||||
processIncarnation: 'process_coord'
|
||||
})
|
||||
).toThrow(/revoked/)
|
||||
})
|
||||
|
||||
it('promotes only mail addressed to the replaced legacy coordinator', () => {
|
||||
const state = openAdoptedFixture()
|
||||
const coordinatorMail = db!.insertMessage({
|
||||
runId: state.adoptedRunId,
|
||||
deliveryContract: 'legacy_direct',
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'coordinator outcome'
|
||||
})
|
||||
const workerMail = db!.insertMessage({
|
||||
runId: state.adoptedRunId,
|
||||
deliveryContract: 'legacy_direct',
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_other_worker',
|
||||
subject: 'worker-only guidance'
|
||||
})
|
||||
|
||||
db!.bindRun({
|
||||
runId: state.adoptedRunId,
|
||||
coordinatorHandle: 'term_current_coord',
|
||||
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111',
|
||||
takeoverLegacy: true
|
||||
})
|
||||
|
||||
expect(db!.getMessageById(coordinatorMail.id)).toMatchObject({
|
||||
to_handle: `run:${state.adoptedRunId}`,
|
||||
delivery_contract: 'current_delivery'
|
||||
})
|
||||
expect(db!.getMessageById(workerMail.id)).toMatchObject({
|
||||
to_handle: 'term_other_worker',
|
||||
delivery_contract: 'legacy_direct'
|
||||
})
|
||||
})
|
||||
|
||||
it('commits legacy messages, lifecycle effects, and invocation receipts exactly once', () => {
|
||||
const state = openAdoptedFixture()
|
||||
const params = {
|
||||
@@ -508,6 +479,65 @@ describe('OrchestrationDb legacy contract storage', () => {
|
||||
expect(db!.getLegacyCompatibilityPrincipal(state.workerPrincipalId)?.status).toBe('settled')
|
||||
})
|
||||
|
||||
it('reconstructs a read pre-takeover completion through its original legacy route', () => {
|
||||
const state = openAdoptedFixture()
|
||||
const taskId = db!.getDispatchContextById(state.fixture.legacyDispatchId)!.task_id
|
||||
const accepted = db!.insertMessage({
|
||||
runId: state.adoptedRunId,
|
||||
deliveryContract: 'legacy_direct',
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'Completed',
|
||||
body: 'accepted before takeover',
|
||||
type: 'worker_done'
|
||||
})
|
||||
db!.markAsRead([accepted.id])
|
||||
db!.settleWorkerReport({
|
||||
taskId,
|
||||
dispatchId: state.fixture.legacyDispatchId,
|
||||
outcome: 'succeeded',
|
||||
result: 'accepted before takeover'
|
||||
})
|
||||
db!.bindRun({
|
||||
runId: state.adoptedRunId,
|
||||
coordinatorHandle: 'term_current_coord',
|
||||
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111',
|
||||
takeoverLegacy: true
|
||||
})
|
||||
const beforeIds = db!.getInbox(100).map((message) => message.id)
|
||||
|
||||
const reconstructed = db!.commitLegacyLifecycleOperation({
|
||||
principalId: state.workerPrincipalId,
|
||||
operationKey: 'read_completion_after_takeover',
|
||||
method: 'orchestration.send',
|
||||
payloadHash: 'read_completion_after_takeover_payload',
|
||||
message: {
|
||||
existingId: accepted.id,
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'Completed',
|
||||
body: 'accepted before takeover',
|
||||
type: 'worker_done'
|
||||
},
|
||||
lifecycle: {
|
||||
kind: 'worker_report',
|
||||
taskId,
|
||||
outcome: 'succeeded',
|
||||
result: 'accepted before takeover'
|
||||
}
|
||||
})
|
||||
|
||||
expect(reconstructed).toMatchObject({
|
||||
duplicate: false,
|
||||
message: {
|
||||
id: accepted.id,
|
||||
to_handle: 'term_legacy_coord',
|
||||
delivery_contract: 'legacy_direct'
|
||||
},
|
||||
settlement: { action: 'settled', outcome: 'succeeded', duplicate: true }
|
||||
})
|
||||
expect(db!.getInbox(100).map((message) => message.id)).toEqual(beforeIds)
|
||||
})
|
||||
|
||||
it('rejects cross-cutover completion reconstruction for another recipient', () => {
|
||||
const state = openAdoptedFixture()
|
||||
const taskId = db!.getDispatchContextById(state.fixture.legacyDispatchId)!.task_id
|
||||
@@ -625,6 +655,38 @@ describe('OrchestrationDb legacy contract storage', () => {
|
||||
})
|
||||
).toHaveLength(2)
|
||||
|
||||
const inherited = db!.createQuestion({
|
||||
runId: state.adoptedRunId,
|
||||
dispatchId: state.fixture.legacyDispatchId,
|
||||
askerHandle: 'term_legacy_worker',
|
||||
question: 'Inherited?'
|
||||
})
|
||||
const sqlite = (db as unknown as { db: Database.Database }).db
|
||||
sqlite
|
||||
.prepare(
|
||||
`UPDATE messages
|
||||
SET from_handle = 'term_legacy_worker', to_handle = 'term_legacy_coord',
|
||||
delivery_contract = 'legacy_direct'
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run(inherited.message.id)
|
||||
const inheritedAsk = {
|
||||
...ask,
|
||||
operationKey: 'ask_inherited_1',
|
||||
payloadHash: 'ask_inherited_payload_1',
|
||||
question: 'Inherited?',
|
||||
options: [],
|
||||
existingQuestionId: inherited.message.id
|
||||
}
|
||||
const adopted = db!.commitLegacyAskOperation(inheritedAsk)
|
||||
const distinct = db!.commitLegacyAskOperation({
|
||||
...inheritedAsk,
|
||||
operationKey: 'ask_inherited_2',
|
||||
payloadHash: 'ask_inherited_payload_2'
|
||||
})
|
||||
expect(adopted.message.id).toBe(inherited.message.id)
|
||||
expect(distinct.message.id).not.toBe(inherited.message.id)
|
||||
|
||||
const reply = {
|
||||
principalId: state.coordinatorPrincipalId,
|
||||
operationKey: 'reply_invocation_1',
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
import { mkdtempSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import Database from '../../sqlite/sync-database'
|
||||
import { LEGACY_RUN_ID, OrchestrationDb } from './db'
|
||||
|
||||
export type LegacyStorageCutoverFixture = {
|
||||
dbPath: string
|
||||
currentRunId: string
|
||||
unrelatedRunId: string
|
||||
currentDispatchId: string
|
||||
legacyTaskId: string
|
||||
legacyDispatchId: string
|
||||
legacyGateId: string
|
||||
legacyMessageIds: string[]
|
||||
legacyQuestionId: string
|
||||
legacyDeliveryId: string
|
||||
rejectionMessageId: string
|
||||
lookalikeMessageId: string
|
||||
}
|
||||
|
||||
export function createLegacyStorageCutoverFixture(): {
|
||||
fixture: LegacyStorageCutoverFixture
|
||||
tempDir: string
|
||||
} {
|
||||
const tempDir = mkdtempSync(join(tmpdir(), 'orca-legacy-storage-'))
|
||||
const dbPath = join(tempDir, 'orchestration.db')
|
||||
const first = new OrchestrationDb(dbPath)
|
||||
const currentRun = first.createRun({
|
||||
objective: 'Current work',
|
||||
coordinatorHandle: 'term_current_coord',
|
||||
coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111'
|
||||
})
|
||||
const currentTask = first.createTask({ spec: 'current', runId: currentRun.id })
|
||||
const unrelatedRun = first.createRun({
|
||||
objective: 'Unrelated current work',
|
||||
coordinatorHandle: 'term_unrelated_coord',
|
||||
coordinatorPaneKey: 'tab_unrelated:55555555-5555-4555-8555-555555555555'
|
||||
})
|
||||
const currentDispatch = first.createDispatchContext(
|
||||
currentTask.id,
|
||||
'term_current_worker',
|
||||
'tab_current:22222222-2222-4222-9222-222222222222',
|
||||
'current_launch_hash'
|
||||
)
|
||||
first.insertMessage({
|
||||
runId: currentRun.id,
|
||||
from: 'term_current_worker',
|
||||
to: `run:${currentRun.id}`,
|
||||
subject: 'current mail'
|
||||
})
|
||||
|
||||
const legacyTask = first.createTask({
|
||||
spec: 'legacy',
|
||||
createdByTerminalHandle: 'term_legacy_coord'
|
||||
})
|
||||
first.createDispatchContext(
|
||||
legacyTask.id,
|
||||
'term_legacy_worker',
|
||||
'tab_legacy:33333333-3333-4333-8333-333333333333'
|
||||
)
|
||||
const legacyGate = first.createGate({
|
||||
taskId: legacyTask.id,
|
||||
question: 'Retained gate?'
|
||||
})
|
||||
first.resolveGate(legacyGate.id, 'continue')
|
||||
const retryDispatch = first.createDispatchContext(
|
||||
legacyTask.id,
|
||||
'term_legacy_worker',
|
||||
'tab_legacy:33333333-3333-4333-8333-333333333333'
|
||||
)
|
||||
const legacyMessages = [
|
||||
first.insertMessage({
|
||||
from: 'term_legacy_coord',
|
||||
to: 'term_legacy_worker',
|
||||
subject: 'read worker mail'
|
||||
}),
|
||||
first.insertMessage({
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'read coordinator mail'
|
||||
}),
|
||||
first.insertMessage({
|
||||
from: 'term_legacy_coord',
|
||||
to: 'term_legacy_worker',
|
||||
subject: 'second worker page'
|
||||
})
|
||||
]
|
||||
first.markAsRead(legacyMessages.map((message) => message.id))
|
||||
const question = first.createQuestion({
|
||||
runId: LEGACY_RUN_ID,
|
||||
dispatchId: retryDispatch.id,
|
||||
askerHandle: 'term_legacy_worker',
|
||||
question: 'Retained question?'
|
||||
})
|
||||
const rejection = first.insertMessage({
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'Rejected heartbeat',
|
||||
type: 'heartbeat',
|
||||
payload: JSON.stringify({ _orcaLifecycleRejection: { code: 'migration', reason: 'cutover' } })
|
||||
})
|
||||
const lookalike = first.insertMessage({
|
||||
from: 'term_legacy_worker',
|
||||
to: 'term_legacy_coord',
|
||||
subject: 'Ordinary legacy mail',
|
||||
payload: JSON.stringify({ '.orcaLifecycleRejection': 'not an audit marker' })
|
||||
})
|
||||
first.close()
|
||||
|
||||
const raw = new Database(dbPath)
|
||||
const legacyDeliveryId = 'delivery_legacy_outstanding'
|
||||
raw
|
||||
.prepare(
|
||||
`INSERT INTO deliveries (
|
||||
id, run_id, consumer_generation, message_ids, status
|
||||
) VALUES (?, ?, 0, ?, 'outstanding')`
|
||||
)
|
||||
.run(legacyDeliveryId, LEGACY_RUN_ID, JSON.stringify([legacyMessages[0].id]))
|
||||
raw.exec(`
|
||||
DROP INDEX IF EXISTS idx_messages_delivery_contract;
|
||||
DROP TABLE legacy_mail_receipts;
|
||||
DROP TABLE legacy_operation_receipts;
|
||||
DROP TABLE legacy_compatibility_principals;
|
||||
DROP TABLE legacy_adoptions;
|
||||
`)
|
||||
raw.pragma('user_version = 18')
|
||||
raw.close()
|
||||
|
||||
return {
|
||||
tempDir,
|
||||
fixture: {
|
||||
dbPath,
|
||||
currentRunId: currentRun.id,
|
||||
unrelatedRunId: unrelatedRun.id,
|
||||
currentDispatchId: currentDispatch.id,
|
||||
legacyTaskId: legacyTask.id,
|
||||
legacyDispatchId: retryDispatch.id,
|
||||
legacyGateId: legacyGate.id,
|
||||
legacyMessageIds: legacyMessages.map((message) => message.id),
|
||||
legacyQuestionId: question.message.id,
|
||||
legacyDeliveryId,
|
||||
rejectionMessageId: rejection.id,
|
||||
lookalikeMessageId: lookalike.id
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
-2
@@ -29,7 +29,8 @@ describe('legacy worker terminal recovery planning', () => {
|
||||
blockedPanes: [
|
||||
{
|
||||
worktreeId: 'repo::/workspace',
|
||||
paneKey: `tab-worker:${LEAF_ID}`
|
||||
paneKey: `tab-worker:${LEAF_ID}`,
|
||||
contractVersion: 0
|
||||
}
|
||||
],
|
||||
candidates: [
|
||||
@@ -50,7 +51,8 @@ describe('legacy worker terminal recovery planning', () => {
|
||||
blockedPanes: [
|
||||
{
|
||||
worktreeId: 'repo::/workspace',
|
||||
paneKey: `tab-worker:${LEAF_ID}`
|
||||
paneKey: `tab-worker:${LEAF_ID}`,
|
||||
contractVersion: 0
|
||||
}
|
||||
],
|
||||
candidates: [],
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { LegacyWorkerTerminalRecoveryRow } from './types'
|
||||
|
||||
export type LegacyWorkerTerminalRecoveryCandidate = {
|
||||
dispatchId: string
|
||||
contractVersion: number
|
||||
taskId: string
|
||||
worktreeId: string
|
||||
terminalHandle: string
|
||||
@@ -16,7 +17,7 @@ export type LegacyWorkerTerminalRecoveryCandidate = {
|
||||
}
|
||||
|
||||
export type LegacyWorkerTerminalRecoveryPlan = {
|
||||
blockedPanes: { worktreeId: string; paneKey: string }[]
|
||||
blockedPanes: { worktreeId: string; paneKey: string; contractVersion: number }[]
|
||||
candidates: LegacyWorkerTerminalRecoveryCandidate[]
|
||||
ambiguousDispatchIds: string[]
|
||||
}
|
||||
@@ -48,14 +49,21 @@ function countCandidateKeys(
|
||||
export function planLegacyWorkerTerminalRecovery(
|
||||
rows: readonly LegacyWorkerTerminalRecoveryRow[]
|
||||
): LegacyWorkerTerminalRecoveryPlan {
|
||||
const blockedPanes = new Map<string, { worktreeId: string; paneKey: string }>()
|
||||
const blockedPanes = new Map<
|
||||
string,
|
||||
{ worktreeId: string; paneKey: string; contractVersion: number }
|
||||
>()
|
||||
const parsedCandidates: LegacyWorkerTerminalRecoveryCandidate[] = []
|
||||
for (const row of rows) {
|
||||
const worktreeId = row.worktree_id?.trim()
|
||||
const paneKey = row.assignee_pane_key?.trim()
|
||||
const pane = paneKey ? parsePaneKey(paneKey) : null
|
||||
if (worktreeId && paneKey && pane) {
|
||||
blockedPanes.set(`${worktreeId}\0${paneKey}`, { worktreeId, paneKey })
|
||||
blockedPanes.set(`${worktreeId}\0${paneKey}`, {
|
||||
worktreeId,
|
||||
paneKey,
|
||||
contractVersion: row.contract_version
|
||||
})
|
||||
}
|
||||
const terminalHandle = row.assignee_handle?.trim()
|
||||
const workerHandle = row.agent_terminal_handle?.trim()
|
||||
@@ -74,6 +82,7 @@ export function planLegacyWorkerTerminalRecovery(
|
||||
}
|
||||
parsedCandidates.push({
|
||||
dispatchId: row.dispatch_id,
|
||||
contractVersion: row.contract_version,
|
||||
taskId: row.task_id,
|
||||
worktreeId,
|
||||
terminalHandle,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { OrchestrationDb } from './db'
|
||||
import { CURRENT_CONTRACT_VERSION, OrchestrationDb } from './db'
|
||||
|
||||
describe('OrchestrationDb worker Dispatch state', () => {
|
||||
let db: OrchestrationDb | undefined
|
||||
@@ -44,6 +44,14 @@ describe('OrchestrationDb worker Dispatch state', () => {
|
||||
status: 'dispatched',
|
||||
assignee_handle: 'term_worker'
|
||||
})
|
||||
expect(d.listLegacyWorkerTerminalRecoveryRows()).toEqual([
|
||||
expect.objectContaining({
|
||||
dispatch_id: started.dispatch.id,
|
||||
contract_version: CURRENT_CONTRACT_VERSION,
|
||||
worker_state: 'ready',
|
||||
agent_terminal_handle: 'term_worker'
|
||||
})
|
||||
])
|
||||
})
|
||||
|
||||
it('commits worker-start mutation acceptance with the starting Dispatch', () => {
|
||||
|
||||
@@ -53,6 +53,14 @@ export type RpcRequest = {
|
||||
orchestrationCompatibilityEvidence?: OrchestrationCompatibilityEvidence
|
||||
}
|
||||
|
||||
export type LegacyCoordinatorAuthorityProof = Readonly<{
|
||||
runId: string
|
||||
principalId: string | null
|
||||
terminalHandle: string
|
||||
paneKey: string
|
||||
consumerGeneration: number
|
||||
}>
|
||||
|
||||
export type RpcContext = {
|
||||
runtime: OrcaRuntimeService
|
||||
// Why: lets long-poll handlers release immediately on client disconnect instead of running down timeoutMs. See design doc §3.1.
|
||||
@@ -82,6 +90,8 @@ export type RpcContext = {
|
||||
}
|
||||
// Why: only the compatibility authority router can set this trusted scope; user params cannot bypass Run consumer binding.
|
||||
legacyCoordinatorRunId?: string
|
||||
legacyCoordinatorAuthority?: LegacyCoordinatorAuthorityProof
|
||||
revalidateLegacyCoordinator?: () => string
|
||||
// Why: federation pins the authenticated saved-environment caller without exposing its token to handlers or storage.
|
||||
authenticatedCallerFingerprint?: string
|
||||
pairing?: PairingRpcContext
|
||||
|
||||
@@ -46,6 +46,10 @@ function makeRuntime(ui: PersistedUIState = getDefaultUIState()): OrcaRuntimeSer
|
||||
updateUIState: vi.fn((updates: Partial<PersistedUIState>) => {
|
||||
currentUI = { ...currentUI, ...updates }
|
||||
return currentUI
|
||||
}),
|
||||
getOrchestrationDb: () => ({
|
||||
getLegacyAdoption: () => undefined,
|
||||
resolveLegacyWorkerCandidate: () => undefined
|
||||
})
|
||||
} as unknown as OrcaRuntimeService
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { RuntimeCapability } from '../../../shared/protocol-version'
|
||||
import type { TerminalStreamFrame } from '../../../shared/terminal-stream-protocol'
|
||||
import type { PairingRpcContext } from './core'
|
||||
|
||||
export type RpcDispatchStreamingOptions = {
|
||||
connectionId?: string
|
||||
signal?: AbortSignal
|
||||
clientId?: string
|
||||
pairedDeviceId?: string
|
||||
clientKind?: 'mobile' | 'runtime'
|
||||
clientCapabilities?: readonly RuntimeCapability[]
|
||||
pairing?: PairingRpcContext
|
||||
sendBinary?: (bytes: Uint8Array<ArrayBufferLike>) => boolean | void
|
||||
registerBinaryStreamHandler?: (
|
||||
streamId: number,
|
||||
handler: (frame: TerminalStreamFrame) => void
|
||||
) => () => void
|
||||
}
|
||||
@@ -10,13 +10,11 @@ import {
|
||||
isStreamingMethod,
|
||||
type RpcAnyMethod,
|
||||
type RpcEnvelopeMeta,
|
||||
type PairingRpcContext,
|
||||
type RpcRegistry,
|
||||
type RpcRequest,
|
||||
type RpcResponse
|
||||
} from './core'
|
||||
|
||||
import type { TerminalStreamFrame } from '../../../shared/terminal-stream-protocol'
|
||||
import type { FeatureInteractionId } from '../../../shared/feature-interactions'
|
||||
import {
|
||||
computerErrorData,
|
||||
@@ -29,20 +27,18 @@ import {
|
||||
import { ALL_RPC_METHODS } from './methods'
|
||||
import { emulatorProbe, emulatorProbeError } from '../../emulator/emulator-probe'
|
||||
import type { OrcaRuntimeService } from '../orca-runtime'
|
||||
import type { RuntimeCapability } from '../../../shared/protocol-version'
|
||||
import {
|
||||
OrchestrationMutationExecutor,
|
||||
authenticatedCallerFingerprint,
|
||||
getOrchestrationMutationExecutor,
|
||||
type OrchestrationMutationExecutor,
|
||||
type DurableMutationInvocation
|
||||
} from './orchestration-mutation-executor'
|
||||
import { orchestrationMigrationFence } from './orchestration-contract-fence'
|
||||
import { getRuntimeFeatureInteractionId } from './runtime-feature-interaction'
|
||||
import { recordRuntimeFeatureInteraction } from './runtime-feature-interaction'
|
||||
import { OrchestrationLegacyCompatibility } from './orchestration-legacy-compatibility'
|
||||
import type { RpcDispatchStreamingOptions } from './dispatcher-stream-options'
|
||||
|
||||
export type DispatcherOptions = {
|
||||
runtime: OrcaRuntimeService
|
||||
methods?: readonly RpcAnyMethod[]
|
||||
}
|
||||
export type DispatcherOptions = { runtime: OrcaRuntimeService; methods?: readonly RpcAnyMethod[] }
|
||||
|
||||
export class RpcDispatcher {
|
||||
private readonly runtime: OrcaRuntimeService
|
||||
@@ -53,7 +49,7 @@ export class RpcDispatcher {
|
||||
constructor({ runtime, methods = ALL_RPC_METHODS }: DispatcherOptions) {
|
||||
this.runtime = runtime
|
||||
this.registry = buildRegistry(methods)
|
||||
this.orchestrationMutations = new OrchestrationMutationExecutor(runtime)
|
||||
this.orchestrationMutations = getOrchestrationMutationExecutor(runtime)
|
||||
this.legacyOrchestration = new OrchestrationLegacyCompatibility(runtime)
|
||||
}
|
||||
|
||||
@@ -91,8 +87,7 @@ export class RpcDispatcher {
|
||||
)
|
||||
}
|
||||
|
||||
const isEmulator = request.method.startsWith('emulator.')
|
||||
if (isEmulator) {
|
||||
if (request.method.startsWith('emulator.')) {
|
||||
emulatorProbe(`rpc ${request.method}`, request.params)
|
||||
}
|
||||
try {
|
||||
@@ -105,8 +100,13 @@ export class RpcDispatcher {
|
||||
return successResponse(request.id, meta, compatibility.result)
|
||||
}
|
||||
const effectiveParams = compatibility.params ?? parsedParams.value
|
||||
const invoke = (mutation?: DurableMutationInvocation) =>
|
||||
method.handler(effectiveParams, {
|
||||
const legacyCoordinator = this.legacyOrchestration.createCoordinatorInvocation(
|
||||
request,
|
||||
compatibility.legacyCoordinatorAuthority
|
||||
)
|
||||
const invoke = (mutation?: DurableMutationInvocation) => {
|
||||
const legacyCoordinatorRunId = legacyCoordinator?.revalidate()
|
||||
return method.handler(effectiveParams, {
|
||||
runtime: this.runtime,
|
||||
signal: options?.signal,
|
||||
requestId: request.id,
|
||||
@@ -114,13 +114,22 @@ export class RpcDispatcher {
|
||||
authenticatedCallerFingerprint: authenticatedCallerFingerprint(request),
|
||||
recordMutationReceipt: mutation?.recordReceipt,
|
||||
orchestrationMutation: mutation?.identity,
|
||||
legacyCoordinatorRunId: compatibility.legacyCoordinatorRunId
|
||||
legacyCoordinatorRunId,
|
||||
legacyCoordinatorAuthority: legacyCoordinator?.authority,
|
||||
revalidateLegacyCoordinator: legacyCoordinator?.revalidate
|
||||
})
|
||||
}
|
||||
const result = await this.orchestrationMutations.run(request, effectiveParams, invoke)
|
||||
this.recordRuntimeFeatureInteraction(request.method, result, undefined, request.params)
|
||||
recordRuntimeFeatureInteraction(
|
||||
this.runtime,
|
||||
request.method,
|
||||
result,
|
||||
undefined,
|
||||
request.params
|
||||
)
|
||||
return successResponse(request.id, meta, result)
|
||||
} catch (error) {
|
||||
if (isEmulator) {
|
||||
if (request.method.startsWith('emulator.')) {
|
||||
emulatorProbeError(`rpc ${request.method}`, error, { params: request.params })
|
||||
}
|
||||
return this.mapError(request, meta, error)
|
||||
@@ -133,20 +142,7 @@ export class RpcDispatcher {
|
||||
async dispatchStreaming(
|
||||
request: RpcRequest,
|
||||
reply: (response: string) => void,
|
||||
options?: {
|
||||
connectionId?: string
|
||||
signal?: AbortSignal
|
||||
clientId?: string
|
||||
pairedDeviceId?: string
|
||||
clientKind?: 'mobile' | 'runtime'
|
||||
clientCapabilities?: readonly RuntimeCapability[]
|
||||
pairing?: PairingRpcContext
|
||||
sendBinary?: (bytes: Uint8Array<ArrayBufferLike>) => boolean | void
|
||||
registerBinaryStreamHandler?: (
|
||||
streamId: number,
|
||||
handler: (frame: TerminalStreamFrame) => void
|
||||
) => () => void
|
||||
}
|
||||
options?: RpcDispatchStreamingOptions
|
||||
): Promise<void> {
|
||||
const meta = this.meta()
|
||||
const method = this.registry.get(request.method)
|
||||
@@ -183,8 +179,13 @@ export class RpcDispatcher {
|
||||
return
|
||||
}
|
||||
const effectiveParams = compatibility.params ?? parsedParams.value
|
||||
const invoke = (mutation?: DurableMutationInvocation) =>
|
||||
method.handler(effectiveParams, {
|
||||
const legacyCoordinator = this.legacyOrchestration.createCoordinatorInvocation(
|
||||
request,
|
||||
compatibility.legacyCoordinatorAuthority
|
||||
)
|
||||
const invoke = (mutation?: DurableMutationInvocation) => {
|
||||
const legacyCoordinatorRunId = legacyCoordinator?.revalidate()
|
||||
return method.handler(effectiveParams, {
|
||||
runtime: this.runtime,
|
||||
signal: options?.signal,
|
||||
requestId: request.id,
|
||||
@@ -200,10 +201,19 @@ export class RpcDispatcher {
|
||||
pairing: options?.pairing,
|
||||
sendBinary: options?.sendBinary,
|
||||
registerBinaryStreamHandler: options?.registerBinaryStreamHandler,
|
||||
legacyCoordinatorRunId: compatibility.legacyCoordinatorRunId
|
||||
legacyCoordinatorRunId,
|
||||
legacyCoordinatorAuthority: legacyCoordinator?.authority,
|
||||
revalidateLegacyCoordinator: legacyCoordinator?.revalidate
|
||||
})
|
||||
}
|
||||
const result = await this.orchestrationMutations.run(request, effectiveParams, invoke)
|
||||
this.recordRuntimeFeatureInteraction(request.method, result, undefined, request.params)
|
||||
recordRuntimeFeatureInteraction(
|
||||
this.runtime,
|
||||
request.method,
|
||||
result,
|
||||
undefined,
|
||||
request.params
|
||||
)
|
||||
reply(JSON.stringify(successResponse(request.id, meta, result)))
|
||||
} catch (error) {
|
||||
reply(JSON.stringify(this.mapError(request, meta, error)))
|
||||
@@ -213,7 +223,8 @@ export class RpcDispatcher {
|
||||
|
||||
const recordedStreamingFeatureInteractions = new Set<FeatureInteractionId>()
|
||||
const emit = (result: unknown): void => {
|
||||
this.recordRuntimeFeatureInteraction(
|
||||
recordRuntimeFeatureInteraction(
|
||||
this.runtime,
|
||||
request.method,
|
||||
result,
|
||||
recordedStreamingFeatureInteractions,
|
||||
@@ -243,7 +254,8 @@ export class RpcDispatcher {
|
||||
},
|
||||
emit
|
||||
)
|
||||
this.recordRuntimeFeatureInteraction(
|
||||
recordRuntimeFeatureInteraction(
|
||||
this.runtime,
|
||||
request.method,
|
||||
result,
|
||||
recordedStreamingFeatureInteractions,
|
||||
@@ -310,25 +322,4 @@ export class RpcDispatcher {
|
||||
private meta(): RpcEnvelopeMeta {
|
||||
return { runtimeId: this.runtime.getRuntimeId() }
|
||||
}
|
||||
|
||||
private recordRuntimeFeatureInteraction(
|
||||
method: string,
|
||||
result: unknown,
|
||||
alreadyRecorded?: Set<FeatureInteractionId>,
|
||||
rawParams?: unknown
|
||||
): void {
|
||||
const id = getRuntimeFeatureInteractionId(method, result, rawParams)
|
||||
if (!id) {
|
||||
return
|
||||
}
|
||||
if (alreadyRecorded?.has(id)) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
this.runtime.recordFeatureInteraction(id)
|
||||
alreadyRecorded?.add(id)
|
||||
} catch {
|
||||
// Best-effort education state must not break runtime tools.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { z } from 'zod'
|
||||
import { defineMethod, type RpcMethod } from '../core'
|
||||
import { OptionalString, requiredString } from '../schemas'
|
||||
import { OptionalBoolean, OptionalString, requiredString } from '../schemas'
|
||||
import type { OrcaRuntimeService } from '../../orca-runtime'
|
||||
import { OrchestrationError } from '../../orchestration/orchestration-error'
|
||||
|
||||
@@ -11,7 +11,8 @@ const RunCreateParams = z.object({
|
||||
|
||||
const RunUseParams = z.object({
|
||||
id: requiredString('Missing --id'),
|
||||
from: requiredString('Missing coordinator terminal')
|
||||
from: requiredString('Missing coordinator terminal'),
|
||||
takeoverLegacy: OptionalBoolean
|
||||
})
|
||||
|
||||
const RunCurrentParams = z.object({ from: requiredString('Missing coordinator terminal') })
|
||||
@@ -51,7 +52,7 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [
|
||||
defineMethod({
|
||||
name: 'orchestration.runUse',
|
||||
params: RunUseParams,
|
||||
handler: (params, { runtime, legacyCoordinatorRunId }) => {
|
||||
handler: (params, { runtime, legacyCoordinatorAuthority }) => {
|
||||
const paneKey = requireCallerPane(runtime, params.from)
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const priorRun = db.getCurrentRunForPane(paneKey)
|
||||
@@ -59,7 +60,8 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [
|
||||
runId: params.id,
|
||||
coordinatorHandle: params.from,
|
||||
coordinatorPaneKey: paneKey,
|
||||
allowLegacyCompatibility: legacyCoordinatorRunId === params.id
|
||||
takeoverLegacy: params.takeoverLegacy,
|
||||
legacyCoordinatorAuthority
|
||||
})
|
||||
if (!run) {
|
||||
throw new OrchestrationError(
|
||||
|
||||
@@ -2,7 +2,12 @@
|
||||
import { z } from 'zod'
|
||||
import { defineMethod, type RpcMethod } from '../core'
|
||||
import { OptionalFiniteNumber, OptionalString, OptionalBoolean, requiredString } from '../schemas'
|
||||
import type { MessageType, MessagePriority, TaskStatus } from '../../orchestration/db'
|
||||
import {
|
||||
LEGACY_CONTRACT_VERSION,
|
||||
type MessageType,
|
||||
type MessagePriority,
|
||||
type TaskStatus
|
||||
} from '../../orchestration/db'
|
||||
import { MESSAGE_TYPES } from '../../orchestration/types'
|
||||
import { buildDispatchPreamble } from '../../orchestration/preamble'
|
||||
import { formatMessageBanner } from '../../orchestration/formatter'
|
||||
@@ -379,6 +384,49 @@ function resolveMessageRun(
|
||||
return { run, dispatchId: dispatch?.id ?? dispatchId }
|
||||
}
|
||||
|
||||
function legacyWorkerDeliveryContract(
|
||||
runtime: OrcaRuntimeService,
|
||||
runId: string | undefined,
|
||||
recipient: string
|
||||
): 'legacy_direct' | undefined {
|
||||
if (!runId) {
|
||||
return undefined
|
||||
}
|
||||
if (!recipient.startsWith('dispatch:')) {
|
||||
return runtime
|
||||
.getOrchestrationDb()
|
||||
.resolveLegacyWorkerCandidate({ runId, terminalHandle: recipient })
|
||||
? 'legacy_direct'
|
||||
: undefined
|
||||
}
|
||||
const dispatch = runtime
|
||||
.getOrchestrationDb()
|
||||
.getDispatchContextById(recipient.slice('dispatch:'.length))
|
||||
return dispatch?.run_id === runId &&
|
||||
dispatch.contract_version === LEGACY_CONTRACT_VERSION &&
|
||||
(dispatch.status === 'pending' || dispatch.status === 'dispatched')
|
||||
? 'legacy_direct'
|
||||
: undefined
|
||||
}
|
||||
|
||||
function interruptedAcknowledgedCheck(
|
||||
runId: string,
|
||||
acknowledged: string,
|
||||
reason: 'consumer_fenced' | 'outcome_unknown' | 'waiter_exists'
|
||||
): Record<string, unknown> {
|
||||
return {
|
||||
runId,
|
||||
deliveryId: null,
|
||||
messages: [],
|
||||
count: 0,
|
||||
acknowledged,
|
||||
timedOut: false,
|
||||
cancelled: false,
|
||||
connectionLost: false,
|
||||
waitInterrupted: reason
|
||||
}
|
||||
}
|
||||
|
||||
function rejectFederatedExplicitTarget(params: { to?: string; run?: string }): void {
|
||||
if (params.to || params.run) {
|
||||
throw new OrchestrationError(
|
||||
@@ -395,7 +443,10 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
defineMethod({
|
||||
name: 'orchestration.send',
|
||||
params: SendParams,
|
||||
handler: async (params, { runtime, orchestrationCapability }) => {
|
||||
handler: async (
|
||||
params,
|
||||
{ runtime, orchestrationCapability, legacyCoordinatorRunId, revalidateLegacyCoordinator }
|
||||
) => {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const from = params.from ?? 'unknown'
|
||||
// Why: caller-supplied pane fields are only compatibility metadata; lifecycle authority uses the runtime-observed pane plus capability.
|
||||
@@ -536,6 +587,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
'Coordinator-to-worker control mail cannot report worker lifecycle.'
|
||||
)
|
||||
}
|
||||
revalidateLegacyCoordinator?.()
|
||||
const relay = db.enqueueFederationRelay({
|
||||
dispatchId,
|
||||
direction: 'to_worker',
|
||||
@@ -562,6 +614,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
}
|
||||
}
|
||||
// Point-to-point — existing single-recipient behavior
|
||||
revalidateLegacyCoordinator?.()
|
||||
const msg = db.insertMessage({
|
||||
from,
|
||||
to,
|
||||
@@ -572,7 +625,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
threadId: params.threadId,
|
||||
payload: params.payload,
|
||||
senderPaneKey,
|
||||
runId: routing.run?.id
|
||||
runId: routing.run?.id,
|
||||
deliveryContract: legacyWorkerDeliveryContract(
|
||||
runtime,
|
||||
routing.run?.id ?? legacyCoordinatorRunId,
|
||||
to
|
||||
)
|
||||
})
|
||||
const dispatch = routing.dispatchId
|
||||
? db.getDispatchContextById(routing.dispatchId)
|
||||
@@ -629,6 +687,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
throw new Error(`No recipients resolved for group address: ${to}`)
|
||||
}
|
||||
|
||||
revalidateLegacyCoordinator?.()
|
||||
const threadId = params.threadId ?? `thread_${Date.now()}`
|
||||
const messages = handles.map((handle) =>
|
||||
db.insertMessage({
|
||||
@@ -641,7 +700,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
threadId,
|
||||
payload: params.payload,
|
||||
senderPaneKey,
|
||||
runId: routing.run?.id
|
||||
runId: routing.run?.id,
|
||||
deliveryContract: legacyWorkerDeliveryContract(
|
||||
runtime,
|
||||
routing.run?.id ?? legacyCoordinatorRunId,
|
||||
handle
|
||||
)
|
||||
})
|
||||
)
|
||||
for (const message of messages) {
|
||||
@@ -655,7 +719,16 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
defineMethod({
|
||||
name: 'orchestration.check',
|
||||
params: CheckParams,
|
||||
handler: async (params, { runtime, signal, legacyCoordinatorRunId }) => {
|
||||
handler: async (
|
||||
params,
|
||||
{
|
||||
runtime,
|
||||
signal,
|
||||
legacyCoordinatorRunId,
|
||||
revalidateLegacyCoordinator,
|
||||
recordMutationReceipt
|
||||
}
|
||||
) => {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const handle = params.terminal ?? 'unknown'
|
||||
const typeFilter = parseMessageTypes(params.types)
|
||||
@@ -682,6 +755,11 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
deliveryId: params.ack
|
||||
})
|
||||
: undefined
|
||||
if (acknowledged) {
|
||||
recordMutationReceipt?.(
|
||||
interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'outcome_unknown')
|
||||
)
|
||||
}
|
||||
if (params.peek || params.all || params.unread === false) {
|
||||
const history = db.getRunMailboxHistory(run.id, 100, typeFilter)
|
||||
const messages =
|
||||
@@ -745,14 +823,28 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
signal,
|
||||
exclusive: true
|
||||
})
|
||||
try {
|
||||
revalidateLegacyCoordinator?.()
|
||||
} catch (error) {
|
||||
if (!acknowledged) {
|
||||
throw error
|
||||
}
|
||||
return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'consumer_fenced')
|
||||
}
|
||||
const latestRun = db.getRun(run.id)
|
||||
if (!latestRun || latestRun.consumer_generation !== generation) {
|
||||
if (acknowledged) {
|
||||
return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'consumer_fenced')
|
||||
}
|
||||
throw new OrchestrationError(
|
||||
'consumer_fenced',
|
||||
'This mailbox consumer was replaced while waiting.'
|
||||
)
|
||||
}
|
||||
if (waitResult === 'waiter_exists') {
|
||||
if (acknowledged) {
|
||||
return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'waiter_exists')
|
||||
}
|
||||
throw new OrchestrationError(
|
||||
'waiter_exists',
|
||||
`Run ${run.id} already has an active actionable waiter.`
|
||||
@@ -1119,7 +1211,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
defineMethod({
|
||||
name: 'orchestration.dispatch',
|
||||
params: DispatchParams,
|
||||
handler: async (params, { runtime, legacyCoordinatorRunId }) => {
|
||||
handler: async (params, { runtime, legacyCoordinatorRunId, revalidateLegacyCoordinator }) => {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const task = db.getTask(params.task)
|
||||
if (!task) {
|
||||
@@ -1184,6 +1276,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
)
|
||||
}
|
||||
|
||||
revalidateLegacyCoordinator?.()
|
||||
const ctx = db.createDispatchContext(params.task, to, assigneePaneKey)
|
||||
const dispatchCapability = params.inject
|
||||
? db.mintDispatchCapability({
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { parsePaneKey } from '../../../shared/stable-pane-id'
|
||||
import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence'
|
||||
import type {
|
||||
DispatchContextRow,
|
||||
@@ -6,9 +5,14 @@ import type {
|
||||
LegacyPrincipalRole
|
||||
} from '../orchestration/types'
|
||||
import { OrchestrationError } from '../orchestration/orchestration-error'
|
||||
import type { OrcaRuntimeService } from '../orca-runtime'
|
||||
import type { OrcaRuntimeService, OrchestrationCompatibilityCallerAuthority } from '../orca-runtime'
|
||||
import { LEGACY_CONTRACT_VERSION } from '../orchestration/db'
|
||||
import type { RpcRequest } from './core'
|
||||
import {
|
||||
equivalentLegacyPaneKey,
|
||||
legacyCoordinatorReadOnly,
|
||||
legacyReadOnlyError
|
||||
} from './orchestration-legacy-process-identity'
|
||||
|
||||
export type LegacyPrincipalCandidate = {
|
||||
runId: string
|
||||
@@ -22,8 +26,9 @@ export function resolveAttestedLegacyPrincipal(args: {
|
||||
runtime: OrcaRuntimeService
|
||||
evidence?: OrchestrationCompatibilityEvidence
|
||||
candidate: LegacyPrincipalCandidate
|
||||
authority?: OrchestrationCompatibilityCallerAuthority
|
||||
}): LegacyCompatibilityPrincipalRow {
|
||||
const authority = verifyAttestedLegacyCandidate(args)
|
||||
const authority = args.authority ?? verifyAttestedLegacyCandidate(args)
|
||||
return args.runtime.getOrchestrationDb().commitLegacyCompatibilityPrincipal({
|
||||
runId: args.candidate.runId,
|
||||
dispatchId: args.candidate.dispatchId,
|
||||
@@ -36,7 +41,7 @@ export function resolveAttestedLegacyPrincipal(args: {
|
||||
}).principal
|
||||
}
|
||||
|
||||
function verifyAttestedLegacyCandidate(args: {
|
||||
export function verifyAttestedLegacyCandidate(args: {
|
||||
runtime: OrcaRuntimeService
|
||||
evidence?: OrchestrationCompatibilityEvidence
|
||||
candidate: LegacyPrincipalCandidate
|
||||
@@ -44,7 +49,7 @@ function verifyAttestedLegacyCandidate(args: {
|
||||
const authority = args.runtime.verifyOrchestrationCompatibilityCaller(args.evidence)
|
||||
if (
|
||||
!authority ||
|
||||
!equivalentPaneKey(args.candidate.paneKey, authority.paneKey) ||
|
||||
!equivalentLegacyPaneKey(args.candidate.paneKey, authority.paneKey) ||
|
||||
args.candidate.terminalHandle !== authority.terminalHandle
|
||||
) {
|
||||
throw legacyReadOnlyError()
|
||||
@@ -221,37 +226,6 @@ export class LegacyCompatibilityAuthority {
|
||||
})
|
||||
return principal
|
||||
}
|
||||
|
||||
resolveProvenCoordinatorScope(request: RpcRequest, requestedRunId?: string): string | undefined {
|
||||
const db = this.runtime.getOrchestrationDb()
|
||||
const adoption = db.getLegacyAdoption()
|
||||
const requestedRun = requestedRunId ?? adoption?.adopted_run_id
|
||||
if (!adoption || requestedRun !== adoption.adopted_run_id) {
|
||||
return undefined
|
||||
}
|
||||
const candidate = db.resolveLegacyCoordinatorCandidate({
|
||||
runId: adoption.adopted_run_id,
|
||||
terminalHandle: request.orchestrationCompatibilityEvidence?.terminalHandle,
|
||||
paneKey: request.orchestrationCompatibilityEvidence?.paneKey
|
||||
})
|
||||
if (!candidate) {
|
||||
if (request.orchestrationCompatibilityEvidence) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
verifyAttestedLegacyCandidate({
|
||||
runtime: this.runtime,
|
||||
evidence: request.orchestrationCompatibilityEvidence,
|
||||
candidate: {
|
||||
runId: adoption.adopted_run_id,
|
||||
role: 'coordinator',
|
||||
terminalHandle: candidate.terminalHandle,
|
||||
paneKey: candidate.paneKey
|
||||
}
|
||||
})
|
||||
return adoption.adopted_run_id
|
||||
}
|
||||
}
|
||||
|
||||
function candidateFromDispatch(dispatch: DispatchContextRow): LegacyPrincipalCandidate {
|
||||
@@ -281,31 +255,3 @@ function candidateFromPrincipal(
|
||||
paneKey: principal.pane_key
|
||||
}
|
||||
}
|
||||
|
||||
function equivalentPaneKey(a: string | null | undefined, b: string | null | undefined): boolean {
|
||||
if (!a || !b) {
|
||||
return false
|
||||
}
|
||||
if (a === b) {
|
||||
return true
|
||||
}
|
||||
const aLeaf = parsePaneKey(a)?.leafId
|
||||
const bLeaf = parsePaneKey(b)?.leafId
|
||||
return Boolean(aLeaf && bLeaf && aLeaf === bLeaf)
|
||||
}
|
||||
|
||||
function legacyReadOnlyError(): OrchestrationError {
|
||||
return new OrchestrationError(
|
||||
'legacy_read_only',
|
||||
'This retained legacy assignment could not prove authority from its original live process. No effects were applied.',
|
||||
{ effectsApplied: false }
|
||||
)
|
||||
}
|
||||
|
||||
function legacyCoordinatorReadOnly(): OrchestrationError {
|
||||
return new OrchestrationError(
|
||||
'legacy_read_only',
|
||||
'This retained legacy coordinator could not prove its original process identity. No effects were applied.',
|
||||
{ effectsApplied: false }
|
||||
)
|
||||
}
|
||||
|
||||
@@ -300,6 +300,82 @@ describe('legacy compatibility through RpcDispatcher', () => {
|
||||
expect(harness.notify).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('replays an A-era completion without touching a newer current attempt', async () => {
|
||||
const harness = createHarness()
|
||||
const payload = JSON.stringify({
|
||||
taskId: harness.taskId,
|
||||
dispatchId: harness.dispatchId
|
||||
})
|
||||
const completion = harness.db.insertMessage({
|
||||
runId: harness.adoptedRunId,
|
||||
from: WORKER_HANDLE,
|
||||
to: COORDINATOR_HANDLE,
|
||||
subject: 'Failed: pinned A treated this as complete',
|
||||
body: 'persisted result',
|
||||
type: 'worker_done',
|
||||
payload,
|
||||
senderPaneKey: WORKER_PANE,
|
||||
deliveryContract: 'legacy_direct'
|
||||
})
|
||||
harness.db.settleWorkerReport({
|
||||
taskId: harness.taskId,
|
||||
dispatchId: harness.dispatchId,
|
||||
outcome: 'succeeded',
|
||||
result: 'persisted result'
|
||||
})
|
||||
harness.db.commitLegacyCompatibilityPrincipal({
|
||||
runId: harness.adoptedRunId,
|
||||
dispatchId: harness.dispatchId,
|
||||
role: 'worker',
|
||||
hostScope: JSON.stringify({ kind: 'local', hostId: 'local' }),
|
||||
terminalHandle: WORKER_HANDLE,
|
||||
paneKey: WORKER_PANE,
|
||||
launchTokenHash: createHash('sha256').update('worker-token').digest('hex'),
|
||||
processIncarnation: 'process-1'
|
||||
})
|
||||
harness.db.updateTaskStatus(harness.taskId, 'ready')
|
||||
const currentDispatch = harness.db.createDispatchContext(
|
||||
harness.taskId,
|
||||
'term_current_worker',
|
||||
'tab_current_worker:77777777-7777-4777-8777-777777777777',
|
||||
'current-launch-hash'
|
||||
)
|
||||
const currentTaskBefore = harness.db.getTask(harness.taskId)
|
||||
const currentDispatchBefore = harness.db.getDispatchContextById(currentDispatch.id)
|
||||
const before = counts(harness.db)
|
||||
|
||||
const response = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.send',
|
||||
{
|
||||
from: WORKER_HANDLE,
|
||||
to: COORDINATOR_HANDLE,
|
||||
subject: completion.subject,
|
||||
body: completion.body,
|
||||
type: 'worker_done',
|
||||
payload
|
||||
},
|
||||
evidence('worker'),
|
||||
'reconstruct-pinned-a-completion'
|
||||
)
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
message: { id: completion.id },
|
||||
lifecycle: { action: 'settled', outcome: 'succeeded', duplicate: true }
|
||||
}
|
||||
})
|
||||
expect(harness.db.getTask(harness.taskId)).toEqual(currentTaskBefore)
|
||||
expect(harness.db.getDispatchContextById(harness.dispatchId)?.status).toBe('completed')
|
||||
expect(harness.db.getDispatchContextById(currentDispatch.id)).toEqual(currentDispatchBefore)
|
||||
expect(counts(harness.db)).toEqual({
|
||||
...before,
|
||||
legacy_operation_receipts: before.legacy_operation_receipts + 1
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects a legacy lifecycle recipient outside the adopted Run with zero effects', async () => {
|
||||
const harness = createHarness()
|
||||
const before = counts(harness.db)
|
||||
@@ -367,6 +443,35 @@ describe('legacy compatibility through RpcDispatcher', () => {
|
||||
expect(harness.notify).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps distinct compatibility ask invocations on distinct questions', async () => {
|
||||
const harness = createHarness()
|
||||
const ask = {
|
||||
from: WORKER_HANDLE,
|
||||
to: COORDINATOR_HANDLE,
|
||||
question: 'Same question?',
|
||||
options: ['yes', 'no'],
|
||||
timeoutMs: 1
|
||||
}
|
||||
const first = await harness.dispatcher.dispatch(
|
||||
request('orchestration.ask', ask, evidence('worker'), 'ask-one')
|
||||
)
|
||||
const replay = await harness.dispatcher.dispatch(
|
||||
request('orchestration.ask', ask, evidence('worker'), 'ask-one')
|
||||
)
|
||||
const second = await harness.dispatcher.dispatch(
|
||||
request('orchestration.ask', ask, evidence('worker'), 'ask-two')
|
||||
)
|
||||
|
||||
expect(first).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: false } } })
|
||||
expect(replay).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: true } } })
|
||||
expect(second).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: false } } })
|
||||
const firstId = (first as { result: { messageId: string } }).result.messageId
|
||||
const replayId = (replay as { result: { messageId: string } }).result.messageId
|
||||
const secondId = (second as { result: { messageId: string } }).result.messageId
|
||||
expect(replayId).toBe(firstId)
|
||||
expect(secondId).not.toBe(firstId)
|
||||
})
|
||||
|
||||
it('does not infer an outcome for a current Dispatch when legacy adoption exists', async () => {
|
||||
const harness = createHarness()
|
||||
const run = harness.db.createRun({
|
||||
@@ -524,6 +629,56 @@ describe('legacy compatibility through RpcDispatcher', () => {
|
||||
expect(harness.notify).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('advertises only Run-addressed current delivery to a legacy coordinator', async () => {
|
||||
const harness = createHarness()
|
||||
harness.db.insertMessage({
|
||||
runId: harness.adoptedRunId,
|
||||
from: 'term_current_worker',
|
||||
to: WORKER_HANDLE,
|
||||
subject: 'worker-only current mail',
|
||||
deliveryContract: 'current_delivery'
|
||||
})
|
||||
const workerOnly = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{ terminal: COORDINATOR_HANDLE },
|
||||
evidence('coordinator'),
|
||||
'worker-current-mail'
|
||||
)
|
||||
)
|
||||
expect(workerOnly).toMatchObject({
|
||||
ok: true,
|
||||
result: { legacyCompatibility: { currentDelivery: undefined } }
|
||||
})
|
||||
|
||||
harness.db.insertMessage({
|
||||
runId: harness.adoptedRunId,
|
||||
from: 'term_current_worker',
|
||||
to: `run:${harness.adoptedRunId}`,
|
||||
subject: 'Run current mail',
|
||||
deliveryContract: 'current_delivery'
|
||||
})
|
||||
const runMail = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{ terminal: COORDINATOR_HANDLE },
|
||||
evidence('coordinator'),
|
||||
'run-current-mail'
|
||||
)
|
||||
)
|
||||
expect(runMail).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
legacyCompatibility: {
|
||||
currentDelivery: {
|
||||
runId: harness.adoptedRunId,
|
||||
checkCommand: expect.stringContaining(`--run ${harness.adoptedRunId}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it.each(['dispatch', 'websocket'] as const)(
|
||||
'%s binds an attested coordinator once and rejects contradictory proof before binding',
|
||||
async (transport) => {
|
||||
@@ -588,4 +743,42 @@ describe('legacy compatibility through RpcDispatcher', () => {
|
||||
expect(harness.db.getRun(harness.adoptedRunId)?.consumer_generation).toBe(0)
|
||||
expect(counts(harness.db)).toEqual(before)
|
||||
})
|
||||
|
||||
it('rejects stale legacy coordinator proof after current takeover', async () => {
|
||||
const harness = createHarness()
|
||||
const principal = harness.db.commitLegacyCompatibilityPrincipal({
|
||||
runId: harness.adoptedRunId,
|
||||
role: 'coordinator',
|
||||
hostScope: JSON.stringify({ kind: 'local', hostId: 'local' }),
|
||||
terminalHandle: COORDINATOR_HANDLE,
|
||||
paneKey: COORDINATOR_PANE,
|
||||
launchTokenHash: createHash('sha256').update('coordinator-token').digest('hex'),
|
||||
processIncarnation: 'process-1'
|
||||
}).principal
|
||||
harness.db.settleWorkerReport({
|
||||
taskId: harness.taskId,
|
||||
dispatchId: harness.dispatchId,
|
||||
outcome: 'succeeded',
|
||||
result: 'done'
|
||||
})
|
||||
harness.db.bindRun({
|
||||
runId: harness.adoptedRunId,
|
||||
coordinatorHandle: 'term_current_coord',
|
||||
coordinatorPaneKey: 'tab_current:55555555-5555-4555-8555-555555555555'
|
||||
})
|
||||
expect(harness.db.getLegacyCompatibilityPrincipal(principal.id)?.status).toBe('revoked')
|
||||
const before = counts(harness.db)
|
||||
|
||||
const response = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.taskList',
|
||||
{},
|
||||
evidence('coordinator'),
|
||||
'stale-coordinator-after-takeover'
|
||||
)
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
|
||||
expect(counts(harness.db)).toEqual(before)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { RpcRequest } from './core'
|
||||
import type { LegacyCoordinatorAuthorityProof, RpcRequest } from './core'
|
||||
import type { OrcaRuntimeService } from '../orca-runtime'
|
||||
import { LegacyCompatibilityAuthority } from './orchestration-legacy-authority'
|
||||
import { handleLegacyLifecycleSend } from './orchestration-legacy-lifecycle'
|
||||
@@ -10,6 +10,7 @@ import type {
|
||||
LegacyReplyParams,
|
||||
LegacySendParams
|
||||
} from './orchestration-legacy-operation'
|
||||
import { LegacyCoordinatorAuthority } from './orchestration-legacy-coordinator-authority'
|
||||
|
||||
const COORDINATOR_PREFLIGHT_METHODS = new Set([
|
||||
'orchestration.taskCreate',
|
||||
@@ -19,19 +20,31 @@ const COORDINATOR_PREFLIGHT_METHODS = new Set([
|
||||
'orchestration.gateCreate',
|
||||
'orchestration.gateResolve',
|
||||
'orchestration.runUse',
|
||||
'orchestration.send',
|
||||
'orchestration.check',
|
||||
'orchestration.reply'
|
||||
])
|
||||
|
||||
export type LegacyCompatibilityRoute =
|
||||
| { handled: true; result: unknown }
|
||||
| { handled: false; params?: unknown; legacyCoordinatorRunId?: string }
|
||||
| {
|
||||
handled: false
|
||||
params?: unknown
|
||||
legacyCoordinatorAuthority?: LegacyCoordinatorAuthorityProof
|
||||
}
|
||||
|
||||
export type LegacyCoordinatorInvocation = Readonly<{
|
||||
authority: LegacyCoordinatorAuthorityProof
|
||||
revalidate: () => string
|
||||
}>
|
||||
|
||||
export class OrchestrationLegacyCompatibility {
|
||||
private readonly authority: LegacyCompatibilityAuthority
|
||||
private readonly coordinatorAuthority: LegacyCoordinatorAuthority
|
||||
|
||||
constructor(private readonly runtime: OrcaRuntimeService) {
|
||||
this.authority = new LegacyCompatibilityAuthority(runtime)
|
||||
this.coordinatorAuthority = new LegacyCoordinatorAuthority(runtime)
|
||||
}
|
||||
|
||||
async tryHandle(
|
||||
@@ -50,18 +63,40 @@ export class OrchestrationLegacyCompatibility {
|
||||
return { handled: false }
|
||||
}
|
||||
const values = params as Record<string, unknown>
|
||||
if (request.method === 'orchestration.runUse' && values.takeoverLegacy === true) {
|
||||
return { handled: false }
|
||||
}
|
||||
const requestedRunId =
|
||||
request.method === 'orchestration.runUse' ? stringValue(values.id) : stringValue(values.run)
|
||||
const runId = this.authority.resolveProvenCoordinatorScope(request, requestedRunId)
|
||||
return runId
|
||||
const authority = this.coordinatorAuthority.resolve(request, requestedRunId)
|
||||
return authority
|
||||
? {
|
||||
handled: false,
|
||||
params: { ...values, run: runId },
|
||||
legacyCoordinatorRunId: runId
|
||||
params: { ...values, run: authority.runId },
|
||||
legacyCoordinatorAuthority: authority
|
||||
}
|
||||
: { handled: false }
|
||||
}
|
||||
|
||||
revalidateCoordinatorAuthority(
|
||||
request: RpcRequest,
|
||||
proof: LegacyCoordinatorAuthorityProof
|
||||
): string {
|
||||
return this.coordinatorAuthority.revalidate(request, proof)
|
||||
}
|
||||
|
||||
createCoordinatorInvocation(
|
||||
request: RpcRequest,
|
||||
authority?: LegacyCoordinatorAuthorityProof
|
||||
): LegacyCoordinatorInvocation | undefined {
|
||||
return authority
|
||||
? {
|
||||
authority,
|
||||
revalidate: () => this.revalidateCoordinatorAuthority(request, authority)
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
|
||||
private async route(
|
||||
request: RpcRequest,
|
||||
params: unknown,
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
import type { OrcaRuntimeService, OrchestrationCompatibilityCallerAuthority } from '../orca-runtime'
|
||||
import type { LegacyCompatibilityPrincipalRow } from '../orchestration/types'
|
||||
import type { LegacyCoordinatorAuthorityProof, RpcRequest } from './core'
|
||||
import {
|
||||
resolveAttestedLegacyPrincipal,
|
||||
verifyAttestedLegacyCandidate,
|
||||
type LegacyPrincipalCandidate
|
||||
} from './orchestration-legacy-authority'
|
||||
import {
|
||||
equivalentLegacyPaneKey,
|
||||
legacyCoordinatorReadOnly
|
||||
} from './orchestration-legacy-process-identity'
|
||||
|
||||
export class LegacyCoordinatorAuthority {
|
||||
constructor(private readonly runtime: OrcaRuntimeService) {}
|
||||
|
||||
resolve(
|
||||
request: RpcRequest,
|
||||
requestedRunId?: string
|
||||
): LegacyCoordinatorAuthorityProof | undefined {
|
||||
const db = this.runtime.getOrchestrationDb()
|
||||
const adoption = db.getLegacyAdoption()
|
||||
const requestedRun = requestedRunId ?? adoption?.adopted_run_id
|
||||
if (!adoption || requestedRun !== adoption.adopted_run_id) {
|
||||
return undefined
|
||||
}
|
||||
const candidate = db.resolveLegacyCoordinatorCandidate({
|
||||
runId: adoption.adopted_run_id,
|
||||
terminalHandle: request.orchestrationCompatibilityEvidence?.terminalHandle,
|
||||
paneKey: request.orchestrationCompatibilityEvidence?.paneKey
|
||||
})
|
||||
if (!candidate) {
|
||||
if (request.orchestrationCompatibilityEvidence) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
const existing = db.getLegacyCoordinatorPrincipal(adoption.adopted_run_id)
|
||||
const proofCandidate = existing
|
||||
? this.candidate(adoption.adopted_run_id, existing)
|
||||
: this.candidate(adoption.adopted_run_id, candidate)
|
||||
const attestation = verifyAttestedLegacyCandidate({
|
||||
runtime: this.runtime,
|
||||
evidence: request.orchestrationCompatibilityEvidence,
|
||||
candidate: proofCandidate
|
||||
})
|
||||
const run = db.getRun(adoption.adopted_run_id)
|
||||
if (!run || !this.bindingMatches(run, proofCandidate)) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
if (existing && !this.principalMatchesAttestation(existing, attestation)) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
const principal = existing
|
||||
? existing
|
||||
: request.method === 'orchestration.runUse'
|
||||
? resolveAttestedLegacyPrincipal({
|
||||
runtime: this.runtime,
|
||||
evidence: request.orchestrationCompatibilityEvidence,
|
||||
candidate: proofCandidate,
|
||||
authority: attestation
|
||||
})
|
||||
: undefined
|
||||
return {
|
||||
runId: run.id,
|
||||
principalId: principal?.id ?? null,
|
||||
terminalHandle: proofCandidate.terminalHandle,
|
||||
paneKey: proofCandidate.paneKey,
|
||||
consumerGeneration: run.consumer_generation
|
||||
}
|
||||
}
|
||||
|
||||
revalidate(request: RpcRequest, proof: LegacyCoordinatorAuthorityProof): string {
|
||||
const db = this.runtime.getOrchestrationDb()
|
||||
const adoption = db.getLegacyAdoption()
|
||||
const run = db.getRun(proof.runId)
|
||||
if (
|
||||
adoption?.adopted_run_id !== proof.runId ||
|
||||
!run ||
|
||||
run.consumer_generation !== proof.consumerGeneration
|
||||
) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
const candidate = this.candidate(proof.runId, proof)
|
||||
const principal = db.getLegacyCoordinatorPrincipal(proof.runId)
|
||||
if (proof.principalId) {
|
||||
if (
|
||||
principal?.id !== proof.principalId ||
|
||||
principal.status !== 'committed' ||
|
||||
principal.terminal_handle !== proof.terminalHandle ||
|
||||
!equivalentLegacyPaneKey(principal.pane_key, proof.paneKey)
|
||||
) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
} else if (
|
||||
principal &&
|
||||
(principal.status !== 'committed' ||
|
||||
principal.terminal_handle !== proof.terminalHandle ||
|
||||
!equivalentLegacyPaneKey(principal.pane_key, proof.paneKey))
|
||||
) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
const attestation = verifyAttestedLegacyCandidate({
|
||||
runtime: this.runtime,
|
||||
evidence: request.orchestrationCompatibilityEvidence,
|
||||
candidate
|
||||
})
|
||||
if (principal && !this.principalMatchesAttestation(principal, attestation)) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
if (!this.bindingMatches(run, candidate)) {
|
||||
throw legacyCoordinatorReadOnly()
|
||||
}
|
||||
return run.id
|
||||
}
|
||||
|
||||
private candidate(
|
||||
runId: string,
|
||||
identity: {
|
||||
terminalHandle?: string
|
||||
terminal_handle?: string
|
||||
paneKey?: string
|
||||
pane_key?: string
|
||||
}
|
||||
): LegacyPrincipalCandidate {
|
||||
return {
|
||||
runId,
|
||||
role: 'coordinator',
|
||||
terminalHandle: identity.terminalHandle ?? (identity.terminal_handle as string),
|
||||
paneKey: identity.paneKey ?? (identity.pane_key as string)
|
||||
}
|
||||
}
|
||||
|
||||
private bindingMatches(
|
||||
run: { coordinator_handle: string | null; coordinator_pane_key: string | null },
|
||||
candidate: LegacyPrincipalCandidate
|
||||
): boolean {
|
||||
return (
|
||||
!run.coordinator_pane_key ||
|
||||
(run.coordinator_handle === candidate.terminalHandle &&
|
||||
equivalentLegacyPaneKey(run.coordinator_pane_key, candidate.paneKey))
|
||||
)
|
||||
}
|
||||
|
||||
private principalMatchesAttestation(
|
||||
principal: LegacyCompatibilityPrincipalRow,
|
||||
attestation: OrchestrationCompatibilityCallerAuthority
|
||||
): boolean {
|
||||
return (
|
||||
principal.status === 'committed' &&
|
||||
principal.host_scope === JSON.stringify(attestation.hostScope) &&
|
||||
principal.terminal_handle === attestation.terminalHandle &&
|
||||
equivalentLegacyPaneKey(principal.pane_key, attestation.paneKey) &&
|
||||
principal.launch_token_hash === attestation.launchTokenHash &&
|
||||
principal.process_incarnation === attestation.processIncarnation
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,849 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { ORCHESTRATION_CONTRACT_VERSION } from '../../../shared/protocol-version'
|
||||
import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence'
|
||||
import Database from '../../sqlite/sync-database'
|
||||
import { OrcaRuntimeService } from '../orca-runtime'
|
||||
import { OrchestrationDb } from '../orchestration/db'
|
||||
import type { RpcRequest, RpcResponse } from './core'
|
||||
import { RpcDispatcher } from './dispatcher'
|
||||
import { ORCHESTRATION_METHODS } from './methods/orchestration'
|
||||
|
||||
const COORDINATOR_HANDLE = 'term_legacy_coord'
|
||||
const COORDINATOR_PANE = 'tab_coord:44444444-4444-4444-8444-444444444444'
|
||||
const WORKER_HANDLE = 'term_legacy_worker'
|
||||
const WORKER_PANE = 'tab_worker:33333333-3333-4333-8333-333333333333'
|
||||
|
||||
type Transport = 'dispatch' | 'websocket'
|
||||
|
||||
type Harness = {
|
||||
db: OrchestrationDb
|
||||
dispatcher: RpcDispatcher
|
||||
runtime: OrcaRuntimeService
|
||||
adoptedRunId: string
|
||||
taskId: string
|
||||
dispatchId: string
|
||||
}
|
||||
|
||||
const tempDirs: string[] = []
|
||||
const databases: OrchestrationDb[] = []
|
||||
|
||||
afterEach(() => {
|
||||
databases.splice(0).forEach((database) => database.close())
|
||||
tempDirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }))
|
||||
})
|
||||
|
||||
function createHarness(): Harness {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-legacy-coordinator-race-'))
|
||||
tempDirs.push(dir)
|
||||
const dbPath = join(dir, 'orchestration.db')
|
||||
const before = new OrchestrationDb(dbPath)
|
||||
const task = before.createTask({
|
||||
spec: 'legacy assignment',
|
||||
createdByTerminalHandle: COORDINATOR_HANDLE
|
||||
})
|
||||
const dispatch = before.createDispatchContext(task.id, WORKER_HANDLE, WORKER_PANE)
|
||||
before.close()
|
||||
|
||||
const raw = new Database(dbPath)
|
||||
raw.exec(`
|
||||
UPDATE dispatch_contexts SET process_incarnation = 'process-1';
|
||||
DROP INDEX IF EXISTS idx_messages_delivery_contract;
|
||||
DROP TABLE legacy_mail_receipts;
|
||||
DROP TABLE legacy_operation_receipts;
|
||||
DROP TABLE legacy_compatibility_principals;
|
||||
DROP TABLE legacy_adoptions;
|
||||
`)
|
||||
raw.pragma('user_version = 18')
|
||||
raw.close()
|
||||
|
||||
const db = new OrchestrationDb(dbPath)
|
||||
databases.push(db)
|
||||
const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string
|
||||
const runtime = new OrcaRuntimeService()
|
||||
runtime.setOrchestrationDb(db)
|
||||
vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) =>
|
||||
handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : handle === WORKER_HANDLE ? WORKER_PANE : null
|
||||
)
|
||||
vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation(
|
||||
(compatibilityEvidence) => {
|
||||
const coordinator =
|
||||
compatibilityEvidence?.terminalHandle === COORDINATOR_HANDLE &&
|
||||
compatibilityEvidence.paneKey === COORDINATOR_PANE &&
|
||||
compatibilityEvidence.launchToken === 'coordinator-token'
|
||||
const worker =
|
||||
compatibilityEvidence?.terminalHandle === WORKER_HANDLE &&
|
||||
compatibilityEvidence.paneKey === WORKER_PANE &&
|
||||
compatibilityEvidence.launchToken === 'worker-token'
|
||||
return coordinator || worker
|
||||
? {
|
||||
hostScope: { kind: 'local', hostId: 'local' },
|
||||
terminalHandle: coordinator ? COORDINATOR_HANDLE : WORKER_HANDLE,
|
||||
paneKey: coordinator ? COORDINATOR_PANE : WORKER_PANE,
|
||||
processIncarnation: 'process-1',
|
||||
launchTokenHash: createHash('sha256')
|
||||
.update(coordinator ? 'coordinator-token' : 'worker-token')
|
||||
.digest('hex')
|
||||
}
|
||||
: null
|
||||
}
|
||||
)
|
||||
return {
|
||||
db,
|
||||
dispatcher: new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }),
|
||||
runtime,
|
||||
adoptedRunId,
|
||||
taskId: task.id,
|
||||
dispatchId: dispatch.id
|
||||
}
|
||||
}
|
||||
|
||||
function evidence(
|
||||
role: 'coordinator' | 'worker' = 'coordinator'
|
||||
): OrchestrationCompatibilityEvidence {
|
||||
return {
|
||||
terminalHandle: role === 'coordinator' ? COORDINATOR_HANDLE : WORKER_HANDLE,
|
||||
paneKey: role === 'coordinator' ? COORDINATOR_PANE : WORKER_PANE,
|
||||
launchToken: role === 'coordinator' ? 'coordinator-token' : 'worker-token'
|
||||
}
|
||||
}
|
||||
|
||||
function request(
|
||||
method: string,
|
||||
params: unknown,
|
||||
invocationId: string,
|
||||
callerEvidence = evidence()
|
||||
): RpcRequest {
|
||||
return {
|
||||
id: `rpc_${invocationId}`,
|
||||
authToken: 'caller-token',
|
||||
method,
|
||||
params,
|
||||
orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION,
|
||||
orchestrationRequestId: invocationId,
|
||||
compatibilityInvocationId: invocationId,
|
||||
orchestrationCompatibilityEvidence: callerEvidence
|
||||
}
|
||||
}
|
||||
|
||||
async function invoke(
|
||||
dispatcher: RpcDispatcher,
|
||||
rpcRequest: RpcRequest,
|
||||
transport: Transport
|
||||
): Promise<RpcResponse> {
|
||||
if (transport === 'dispatch') {
|
||||
return await dispatcher.dispatch(rpcRequest)
|
||||
}
|
||||
const replies: string[] = []
|
||||
await dispatcher.dispatchStreaming(rpcRequest, (reply) => replies.push(reply))
|
||||
expect(replies).toHaveLength(1)
|
||||
return JSON.parse(replies[0]) as RpcResponse
|
||||
}
|
||||
|
||||
function installTakeoverDuringMutationPreflight(harness: Harness): void {
|
||||
const originalBegin = harness.db.beginMutationReceipt.bind(harness.db)
|
||||
vi.spyOn(harness.db, 'beginMutationReceipt').mockImplementation((identity) => {
|
||||
const begun = originalBegin(identity)
|
||||
settleLegacyWorkerAndTakeOver(harness, 'settled during takeover')
|
||||
return begun
|
||||
})
|
||||
}
|
||||
|
||||
function settleLegacyWorkerAndTakeOver(harness: Harness, result: string): void {
|
||||
harness.db.settleWorkerReport({
|
||||
taskId: harness.taskId,
|
||||
dispatchId: harness.dispatchId,
|
||||
outcome: 'succeeded',
|
||||
result
|
||||
})
|
||||
harness.db.bindRun({
|
||||
runId: harness.adoptedRunId,
|
||||
coordinatorHandle: 'term_current_coord',
|
||||
coordinatorPaneKey: 'tab_current:55555555-5555-4555-8555-555555555555'
|
||||
})
|
||||
}
|
||||
|
||||
function mutationReceiptCount(db: OrchestrationDb): number {
|
||||
const sqlite = (db as unknown as { db: Database.Database }).db
|
||||
return (
|
||||
sqlite.prepare('SELECT COUNT(*) AS count FROM mutation_receipts').get() as { count: number }
|
||||
).count
|
||||
}
|
||||
|
||||
function messageCount(db: OrchestrationDb): number {
|
||||
const sqlite = (db as unknown as { db: Database.Database }).db
|
||||
return (sqlite.prepare('SELECT COUNT(*) AS count FROM messages').get() as { count: number }).count
|
||||
}
|
||||
|
||||
describe('legacy coordinator takeover races', () => {
|
||||
it.each(['dispatch', 'websocket'] as const)(
|
||||
'%s routes a pre-bind coordinator send to the retained worker mailbox',
|
||||
async (transport) => {
|
||||
const harness = createHarness()
|
||||
|
||||
const response = await invoke(
|
||||
harness.dispatcher,
|
||||
request(
|
||||
'orchestration.send',
|
||||
{ from: COORDINATOR_HANDLE, to: WORKER_HANDLE, subject: 'continue safely' },
|
||||
`send-direct-${transport}`
|
||||
),
|
||||
transport
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
message: {
|
||||
run_id: harness.adoptedRunId,
|
||||
to_handle: WORKER_HANDLE,
|
||||
delivery_contract: 'legacy_direct'
|
||||
}
|
||||
}
|
||||
})
|
||||
const workerCheck = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{ terminal: WORKER_HANDLE },
|
||||
`worker-check-${transport}`,
|
||||
evidence('worker')
|
||||
)
|
||||
)
|
||||
expect(workerCheck).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
messages: [
|
||||
{
|
||||
subject: 'continue safely',
|
||||
run_id: harness.adoptedRunId,
|
||||
delivery_contract: 'legacy_direct'
|
||||
}
|
||||
],
|
||||
legacyCompatibility: { ackMessageIds: [expect.any(String)] }
|
||||
}
|
||||
})
|
||||
expect(
|
||||
harness.db.getInbox(100).some((message) => message.run_id === 'run_legacy_local')
|
||||
).toBe(false)
|
||||
}
|
||||
)
|
||||
|
||||
it('routes an exact legacy Dispatch recipient to compatibility delivery', async () => {
|
||||
const harness = createHarness()
|
||||
|
||||
const response = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.send',
|
||||
{
|
||||
from: COORDINATOR_HANDLE,
|
||||
to: `dispatch:${harness.dispatchId}`,
|
||||
subject: 'dispatch guidance'
|
||||
},
|
||||
'send-dispatch'
|
||||
)
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
message: {
|
||||
run_id: harness.adoptedRunId,
|
||||
to_handle: `dispatch:${harness.dispatchId}`,
|
||||
delivery_contract: 'legacy_direct'
|
||||
}
|
||||
}
|
||||
})
|
||||
const workerCheck = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{ terminal: WORKER_HANDLE },
|
||||
'worker-check-dispatch',
|
||||
evidence('worker')
|
||||
)
|
||||
)
|
||||
expect(workerCheck).toMatchObject({
|
||||
ok: true,
|
||||
result: { messages: [{ subject: 'dispatch guidance' }] }
|
||||
})
|
||||
})
|
||||
|
||||
it('partitions a coordinator group send by legacy recipient contract', async () => {
|
||||
const harness = createHarness()
|
||||
vi.spyOn(harness.runtime, 'listTerminals').mockResolvedValue({
|
||||
terminals: [
|
||||
{ handle: COORDINATOR_HANDLE },
|
||||
{ handle: WORKER_HANDLE },
|
||||
{ handle: 'term_current_worker' }
|
||||
],
|
||||
totalCount: 3,
|
||||
truncated: false
|
||||
} as never)
|
||||
|
||||
const response = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.send',
|
||||
{ from: COORDINATOR_HANDLE, to: '@all', subject: 'group guidance' },
|
||||
'send-group'
|
||||
)
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({ ok: true, result: { recipients: 2 } })
|
||||
const messages = harness.db
|
||||
.getInbox(100)
|
||||
.filter((message) => message.subject === 'group guidance')
|
||||
expect(messages).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
run_id: harness.adoptedRunId,
|
||||
to_handle: WORKER_HANDLE,
|
||||
delivery_contract: 'legacy_direct'
|
||||
}),
|
||||
expect.objectContaining({
|
||||
run_id: harness.adoptedRunId,
|
||||
to_handle: 'term_current_worker',
|
||||
delivery_contract: 'current_delivery'
|
||||
})
|
||||
])
|
||||
)
|
||||
})
|
||||
|
||||
it.each(['dispatch', 'websocket'] as const)(
|
||||
'%s rejects a coordinator send after takeover during durable preflight',
|
||||
async (transport) => {
|
||||
const harness = createHarness()
|
||||
installTakeoverDuringMutationPreflight(harness)
|
||||
const before = messageCount(harness.db)
|
||||
|
||||
const response = await invoke(
|
||||
harness.dispatcher,
|
||||
request(
|
||||
'orchestration.send',
|
||||
{ from: COORDINATOR_HANDLE, to: WORKER_HANDLE, subject: 'must not send' },
|
||||
`send-takeover-${transport}`
|
||||
),
|
||||
transport
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
|
||||
expect(messageCount(harness.db)).toBe(before)
|
||||
expect(mutationReceiptCount(harness.db)).toBe(0)
|
||||
}
|
||||
)
|
||||
|
||||
it('revalidates a group send after asynchronous terminal discovery', async () => {
|
||||
const harness = createHarness()
|
||||
let resolveTerminals:
|
||||
| ((value: {
|
||||
terminals: { handle: string }[]
|
||||
totalCount: number
|
||||
truncated: boolean
|
||||
}) => void)
|
||||
| undefined
|
||||
let signalListingStarted: (() => void) | undefined
|
||||
const listingStarted = new Promise<void>((resolve) => {
|
||||
signalListingStarted = resolve
|
||||
})
|
||||
vi.spyOn(harness.runtime, 'listTerminals').mockImplementation(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
resolveTerminals = resolve as typeof resolveTerminals
|
||||
signalListingStarted?.()
|
||||
}) as never
|
||||
)
|
||||
const before = messageCount(harness.db)
|
||||
const pending = harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.send',
|
||||
{ from: COORDINATOR_HANDLE, to: '@all', subject: 'must remain unsent' },
|
||||
'send-group-takeover'
|
||||
)
|
||||
)
|
||||
await listingStarted
|
||||
settleLegacyWorkerAndTakeOver(harness, 'settled during terminal discovery')
|
||||
resolveTerminals?.({
|
||||
terminals: [{ handle: COORDINATOR_HANDLE }, { handle: WORKER_HANDLE }],
|
||||
totalCount: 2,
|
||||
truncated: false
|
||||
})
|
||||
|
||||
await expect(pending).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { code: 'legacy_read_only' }
|
||||
})
|
||||
expect(messageCount(harness.db)).toBe(before)
|
||||
expect(mutationReceiptCount(harness.db)).toBe(0)
|
||||
})
|
||||
|
||||
it('reports a committed ACK when coordinator takeover interrupts its wait', async () => {
|
||||
const harness = createHarness()
|
||||
const incoming = harness.db.insertMessage({
|
||||
runId: harness.adoptedRunId,
|
||||
from: WORKER_HANDLE,
|
||||
to: `run:${harness.adoptedRunId}`,
|
||||
subject: 'ready to acknowledge'
|
||||
})
|
||||
const first = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{ terminal: COORDINATOR_HANDLE, run: harness.adoptedRunId },
|
||||
'check-delivery-before-takeover'
|
||||
)
|
||||
)
|
||||
expect(first).toMatchObject({
|
||||
ok: true,
|
||||
result: { deliveryId: expect.any(String), messages: [{ id: incoming.id }] }
|
||||
})
|
||||
const deliveryId = (first as { result: { deliveryId: string } }).result.deliveryId
|
||||
let resolveWait: (() => void) | undefined
|
||||
let signalWaitStarted: (() => void) | undefined
|
||||
const waitStarted = new Promise<void>((resolve) => {
|
||||
signalWaitStarted = resolve
|
||||
})
|
||||
vi.spyOn(harness.runtime, 'waitForMessage').mockImplementation(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
resolveWait = () => resolve('notified')
|
||||
signalWaitStarted?.()
|
||||
})
|
||||
)
|
||||
const pending = harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{
|
||||
terminal: COORDINATOR_HANDLE,
|
||||
run: harness.adoptedRunId,
|
||||
ack: deliveryId,
|
||||
wait: true,
|
||||
timeoutMs: 10_000
|
||||
},
|
||||
'check-ack-takeover'
|
||||
)
|
||||
)
|
||||
await waitStarted
|
||||
expect(harness.db.getMessageById(incoming.id)?.read).toBe(1)
|
||||
settleLegacyWorkerAndTakeOver(harness, 'settled during acknowledged wait')
|
||||
resolveWait?.()
|
||||
|
||||
await expect(pending).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
acknowledged: deliveryId,
|
||||
count: 0,
|
||||
waitInterrupted: 'consumer_fenced',
|
||||
mutation: { requestId: 'check-ack-takeover', replayed: false }
|
||||
}
|
||||
})
|
||||
const sqlite = (harness.db as unknown as { db: Database.Database }).db
|
||||
expect(sqlite.prepare('SELECT state FROM mutation_receipts').get()).toEqual({
|
||||
state: 'completed'
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps concurrent duplicates on the live ACK-and-wait result', async () => {
|
||||
const harness = createHarness()
|
||||
harness.db.insertMessage({
|
||||
runId: harness.adoptedRunId,
|
||||
from: WORKER_HANDLE,
|
||||
to: `run:${harness.adoptedRunId}`,
|
||||
subject: 'first delivery'
|
||||
})
|
||||
const firstCheck = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{ terminal: COORDINATOR_HANDLE, run: harness.adoptedRunId },
|
||||
'check-first-delivery'
|
||||
)
|
||||
)
|
||||
const deliveryId = (firstCheck as { result: { deliveryId: string } }).result.deliveryId
|
||||
let resolveWait: (() => void) | undefined
|
||||
let signalWaitStarted: (() => void) | undefined
|
||||
const waitStarted = new Promise<void>((resolve) => {
|
||||
signalWaitStarted = resolve
|
||||
})
|
||||
vi.spyOn(harness.runtime, 'waitForMessage').mockImplementation(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
resolveWait = () => resolve('notified')
|
||||
signalWaitStarted?.()
|
||||
})
|
||||
)
|
||||
const ackRequest = request(
|
||||
'orchestration.check',
|
||||
{
|
||||
terminal: COORDINATOR_HANDLE,
|
||||
run: harness.adoptedRunId,
|
||||
ack: deliveryId,
|
||||
wait: true,
|
||||
timeoutMs: 10_000
|
||||
},
|
||||
'check-ack-concurrent'
|
||||
)
|
||||
const original = harness.dispatcher.dispatch(ackRequest)
|
||||
await waitStarted
|
||||
const secondLiveDispatcher = new RpcDispatcher({
|
||||
runtime: harness.runtime,
|
||||
methods: ORCHESTRATION_METHODS
|
||||
})
|
||||
const duplicate = secondLiveDispatcher.dispatch({
|
||||
...ackRequest,
|
||||
id: 'rpc_check-ack-concurrent-duplicate'
|
||||
})
|
||||
let duplicateSettled = false
|
||||
void duplicate.then(() => {
|
||||
duplicateSettled = true
|
||||
})
|
||||
await Promise.resolve()
|
||||
expect(duplicateSettled).toBe(false)
|
||||
|
||||
const arrived = harness.db.insertMessage({
|
||||
runId: harness.adoptedRunId,
|
||||
from: WORKER_HANDLE,
|
||||
to: `run:${harness.adoptedRunId}`,
|
||||
subject: 'arrived while waiting'
|
||||
})
|
||||
resolveWait?.()
|
||||
const [originalResult, duplicateResult] = await Promise.all([original, duplicate])
|
||||
expect(originalResult).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
messages: [{ id: arrived.id }],
|
||||
mutation: { requestId: 'check-ack-concurrent', replayed: false }
|
||||
}
|
||||
})
|
||||
expect(duplicateResult).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
messages: [{ id: arrived.id }],
|
||||
mutation: { requestId: 'check-ack-concurrent', replayed: true }
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it.each(['dispatch', 'websocket'] as const)(
|
||||
'%s rejects run-use after takeover during durable preflight',
|
||||
async (transport) => {
|
||||
const harness = createHarness()
|
||||
installTakeoverDuringMutationPreflight(harness)
|
||||
|
||||
const response = await invoke(
|
||||
harness.dispatcher,
|
||||
request(
|
||||
'orchestration.runUse',
|
||||
{ id: harness.adoptedRunId, from: COORDINATOR_HANDLE },
|
||||
`run-use-takeover-${transport}`
|
||||
),
|
||||
transport
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
|
||||
expect(harness.db.getRun(harness.adoptedRunId)).toMatchObject({
|
||||
coordinator_handle: 'term_current_coord',
|
||||
consumer_generation: 1
|
||||
})
|
||||
expect(mutationReceiptCount(harness.db)).toBe(0)
|
||||
expect(harness.db.getLegacyCoordinatorPrincipal(harness.adoptedRunId)?.status).toBe('revoked')
|
||||
}
|
||||
)
|
||||
|
||||
it.each(['dispatch', 'websocket'] as const)(
|
||||
'%s rejects a mutation after takeover during durable preflight',
|
||||
async (transport) => {
|
||||
const harness = createHarness()
|
||||
const target = harness.db.createTask({
|
||||
spec: 'must remain ready',
|
||||
runId: harness.adoptedRunId
|
||||
})
|
||||
installTakeoverDuringMutationPreflight(harness)
|
||||
|
||||
const response = await invoke(
|
||||
harness.dispatcher,
|
||||
request(
|
||||
'orchestration.taskUpdate',
|
||||
{
|
||||
id: target.id,
|
||||
status: 'completed',
|
||||
callerTerminalHandle: COORDINATOR_HANDLE
|
||||
},
|
||||
`task-update-takeover-${transport}`
|
||||
),
|
||||
transport
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
|
||||
expect(harness.db.getTask(target.id)?.status).toBe('ready')
|
||||
expect(mutationReceiptCount(harness.db)).toBe(0)
|
||||
}
|
||||
)
|
||||
|
||||
it('revalidates after asynchronous agent detection', async () => {
|
||||
const harness = createHarness()
|
||||
const target = harness.db.createTask({
|
||||
spec: 'must not dispatch after takeover',
|
||||
runId: harness.adoptedRunId
|
||||
})
|
||||
const targetHandle = 'term_current_worker'
|
||||
const targetPane = 'tab_current_worker:77777777-7777-4777-8777-777777777777'
|
||||
vi.spyOn(harness.runtime, 'getTerminalPaneKey').mockImplementation((handle: string) =>
|
||||
handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : handle === targetHandle ? targetPane : null
|
||||
)
|
||||
vi.spyOn(harness.runtime, 'getTerminalProcessIncarnation').mockReturnValue('current-process')
|
||||
let resolveDetection: ((detected: boolean) => void) | undefined
|
||||
let signalDetectionStarted: (() => void) | undefined
|
||||
const detectionStarted = new Promise<void>((resolve) => {
|
||||
signalDetectionStarted = resolve
|
||||
})
|
||||
vi.spyOn(harness.runtime, 'isTerminalRunningAgent').mockImplementation(
|
||||
() =>
|
||||
new Promise<boolean>((resolve) => {
|
||||
resolveDetection = resolve
|
||||
signalDetectionStarted?.()
|
||||
})
|
||||
)
|
||||
const sendPrompt = vi
|
||||
.spyOn(harness.runtime, 'sendTerminalAgentPrompt')
|
||||
.mockResolvedValue({ handle: targetHandle, accepted: true, bytesWritten: 1 })
|
||||
|
||||
const pending = harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.dispatch',
|
||||
{ task: target.id, to: targetHandle, from: COORDINATOR_HANDLE, inject: true },
|
||||
'dispatch-detection-takeover'
|
||||
)
|
||||
)
|
||||
await detectionStarted
|
||||
settleLegacyWorkerAndTakeOver(harness, 'settled during detection')
|
||||
resolveDetection?.(true)
|
||||
const response = await pending
|
||||
|
||||
expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
|
||||
expect(harness.db.getTask(target.id)?.status).toBe('ready')
|
||||
expect(harness.db.getDispatchContext(target.id)).toBeUndefined()
|
||||
expect(sendPrompt).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('explicitly takes over an active adopted Run without stranding its legacy worker', async () => {
|
||||
const harness = createHarness()
|
||||
const currentHandle = 'term_current_coord'
|
||||
const currentPane = 'tab_current:55555555-5555-4555-8555-555555555555'
|
||||
vi.mocked(harness.runtime.getTerminalPaneKey).mockImplementation((handle) =>
|
||||
handle === COORDINATOR_HANDLE
|
||||
? COORDINATOR_PANE
|
||||
: handle === WORKER_HANDLE
|
||||
? WORKER_PANE
|
||||
: handle === currentHandle
|
||||
? currentPane
|
||||
: null
|
||||
)
|
||||
|
||||
await expect(
|
||||
harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.runUse',
|
||||
{ id: harness.adoptedRunId, from: COORDINATOR_HANDLE },
|
||||
'bind-original-coordinator'
|
||||
)
|
||||
)
|
||||
).resolves.toMatchObject({ ok: true })
|
||||
const pendingAsk = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.ask',
|
||||
{
|
||||
from: WORKER_HANDLE,
|
||||
to: COORDINATOR_HANDLE,
|
||||
question: 'Continue with the migration?',
|
||||
timeoutMs: 0
|
||||
},
|
||||
'pending-before-takeover',
|
||||
evidence('worker')
|
||||
)
|
||||
)
|
||||
const questionId = (pendingAsk as { result: { messageId: string } }).result.messageId
|
||||
const unrelated = harness.db.insertMessage({
|
||||
runId: harness.adoptedRunId,
|
||||
from: WORKER_HANDLE,
|
||||
to: 'term_peer_worker',
|
||||
subject: 'peer-only',
|
||||
deliveryContract: 'legacy_direct'
|
||||
})
|
||||
|
||||
const currentRequest = (method: string, params: unknown, invocationId: string): RpcRequest => ({
|
||||
...request(method, params, invocationId),
|
||||
orchestrationCompatibilityEvidence: undefined
|
||||
})
|
||||
const takeoverParams = {
|
||||
id: harness.adoptedRunId,
|
||||
from: currentHandle,
|
||||
takeoverLegacy: true
|
||||
}
|
||||
const takeover = await harness.dispatcher.dispatch(
|
||||
currentRequest('orchestration.runUse', takeoverParams, 'explicit-takeover')
|
||||
)
|
||||
const repeated = await harness.dispatcher.dispatch(
|
||||
currentRequest('orchestration.runUse', takeoverParams, 'explicit-takeover-repeat')
|
||||
)
|
||||
|
||||
expect(takeover).toMatchObject({
|
||||
ok: true,
|
||||
result: { binding: { consumerGeneration: 2 } }
|
||||
})
|
||||
expect(repeated).toMatchObject({
|
||||
ok: true,
|
||||
result: { binding: { consumerGeneration: 2 } }
|
||||
})
|
||||
expect(harness.db.getDispatchContextById(harness.dispatchId)?.status).toBe('dispatched')
|
||||
expect(harness.db.getLegacyCoordinatorPrincipal(harness.adoptedRunId)?.status).toBe('revoked')
|
||||
expect(harness.db.getMessageById(unrelated.id)).toMatchObject({
|
||||
to_handle: 'term_peer_worker',
|
||||
delivery_contract: 'legacy_direct'
|
||||
})
|
||||
|
||||
const promoted = await harness.dispatcher.dispatch(
|
||||
currentRequest(
|
||||
'orchestration.check',
|
||||
{ terminal: currentHandle, run: harness.adoptedRunId, format: true },
|
||||
'read-promoted-question'
|
||||
)
|
||||
)
|
||||
expect(promoted).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
messages: [
|
||||
{
|
||||
id: questionId,
|
||||
to_handle: `run:${harness.adoptedRunId}`,
|
||||
delivery_contract: 'current_delivery'
|
||||
}
|
||||
],
|
||||
formatted: expect.not.stringContaining(`--from run:${harness.adoptedRunId}`)
|
||||
}
|
||||
})
|
||||
const deliveryId = (promoted as { result: { deliveryId: string } }).result.deliveryId
|
||||
await expect(
|
||||
harness.dispatcher.dispatch(
|
||||
currentRequest(
|
||||
'orchestration.reply',
|
||||
{
|
||||
id: questionId,
|
||||
body: 'Yes, continue.',
|
||||
from: currentHandle,
|
||||
run: harness.adoptedRunId
|
||||
},
|
||||
'reply-after-takeover'
|
||||
)
|
||||
)
|
||||
).resolves.toMatchObject({ ok: true, result: { question: { status: 'answered' } } })
|
||||
await expect(
|
||||
harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.ask',
|
||||
{ from: WORKER_HANDLE, resume: questionId, timeoutMs: 100 },
|
||||
'resume-after-takeover',
|
||||
evidence('worker')
|
||||
)
|
||||
)
|
||||
).resolves.toMatchObject({ ok: true, result: { answer: 'Yes, continue.' } })
|
||||
|
||||
const followUp = await harness.dispatcher.dispatch(
|
||||
currentRequest(
|
||||
'orchestration.send',
|
||||
{
|
||||
from: currentHandle,
|
||||
to: `dispatch:${harness.dispatchId}`,
|
||||
subject: 'Continue from the new coordinator'
|
||||
},
|
||||
'current-coordinator-to-legacy-worker'
|
||||
)
|
||||
)
|
||||
expect(followUp).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
message: {
|
||||
run_id: harness.adoptedRunId,
|
||||
to_handle: `dispatch:${harness.dispatchId}`,
|
||||
delivery_contract: 'legacy_direct'
|
||||
}
|
||||
}
|
||||
})
|
||||
const workerFollowUp = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{ terminal: WORKER_HANDLE },
|
||||
'legacy-worker-check-after-takeover',
|
||||
evidence('worker')
|
||||
)
|
||||
)
|
||||
expect(workerFollowUp).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
messages: [{ subject: 'Continue from the new coordinator' }],
|
||||
legacyCompatibility: { ackMessageIds: [expect.any(String)] }
|
||||
}
|
||||
})
|
||||
const [followUpMessageId] = (
|
||||
workerFollowUp as { result: { legacyCompatibility: { ackMessageIds: string[] } } }
|
||||
).result.legacyCompatibility.ackMessageIds
|
||||
await expect(
|
||||
harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.check',
|
||||
{
|
||||
terminal: WORKER_HANDLE,
|
||||
compatibilityAck: JSON.stringify({ messageIds: [followUpMessageId] })
|
||||
},
|
||||
'legacy-worker-ack-after-takeover',
|
||||
evidence('worker')
|
||||
)
|
||||
)
|
||||
).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
acknowledged: [followUpMessageId],
|
||||
legacyCompatibility: { acknowledged: true }
|
||||
}
|
||||
})
|
||||
|
||||
await harness.dispatcher.dispatch(
|
||||
currentRequest(
|
||||
'orchestration.check',
|
||||
{ terminal: currentHandle, run: harness.adoptedRunId, ack: deliveryId },
|
||||
'ack-promoted-question'
|
||||
)
|
||||
)
|
||||
await expect(
|
||||
harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.send',
|
||||
{
|
||||
from: WORKER_HANDLE,
|
||||
to: COORDINATOR_HANDLE,
|
||||
subject: 'Need current coordinator',
|
||||
type: 'escalation',
|
||||
payload: JSON.stringify({ taskId: harness.taskId })
|
||||
},
|
||||
'escalation-after-takeover',
|
||||
evidence('worker')
|
||||
)
|
||||
)
|
||||
).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
message: {
|
||||
to_handle: `run:${harness.adoptedRunId}`,
|
||||
delivery_contract: 'current_delivery'
|
||||
}
|
||||
}
|
||||
})
|
||||
await expect(
|
||||
harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.taskList',
|
||||
{ callerTerminalHandle: COORDINATOR_HANDLE },
|
||||
'old-coordinator-fenced'
|
||||
)
|
||||
)
|
||||
).resolves.toMatchObject({ ok: false, error: { code: 'legacy_read_only' } })
|
||||
})
|
||||
})
|
||||
@@ -104,7 +104,7 @@ export async function handleLegacyLifecycleSend(args: {
|
||||
: { kind: 'message_only' }
|
||||
})
|
||||
if (!committed.duplicate) {
|
||||
runtime.notifyMessageArrived(params.to, committed.message.type)
|
||||
runtime.notifyMessageArrived(committed.message.to_handle, committed.message.type)
|
||||
}
|
||||
return {
|
||||
message: committed.message,
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import { parsePaneKey } from '../../../shared/stable-pane-id'
|
||||
import { OrchestrationError } from '../orchestration/orchestration-error'
|
||||
|
||||
export function equivalentLegacyPaneKey(
|
||||
a: string | null | undefined,
|
||||
b: string | null | undefined
|
||||
): boolean {
|
||||
if (!a || !b) {
|
||||
return false
|
||||
}
|
||||
if (a === b) {
|
||||
return true
|
||||
}
|
||||
const aLeaf = parsePaneKey(a)?.leafId
|
||||
const bLeaf = parsePaneKey(b)?.leafId
|
||||
return Boolean(aLeaf && bLeaf && aLeaf === bLeaf)
|
||||
}
|
||||
|
||||
export function legacyReadOnlyError(): OrchestrationError {
|
||||
return new OrchestrationError(
|
||||
'legacy_read_only',
|
||||
'This retained legacy assignment could not prove authority from its original live process. No effects were applied.',
|
||||
{ effectsApplied: false }
|
||||
)
|
||||
}
|
||||
|
||||
export function legacyCoordinatorReadOnly(): OrchestrationError {
|
||||
return new OrchestrationError(
|
||||
'legacy_read_only',
|
||||
'This retained legacy coordinator could not prove its original process identity. No effects were applied.',
|
||||
{ effectsApplied: false }
|
||||
)
|
||||
}
|
||||
@@ -65,8 +65,9 @@ export async function handleLegacyAsk(args: {
|
||||
options,
|
||||
recipientHandle: recipient
|
||||
})
|
||||
const pending = matches.filter((match) => match.question.status === 'pending')
|
||||
const lostAnswer = matches.find(
|
||||
const unclaimed = matches.filter((match) => !match.claimedByOperation)
|
||||
const pending = unclaimed.filter((match) => match.question.status === 'pending')
|
||||
const lostAnswer = unclaimed.find(
|
||||
(match) => match.question.status === 'answered' && !match.answerAcknowledged
|
||||
)
|
||||
if (lostAnswer) {
|
||||
@@ -98,7 +99,7 @@ export async function handleLegacyAsk(args: {
|
||||
questionId = committed.question.message_id
|
||||
duplicate = committed.duplicate || Boolean(existingQuestionId)
|
||||
if (!duplicate) {
|
||||
runtime.notifyMessageArrived(recipient, committed.message.type)
|
||||
runtime.notifyMessageArrived(committed.message.to_handle, committed.message.type)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -55,6 +55,10 @@ export class OrchestrationMutationExecutor {
|
||||
: db.beginMutationReceipt(identity)
|
||||
|
||||
if (begun.disposition === 'completed') {
|
||||
const active = this.inFlight.get(key)
|
||||
if (active) {
|
||||
return attachMutationReceipt(await active, requestId, true)
|
||||
}
|
||||
return attachMutationReceipt(JSON.parse(begun.row.receipt ?? 'null'), requestId, true)
|
||||
}
|
||||
if (begun.disposition === 'pending') {
|
||||
@@ -102,6 +106,20 @@ export class OrchestrationMutationExecutor {
|
||||
}
|
||||
}
|
||||
|
||||
const executorsByRuntime = new WeakMap<OrcaRuntimeService, OrchestrationMutationExecutor>()
|
||||
|
||||
export function getOrchestrationMutationExecutor(
|
||||
runtime: OrcaRuntimeService
|
||||
): OrchestrationMutationExecutor {
|
||||
const existing = executorsByRuntime.get(runtime)
|
||||
if (existing) {
|
||||
return existing
|
||||
}
|
||||
const executor = new OrchestrationMutationExecutor(runtime)
|
||||
executorsByRuntime.set(runtime, executor)
|
||||
return executor
|
||||
}
|
||||
|
||||
export function authenticatedCallerFingerprint(request: RpcRequest): string {
|
||||
const callerToken =
|
||||
request.authToken ||
|
||||
|
||||
@@ -252,7 +252,10 @@ describe('durable orchestration mutation ledger', () => {
|
||||
})
|
||||
|
||||
it('recovers a lost ask acceptance without creating a second question', async () => {
|
||||
const db = new OrchestrationDb(':memory:')
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-mutation-ask-recovery-'))
|
||||
paths.push(dir)
|
||||
const dbPath = join(dir, 'orchestration.db')
|
||||
const db = new OrchestrationDb(dbPath)
|
||||
const runtime = new OrcaRuntimeService()
|
||||
runtime.setOrchestrationDb(db)
|
||||
vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_worker:leaf_worker')
|
||||
@@ -290,7 +293,16 @@ describe('durable orchestration mutation ledger', () => {
|
||||
const first = firstDispatcher.dispatch(askRequest, { signal: controller.signal })
|
||||
await vi.waitFor(() => expect(db.getInbox(10)).toHaveLength(1))
|
||||
|
||||
const restartedDispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS })
|
||||
const restartedDb = new OrchestrationDb(dbPath)
|
||||
const restartedRuntime = new OrcaRuntimeService()
|
||||
restartedRuntime.setOrchestrationDb(restartedDb)
|
||||
vi.spyOn(restartedRuntime, 'getTerminalPaneKey').mockReturnValue('tab_worker:leaf_worker')
|
||||
vi.spyOn(restartedRuntime, 'getTerminalProcessIncarnation').mockReturnValue('runtime:pty:1')
|
||||
vi.spyOn(restartedRuntime, 'notifyMessageArrived').mockImplementation(() => {})
|
||||
const restartedDispatcher = new RpcDispatcher({
|
||||
runtime: restartedRuntime,
|
||||
methods: ORCHESTRATION_METHODS
|
||||
})
|
||||
const recovered = await restartedDispatcher.dispatch({ ...askRequest, id: 'rpc_ask_2' })
|
||||
expect(recovered).toMatchObject({
|
||||
ok: true,
|
||||
@@ -304,6 +316,7 @@ describe('durable orchestration mutation ledger', () => {
|
||||
|
||||
controller.abort()
|
||||
await first
|
||||
restartedDb.close()
|
||||
db.close()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { FeatureInteractionId } from '../../../shared/feature-interactions'
|
||||
import { isBrowserPaneUiRuntimeRpcParams } from '../../../shared/runtime-rpc-feature-interaction-source'
|
||||
import type { OrcaRuntimeService } from '../orca-runtime'
|
||||
|
||||
export function getRuntimeFeatureInteractionId(
|
||||
method: string,
|
||||
@@ -37,6 +38,25 @@ export function getRuntimeFeatureInteractionId(
|
||||
return method.startsWith('orchestration.') ? 'agent-orchestration' : null
|
||||
}
|
||||
|
||||
export function recordRuntimeFeatureInteraction(
|
||||
runtime: OrcaRuntimeService,
|
||||
method: string,
|
||||
result: unknown,
|
||||
alreadyRecorded?: Set<FeatureInteractionId>,
|
||||
rawParams?: unknown
|
||||
): void {
|
||||
const id = getRuntimeFeatureInteractionId(method, result, rawParams)
|
||||
if (!id || alreadyRecorded?.has(id)) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
runtime.recordFeatureInteraction(id)
|
||||
alreadyRecorded?.add(id)
|
||||
} catch {
|
||||
// Best-effort education state must not break runtime tools.
|
||||
}
|
||||
}
|
||||
|
||||
function hasBooleanResult(value: unknown, key: string): boolean {
|
||||
return (
|
||||
value !== null && typeof value === 'object' && (value as Record<string, unknown>)[key] === true
|
||||
|
||||
@@ -66,6 +66,10 @@ import {
|
||||
import type { Store } from '../persistence'
|
||||
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
|
||||
import { runRemoteOrcaCli } from './ssh-remote-orca-cli'
|
||||
import {
|
||||
acknowledgeRemoteOrcaCliPostOutput,
|
||||
parseRemoteOrcaCliPostOutput
|
||||
} from './ssh-remote-orchestration-post-output'
|
||||
import { toSshExecutionHostId, type ExecutionHostId } from '../../shared/execution-host'
|
||||
import { isTerminalLeafId, makePaneKey } from '../../shared/stable-pane-id'
|
||||
import { isValidTerminalTabId } from '../../shared/terminal-tab-id'
|
||||
@@ -716,6 +720,37 @@ export class SshRelaySession {
|
||||
this.runtime.releaseOrchestrationCompatibilitySshAttachment(runtimeAuthority.attachmentId)
|
||||
}
|
||||
})
|
||||
mux.onRequest('orca.cli.postOutput', async (params) => {
|
||||
if (!this.runtime) {
|
||||
throw new Error('Orca runtime is unavailable')
|
||||
}
|
||||
const rawEnv = params.env
|
||||
const env =
|
||||
rawEnv && typeof rawEnv === 'object' && !Array.isArray(rawEnv)
|
||||
? Object.fromEntries(
|
||||
Object.entries(rawEnv).filter(
|
||||
(entry): entry is [string, string] =>
|
||||
typeof entry[0] === 'string' && typeof entry[1] === 'string'
|
||||
)
|
||||
)
|
||||
: {}
|
||||
const runtimeAuthority = this.runtime.registerOrchestrationCompatibilitySshAttachment(
|
||||
this.targetId,
|
||||
connectionIncarnation
|
||||
)
|
||||
this.activeCompatibilityAttachmentIds.add(runtimeAuthority.attachmentId)
|
||||
try {
|
||||
await acknowledgeRemoteOrcaCliPostOutput(this.runtime, {
|
||||
postOutput: parseRemoteOrcaCliPostOutput(params.postOutput),
|
||||
env,
|
||||
runtimeAuthority
|
||||
})
|
||||
return { acknowledged: true }
|
||||
} finally {
|
||||
this.activeCompatibilityAttachmentIds.delete(runtimeAuthority.attachmentId)
|
||||
this.runtime.releaseOrchestrationCompatibilitySshAttachment(runtimeAuthority.attachmentId)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Why: ship plugin/extension source from Orca so agent-event changes don't force a relay redeploy (agent-status-over-ssh.md §4/§8). Best-effort.
|
||||
|
||||
@@ -43,8 +43,23 @@ export type RemoteOrcaCliResult = {
|
||||
stdout: string
|
||||
stderr: string
|
||||
exitCode: number
|
||||
postOutput?: RemoteOrcaCliPostOutput
|
||||
}
|
||||
|
||||
export type RemoteOrcaCliPostOutput =
|
||||
| {
|
||||
kind: 'legacy_check_ack'
|
||||
terminal: string
|
||||
messageIds: string[]
|
||||
types?: string[]
|
||||
}
|
||||
| {
|
||||
kind: 'legacy_question_ack'
|
||||
terminal: string
|
||||
questionId: string
|
||||
answerMessageId: string
|
||||
}
|
||||
|
||||
export type HostCliPassthroughOptions = {
|
||||
execPath?: string
|
||||
cliEntryPath?: string
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import type { RpcResponse } from '../runtime/rpc/core'
|
||||
import type { ParsedRemoteCli } from './ssh-remote-cli-argument-error'
|
||||
import type { RemoteOrcaCliResult } from './ssh-remote-cli-host-passthrough'
|
||||
import { formatRemoteCli } from './ssh-remote-cli-format'
|
||||
import {
|
||||
formatRemoteOrchestrationAsk,
|
||||
getRemoteCliExitCode
|
||||
} from './ssh-remote-orchestration-ask-output'
|
||||
import { formatRemoteOrchestrationCheck } from './ssh-remote-orchestration-check-output'
|
||||
import { getRemoteCliPostOutput } from './ssh-remote-orchestration-post-output'
|
||||
import { resolveRemoteCliHandle } from './ssh-remote-cli-args'
|
||||
|
||||
export function formatInProcessRemoteCliResult(
|
||||
parsed: ParsedRemoteCli,
|
||||
env: Record<string, string>,
|
||||
response: RpcResponse,
|
||||
json: boolean
|
||||
): RemoteOrcaCliResult {
|
||||
const command = parsed.commandPath.join(' ')
|
||||
const formatted =
|
||||
command === 'orchestration check'
|
||||
? formatRemoteOrchestrationCheck(
|
||||
response,
|
||||
json,
|
||||
resolveRemoteCliHandle(parsed.flags, env, 'terminal'),
|
||||
parsed.flags.has('format')
|
||||
)
|
||||
: command === 'orchestration ask'
|
||||
? formatRemoteOrchestrationAsk(response, json)
|
||||
: json
|
||||
? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' }
|
||||
: formatRemoteCli(response)
|
||||
const postOutput = getRemoteCliPostOutput(parsed, env, response)
|
||||
return {
|
||||
stdout: formatted.stdout,
|
||||
stderr: formatted.stderr,
|
||||
exitCode: getRemoteCliExitCode(command, response),
|
||||
...(postOutput ? { postOutput } : {})
|
||||
}
|
||||
}
|
||||
@@ -24,7 +24,6 @@ const LEGACY_FALLBACK_OPTIONS: HostCliPassthroughOptions = {
|
||||
userDataPath: '/host/user-data',
|
||||
entryExists: () => false
|
||||
}
|
||||
|
||||
type FakeChild = EventEmitter & {
|
||||
stdout: EventEmitter
|
||||
stderr: EventEmitter
|
||||
|
||||
@@ -6,7 +6,6 @@ import { ORCHESTRATION_CONTRACT_VERSION } from '../../shared/protocol-version'
|
||||
import { RpcDispatcher } from '../runtime/rpc/dispatcher'
|
||||
import type { RpcResponse } from '../runtime/rpc/core'
|
||||
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
|
||||
import { formatRemoteCli } from './ssh-remote-cli-format'
|
||||
import {
|
||||
HostCliUnavailableError,
|
||||
runHostOrcaCliPassthrough,
|
||||
@@ -26,9 +25,9 @@ import { buildRemoteCliError } from './ssh-remote-cli-error-response'
|
||||
import { getRemoteLinearHelp, tryDispatchRemoteLinearCli } from './ssh-remote-linear-cli'
|
||||
import {
|
||||
getRemoteOrchestrationPayload,
|
||||
hasRemoteLifecycleRejection,
|
||||
resolveRemoteOrchestrationSender
|
||||
} from './ssh-remote-orchestration-send'
|
||||
import { formatInProcessRemoteCliResult } from './ssh-remote-cli-in-process-result'
|
||||
|
||||
export type { RemoteOrcaCliRequest, RemoteOrcaCliResult } from './ssh-remote-cli-host-passthrough'
|
||||
|
||||
@@ -51,6 +50,7 @@ export async function runRemoteOrcaCli(
|
||||
): Promise<RemoteOrcaCliResult> {
|
||||
const parsed = parseRemoteCliArgs(request.argv)
|
||||
const json = parsed.flags.has('json')
|
||||
const command = parsed.commandPath.join(' ')
|
||||
|
||||
const interactiveMessage = HOST_INTERACTIVE_COMMANDS[parsed.commandPath[0] ?? '']
|
||||
if (interactiveMessage) {
|
||||
@@ -64,6 +64,17 @@ export async function runRemoteOrcaCli(
|
||||
return { stdout: '', stderr: `${interactiveMessage}\n`, exitCode: 1 }
|
||||
}
|
||||
|
||||
if (command === 'orchestration check' || command === 'orchestration ask') {
|
||||
// Why: compatibility ACKs must wait until relay stdout is observable; a host CLI child can only flush into main's capture pipe.
|
||||
return await runLegacyRemoteOrcaCli(
|
||||
runtime,
|
||||
request,
|
||||
parsed,
|
||||
json,
|
||||
new HostCliUnavailableError('output-ordered orchestration bridge required')
|
||||
)
|
||||
}
|
||||
|
||||
let passthroughFailure: HostCliUnavailableError | null = null
|
||||
try {
|
||||
return await runHostOrcaCliPassthrough(request, passthroughOptions)
|
||||
@@ -101,16 +112,7 @@ async function runLegacyRemoteOrcaCli(
|
||||
passthroughFailure.message,
|
||||
request.runtimeAuthority
|
||||
)
|
||||
const formatted = json
|
||||
? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' }
|
||||
: formatRemoteCli(response)
|
||||
return {
|
||||
stdout: formatted.stdout,
|
||||
stderr: formatted.stderr,
|
||||
// Why: the legacy SSH bridge bypasses the local CLI handler that turns
|
||||
// a persisted lifecycle rejection into an unsuccessful command.
|
||||
exitCode: response.ok && !hasRemoteLifecycleRejection(response.result) ? 0 : 1
|
||||
}
|
||||
return formatInProcessRemoteCliResult(parsed, request.env, response, json)
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err)
|
||||
const code =
|
||||
@@ -141,13 +143,17 @@ async function dispatchRemoteCli(
|
||||
runtimeAuthority: RemoteOrcaCliRequest['runtimeAuthority']
|
||||
): Promise<RpcResponse> {
|
||||
const command = parsed.commandPath.join(' ')
|
||||
const compatibilityInvocationId = randomUUID()
|
||||
const inheritedEvidence = readOrchestrationCompatibilityEvidence(env)
|
||||
const orchestrationCompatibilityEvidence = runtimeAuthority
|
||||
? { ...inheritedEvidence, host: runtimeAuthority }
|
||||
: inheritedEvidence
|
||||
const compatibilityEnvelope: RuntimeOrchestrationEnvelope = {
|
||||
compatibilityInvocationId,
|
||||
compatibilityInvocationId: randomUUID(),
|
||||
orchestrationRequestId:
|
||||
optionalRemoteCliString(parsed.flags, 'retry-request') ??
|
||||
(command === 'orchestration check' || command === 'orchestration ask'
|
||||
? randomUUID()
|
||||
: undefined),
|
||||
orchestrationCompatibilityEvidence
|
||||
}
|
||||
const linearResponse = await tryDispatchRemoteLinearCli(dispatcher, parsed, env, stdin)
|
||||
@@ -201,8 +207,7 @@ async function dispatchRemoteCli(
|
||||
},
|
||||
{
|
||||
...compatibilityEnvelope,
|
||||
orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability'),
|
||||
orchestrationRequestId: optionalRemoteCliString(parsed.flags, 'retry-request')
|
||||
orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability')
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -215,15 +220,39 @@ async function dispatchRemoteCli(
|
||||
terminalPaneKey: parsed.flags.has('terminal')
|
||||
? undefined
|
||||
: env.ORCA_PANE_KEY || undefined,
|
||||
unread: parsed.flags.has('unread') ? true : undefined,
|
||||
unread: parsed.flags.has('unread') ? true : parsed.flags.has('peek') ? false : undefined,
|
||||
peek: parsed.flags.has('peek') ? true : undefined,
|
||||
all: parsed.flags.has('all') ? true : undefined,
|
||||
types: optionalRemoteCliString(parsed.flags, 'types'),
|
||||
format: parsed.flags.has('format') ? true : undefined,
|
||||
inject: parsed.flags.has('inject') ? true : undefined,
|
||||
compatibilityCliCommand: 'orca',
|
||||
run: optionalRemoteCliString(parsed.flags, 'run'),
|
||||
ack: optionalRemoteCliString(parsed.flags, 'ack'),
|
||||
wait: parsed.flags.has('wait') ? true : undefined,
|
||||
timeoutMs: optionalRemoteCliNumber(parsed.flags, 'timeout-ms')
|
||||
},
|
||||
compatibilityEnvelope
|
||||
)
|
||||
case 'orchestration ask':
|
||||
return await call(
|
||||
dispatcher,
|
||||
'orchestration.ask',
|
||||
{
|
||||
to: optionalRemoteCliString(parsed.flags, 'to'),
|
||||
question: optionalRemoteCliString(parsed.flags, 'question'),
|
||||
resume: optionalRemoteCliString(parsed.flags, 'resume'),
|
||||
options: optionalRemoteCliString(parsed.flags, 'options'),
|
||||
timeoutMs: optionalRemoteCliNumber(parsed.flags, 'timeout-ms'),
|
||||
from: resolveRemoteCliHandle(parsed.flags, env, 'from'),
|
||||
run: optionalRemoteCliString(parsed.flags, 'run'),
|
||||
compatibilityCliCommand: 'orca'
|
||||
},
|
||||
{
|
||||
...compatibilityEnvelope,
|
||||
orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability')
|
||||
}
|
||||
)
|
||||
case 'orchestration reply':
|
||||
return await call(
|
||||
dispatcher,
|
||||
@@ -271,7 +300,8 @@ async function call(
|
||||
? ORCHESTRATION_CONTRACT_VERSION
|
||||
: undefined,
|
||||
orchestrationRequestId: envelope?.orchestrationRequestId,
|
||||
compatibilityInvocationId: envelope?.compatibilityInvocationId,
|
||||
compatibilityInvocationId:
|
||||
envelope?.orchestrationRequestId ?? envelope?.compatibilityInvocationId,
|
||||
orchestrationCompatibilityEvidence: envelope?.orchestrationCompatibilityEvidence
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import type { RpcResponse } from '../runtime/rpc/core'
|
||||
import { formatRemoteCli } from './ssh-remote-cli-format'
|
||||
import { hasRemoteLifecycleRejection } from './ssh-remote-orchestration-send'
|
||||
|
||||
export function formatRemoteOrchestrationAsk(
|
||||
response: RpcResponse,
|
||||
json: boolean
|
||||
): { stdout: string; stderr: string } {
|
||||
if (!response.ok || !isRecord(response.result)) {
|
||||
return json
|
||||
? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' }
|
||||
: formatRemoteCli(response)
|
||||
}
|
||||
if (json) {
|
||||
return { stdout: `${JSON.stringify(response.result)}\n`, stderr: '' }
|
||||
}
|
||||
if (isRecord(response.result.legacyCompatibility)) {
|
||||
const compatibility = response.result.legacyCompatibility
|
||||
if (compatibility.resumeRequired === true && typeof compatibility.resumeCommand === 'string') {
|
||||
return {
|
||||
stdout:
|
||||
`Question ${String(response.result.messageId ?? 'unknown')} committed.\n` +
|
||||
`Resume with: ${compatibility.resumeCommand}\n`,
|
||||
stderr: ''
|
||||
}
|
||||
}
|
||||
}
|
||||
const answer = typeof response.result.answer === 'string' ? response.result.answer : ''
|
||||
const thread = typeof response.result.threadId === 'string' ? response.result.threadId : 'unknown'
|
||||
const timeoutMs =
|
||||
typeof response.result.timeoutMs === 'number' ? response.result.timeoutMs : undefined
|
||||
const stderr = response.result.timedOut
|
||||
? `ask timeout after ${timeoutMs ?? 0}ms (thread ${thread})\n`
|
||||
: response.result.cancelled
|
||||
? response.result.connectionLost
|
||||
? `ask connection closed (question ${String(response.result.messageId ?? 'unknown')})\n`
|
||||
: `ask cancelled (question ${String(response.result.messageId ?? 'unknown')})\n`
|
||||
: ''
|
||||
return { stdout: answer ? `${answer}\n` : '', stderr }
|
||||
}
|
||||
|
||||
export function getRemoteCliExitCode(command: string, response: RpcResponse): number {
|
||||
if (!response.ok || hasRemoteLifecycleRejection(response.result)) {
|
||||
return 1
|
||||
}
|
||||
if (
|
||||
command === 'orchestration ask' &&
|
||||
isRecord(response.result) &&
|
||||
isRecord(response.result.legacyCompatibility) &&
|
||||
response.result.legacyCompatibility.resumeRequired === true
|
||||
) {
|
||||
return 75
|
||||
}
|
||||
if (
|
||||
command === 'orchestration ask' &&
|
||||
isRecord(response.result) &&
|
||||
(response.result.timedOut === true || response.result.cancelled === true)
|
||||
) {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import {
|
||||
formatOrchestrationCheckText,
|
||||
prepareOrchestrationCheckOutput,
|
||||
type OrchestrationCheckOutput,
|
||||
type OrchestrationMessageSummary
|
||||
} from '../../shared/orchestration-check-output'
|
||||
import type { RpcResponse } from '../runtime/rpc/core'
|
||||
import { formatRemoteCli } from './ssh-remote-cli-format'
|
||||
|
||||
export function formatRemoteOrchestrationCheck(
|
||||
response: RpcResponse,
|
||||
json: boolean,
|
||||
terminal: string,
|
||||
formattedRequested: boolean
|
||||
): { stdout: string; stderr: string } {
|
||||
if (!response.ok || !isRecord(response.result)) {
|
||||
return json
|
||||
? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' }
|
||||
: formatRemoteCli(response)
|
||||
}
|
||||
const messages = Array.isArray(response.result.messages)
|
||||
? response.result.messages.filter(isMessageSummary)
|
||||
: []
|
||||
const count = typeof response.result.count === 'number' ? response.result.count : messages.length
|
||||
const result = prepareOrchestrationCheckOutput(
|
||||
{ ...(response.result as OrchestrationCheckOutput), messages, count },
|
||||
terminal,
|
||||
formattedRequested
|
||||
)
|
||||
if (json) {
|
||||
return {
|
||||
stdout: `${JSON.stringify({ ...response, result }, null, 2)}\n`,
|
||||
stderr: ''
|
||||
}
|
||||
}
|
||||
return {
|
||||
stdout: `${formatOrchestrationCheckText(result, terminal)}\n`,
|
||||
stderr: ''
|
||||
}
|
||||
}
|
||||
|
||||
function isMessageSummary(value: unknown): value is OrchestrationMessageSummary {
|
||||
return isRecord(value) && typeof value.id === 'string' && typeof value.from_handle === 'string'
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
|
||||
}
|
||||
@@ -0,0 +1,394 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
app: {
|
||||
isPackaged: false,
|
||||
getAppPath: () => '/host/app'
|
||||
}
|
||||
}))
|
||||
vi.mock('../persistence', () => ({
|
||||
getCanonicalUserDataPath: () => '/host/user-data'
|
||||
}))
|
||||
|
||||
import { OrchestrationDb } from '../runtime/orchestration/db'
|
||||
import { OrcaRuntimeService } from '../runtime/orca-runtime'
|
||||
import type Database from '../sqlite/sync-database'
|
||||
import type { HostCliPassthroughOptions } from './ssh-remote-cli-host-passthrough'
|
||||
import { runRemoteOrcaCli } from './ssh-remote-orca-cli'
|
||||
import { acknowledgeRemoteOrcaCliPostOutput } from './ssh-remote-orchestration-post-output'
|
||||
|
||||
const LEGACY_FALLBACK_OPTIONS: HostCliPassthroughOptions = {
|
||||
execPath: '/host/electron',
|
||||
cliEntryPath: '/host/app/out/cli/index.js',
|
||||
userDataPath: '/host/user-data',
|
||||
entryExists: () => false
|
||||
}
|
||||
const WORKER_HANDLE = 'term_legacy_ssh_worker'
|
||||
const WORKER_PANE = 'tab_legacy_ssh:33333333-3333-4333-8333-333333333333'
|
||||
const COORDINATOR_HANDLE = 'term_legacy_ssh_coord'
|
||||
const COORDINATOR_PANE = 'tab_legacy_coord:44444444-4444-4444-8444-444444444444'
|
||||
const WORKER_ENV = {
|
||||
ORCA_TERMINAL_HANDLE: WORKER_HANDLE,
|
||||
ORCA_PANE_KEY: WORKER_PANE,
|
||||
ORCA_AGENT_LAUNCH_TOKEN: 'legacy-ssh-token'
|
||||
}
|
||||
const COORDINATOR_ENV = {
|
||||
ORCA_TERMINAL_HANDLE: COORDINATOR_HANDLE,
|
||||
ORCA_PANE_KEY: COORDINATOR_PANE,
|
||||
ORCA_AGENT_LAUNCH_TOKEN: 'legacy-ssh-coordinator-token'
|
||||
}
|
||||
const RUNTIME_AUTHORITY = {
|
||||
kind: 'ssh' as const,
|
||||
targetId: 'saved-target',
|
||||
connectionIncarnation: 'connection-1',
|
||||
attachmentId: 'attachment-1'
|
||||
}
|
||||
|
||||
function createLegacyRuntime() {
|
||||
const db = new OrchestrationDb(':memory:')
|
||||
const run = db.createRun({
|
||||
objective: 'Adopted legacy SSH work',
|
||||
coordinatorHandle: COORDINATOR_HANDLE,
|
||||
coordinatorPaneKey: COORDINATOR_PANE
|
||||
})
|
||||
const task = db.createTask({
|
||||
spec: 'legacy SSH assignment',
|
||||
runId: run.id,
|
||||
createdByTerminalHandle: COORDINATOR_HANDLE
|
||||
})
|
||||
const dispatch = db.createDispatchContext(task.id, WORKER_HANDLE, WORKER_PANE)
|
||||
const sqlite = (db as unknown as { db: Database.Database }).db
|
||||
sqlite
|
||||
.prepare(
|
||||
`UPDATE dispatch_contexts
|
||||
SET contract_version = 0, process_incarnation = 'process-1'
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run(dispatch.id)
|
||||
sqlite
|
||||
.prepare(
|
||||
`INSERT INTO legacy_adoptions (source_run_id, adopted_run_id, scheduler_state_lost)
|
||||
VALUES ('run_legacy_local', ?, 1)`
|
||||
)
|
||||
.run(run.id)
|
||||
|
||||
const runtime = new OrcaRuntimeService()
|
||||
runtime.setOrchestrationDb(db)
|
||||
vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) =>
|
||||
handle === WORKER_HANDLE ? WORKER_PANE : handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : null
|
||||
)
|
||||
vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation((evidence) => {
|
||||
const worker =
|
||||
evidence?.terminalHandle === WORKER_HANDLE &&
|
||||
evidence.paneKey === WORKER_PANE &&
|
||||
evidence.launchToken === WORKER_ENV.ORCA_AGENT_LAUNCH_TOKEN
|
||||
const coordinator =
|
||||
evidence?.terminalHandle === COORDINATOR_HANDLE &&
|
||||
evidence.paneKey === COORDINATOR_PANE &&
|
||||
evidence.launchToken === COORDINATOR_ENV.ORCA_AGENT_LAUNCH_TOKEN
|
||||
if (
|
||||
(!worker && !coordinator) ||
|
||||
evidence.host?.kind !== 'ssh' ||
|
||||
evidence.host.targetId !== RUNTIME_AUTHORITY.targetId ||
|
||||
evidence.host.connectionIncarnation !== RUNTIME_AUTHORITY.connectionIncarnation
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const identity = worker
|
||||
? {
|
||||
terminalHandle: WORKER_HANDLE,
|
||||
paneKey: WORKER_PANE,
|
||||
launchToken: WORKER_ENV.ORCA_AGENT_LAUNCH_TOKEN
|
||||
}
|
||||
: {
|
||||
terminalHandle: COORDINATOR_HANDLE,
|
||||
paneKey: COORDINATOR_PANE,
|
||||
launchToken: COORDINATOR_ENV.ORCA_AGENT_LAUNCH_TOKEN
|
||||
}
|
||||
return {
|
||||
hostScope: { kind: 'ssh', targetId: RUNTIME_AUTHORITY.targetId },
|
||||
terminalHandle: identity.terminalHandle,
|
||||
paneKey: identity.paneKey,
|
||||
processIncarnation: 'process-1',
|
||||
launchTokenHash: createHash('sha256').update(identity.launchToken).digest('hex')
|
||||
}
|
||||
})
|
||||
vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {})
|
||||
return { db, runtime, run, dispatch }
|
||||
}
|
||||
|
||||
describe('legacy SSH orchestration fallback', () => {
|
||||
it('acknowledges a consuming check only after remote output', async () => {
|
||||
const { db, runtime, run } = createLegacyRuntime()
|
||||
const message = db.insertMessage({
|
||||
runId: run.id,
|
||||
deliveryContract: 'legacy_direct',
|
||||
from: COORDINATOR_HANDLE,
|
||||
to: WORKER_HANDLE,
|
||||
subject: 'retained SSH mail'
|
||||
})
|
||||
const request = {
|
||||
argv: ['orchestration', 'check', '--unread', '--inject', '--json'],
|
||||
cwd: '/home/alice/repo',
|
||||
env: WORKER_ENV,
|
||||
runtimeAuthority: RUNTIME_AUTHORITY
|
||||
}
|
||||
|
||||
try {
|
||||
const first = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
|
||||
const replay = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
|
||||
|
||||
expect(JSON.parse(first.stdout)).toMatchObject({
|
||||
result: {
|
||||
messages: [{ id: message.id }],
|
||||
formatted: expect.stringContaining('retained SSH mail')
|
||||
}
|
||||
})
|
||||
expect(first.postOutput).toEqual({
|
||||
kind: 'legacy_check_ack',
|
||||
terminal: WORKER_HANDLE,
|
||||
messageIds: [message.id]
|
||||
})
|
||||
expect(JSON.parse(replay.stdout)).toMatchObject({
|
||||
result: { messages: [{ id: message.id }] }
|
||||
})
|
||||
expect(db.getMessageById(message.id)?.read).toBe(0)
|
||||
|
||||
await acknowledgeRemoteOrcaCliPostOutput(runtime, {
|
||||
postOutput: first.postOutput!,
|
||||
env: WORKER_ENV,
|
||||
runtimeAuthority: RUNTIME_AUTHORITY
|
||||
})
|
||||
const afterAck = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
|
||||
|
||||
expect(JSON.parse(afterAck.stdout)).toMatchObject({ result: { messages: [], count: 0 } })
|
||||
expect(afterAck.postOutput).toBeUndefined()
|
||||
expect(db.getMessageById(message.id)?.read).toBe(1)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps peek formatted and non-consuming', async () => {
|
||||
const { db, runtime, run } = createLegacyRuntime()
|
||||
const message = db.insertMessage({
|
||||
runId: run.id,
|
||||
deliveryContract: 'legacy_direct',
|
||||
from: COORDINATOR_HANDLE,
|
||||
to: WORKER_HANDLE,
|
||||
subject: 'inspect retained SSH mail'
|
||||
})
|
||||
|
||||
try {
|
||||
const peek = await runRemoteOrcaCli(
|
||||
runtime,
|
||||
{
|
||||
argv: ['orchestration', 'check', '--peek', '--format', '--json'],
|
||||
cwd: '/home/alice/repo',
|
||||
env: WORKER_ENV,
|
||||
runtimeAuthority: RUNTIME_AUTHORITY
|
||||
},
|
||||
LEGACY_FALLBACK_OPTIONS
|
||||
)
|
||||
|
||||
const output = JSON.parse(peek.stdout) as {
|
||||
result: { formatted: string }
|
||||
}
|
||||
expect(output).toMatchObject({
|
||||
result: {
|
||||
messages: [{ id: message.id }],
|
||||
formatted: expect.stringContaining('inspect retained SSH mail'),
|
||||
legacyCompatibility: { readOnly: true }
|
||||
}
|
||||
})
|
||||
expect(output.result.formatted).toContain(`${message.id} [legacy, read-only]`)
|
||||
expect(output.result.formatted).toContain(
|
||||
'Inspection only: reply and acknowledgment are unavailable.'
|
||||
)
|
||||
expect(output.result.formatted).not.toContain('orchestration reply')
|
||||
expect(peek.postOutput).toBeUndefined()
|
||||
expect(db.getMessageById(message.id)?.read).toBe(0)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('reads and acknowledges current Run delivery', async () => {
|
||||
const { db, runtime, run } = createLegacyRuntime()
|
||||
const message = db.insertMessage({
|
||||
runId: run.id,
|
||||
from: WORKER_HANDLE,
|
||||
to: `run:${run.id}`,
|
||||
subject: 'current Run mail'
|
||||
})
|
||||
const baseRequest = {
|
||||
cwd: '/home/alice/repo',
|
||||
env: COORDINATOR_ENV,
|
||||
runtimeAuthority: RUNTIME_AUTHORITY
|
||||
}
|
||||
const spawn = vi.fn()
|
||||
const hostCliAvailable = {
|
||||
...LEGACY_FALLBACK_OPTIONS,
|
||||
entryExists: () => true,
|
||||
spawn: spawn as HostCliPassthroughOptions['spawn']
|
||||
}
|
||||
|
||||
try {
|
||||
const checked = await runRemoteOrcaCli(
|
||||
runtime,
|
||||
{
|
||||
...baseRequest,
|
||||
argv: ['orchestration', 'check', '--run', run.id]
|
||||
},
|
||||
hostCliAvailable
|
||||
)
|
||||
expect(checked.stdout).toContain('Delivery ')
|
||||
expect(checked.stdout).toContain(`${message.id} [status] from=${WORKER_HANDLE}`)
|
||||
expect(spawn).not.toHaveBeenCalled()
|
||||
|
||||
const checkedJson = await runRemoteOrcaCli(
|
||||
runtime,
|
||||
{
|
||||
...baseRequest,
|
||||
argv: ['orchestration', 'check', '--run', run.id, '--json']
|
||||
},
|
||||
hostCliAvailable
|
||||
)
|
||||
const result = JSON.parse(checkedJson.stdout) as {
|
||||
result: { deliveryId: string; messages: { id: string }[] }
|
||||
}
|
||||
expect(result.result.messages).toEqual([expect.objectContaining({ id: message.id })])
|
||||
|
||||
const acknowledged = await runRemoteOrcaCli(
|
||||
runtime,
|
||||
{
|
||||
...baseRequest,
|
||||
argv: [
|
||||
'orchestration',
|
||||
'check',
|
||||
'--run',
|
||||
run.id,
|
||||
'--ack',
|
||||
result.result.deliveryId,
|
||||
'--json'
|
||||
]
|
||||
},
|
||||
hostCliAvailable
|
||||
)
|
||||
|
||||
expect(JSON.parse(acknowledged.stdout)).toMatchObject({
|
||||
result: { acknowledged: result.result.deliveryId }
|
||||
})
|
||||
expect(db.getMessageById(message.id)?.read).toBe(1)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('replays an SSH legacy ask by retry request without creating another question', async () => {
|
||||
const { db, runtime } = createLegacyRuntime()
|
||||
const argv = [
|
||||
'orchestration',
|
||||
'ask',
|
||||
'--to',
|
||||
COORDINATOR_HANDLE,
|
||||
'--question',
|
||||
'Continue?',
|
||||
'--timeout-ms',
|
||||
'1',
|
||||
'--retry-request',
|
||||
'ssh-question-1',
|
||||
'--json'
|
||||
]
|
||||
const request = {
|
||||
argv,
|
||||
cwd: '/home/alice/repo',
|
||||
env: WORKER_ENV,
|
||||
runtimeAuthority: RUNTIME_AUTHORITY
|
||||
}
|
||||
|
||||
try {
|
||||
const first = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
|
||||
const replay = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS)
|
||||
const sqlite = (db as unknown as { db: Database.Database }).db
|
||||
|
||||
const firstResult = JSON.parse(first.stdout) as { messageId: string; timedOut: boolean }
|
||||
const replayResult = JSON.parse(replay.stdout) as { messageId: string; timedOut: boolean }
|
||||
expect(first).toMatchObject({ exitCode: 1 })
|
||||
expect(firstResult).toMatchObject({ messageId: expect.any(String), timedOut: true })
|
||||
expect(replay).toMatchObject({ exitCode: 1 })
|
||||
expect(replayResult).toMatchObject({ messageId: firstResult.messageId, timedOut: true })
|
||||
expect(
|
||||
(
|
||||
sqlite.prepare('SELECT COUNT(*) AS count FROM question_threads').get() as {
|
||||
count: number
|
||||
}
|
||||
).count
|
||||
).toBe(1)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('resumes and acknowledges a question answer', async () => {
|
||||
const { db, runtime, run, dispatch } = createLegacyRuntime()
|
||||
const pending = db.createQuestion({
|
||||
runId: run.id,
|
||||
dispatchId: dispatch.id,
|
||||
askerHandle: WORKER_HANDLE,
|
||||
question: 'Proceed?'
|
||||
})
|
||||
const answer = db.answerQuestion({
|
||||
messageId: pending.question.message_id,
|
||||
runId: run.id,
|
||||
consumerGeneration: run.consumer_generation,
|
||||
body: 'yes'
|
||||
})
|
||||
const sqlite = (db as unknown as { db: Database.Database }).db
|
||||
sqlite
|
||||
.prepare(
|
||||
`UPDATE messages
|
||||
SET delivery_contract = 'legacy_direct', read = 0
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run(answer.message.id)
|
||||
|
||||
try {
|
||||
const result = await runRemoteOrcaCli(
|
||||
runtime,
|
||||
{
|
||||
argv: ['orchestration', 'ask', '--resume', pending.question.message_id, '--json'],
|
||||
cwd: '/home/alice/repo',
|
||||
env: WORKER_ENV,
|
||||
runtimeAuthority: RUNTIME_AUTHORITY
|
||||
},
|
||||
LEGACY_FALLBACK_OPTIONS
|
||||
)
|
||||
|
||||
expect(result.exitCode).toBe(0)
|
||||
expect(JSON.parse(result.stdout)).toMatchObject({
|
||||
answer: 'yes',
|
||||
messageId: pending.question.message_id,
|
||||
timedOut: false
|
||||
})
|
||||
expect(result.postOutput).toEqual({
|
||||
kind: 'legacy_question_ack',
|
||||
terminal: WORKER_HANDLE,
|
||||
questionId: pending.question.message_id,
|
||||
answerMessageId: answer.message.id
|
||||
})
|
||||
expect(db.getMessageById(answer.message.id)?.read).toBe(0)
|
||||
|
||||
await acknowledgeRemoteOrcaCliPostOutput(runtime, {
|
||||
postOutput: result.postOutput!,
|
||||
env: WORKER_ENV,
|
||||
runtimeAuthority: RUNTIME_AUTHORITY
|
||||
})
|
||||
|
||||
expect(db.getMessageById(answer.message.id)?.read).toBe(1)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,143 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { readOrchestrationCompatibilityEvidence } from '../../shared/orchestration-compatibility-evidence'
|
||||
import { ORCHESTRATION_CONTRACT_VERSION } from '../../shared/protocol-version'
|
||||
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
|
||||
import type { RpcResponse } from '../runtime/rpc/core'
|
||||
import { RpcDispatcher } from '../runtime/rpc/dispatcher'
|
||||
import type {
|
||||
RemoteOrcaCliPostOutput,
|
||||
RemoteOrcaCliRequest
|
||||
} from './ssh-remote-cli-host-passthrough'
|
||||
import { RemoteCliArgumentError, type ParsedRemoteCli } from './ssh-remote-cli-argument-error'
|
||||
import { optionalRemoteCliString, resolveRemoteCliHandle } from './ssh-remote-cli-args'
|
||||
|
||||
export async function acknowledgeRemoteOrcaCliPostOutput(
|
||||
runtime: OrcaRuntimeService,
|
||||
args: {
|
||||
postOutput: RemoteOrcaCliPostOutput
|
||||
env: Record<string, string>
|
||||
runtimeAuthority?: RemoteOrcaCliRequest['runtimeAuthority']
|
||||
}
|
||||
): Promise<void> {
|
||||
const inheritedEvidence = readOrchestrationCompatibilityEvidence(args.env)
|
||||
const orchestrationCompatibilityEvidence = args.runtimeAuthority
|
||||
? { ...inheritedEvidence, host: args.runtimeAuthority }
|
||||
: inheritedEvidence
|
||||
const params =
|
||||
args.postOutput.kind === 'legacy_check_ack'
|
||||
? {
|
||||
terminal: args.postOutput.terminal,
|
||||
compatibilityAck: JSON.stringify({
|
||||
messageIds: args.postOutput.messageIds,
|
||||
types: args.postOutput.types
|
||||
})
|
||||
}
|
||||
: {
|
||||
terminal: args.postOutput.terminal,
|
||||
compatibilityQuestionAck: JSON.stringify({
|
||||
questionId: args.postOutput.questionId,
|
||||
answerMessageId: args.postOutput.answerMessageId
|
||||
})
|
||||
}
|
||||
const response = await new RpcDispatcher({ runtime }).dispatch({
|
||||
id: `remote-cli-post-output-${randomUUID()}`,
|
||||
authToken: 'remote-cli',
|
||||
method: 'orchestration.check',
|
||||
params,
|
||||
orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION,
|
||||
compatibilityInvocationId: randomUUID(),
|
||||
orchestrationCompatibilityEvidence
|
||||
})
|
||||
if (!response.ok) {
|
||||
throw new Error(response.error.message)
|
||||
}
|
||||
}
|
||||
|
||||
export function parseRemoteOrcaCliPostOutput(value: unknown): RemoteOrcaCliPostOutput {
|
||||
if (!isRecord(value) || typeof value.kind !== 'string' || typeof value.terminal !== 'string') {
|
||||
throw invalidPostOutput()
|
||||
}
|
||||
if (
|
||||
value.kind === 'legacy_check_ack' &&
|
||||
Array.isArray(value.messageIds) &&
|
||||
value.messageIds.every((id) => typeof id === 'string') &&
|
||||
(value.types === undefined ||
|
||||
(Array.isArray(value.types) && value.types.every((type) => typeof type === 'string')))
|
||||
) {
|
||||
return {
|
||||
kind: value.kind,
|
||||
terminal: value.terminal,
|
||||
messageIds: value.messageIds,
|
||||
...(value.types === undefined ? {} : { types: value.types })
|
||||
}
|
||||
}
|
||||
if (
|
||||
value.kind === 'legacy_question_ack' &&
|
||||
typeof value.questionId === 'string' &&
|
||||
typeof value.answerMessageId === 'string'
|
||||
) {
|
||||
return {
|
||||
kind: value.kind,
|
||||
terminal: value.terminal,
|
||||
questionId: value.questionId,
|
||||
answerMessageId: value.answerMessageId
|
||||
}
|
||||
}
|
||||
throw invalidPostOutput()
|
||||
}
|
||||
|
||||
export function getRemoteCliPostOutput(
|
||||
parsed: ParsedRemoteCli,
|
||||
env: Record<string, string>,
|
||||
response: RpcResponse
|
||||
): RemoteOrcaCliPostOutput | undefined {
|
||||
if (!response.ok || !isRecord(response.result)) {
|
||||
return undefined
|
||||
}
|
||||
const compatibility = response.result.legacyCompatibility
|
||||
if (!isRecord(compatibility)) {
|
||||
return undefined
|
||||
}
|
||||
const command = parsed.commandPath.join(' ')
|
||||
if (
|
||||
command === 'orchestration check' &&
|
||||
Array.isArray(compatibility.ackMessageIds) &&
|
||||
compatibility.ackMessageIds.length > 0 &&
|
||||
compatibility.ackMessageIds.every((id) => typeof id === 'string')
|
||||
) {
|
||||
const types = optionalRemoteCliString(parsed.flags, 'types')
|
||||
?.split(',')
|
||||
.map((type) => type.trim())
|
||||
.filter(Boolean)
|
||||
return {
|
||||
kind: 'legacy_check_ack',
|
||||
terminal: resolveRemoteCliHandle(parsed.flags, env, 'terminal'),
|
||||
messageIds: compatibility.ackMessageIds,
|
||||
...(types ? { types } : {})
|
||||
}
|
||||
}
|
||||
const acknowledgement = compatibility.answerAcknowledgement
|
||||
if (
|
||||
command === 'orchestration ask' &&
|
||||
response.result.answer !== null &&
|
||||
isRecord(acknowledgement) &&
|
||||
typeof acknowledgement.questionId === 'string' &&
|
||||
typeof acknowledgement.answerMessageId === 'string'
|
||||
) {
|
||||
return {
|
||||
kind: 'legacy_question_ack',
|
||||
terminal: resolveRemoteCliHandle(parsed.flags, env, 'from'),
|
||||
questionId: acknowledgement.questionId,
|
||||
answerMessageId: acknowledgement.answerMessageId
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
function invalidPostOutput(): RemoteCliArgumentError {
|
||||
return new RemoteCliArgumentError('invalid_argument', 'Invalid SSH CLI post-output action.')
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
|
||||
}
|
||||
@@ -829,4 +829,73 @@ describe('attachMainWindowServices', () => {
|
||||
await expect(revealPromise).resolves.toEqual({ tabId: 'tab-1', title: 'SSH tmux' })
|
||||
expect(removeListenerMock).toHaveBeenCalledWith('terminal:tabCreateReply', handler)
|
||||
})
|
||||
|
||||
it('requires an exact renderer identity receipt for recovered worker reveals', async () => {
|
||||
const sendMock = vi.fn()
|
||||
const mainWindow = createMainWindow({ send: sendMock })
|
||||
const runtime = createRuntime()
|
||||
|
||||
attachMainWindowServices(mainWindow as never, createStore(), runtime as never)
|
||||
|
||||
const notifier = runtime.setNotifier.mock.calls[0][0] as {
|
||||
revealTerminalSession: (
|
||||
worktreeId: string,
|
||||
opts: {
|
||||
ptyId: string
|
||||
tabId: string
|
||||
leafId: string
|
||||
expectedProcessIdentity: { terminalHandle: string; incarnationId: string }
|
||||
}
|
||||
) => Promise<unknown>
|
||||
}
|
||||
const opts = {
|
||||
ptyId: 'pty-worker',
|
||||
tabId: 'tab-worker',
|
||||
leafId: 'leaf-worker',
|
||||
expectedProcessIdentity: {
|
||||
terminalHandle: 'term_worker',
|
||||
incarnationId: 'inc-worker'
|
||||
}
|
||||
}
|
||||
const mismatch = notifier.revealTerminalSession('worktree-1', opts)
|
||||
const mismatchPayload = sendMock.mock.calls.at(-1)?.[1]
|
||||
const mismatchHandler = onMock.mock.calls.findLast(
|
||||
([channel]) => channel === 'terminal:tabCreateReply'
|
||||
)?.[1]
|
||||
mismatchHandler?.(
|
||||
{ sender: mainWindow.webContents },
|
||||
{
|
||||
requestId: mismatchPayload.requestId,
|
||||
tabId: 'tab-worker',
|
||||
identity: {
|
||||
worktreeId: 'worktree-1',
|
||||
tabId: 'tab-worker',
|
||||
leafId: 'leaf-worker',
|
||||
ptyId: 'pty-replacement'
|
||||
}
|
||||
}
|
||||
)
|
||||
await expect(mismatch).rejects.toThrow('terminal_reveal_identity_mismatch')
|
||||
|
||||
const exact = notifier.revealTerminalSession('worktree-1', opts)
|
||||
const exactPayload = sendMock.mock.calls.at(-1)?.[1]
|
||||
const exactHandler = onMock.mock.calls.findLast(
|
||||
([channel]) => channel === 'terminal:tabCreateReply'
|
||||
)?.[1]
|
||||
const identity = {
|
||||
worktreeId: 'worktree-1',
|
||||
tabId: 'tab-worker',
|
||||
leafId: 'leaf-worker',
|
||||
ptyId: 'pty-worker'
|
||||
}
|
||||
exactHandler?.(
|
||||
{ sender: mainWindow.webContents },
|
||||
{ requestId: exactPayload.requestId, tabId: 'tab-worker', identity }
|
||||
)
|
||||
await expect(exact).resolves.toEqual({
|
||||
tabId: 'tab-worker',
|
||||
title: undefined,
|
||||
identity
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -49,6 +49,7 @@ import type {
|
||||
RuntimeMarkdownSaveTabResult
|
||||
} from '../../shared/mobile-markdown-document'
|
||||
import type { RuntimeMobileSessionTabMove } from '../../shared/runtime-types'
|
||||
import type { TerminalTabCreateReply } from '../../shared/terminal-reveal-identity'
|
||||
import { isNativeFileDropPayload, type NativeFileDropPayload } from '../../shared/native-file-drop'
|
||||
import { requestMobileMarkdownFromRenderer } from './mobile-markdown-request-relay'
|
||||
import { requestTerminalTabCloseFromRenderer } from './terminal-tab-close-request-relay'
|
||||
@@ -347,14 +348,20 @@ function registerRuntimeWindowLifecycle(
|
||||
revealTerminalSession: (worktreeId, opts) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const requestId = randomUUID()
|
||||
const expectedIdentity = opts.expectedProcessIdentity
|
||||
? opts.tabId && opts.leafId
|
||||
? { worktreeId, tabId: opts.tabId, leafId: opts.leafId, ptyId: opts.ptyId }
|
||||
: null
|
||||
: undefined
|
||||
if (expectedIdentity === null) {
|
||||
reject(new Error('terminal_reveal_identity_required'))
|
||||
return
|
||||
}
|
||||
const timer = setTimeout(() => {
|
||||
ipcMain.removeListener('terminal:tabCreateReply', handler)
|
||||
reject(new Error('Terminal reveal timed out'))
|
||||
}, 10_000)
|
||||
const handler = (
|
||||
event: Electron.IpcMainEvent,
|
||||
reply: { requestId: string; tabId?: string; title?: string; error?: string }
|
||||
): void => {
|
||||
const handler = (event: Electron.IpcMainEvent, reply: TerminalTabCreateReply): void => {
|
||||
// Why: requestId is renderer-supplied, so only the targeted main window may satisfy the reveal.
|
||||
if (event.sender !== mainWindow.webContents || reply.requestId !== requestId) {
|
||||
return
|
||||
@@ -365,7 +372,22 @@ function registerRuntimeWindowLifecycle(
|
||||
reject(new Error(reply.error))
|
||||
return
|
||||
}
|
||||
resolve({ tabId: reply.tabId!, title: reply.title })
|
||||
if (
|
||||
expectedIdentity &&
|
||||
(!reply.identity ||
|
||||
reply.identity.worktreeId !== expectedIdentity.worktreeId ||
|
||||
reply.identity.tabId !== expectedIdentity.tabId ||
|
||||
reply.identity.leafId !== expectedIdentity.leafId ||
|
||||
reply.identity.ptyId !== expectedIdentity.ptyId)
|
||||
) {
|
||||
reject(new Error('terminal_reveal_identity_mismatch'))
|
||||
return
|
||||
}
|
||||
resolve({
|
||||
tabId: reply.tabId!,
|
||||
title: reply.title,
|
||||
...(reply.identity ? { identity: reply.identity } : {})
|
||||
})
|
||||
}
|
||||
ipcMain.on('terminal:tabCreateReply', handler)
|
||||
send('ui:createTerminal', {
|
||||
|
||||
@@ -18,6 +18,7 @@ import type {
|
||||
TerminalTabCloseRequest,
|
||||
TerminalTabCloseResponse
|
||||
} from '../shared/terminal-tab-close'
|
||||
import type { TerminalTabCreateReply } from '../shared/terminal-reveal-identity'
|
||||
import type {
|
||||
LocalLogTailChangedPayload,
|
||||
LocalLogTailReadArgs,
|
||||
@@ -3149,12 +3150,7 @@ export type PreloadApi = {
|
||||
onRequestTerminalTabMount: (
|
||||
callback: (data: { worktreeId: string; tabId?: string; ptyId?: string }) => void
|
||||
) => () => void
|
||||
replyTerminalCreate: (reply: {
|
||||
requestId: string
|
||||
tabId?: string
|
||||
title?: string
|
||||
error?: string
|
||||
}) => void
|
||||
replyTerminalCreate: (reply: TerminalTabCreateReply) => void
|
||||
onSplitTerminal: (
|
||||
callback: (data: {
|
||||
tabId: string
|
||||
|
||||
@@ -13,6 +13,7 @@ import type { CliInstallStatus } from '../shared/cli-install-types'
|
||||
import type { AgentHookInstallStatus } from '../shared/agent-hook-types'
|
||||
import type { CodexConfigSyncStatus } from '../shared/codex-config-sync-types'
|
||||
import type { TerminalPaneSplitSource } from '../shared/feature-education-telemetry'
|
||||
import type { TerminalTabCreateReply } from '../shared/terminal-reveal-identity'
|
||||
import type { ProjectExecutionRuntimeResolution } from '../shared/project-execution-runtime'
|
||||
import type { StartupCommandDelivery } from '../shared/codex-startup-delivery'
|
||||
import type {
|
||||
@@ -3752,12 +3753,7 @@ const api = {
|
||||
ipcRenderer.on('terminal:requestTabMount', listener)
|
||||
return () => ipcRenderer.removeListener('terminal:requestTabMount', listener)
|
||||
},
|
||||
replyTerminalCreate: (reply: {
|
||||
requestId: string
|
||||
tabId?: string
|
||||
title?: string
|
||||
error?: string
|
||||
}): void => {
|
||||
replyTerminalCreate: (reply: TerminalTabCreateReply): void => {
|
||||
ipcRenderer.send('terminal:tabCreateReply', reply)
|
||||
},
|
||||
onSplitTerminal: (
|
||||
|
||||
+85
-11
@@ -198,6 +198,8 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise<void> {
|
||||
let nextSeq = 1
|
||||
let highestReceivedSeq = 0
|
||||
const requestId = 1
|
||||
const postOutputRequestId = 2
|
||||
let initialExitCode = 0
|
||||
|
||||
const sendRequest = (): void => {
|
||||
const env = pickRemoteCliEnv(process.env)
|
||||
@@ -219,6 +221,59 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise<void> {
|
||||
sock.write(frame)
|
||||
}
|
||||
|
||||
const finish = (exitCode: number): void => {
|
||||
sock.destroy()
|
||||
process.exit(exitCode)
|
||||
}
|
||||
|
||||
const sendPostOutput = (postOutput: unknown): void => {
|
||||
sock.write(
|
||||
encodeJsonRpcFrame(
|
||||
{
|
||||
jsonrpc: '2.0',
|
||||
id: postOutputRequestId,
|
||||
method: 'orca.cli.postOutput',
|
||||
params: { postOutput, env: pickRemoteCliEnv(process.env) }
|
||||
},
|
||||
nextSeq++,
|
||||
highestReceivedSeq
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
const writeOutput = (
|
||||
result: { stdout?: unknown; stderr?: unknown },
|
||||
onFlushed: (error?: Error) => void
|
||||
): void => {
|
||||
const writes = [
|
||||
[process.stdout, result.stdout],
|
||||
[process.stderr, result.stderr]
|
||||
].filter((entry): entry is [NodeJS.WriteStream, string] => typeof entry[1] === 'string')
|
||||
if (writes.length === 0) {
|
||||
onFlushed()
|
||||
return
|
||||
}
|
||||
let pending = writes.length
|
||||
let completed = false
|
||||
for (const [stream, output] of writes) {
|
||||
stream.write(output, (error) => {
|
||||
if (completed) {
|
||||
return
|
||||
}
|
||||
if (error) {
|
||||
completed = true
|
||||
onFlushed(error)
|
||||
return
|
||||
}
|
||||
pending -= 1
|
||||
if (pending === 0) {
|
||||
completed = true
|
||||
onFlushed()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const decoder = new FrameDecoder((frame: DecodedFrame) => {
|
||||
if (frame.id > highestReceivedSeq) {
|
||||
highestReceivedSeq = frame.id
|
||||
@@ -227,28 +282,41 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise<void> {
|
||||
return
|
||||
}
|
||||
const msg = parseJsonRpcMessage(frame.payload)
|
||||
if (!('id' in msg) || msg.id !== requestId || !('result' in msg || 'error' in msg)) {
|
||||
if (
|
||||
!('id' in msg) ||
|
||||
(msg.id !== requestId && msg.id !== postOutputRequestId) ||
|
||||
!('result' in msg || 'error' in msg)
|
||||
) {
|
||||
return
|
||||
}
|
||||
const response = msg as JsonRpcResponse
|
||||
if (response.error) {
|
||||
process.stderr.write(`${response.error.message}\n`)
|
||||
sock.destroy()
|
||||
process.exit(1)
|
||||
finish(1)
|
||||
return
|
||||
}
|
||||
if (response.id === postOutputRequestId) {
|
||||
finish(initialExitCode)
|
||||
return
|
||||
}
|
||||
const result = (response.result ?? {}) as {
|
||||
stdout?: unknown
|
||||
stderr?: unknown
|
||||
exitCode?: unknown
|
||||
postOutput?: unknown
|
||||
}
|
||||
if (typeof result.stdout === 'string' && result.stdout.length > 0) {
|
||||
process.stdout.write(result.stdout)
|
||||
}
|
||||
if (typeof result.stderr === 'string' && result.stderr.length > 0) {
|
||||
process.stderr.write(result.stderr)
|
||||
}
|
||||
sock.destroy()
|
||||
process.exit(typeof result.exitCode === 'number' ? result.exitCode : 0)
|
||||
initialExitCode = typeof result.exitCode === 'number' ? result.exitCode : 0
|
||||
writeOutput(result, (error) => {
|
||||
if (error) {
|
||||
finish(1)
|
||||
return
|
||||
}
|
||||
if (result.postOutput === undefined) {
|
||||
finish(initialExitCode)
|
||||
return
|
||||
}
|
||||
sendPostOutput(result.postOutput)
|
||||
})
|
||||
})
|
||||
|
||||
const connectTimeout = setTimeout(() => {
|
||||
@@ -443,6 +511,12 @@ async function main(): Promise<void> {
|
||||
timeoutMs: remoteCliRequestTimeoutMs(params)
|
||||
})
|
||||
})
|
||||
dispatcher.onRequest('orca.cli.postOutput', async (params, context) => {
|
||||
return await dispatcher.requestAnyClient('orca.cli.postOutput', params, {
|
||||
excludeClientId: context.clientId,
|
||||
timeoutMs: remoteCliRequestTimeoutMs(params)
|
||||
})
|
||||
})
|
||||
|
||||
function configureRelayGraceTime(params: Record<string, unknown>): { graceTimeMs: number } {
|
||||
const seconds = Number(params.graceTimeSeconds)
|
||||
|
||||
@@ -1504,7 +1504,6 @@ describe('useIpcEvents updater integration', () => {
|
||||
openAgentTabsInChatByDefault: false
|
||||
}
|
||||
}
|
||||
|
||||
vi.doMock('react', async () => {
|
||||
const actual = await vi.importActual<typeof ReactModule>('react')
|
||||
return {
|
||||
@@ -1789,7 +1788,11 @@ describe('useIpcEvents updater integration', () => {
|
||||
})
|
||||
|
||||
it('surfaces terminal creates without stealing focus unless requested', async () => {
|
||||
const createTab = vi.fn(() => ({ id: 'tab-new' }))
|
||||
const createTab = vi.fn(
|
||||
(_worktreeId: string, _groupId?: string, _tabType?: string, options?: { id?: string }) => ({
|
||||
id: options?.id ?? 'tab-new'
|
||||
})
|
||||
)
|
||||
const setActiveView = vi.fn()
|
||||
const setActiveWorktree = vi.fn()
|
||||
const markWorktreeVisited = vi.fn()
|
||||
@@ -1876,6 +1879,20 @@ describe('useIpcEvents updater integration', () => {
|
||||
activeRuntimeEnvironmentId: undefined as string | undefined
|
||||
}
|
||||
}
|
||||
updateTabPtyId.mockImplementation((tabId: string, ptyId: string) => {
|
||||
storeState.ptyIdsByTabId[tabId] = [
|
||||
...new Set([...(storeState.ptyIdsByTabId[tabId] ?? []), ptyId])
|
||||
]
|
||||
for (const tabs of Object.values(storeState.tabsByWorktree)) {
|
||||
const tab = tabs.find((candidate) => candidate.id === tabId)
|
||||
if (tab) {
|
||||
tab.ptyId = ptyId
|
||||
}
|
||||
}
|
||||
})
|
||||
setTabLayout.mockImplementation((tabId: string, layout: unknown) => {
|
||||
storeState.terminalLayoutsByTabId[tabId] = layout
|
||||
})
|
||||
const createTerminalListenerRef: {
|
||||
current:
|
||||
| ((data: {
|
||||
@@ -2280,6 +2297,7 @@ describe('useIpcEvents updater integration', () => {
|
||||
setActiveTabType.mockClear()
|
||||
setActiveTab.mockClear()
|
||||
revealWorktreeInSidebar.mockClear()
|
||||
dispatchEvent.mockClear()
|
||||
createTerminalListenerRef.current({
|
||||
worktreeId: 'wt-2',
|
||||
title: 'Runner',
|
||||
@@ -2588,6 +2606,7 @@ describe('useIpcEvents updater integration', () => {
|
||||
setActiveTab.mockClear()
|
||||
revealWorktreeInSidebar.mockClear()
|
||||
replyTerminalCreate.mockClear()
|
||||
dispatchEvent.mockClear()
|
||||
requestTerminalCreateListenerRef.current({
|
||||
requestId: 'req-renderer-backed-background',
|
||||
worktreeId: 'wt-2',
|
||||
@@ -2605,6 +2624,12 @@ describe('useIpcEvents updater integration', () => {
|
||||
expect(setActiveTabType).not.toHaveBeenCalled()
|
||||
expect(setActiveTab).not.toHaveBeenCalled()
|
||||
expect(revealWorktreeInSidebar).not.toHaveBeenCalled()
|
||||
expect(dispatchEvent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: 'orca-background-mount-terminal-worktree',
|
||||
detail: { worktreeId: 'wt-2', tabIds: ['tab-new'] }
|
||||
})
|
||||
)
|
||||
expect(replyTerminalCreate).toHaveBeenCalledWith({
|
||||
requestId: 'req-renderer-backed-background',
|
||||
tabId: 'tab-new',
|
||||
@@ -2708,6 +2733,7 @@ describe('useIpcEvents updater integration', () => {
|
||||
setActiveTabType.mockClear()
|
||||
setActiveTab.mockClear()
|
||||
revealWorktreeInSidebar.mockClear()
|
||||
dispatchEvent.mockClear()
|
||||
createTerminalListenerRef.current({
|
||||
worktreeId: 'wt-2',
|
||||
ptyId: 'pty-bg-3',
|
||||
@@ -2725,6 +2751,13 @@ describe('useIpcEvents updater integration', () => {
|
||||
expect(setActiveTabType).not.toHaveBeenCalled()
|
||||
expect(setActiveTab).not.toHaveBeenCalled()
|
||||
expect(revealWorktreeInSidebar).not.toHaveBeenCalled()
|
||||
expect(dispatchEvent).toHaveBeenCalledOnce()
|
||||
expect(dispatchEvent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: 'orca-background-mount-terminal-worktree',
|
||||
detail: { worktreeId: 'wt-2', tabIds: ['tab-cli-bg-reveal'] }
|
||||
})
|
||||
)
|
||||
|
||||
createTab.mockClear()
|
||||
setActiveView.mockClear()
|
||||
@@ -2734,6 +2767,7 @@ describe('useIpcEvents updater integration', () => {
|
||||
revealWorktreeInSidebar.mockClear()
|
||||
focusRuntimeTerminalSurface.mockClear()
|
||||
focusTerminalTabSurface.mockClear()
|
||||
dispatchEvent.mockClear()
|
||||
createTerminalListenerRef.current({
|
||||
worktreeId: 'wt-2',
|
||||
ptyId: 'pty-recovery-bg',
|
||||
@@ -2754,6 +2788,13 @@ describe('useIpcEvents updater integration', () => {
|
||||
expect(revealWorktreeInSidebar).not.toHaveBeenCalled()
|
||||
expect(focusRuntimeTerminalSurface).not.toHaveBeenCalled()
|
||||
expect(focusTerminalTabSurface).not.toHaveBeenCalled()
|
||||
expect(dispatchEvent).toHaveBeenCalledOnce()
|
||||
expect(dispatchEvent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: 'orca-background-mount-terminal-worktree',
|
||||
detail: { worktreeId: 'wt-2', tabIds: ['tab-recovery-bg'] }
|
||||
})
|
||||
)
|
||||
|
||||
storeState.tabsByWorktree = {
|
||||
'wt-2': [{ id: 'tab-existing', ptyId: 'pty-bg', title: 'Terminal 1' }]
|
||||
@@ -2836,7 +2877,13 @@ describe('useIpcEvents updater integration', () => {
|
||||
expect(replyTerminalCreate).toHaveBeenCalledWith({
|
||||
requestId: 'req-adopt-pending',
|
||||
tabId: pendingTabId,
|
||||
title: 'Terminal 3'
|
||||
title: 'Terminal 3',
|
||||
identity: {
|
||||
worktreeId: 'wt-2',
|
||||
tabId: pendingTabId,
|
||||
leafId: pendingLeafId,
|
||||
ptyId: 'serve-cf39bedb-a33a-417c-9ab6-f304dc27a6c0'
|
||||
}
|
||||
})
|
||||
|
||||
storeState.tabsByWorktree = {
|
||||
@@ -2902,7 +2949,13 @@ describe('useIpcEvents updater integration', () => {
|
||||
expect(replyTerminalCreate).toHaveBeenCalledWith({
|
||||
requestId: 'req-split',
|
||||
tabId: 'tab-existing',
|
||||
title: 'Terminal 1'
|
||||
title: 'Terminal 1',
|
||||
identity: {
|
||||
worktreeId: 'wt-2',
|
||||
tabId: 'tab-existing',
|
||||
leafId: 'leaf-split',
|
||||
ptyId: 'pty-split'
|
||||
}
|
||||
})
|
||||
|
||||
storeState.terminalLayoutsByTabId = {
|
||||
@@ -5234,7 +5287,7 @@ describe('useIpcEvents agent status snapshot integration', () => {
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
it('reconciles adopted and exited legacy worker recovery without broad record cleanup', async () => {
|
||||
it('retires the exact sleeping record after adopted or exited legacy worker recovery', async () => {
|
||||
const clearSleepingAgentSession = vi.fn()
|
||||
const setSleepingAgentAutomaticResumeBlocked = vi.fn()
|
||||
let listener:
|
||||
@@ -5275,11 +5328,8 @@ describe('useIpcEvents agent status snapshot integration', () => {
|
||||
|
||||
clearSleepingAgentSession.mockClear()
|
||||
listener?.({ paneKey: 'tab-exited:leaf-exited', resolution: 'exited' })
|
||||
expect(clearSleepingAgentSession).not.toHaveBeenCalled()
|
||||
expect(setSleepingAgentAutomaticResumeBlocked).toHaveBeenCalledWith(
|
||||
'tab-exited:leaf-exited',
|
||||
false
|
||||
)
|
||||
expect(clearSleepingAgentSession).toHaveBeenCalledWith('tab-exited:leaf-exited')
|
||||
expect(setSleepingAgentAutomaticResumeBlocked).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each([
|
||||
|
||||
@@ -101,6 +101,7 @@ import { track } from '@/lib/telemetry'
|
||||
import { singlePaneLayoutSnapshot } from '@/store/slices/terminal-helpers'
|
||||
import { buildWorkspaceSessionPayload } from '@/lib/workspace-session'
|
||||
import { persistWorkspaceSessionByHost } from '@/lib/workspace-session-host-persistence'
|
||||
import { verifyTerminalRevealIdentity } from '@/lib/terminal-reveal-identity'
|
||||
import { getLinearIssueWorkspaceName } from '../../../shared/workspace-name'
|
||||
import type { RuntimeClientEvent } from '../../../shared/runtime-client-events'
|
||||
import { applyHostWorktreeTerminalSleepState } from '@/components/terminal-pane/pty-shutdown-exit-deferral'
|
||||
@@ -1576,11 +1577,24 @@ export function useIpcEvents(): void {
|
||||
...(launchAgent ? { launchAgent } : {})
|
||||
})
|
||||
}
|
||||
if (ptyId && terminalPresentation === 'background') {
|
||||
requestBackgroundTerminalWorktreeMount({ worktreeId, tabIds: [tab.id] })
|
||||
}
|
||||
if (requestId) {
|
||||
const identity =
|
||||
ptyId && tabId && leafId
|
||||
? verifyTerminalRevealIdentity(useAppStore.getState(), {
|
||||
worktreeId,
|
||||
tabId,
|
||||
leafId,
|
||||
ptyId
|
||||
})
|
||||
: undefined
|
||||
window.api.ui.replyTerminalCreate({
|
||||
requestId,
|
||||
tabId: tab.id,
|
||||
title: title ?? tab.title
|
||||
title: title ?? tab.title,
|
||||
...(identity ? { identity } : {})
|
||||
})
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -3348,13 +3362,8 @@ export function useIpcEvents(): void {
|
||||
unsubs.push(unsubscribeMigrationUnsupportedClear)
|
||||
}
|
||||
const unsubscribeLegacyWorkerTerminalRecovery =
|
||||
window.api.agentStatus.onLegacyWorkerTerminalRecovery?.(({ paneKey, resolution }) => {
|
||||
const store = useAppStore.getState()
|
||||
if (resolution === 'adopted') {
|
||||
store.clearSleepingAgentSession(paneKey)
|
||||
return
|
||||
}
|
||||
store.setSleepingAgentAutomaticResumeBlocked(paneKey, false)
|
||||
window.api.agentStatus.onLegacyWorkerTerminalRecovery?.(({ paneKey }) => {
|
||||
useAppStore.getState().clearSleepingAgentSession(paneKey)
|
||||
})
|
||||
if (unsubscribeLegacyWorkerTerminalRecovery) {
|
||||
unsubs.push(unsubscribeLegacyWorkerTerminalRecovery)
|
||||
|
||||
@@ -11,7 +11,7 @@ afterEach(() => {
|
||||
})
|
||||
|
||||
describe('legacy worker sleeping-session recovery', () => {
|
||||
it('keeps the session dormant until exited recovery removes only its resume block', () => {
|
||||
it('never resumes a proven-exited legacy worker on workspace activation', () => {
|
||||
const record: SleepingAgentSessionRecord = {
|
||||
paneKey: 'tab-legacy:leaf-legacy',
|
||||
tabId: 'tab-legacy',
|
||||
@@ -46,16 +46,8 @@ describe('legacy worker sleeping-session recovery', () => {
|
||||
expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(0)
|
||||
expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey]).toBe(record)
|
||||
|
||||
useAppStore.getState().setSleepingAgentAutomaticResumeBlocked(record.paneKey, false)
|
||||
const unblocked = useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey]
|
||||
expect(unblocked).toMatchObject({
|
||||
paneKey: record.paneKey,
|
||||
providerSession: record.providerSession,
|
||||
origin: 'live'
|
||||
})
|
||||
expect(unblocked?.automaticResumeBlockedBy).toBeUndefined()
|
||||
|
||||
expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(1)
|
||||
useAppStore.getState().clearSleepingAgentSession(record.paneKey)
|
||||
expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(0)
|
||||
expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey]).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { verifyTerminalRevealIdentity } from './terminal-reveal-identity'
|
||||
|
||||
const expected = {
|
||||
worktreeId: 'worktree-1',
|
||||
tabId: 'tab-1',
|
||||
leafId: 'leaf-1',
|
||||
ptyId: 'pty-1'
|
||||
}
|
||||
|
||||
const mismatchCases: {
|
||||
name: string
|
||||
state: Parameters<typeof verifyTerminalRevealIdentity>[0]
|
||||
}[] = [
|
||||
{
|
||||
name: 'worktree',
|
||||
state: {
|
||||
tabsByWorktree: { 'worktree-2': [{ id: 'tab-1' }] },
|
||||
terminalLayoutsByTabId: {
|
||||
'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-1' } }
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'leaf',
|
||||
state: {
|
||||
tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] },
|
||||
terminalLayoutsByTabId: {
|
||||
'tab-1': { ptyIdsByLeafId: { 'leaf-2': 'pty-1' } }
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'pty',
|
||||
state: {
|
||||
tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] },
|
||||
terminalLayoutsByTabId: {
|
||||
'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-2' } }
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
describe('verifyTerminalRevealIdentity', () => {
|
||||
it('returns the exact fresh renderer binding', () => {
|
||||
expect(
|
||||
verifyTerminalRevealIdentity(
|
||||
{
|
||||
tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] },
|
||||
terminalLayoutsByTabId: {
|
||||
'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-1' } }
|
||||
}
|
||||
},
|
||||
expected
|
||||
)
|
||||
).toEqual(expected)
|
||||
})
|
||||
|
||||
it.each(mismatchCases)('rejects a mismatched $name owner', ({ state }) => {
|
||||
expect(() => verifyTerminalRevealIdentity(state, expected)).toThrow(
|
||||
'terminal_reveal_identity_mismatch'
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,23 @@
|
||||
import type { TerminalRevealIdentity } from '../../../shared/terminal-reveal-identity'
|
||||
|
||||
type TerminalRevealState = {
|
||||
tabsByWorktree: Record<string, readonly { id: string }[]>
|
||||
terminalLayoutsByTabId: Record<
|
||||
string,
|
||||
{ ptyIdsByLeafId?: Record<string, string | undefined> } | undefined
|
||||
>
|
||||
}
|
||||
|
||||
export function verifyTerminalRevealIdentity(
|
||||
state: TerminalRevealState,
|
||||
expected: TerminalRevealIdentity
|
||||
): TerminalRevealIdentity {
|
||||
const ownsTab = state.tabsByWorktree[expected.worktreeId]?.some(
|
||||
(tab) => tab.id === expected.tabId
|
||||
)
|
||||
const boundPtyId = state.terminalLayoutsByTabId[expected.tabId]?.ptyIdsByLeafId?.[expected.leafId]
|
||||
if (!ownsTab || boundPtyId !== expected.ptyId) {
|
||||
throw new Error('terminal_reveal_identity_mismatch')
|
||||
}
|
||||
return expected
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { prepareOrchestrationCheckOutput } from './orchestration-check-output'
|
||||
|
||||
describe('prepareOrchestrationCheckOutput', () => {
|
||||
it('keeps mixed read-only mail safe and current Run replies executable', () => {
|
||||
const prepared = prepareOrchestrationCheckOutput(
|
||||
{
|
||||
count: 2,
|
||||
messages: [
|
||||
{
|
||||
id: 'msg_current',
|
||||
run_id: 'run_adopted',
|
||||
delivery_contract: 'current_delivery',
|
||||
from_handle: 'term_worker',
|
||||
to_handle: 'run:run_adopted',
|
||||
subject: 'Question'
|
||||
},
|
||||
{
|
||||
id: 'msg_legacy',
|
||||
run_id: 'run_legacy_local',
|
||||
delivery_contract: 'audit_only',
|
||||
from_handle: 'term_legacy',
|
||||
to_handle: 'term_coord',
|
||||
subject: 'Old reply'
|
||||
}
|
||||
],
|
||||
formatted: '[Reply: unsafe stale formatter output]'
|
||||
},
|
||||
'term_current_coord',
|
||||
true
|
||||
)
|
||||
|
||||
expect(prepared.formatted).toContain(
|
||||
'[Reply: orca orchestration reply --id msg_current --body "..."]'
|
||||
)
|
||||
expect(prepared.formatted).not.toContain('--from run:run_adopted')
|
||||
expect(prepared.formatted).toContain(
|
||||
'[Inspection only: reply and acknowledgment are unavailable.]'
|
||||
)
|
||||
expect(prepared.formatted).not.toContain('unsafe stale formatter output')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,199 @@
|
||||
import { ORCHESTRATION_LEGACY_RUN_ID } from './orchestration-rpc-contract'
|
||||
|
||||
export type OrchestrationMessageSummary = {
|
||||
id: string
|
||||
run_id?: string
|
||||
delivery_contract?: 'legacy_direct' | 'current_delivery' | 'audit_only'
|
||||
from_handle: string
|
||||
to_handle?: string
|
||||
subject?: string
|
||||
type?: string
|
||||
body?: string
|
||||
payload?: string | null
|
||||
priority?: string
|
||||
read?: number
|
||||
}
|
||||
|
||||
export type LegacyCompatibilityResult = {
|
||||
recovery?: boolean
|
||||
readOnly?: boolean
|
||||
ackMessageIds?: string[]
|
||||
answerAcknowledgement?: {
|
||||
questionId: string
|
||||
answerMessageId: string
|
||||
}
|
||||
currentDelivery?: {
|
||||
runId: string
|
||||
checkCommand: string
|
||||
ackCommand: string
|
||||
}
|
||||
resumeRequired?: boolean
|
||||
resumeCommand?: string
|
||||
}
|
||||
|
||||
export type OrchestrationCheckOutput = {
|
||||
messages: OrchestrationMessageSummary[]
|
||||
count: number
|
||||
formatted?: string
|
||||
deliveryId?: string | null
|
||||
timedOut?: boolean
|
||||
cancelled?: boolean
|
||||
connectionLost?: boolean
|
||||
legacyCompatibility?: LegacyCompatibilityResult
|
||||
}
|
||||
|
||||
export function formatMessageReadOnlyTag(
|
||||
message: OrchestrationMessageSummary,
|
||||
legacyCompatibilityActive = false
|
||||
): string {
|
||||
return isLegacyReadOnlyMessage(message, legacyCompatibilityActive) ? ' [legacy, read-only]' : ''
|
||||
}
|
||||
|
||||
export function isLegacyReadOnlyMessage(
|
||||
message: OrchestrationMessageSummary,
|
||||
legacyCompatibilityActive = false
|
||||
): boolean {
|
||||
return (
|
||||
message.run_id === ORCHESTRATION_LEGACY_RUN_ID ||
|
||||
(message.delivery_contract === 'legacy_direct' && !legacyCompatibilityActive) ||
|
||||
message.delivery_contract === 'audit_only'
|
||||
)
|
||||
}
|
||||
|
||||
export function formatOrchestrationCheckText(
|
||||
result: OrchestrationCheckOutput,
|
||||
checkedTerminal: string
|
||||
): string {
|
||||
const prepared = prepareOrchestrationCheckOutput(
|
||||
result,
|
||||
checkedTerminal,
|
||||
result.formatted !== undefined
|
||||
)
|
||||
const compatibilityActive = Boolean(
|
||||
prepared.legacyCompatibility && !prepared.legacyCompatibility.readOnly
|
||||
)
|
||||
const legacyHeader = prepared.legacyCompatibility
|
||||
? prepared.legacyCompatibility.readOnly
|
||||
? '[LEGACY READ-ONLY]\n'
|
||||
: prepared.legacyCompatibility.recovery
|
||||
? '[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]\n'
|
||||
: '[LEGACY COMPATIBILITY]\n'
|
||||
: ''
|
||||
const deliveryNotice = formatCurrentDeliveryNotice(prepared.legacyCompatibility?.currentDelivery)
|
||||
if (prepared.formatted) {
|
||||
return `${legacyHeader}${prepared.formatted}${deliveryNotice}`
|
||||
}
|
||||
if (prepared.count === 0) {
|
||||
if (prepared.timedOut) {
|
||||
return `${legacyHeader}Wait timed out; no messages were consumed.${deliveryNotice}`
|
||||
}
|
||||
if (prepared.cancelled) {
|
||||
const cancelled = prepared.connectionLost
|
||||
? 'Wait cancelled because the connection closed; no messages were consumed.'
|
||||
: 'Wait cancelled; no messages were consumed.'
|
||||
return `${legacyHeader}${cancelled}${deliveryNotice}`
|
||||
}
|
||||
return `${legacyHeader}No messages.${deliveryNotice}`
|
||||
}
|
||||
const rendered = prepared.messages
|
||||
.map(
|
||||
(message) =>
|
||||
`${message.id}${formatMessageReadOnlyTag(
|
||||
message,
|
||||
compatibilityActive
|
||||
)} [${message.type ?? 'status'}] from=${message.from_handle} "${message.subject}"`
|
||||
)
|
||||
.join('\n')
|
||||
const output = prepared.deliveryId ? `Delivery ${prepared.deliveryId}\n${rendered}` : rendered
|
||||
return `${legacyHeader}${output}${deliveryNotice}`
|
||||
}
|
||||
|
||||
export function prepareOrchestrationCheckOutput<T extends OrchestrationCheckOutput>(
|
||||
result: T,
|
||||
checkedTerminal: string,
|
||||
formattedRequested: boolean
|
||||
): T {
|
||||
const compatibilityActive = Boolean(
|
||||
result.legacyCompatibility && !result.legacyCompatibility.readOnly
|
||||
)
|
||||
if (
|
||||
!formattedRequested ||
|
||||
!result.messages.some((message) => isLegacyReadOnlyMessage(message, compatibilityActive))
|
||||
) {
|
||||
return result
|
||||
}
|
||||
return {
|
||||
...result,
|
||||
formatted: formatLegacyAwareCheckMessages(result.messages, checkedTerminal, compatibilityActive)
|
||||
}
|
||||
}
|
||||
|
||||
function formatMessagePriorityTag(message: OrchestrationMessageSummary): string {
|
||||
return message.priority === 'urgent' ? ' [URGENT]' : message.priority === 'high' ? ' [HIGH]' : ''
|
||||
}
|
||||
|
||||
function escapeTerminalControlCharacters(value: string): string {
|
||||
return [...value]
|
||||
.map((character) => {
|
||||
const code = character.charCodeAt(0)
|
||||
if (character === '\n' || (code >= 0x20 && code < 0x7f) || code > 0x9f) {
|
||||
return character
|
||||
}
|
||||
return `\\x${code.toString(16).padStart(2, '0')}`
|
||||
})
|
||||
.join('')
|
||||
}
|
||||
|
||||
function formatQuotedMessageField(label: string, value?: string): string {
|
||||
return `[${label}]\n${escapeTerminalControlCharacters(value ?? '')
|
||||
.split('\n')
|
||||
.map((line) => ` ${line}`)
|
||||
.join('\n')}`
|
||||
}
|
||||
|
||||
function formatLegacyAwareCheckMessages(
|
||||
messages: OrchestrationMessageSummary[],
|
||||
checkedTerminal: string,
|
||||
legacyCompatibilityActive = false
|
||||
): string {
|
||||
return messages
|
||||
.map((message) => {
|
||||
const legacyReadOnly = isLegacyReadOnlyMessage(message, legacyCompatibilityActive)
|
||||
const lines = [
|
||||
`${message.id}${formatMessageReadOnlyTag(message, legacyCompatibilityActive)}${formatMessagePriorityTag(message)} [${message.type ?? 'status'}] from=${message.from_handle}`,
|
||||
formatQuotedMessageField('subject', message.subject)
|
||||
]
|
||||
if (legacyReadOnly) {
|
||||
lines.push('[Inspection only: reply and acknowledgment are unavailable.]')
|
||||
}
|
||||
if (message.body) {
|
||||
lines.push(formatQuotedMessageField('body', message.body))
|
||||
}
|
||||
if (message.payload) {
|
||||
lines.push(formatQuotedMessageField('payload', message.payload))
|
||||
}
|
||||
if (!legacyReadOnly) {
|
||||
const replyTarget = message.to_handle ?? checkedTerminal
|
||||
const replyFrom =
|
||||
replyTarget.startsWith('run:') || replyTarget.startsWith('dispatch:')
|
||||
? ''
|
||||
: ` --from ${replyTarget}`
|
||||
lines.push(`[Reply: orca orchestration reply --id ${message.id}${replyFrom} --body "..."]`)
|
||||
}
|
||||
return lines.join('\n')
|
||||
})
|
||||
.join('\n\n')
|
||||
}
|
||||
|
||||
function formatCurrentDeliveryNotice(
|
||||
delivery: LegacyCompatibilityResult['currentDelivery']
|
||||
): string {
|
||||
if (!delivery) {
|
||||
return ''
|
||||
}
|
||||
return (
|
||||
`\n[CURRENT RUN MAIL WAITING]\n` +
|
||||
`Read: ${delivery.checkCommand}\n` +
|
||||
`Then acknowledge the delivery ID it prints: ${delivery.ackCommand}`
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
export type TerminalRevealIdentity = {
|
||||
worktreeId: string
|
||||
tabId: string
|
||||
leafId: string
|
||||
ptyId: string
|
||||
}
|
||||
|
||||
export type TerminalTabCreateReply = {
|
||||
requestId: string
|
||||
tabId?: string
|
||||
title?: string
|
||||
identity?: TerminalRevealIdentity
|
||||
error?: string
|
||||
}
|
||||
Reference in New Issue
Block a user