feat(orchestration): start chat workers on a paired server

A server that prefers chats now starts a remote worker as a chat, the
same way a local start does. The chat takes its task as its first turn
(after a wait-for-setup setup finishes), reports home over the relay
(mail, turn stops, worker_done), takes its coordinator's mail as new
turns, and closes on stop or release, including automatic release.
Reading a remote chat worker's state says plainly that it is not
supported yet.
This commit is contained in:
Jinwoo-H
2026-11-09 03:28:26 -05:00
parent 31f28d6a93
commit b399ba20cd
20 changed files with 1012 additions and 202 deletions
@@ -0,0 +1,30 @@
import { isStructuredWorkerHandle } from '../../../../shared/orchestration/workers/structured-worker-handle'
import type { OrchestrationDb } from '../db/orchestration-db'
import type { RemoteDispatchAttachmentRow } from '../db/orchestration-records'
import { resolveStructuredWorkerIdentity } from '../terminal-authority/structured-sessions/structured-worker-authority'
import type { StructuredWorkerIdentity } from '../terminal-authority/structured-sessions/structured-worker-identity'
/** Said wherever this host would read a remote chat worker's state, until reads of one are supported. */
export const REMOTE_CHAT_WORKER_READ_UNSUPPORTED =
'Orca cannot read the state of a chat worker on a paired server yet; open its chat on that server to see it.'
/** The chat session a coordinator on another computer runs here, with whether it is still the attached one. */
export function remoteChatWorkerOf(
db: OrchestrationDb,
attachment: Pick<RemoteDispatchAttachmentRow, 'dispatch_id' | 'terminal_handle'> | undefined
): { identity: StructuredWorkerIdentity; exact: boolean } | null {
const handle = attachment?.terminal_handle
const identity =
handle && isStructuredWorkerHandle(handle) ? resolveStructuredWorkerIdentity(handle, db) : null
if (!attachment || !identity) {
return null
}
return {
identity,
exact: db.isRemoteAttachmentProcessCurrent({
dispatchId: attachment.dispatch_id,
paneKey: identity.paneKey,
processIncarnation: identity.processIncarnation
})
}
}
@@ -21,6 +21,7 @@ import {
import type { RunRow } from '../../db/orchestration-records'
import { chatAssigneeSessionId } from '../../workers/lifecycle/chat-assignee'
import { structuredWorkerCustody } from '../../terminal-authority/structured-sessions/structured-worker-authority'
import { remoteChatWorkerOf } from '../../federation/remote-chat-worker'
/** A handle-less coordinator uses its generation-current session id; structured workers resolve by handle. */
export function handleLessCoordinatorSessionId(
@@ -118,7 +119,11 @@ export function structuredMailboxCustody(
if (!mailboxHandle.startsWith('dispatch:')) {
return workerMailCustody(mailboxHandle, db)
}
const dispatch = db?.getDispatchContextById?.(mailboxHandle.slice('dispatch:'.length))
const dispatchId = mailboxHandle.slice('dispatch:'.length)
const dispatch = db?.getDispatchContextById?.(dispatchId)
if (!dispatch && db) {
return remoteDispatchMailCustody(dispatchId, db)
}
// A Dispatch's mail follows its Task: none reaches the chat before it took the Task, or after it ended.
if (!dispatch?.assignee_handle || dispatch.status !== 'dispatched') {
return 'elsewhere'
@@ -129,6 +134,16 @@ export function structuredMailboxCustody(
: workerMailCustody(dispatch.assignee_handle, db, dispatch.id)
}
/** A Dispatch a coordinator on another computer runs here reaches its chat only while that chat still works it. */
function remoteDispatchMailCustody(
dispatchId: string,
db: OrchestrationDb
): StructuredMailboxCustody {
const attachment = db.getRemoteDispatchAttachment?.(dispatchId)
const chat = attachment?.state === 'ready' ? remoteChatWorkerOf(db, attachment) : null
return chat?.exact ? workerMailCustody(chat.identity.handle, db, dispatchId) : 'elsewhere'
}
/** Re-derive Run, direct and active-Dispatch mail on each idle edge, including root mail for /clear successors. */
export function structuredSessionOwnedMailboxes(sessionId: string, db: OrchestrationDb): string[] {
const party = isOrcaSessionId(sessionId) ? addressableSessionParty(sessionId, db) : null
@@ -144,6 +159,10 @@ export function structuredSessionOwnedMailboxes(sessionId: string, db: Orchestra
if (dispatch) {
mailboxes.push(`dispatch:${dispatch.id}`)
}
const remote = party.paneKey ? db.findActiveRemoteAttachmentForPane(party.paneKey) : undefined
if (remote && remoteChatWorkerOf(db, remote)?.exact) {
mailboxes.push(`dispatch:${remote.dispatch_id}`)
}
return mailboxes
}
@@ -26,13 +26,19 @@ export type StructuredMailResend = { row: StructuredMailSendRow; renew: boolean
const REUSABLE_OPERATION_AGE_MS =
AGENT_SESSION_MAX_NEW_OPERATION_AGE_MS - AGENT_SESSION_OPERATION_FUTURE_SKEW_MS
/** A mailbox nothing can own again: its Dispatch settled or is gone, or its Run is gone. */
/** A mailbox nothing can own again: its Dispatch (or remote attachment) settled or is gone, or its Run is gone. */
export function structuredMailboxEnded(
mailboxHandle: string,
db: OrchestrationDb | null | undefined
): boolean {
if (mailboxHandle.startsWith('dispatch:')) {
const status = db?.getDispatchContextById?.(mailboxHandle.slice('dispatch:'.length))?.status
const dispatchId = mailboxHandle.slice('dispatch:'.length)
const status = db?.getDispatchContextById?.(dispatchId)?.status
if (status === undefined) {
// A coordinator on another computer runs this Dispatch; here it is only an attachment.
const remote = db?.getRemoteDispatchAttachment?.(dispatchId)?.state
return remote !== 'starting' && remote !== 'ready'
}
return status !== 'pending' && status !== 'dispatched'
}
return mailboxHandle.startsWith('run:') && !db?.getRun?.(mailboxHandle.slice('run:'.length))
@@ -0,0 +1,301 @@
// A coordinator on another computer starts a chat worker here; the worker reports home over the relay.
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { EVERY_AGENT_CHAT } from '../../../../../shared/native-chat/every-agent-chat.test-fixture'
import { isStructuredWorkerHandle } from '../../../../../shared/orchestration/workers/structured-worker-handle'
import * as sessionCreate from '../../../rpc/methods/structured-sessions/creation/structured-agent-session-create'
import { LOCAL_EXECUTION_HOST_ID } from '../../../../../shared/execution-host'
import { chatWorkerTurnStopReporter } from '../../workers/turn-stops/chat-worker-turn-stops'
import { encodeFederatedControlMessage } from '../../federation/relay/federation-control-message'
import * as placement from '../../../rpc/methods/orchestration/worker/folder-worktree-placement'
import * as worktreeName from '../../../rpc/methods/orchestration/worker/worker-worktree-creation'
import { attachParams } from './structured-chat-coordinator-fake-codex-fixture'
import {
WAIT,
call,
codex,
db,
dispatcher,
host,
request,
runtime,
settleTurn,
turnText
} from './structured-chat-coordinator-mail-rig.test-fixture'
import { resolveStructuredWorkerIdentity } from './structured-worker-authority'
import { reconcileRequestedWorkerTerminalReleases } from '../release/worker-terminal-release-reconciliation'
const DISPATCH = 'ctx_remote_chat'
const HOME = 'home_peer'
beforeEach(() => {
vi.spyOn(runtime, 'validateOrchestrationAgentLauncher').mockImplementation(() => {})
vi.spyOn(runtime, 'getStructuredAgentSessionCreateSupport').mockResolvedValue({ supported: true })
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: placement reads only the workspace identity in this rig.
vi.spyOn(runtime, 'showManagedTerminalWorkspace').mockResolvedValue({
id: 'workspace-1',
repoId: 'repo'
} as never)
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: launch-mode selection reads only getSettings on this store double.
vi.spyOn(runtime, 'requireStore').mockReturnValue({
getSettings: () => ({
nativeChatProviders: EVERY_AGENT_CHAT,
agentCmdOverrides: {},
agentDefaultArgs: {},
agentDefaultEnv: {}
})
} as never)
vi.spyOn(sessionCreate, 'createStructuredAgentSessionForWorktree').mockImplementation(
async (args) => {
const attached = await host.attach(args.caller, attachParams(args.envelope.sessionId))
if (!attached.ok) {
throw new Error('scripted worker attach refused')
}
return {
...attached,
value: { ...attached.value, sessionId: args.envelope.sessionId, fence: attached.fence }
}
}
)
// The rig fakes PTY identity for its own terminals; a chat worker's comes from its minted identity.
const identity = (handle: string) =>
isStructuredWorkerHandle(handle) ? resolveStructuredWorkerIdentity(handle, db) : null
vi.mocked(runtime.getTerminalPaneKey).mockImplementation(
(handle) => identity(handle)?.paneKey ?? null
)
vi.mocked(runtime.getTerminalProcessIncarnation).mockImplementation(
(handle) => identity(handle)?.processIncarnation ?? null
)
})
async function federation(name: string, params: Record<string, unknown>): Promise<unknown> {
const response = await dispatcher.dispatch(request(name, params), {
authenticatedCallerFingerprint: HOME
})
if (!response.ok) {
throw new Error(`${name} failed: ${JSON.stringify(response)}`)
}
return response.result
}
/** The worker started and took its task, as a coordinator on another computer starts it. */
async function startedChatWorker() {
const started = federation('orchestration.federationAttachStart', {
runId: 'run_home',
dispatchId: DISPATCH,
taskId: 'task_home',
taskSpec: 'Check the docs and report.',
protocolVersion: 4,
worktree: 'id:workspace-1',
agent: 'codex'
})
await vi.waitFor(() => expect(codex.connections.at(-1)?.turns).toHaveLength(1), WAIT)
const handle = String(db.getRemoteDispatchAttachment(DISPATCH)?.terminal_handle)
const identity = resolveStructuredWorkerIdentity(handle, db)!
const worker = codex.connections.at(-1)!
await settleTurn(identity.sessionId, 0)
return { receipt: await started, handle, identity, worker }
}
const homeBound = () =>
db.listPendingFederationRelay(DISPATCH, 'to_home').map((item) => JSON.parse(item.payload))
describe('a chat worker that a coordinator on another computer starts here', () => {
it('starts as a chat when this host prefers chats, and takes its task as its first turn', async () => {
const { receipt, handle, identity, worker } = await startedChatWorker()
expect(receipt).toMatchObject({
state: 'ready',
terminalHandle: handle,
mode: { mode: 'structured' }
})
expect(isStructuredWorkerHandle(handle)).toBe(true)
expect(worker.turns[0]!.text).toContain('Check the docs and report.')
expect(db.getRemoteDispatchAttachment(DISPATCH)).toMatchObject({
state: 'ready',
pane_key: identity.paneKey,
process_incarnation: identity.processIncarnation
})
})
it('sends its mail home on the relay', async () => {
const { identity } = await startedChatWorker()
await expect(
call(
'orchestration.send',
{ subject: 'Halfway', body: 'docs checked' },
{ sessionId: identity.sessionId }
)
).resolves.toMatchObject({ relay: { destination: 'run_home', dispatchId: DISPATCH } })
expect(homeBound()).toEqual([expect.objectContaining({ subject: 'Halfway', type: 'status' })])
})
it('tells home when its turn ends without a report', async () => {
const { identity } = await startedChatWorker()
const timers: (() => void)[] = []
const report = chatWorkerTurnStopReporter(
{ getDb: () => db, isWaitingForMail: () => false, notify: vi.fn() },
(run) => timers.push(run)
)
report(
{
type: 'completion',
completion: {
scope: {
executionHostId: LOCAL_EXECUTION_HOST_ID,
wslDistro: null,
workspaceId: 'workspace-1',
workspaceKind: 'git-worktree'
},
sessionId: identity.sessionId,
turnId: 'turn-1',
outcome: 'success',
completedAt: 1
}
},
() => ({ status: 'idle', lastAssistantMessage: 'Docs look fine.' })
)
for (const run of timers.splice(0)) {
run()
}
expect(homeBound()).toEqual([
expect.objectContaining({ subject: 'Worker stopped without reporting' })
])
})
it("takes its coordinator's mail as a new turn", async () => {
const { worker } = await startedChatWorker()
await federation('orchestration.federationImport', {
dispatchId: DISPATCH,
items: [
{
dispatch_id: DISPATCH,
direction: 'to_worker',
sequence: 1,
message_id: 'relay_home_1',
kind: 'control_message',
payload: encodeFederatedControlMessage({
from: 'term_coord',
subject: 'Also the changelog',
body: 'Update the changelog too.',
type: 'status',
priority: 'normal',
threadId: null,
payload: null
})
}
]
})
await vi.waitFor(() => expect(worker.turns).toHaveLength(2), WAIT)
expect(turnText(worker.turns[1]!)).toContain('Also the changelog')
})
it('says plainly that its state cannot be read here yet', async () => {
await startedChatWorker()
await expect(
federation('orchestration.federationShow', { dispatchId: DISPATCH })
).resolves.toMatchObject({
observation: {
status: 'unverifiable',
reason: expect.stringContaining('cannot read the state of a chat worker on a paired server')
}
})
for (const read of ['orchestration.federationRead', 'orchestration.federationReadOutput']) {
await expect(federation(read, { dispatchId: DISPATCH })).rejects.toThrow(
/cannot read the state of a chat worker on a paired server/
)
}
})
it('closes its chat when its coordinator stops it', async () => {
const { identity } = await startedChatWorker()
await expect(
federation('orchestration.federationStop', { dispatchId: DISPATCH })
).resolves.toMatchObject({ state: 'stopped', processAction: 'closed_agent_terminal' })
expect(host.hasSession(identity.sessionId)).toBe(false)
})
it('closes its chat when its coordinator releases it after its report', async () => {
const { identity } = await startedChatWorker()
// Its report settled at the Run home, as a confirmed worker_done leaves it.
db.recordRemoteAttachmentStage({
dispatchId: DISPATCH,
state: 'succeeded',
stage: 'worker_reported'
})
await expect(
federation('orchestration.federationRelease', { dispatchId: DISPATCH })
).resolves.toMatchObject({ state: 'released' })
expect(host.hasSession(identity.sessionId)).toBe(false)
expect(db.getRemoteDispatchAttachment(DISPATCH)).toMatchObject({ stage: 'released' })
})
it('closes its chat when its automatic release comes due', async () => {
const { identity } = await startedChatWorker()
db.recordRemoteAttachmentStage({
dispatchId: DISPATCH,
state: 'succeeded',
stage: 'worker_reported'
})
const settledAt = Date.UTC(2026, 0, 1)
db.armWorkerTerminalAutoReleaseStatement(DISPATCH, settledAt)
expect(db.claimDueWorkerTerminalAutoReleases(settledAt + 3_600_000)).toBe(1)
await reconcileRequestedWorkerTerminalReleases(runtime)
expect(host.hasSession(identity.sessionId)).toBe(false)
expect(db.getRemoteDispatchAttachment(DISPATCH)).toMatchObject({ stage: 'released' })
})
it('in a new worktree, takes its task only after a wait-for-setup setup finishes', async () => {
vi.spyOn(placement, 'assertOrchestrationWorktreeCreationSupported').mockResolvedValue()
vi.spyOn(worktreeName, 'resolveWorkerWorktreeName').mockResolvedValue({ name: 'w2' })
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the launch reads only these receipt fields.
vi.spyOn(runtime, 'createManagedWorktree').mockResolvedValue({
worktree: { id: 'workspace-1', displayName: 'w2' },
setupReceipt: {
hookFound: true,
startupPolicy: 'wait-for-setup',
state: 'running',
terminalHandle: 'term_setup'
}
} as never)
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: effects read only handle and title.
vi.spyOn(runtime, 'listTerminals').mockResolvedValue({
terminals: [{ handle: 'term_setup', title: 'Setup' }]
} as never)
const setupDone = Promise.withResolvers<{ exitCode: number }>()
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the gate reads only exitCode.
vi.spyOn(runtime, 'waitForSetupTerminalCompletion').mockReturnValue(setupDone.promise as never)
const started = federation('orchestration.federationAttachStart', {
runId: 'run_home',
dispatchId: DISPATCH,
taskId: 'task_home',
taskSpec: 'Check the docs and report.',
protocolVersion: 4,
worktree: 'new-top-level',
repo: 'id:repo',
agent: 'codex'
})
await vi.waitFor(
() => expect(runtime.waitForSetupTerminalCompletion).toHaveBeenCalledWith('term_setup'),
WAIT
)
expect(codex.connections.flatMap((connection) => connection.turns)).toHaveLength(0)
setupDone.resolve({ exitCode: 0 })
await vi.waitFor(() => expect(codex.connections.at(-1)?.turns).toHaveLength(1), WAIT)
const handle = String(db.getRemoteDispatchAttachment(DISPATCH)?.terminal_handle)
await settleTurn(resolveStructuredWorkerIdentity(handle, db)!.sessionId, 0)
await expect(started).resolves.toMatchObject({ state: 'ready', mode: { mode: 'structured' } })
})
})
@@ -4,9 +4,46 @@ import { AGENT_DONE_QUIET_MS } from '../../../../../shared/agent-status/lifecycl
import { chatWorkerDispatches } from './chat-worker-error-escalation'
import {
reportWorkerTurnStop,
workerTurnStopDispatchId,
type WorkerTurnStopDispatch,
type WorkerTurnStopReason,
type WorkerTurnStopReportDeps
} from './worker-turn-stop-report'
import type { OrchestrationDb } from '../../db/orchestration-db'
import { resolveStructuredWorkerIdentityForSession } from '../../terminal-authority/structured-sessions/structured-worker-authority'
/** The open Dispatches a chat works: its own here, or one a coordinator on another computer started here. */
function chatWorkerTargets(
sessionId: string,
db: OrchestrationDb
): { target: WorkerTurnStopDispatch; from: string; starting: boolean }[] {
const local = chatWorkerDispatches(sessionId, db).map(({ dispatch, from }) => ({
target: { kind: 'local' as const, dispatch },
from,
starting: db.getWorkerDispatch(dispatch.id)?.state === 'starting'
}))
if (local.length > 0) {
return local
}
// A paired server holds its remote workers as attachments; their Dispatch lives at the Run home.
const worker = resolveStructuredWorkerIdentityForSession(sessionId, db)
const attachment = worker && db.findActiveRemoteAttachmentForPane(worker.paneKey)
return worker &&
attachment &&
db.isRemoteAttachmentProcessCurrent({
dispatchId: attachment.dispatch_id,
paneKey: worker.paneKey,
processIncarnation: worker.processIncarnation
})
? [
{
target: { kind: 'remote', attachment },
from: worker.handle,
starting: attachment.state === 'starting'
}
]
: []
}
type ChatStop = {
sessionId: string
@@ -67,7 +104,7 @@ export function chatWorkerTurnStopReporter(
}
// The Dispatch a stop belongs to is one open when it was seen.
const seen = new Set(
chatWorkerDispatches(stop.sessionId, db).map(({ dispatch }) => dispatch.id)
chatWorkerTargets(stop.sessionId, db).map(({ target }) => workerTurnStopDispatchId(target))
)
const deliver = (): void => {
try {
@@ -81,22 +118,17 @@ export function chatWorkerTurnStopReporter(
) {
return
}
const open = chatWorkerDispatches(stop.sessionId, current).filter(({ dispatch }) =>
seen.has(dispatch.id)
const open = chatWorkerTargets(stop.sessionId, current).filter(({ target }) =>
seen.has(workerTurnStopDispatchId(target))
)
if (
stop.reason === 'finished' &&
open.some(
({ dispatch }) => current.getWorkerDispatch(dispatch.id)?.state === 'starting'
)
) {
if (stop.reason === 'finished' && open.some(({ starting }) => starting)) {
setTimer(deliver, AGENT_DONE_QUIET_MS)
return
}
for (const { dispatch, from } of open) {
for (const { target, from } of open) {
reportWorkerTurnStop(
current,
{ kind: 'local', dispatch },
target,
{
from,
stopKey: stop.stopKey,
@@ -1,4 +1,5 @@
import type { OrchestrationWorkerLaunchReceipt } from '../worker/worker-launch-preferences'
import type { AgentLaunchModeReceipt } from '../../../../../../shared/agent-launch/contracts/agent-launch-intent'
export type RemoteFederatedWorkerStartReceipt = {
dispatchId: string
@@ -8,6 +9,8 @@ export type RemoteFederatedWorkerStartReceipt = {
terminalHandle?: string
setup?: { state: string }
launch?: OrchestrationWorkerLaunchReceipt
/** Chat or terminal, as the server decided; absent from servers that start terminals only. */
mode?: AgentLaunchModeReceipt
effects?: unknown[]
residualResources?: unknown[]
failedStage?: string
@@ -44,6 +47,16 @@ export function isReadyRemoteFederatedWorkerStartReceipt(
return receipt.state === 'ready'
}
/** The ready receipt's parts the server reports as is. */
export function readyRemoteReceiptFields(remote: ReadyRemoteFederatedWorkerStartReceipt) {
return {
setup: remote.setup,
...(remote.mode ? { mode: remote.mode } : {}),
effects: remote.effects ?? [],
residualResources: remote.residualResources ?? []
}
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value)
}
@@ -1,5 +1,6 @@
import type { RuntimeTerminalCreate } from '../../../../../../shared/runtime-terminal-create-contract'
/** Execute terminalOnly on the receiving host with federation's agent-first worktree/background-terminal factories. The dispatch brief is delivered later. */
/** Launch on the receiving host in the mode it decided, with federation's worktree, background-terminal and
* chat-session factories. The dispatch brief is delivered later. */
import type { AgentLaunchPreferences } from '../../../../../../shared/agent-session/agent-session-host-authority'
import type { TuiAgent } from '../../../../../../shared/agent-launch/catalog/tui-agent'
@@ -16,8 +17,11 @@ import {
} from './federation-effects'
import type { FederationAttachStartInput } from '../../../../../../shared/rpc-contract/orchestration-federation-start-params'
import type { Worktree } from '../../../../../../shared/worktree/types'
import type { AgentLaunchModeReceipt } from '../../../../../../shared/agent-launch/contracts/agent-launch-intent'
import { createStructuredWorkerSessionForWorktree } from '../worker/worker-topology'
import { tearDownFailedWorkerStart } from '../worker/failed-worker-start-teardown'
const TERMINAL_ONLY_MESSAGE = 'A federated worker starts a terminal agent only.'
type StructuredSession = Awaited<ReturnType<typeof createStructuredWorkerSessionForWorktree>>
export async function launchFederatedWorkerAgent(args: {
runtime: OrcaRuntimeService
@@ -25,6 +29,8 @@ export async function launchFederatedWorkerAgent(args: {
params: FederationAttachStartInput
agent: TuiAgent | undefined
launchPreferences: AgentLaunchPreferences | undefined
/** Decided here, on the host that runs the worker, from this host's own settings. */
mode: AgentLaunchModeReceipt
/** The workspace to start in; absent creates the new top-level worktree the params describe. */
worktree?: { id: string }
/** The paired client that asked, so its own sidebar counts the new worktree as its own. */
@@ -38,6 +44,10 @@ export async function launchFederatedWorkerAgent(args: {
worktree: { id: string }
terminalHandle: string
terminalCreation?: RuntimeTerminalCreate
/** Set when the worker is a chat session rather than a terminal agent. */
structuredSession: StructuredSession | null
/** The mode that actually ran; a created worktree can settle it later than the caller could. */
mode: AgentLaunchModeReceipt
}> {
const { runtime, db, params, effects } = args
if (!args.agent) {
@@ -49,123 +59,165 @@ export async function launchFederatedWorkerAgent(args: {
const agent = args.agent
let created: { id: string } | undefined
let terminalCreation: RuntimeTerminalCreate | undefined
const launched = await executeAgentLaunch({
runtime,
intent: {
agent,
// Federation's own factory builds the create from the attachment, so the target carries none.
target: args.worktree
? { kind: 'existing', worktree: args.worktree.id }
: { kind: 'create-worktree', create: {} },
launchSource: 'orchestration'
},
terminalOnly: true,
vocabulary: WORKER_START_VOCABULARY,
onStage: (stage) =>
args.onStage(stage === 'worktree_create' ? 'worktree_create' : 'terminal_create'),
workspaces: {
createWorktree: async ({ startupAgent }) => {
// Agent-first is the only create here; a structured pre-flight must not reach it.
if (startupAgent === undefined) {
throw new Error(TERMINAL_ONLY_MESSAGE)
}
const { repo } = params
if (!repo) {
// Validation refuses this before the attachment exists; this only narrows the types.
throw new Error('A remote new-top-level worktree requires an explicit --repo.')
}
db.recordRemoteAttachmentStage({
dispatchId: params.dispatchId,
stage: 'worktree_creating'
})
const setupDecision = params.setup ?? 'run'
const result = await runtime.createManagedWorktree({
onStartupTerminalCreated: (terminal) => {
terminalCreation = terminal
},
repoSelector: repo,
...(await resolveWorkerWorktreeName(runtime, repo, params.name)),
baseBranch: params.baseBranch,
displayName: params.displayName,
displayNameKind: params.displayNameKind,
comment: params.comment,
// setupDecision runs setup without the legacy runHooks activation side effect.
runHooks: false,
setupDecision,
awaitTerminalProvisioning: true,
observeSetupCompletion: true,
createdWithAgent: agent,
startupAgent: agent,
startupLaunchSource: 'orchestration',
startupSuppressUpdatePrompt: true,
...(args.launchPreferences ? { startupLaunchPreferences: args.launchPreferences } : {}),
activate: false,
...(args.creatorProvenance ? { creatorProvenance: args.creatorProvenance } : {}),
lineage: params.crossStoreParent
? {
crossStoreParent: params.crossStoreParent,
...(params.crossStoreParentInferredFrom
? { crossStoreParentInferredFrom: params.crossStoreParentInferredFrom }
: {})
}
: { noParent: true }
})
created = result.worktree
const terminalHandle = result.startupTerminal?.handle
effects.push({
kind: 'worktree',
action: params.crossStoreParent ? 'created_child' : 'created_top_level',
id: result.worktree.id,
name: result.worktree.displayName
})
const setup: WorkerSetupReceipt = {
requested: setupDecision,
effective: setupDecision,
source: args.setupSource,
hookFound: result.setupReceipt?.hookFound ?? false,
startupPolicy: result.setupReceipt?.startupPolicy ?? 'start-immediately',
state: result.setupReceipt?.state ?? 'not_configured'
}
args.onSetup(setup)
if (!terminalHandle) {
throw new Error(result.warning ?? 'Agent-first worktree creation returned no terminal.')
}
const listed = await runtime.listTerminals(`id:${result.worktree.id}`, undefined, {
includeVisualLayouts: false
})
appendFederationTerminalEffects(
effects,
listed.terminals,
terminalHandle,
result.setupReceipt?.terminalHandle
)
appendFederationSetupEffect(effects, setup)
return { worktreeId: result.worktree.id, startupTerminalHandle: terminalHandle }
}
},
surfaces: {
createStructuredSession: () => {
throw new Error(TERMINAL_ONLY_MESSAGE)
let structuredSession: StructuredSession | null = null
let launched: Awaited<ReturnType<typeof executeAgentLaunch>>
try {
launched = await executeAgentLaunch({
runtime,
intent: {
agent,
// Federation's own factory builds the create from the attachment, so the target carries none.
target: args.worktree
? { kind: 'existing', worktree: args.worktree.id }
: { kind: 'create-worktree', create: {} },
launchSource: 'orchestration'
},
createTerminalAgent: async ({ worktreeId }) => {
const terminal = await runtime.createTerminal(`id:${worktreeId}`, {
// Resolve agent ids through TUI config: cursor is the desktop app, its CLI is cursor-agent.
startupAgent: agent,
launchSource: 'orchestration',
suppressStartupUpdatePrompt: true,
...(args.launchPreferences ? { launchPreferences: args.launchPreferences } : {}),
title: `worker-${params.taskId}`,
presentation: 'background'
})
effects.push({ kind: 'terminal', role: 'agent', action: 'created', id: terminal.handle })
terminalCreation = terminal
return { handle: terminal.handle, terminalCreation: terminal }
decidedMode: args.mode,
vocabulary: WORKER_START_VOCABULARY,
onStage: (stage) =>
args.onStage(stage === 'worktree_create' ? 'worktree_create' : 'terminal_create'),
workspaces: {
createWorktree: async ({ startupAgent, promisedAgentChat, onWorkspaceCreated }) => {
const { repo } = params
if (!repo) {
// Validation refuses this before the attachment exists; this only narrows the types.
throw new Error('A remote new-top-level worktree requires an explicit --repo.')
}
db.recordRemoteAttachmentStage({
dispatchId: params.dispatchId,
stage: 'worktree_creating'
})
const setupDecision = params.setup ?? 'run'
const result = await runtime.createManagedWorktree({
onStartupTerminalCreated: (terminal) => {
terminalCreation = terminal
},
repoSelector: repo,
...(await resolveWorkerWorktreeName(runtime, repo, params.name)),
baseBranch: params.baseBranch,
displayName: params.displayName,
displayNameKind: params.displayNameKind,
comment: params.comment,
// setupDecision runs setup without the legacy runHooks activation side effect.
runHooks: false,
setupDecision,
awaitTerminalProvisioning: true,
observeSetupCompletion: true,
createdWithAgent: agent,
...(promisedAgentChat ? { promisedAgentChat: true } : {}),
...(onWorkspaceCreated ? { onWorkspaceCreated } : {}),
// The executor withholds the startup agent exactly when the worker is to be a chat session.
...(startupAgent !== undefined
? {
startupAgent: agent,
startupLaunchSource: 'orchestration' as const,
startupSuppressUpdatePrompt: true,
...(args.launchPreferences
? { startupLaunchPreferences: args.launchPreferences }
: {})
}
: {}),
activate: false,
...(args.creatorProvenance ? { creatorProvenance: args.creatorProvenance } : {}),
lineage: params.crossStoreParent
? {
crossStoreParent: params.crossStoreParent,
...(params.crossStoreParentInferredFrom
? { crossStoreParentInferredFrom: params.crossStoreParentInferredFrom }
: {})
}
: { noParent: true }
})
created = result.worktree
const terminalHandle = result.startupTerminal?.handle
effects.push({
kind: 'worktree',
action: params.crossStoreParent ? 'created_child' : 'created_top_level',
id: result.worktree.id,
name: result.worktree.displayName
})
const setup: WorkerSetupReceipt = {
requested: setupDecision,
effective: setupDecision,
source: args.setupSource,
hookFound: result.setupReceipt?.hookFound ?? false,
startupPolicy: result.setupReceipt?.startupPolicy ?? 'start-immediately',
state: result.setupReceipt?.state ?? 'not_configured'
}
args.onSetup(setup)
// A chat has no startup terminal; its setup terminal is still recorded so the setup gate finds it.
const chat = startupAgent === undefined
if (!chat && !terminalHandle) {
throw new Error(result.warning ?? 'Agent-first worktree creation returned no terminal.')
}
const listed = await runtime.listTerminals(`id:${result.worktree.id}`, undefined, {
includeVisualLayouts: false
})
appendFederationTerminalEffects(
effects,
listed.terminals,
chat ? undefined : terminalHandle,
result.setupReceipt?.terminalHandle
)
appendFederationSetupEffect(effects, setup)
return {
worktreeId: result.worktree.id,
startupTerminalHandle: chat ? undefined : terminalHandle
}
}
},
surfaces: {
createStructuredSession: async ({ worktreeId }) => {
args.onStage('terminal_create')
structuredSession = await createStructuredWorkerSessionForWorktree({
runtime,
worktreeId,
agent,
dispatchId: params.dispatchId,
...(args.launchPreferences ? { launchPreferences: args.launchPreferences } : {}),
effects
})
return {
sessionId: structuredSession.identity.sessionId,
handle: structuredSession.identity.handle,
fence: structuredSession.fence
}
},
createTerminalAgent: async ({ worktreeId }) => {
const terminal = await runtime.createTerminal(`id:${worktreeId}`, {
// Resolve agent ids through TUI config: cursor is the desktop app, its CLI is cursor-agent.
startupAgent: agent,
launchSource: 'orchestration',
suppressStartupUpdatePrompt: true,
...(args.launchPreferences ? { launchPreferences: args.launchPreferences } : {}),
title: `worker-${params.taskId}`,
presentation: 'background'
})
effects.push({ kind: 'terminal', role: 'agent', action: 'created', id: terminal.handle })
terminalCreation = terminal
return { handle: terminal.handle, terminalCreation: terminal }
}
}
}
})
})
} catch (error) {
// The caller only sees a returned launch, so a session made before this throw is ours.
await tearDownFailedWorkerStart({
runtime,
structuredSession,
dispatchId: params.dispatchId,
error
})
throw error
}
const worktree = args.worktree ?? created
if (!worktree) {
throw new Error('Federated worker topology did not resolve.')
}
return { worktree, terminalHandle: launched.outcome.handle, terminalCreation }
return {
worktree,
terminalHandle: launched.outcome.handle,
terminalCreation,
structuredSession,
mode: launched.receipt
}
}
@@ -0,0 +1,134 @@
import { waitForWorkerAgentReady } from '../../../../terminal/wait/launched-agent-composer-readiness'
import { sendAgentTurn } from '../../../../orchestration/turn-delivery/send-agent-turn'
import { createWorkerBriefWriteGuard } from '../../../../terminal/wait/launched-agent-write-guard'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import type { TuiAgent } from '../../../../../../shared/agent-launch/catalog/tui-agent'
import type { RuntimeTerminalCreate } from '../../../../../../shared/runtime-terminal-create-contract'
import {
formatOrcaSessionAddress,
isOrcaSessionId
} from '../../../../../../shared/orca-session-address'
import { canonicalOrcaSessionId } from '../../../../orchestration/addressing/session-lineage/canonical-orca-session-id'
import { sendStructuredWorkerPreamble } from '../structured-workers/orchestration-structured-worker-session'
import { awaitStructuredWorkerSetupGate } from '../worker/worker-start-structured-setup-gate'
import type {
createStructuredWorkerSessionForWorktree,
WorkerSetupReceipt
} from '../worker/worker-topology'
import type { FederationEffect } from './federation-effects'
export type StructuredSession = Awaited<ReturnType<typeof createStructuredWorkerSessionForWorktree>>
/** A chat's readiness is its brief being taken, so only a wait-for-setup policy gates it here. */
export function awaitFederatedWorkerReady(input: {
runtime: OrcaRuntimeService
structuredSession: StructuredSession | null
terminalHandle: string
agent: TuiAgent | undefined
reusesTerminal: boolean
creation: RuntimeTerminalCreate | undefined
setup: WorkerSetupReceipt
effects: FederationEffect[]
timeoutMs: number
}) {
const { runtime, timeoutMs } = input
return input.structuredSession
? awaitStructuredWorkerSetupGate({
runtime,
setup: input.setup,
effects: input.effects,
timeoutMs
})
: waitForWorkerAgentReady(runtime, input.terminalHandle, {
agent: input.agent,
reusesTerminal: input.reusesTerminal,
creation: input.creation,
timeoutMs
})
}
/** A chat worker's commands name its Orca session ID, which binds to it, as a local one's do. */
export function federatedWorkerAddress(
structuredSession: StructuredSession | null,
terminalHandle: string
): string {
const sessionId = structuredSession?.identity.sessionId
return sessionId && isOrcaSessionId(sessionId)
? formatOrcaSessionAddress(canonicalOrcaSessionId(sessionId))
: terminalHandle
}
export function sendFederatedWorkerBrief(input: {
runtime: OrcaRuntimeService
structuredSession: StructuredSession | null
terminalHandle: string
agent: TuiAgent | undefined
params: { dispatchId: string; runId?: string; taskId: string; terminal?: string }
operationId: string
preamble: string
budgetMs: number
}) {
return input.structuredSession
? sendRemoteChatWorkerBrief(
input.structuredSession,
input.params,
input.preamble,
input.budgetMs
)
: sendTerminalWorkerBrief(input.runtime, input.terminalHandle, {
agent: input.agent,
launched: !input.params.terminal,
operationId: input.operationId,
preamble: input.preamble
})
}
async function sendTerminalWorkerBrief(
runtime: OrcaRuntimeService,
handle: string,
brief: { agent: TuiAgent | undefined; launched: boolean; operationId: string; preamble: string }
) {
// A shell back at its prompt also reads as ready, so the brief needs the agent found in front.
const briefGuard = createWorkerBriefWriteGuard(runtime, brief.agent, brief.launched)
return sendAgentTurn({
kind: 'terminal',
runtime,
handle,
...(briefGuard ? { beforeWrite: briefGuard.beforeWrite } : {}),
turn: { purpose: 'dispatch-preamble', operationId: brief.operationId, body: brief.preamble }
}).finally(() => briefGuard?.dispose())
}
/** The brief as the chat's first turn, as a local chat worker gets it; one not yet taken leaves the start unknown. */
async function sendRemoteChatWorkerBrief(
session: StructuredSession,
params: { dispatchId: string; runId?: string; taskId: string },
preamble: string,
budgetMs: number
): Promise<undefined> {
const delivery = await sendStructuredWorkerPreamble({
host: session.host,
sessionId: session.identity.sessionId,
dispatchId: params.dispatchId,
preamble,
from: {
kind: 'agent',
senders: [],
orchestration: {
message: 'task',
runId: params.runId ?? '',
taskId: params.taskId,
dispatchId: params.dispatchId
}
},
budgetMs
})
if (delivery !== 'accepted') {
throw new OrchestrationError(
'operation_unknown',
'The task was accepted, but the chat agent had not started to take it. It is delivered when the agent starts; if the worker then reports, this Dispatch settles normally.'
)
}
return undefined
}
@@ -2,8 +2,7 @@ import { describeUnconfirmedAgentStop } from '../../../../../../shared/terminal/
import type { RemoteDispatchAttachmentRow } from '../../../../orchestration/db/orchestration-records'
import type {
WorkerTerminalArchiveRow,
WorkerTerminalResourceRow,
WorkerTerminalRetainedReason
WorkerTerminalResourceRow
} from '../../../../orchestration/terminal-authority/release/worker-terminal-ownership'
import {
captureWorkerOutputArchive,
@@ -14,10 +13,14 @@ import {
import type { WorkerTerminalReleaseRuntime } from '../../../../orchestration/terminal-authority/release/worker-terminal-release-runtime'
import { readRemoteAttachmentArchive } from './remote-attachment-archive-read'
import { WorkerTerminalReleaseSingleFlight } from '../worker/worker-terminal-release-single-flight'
import { remoteChatWorkerOf } from '../../../../orchestration/federation/remote-chat-worker'
import { releaseRemoteChatWorker } from './remote-chat-worker-host'
import { yieldAutoReleaseToWorkingAgent } from '../../../../orchestration/terminal-authority/release/worker-terminal-auto-release-working-yield'
import { archiveSummary } from '../worker/worker-terminal-resource-presentation'
import {
isStillReleasing,
releaseUnknownRecovery,
retainedReason,
type WorkerReleaseReceipt
} from '../worker/worker-release-completion'
import { orchestrationTimestampToMs } from '../worker/worker-output'
@@ -113,6 +116,10 @@ async function completeRemoteAttachmentRelease(
): Promise<RemoteReleaseReceipt> {
const { runtime, attachment, observation } = args
const db = runtime.getOrchestrationDb()
const chat = remoteChatWorkerOf(db, attachment)
if (chat?.exact) {
return releaseRemoteChatWorker(runtime, attachment, chat.identity, resource, storedArchive)
}
if (!observation.exact || !observation.terminal) {
if (
args.mode === 'recovery' &&
@@ -280,16 +287,6 @@ function remoteAttachmentLeaseIsCurrent(
)
}
function retainedReason(resource: WorkerTerminalResourceRow): WorkerTerminalRetainedReason {
if (resource.retained_reason) {
return resource.retained_reason as WorkerTerminalRetainedReason
}
if (resource.ownership_state === 'user_owned') {
return 'user_takeover'
}
return 'identity_unproven'
}
function projectArchivedOutputLiveness<
T extends { status: { terminal: string; liveness: string } }
>(output: T, liveness: 'live' | 'unverifiable' | 'exited'): T {
@@ -302,7 +299,3 @@ function projectArchivedOutputLiveness<
}
}
}
function isStillReleasing(resource: WorkerTerminalResourceRow): boolean {
return resource.ownership_state === 'owned' && resource.release_state === 'releasing'
}
@@ -20,7 +20,8 @@ import { resolveFederatedWorkerStartBudgets } from '../worker/worker-start-budge
import { resolveDispatchCreator } from '../runs/dispatch-creator'
import {
isReadyRemoteFederatedWorkerStartReceipt,
parseRemoteFederatedWorkerStartReceipt
parseRemoteFederatedWorkerStartReceipt,
readyRemoteReceiptFields
} from './federated-attach-receipt'
import { isWorkerStartTimeoutWithinTimerLimit } from '../../../../../../shared/orchestration/orchestration-timing-budgets'
import {
@@ -232,11 +233,9 @@ export async function startFederatedWorker(args: FederatedWorkerStartRequest): P
state: 'ready',
stage: readyWorker.stage,
server: { environmentId: server.environmentId, name: server.name },
setup: remote.setup,
...readyRemoteReceiptFields(remote),
launch,
timeoutMs: budgets.readinessTimeoutMs,
effects: remote.effects ?? [],
residualResources: remote.residualResources ?? [],
...(childParent && 'warning' in childParent ? { warning: childParent.warning } : {})
},
reportsTurnStops
@@ -3,6 +3,8 @@ import type { WorkerStopReceipt } from '../../../../../../shared/orchestration/w
import { archiveWorkerOutputBeforeStop } from '../worker/worker-stop-archive'
import { describeUnconfirmedAgentStop } from '../../../../../../shared/terminal/pty/lifecycle/pty-liveness-verdict'
import { inspectRemoteAttachment } from './federation-attachment-observation'
import { remoteChatWorkerOf } from '../../../../orchestration/federation/remote-chat-worker'
import { stopRemoteChatWorker } from './remote-chat-worker-host'
/** Close the exact attached worker this host runs for its home. */
export async function stopRemoteAttachment(
@@ -19,6 +21,10 @@ export async function stopRemoteAttachment(
processAction: 'none'
}
}
const chat = remoteChatWorkerOf(db, begun)
if (chat) {
return stopRemoteChatWorker(runtime, params.dispatchId, chat)
}
const observation = await inspectRemoteAttachment(runtime, params.dispatchId)
if (!observation.exact || !observation.terminal) {
const attachment = db.markRemoteAttachmentStopUnknown(
@@ -2,6 +2,7 @@ import type { OrcaRuntimeService } from '../../../../orca-runtime'
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import type { FederationEffect } from './federation-effects'
import type { WorkerSetupReceipt } from '../worker/worker-topology'
import type { StructuredWorkerIdentity } from '../../../../orchestration/terminal-authority/structured-sessions/structured-worker-identity'
export function prepareFederatedAttachmentAuthority(args: {
runtime: OrcaRuntimeService
@@ -12,9 +13,11 @@ export function prepareFederatedAttachmentAuthority(args: {
setup: WorkerSetupReceipt
effects: FederationEffect[]
reusesTerminal: boolean
/** A chat worker's minted identity, which stands in for a terminal's pane and process. */
identity?: Pick<StructuredWorkerIdentity, 'paneKey' | 'processIncarnation' | 'hostScope'>
}): void {
const { runtime, terminalHandle } = args
const authority = runtime.getOrchestrationDispatchAuthority(terminalHandle)
const authority = args.identity ?? runtime.getOrchestrationDispatchAuthority(terminalHandle)
const paneKey = authority?.paneKey ?? runtime.getTerminalPaneKey(terminalHandle)
const processIncarnation =
authority?.processIncarnation ?? runtime.getTerminalProcessIncarnation(terminalHandle)
@@ -4,6 +4,10 @@ import { OrchestrationError } from '../../../../orchestration/orchestration-erro
import { parseWorkerTerminalHostScope } from '../../../../../../shared/orchestration/workers/worker-terminal-host-scope'
import type { RemoteDispatchAttachmentRow } from '../../../../orchestration/db/orchestration-records'
import { orchestrationTimestampToMs } from '../worker/worker-output'
import {
REMOTE_CHAT_WORKER_READ_UNSUPPORTED,
remoteChatWorkerOf
} from '../../../../orchestration/federation/remote-chat-worker'
export function requireHomeAttachment(
runtime: WorkerTerminalReleaseRuntime,
@@ -20,6 +24,16 @@ export function requireHomeAttachment(
return attachment
}
/** A live read of a remote chat worker is refused plainly, until reads of one are supported. */
export function refuseRemoteChatWorkerRead(
runtime: WorkerTerminalReleaseRuntime,
attachment: RemoteDispatchAttachmentRow
): void {
if (remoteChatWorkerOf(runtime.getOrchestrationDb(), attachment)?.exact) {
throw new OrchestrationError('capability_unsupported', REMOTE_CHAT_WORKER_READ_UNSUPPORTED)
}
}
export async function inspectRemoteAttachment(
runtime: WorkerTerminalReleaseRuntime,
dispatchId: string
@@ -34,6 +48,16 @@ export async function inspectRemoteAttachment(
}> {
const db = runtime.getOrchestrationDb()
const attachment = db.getRemoteDispatchAttachment(dispatchId)
const chat = remoteChatWorkerOf(db, attachment)
if (chat) {
// Said plainly rather than read through a terminal it does not have.
return {
terminal: null,
exact: chat.exact,
status: chat.exact ? 'unverifiable' : 'identity_changed',
...(chat.exact ? { reason: REMOTE_CHAT_WORKER_READ_UNSUPPORTED } : {})
}
}
if (!attachment?.terminal_handle) {
return { terminal: null, exact: false, status: 'unattached' }
}
@@ -10,6 +10,7 @@ import { stopRemoteAttachment } from './federated-worker-stop-host'
import {
inspectRemoteAttachment,
observeRemoteAttachmentLiveness,
refuseRemoteChatWorkerRead,
requireHomeAttachment
} from './federation-attachment-observation'
import { releaseRemoteAttachment } from './federated-worker-release-host'
@@ -111,7 +112,10 @@ export const ORCHESTRATION_FEDERATION_CONTROL_METHODS = [
permission: 'workspace',
params: FederationReadParams,
handler: async (params, { runtime, authenticatedCallerFingerprint }) => {
requireHomeAttachment(runtime, params.dispatchId, authenticatedCallerFingerprint)
refuseRemoteChatWorkerRead(
runtime,
requireHomeAttachment(runtime, params.dispatchId, authenticatedCallerFingerprint)
)
const observation = await inspectRemoteAttachment(runtime, params.dispatchId)
// Only exited proves death; an unverifiable terminal may still be readable after stop-contact loss.
if (!observation.exact || !observation.terminal || observation.status === 'exited') {
@@ -171,6 +175,7 @@ export const ORCHESTRATION_FEDERATION_CONTROL_METHODS = [
}
}
}
refuseRemoteChatWorkerRead(runtime, attachment)
const observation = await inspectRemoteAttachment(runtime, params.dispatchId)
if (!observation.exact || !observation.terminal) {
throw new OrchestrationError(
@@ -19,7 +19,7 @@ export type FederationEffect = {
export function appendFederationTerminalEffects(
effects: FederationEffect[],
terminals: { handle: string; title: string | null; tabId?: string; leafId?: string }[],
agentHandle: string,
agentHandle: string | undefined,
setupHandle?: string
): void {
for (const terminal of terminals) {
@@ -94,3 +94,28 @@ export function monitorFederatedSetup(
})
.catch(() => undefined)
}
export function federatedSetupSource(
params: { setup?: WorkerSetupReceipt['requested']; setupSource?: WorkerSetupReceipt['source'] },
createsWorktree: boolean
): WorkerSetupReceipt['source'] {
return createsWorktree
? (params.setupSource ?? (params.setup ? 'explicit_request' : 'orchestration_default'))
: 'existing_worktree'
}
/** The setup a start reports before any setup ran: none for an existing worktree. */
export function initialFederatedSetup(
params: { setup?: WorkerSetupReceipt['requested'] },
createsWorktree: boolean,
source: WorkerSetupReceipt['source']
): WorkerSetupReceipt {
return {
requested: createsWorktree ? (params.setup ?? 'run') : 'not_applicable',
effective: createsWorktree ? (params.setup ?? 'run') : 'not_applicable',
source,
hookFound: false,
startupPolicy: 'start-immediately',
state: createsWorktree ? 'not_configured' : 'not_applicable'
}
}
@@ -1,16 +1,14 @@
import type { RuntimeTerminalCreate } from '../../../../../../shared/runtime-terminal-create-contract'
import { waitForWorkerAgentReady } from '../../../../terminal/wait/launched-agent-composer-readiness'
import { describeTerminalWaitBlockedReason } from '../../../../../../shared/terminal/runtime-policy/terminal-wait-blocked-reason-legacy-alias'
import { buildDispatchPreamble } from '../../../../orchestration/workers/launch/worker-dispatch-preamble'
import { sendAgentTurn } from '../../../../orchestration/turn-delivery/send-agent-turn'
import { createWorkerBriefWriteGuard } from '../../../../terminal/wait/launched-agent-write-guard'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import { defineMethod } from '../../../core'
import { assertOrchestrationWorktreeCreationSupported } from '../worker/folder-worktree-placement'
import type { FederationEffect } from './federation-effects'
import { isFederationEffectUnknown, type FederationEffect } from './federation-effects'
import { launchFederatedWorkerAgent } from './federated-worker-agent-launch'
import type { WorkerSetupReceipt } from '../worker/worker-topology'
import {
federatedSetupSource,
initialFederatedSetup,
monitorFederatedSetup,
persistFederatedReadinessStage,
persistFederatedSetupSpawnFailure,
@@ -26,6 +24,18 @@ import {
import { assertWorkerStartTaskSpecWithinPromptBudget } from '../worker/worker-start-prompt-budget'
import { prepareFederatedAttachmentAuthority } from './federation-attachment-authority'
import { resolveRpcWorkspaceCreatorProvenance } from '../../../workspace-creator-context'
import {
decideWorkerStartMode,
readWorkerStartModeSettings,
resolveWorkerStartModeOnHost
} from '../structured-workers/orchestration-worker-start-mode'
import { tearDownFailedWorkerStart } from '../worker/failed-worker-start-teardown'
import {
awaitFederatedWorkerReady,
federatedWorkerAddress,
sendFederatedWorkerBrief,
type StructuredSession
} from './federated-worker-brief'
import { claimTerminalForRemoteAttach } from '../../../../orchestration/db/worker-dispatch/worker-terminal-claim'
export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
@@ -96,17 +106,14 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
let worktree
let terminalHandle = params.terminal
let terminalCreation: RuntimeTerminalCreate | undefined
const setupSource = createsWorktree
? (params.setupSource ?? (params.setup ? 'explicit_request' : 'orchestration_default'))
: 'existing_worktree'
let setup: WorkerSetupReceipt = {
requested: createsWorktree ? (params.setup ?? 'run') : 'not_applicable',
effective: createsWorktree ? (params.setup ?? 'run') : 'not_applicable',
source: setupSource,
hookFound: false,
startupPolicy: 'start-immediately',
state: createsWorktree ? 'not_configured' : 'not_applicable'
}
let structuredSession: StructuredSession | null = null
// This host decides chat or terminal from its own settings, as a local start does.
let mode = decideWorkerStartMode({
params: { agent, terminal: params.terminal, worktree: params.worktree },
settings: readWorkerStartModeSettings(runtime)
})
const setupSource = federatedSetupSource(params, createsWorktree)
let setup = initialFederatedSetup(params, createsWorktree, setupSource)
try {
const launchAgent = (existing?: { id: string }) =>
launchFederatedWorkerAgent({
@@ -115,6 +122,7 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
params,
agent,
launchPreferences: launch.preferences,
mode,
...(existing
? { worktree: existing }
: { creatorProvenance: resolveRpcWorkspaceCreatorProvenance(context) }),
@@ -128,7 +136,8 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
}
})
if (createsWorktree) {
;({ worktree, terminalHandle, terminalCreation } = await launchAgent())
;({ worktree, terminalHandle, terminalCreation, structuredSession, mode } =
await launchAgent())
} else {
worktree = await runtime.showManagedTerminalWorkspace(params.worktree).catch(() => {
throw new OrchestrationError(
@@ -162,7 +171,9 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
})
} else {
failedStage = 'terminal_create'
;({ terminalHandle, terminalCreation } = await launchAgent(worktree))
mode = await resolveWorkerStartModeOnHost(runtime, mode, worktree.id, agent)
;({ terminalHandle, terminalCreation, structuredSession, mode } =
await launchAgent(worktree))
}
}
if (!worktree || !terminalHandle) {
@@ -182,14 +193,21 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
}
persistFederatedReadinessStage(setupStage)
failedStage = 'agent_readiness'
const wait = await waitForWorkerAgentReady(runtime, terminalHandle, {
const wait = await awaitFederatedWorkerReady({
runtime,
structuredSession,
terminalHandle,
agent,
reusesTerminal: Boolean(params.terminal),
creation: terminalCreation,
setup,
effects,
timeoutMs: readinessTimeoutMs
})
persistFederatedSetupWaitOutcome({ ...setupStage, wait })
if (!wait.satisfied) {
if (wait) {
persistFederatedSetupWaitOutcome({ ...setupStage, wait })
}
if (wait && !wait.satisfied) {
if (setup.state === 'failed') {
failedStage = 'setup_wait'
}
@@ -207,33 +225,32 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
terminalHandle,
setup,
effects,
reusesTerminal: Boolean(params.terminal)
reusesTerminal: Boolean(params.terminal),
...(structuredSession ? { identity: structuredSession.identity } : {})
})
failedStage = 'dispatch_input'
// A shell back at its prompt also reads as ready, so the brief needs the agent found in front.
const briefGuard = createWorkerBriefWriteGuard(runtime, agent, !params.terminal)
const prompt = await sendAgentTurn({
kind: 'terminal',
const preamble = buildDispatchPreamble({
taskId: params.taskId,
dispatchId: params.dispatchId,
taskSpec: params.taskSpec,
coordinatorHandle: 'Run home (relayed by Orca)',
workerHandle: federatedWorkerAddress(structuredSession, terminalHandle),
devMode: params.devMode,
// Enforce the execution host's nesting cap using its own setting.
canDispatchSubWorkers: (params.depth ?? 1) < runtime.getNestedWorkerMaxDepth(),
subWorkersInOwnRun: true,
cliCommand: runtime.getTerminalOrchestrationCliCommand(terminalHandle)
})
const prompt = await sendFederatedWorkerBrief({
runtime,
handle: terminalHandle,
...(briefGuard ? { beforeWrite: briefGuard.beforeWrite } : {}),
turn: {
purpose: 'dispatch-preamble',
operationId: orchestrationMutation.requestId,
body: buildDispatchPreamble({
taskId: params.taskId,
dispatchId: params.dispatchId,
taskSpec: params.taskSpec,
coordinatorHandle: 'Run home (relayed by Orca)',
workerHandle: terminalHandle,
devMode: params.devMode,
// Enforce the execution host's nesting cap using its own setting.
canDispatchSubWorkers: (params.depth ?? 1) < runtime.getNestedWorkerMaxDepth(),
subWorkersInOwnRun: true,
cliCommand: runtime.getTerminalOrchestrationCliCommand(terminalHandle)
})
}
}).finally(() => briefGuard?.dispose())
structuredSession,
terminalHandle,
agent,
params,
operationId: orchestrationMutation.requestId,
preamble,
budgetMs: readinessTimeoutMs
})
effects.push({
kind: 'dispatch_input',
role: 'agent',
@@ -251,11 +268,21 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
terminalHandle,
setup,
launch: launch.receipt,
mode,
effects,
...(prompt.prompt ? { prompt: prompt.prompt } : {}),
...(prompt?.prompt ? { prompt: prompt.prompt } : {}),
residualResources: []
}
} catch (error) {
// A chat that may hold its brief stays for the worker to report; a refused start retires it.
if (structuredSession && !isFederationEffectUnknown(error, failedStage)) {
await tearDownFailedWorkerStart({
runtime,
structuredSession,
dispatchId: params.dispatchId,
error
})
}
return failFederatedAttachmentWithReceipt({
db,
dispatchId: params.dispatchId,
@@ -0,0 +1,141 @@
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import type { WorkerStopReceipt } from '../../../../../../shared/orchestration/workers/orchestration-worker-stop-receipt'
import type {
WorkerTerminalArchiveRow,
WorkerTerminalResourceRow
} from '../../../../orchestration/terminal-authority/release/worker-terminal-ownership'
import { remoteChatWorkerOf } from '../../../../orchestration/federation/remote-chat-worker'
import type { StructuredWorkerIdentity } from '../../../../orchestration/terminal-authority/structured-sessions/structured-worker-identity'
import type { WorkerTerminalReleaseRuntime } from '../../../../orchestration/terminal-authority/release/worker-terminal-release-runtime'
import { stopStructuredWorker } from '../structured-workers/orchestration-structured-worker-lifecycle'
import { stopStructuredWorkerForRelease } from '../worker/structured-worker-release-stop'
import { captureWorkerReleaseArchive } from '../worker/worker-release-archive-capture'
import {
isStillReleasing,
retainedReason,
type WorkerReleaseReceipt
} from '../worker/worker-release-completion'
import { orchestrationTimestampToMs } from '../worker/worker-output'
import { archiveSummary } from '../worker/worker-terminal-resource-presentation'
import { summarizeWorkerOutputArchive } from '../../../../orchestration/workers/archives/worker-output-archive'
import { yieldAutoReleaseToWorkingAgent } from '../../../../orchestration/terminal-authority/release/worker-terminal-auto-release-working-yield'
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import type { RemoteDispatchAttachmentRow } from '../../../../orchestration/db/orchestration-records'
/** Close a chat worker the way a local stop does; only a proven close settles. */
export async function stopRemoteChatWorker(
runtime: OrcaRuntimeService,
dispatchId: string,
chat: NonNullable<ReturnType<typeof remoteChatWorkerOf>>
): Promise<WorkerStopReceipt> {
const db = runtime.getOrchestrationDb()
const unknown = (reason: string, processAction: string): WorkerStopReceipt => {
const attachment = db.markRemoteAttachmentStopUnknown(dispatchId, reason)
return {
dispatchId,
state: attachment.state,
alreadySettled: false,
processAction,
lastError: attachment.last_error
}
}
if (!chat.exact) {
return unknown(
'The recorded chat worker is no longer this Dispatch; nothing was closed.',
'none'
)
}
// Install the chat host after a restart, or the close cannot reach the session.
await runtime.ensureStructuredAgentSessionHost().catch(() => undefined)
const stop = await stopStructuredWorker(chat.identity, runtime)
if (!stop.stopped) {
return unknown(
stop.reason ?? 'The close was not proven.',
stop.closeAttempted ? 'closed_agent_terminal' : 'none'
)
}
const attachment = db.settleRemoteAttachmentStop(dispatchId)
return {
dispatchId,
state: attachment.state,
alreadySettled: false,
processAction: 'closed_agent_terminal'
}
}
/** A chat releases the way a local chat worker does: its transcript is kept, the release commits,
* a working agent defers an automatic one, and only then does its chat close. */
export async function releaseRemoteChatWorker(
runtime: WorkerTerminalReleaseRuntime,
attachment: RemoteDispatchAttachmentRow,
identity: StructuredWorkerIdentity,
resource: WorkerTerminalResourceRow,
storedArchive: WorkerTerminalArchiveRow | null
): Promise<WorkerReleaseReceipt> {
const db = runtime.getOrchestrationDb()
const dispatchId = attachment.dispatch_id
await runtime.ensureStructuredAgentSessionHost().catch(() => undefined)
let archive: Pick<
Parameters<OrchestrationDb['commitWorkerTerminalArchiveForRelease']>[0],
'kind' | 'content' | 'archiveSource' | 'archiveStatus'
>
if (storedArchive) {
const stored = summarizeWorkerOutputArchive(storedArchive)
archive = {
archiveSource: stored.source,
archiveStatus: stored.status === 'empty' ? 'empty' : 'captured'
}
} else {
const capture = await captureWorkerReleaseArchive({
db,
resource,
runtime,
dispatchId,
terminalHandle: identity.handle,
attachedAtMs: orchestrationTimestampToMs(attachment.created_at),
structuredWorker: identity
})
if ('receipt' in capture) {
return capture.receipt
}
const { captured } = capture
archive = {
kind: captured.kind,
content: JSON.stringify(captured.content),
archiveSource: captured.kind === 'terminal_tail' ? 'terminal' : 'transcript',
archiveStatus: captured.status === 'empty' ? 'empty' : 'captured'
}
}
const committed = db.commitWorkerTerminalArchiveForRelease({
dispatchId,
resourceId: resource.id,
...archive
})
const releasing = !isStillReleasing(committed)
? committed
: !remoteChatWorkerOf(db, attachment)?.exact
? db.revertWorkerTerminalReleaseToRetained(resource.id, 'identity_unproven')
: yieldAutoReleaseToWorkingAgent(runtime, committed)
if (!isStillReleasing(releasing)) {
return {
dispatchId,
state: 'retained',
reason: retainedReason(releasing),
processAction: 'none',
archive: archiveSummary(releasing)
}
}
const released = await stopStructuredWorkerForRelease({
structured: identity,
dispatchId,
resource: releasing,
runtime,
db,
archiveSource: releasing.archive_source,
archiveStatus: releasing.archive_status
})
if (released.state === 'released') {
db.recordRemoteAttachmentStage({ dispatchId, stage: 'released' })
}
return released
}
@@ -271,7 +271,7 @@ export function releaseUnknownRecovery(dispatchId: string): string {
return `Inspect with: orca orchestration worker-show --dispatch ${dispatchId} --json — then retry worker-release with a fresh request ID (omit --retry-request to let the CLI generate one). Reusing the prior request ID only replays this release_unknown receipt. Never substitute a broad terminal close.`
}
function retainedReason(resource: WorkerTerminalResourceRow): WorkerTerminalRetainedReason {
export function retainedReason(resource: WorkerTerminalResourceRow): WorkerTerminalRetainedReason {
if (resource.retained_reason) {
return resource.retained_reason as WorkerTerminalRetainedReason
}
@@ -281,6 +281,6 @@ function retainedReason(resource: WorkerTerminalResourceRow): WorkerTerminalReta
return 'identity_unproven'
}
function isStillReleasing(resource: WorkerTerminalResourceRow): boolean {
export function isStillReleasing(resource: WorkerTerminalResourceRow): boolean {
return resource.ownership_state === 'owned' && resource.release_state === 'releasing'
}
@@ -171,7 +171,7 @@ export async function startWorker(
settings: readWorkerStartModeSettings(runtime)
})
if (params.on) {
// Remote workers are terminal-only; carry the mode receipt to explain structured-preference fallback.
// The server decides chat or terminal; one that cannot run chats leaves this host's terminal receipt.
const receipt = await startFederatedWorker({
params,
runtime,
@@ -184,7 +184,7 @@ export async function startWorker(
followUp
})
await stopHelperAfterStart(runtime, receipt, orchestrationMutation)
return withRun(receipt && typeof receipt === 'object' ? { ...receipt, mode } : receipt)
return withRun(receipt && typeof receipt === 'object' ? { mode, ...receipt } : receipt)
}
const receipt = await startLocalWorker({
params: { ...params, timeoutMs: readinessTimeoutMs },