diff --git a/src/main/runtime/orchestration/federation/remote-chat-worker.ts b/src/main/runtime/orchestration/federation/remote-chat-worker.ts new file mode 100644 index 00000000000..cbccf620a2e --- /dev/null +++ b/src/main/runtime/orchestration/federation/remote-chat-worker.ts @@ -0,0 +1,30 @@ +import { isStructuredWorkerHandle } from '../../../../shared/orchestration/workers/structured-worker-handle' +import type { OrchestrationDb } from '../db/orchestration-db' +import type { RemoteDispatchAttachmentRow } from '../db/orchestration-records' +import { resolveStructuredWorkerIdentity } from '../terminal-authority/structured-sessions/structured-worker-authority' +import type { StructuredWorkerIdentity } from '../terminal-authority/structured-sessions/structured-worker-identity' + +/** Said wherever this host would read a remote chat worker's state, until reads of one are supported. */ +export const REMOTE_CHAT_WORKER_READ_UNSUPPORTED = + 'Orca cannot read the state of a chat worker on a paired server yet; open its chat on that server to see it.' + +/** The chat session a coordinator on another computer runs here, with whether it is still the attached one. */ +export function remoteChatWorkerOf( + db: OrchestrationDb, + attachment: Pick | undefined +): { identity: StructuredWorkerIdentity; exact: boolean } | null { + const handle = attachment?.terminal_handle + const identity = + handle && isStructuredWorkerHandle(handle) ? resolveStructuredWorkerIdentity(handle, db) : null + if (!attachment || !identity) { + return null + } + return { + identity, + exact: db.isRemoteAttachmentProcessCurrent({ + dispatchId: attachment.dispatch_id, + paneKey: identity.paneKey, + processIncarnation: identity.processIncarnation + }) + } +} diff --git a/src/main/runtime/orchestration/mailbox/routing/structured-session-mail-target.ts b/src/main/runtime/orchestration/mailbox/routing/structured-session-mail-target.ts index 107841cb7e6..0e0852ad835 100644 --- a/src/main/runtime/orchestration/mailbox/routing/structured-session-mail-target.ts +++ b/src/main/runtime/orchestration/mailbox/routing/structured-session-mail-target.ts @@ -21,6 +21,7 @@ import { import type { RunRow } from '../../db/orchestration-records' import { chatAssigneeSessionId } from '../../workers/lifecycle/chat-assignee' import { structuredWorkerCustody } from '../../terminal-authority/structured-sessions/structured-worker-authority' +import { remoteChatWorkerOf } from '../../federation/remote-chat-worker' /** A handle-less coordinator uses its generation-current session id; structured workers resolve by handle. */ export function handleLessCoordinatorSessionId( @@ -118,7 +119,11 @@ export function structuredMailboxCustody( if (!mailboxHandle.startsWith('dispatch:')) { return workerMailCustody(mailboxHandle, db) } - const dispatch = db?.getDispatchContextById?.(mailboxHandle.slice('dispatch:'.length)) + const dispatchId = mailboxHandle.slice('dispatch:'.length) + const dispatch = db?.getDispatchContextById?.(dispatchId) + if (!dispatch && db) { + return remoteDispatchMailCustody(dispatchId, db) + } // A Dispatch's mail follows its Task: none reaches the chat before it took the Task, or after it ended. if (!dispatch?.assignee_handle || dispatch.status !== 'dispatched') { return 'elsewhere' @@ -129,6 +134,16 @@ export function structuredMailboxCustody( : workerMailCustody(dispatch.assignee_handle, db, dispatch.id) } +/** A Dispatch a coordinator on another computer runs here reaches its chat only while that chat still works it. */ +function remoteDispatchMailCustody( + dispatchId: string, + db: OrchestrationDb +): StructuredMailboxCustody { + const attachment = db.getRemoteDispatchAttachment?.(dispatchId) + const chat = attachment?.state === 'ready' ? remoteChatWorkerOf(db, attachment) : null + return chat?.exact ? workerMailCustody(chat.identity.handle, db, dispatchId) : 'elsewhere' +} + /** Re-derive Run, direct and active-Dispatch mail on each idle edge, including root mail for /clear successors. */ export function structuredSessionOwnedMailboxes(sessionId: string, db: OrchestrationDb): string[] { const party = isOrcaSessionId(sessionId) ? addressableSessionParty(sessionId, db) : null @@ -144,6 +159,10 @@ export function structuredSessionOwnedMailboxes(sessionId: string, db: Orchestra if (dispatch) { mailboxes.push(`dispatch:${dispatch.id}`) } + const remote = party.paneKey ? db.findActiveRemoteAttachmentForPane(party.paneKey) : undefined + if (remote && remoteChatWorkerOf(db, remote)?.exact) { + mailboxes.push(`dispatch:${remote.dispatch_id}`) + } return mailboxes } diff --git a/src/main/runtime/orchestration/mailbox/structured-mail/structured-mail-custody.ts b/src/main/runtime/orchestration/mailbox/structured-mail/structured-mail-custody.ts index ece9005e896..f82c557e162 100644 --- a/src/main/runtime/orchestration/mailbox/structured-mail/structured-mail-custody.ts +++ b/src/main/runtime/orchestration/mailbox/structured-mail/structured-mail-custody.ts @@ -26,13 +26,19 @@ export type StructuredMailResend = { row: StructuredMailSendRow; renew: boolean const REUSABLE_OPERATION_AGE_MS = AGENT_SESSION_MAX_NEW_OPERATION_AGE_MS - AGENT_SESSION_OPERATION_FUTURE_SKEW_MS -/** A mailbox nothing can own again: its Dispatch settled or is gone, or its Run is gone. */ +/** A mailbox nothing can own again: its Dispatch (or remote attachment) settled or is gone, or its Run is gone. */ export function structuredMailboxEnded( mailboxHandle: string, db: OrchestrationDb | null | undefined ): boolean { if (mailboxHandle.startsWith('dispatch:')) { - const status = db?.getDispatchContextById?.(mailboxHandle.slice('dispatch:'.length))?.status + const dispatchId = mailboxHandle.slice('dispatch:'.length) + const status = db?.getDispatchContextById?.(dispatchId)?.status + if (status === undefined) { + // A coordinator on another computer runs this Dispatch; here it is only an attachment. + const remote = db?.getRemoteDispatchAttachment?.(dispatchId)?.state + return remote !== 'starting' && remote !== 'ready' + } return status !== 'pending' && status !== 'dispatched' } return mailboxHandle.startsWith('run:') && !db?.getRun?.(mailboxHandle.slice('run:'.length)) diff --git a/src/main/runtime/orchestration/terminal-authority/structured-sessions/remote-chat-worker-attach.test.ts b/src/main/runtime/orchestration/terminal-authority/structured-sessions/remote-chat-worker-attach.test.ts new file mode 100644 index 00000000000..111e37c1b87 --- /dev/null +++ b/src/main/runtime/orchestration/terminal-authority/structured-sessions/remote-chat-worker-attach.test.ts @@ -0,0 +1,301 @@ +// A coordinator on another computer starts a chat worker here; the worker reports home over the relay. + +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { EVERY_AGENT_CHAT } from '../../../../../shared/native-chat/every-agent-chat.test-fixture' +import { isStructuredWorkerHandle } from '../../../../../shared/orchestration/workers/structured-worker-handle' +import * as sessionCreate from '../../../rpc/methods/structured-sessions/creation/structured-agent-session-create' +import { LOCAL_EXECUTION_HOST_ID } from '../../../../../shared/execution-host' +import { chatWorkerTurnStopReporter } from '../../workers/turn-stops/chat-worker-turn-stops' +import { encodeFederatedControlMessage } from '../../federation/relay/federation-control-message' +import * as placement from '../../../rpc/methods/orchestration/worker/folder-worktree-placement' +import * as worktreeName from '../../../rpc/methods/orchestration/worker/worker-worktree-creation' +import { attachParams } from './structured-chat-coordinator-fake-codex-fixture' +import { + WAIT, + call, + codex, + db, + dispatcher, + host, + request, + runtime, + settleTurn, + turnText +} from './structured-chat-coordinator-mail-rig.test-fixture' +import { resolveStructuredWorkerIdentity } from './structured-worker-authority' +import { reconcileRequestedWorkerTerminalReleases } from '../release/worker-terminal-release-reconciliation' + +const DISPATCH = 'ctx_remote_chat' +const HOME = 'home_peer' + +beforeEach(() => { + vi.spyOn(runtime, 'validateOrchestrationAgentLauncher').mockImplementation(() => {}) + vi.spyOn(runtime, 'getStructuredAgentSessionCreateSupport').mockResolvedValue({ supported: true }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: placement reads only the workspace identity in this rig. + vi.spyOn(runtime, 'showManagedTerminalWorkspace').mockResolvedValue({ + id: 'workspace-1', + repoId: 'repo' + } as never) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: launch-mode selection reads only getSettings on this store double. + vi.spyOn(runtime, 'requireStore').mockReturnValue({ + getSettings: () => ({ + nativeChatProviders: EVERY_AGENT_CHAT, + agentCmdOverrides: {}, + agentDefaultArgs: {}, + agentDefaultEnv: {} + }) + } as never) + vi.spyOn(sessionCreate, 'createStructuredAgentSessionForWorktree').mockImplementation( + async (args) => { + const attached = await host.attach(args.caller, attachParams(args.envelope.sessionId)) + if (!attached.ok) { + throw new Error('scripted worker attach refused') + } + return { + ...attached, + value: { ...attached.value, sessionId: args.envelope.sessionId, fence: attached.fence } + } + } + ) + // The rig fakes PTY identity for its own terminals; a chat worker's comes from its minted identity. + const identity = (handle: string) => + isStructuredWorkerHandle(handle) ? resolveStructuredWorkerIdentity(handle, db) : null + vi.mocked(runtime.getTerminalPaneKey).mockImplementation( + (handle) => identity(handle)?.paneKey ?? null + ) + vi.mocked(runtime.getTerminalProcessIncarnation).mockImplementation( + (handle) => identity(handle)?.processIncarnation ?? null + ) +}) + +async function federation(name: string, params: Record): Promise { + const response = await dispatcher.dispatch(request(name, params), { + authenticatedCallerFingerprint: HOME + }) + if (!response.ok) { + throw new Error(`${name} failed: ${JSON.stringify(response)}`) + } + return response.result +} + +/** The worker started and took its task, as a coordinator on another computer starts it. */ +async function startedChatWorker() { + const started = federation('orchestration.federationAttachStart', { + runId: 'run_home', + dispatchId: DISPATCH, + taskId: 'task_home', + taskSpec: 'Check the docs and report.', + protocolVersion: 4, + worktree: 'id:workspace-1', + agent: 'codex' + }) + await vi.waitFor(() => expect(codex.connections.at(-1)?.turns).toHaveLength(1), WAIT) + const handle = String(db.getRemoteDispatchAttachment(DISPATCH)?.terminal_handle) + const identity = resolveStructuredWorkerIdentity(handle, db)! + const worker = codex.connections.at(-1)! + await settleTurn(identity.sessionId, 0) + return { receipt: await started, handle, identity, worker } +} + +const homeBound = () => + db.listPendingFederationRelay(DISPATCH, 'to_home').map((item) => JSON.parse(item.payload)) + +describe('a chat worker that a coordinator on another computer starts here', () => { + it('starts as a chat when this host prefers chats, and takes its task as its first turn', async () => { + const { receipt, handle, identity, worker } = await startedChatWorker() + + expect(receipt).toMatchObject({ + state: 'ready', + terminalHandle: handle, + mode: { mode: 'structured' } + }) + expect(isStructuredWorkerHandle(handle)).toBe(true) + expect(worker.turns[0]!.text).toContain('Check the docs and report.') + expect(db.getRemoteDispatchAttachment(DISPATCH)).toMatchObject({ + state: 'ready', + pane_key: identity.paneKey, + process_incarnation: identity.processIncarnation + }) + }) + + it('sends its mail home on the relay', async () => { + const { identity } = await startedChatWorker() + + await expect( + call( + 'orchestration.send', + { subject: 'Halfway', body: 'docs checked' }, + { sessionId: identity.sessionId } + ) + ).resolves.toMatchObject({ relay: { destination: 'run_home', dispatchId: DISPATCH } }) + expect(homeBound()).toEqual([expect.objectContaining({ subject: 'Halfway', type: 'status' })]) + }) + + it('tells home when its turn ends without a report', async () => { + const { identity } = await startedChatWorker() + const timers: (() => void)[] = [] + const report = chatWorkerTurnStopReporter( + { getDb: () => db, isWaitingForMail: () => false, notify: vi.fn() }, + (run) => timers.push(run) + ) + + report( + { + type: 'completion', + completion: { + scope: { + executionHostId: LOCAL_EXECUTION_HOST_ID, + wslDistro: null, + workspaceId: 'workspace-1', + workspaceKind: 'git-worktree' + }, + sessionId: identity.sessionId, + turnId: 'turn-1', + outcome: 'success', + completedAt: 1 + } + }, + () => ({ status: 'idle', lastAssistantMessage: 'Docs look fine.' }) + ) + for (const run of timers.splice(0)) { + run() + } + + expect(homeBound()).toEqual([ + expect.objectContaining({ subject: 'Worker stopped without reporting' }) + ]) + }) + + it("takes its coordinator's mail as a new turn", async () => { + const { worker } = await startedChatWorker() + + await federation('orchestration.federationImport', { + dispatchId: DISPATCH, + items: [ + { + dispatch_id: DISPATCH, + direction: 'to_worker', + sequence: 1, + message_id: 'relay_home_1', + kind: 'control_message', + payload: encodeFederatedControlMessage({ + from: 'term_coord', + subject: 'Also the changelog', + body: 'Update the changelog too.', + type: 'status', + priority: 'normal', + threadId: null, + payload: null + }) + } + ] + }) + + await vi.waitFor(() => expect(worker.turns).toHaveLength(2), WAIT) + expect(turnText(worker.turns[1]!)).toContain('Also the changelog') + }) + + it('says plainly that its state cannot be read here yet', async () => { + await startedChatWorker() + + await expect( + federation('orchestration.federationShow', { dispatchId: DISPATCH }) + ).resolves.toMatchObject({ + observation: { + status: 'unverifiable', + reason: expect.stringContaining('cannot read the state of a chat worker on a paired server') + } + }) + for (const read of ['orchestration.federationRead', 'orchestration.federationReadOutput']) { + await expect(federation(read, { dispatchId: DISPATCH })).rejects.toThrow( + /cannot read the state of a chat worker on a paired server/ + ) + } + }) + + it('closes its chat when its coordinator stops it', async () => { + const { identity } = await startedChatWorker() + + await expect( + federation('orchestration.federationStop', { dispatchId: DISPATCH }) + ).resolves.toMatchObject({ state: 'stopped', processAction: 'closed_agent_terminal' }) + expect(host.hasSession(identity.sessionId)).toBe(false) + }) + + it('closes its chat when its coordinator releases it after its report', async () => { + const { identity } = await startedChatWorker() + // Its report settled at the Run home, as a confirmed worker_done leaves it. + db.recordRemoteAttachmentStage({ + dispatchId: DISPATCH, + state: 'succeeded', + stage: 'worker_reported' + }) + + await expect( + federation('orchestration.federationRelease', { dispatchId: DISPATCH }) + ).resolves.toMatchObject({ state: 'released' }) + expect(host.hasSession(identity.sessionId)).toBe(false) + expect(db.getRemoteDispatchAttachment(DISPATCH)).toMatchObject({ stage: 'released' }) + }) + + it('closes its chat when its automatic release comes due', async () => { + const { identity } = await startedChatWorker() + db.recordRemoteAttachmentStage({ + dispatchId: DISPATCH, + state: 'succeeded', + stage: 'worker_reported' + }) + const settledAt = Date.UTC(2026, 0, 1) + db.armWorkerTerminalAutoReleaseStatement(DISPATCH, settledAt) + expect(db.claimDueWorkerTerminalAutoReleases(settledAt + 3_600_000)).toBe(1) + + await reconcileRequestedWorkerTerminalReleases(runtime) + + expect(host.hasSession(identity.sessionId)).toBe(false) + expect(db.getRemoteDispatchAttachment(DISPATCH)).toMatchObject({ stage: 'released' }) + }) + + it('in a new worktree, takes its task only after a wait-for-setup setup finishes', async () => { + vi.spyOn(placement, 'assertOrchestrationWorktreeCreationSupported').mockResolvedValue() + vi.spyOn(worktreeName, 'resolveWorkerWorktreeName').mockResolvedValue({ name: 'w2' }) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the launch reads only these receipt fields. + vi.spyOn(runtime, 'createManagedWorktree').mockResolvedValue({ + worktree: { id: 'workspace-1', displayName: 'w2' }, + setupReceipt: { + hookFound: true, + startupPolicy: 'wait-for-setup', + state: 'running', + terminalHandle: 'term_setup' + } + } as never) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: effects read only handle and title. + vi.spyOn(runtime, 'listTerminals').mockResolvedValue({ + terminals: [{ handle: 'term_setup', title: 'Setup' }] + } as never) + const setupDone = Promise.withResolvers<{ exitCode: number }>() + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the gate reads only exitCode. + vi.spyOn(runtime, 'waitForSetupTerminalCompletion').mockReturnValue(setupDone.promise as never) + + const started = federation('orchestration.federationAttachStart', { + runId: 'run_home', + dispatchId: DISPATCH, + taskId: 'task_home', + taskSpec: 'Check the docs and report.', + protocolVersion: 4, + worktree: 'new-top-level', + repo: 'id:repo', + agent: 'codex' + }) + await vi.waitFor( + () => expect(runtime.waitForSetupTerminalCompletion).toHaveBeenCalledWith('term_setup'), + WAIT + ) + expect(codex.connections.flatMap((connection) => connection.turns)).toHaveLength(0) + + setupDone.resolve({ exitCode: 0 }) + await vi.waitFor(() => expect(codex.connections.at(-1)?.turns).toHaveLength(1), WAIT) + const handle = String(db.getRemoteDispatchAttachment(DISPATCH)?.terminal_handle) + await settleTurn(resolveStructuredWorkerIdentity(handle, db)!.sessionId, 0) + await expect(started).resolves.toMatchObject({ state: 'ready', mode: { mode: 'structured' } }) + }) +}) diff --git a/src/main/runtime/orchestration/workers/turn-stops/chat-worker-turn-stops.ts b/src/main/runtime/orchestration/workers/turn-stops/chat-worker-turn-stops.ts index 906c8d7dfae..831d326ed61 100644 --- a/src/main/runtime/orchestration/workers/turn-stops/chat-worker-turn-stops.ts +++ b/src/main/runtime/orchestration/workers/turn-stops/chat-worker-turn-stops.ts @@ -4,9 +4,46 @@ import { AGENT_DONE_QUIET_MS } from '../../../../../shared/agent-status/lifecycl import { chatWorkerDispatches } from './chat-worker-error-escalation' import { reportWorkerTurnStop, + workerTurnStopDispatchId, + type WorkerTurnStopDispatch, type WorkerTurnStopReason, type WorkerTurnStopReportDeps } from './worker-turn-stop-report' +import type { OrchestrationDb } from '../../db/orchestration-db' +import { resolveStructuredWorkerIdentityForSession } from '../../terminal-authority/structured-sessions/structured-worker-authority' + +/** The open Dispatches a chat works: its own here, or one a coordinator on another computer started here. */ +function chatWorkerTargets( + sessionId: string, + db: OrchestrationDb +): { target: WorkerTurnStopDispatch; from: string; starting: boolean }[] { + const local = chatWorkerDispatches(sessionId, db).map(({ dispatch, from }) => ({ + target: { kind: 'local' as const, dispatch }, + from, + starting: db.getWorkerDispatch(dispatch.id)?.state === 'starting' + })) + if (local.length > 0) { + return local + } + // A paired server holds its remote workers as attachments; their Dispatch lives at the Run home. + const worker = resolveStructuredWorkerIdentityForSession(sessionId, db) + const attachment = worker && db.findActiveRemoteAttachmentForPane(worker.paneKey) + return worker && + attachment && + db.isRemoteAttachmentProcessCurrent({ + dispatchId: attachment.dispatch_id, + paneKey: worker.paneKey, + processIncarnation: worker.processIncarnation + }) + ? [ + { + target: { kind: 'remote', attachment }, + from: worker.handle, + starting: attachment.state === 'starting' + } + ] + : [] +} type ChatStop = { sessionId: string @@ -67,7 +104,7 @@ export function chatWorkerTurnStopReporter( } // The Dispatch a stop belongs to is one open when it was seen. const seen = new Set( - chatWorkerDispatches(stop.sessionId, db).map(({ dispatch }) => dispatch.id) + chatWorkerTargets(stop.sessionId, db).map(({ target }) => workerTurnStopDispatchId(target)) ) const deliver = (): void => { try { @@ -81,22 +118,17 @@ export function chatWorkerTurnStopReporter( ) { return } - const open = chatWorkerDispatches(stop.sessionId, current).filter(({ dispatch }) => - seen.has(dispatch.id) + const open = chatWorkerTargets(stop.sessionId, current).filter(({ target }) => + seen.has(workerTurnStopDispatchId(target)) ) - if ( - stop.reason === 'finished' && - open.some( - ({ dispatch }) => current.getWorkerDispatch(dispatch.id)?.state === 'starting' - ) - ) { + if (stop.reason === 'finished' && open.some(({ starting }) => starting)) { setTimer(deliver, AGENT_DONE_QUIET_MS) return } - for (const { dispatch, from } of open) { + for (const { target, from } of open) { reportWorkerTurnStop( current, - { kind: 'local', dispatch }, + target, { from, stopKey: stop.stopKey, diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federated-attach-receipt.ts b/src/main/runtime/rpc/methods/orchestration/federation/federated-attach-receipt.ts index 6fac7950954..c6c774c1452 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federated-attach-receipt.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federated-attach-receipt.ts @@ -1,4 +1,5 @@ import type { OrchestrationWorkerLaunchReceipt } from '../worker/worker-launch-preferences' +import type { AgentLaunchModeReceipt } from '../../../../../../shared/agent-launch/contracts/agent-launch-intent' export type RemoteFederatedWorkerStartReceipt = { dispatchId: string @@ -8,6 +9,8 @@ export type RemoteFederatedWorkerStartReceipt = { terminalHandle?: string setup?: { state: string } launch?: OrchestrationWorkerLaunchReceipt + /** Chat or terminal, as the server decided; absent from servers that start terminals only. */ + mode?: AgentLaunchModeReceipt effects?: unknown[] residualResources?: unknown[] failedStage?: string @@ -44,6 +47,16 @@ export function isReadyRemoteFederatedWorkerStartReceipt( return receipt.state === 'ready' } +/** The ready receipt's parts the server reports as is. */ +export function readyRemoteReceiptFields(remote: ReadyRemoteFederatedWorkerStartReceipt) { + return { + setup: remote.setup, + ...(remote.mode ? { mode: remote.mode } : {}), + effects: remote.effects ?? [], + residualResources: remote.residualResources ?? [] + } +} + function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value) } diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-agent-launch.ts b/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-agent-launch.ts index 0a3b9d07f9f..c683ffe70a3 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-agent-launch.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-agent-launch.ts @@ -1,5 +1,6 @@ import type { RuntimeTerminalCreate } from '../../../../../../shared/runtime-terminal-create-contract' -/** Execute terminalOnly on the receiving host with federation's agent-first worktree/background-terminal factories. The dispatch brief is delivered later. */ +/** Launch on the receiving host in the mode it decided, with federation's worktree, background-terminal and + * chat-session factories. The dispatch brief is delivered later. */ import type { AgentLaunchPreferences } from '../../../../../../shared/agent-session/agent-session-host-authority' import type { TuiAgent } from '../../../../../../shared/agent-launch/catalog/tui-agent' @@ -16,8 +17,11 @@ import { } from './federation-effects' import type { FederationAttachStartInput } from '../../../../../../shared/rpc-contract/orchestration-federation-start-params' import type { Worktree } from '../../../../../../shared/worktree/types' +import type { AgentLaunchModeReceipt } from '../../../../../../shared/agent-launch/contracts/agent-launch-intent' +import { createStructuredWorkerSessionForWorktree } from '../worker/worker-topology' +import { tearDownFailedWorkerStart } from '../worker/failed-worker-start-teardown' -const TERMINAL_ONLY_MESSAGE = 'A federated worker starts a terminal agent only.' +type StructuredSession = Awaited> export async function launchFederatedWorkerAgent(args: { runtime: OrcaRuntimeService @@ -25,6 +29,8 @@ export async function launchFederatedWorkerAgent(args: { params: FederationAttachStartInput agent: TuiAgent | undefined launchPreferences: AgentLaunchPreferences | undefined + /** Decided here, on the host that runs the worker, from this host's own settings. */ + mode: AgentLaunchModeReceipt /** The workspace to start in; absent creates the new top-level worktree the params describe. */ worktree?: { id: string } /** The paired client that asked, so its own sidebar counts the new worktree as its own. */ @@ -38,6 +44,10 @@ export async function launchFederatedWorkerAgent(args: { worktree: { id: string } terminalHandle: string terminalCreation?: RuntimeTerminalCreate + /** Set when the worker is a chat session rather than a terminal agent. */ + structuredSession: StructuredSession | null + /** The mode that actually ran; a created worktree can settle it later than the caller could. */ + mode: AgentLaunchModeReceipt }> { const { runtime, db, params, effects } = args if (!args.agent) { @@ -49,123 +59,165 @@ export async function launchFederatedWorkerAgent(args: { const agent = args.agent let created: { id: string } | undefined let terminalCreation: RuntimeTerminalCreate | undefined - const launched = await executeAgentLaunch({ - runtime, - intent: { - agent, - // Federation's own factory builds the create from the attachment, so the target carries none. - target: args.worktree - ? { kind: 'existing', worktree: args.worktree.id } - : { kind: 'create-worktree', create: {} }, - launchSource: 'orchestration' - }, - terminalOnly: true, - vocabulary: WORKER_START_VOCABULARY, - onStage: (stage) => - args.onStage(stage === 'worktree_create' ? 'worktree_create' : 'terminal_create'), - workspaces: { - createWorktree: async ({ startupAgent }) => { - // Agent-first is the only create here; a structured pre-flight must not reach it. - if (startupAgent === undefined) { - throw new Error(TERMINAL_ONLY_MESSAGE) - } - const { repo } = params - if (!repo) { - // Validation refuses this before the attachment exists; this only narrows the types. - throw new Error('A remote new-top-level worktree requires an explicit --repo.') - } - db.recordRemoteAttachmentStage({ - dispatchId: params.dispatchId, - stage: 'worktree_creating' - }) - const setupDecision = params.setup ?? 'run' - const result = await runtime.createManagedWorktree({ - onStartupTerminalCreated: (terminal) => { - terminalCreation = terminal - }, - repoSelector: repo, - ...(await resolveWorkerWorktreeName(runtime, repo, params.name)), - baseBranch: params.baseBranch, - displayName: params.displayName, - displayNameKind: params.displayNameKind, - comment: params.comment, - // setupDecision runs setup without the legacy runHooks activation side effect. - runHooks: false, - setupDecision, - awaitTerminalProvisioning: true, - observeSetupCompletion: true, - createdWithAgent: agent, - startupAgent: agent, - startupLaunchSource: 'orchestration', - startupSuppressUpdatePrompt: true, - ...(args.launchPreferences ? { startupLaunchPreferences: args.launchPreferences } : {}), - activate: false, - ...(args.creatorProvenance ? { creatorProvenance: args.creatorProvenance } : {}), - lineage: params.crossStoreParent - ? { - crossStoreParent: params.crossStoreParent, - ...(params.crossStoreParentInferredFrom - ? { crossStoreParentInferredFrom: params.crossStoreParentInferredFrom } - : {}) - } - : { noParent: true } - }) - created = result.worktree - const terminalHandle = result.startupTerminal?.handle - effects.push({ - kind: 'worktree', - action: params.crossStoreParent ? 'created_child' : 'created_top_level', - id: result.worktree.id, - name: result.worktree.displayName - }) - const setup: WorkerSetupReceipt = { - requested: setupDecision, - effective: setupDecision, - source: args.setupSource, - hookFound: result.setupReceipt?.hookFound ?? false, - startupPolicy: result.setupReceipt?.startupPolicy ?? 'start-immediately', - state: result.setupReceipt?.state ?? 'not_configured' - } - args.onSetup(setup) - if (!terminalHandle) { - throw new Error(result.warning ?? 'Agent-first worktree creation returned no terminal.') - } - const listed = await runtime.listTerminals(`id:${result.worktree.id}`, undefined, { - includeVisualLayouts: false - }) - appendFederationTerminalEffects( - effects, - listed.terminals, - terminalHandle, - result.setupReceipt?.terminalHandle - ) - appendFederationSetupEffect(effects, setup) - return { worktreeId: result.worktree.id, startupTerminalHandle: terminalHandle } - } - }, - surfaces: { - createStructuredSession: () => { - throw new Error(TERMINAL_ONLY_MESSAGE) + let structuredSession: StructuredSession | null = null + let launched: Awaited> + try { + launched = await executeAgentLaunch({ + runtime, + intent: { + agent, + // Federation's own factory builds the create from the attachment, so the target carries none. + target: args.worktree + ? { kind: 'existing', worktree: args.worktree.id } + : { kind: 'create-worktree', create: {} }, + launchSource: 'orchestration' }, - createTerminalAgent: async ({ worktreeId }) => { - const terminal = await runtime.createTerminal(`id:${worktreeId}`, { - // Resolve agent ids through TUI config: cursor is the desktop app, its CLI is cursor-agent. - startupAgent: agent, - launchSource: 'orchestration', - suppressStartupUpdatePrompt: true, - ...(args.launchPreferences ? { launchPreferences: args.launchPreferences } : {}), - title: `worker-${params.taskId}`, - presentation: 'background' - }) - effects.push({ kind: 'terminal', role: 'agent', action: 'created', id: terminal.handle }) - terminalCreation = terminal - return { handle: terminal.handle, terminalCreation: terminal } + decidedMode: args.mode, + vocabulary: WORKER_START_VOCABULARY, + onStage: (stage) => + args.onStage(stage === 'worktree_create' ? 'worktree_create' : 'terminal_create'), + workspaces: { + createWorktree: async ({ startupAgent, promisedAgentChat, onWorkspaceCreated }) => { + const { repo } = params + if (!repo) { + // Validation refuses this before the attachment exists; this only narrows the types. + throw new Error('A remote new-top-level worktree requires an explicit --repo.') + } + db.recordRemoteAttachmentStage({ + dispatchId: params.dispatchId, + stage: 'worktree_creating' + }) + const setupDecision = params.setup ?? 'run' + const result = await runtime.createManagedWorktree({ + onStartupTerminalCreated: (terminal) => { + terminalCreation = terminal + }, + repoSelector: repo, + ...(await resolveWorkerWorktreeName(runtime, repo, params.name)), + baseBranch: params.baseBranch, + displayName: params.displayName, + displayNameKind: params.displayNameKind, + comment: params.comment, + // setupDecision runs setup without the legacy runHooks activation side effect. + runHooks: false, + setupDecision, + awaitTerminalProvisioning: true, + observeSetupCompletion: true, + createdWithAgent: agent, + ...(promisedAgentChat ? { promisedAgentChat: true } : {}), + ...(onWorkspaceCreated ? { onWorkspaceCreated } : {}), + // The executor withholds the startup agent exactly when the worker is to be a chat session. + ...(startupAgent !== undefined + ? { + startupAgent: agent, + startupLaunchSource: 'orchestration' as const, + startupSuppressUpdatePrompt: true, + ...(args.launchPreferences + ? { startupLaunchPreferences: args.launchPreferences } + : {}) + } + : {}), + activate: false, + ...(args.creatorProvenance ? { creatorProvenance: args.creatorProvenance } : {}), + lineage: params.crossStoreParent + ? { + crossStoreParent: params.crossStoreParent, + ...(params.crossStoreParentInferredFrom + ? { crossStoreParentInferredFrom: params.crossStoreParentInferredFrom } + : {}) + } + : { noParent: true } + }) + created = result.worktree + const terminalHandle = result.startupTerminal?.handle + effects.push({ + kind: 'worktree', + action: params.crossStoreParent ? 'created_child' : 'created_top_level', + id: result.worktree.id, + name: result.worktree.displayName + }) + const setup: WorkerSetupReceipt = { + requested: setupDecision, + effective: setupDecision, + source: args.setupSource, + hookFound: result.setupReceipt?.hookFound ?? false, + startupPolicy: result.setupReceipt?.startupPolicy ?? 'start-immediately', + state: result.setupReceipt?.state ?? 'not_configured' + } + args.onSetup(setup) + // A chat has no startup terminal; its setup terminal is still recorded so the setup gate finds it. + const chat = startupAgent === undefined + if (!chat && !terminalHandle) { + throw new Error(result.warning ?? 'Agent-first worktree creation returned no terminal.') + } + const listed = await runtime.listTerminals(`id:${result.worktree.id}`, undefined, { + includeVisualLayouts: false + }) + appendFederationTerminalEffects( + effects, + listed.terminals, + chat ? undefined : terminalHandle, + result.setupReceipt?.terminalHandle + ) + appendFederationSetupEffect(effects, setup) + return { + worktreeId: result.worktree.id, + startupTerminalHandle: chat ? undefined : terminalHandle + } + } + }, + surfaces: { + createStructuredSession: async ({ worktreeId }) => { + args.onStage('terminal_create') + structuredSession = await createStructuredWorkerSessionForWorktree({ + runtime, + worktreeId, + agent, + dispatchId: params.dispatchId, + ...(args.launchPreferences ? { launchPreferences: args.launchPreferences } : {}), + effects + }) + return { + sessionId: structuredSession.identity.sessionId, + handle: structuredSession.identity.handle, + fence: structuredSession.fence + } + }, + createTerminalAgent: async ({ worktreeId }) => { + const terminal = await runtime.createTerminal(`id:${worktreeId}`, { + // Resolve agent ids through TUI config: cursor is the desktop app, its CLI is cursor-agent. + startupAgent: agent, + launchSource: 'orchestration', + suppressStartupUpdatePrompt: true, + ...(args.launchPreferences ? { launchPreferences: args.launchPreferences } : {}), + title: `worker-${params.taskId}`, + presentation: 'background' + }) + effects.push({ kind: 'terminal', role: 'agent', action: 'created', id: terminal.handle }) + terminalCreation = terminal + return { handle: terminal.handle, terminalCreation: terminal } + } } - } - }) + }) + } catch (error) { + // The caller only sees a returned launch, so a session made before this throw is ours. + await tearDownFailedWorkerStart({ + runtime, + structuredSession, + dispatchId: params.dispatchId, + error + }) + throw error + } const worktree = args.worktree ?? created if (!worktree) { throw new Error('Federated worker topology did not resolve.') } - return { worktree, terminalHandle: launched.outcome.handle, terminalCreation } + return { + worktree, + terminalHandle: launched.outcome.handle, + terminalCreation, + structuredSession, + mode: launched.receipt + } } diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-brief.ts b/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-brief.ts new file mode 100644 index 00000000000..0d4966d2d17 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-brief.ts @@ -0,0 +1,134 @@ +import { waitForWorkerAgentReady } from '../../../../terminal/wait/launched-agent-composer-readiness' +import { sendAgentTurn } from '../../../../orchestration/turn-delivery/send-agent-turn' +import { createWorkerBriefWriteGuard } from '../../../../terminal/wait/launched-agent-write-guard' +import { OrchestrationError } from '../../../../orchestration/orchestration-error' +import type { OrcaRuntimeService } from '../../../../orca-runtime' +import type { TuiAgent } from '../../../../../../shared/agent-launch/catalog/tui-agent' +import type { RuntimeTerminalCreate } from '../../../../../../shared/runtime-terminal-create-contract' +import { + formatOrcaSessionAddress, + isOrcaSessionId +} from '../../../../../../shared/orca-session-address' +import { canonicalOrcaSessionId } from '../../../../orchestration/addressing/session-lineage/canonical-orca-session-id' +import { sendStructuredWorkerPreamble } from '../structured-workers/orchestration-structured-worker-session' +import { awaitStructuredWorkerSetupGate } from '../worker/worker-start-structured-setup-gate' +import type { + createStructuredWorkerSessionForWorktree, + WorkerSetupReceipt +} from '../worker/worker-topology' +import type { FederationEffect } from './federation-effects' + +export type StructuredSession = Awaited> + +/** A chat's readiness is its brief being taken, so only a wait-for-setup policy gates it here. */ +export function awaitFederatedWorkerReady(input: { + runtime: OrcaRuntimeService + structuredSession: StructuredSession | null + terminalHandle: string + agent: TuiAgent | undefined + reusesTerminal: boolean + creation: RuntimeTerminalCreate | undefined + setup: WorkerSetupReceipt + effects: FederationEffect[] + timeoutMs: number +}) { + const { runtime, timeoutMs } = input + return input.structuredSession + ? awaitStructuredWorkerSetupGate({ + runtime, + setup: input.setup, + effects: input.effects, + timeoutMs + }) + : waitForWorkerAgentReady(runtime, input.terminalHandle, { + agent: input.agent, + reusesTerminal: input.reusesTerminal, + creation: input.creation, + timeoutMs + }) +} + +/** A chat worker's commands name its Orca session ID, which binds to it, as a local one's do. */ +export function federatedWorkerAddress( + structuredSession: StructuredSession | null, + terminalHandle: string +): string { + const sessionId = structuredSession?.identity.sessionId + return sessionId && isOrcaSessionId(sessionId) + ? formatOrcaSessionAddress(canonicalOrcaSessionId(sessionId)) + : terminalHandle +} + +export function sendFederatedWorkerBrief(input: { + runtime: OrcaRuntimeService + structuredSession: StructuredSession | null + terminalHandle: string + agent: TuiAgent | undefined + params: { dispatchId: string; runId?: string; taskId: string; terminal?: string } + operationId: string + preamble: string + budgetMs: number +}) { + return input.structuredSession + ? sendRemoteChatWorkerBrief( + input.structuredSession, + input.params, + input.preamble, + input.budgetMs + ) + : sendTerminalWorkerBrief(input.runtime, input.terminalHandle, { + agent: input.agent, + launched: !input.params.terminal, + operationId: input.operationId, + preamble: input.preamble + }) +} + +async function sendTerminalWorkerBrief( + runtime: OrcaRuntimeService, + handle: string, + brief: { agent: TuiAgent | undefined; launched: boolean; operationId: string; preamble: string } +) { + // A shell back at its prompt also reads as ready, so the brief needs the agent found in front. + const briefGuard = createWorkerBriefWriteGuard(runtime, brief.agent, brief.launched) + return sendAgentTurn({ + kind: 'terminal', + runtime, + handle, + ...(briefGuard ? { beforeWrite: briefGuard.beforeWrite } : {}), + turn: { purpose: 'dispatch-preamble', operationId: brief.operationId, body: brief.preamble } + }).finally(() => briefGuard?.dispose()) +} + +/** The brief as the chat's first turn, as a local chat worker gets it; one not yet taken leaves the start unknown. */ +async function sendRemoteChatWorkerBrief( + session: StructuredSession, + params: { dispatchId: string; runId?: string; taskId: string }, + preamble: string, + budgetMs: number +): Promise { + const delivery = await sendStructuredWorkerPreamble({ + host: session.host, + sessionId: session.identity.sessionId, + dispatchId: params.dispatchId, + preamble, + from: { + kind: 'agent', + senders: [], + orchestration: { + message: 'task', + runId: params.runId ?? '', + taskId: params.taskId, + dispatchId: params.dispatchId + } + }, + budgetMs + }) + if (delivery !== 'accepted') { + throw new OrchestrationError( + 'operation_unknown', + 'The task was accepted, but the chat agent had not started to take it. It is delivered when the agent starts; if the worker then reports, this Dispatch settles normally.' + ) + } + return undefined +} diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-release-host.ts b/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-release-host.ts index 74ba608fcc8..1aa4c9731ac 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-release-host.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-release-host.ts @@ -2,8 +2,7 @@ import { describeUnconfirmedAgentStop } from '../../../../../../shared/terminal/ import type { RemoteDispatchAttachmentRow } from '../../../../orchestration/db/orchestration-records' import type { WorkerTerminalArchiveRow, - WorkerTerminalResourceRow, - WorkerTerminalRetainedReason + WorkerTerminalResourceRow } from '../../../../orchestration/terminal-authority/release/worker-terminal-ownership' import { captureWorkerOutputArchive, @@ -14,10 +13,14 @@ import { import type { WorkerTerminalReleaseRuntime } from '../../../../orchestration/terminal-authority/release/worker-terminal-release-runtime' import { readRemoteAttachmentArchive } from './remote-attachment-archive-read' import { WorkerTerminalReleaseSingleFlight } from '../worker/worker-terminal-release-single-flight' +import { remoteChatWorkerOf } from '../../../../orchestration/federation/remote-chat-worker' +import { releaseRemoteChatWorker } from './remote-chat-worker-host' import { yieldAutoReleaseToWorkingAgent } from '../../../../orchestration/terminal-authority/release/worker-terminal-auto-release-working-yield' import { archiveSummary } from '../worker/worker-terminal-resource-presentation' import { + isStillReleasing, releaseUnknownRecovery, + retainedReason, type WorkerReleaseReceipt } from '../worker/worker-release-completion' import { orchestrationTimestampToMs } from '../worker/worker-output' @@ -113,6 +116,10 @@ async function completeRemoteAttachmentRelease( ): Promise { const { runtime, attachment, observation } = args const db = runtime.getOrchestrationDb() + const chat = remoteChatWorkerOf(db, attachment) + if (chat?.exact) { + return releaseRemoteChatWorker(runtime, attachment, chat.identity, resource, storedArchive) + } if (!observation.exact || !observation.terminal) { if ( args.mode === 'recovery' && @@ -280,16 +287,6 @@ function remoteAttachmentLeaseIsCurrent( ) } -function retainedReason(resource: WorkerTerminalResourceRow): WorkerTerminalRetainedReason { - if (resource.retained_reason) { - return resource.retained_reason as WorkerTerminalRetainedReason - } - if (resource.ownership_state === 'user_owned') { - return 'user_takeover' - } - return 'identity_unproven' -} - function projectArchivedOutputLiveness< T extends { status: { terminal: string; liveness: string } } >(output: T, liveness: 'live' | 'unverifiable' | 'exited'): T { @@ -302,7 +299,3 @@ function projectArchivedOutputLiveness< } } } - -function isStillReleasing(resource: WorkerTerminalResourceRow): boolean { - return resource.ownership_state === 'owned' && resource.release_state === 'releasing' -} diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-start.ts b/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-start.ts index 781aa443f20..0c56218f58a 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-start.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-start.ts @@ -20,7 +20,8 @@ import { resolveFederatedWorkerStartBudgets } from '../worker/worker-start-budge import { resolveDispatchCreator } from '../runs/dispatch-creator' import { isReadyRemoteFederatedWorkerStartReceipt, - parseRemoteFederatedWorkerStartReceipt + parseRemoteFederatedWorkerStartReceipt, + readyRemoteReceiptFields } from './federated-attach-receipt' import { isWorkerStartTimeoutWithinTimerLimit } from '../../../../../../shared/orchestration/orchestration-timing-budgets' import { @@ -232,11 +233,9 @@ export async function startFederatedWorker(args: FederatedWorkerStartRequest): P state: 'ready', stage: readyWorker.stage, server: { environmentId: server.environmentId, name: server.name }, - setup: remote.setup, + ...readyRemoteReceiptFields(remote), launch, timeoutMs: budgets.readinessTimeoutMs, - effects: remote.effects ?? [], - residualResources: remote.residualResources ?? [], ...(childParent && 'warning' in childParent ? { warning: childParent.warning } : {}) }, reportsTurnStops diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-stop-host.ts b/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-stop-host.ts index d73f42f565a..8fde0534a34 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-stop-host.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federated-worker-stop-host.ts @@ -3,6 +3,8 @@ import type { WorkerStopReceipt } from '../../../../../../shared/orchestration/w import { archiveWorkerOutputBeforeStop } from '../worker/worker-stop-archive' import { describeUnconfirmedAgentStop } from '../../../../../../shared/terminal/pty/lifecycle/pty-liveness-verdict' import { inspectRemoteAttachment } from './federation-attachment-observation' +import { remoteChatWorkerOf } from '../../../../orchestration/federation/remote-chat-worker' +import { stopRemoteChatWorker } from './remote-chat-worker-host' /** Close the exact attached worker this host runs for its home. */ export async function stopRemoteAttachment( @@ -19,6 +21,10 @@ export async function stopRemoteAttachment( processAction: 'none' } } + const chat = remoteChatWorkerOf(db, begun) + if (chat) { + return stopRemoteChatWorker(runtime, params.dispatchId, chat) + } const observation = await inspectRemoteAttachment(runtime, params.dispatchId) if (!observation.exact || !observation.terminal) { const attachment = db.markRemoteAttachmentStopUnknown( diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation-attachment-authority.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation-attachment-authority.ts index 28448ff5991..a704a4ff46d 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation-attachment-authority.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation-attachment-authority.ts @@ -2,6 +2,7 @@ import type { OrcaRuntimeService } from '../../../../orca-runtime' import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db' import type { FederationEffect } from './federation-effects' import type { WorkerSetupReceipt } from '../worker/worker-topology' +import type { StructuredWorkerIdentity } from '../../../../orchestration/terminal-authority/structured-sessions/structured-worker-identity' export function prepareFederatedAttachmentAuthority(args: { runtime: OrcaRuntimeService @@ -12,9 +13,11 @@ export function prepareFederatedAttachmentAuthority(args: { setup: WorkerSetupReceipt effects: FederationEffect[] reusesTerminal: boolean + /** A chat worker's minted identity, which stands in for a terminal's pane and process. */ + identity?: Pick }): void { const { runtime, terminalHandle } = args - const authority = runtime.getOrchestrationDispatchAuthority(terminalHandle) + const authority = args.identity ?? runtime.getOrchestrationDispatchAuthority(terminalHandle) const paneKey = authority?.paneKey ?? runtime.getTerminalPaneKey(terminalHandle) const processIncarnation = authority?.processIncarnation ?? runtime.getTerminalProcessIncarnation(terminalHandle) diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation-attachment-observation.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation-attachment-observation.ts index cddf8efcfa7..18781c32db6 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation-attachment-observation.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation-attachment-observation.ts @@ -4,6 +4,10 @@ import { OrchestrationError } from '../../../../orchestration/orchestration-erro import { parseWorkerTerminalHostScope } from '../../../../../../shared/orchestration/workers/worker-terminal-host-scope' import type { RemoteDispatchAttachmentRow } from '../../../../orchestration/db/orchestration-records' import { orchestrationTimestampToMs } from '../worker/worker-output' +import { + REMOTE_CHAT_WORKER_READ_UNSUPPORTED, + remoteChatWorkerOf +} from '../../../../orchestration/federation/remote-chat-worker' export function requireHomeAttachment( runtime: WorkerTerminalReleaseRuntime, @@ -20,6 +24,16 @@ export function requireHomeAttachment( return attachment } +/** A live read of a remote chat worker is refused plainly, until reads of one are supported. */ +export function refuseRemoteChatWorkerRead( + runtime: WorkerTerminalReleaseRuntime, + attachment: RemoteDispatchAttachmentRow +): void { + if (remoteChatWorkerOf(runtime.getOrchestrationDb(), attachment)?.exact) { + throw new OrchestrationError('capability_unsupported', REMOTE_CHAT_WORKER_READ_UNSUPPORTED) + } +} + export async function inspectRemoteAttachment( runtime: WorkerTerminalReleaseRuntime, dispatchId: string @@ -34,6 +48,16 @@ export async function inspectRemoteAttachment( }> { const db = runtime.getOrchestrationDb() const attachment = db.getRemoteDispatchAttachment(dispatchId) + const chat = remoteChatWorkerOf(db, attachment) + if (chat) { + // Said plainly rather than read through a terminal it does not have. + return { + terminal: null, + exact: chat.exact, + status: chat.exact ? 'unverifiable' : 'identity_changed', + ...(chat.exact ? { reason: REMOTE_CHAT_WORKER_READ_UNSUPPORTED } : {}) + } + } if (!attachment?.terminal_handle) { return { terminal: null, exact: false, status: 'unattached' } } diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation-control.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation-control.ts index 4fbe9d527ae..faa8fca046b 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation-control.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation-control.ts @@ -10,6 +10,7 @@ import { stopRemoteAttachment } from './federated-worker-stop-host' import { inspectRemoteAttachment, observeRemoteAttachmentLiveness, + refuseRemoteChatWorkerRead, requireHomeAttachment } from './federation-attachment-observation' import { releaseRemoteAttachment } from './federated-worker-release-host' @@ -111,7 +112,10 @@ export const ORCHESTRATION_FEDERATION_CONTROL_METHODS = [ permission: 'workspace', params: FederationReadParams, handler: async (params, { runtime, authenticatedCallerFingerprint }) => { - requireHomeAttachment(runtime, params.dispatchId, authenticatedCallerFingerprint) + refuseRemoteChatWorkerRead( + runtime, + requireHomeAttachment(runtime, params.dispatchId, authenticatedCallerFingerprint) + ) const observation = await inspectRemoteAttachment(runtime, params.dispatchId) // Only exited proves death; an unverifiable terminal may still be readable after stop-contact loss. if (!observation.exact || !observation.terminal || observation.status === 'exited') { @@ -171,6 +175,7 @@ export const ORCHESTRATION_FEDERATION_CONTROL_METHODS = [ } } } + refuseRemoteChatWorkerRead(runtime, attachment) const observation = await inspectRemoteAttachment(runtime, params.dispatchId) if (!observation.exact || !observation.terminal) { throw new OrchestrationError( diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation-effects.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation-effects.ts index 116f51e9ee8..1dff5af053b 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation-effects.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation-effects.ts @@ -19,7 +19,7 @@ export type FederationEffect = { export function appendFederationTerminalEffects( effects: FederationEffect[], terminals: { handle: string; title: string | null; tabId?: string; leafId?: string }[], - agentHandle: string, + agentHandle: string | undefined, setupHandle?: string ): void { for (const terminal of terminals) { diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation-setup.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation-setup.ts index b4dc2c5e86b..3d7119f853f 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation-setup.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation-setup.ts @@ -94,3 +94,28 @@ export function monitorFederatedSetup( }) .catch(() => undefined) } + +export function federatedSetupSource( + params: { setup?: WorkerSetupReceipt['requested']; setupSource?: WorkerSetupReceipt['source'] }, + createsWorktree: boolean +): WorkerSetupReceipt['source'] { + return createsWorktree + ? (params.setupSource ?? (params.setup ? 'explicit_request' : 'orchestration_default')) + : 'existing_worktree' +} + +/** The setup a start reports before any setup ran: none for an existing worktree. */ +export function initialFederatedSetup( + params: { setup?: WorkerSetupReceipt['requested'] }, + createsWorktree: boolean, + source: WorkerSetupReceipt['source'] +): WorkerSetupReceipt { + return { + requested: createsWorktree ? (params.setup ?? 'run') : 'not_applicable', + effective: createsWorktree ? (params.setup ?? 'run') : 'not_applicable', + source, + hookFound: false, + startupPolicy: 'start-immediately', + state: createsWorktree ? 'not_configured' : 'not_applicable' + } +} diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation.ts index 995d88ddb8b..b7f0e3f66d7 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation.ts @@ -1,16 +1,14 @@ import type { RuntimeTerminalCreate } from '../../../../../../shared/runtime-terminal-create-contract' -import { waitForWorkerAgentReady } from '../../../../terminal/wait/launched-agent-composer-readiness' import { describeTerminalWaitBlockedReason } from '../../../../../../shared/terminal/runtime-policy/terminal-wait-blocked-reason-legacy-alias' import { buildDispatchPreamble } from '../../../../orchestration/workers/launch/worker-dispatch-preamble' -import { sendAgentTurn } from '../../../../orchestration/turn-delivery/send-agent-turn' -import { createWorkerBriefWriteGuard } from '../../../../terminal/wait/launched-agent-write-guard' import { OrchestrationError } from '../../../../orchestration/orchestration-error' import { defineMethod } from '../../../core' import { assertOrchestrationWorktreeCreationSupported } from '../worker/folder-worktree-placement' -import type { FederationEffect } from './federation-effects' +import { isFederationEffectUnknown, type FederationEffect } from './federation-effects' import { launchFederatedWorkerAgent } from './federated-worker-agent-launch' -import type { WorkerSetupReceipt } from '../worker/worker-topology' import { + federatedSetupSource, + initialFederatedSetup, monitorFederatedSetup, persistFederatedReadinessStage, persistFederatedSetupSpawnFailure, @@ -26,6 +24,18 @@ import { import { assertWorkerStartTaskSpecWithinPromptBudget } from '../worker/worker-start-prompt-budget' import { prepareFederatedAttachmentAuthority } from './federation-attachment-authority' import { resolveRpcWorkspaceCreatorProvenance } from '../../../workspace-creator-context' +import { + decideWorkerStartMode, + readWorkerStartModeSettings, + resolveWorkerStartModeOnHost +} from '../structured-workers/orchestration-worker-start-mode' +import { tearDownFailedWorkerStart } from '../worker/failed-worker-start-teardown' +import { + awaitFederatedWorkerReady, + federatedWorkerAddress, + sendFederatedWorkerBrief, + type StructuredSession +} from './federated-worker-brief' import { claimTerminalForRemoteAttach } from '../../../../orchestration/db/worker-dispatch/worker-terminal-claim' export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [ @@ -96,17 +106,14 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [ let worktree let terminalHandle = params.terminal let terminalCreation: RuntimeTerminalCreate | undefined - const setupSource = createsWorktree - ? (params.setupSource ?? (params.setup ? 'explicit_request' : 'orchestration_default')) - : 'existing_worktree' - let setup: WorkerSetupReceipt = { - requested: createsWorktree ? (params.setup ?? 'run') : 'not_applicable', - effective: createsWorktree ? (params.setup ?? 'run') : 'not_applicable', - source: setupSource, - hookFound: false, - startupPolicy: 'start-immediately', - state: createsWorktree ? 'not_configured' : 'not_applicable' - } + let structuredSession: StructuredSession | null = null + // This host decides chat or terminal from its own settings, as a local start does. + let mode = decideWorkerStartMode({ + params: { agent, terminal: params.terminal, worktree: params.worktree }, + settings: readWorkerStartModeSettings(runtime) + }) + const setupSource = federatedSetupSource(params, createsWorktree) + let setup = initialFederatedSetup(params, createsWorktree, setupSource) try { const launchAgent = (existing?: { id: string }) => launchFederatedWorkerAgent({ @@ -115,6 +122,7 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [ params, agent, launchPreferences: launch.preferences, + mode, ...(existing ? { worktree: existing } : { creatorProvenance: resolveRpcWorkspaceCreatorProvenance(context) }), @@ -128,7 +136,8 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [ } }) if (createsWorktree) { - ;({ worktree, terminalHandle, terminalCreation } = await launchAgent()) + ;({ worktree, terminalHandle, terminalCreation, structuredSession, mode } = + await launchAgent()) } else { worktree = await runtime.showManagedTerminalWorkspace(params.worktree).catch(() => { throw new OrchestrationError( @@ -162,7 +171,9 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [ }) } else { failedStage = 'terminal_create' - ;({ terminalHandle, terminalCreation } = await launchAgent(worktree)) + mode = await resolveWorkerStartModeOnHost(runtime, mode, worktree.id, agent) + ;({ terminalHandle, terminalCreation, structuredSession, mode } = + await launchAgent(worktree)) } } if (!worktree || !terminalHandle) { @@ -182,14 +193,21 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [ } persistFederatedReadinessStage(setupStage) failedStage = 'agent_readiness' - const wait = await waitForWorkerAgentReady(runtime, terminalHandle, { + const wait = await awaitFederatedWorkerReady({ + runtime, + structuredSession, + terminalHandle, agent, reusesTerminal: Boolean(params.terminal), creation: terminalCreation, + setup, + effects, timeoutMs: readinessTimeoutMs }) - persistFederatedSetupWaitOutcome({ ...setupStage, wait }) - if (!wait.satisfied) { + if (wait) { + persistFederatedSetupWaitOutcome({ ...setupStage, wait }) + } + if (wait && !wait.satisfied) { if (setup.state === 'failed') { failedStage = 'setup_wait' } @@ -207,33 +225,32 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [ terminalHandle, setup, effects, - reusesTerminal: Boolean(params.terminal) + reusesTerminal: Boolean(params.terminal), + ...(structuredSession ? { identity: structuredSession.identity } : {}) }) failedStage = 'dispatch_input' - // A shell back at its prompt also reads as ready, so the brief needs the agent found in front. - const briefGuard = createWorkerBriefWriteGuard(runtime, agent, !params.terminal) - const prompt = await sendAgentTurn({ - kind: 'terminal', + const preamble = buildDispatchPreamble({ + taskId: params.taskId, + dispatchId: params.dispatchId, + taskSpec: params.taskSpec, + coordinatorHandle: 'Run home (relayed by Orca)', + workerHandle: federatedWorkerAddress(structuredSession, terminalHandle), + devMode: params.devMode, + // Enforce the execution host's nesting cap using its own setting. + canDispatchSubWorkers: (params.depth ?? 1) < runtime.getNestedWorkerMaxDepth(), + subWorkersInOwnRun: true, + cliCommand: runtime.getTerminalOrchestrationCliCommand(terminalHandle) + }) + const prompt = await sendFederatedWorkerBrief({ runtime, - handle: terminalHandle, - ...(briefGuard ? { beforeWrite: briefGuard.beforeWrite } : {}), - turn: { - purpose: 'dispatch-preamble', - operationId: orchestrationMutation.requestId, - body: buildDispatchPreamble({ - taskId: params.taskId, - dispatchId: params.dispatchId, - taskSpec: params.taskSpec, - coordinatorHandle: 'Run home (relayed by Orca)', - workerHandle: terminalHandle, - devMode: params.devMode, - // Enforce the execution host's nesting cap using its own setting. - canDispatchSubWorkers: (params.depth ?? 1) < runtime.getNestedWorkerMaxDepth(), - subWorkersInOwnRun: true, - cliCommand: runtime.getTerminalOrchestrationCliCommand(terminalHandle) - }) - } - }).finally(() => briefGuard?.dispose()) + structuredSession, + terminalHandle, + agent, + params, + operationId: orchestrationMutation.requestId, + preamble, + budgetMs: readinessTimeoutMs + }) effects.push({ kind: 'dispatch_input', role: 'agent', @@ -251,11 +268,21 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [ terminalHandle, setup, launch: launch.receipt, + mode, effects, - ...(prompt.prompt ? { prompt: prompt.prompt } : {}), + ...(prompt?.prompt ? { prompt: prompt.prompt } : {}), residualResources: [] } } catch (error) { + // A chat that may hold its brief stays for the worker to report; a refused start retires it. + if (structuredSession && !isFederationEffectUnknown(error, failedStage)) { + await tearDownFailedWorkerStart({ + runtime, + structuredSession, + dispatchId: params.dispatchId, + error + }) + } return failFederatedAttachmentWithReceipt({ db, dispatchId: params.dispatchId, diff --git a/src/main/runtime/rpc/methods/orchestration/federation/remote-chat-worker-host.ts b/src/main/runtime/rpc/methods/orchestration/federation/remote-chat-worker-host.ts new file mode 100644 index 00000000000..8af65c2f403 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration/federation/remote-chat-worker-host.ts @@ -0,0 +1,141 @@ +import type { OrcaRuntimeService } from '../../../../orca-runtime' +import type { WorkerStopReceipt } from '../../../../../../shared/orchestration/workers/orchestration-worker-stop-receipt' +import type { + WorkerTerminalArchiveRow, + WorkerTerminalResourceRow +} from '../../../../orchestration/terminal-authority/release/worker-terminal-ownership' +import { remoteChatWorkerOf } from '../../../../orchestration/federation/remote-chat-worker' +import type { StructuredWorkerIdentity } from '../../../../orchestration/terminal-authority/structured-sessions/structured-worker-identity' +import type { WorkerTerminalReleaseRuntime } from '../../../../orchestration/terminal-authority/release/worker-terminal-release-runtime' +import { stopStructuredWorker } from '../structured-workers/orchestration-structured-worker-lifecycle' +import { stopStructuredWorkerForRelease } from '../worker/structured-worker-release-stop' +import { captureWorkerReleaseArchive } from '../worker/worker-release-archive-capture' +import { + isStillReleasing, + retainedReason, + type WorkerReleaseReceipt +} from '../worker/worker-release-completion' +import { orchestrationTimestampToMs } from '../worker/worker-output' +import { archiveSummary } from '../worker/worker-terminal-resource-presentation' +import { summarizeWorkerOutputArchive } from '../../../../orchestration/workers/archives/worker-output-archive' +import { yieldAutoReleaseToWorkingAgent } from '../../../../orchestration/terminal-authority/release/worker-terminal-auto-release-working-yield' +import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db' +import type { RemoteDispatchAttachmentRow } from '../../../../orchestration/db/orchestration-records' + +/** Close a chat worker the way a local stop does; only a proven close settles. */ +export async function stopRemoteChatWorker( + runtime: OrcaRuntimeService, + dispatchId: string, + chat: NonNullable> +): Promise { + const db = runtime.getOrchestrationDb() + const unknown = (reason: string, processAction: string): WorkerStopReceipt => { + const attachment = db.markRemoteAttachmentStopUnknown(dispatchId, reason) + return { + dispatchId, + state: attachment.state, + alreadySettled: false, + processAction, + lastError: attachment.last_error + } + } + if (!chat.exact) { + return unknown( + 'The recorded chat worker is no longer this Dispatch; nothing was closed.', + 'none' + ) + } + // Install the chat host after a restart, or the close cannot reach the session. + await runtime.ensureStructuredAgentSessionHost().catch(() => undefined) + const stop = await stopStructuredWorker(chat.identity, runtime) + if (!stop.stopped) { + return unknown( + stop.reason ?? 'The close was not proven.', + stop.closeAttempted ? 'closed_agent_terminal' : 'none' + ) + } + const attachment = db.settleRemoteAttachmentStop(dispatchId) + return { + dispatchId, + state: attachment.state, + alreadySettled: false, + processAction: 'closed_agent_terminal' + } +} + +/** A chat releases the way a local chat worker does: its transcript is kept, the release commits, + * a working agent defers an automatic one, and only then does its chat close. */ +export async function releaseRemoteChatWorker( + runtime: WorkerTerminalReleaseRuntime, + attachment: RemoteDispatchAttachmentRow, + identity: StructuredWorkerIdentity, + resource: WorkerTerminalResourceRow, + storedArchive: WorkerTerminalArchiveRow | null +): Promise { + const db = runtime.getOrchestrationDb() + const dispatchId = attachment.dispatch_id + await runtime.ensureStructuredAgentSessionHost().catch(() => undefined) + let archive: Pick< + Parameters[0], + 'kind' | 'content' | 'archiveSource' | 'archiveStatus' + > + if (storedArchive) { + const stored = summarizeWorkerOutputArchive(storedArchive) + archive = { + archiveSource: stored.source, + archiveStatus: stored.status === 'empty' ? 'empty' : 'captured' + } + } else { + const capture = await captureWorkerReleaseArchive({ + db, + resource, + runtime, + dispatchId, + terminalHandle: identity.handle, + attachedAtMs: orchestrationTimestampToMs(attachment.created_at), + structuredWorker: identity + }) + if ('receipt' in capture) { + return capture.receipt + } + const { captured } = capture + archive = { + kind: captured.kind, + content: JSON.stringify(captured.content), + archiveSource: captured.kind === 'terminal_tail' ? 'terminal' : 'transcript', + archiveStatus: captured.status === 'empty' ? 'empty' : 'captured' + } + } + const committed = db.commitWorkerTerminalArchiveForRelease({ + dispatchId, + resourceId: resource.id, + ...archive + }) + const releasing = !isStillReleasing(committed) + ? committed + : !remoteChatWorkerOf(db, attachment)?.exact + ? db.revertWorkerTerminalReleaseToRetained(resource.id, 'identity_unproven') + : yieldAutoReleaseToWorkingAgent(runtime, committed) + if (!isStillReleasing(releasing)) { + return { + dispatchId, + state: 'retained', + reason: retainedReason(releasing), + processAction: 'none', + archive: archiveSummary(releasing) + } + } + const released = await stopStructuredWorkerForRelease({ + structured: identity, + dispatchId, + resource: releasing, + runtime, + db, + archiveSource: releasing.archive_source, + archiveStatus: releasing.archive_status + }) + if (released.state === 'released') { + db.recordRemoteAttachmentStage({ dispatchId, stage: 'released' }) + } + return released +} diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-release-completion.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-release-completion.ts index cb4b1b5f410..753102fa74d 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-release-completion.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-release-completion.ts @@ -271,7 +271,7 @@ export function releaseUnknownRecovery(dispatchId: string): string { return `Inspect with: orca orchestration worker-show --dispatch ${dispatchId} --json — then retry worker-release with a fresh request ID (omit --retry-request to let the CLI generate one). Reusing the prior request ID only replays this release_unknown receipt. Never substitute a broad terminal close.` } -function retainedReason(resource: WorkerTerminalResourceRow): WorkerTerminalRetainedReason { +export function retainedReason(resource: WorkerTerminalResourceRow): WorkerTerminalRetainedReason { if (resource.retained_reason) { return resource.retained_reason as WorkerTerminalRetainedReason } @@ -281,6 +281,6 @@ function retainedReason(resource: WorkerTerminalResourceRow): WorkerTerminalReta return 'identity_unproven' } -function isStillReleasing(resource: WorkerTerminalResourceRow): boolean { +export function isStillReleasing(resource: WorkerTerminalResourceRow): boolean { return resource.ownership_state === 'owned' && resource.release_state === 'releasing' } diff --git a/src/main/runtime/rpc/methods/orchestration/worker/workers.ts b/src/main/runtime/rpc/methods/orchestration/worker/workers.ts index 51b920e05c9..d20a19184a5 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/workers.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/workers.ts @@ -171,7 +171,7 @@ export async function startWorker( settings: readWorkerStartModeSettings(runtime) }) if (params.on) { - // Remote workers are terminal-only; carry the mode receipt to explain structured-preference fallback. + // The server decides chat or terminal; one that cannot run chats leaves this host's terminal receipt. const receipt = await startFederatedWorker({ params, runtime, @@ -184,7 +184,7 @@ export async function startWorker( followUp }) await stopHelperAfterStart(runtime, receipt, orchestrationMutation) - return withRun(receipt && typeof receipt === 'object' ? { ...receipt, mode } : receipt) + return withRun(receipt && typeof receipt === 'object' ? { mode, ...receipt } : receipt) } const receipt = await startLocalWorker({ params: { ...params, timeoutMs: readinessTimeoutMs },