mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 10:58:10 +00:00
feat(orchestration): start chat workers on a paired server
A server that prefers chats now starts a remote worker as a chat, the same way a local start does. The chat takes its task as its first turn (after a wait-for-setup setup finishes), reports home over the relay (mail, turn stops, worker_done), takes its coordinator's mail as new turns, and closes on stop or release, including automatic release. Reading a remote chat worker's state says plainly that it is not supported yet.
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
import { isStructuredWorkerHandle } from '../../../../shared/orchestration/workers/structured-worker-handle'
|
||||
import type { OrchestrationDb } from '../db/orchestration-db'
|
||||
import type { RemoteDispatchAttachmentRow } from '../db/orchestration-records'
|
||||
import { resolveStructuredWorkerIdentity } from '../terminal-authority/structured-sessions/structured-worker-authority'
|
||||
import type { StructuredWorkerIdentity } from '../terminal-authority/structured-sessions/structured-worker-identity'
|
||||
|
||||
/** Said wherever this host would read a remote chat worker's state, until reads of one are supported. */
|
||||
export const REMOTE_CHAT_WORKER_READ_UNSUPPORTED =
|
||||
'Orca cannot read the state of a chat worker on a paired server yet; open its chat on that server to see it.'
|
||||
|
||||
/** The chat session a coordinator on another computer runs here, with whether it is still the attached one. */
|
||||
export function remoteChatWorkerOf(
|
||||
db: OrchestrationDb,
|
||||
attachment: Pick<RemoteDispatchAttachmentRow, 'dispatch_id' | 'terminal_handle'> | undefined
|
||||
): { identity: StructuredWorkerIdentity; exact: boolean } | null {
|
||||
const handle = attachment?.terminal_handle
|
||||
const identity =
|
||||
handle && isStructuredWorkerHandle(handle) ? resolveStructuredWorkerIdentity(handle, db) : null
|
||||
if (!attachment || !identity) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
identity,
|
||||
exact: db.isRemoteAttachmentProcessCurrent({
|
||||
dispatchId: attachment.dispatch_id,
|
||||
paneKey: identity.paneKey,
|
||||
processIncarnation: identity.processIncarnation
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
import type { RunRow } from '../../db/orchestration-records'
|
||||
import { chatAssigneeSessionId } from '../../workers/lifecycle/chat-assignee'
|
||||
import { structuredWorkerCustody } from '../../terminal-authority/structured-sessions/structured-worker-authority'
|
||||
import { remoteChatWorkerOf } from '../../federation/remote-chat-worker'
|
||||
|
||||
/** A handle-less coordinator uses its generation-current session id; structured workers resolve by handle. */
|
||||
export function handleLessCoordinatorSessionId(
|
||||
@@ -118,7 +119,11 @@ export function structuredMailboxCustody(
|
||||
if (!mailboxHandle.startsWith('dispatch:')) {
|
||||
return workerMailCustody(mailboxHandle, db)
|
||||
}
|
||||
const dispatch = db?.getDispatchContextById?.(mailboxHandle.slice('dispatch:'.length))
|
||||
const dispatchId = mailboxHandle.slice('dispatch:'.length)
|
||||
const dispatch = db?.getDispatchContextById?.(dispatchId)
|
||||
if (!dispatch && db) {
|
||||
return remoteDispatchMailCustody(dispatchId, db)
|
||||
}
|
||||
// A Dispatch's mail follows its Task: none reaches the chat before it took the Task, or after it ended.
|
||||
if (!dispatch?.assignee_handle || dispatch.status !== 'dispatched') {
|
||||
return 'elsewhere'
|
||||
@@ -129,6 +134,16 @@ export function structuredMailboxCustody(
|
||||
: workerMailCustody(dispatch.assignee_handle, db, dispatch.id)
|
||||
}
|
||||
|
||||
/** A Dispatch a coordinator on another computer runs here reaches its chat only while that chat still works it. */
|
||||
function remoteDispatchMailCustody(
|
||||
dispatchId: string,
|
||||
db: OrchestrationDb
|
||||
): StructuredMailboxCustody {
|
||||
const attachment = db.getRemoteDispatchAttachment?.(dispatchId)
|
||||
const chat = attachment?.state === 'ready' ? remoteChatWorkerOf(db, attachment) : null
|
||||
return chat?.exact ? workerMailCustody(chat.identity.handle, db, dispatchId) : 'elsewhere'
|
||||
}
|
||||
|
||||
/** Re-derive Run, direct and active-Dispatch mail on each idle edge, including root mail for /clear successors. */
|
||||
export function structuredSessionOwnedMailboxes(sessionId: string, db: OrchestrationDb): string[] {
|
||||
const party = isOrcaSessionId(sessionId) ? addressableSessionParty(sessionId, db) : null
|
||||
@@ -144,6 +159,10 @@ export function structuredSessionOwnedMailboxes(sessionId: string, db: Orchestra
|
||||
if (dispatch) {
|
||||
mailboxes.push(`dispatch:${dispatch.id}`)
|
||||
}
|
||||
const remote = party.paneKey ? db.findActiveRemoteAttachmentForPane(party.paneKey) : undefined
|
||||
if (remote && remoteChatWorkerOf(db, remote)?.exact) {
|
||||
mailboxes.push(`dispatch:${remote.dispatch_id}`)
|
||||
}
|
||||
return mailboxes
|
||||
}
|
||||
|
||||
|
||||
@@ -26,13 +26,19 @@ export type StructuredMailResend = { row: StructuredMailSendRow; renew: boolean
|
||||
const REUSABLE_OPERATION_AGE_MS =
|
||||
AGENT_SESSION_MAX_NEW_OPERATION_AGE_MS - AGENT_SESSION_OPERATION_FUTURE_SKEW_MS
|
||||
|
||||
/** A mailbox nothing can own again: its Dispatch settled or is gone, or its Run is gone. */
|
||||
/** A mailbox nothing can own again: its Dispatch (or remote attachment) settled or is gone, or its Run is gone. */
|
||||
export function structuredMailboxEnded(
|
||||
mailboxHandle: string,
|
||||
db: OrchestrationDb | null | undefined
|
||||
): boolean {
|
||||
if (mailboxHandle.startsWith('dispatch:')) {
|
||||
const status = db?.getDispatchContextById?.(mailboxHandle.slice('dispatch:'.length))?.status
|
||||
const dispatchId = mailboxHandle.slice('dispatch:'.length)
|
||||
const status = db?.getDispatchContextById?.(dispatchId)?.status
|
||||
if (status === undefined) {
|
||||
// A coordinator on another computer runs this Dispatch; here it is only an attachment.
|
||||
const remote = db?.getRemoteDispatchAttachment?.(dispatchId)?.state
|
||||
return remote !== 'starting' && remote !== 'ready'
|
||||
}
|
||||
return status !== 'pending' && status !== 'dispatched'
|
||||
}
|
||||
return mailboxHandle.startsWith('run:') && !db?.getRun?.(mailboxHandle.slice('run:'.length))
|
||||
|
||||
+301
@@ -0,0 +1,301 @@
|
||||
// A coordinator on another computer starts a chat worker here; the worker reports home over the relay.
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { EVERY_AGENT_CHAT } from '../../../../../shared/native-chat/every-agent-chat.test-fixture'
|
||||
import { isStructuredWorkerHandle } from '../../../../../shared/orchestration/workers/structured-worker-handle'
|
||||
import * as sessionCreate from '../../../rpc/methods/structured-sessions/creation/structured-agent-session-create'
|
||||
import { LOCAL_EXECUTION_HOST_ID } from '../../../../../shared/execution-host'
|
||||
import { chatWorkerTurnStopReporter } from '../../workers/turn-stops/chat-worker-turn-stops'
|
||||
import { encodeFederatedControlMessage } from '../../federation/relay/federation-control-message'
|
||||
import * as placement from '../../../rpc/methods/orchestration/worker/folder-worktree-placement'
|
||||
import * as worktreeName from '../../../rpc/methods/orchestration/worker/worker-worktree-creation'
|
||||
import { attachParams } from './structured-chat-coordinator-fake-codex-fixture'
|
||||
import {
|
||||
WAIT,
|
||||
call,
|
||||
codex,
|
||||
db,
|
||||
dispatcher,
|
||||
host,
|
||||
request,
|
||||
runtime,
|
||||
settleTurn,
|
||||
turnText
|
||||
} from './structured-chat-coordinator-mail-rig.test-fixture'
|
||||
import { resolveStructuredWorkerIdentity } from './structured-worker-authority'
|
||||
import { reconcileRequestedWorkerTerminalReleases } from '../release/worker-terminal-release-reconciliation'
|
||||
|
||||
const DISPATCH = 'ctx_remote_chat'
|
||||
const HOME = 'home_peer'
|
||||
|
||||
beforeEach(() => {
|
||||
vi.spyOn(runtime, 'validateOrchestrationAgentLauncher').mockImplementation(() => {})
|
||||
vi.spyOn(runtime, 'getStructuredAgentSessionCreateSupport').mockResolvedValue({ supported: true })
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: placement reads only the workspace identity in this rig.
|
||||
vi.spyOn(runtime, 'showManagedTerminalWorkspace').mockResolvedValue({
|
||||
id: 'workspace-1',
|
||||
repoId: 'repo'
|
||||
} as never)
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: launch-mode selection reads only getSettings on this store double.
|
||||
vi.spyOn(runtime, 'requireStore').mockReturnValue({
|
||||
getSettings: () => ({
|
||||
nativeChatProviders: EVERY_AGENT_CHAT,
|
||||
agentCmdOverrides: {},
|
||||
agentDefaultArgs: {},
|
||||
agentDefaultEnv: {}
|
||||
})
|
||||
} as never)
|
||||
vi.spyOn(sessionCreate, 'createStructuredAgentSessionForWorktree').mockImplementation(
|
||||
async (args) => {
|
||||
const attached = await host.attach(args.caller, attachParams(args.envelope.sessionId))
|
||||
if (!attached.ok) {
|
||||
throw new Error('scripted worker attach refused')
|
||||
}
|
||||
return {
|
||||
...attached,
|
||||
value: { ...attached.value, sessionId: args.envelope.sessionId, fence: attached.fence }
|
||||
}
|
||||
}
|
||||
)
|
||||
// The rig fakes PTY identity for its own terminals; a chat worker's comes from its minted identity.
|
||||
const identity = (handle: string) =>
|
||||
isStructuredWorkerHandle(handle) ? resolveStructuredWorkerIdentity(handle, db) : null
|
||||
vi.mocked(runtime.getTerminalPaneKey).mockImplementation(
|
||||
(handle) => identity(handle)?.paneKey ?? null
|
||||
)
|
||||
vi.mocked(runtime.getTerminalProcessIncarnation).mockImplementation(
|
||||
(handle) => identity(handle)?.processIncarnation ?? null
|
||||
)
|
||||
})
|
||||
|
||||
async function federation(name: string, params: Record<string, unknown>): Promise<unknown> {
|
||||
const response = await dispatcher.dispatch(request(name, params), {
|
||||
authenticatedCallerFingerprint: HOME
|
||||
})
|
||||
if (!response.ok) {
|
||||
throw new Error(`${name} failed: ${JSON.stringify(response)}`)
|
||||
}
|
||||
return response.result
|
||||
}
|
||||
|
||||
/** The worker started and took its task, as a coordinator on another computer starts it. */
|
||||
async function startedChatWorker() {
|
||||
const started = federation('orchestration.federationAttachStart', {
|
||||
runId: 'run_home',
|
||||
dispatchId: DISPATCH,
|
||||
taskId: 'task_home',
|
||||
taskSpec: 'Check the docs and report.',
|
||||
protocolVersion: 4,
|
||||
worktree: 'id:workspace-1',
|
||||
agent: 'codex'
|
||||
})
|
||||
await vi.waitFor(() => expect(codex.connections.at(-1)?.turns).toHaveLength(1), WAIT)
|
||||
const handle = String(db.getRemoteDispatchAttachment(DISPATCH)?.terminal_handle)
|
||||
const identity = resolveStructuredWorkerIdentity(handle, db)!
|
||||
const worker = codex.connections.at(-1)!
|
||||
await settleTurn(identity.sessionId, 0)
|
||||
return { receipt: await started, handle, identity, worker }
|
||||
}
|
||||
|
||||
const homeBound = () =>
|
||||
db.listPendingFederationRelay(DISPATCH, 'to_home').map((item) => JSON.parse(item.payload))
|
||||
|
||||
describe('a chat worker that a coordinator on another computer starts here', () => {
|
||||
it('starts as a chat when this host prefers chats, and takes its task as its first turn', async () => {
|
||||
const { receipt, handle, identity, worker } = await startedChatWorker()
|
||||
|
||||
expect(receipt).toMatchObject({
|
||||
state: 'ready',
|
||||
terminalHandle: handle,
|
||||
mode: { mode: 'structured' }
|
||||
})
|
||||
expect(isStructuredWorkerHandle(handle)).toBe(true)
|
||||
expect(worker.turns[0]!.text).toContain('Check the docs and report.')
|
||||
expect(db.getRemoteDispatchAttachment(DISPATCH)).toMatchObject({
|
||||
state: 'ready',
|
||||
pane_key: identity.paneKey,
|
||||
process_incarnation: identity.processIncarnation
|
||||
})
|
||||
})
|
||||
|
||||
it('sends its mail home on the relay', async () => {
|
||||
const { identity } = await startedChatWorker()
|
||||
|
||||
await expect(
|
||||
call(
|
||||
'orchestration.send',
|
||||
{ subject: 'Halfway', body: 'docs checked' },
|
||||
{ sessionId: identity.sessionId }
|
||||
)
|
||||
).resolves.toMatchObject({ relay: { destination: 'run_home', dispatchId: DISPATCH } })
|
||||
expect(homeBound()).toEqual([expect.objectContaining({ subject: 'Halfway', type: 'status' })])
|
||||
})
|
||||
|
||||
it('tells home when its turn ends without a report', async () => {
|
||||
const { identity } = await startedChatWorker()
|
||||
const timers: (() => void)[] = []
|
||||
const report = chatWorkerTurnStopReporter(
|
||||
{ getDb: () => db, isWaitingForMail: () => false, notify: vi.fn() },
|
||||
(run) => timers.push(run)
|
||||
)
|
||||
|
||||
report(
|
||||
{
|
||||
type: 'completion',
|
||||
completion: {
|
||||
scope: {
|
||||
executionHostId: LOCAL_EXECUTION_HOST_ID,
|
||||
wslDistro: null,
|
||||
workspaceId: 'workspace-1',
|
||||
workspaceKind: 'git-worktree'
|
||||
},
|
||||
sessionId: identity.sessionId,
|
||||
turnId: 'turn-1',
|
||||
outcome: 'success',
|
||||
completedAt: 1
|
||||
}
|
||||
},
|
||||
() => ({ status: 'idle', lastAssistantMessage: 'Docs look fine.' })
|
||||
)
|
||||
for (const run of timers.splice(0)) {
|
||||
run()
|
||||
}
|
||||
|
||||
expect(homeBound()).toEqual([
|
||||
expect.objectContaining({ subject: 'Worker stopped without reporting' })
|
||||
])
|
||||
})
|
||||
|
||||
it("takes its coordinator's mail as a new turn", async () => {
|
||||
const { worker } = await startedChatWorker()
|
||||
|
||||
await federation('orchestration.federationImport', {
|
||||
dispatchId: DISPATCH,
|
||||
items: [
|
||||
{
|
||||
dispatch_id: DISPATCH,
|
||||
direction: 'to_worker',
|
||||
sequence: 1,
|
||||
message_id: 'relay_home_1',
|
||||
kind: 'control_message',
|
||||
payload: encodeFederatedControlMessage({
|
||||
from: 'term_coord',
|
||||
subject: 'Also the changelog',
|
||||
body: 'Update the changelog too.',
|
||||
type: 'status',
|
||||
priority: 'normal',
|
||||
threadId: null,
|
||||
payload: null
|
||||
})
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
await vi.waitFor(() => expect(worker.turns).toHaveLength(2), WAIT)
|
||||
expect(turnText(worker.turns[1]!)).toContain('Also the changelog')
|
||||
})
|
||||
|
||||
it('says plainly that its state cannot be read here yet', async () => {
|
||||
await startedChatWorker()
|
||||
|
||||
await expect(
|
||||
federation('orchestration.federationShow', { dispatchId: DISPATCH })
|
||||
).resolves.toMatchObject({
|
||||
observation: {
|
||||
status: 'unverifiable',
|
||||
reason: expect.stringContaining('cannot read the state of a chat worker on a paired server')
|
||||
}
|
||||
})
|
||||
for (const read of ['orchestration.federationRead', 'orchestration.federationReadOutput']) {
|
||||
await expect(federation(read, { dispatchId: DISPATCH })).rejects.toThrow(
|
||||
/cannot read the state of a chat worker on a paired server/
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
it('closes its chat when its coordinator stops it', async () => {
|
||||
const { identity } = await startedChatWorker()
|
||||
|
||||
await expect(
|
||||
federation('orchestration.federationStop', { dispatchId: DISPATCH })
|
||||
).resolves.toMatchObject({ state: 'stopped', processAction: 'closed_agent_terminal' })
|
||||
expect(host.hasSession(identity.sessionId)).toBe(false)
|
||||
})
|
||||
|
||||
it('closes its chat when its coordinator releases it after its report', async () => {
|
||||
const { identity } = await startedChatWorker()
|
||||
// Its report settled at the Run home, as a confirmed worker_done leaves it.
|
||||
db.recordRemoteAttachmentStage({
|
||||
dispatchId: DISPATCH,
|
||||
state: 'succeeded',
|
||||
stage: 'worker_reported'
|
||||
})
|
||||
|
||||
await expect(
|
||||
federation('orchestration.federationRelease', { dispatchId: DISPATCH })
|
||||
).resolves.toMatchObject({ state: 'released' })
|
||||
expect(host.hasSession(identity.sessionId)).toBe(false)
|
||||
expect(db.getRemoteDispatchAttachment(DISPATCH)).toMatchObject({ stage: 'released' })
|
||||
})
|
||||
|
||||
it('closes its chat when its automatic release comes due', async () => {
|
||||
const { identity } = await startedChatWorker()
|
||||
db.recordRemoteAttachmentStage({
|
||||
dispatchId: DISPATCH,
|
||||
state: 'succeeded',
|
||||
stage: 'worker_reported'
|
||||
})
|
||||
const settledAt = Date.UTC(2026, 0, 1)
|
||||
db.armWorkerTerminalAutoReleaseStatement(DISPATCH, settledAt)
|
||||
expect(db.claimDueWorkerTerminalAutoReleases(settledAt + 3_600_000)).toBe(1)
|
||||
|
||||
await reconcileRequestedWorkerTerminalReleases(runtime)
|
||||
|
||||
expect(host.hasSession(identity.sessionId)).toBe(false)
|
||||
expect(db.getRemoteDispatchAttachment(DISPATCH)).toMatchObject({ stage: 'released' })
|
||||
})
|
||||
|
||||
it('in a new worktree, takes its task only after a wait-for-setup setup finishes', async () => {
|
||||
vi.spyOn(placement, 'assertOrchestrationWorktreeCreationSupported').mockResolvedValue()
|
||||
vi.spyOn(worktreeName, 'resolveWorkerWorktreeName').mockResolvedValue({ name: 'w2' })
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the launch reads only these receipt fields.
|
||||
vi.spyOn(runtime, 'createManagedWorktree').mockResolvedValue({
|
||||
worktree: { id: 'workspace-1', displayName: 'w2' },
|
||||
setupReceipt: {
|
||||
hookFound: true,
|
||||
startupPolicy: 'wait-for-setup',
|
||||
state: 'running',
|
||||
terminalHandle: 'term_setup'
|
||||
}
|
||||
} as never)
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: effects read only handle and title.
|
||||
vi.spyOn(runtime, 'listTerminals').mockResolvedValue({
|
||||
terminals: [{ handle: 'term_setup', title: 'Setup' }]
|
||||
} as never)
|
||||
const setupDone = Promise.withResolvers<{ exitCode: number }>()
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the gate reads only exitCode.
|
||||
vi.spyOn(runtime, 'waitForSetupTerminalCompletion').mockReturnValue(setupDone.promise as never)
|
||||
|
||||
const started = federation('orchestration.federationAttachStart', {
|
||||
runId: 'run_home',
|
||||
dispatchId: DISPATCH,
|
||||
taskId: 'task_home',
|
||||
taskSpec: 'Check the docs and report.',
|
||||
protocolVersion: 4,
|
||||
worktree: 'new-top-level',
|
||||
repo: 'id:repo',
|
||||
agent: 'codex'
|
||||
})
|
||||
await vi.waitFor(
|
||||
() => expect(runtime.waitForSetupTerminalCompletion).toHaveBeenCalledWith('term_setup'),
|
||||
WAIT
|
||||
)
|
||||
expect(codex.connections.flatMap((connection) => connection.turns)).toHaveLength(0)
|
||||
|
||||
setupDone.resolve({ exitCode: 0 })
|
||||
await vi.waitFor(() => expect(codex.connections.at(-1)?.turns).toHaveLength(1), WAIT)
|
||||
const handle = String(db.getRemoteDispatchAttachment(DISPATCH)?.terminal_handle)
|
||||
await settleTurn(resolveStructuredWorkerIdentity(handle, db)!.sessionId, 0)
|
||||
await expect(started).resolves.toMatchObject({ state: 'ready', mode: { mode: 'structured' } })
|
||||
})
|
||||
})
|
||||
@@ -4,9 +4,46 @@ import { AGENT_DONE_QUIET_MS } from '../../../../../shared/agent-status/lifecycl
|
||||
import { chatWorkerDispatches } from './chat-worker-error-escalation'
|
||||
import {
|
||||
reportWorkerTurnStop,
|
||||
workerTurnStopDispatchId,
|
||||
type WorkerTurnStopDispatch,
|
||||
type WorkerTurnStopReason,
|
||||
type WorkerTurnStopReportDeps
|
||||
} from './worker-turn-stop-report'
|
||||
import type { OrchestrationDb } from '../../db/orchestration-db'
|
||||
import { resolveStructuredWorkerIdentityForSession } from '../../terminal-authority/structured-sessions/structured-worker-authority'
|
||||
|
||||
/** The open Dispatches a chat works: its own here, or one a coordinator on another computer started here. */
|
||||
function chatWorkerTargets(
|
||||
sessionId: string,
|
||||
db: OrchestrationDb
|
||||
): { target: WorkerTurnStopDispatch; from: string; starting: boolean }[] {
|
||||
const local = chatWorkerDispatches(sessionId, db).map(({ dispatch, from }) => ({
|
||||
target: { kind: 'local' as const, dispatch },
|
||||
from,
|
||||
starting: db.getWorkerDispatch(dispatch.id)?.state === 'starting'
|
||||
}))
|
||||
if (local.length > 0) {
|
||||
return local
|
||||
}
|
||||
// A paired server holds its remote workers as attachments; their Dispatch lives at the Run home.
|
||||
const worker = resolveStructuredWorkerIdentityForSession(sessionId, db)
|
||||
const attachment = worker && db.findActiveRemoteAttachmentForPane(worker.paneKey)
|
||||
return worker &&
|
||||
attachment &&
|
||||
db.isRemoteAttachmentProcessCurrent({
|
||||
dispatchId: attachment.dispatch_id,
|
||||
paneKey: worker.paneKey,
|
||||
processIncarnation: worker.processIncarnation
|
||||
})
|
||||
? [
|
||||
{
|
||||
target: { kind: 'remote', attachment },
|
||||
from: worker.handle,
|
||||
starting: attachment.state === 'starting'
|
||||
}
|
||||
]
|
||||
: []
|
||||
}
|
||||
|
||||
type ChatStop = {
|
||||
sessionId: string
|
||||
@@ -67,7 +104,7 @@ export function chatWorkerTurnStopReporter(
|
||||
}
|
||||
// The Dispatch a stop belongs to is one open when it was seen.
|
||||
const seen = new Set(
|
||||
chatWorkerDispatches(stop.sessionId, db).map(({ dispatch }) => dispatch.id)
|
||||
chatWorkerTargets(stop.sessionId, db).map(({ target }) => workerTurnStopDispatchId(target))
|
||||
)
|
||||
const deliver = (): void => {
|
||||
try {
|
||||
@@ -81,22 +118,17 @@ export function chatWorkerTurnStopReporter(
|
||||
) {
|
||||
return
|
||||
}
|
||||
const open = chatWorkerDispatches(stop.sessionId, current).filter(({ dispatch }) =>
|
||||
seen.has(dispatch.id)
|
||||
const open = chatWorkerTargets(stop.sessionId, current).filter(({ target }) =>
|
||||
seen.has(workerTurnStopDispatchId(target))
|
||||
)
|
||||
if (
|
||||
stop.reason === 'finished' &&
|
||||
open.some(
|
||||
({ dispatch }) => current.getWorkerDispatch(dispatch.id)?.state === 'starting'
|
||||
)
|
||||
) {
|
||||
if (stop.reason === 'finished' && open.some(({ starting }) => starting)) {
|
||||
setTimer(deliver, AGENT_DONE_QUIET_MS)
|
||||
return
|
||||
}
|
||||
for (const { dispatch, from } of open) {
|
||||
for (const { target, from } of open) {
|
||||
reportWorkerTurnStop(
|
||||
current,
|
||||
{ kind: 'local', dispatch },
|
||||
target,
|
||||
{
|
||||
from,
|
||||
stopKey: stop.stopKey,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { OrchestrationWorkerLaunchReceipt } from '../worker/worker-launch-preferences'
|
||||
import type { AgentLaunchModeReceipt } from '../../../../../../shared/agent-launch/contracts/agent-launch-intent'
|
||||
|
||||
export type RemoteFederatedWorkerStartReceipt = {
|
||||
dispatchId: string
|
||||
@@ -8,6 +9,8 @@ export type RemoteFederatedWorkerStartReceipt = {
|
||||
terminalHandle?: string
|
||||
setup?: { state: string }
|
||||
launch?: OrchestrationWorkerLaunchReceipt
|
||||
/** Chat or terminal, as the server decided; absent from servers that start terminals only. */
|
||||
mode?: AgentLaunchModeReceipt
|
||||
effects?: unknown[]
|
||||
residualResources?: unknown[]
|
||||
failedStage?: string
|
||||
@@ -44,6 +47,16 @@ export function isReadyRemoteFederatedWorkerStartReceipt(
|
||||
return receipt.state === 'ready'
|
||||
}
|
||||
|
||||
/** The ready receipt's parts the server reports as is. */
|
||||
export function readyRemoteReceiptFields(remote: ReadyRemoteFederatedWorkerStartReceipt) {
|
||||
return {
|
||||
setup: remote.setup,
|
||||
...(remote.mode ? { mode: remote.mode } : {}),
|
||||
effects: remote.effects ?? [],
|
||||
residualResources: remote.residualResources ?? []
|
||||
}
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
}
|
||||
|
||||
+167
-115
@@ -1,5 +1,6 @@
|
||||
import type { RuntimeTerminalCreate } from '../../../../../../shared/runtime-terminal-create-contract'
|
||||
/** Execute terminalOnly on the receiving host with federation's agent-first worktree/background-terminal factories. The dispatch brief is delivered later. */
|
||||
/** Launch on the receiving host in the mode it decided, with federation's worktree, background-terminal and
|
||||
* chat-session factories. The dispatch brief is delivered later. */
|
||||
|
||||
import type { AgentLaunchPreferences } from '../../../../../../shared/agent-session/agent-session-host-authority'
|
||||
import type { TuiAgent } from '../../../../../../shared/agent-launch/catalog/tui-agent'
|
||||
@@ -16,8 +17,11 @@ import {
|
||||
} from './federation-effects'
|
||||
import type { FederationAttachStartInput } from '../../../../../../shared/rpc-contract/orchestration-federation-start-params'
|
||||
import type { Worktree } from '../../../../../../shared/worktree/types'
|
||||
import type { AgentLaunchModeReceipt } from '../../../../../../shared/agent-launch/contracts/agent-launch-intent'
|
||||
import { createStructuredWorkerSessionForWorktree } from '../worker/worker-topology'
|
||||
import { tearDownFailedWorkerStart } from '../worker/failed-worker-start-teardown'
|
||||
|
||||
const TERMINAL_ONLY_MESSAGE = 'A federated worker starts a terminal agent only.'
|
||||
type StructuredSession = Awaited<ReturnType<typeof createStructuredWorkerSessionForWorktree>>
|
||||
|
||||
export async function launchFederatedWorkerAgent(args: {
|
||||
runtime: OrcaRuntimeService
|
||||
@@ -25,6 +29,8 @@ export async function launchFederatedWorkerAgent(args: {
|
||||
params: FederationAttachStartInput
|
||||
agent: TuiAgent | undefined
|
||||
launchPreferences: AgentLaunchPreferences | undefined
|
||||
/** Decided here, on the host that runs the worker, from this host's own settings. */
|
||||
mode: AgentLaunchModeReceipt
|
||||
/** The workspace to start in; absent creates the new top-level worktree the params describe. */
|
||||
worktree?: { id: string }
|
||||
/** The paired client that asked, so its own sidebar counts the new worktree as its own. */
|
||||
@@ -38,6 +44,10 @@ export async function launchFederatedWorkerAgent(args: {
|
||||
worktree: { id: string }
|
||||
terminalHandle: string
|
||||
terminalCreation?: RuntimeTerminalCreate
|
||||
/** Set when the worker is a chat session rather than a terminal agent. */
|
||||
structuredSession: StructuredSession | null
|
||||
/** The mode that actually ran; a created worktree can settle it later than the caller could. */
|
||||
mode: AgentLaunchModeReceipt
|
||||
}> {
|
||||
const { runtime, db, params, effects } = args
|
||||
if (!args.agent) {
|
||||
@@ -49,123 +59,165 @@ export async function launchFederatedWorkerAgent(args: {
|
||||
const agent = args.agent
|
||||
let created: { id: string } | undefined
|
||||
let terminalCreation: RuntimeTerminalCreate | undefined
|
||||
const launched = await executeAgentLaunch({
|
||||
runtime,
|
||||
intent: {
|
||||
agent,
|
||||
// Federation's own factory builds the create from the attachment, so the target carries none.
|
||||
target: args.worktree
|
||||
? { kind: 'existing', worktree: args.worktree.id }
|
||||
: { kind: 'create-worktree', create: {} },
|
||||
launchSource: 'orchestration'
|
||||
},
|
||||
terminalOnly: true,
|
||||
vocabulary: WORKER_START_VOCABULARY,
|
||||
onStage: (stage) =>
|
||||
args.onStage(stage === 'worktree_create' ? 'worktree_create' : 'terminal_create'),
|
||||
workspaces: {
|
||||
createWorktree: async ({ startupAgent }) => {
|
||||
// Agent-first is the only create here; a structured pre-flight must not reach it.
|
||||
if (startupAgent === undefined) {
|
||||
throw new Error(TERMINAL_ONLY_MESSAGE)
|
||||
}
|
||||
const { repo } = params
|
||||
if (!repo) {
|
||||
// Validation refuses this before the attachment exists; this only narrows the types.
|
||||
throw new Error('A remote new-top-level worktree requires an explicit --repo.')
|
||||
}
|
||||
db.recordRemoteAttachmentStage({
|
||||
dispatchId: params.dispatchId,
|
||||
stage: 'worktree_creating'
|
||||
})
|
||||
const setupDecision = params.setup ?? 'run'
|
||||
const result = await runtime.createManagedWorktree({
|
||||
onStartupTerminalCreated: (terminal) => {
|
||||
terminalCreation = terminal
|
||||
},
|
||||
repoSelector: repo,
|
||||
...(await resolveWorkerWorktreeName(runtime, repo, params.name)),
|
||||
baseBranch: params.baseBranch,
|
||||
displayName: params.displayName,
|
||||
displayNameKind: params.displayNameKind,
|
||||
comment: params.comment,
|
||||
// setupDecision runs setup without the legacy runHooks activation side effect.
|
||||
runHooks: false,
|
||||
setupDecision,
|
||||
awaitTerminalProvisioning: true,
|
||||
observeSetupCompletion: true,
|
||||
createdWithAgent: agent,
|
||||
startupAgent: agent,
|
||||
startupLaunchSource: 'orchestration',
|
||||
startupSuppressUpdatePrompt: true,
|
||||
...(args.launchPreferences ? { startupLaunchPreferences: args.launchPreferences } : {}),
|
||||
activate: false,
|
||||
...(args.creatorProvenance ? { creatorProvenance: args.creatorProvenance } : {}),
|
||||
lineage: params.crossStoreParent
|
||||
? {
|
||||
crossStoreParent: params.crossStoreParent,
|
||||
...(params.crossStoreParentInferredFrom
|
||||
? { crossStoreParentInferredFrom: params.crossStoreParentInferredFrom }
|
||||
: {})
|
||||
}
|
||||
: { noParent: true }
|
||||
})
|
||||
created = result.worktree
|
||||
const terminalHandle = result.startupTerminal?.handle
|
||||
effects.push({
|
||||
kind: 'worktree',
|
||||
action: params.crossStoreParent ? 'created_child' : 'created_top_level',
|
||||
id: result.worktree.id,
|
||||
name: result.worktree.displayName
|
||||
})
|
||||
const setup: WorkerSetupReceipt = {
|
||||
requested: setupDecision,
|
||||
effective: setupDecision,
|
||||
source: args.setupSource,
|
||||
hookFound: result.setupReceipt?.hookFound ?? false,
|
||||
startupPolicy: result.setupReceipt?.startupPolicy ?? 'start-immediately',
|
||||
state: result.setupReceipt?.state ?? 'not_configured'
|
||||
}
|
||||
args.onSetup(setup)
|
||||
if (!terminalHandle) {
|
||||
throw new Error(result.warning ?? 'Agent-first worktree creation returned no terminal.')
|
||||
}
|
||||
const listed = await runtime.listTerminals(`id:${result.worktree.id}`, undefined, {
|
||||
includeVisualLayouts: false
|
||||
})
|
||||
appendFederationTerminalEffects(
|
||||
effects,
|
||||
listed.terminals,
|
||||
terminalHandle,
|
||||
result.setupReceipt?.terminalHandle
|
||||
)
|
||||
appendFederationSetupEffect(effects, setup)
|
||||
return { worktreeId: result.worktree.id, startupTerminalHandle: terminalHandle }
|
||||
}
|
||||
},
|
||||
surfaces: {
|
||||
createStructuredSession: () => {
|
||||
throw new Error(TERMINAL_ONLY_MESSAGE)
|
||||
let structuredSession: StructuredSession | null = null
|
||||
let launched: Awaited<ReturnType<typeof executeAgentLaunch>>
|
||||
try {
|
||||
launched = await executeAgentLaunch({
|
||||
runtime,
|
||||
intent: {
|
||||
agent,
|
||||
// Federation's own factory builds the create from the attachment, so the target carries none.
|
||||
target: args.worktree
|
||||
? { kind: 'existing', worktree: args.worktree.id }
|
||||
: { kind: 'create-worktree', create: {} },
|
||||
launchSource: 'orchestration'
|
||||
},
|
||||
createTerminalAgent: async ({ worktreeId }) => {
|
||||
const terminal = await runtime.createTerminal(`id:${worktreeId}`, {
|
||||
// Resolve agent ids through TUI config: cursor is the desktop app, its CLI is cursor-agent.
|
||||
startupAgent: agent,
|
||||
launchSource: 'orchestration',
|
||||
suppressStartupUpdatePrompt: true,
|
||||
...(args.launchPreferences ? { launchPreferences: args.launchPreferences } : {}),
|
||||
title: `worker-${params.taskId}`,
|
||||
presentation: 'background'
|
||||
})
|
||||
effects.push({ kind: 'terminal', role: 'agent', action: 'created', id: terminal.handle })
|
||||
terminalCreation = terminal
|
||||
return { handle: terminal.handle, terminalCreation: terminal }
|
||||
decidedMode: args.mode,
|
||||
vocabulary: WORKER_START_VOCABULARY,
|
||||
onStage: (stage) =>
|
||||
args.onStage(stage === 'worktree_create' ? 'worktree_create' : 'terminal_create'),
|
||||
workspaces: {
|
||||
createWorktree: async ({ startupAgent, promisedAgentChat, onWorkspaceCreated }) => {
|
||||
const { repo } = params
|
||||
if (!repo) {
|
||||
// Validation refuses this before the attachment exists; this only narrows the types.
|
||||
throw new Error('A remote new-top-level worktree requires an explicit --repo.')
|
||||
}
|
||||
db.recordRemoteAttachmentStage({
|
||||
dispatchId: params.dispatchId,
|
||||
stage: 'worktree_creating'
|
||||
})
|
||||
const setupDecision = params.setup ?? 'run'
|
||||
const result = await runtime.createManagedWorktree({
|
||||
onStartupTerminalCreated: (terminal) => {
|
||||
terminalCreation = terminal
|
||||
},
|
||||
repoSelector: repo,
|
||||
...(await resolveWorkerWorktreeName(runtime, repo, params.name)),
|
||||
baseBranch: params.baseBranch,
|
||||
displayName: params.displayName,
|
||||
displayNameKind: params.displayNameKind,
|
||||
comment: params.comment,
|
||||
// setupDecision runs setup without the legacy runHooks activation side effect.
|
||||
runHooks: false,
|
||||
setupDecision,
|
||||
awaitTerminalProvisioning: true,
|
||||
observeSetupCompletion: true,
|
||||
createdWithAgent: agent,
|
||||
...(promisedAgentChat ? { promisedAgentChat: true } : {}),
|
||||
...(onWorkspaceCreated ? { onWorkspaceCreated } : {}),
|
||||
// The executor withholds the startup agent exactly when the worker is to be a chat session.
|
||||
...(startupAgent !== undefined
|
||||
? {
|
||||
startupAgent: agent,
|
||||
startupLaunchSource: 'orchestration' as const,
|
||||
startupSuppressUpdatePrompt: true,
|
||||
...(args.launchPreferences
|
||||
? { startupLaunchPreferences: args.launchPreferences }
|
||||
: {})
|
||||
}
|
||||
: {}),
|
||||
activate: false,
|
||||
...(args.creatorProvenance ? { creatorProvenance: args.creatorProvenance } : {}),
|
||||
lineage: params.crossStoreParent
|
||||
? {
|
||||
crossStoreParent: params.crossStoreParent,
|
||||
...(params.crossStoreParentInferredFrom
|
||||
? { crossStoreParentInferredFrom: params.crossStoreParentInferredFrom }
|
||||
: {})
|
||||
}
|
||||
: { noParent: true }
|
||||
})
|
||||
created = result.worktree
|
||||
const terminalHandle = result.startupTerminal?.handle
|
||||
effects.push({
|
||||
kind: 'worktree',
|
||||
action: params.crossStoreParent ? 'created_child' : 'created_top_level',
|
||||
id: result.worktree.id,
|
||||
name: result.worktree.displayName
|
||||
})
|
||||
const setup: WorkerSetupReceipt = {
|
||||
requested: setupDecision,
|
||||
effective: setupDecision,
|
||||
source: args.setupSource,
|
||||
hookFound: result.setupReceipt?.hookFound ?? false,
|
||||
startupPolicy: result.setupReceipt?.startupPolicy ?? 'start-immediately',
|
||||
state: result.setupReceipt?.state ?? 'not_configured'
|
||||
}
|
||||
args.onSetup(setup)
|
||||
// A chat has no startup terminal; its setup terminal is still recorded so the setup gate finds it.
|
||||
const chat = startupAgent === undefined
|
||||
if (!chat && !terminalHandle) {
|
||||
throw new Error(result.warning ?? 'Agent-first worktree creation returned no terminal.')
|
||||
}
|
||||
const listed = await runtime.listTerminals(`id:${result.worktree.id}`, undefined, {
|
||||
includeVisualLayouts: false
|
||||
})
|
||||
appendFederationTerminalEffects(
|
||||
effects,
|
||||
listed.terminals,
|
||||
chat ? undefined : terminalHandle,
|
||||
result.setupReceipt?.terminalHandle
|
||||
)
|
||||
appendFederationSetupEffect(effects, setup)
|
||||
return {
|
||||
worktreeId: result.worktree.id,
|
||||
startupTerminalHandle: chat ? undefined : terminalHandle
|
||||
}
|
||||
}
|
||||
},
|
||||
surfaces: {
|
||||
createStructuredSession: async ({ worktreeId }) => {
|
||||
args.onStage('terminal_create')
|
||||
structuredSession = await createStructuredWorkerSessionForWorktree({
|
||||
runtime,
|
||||
worktreeId,
|
||||
agent,
|
||||
dispatchId: params.dispatchId,
|
||||
...(args.launchPreferences ? { launchPreferences: args.launchPreferences } : {}),
|
||||
effects
|
||||
})
|
||||
return {
|
||||
sessionId: structuredSession.identity.sessionId,
|
||||
handle: structuredSession.identity.handle,
|
||||
fence: structuredSession.fence
|
||||
}
|
||||
},
|
||||
createTerminalAgent: async ({ worktreeId }) => {
|
||||
const terminal = await runtime.createTerminal(`id:${worktreeId}`, {
|
||||
// Resolve agent ids through TUI config: cursor is the desktop app, its CLI is cursor-agent.
|
||||
startupAgent: agent,
|
||||
launchSource: 'orchestration',
|
||||
suppressStartupUpdatePrompt: true,
|
||||
...(args.launchPreferences ? { launchPreferences: args.launchPreferences } : {}),
|
||||
title: `worker-${params.taskId}`,
|
||||
presentation: 'background'
|
||||
})
|
||||
effects.push({ kind: 'terminal', role: 'agent', action: 'created', id: terminal.handle })
|
||||
terminalCreation = terminal
|
||||
return { handle: terminal.handle, terminalCreation: terminal }
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
} catch (error) {
|
||||
// The caller only sees a returned launch, so a session made before this throw is ours.
|
||||
await tearDownFailedWorkerStart({
|
||||
runtime,
|
||||
structuredSession,
|
||||
dispatchId: params.dispatchId,
|
||||
error
|
||||
})
|
||||
throw error
|
||||
}
|
||||
const worktree = args.worktree ?? created
|
||||
if (!worktree) {
|
||||
throw new Error('Federated worker topology did not resolve.')
|
||||
}
|
||||
return { worktree, terminalHandle: launched.outcome.handle, terminalCreation }
|
||||
return {
|
||||
worktree,
|
||||
terminalHandle: launched.outcome.handle,
|
||||
terminalCreation,
|
||||
structuredSession,
|
||||
mode: launched.receipt
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
import { waitForWorkerAgentReady } from '../../../../terminal/wait/launched-agent-composer-readiness'
|
||||
import { sendAgentTurn } from '../../../../orchestration/turn-delivery/send-agent-turn'
|
||||
import { createWorkerBriefWriteGuard } from '../../../../terminal/wait/launched-agent-write-guard'
|
||||
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
|
||||
import type { OrcaRuntimeService } from '../../../../orca-runtime'
|
||||
import type { TuiAgent } from '../../../../../../shared/agent-launch/catalog/tui-agent'
|
||||
import type { RuntimeTerminalCreate } from '../../../../../../shared/runtime-terminal-create-contract'
|
||||
import {
|
||||
formatOrcaSessionAddress,
|
||||
isOrcaSessionId
|
||||
} from '../../../../../../shared/orca-session-address'
|
||||
import { canonicalOrcaSessionId } from '../../../../orchestration/addressing/session-lineage/canonical-orca-session-id'
|
||||
import { sendStructuredWorkerPreamble } from '../structured-workers/orchestration-structured-worker-session'
|
||||
import { awaitStructuredWorkerSetupGate } from '../worker/worker-start-structured-setup-gate'
|
||||
import type {
|
||||
createStructuredWorkerSessionForWorktree,
|
||||
WorkerSetupReceipt
|
||||
} from '../worker/worker-topology'
|
||||
import type { FederationEffect } from './federation-effects'
|
||||
|
||||
export type StructuredSession = Awaited<ReturnType<typeof createStructuredWorkerSessionForWorktree>>
|
||||
|
||||
/** A chat's readiness is its brief being taken, so only a wait-for-setup policy gates it here. */
|
||||
export function awaitFederatedWorkerReady(input: {
|
||||
runtime: OrcaRuntimeService
|
||||
structuredSession: StructuredSession | null
|
||||
terminalHandle: string
|
||||
agent: TuiAgent | undefined
|
||||
reusesTerminal: boolean
|
||||
creation: RuntimeTerminalCreate | undefined
|
||||
setup: WorkerSetupReceipt
|
||||
effects: FederationEffect[]
|
||||
timeoutMs: number
|
||||
}) {
|
||||
const { runtime, timeoutMs } = input
|
||||
return input.structuredSession
|
||||
? awaitStructuredWorkerSetupGate({
|
||||
runtime,
|
||||
setup: input.setup,
|
||||
effects: input.effects,
|
||||
timeoutMs
|
||||
})
|
||||
: waitForWorkerAgentReady(runtime, input.terminalHandle, {
|
||||
agent: input.agent,
|
||||
reusesTerminal: input.reusesTerminal,
|
||||
creation: input.creation,
|
||||
timeoutMs
|
||||
})
|
||||
}
|
||||
|
||||
/** A chat worker's commands name its Orca session ID, which binds to it, as a local one's do. */
|
||||
export function federatedWorkerAddress(
|
||||
structuredSession: StructuredSession | null,
|
||||
terminalHandle: string
|
||||
): string {
|
||||
const sessionId = structuredSession?.identity.sessionId
|
||||
return sessionId && isOrcaSessionId(sessionId)
|
||||
? formatOrcaSessionAddress(canonicalOrcaSessionId(sessionId))
|
||||
: terminalHandle
|
||||
}
|
||||
|
||||
export function sendFederatedWorkerBrief(input: {
|
||||
runtime: OrcaRuntimeService
|
||||
structuredSession: StructuredSession | null
|
||||
terminalHandle: string
|
||||
agent: TuiAgent | undefined
|
||||
params: { dispatchId: string; runId?: string; taskId: string; terminal?: string }
|
||||
operationId: string
|
||||
preamble: string
|
||||
budgetMs: number
|
||||
}) {
|
||||
return input.structuredSession
|
||||
? sendRemoteChatWorkerBrief(
|
||||
input.structuredSession,
|
||||
input.params,
|
||||
input.preamble,
|
||||
input.budgetMs
|
||||
)
|
||||
: sendTerminalWorkerBrief(input.runtime, input.terminalHandle, {
|
||||
agent: input.agent,
|
||||
launched: !input.params.terminal,
|
||||
operationId: input.operationId,
|
||||
preamble: input.preamble
|
||||
})
|
||||
}
|
||||
|
||||
async function sendTerminalWorkerBrief(
|
||||
runtime: OrcaRuntimeService,
|
||||
handle: string,
|
||||
brief: { agent: TuiAgent | undefined; launched: boolean; operationId: string; preamble: string }
|
||||
) {
|
||||
// A shell back at its prompt also reads as ready, so the brief needs the agent found in front.
|
||||
const briefGuard = createWorkerBriefWriteGuard(runtime, brief.agent, brief.launched)
|
||||
return sendAgentTurn({
|
||||
kind: 'terminal',
|
||||
runtime,
|
||||
handle,
|
||||
...(briefGuard ? { beforeWrite: briefGuard.beforeWrite } : {}),
|
||||
turn: { purpose: 'dispatch-preamble', operationId: brief.operationId, body: brief.preamble }
|
||||
}).finally(() => briefGuard?.dispose())
|
||||
}
|
||||
|
||||
/** The brief as the chat's first turn, as a local chat worker gets it; one not yet taken leaves the start unknown. */
|
||||
async function sendRemoteChatWorkerBrief(
|
||||
session: StructuredSession,
|
||||
params: { dispatchId: string; runId?: string; taskId: string },
|
||||
preamble: string,
|
||||
budgetMs: number
|
||||
): Promise<undefined> {
|
||||
const delivery = await sendStructuredWorkerPreamble({
|
||||
host: session.host,
|
||||
sessionId: session.identity.sessionId,
|
||||
dispatchId: params.dispatchId,
|
||||
preamble,
|
||||
from: {
|
||||
kind: 'agent',
|
||||
senders: [],
|
||||
orchestration: {
|
||||
message: 'task',
|
||||
runId: params.runId ?? '',
|
||||
taskId: params.taskId,
|
||||
dispatchId: params.dispatchId
|
||||
}
|
||||
},
|
||||
budgetMs
|
||||
})
|
||||
if (delivery !== 'accepted') {
|
||||
throw new OrchestrationError(
|
||||
'operation_unknown',
|
||||
'The task was accepted, but the chat agent had not started to take it. It is delivered when the agent starts; if the worker then reports, this Dispatch settles normally.'
|
||||
)
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
+9
-16
@@ -2,8 +2,7 @@ import { describeUnconfirmedAgentStop } from '../../../../../../shared/terminal/
|
||||
import type { RemoteDispatchAttachmentRow } from '../../../../orchestration/db/orchestration-records'
|
||||
import type {
|
||||
WorkerTerminalArchiveRow,
|
||||
WorkerTerminalResourceRow,
|
||||
WorkerTerminalRetainedReason
|
||||
WorkerTerminalResourceRow
|
||||
} from '../../../../orchestration/terminal-authority/release/worker-terminal-ownership'
|
||||
import {
|
||||
captureWorkerOutputArchive,
|
||||
@@ -14,10 +13,14 @@ import {
|
||||
import type { WorkerTerminalReleaseRuntime } from '../../../../orchestration/terminal-authority/release/worker-terminal-release-runtime'
|
||||
import { readRemoteAttachmentArchive } from './remote-attachment-archive-read'
|
||||
import { WorkerTerminalReleaseSingleFlight } from '../worker/worker-terminal-release-single-flight'
|
||||
import { remoteChatWorkerOf } from '../../../../orchestration/federation/remote-chat-worker'
|
||||
import { releaseRemoteChatWorker } from './remote-chat-worker-host'
|
||||
import { yieldAutoReleaseToWorkingAgent } from '../../../../orchestration/terminal-authority/release/worker-terminal-auto-release-working-yield'
|
||||
import { archiveSummary } from '../worker/worker-terminal-resource-presentation'
|
||||
import {
|
||||
isStillReleasing,
|
||||
releaseUnknownRecovery,
|
||||
retainedReason,
|
||||
type WorkerReleaseReceipt
|
||||
} from '../worker/worker-release-completion'
|
||||
import { orchestrationTimestampToMs } from '../worker/worker-output'
|
||||
@@ -113,6 +116,10 @@ async function completeRemoteAttachmentRelease(
|
||||
): Promise<RemoteReleaseReceipt> {
|
||||
const { runtime, attachment, observation } = args
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const chat = remoteChatWorkerOf(db, attachment)
|
||||
if (chat?.exact) {
|
||||
return releaseRemoteChatWorker(runtime, attachment, chat.identity, resource, storedArchive)
|
||||
}
|
||||
if (!observation.exact || !observation.terminal) {
|
||||
if (
|
||||
args.mode === 'recovery' &&
|
||||
@@ -280,16 +287,6 @@ function remoteAttachmentLeaseIsCurrent(
|
||||
)
|
||||
}
|
||||
|
||||
function retainedReason(resource: WorkerTerminalResourceRow): WorkerTerminalRetainedReason {
|
||||
if (resource.retained_reason) {
|
||||
return resource.retained_reason as WorkerTerminalRetainedReason
|
||||
}
|
||||
if (resource.ownership_state === 'user_owned') {
|
||||
return 'user_takeover'
|
||||
}
|
||||
return 'identity_unproven'
|
||||
}
|
||||
|
||||
function projectArchivedOutputLiveness<
|
||||
T extends { status: { terminal: string; liveness: string } }
|
||||
>(output: T, liveness: 'live' | 'unverifiable' | 'exited'): T {
|
||||
@@ -302,7 +299,3 @@ function projectArchivedOutputLiveness<
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isStillReleasing(resource: WorkerTerminalResourceRow): boolean {
|
||||
return resource.ownership_state === 'owned' && resource.release_state === 'releasing'
|
||||
}
|
||||
|
||||
@@ -20,7 +20,8 @@ import { resolveFederatedWorkerStartBudgets } from '../worker/worker-start-budge
|
||||
import { resolveDispatchCreator } from '../runs/dispatch-creator'
|
||||
import {
|
||||
isReadyRemoteFederatedWorkerStartReceipt,
|
||||
parseRemoteFederatedWorkerStartReceipt
|
||||
parseRemoteFederatedWorkerStartReceipt,
|
||||
readyRemoteReceiptFields
|
||||
} from './federated-attach-receipt'
|
||||
import { isWorkerStartTimeoutWithinTimerLimit } from '../../../../../../shared/orchestration/orchestration-timing-budgets'
|
||||
import {
|
||||
@@ -232,11 +233,9 @@ export async function startFederatedWorker(args: FederatedWorkerStartRequest): P
|
||||
state: 'ready',
|
||||
stage: readyWorker.stage,
|
||||
server: { environmentId: server.environmentId, name: server.name },
|
||||
setup: remote.setup,
|
||||
...readyRemoteReceiptFields(remote),
|
||||
launch,
|
||||
timeoutMs: budgets.readinessTimeoutMs,
|
||||
effects: remote.effects ?? [],
|
||||
residualResources: remote.residualResources ?? [],
|
||||
...(childParent && 'warning' in childParent ? { warning: childParent.warning } : {})
|
||||
},
|
||||
reportsTurnStops
|
||||
|
||||
@@ -3,6 +3,8 @@ import type { WorkerStopReceipt } from '../../../../../../shared/orchestration/w
|
||||
import { archiveWorkerOutputBeforeStop } from '../worker/worker-stop-archive'
|
||||
import { describeUnconfirmedAgentStop } from '../../../../../../shared/terminal/pty/lifecycle/pty-liveness-verdict'
|
||||
import { inspectRemoteAttachment } from './federation-attachment-observation'
|
||||
import { remoteChatWorkerOf } from '../../../../orchestration/federation/remote-chat-worker'
|
||||
import { stopRemoteChatWorker } from './remote-chat-worker-host'
|
||||
|
||||
/** Close the exact attached worker this host runs for its home. */
|
||||
export async function stopRemoteAttachment(
|
||||
@@ -19,6 +21,10 @@ export async function stopRemoteAttachment(
|
||||
processAction: 'none'
|
||||
}
|
||||
}
|
||||
const chat = remoteChatWorkerOf(db, begun)
|
||||
if (chat) {
|
||||
return stopRemoteChatWorker(runtime, params.dispatchId, chat)
|
||||
}
|
||||
const observation = await inspectRemoteAttachment(runtime, params.dispatchId)
|
||||
if (!observation.exact || !observation.terminal) {
|
||||
const attachment = db.markRemoteAttachmentStopUnknown(
|
||||
|
||||
+4
-1
@@ -2,6 +2,7 @@ import type { OrcaRuntimeService } from '../../../../orca-runtime'
|
||||
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
|
||||
import type { FederationEffect } from './federation-effects'
|
||||
import type { WorkerSetupReceipt } from '../worker/worker-topology'
|
||||
import type { StructuredWorkerIdentity } from '../../../../orchestration/terminal-authority/structured-sessions/structured-worker-identity'
|
||||
|
||||
export function prepareFederatedAttachmentAuthority(args: {
|
||||
runtime: OrcaRuntimeService
|
||||
@@ -12,9 +13,11 @@ export function prepareFederatedAttachmentAuthority(args: {
|
||||
setup: WorkerSetupReceipt
|
||||
effects: FederationEffect[]
|
||||
reusesTerminal: boolean
|
||||
/** A chat worker's minted identity, which stands in for a terminal's pane and process. */
|
||||
identity?: Pick<StructuredWorkerIdentity, 'paneKey' | 'processIncarnation' | 'hostScope'>
|
||||
}): void {
|
||||
const { runtime, terminalHandle } = args
|
||||
const authority = runtime.getOrchestrationDispatchAuthority(terminalHandle)
|
||||
const authority = args.identity ?? runtime.getOrchestrationDispatchAuthority(terminalHandle)
|
||||
const paneKey = authority?.paneKey ?? runtime.getTerminalPaneKey(terminalHandle)
|
||||
const processIncarnation =
|
||||
authority?.processIncarnation ?? runtime.getTerminalProcessIncarnation(terminalHandle)
|
||||
|
||||
+24
@@ -4,6 +4,10 @@ import { OrchestrationError } from '../../../../orchestration/orchestration-erro
|
||||
import { parseWorkerTerminalHostScope } from '../../../../../../shared/orchestration/workers/worker-terminal-host-scope'
|
||||
import type { RemoteDispatchAttachmentRow } from '../../../../orchestration/db/orchestration-records'
|
||||
import { orchestrationTimestampToMs } from '../worker/worker-output'
|
||||
import {
|
||||
REMOTE_CHAT_WORKER_READ_UNSUPPORTED,
|
||||
remoteChatWorkerOf
|
||||
} from '../../../../orchestration/federation/remote-chat-worker'
|
||||
|
||||
export function requireHomeAttachment(
|
||||
runtime: WorkerTerminalReleaseRuntime,
|
||||
@@ -20,6 +24,16 @@ export function requireHomeAttachment(
|
||||
return attachment
|
||||
}
|
||||
|
||||
/** A live read of a remote chat worker is refused plainly, until reads of one are supported. */
|
||||
export function refuseRemoteChatWorkerRead(
|
||||
runtime: WorkerTerminalReleaseRuntime,
|
||||
attachment: RemoteDispatchAttachmentRow
|
||||
): void {
|
||||
if (remoteChatWorkerOf(runtime.getOrchestrationDb(), attachment)?.exact) {
|
||||
throw new OrchestrationError('capability_unsupported', REMOTE_CHAT_WORKER_READ_UNSUPPORTED)
|
||||
}
|
||||
}
|
||||
|
||||
export async function inspectRemoteAttachment(
|
||||
runtime: WorkerTerminalReleaseRuntime,
|
||||
dispatchId: string
|
||||
@@ -34,6 +48,16 @@ export async function inspectRemoteAttachment(
|
||||
}> {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const attachment = db.getRemoteDispatchAttachment(dispatchId)
|
||||
const chat = remoteChatWorkerOf(db, attachment)
|
||||
if (chat) {
|
||||
// Said plainly rather than read through a terminal it does not have.
|
||||
return {
|
||||
terminal: null,
|
||||
exact: chat.exact,
|
||||
status: chat.exact ? 'unverifiable' : 'identity_changed',
|
||||
...(chat.exact ? { reason: REMOTE_CHAT_WORKER_READ_UNSUPPORTED } : {})
|
||||
}
|
||||
}
|
||||
if (!attachment?.terminal_handle) {
|
||||
return { terminal: null, exact: false, status: 'unattached' }
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import { stopRemoteAttachment } from './federated-worker-stop-host'
|
||||
import {
|
||||
inspectRemoteAttachment,
|
||||
observeRemoteAttachmentLiveness,
|
||||
refuseRemoteChatWorkerRead,
|
||||
requireHomeAttachment
|
||||
} from './federation-attachment-observation'
|
||||
import { releaseRemoteAttachment } from './federated-worker-release-host'
|
||||
@@ -111,7 +112,10 @@ export const ORCHESTRATION_FEDERATION_CONTROL_METHODS = [
|
||||
permission: 'workspace',
|
||||
params: FederationReadParams,
|
||||
handler: async (params, { runtime, authenticatedCallerFingerprint }) => {
|
||||
requireHomeAttachment(runtime, params.dispatchId, authenticatedCallerFingerprint)
|
||||
refuseRemoteChatWorkerRead(
|
||||
runtime,
|
||||
requireHomeAttachment(runtime, params.dispatchId, authenticatedCallerFingerprint)
|
||||
)
|
||||
const observation = await inspectRemoteAttachment(runtime, params.dispatchId)
|
||||
// Only exited proves death; an unverifiable terminal may still be readable after stop-contact loss.
|
||||
if (!observation.exact || !observation.terminal || observation.status === 'exited') {
|
||||
@@ -171,6 +175,7 @@ export const ORCHESTRATION_FEDERATION_CONTROL_METHODS = [
|
||||
}
|
||||
}
|
||||
}
|
||||
refuseRemoteChatWorkerRead(runtime, attachment)
|
||||
const observation = await inspectRemoteAttachment(runtime, params.dispatchId)
|
||||
if (!observation.exact || !observation.terminal) {
|
||||
throw new OrchestrationError(
|
||||
|
||||
@@ -19,7 +19,7 @@ export type FederationEffect = {
|
||||
export function appendFederationTerminalEffects(
|
||||
effects: FederationEffect[],
|
||||
terminals: { handle: string; title: string | null; tabId?: string; leafId?: string }[],
|
||||
agentHandle: string,
|
||||
agentHandle: string | undefined,
|
||||
setupHandle?: string
|
||||
): void {
|
||||
for (const terminal of terminals) {
|
||||
|
||||
@@ -94,3 +94,28 @@ export function monitorFederatedSetup(
|
||||
})
|
||||
.catch(() => undefined)
|
||||
}
|
||||
|
||||
export function federatedSetupSource(
|
||||
params: { setup?: WorkerSetupReceipt['requested']; setupSource?: WorkerSetupReceipt['source'] },
|
||||
createsWorktree: boolean
|
||||
): WorkerSetupReceipt['source'] {
|
||||
return createsWorktree
|
||||
? (params.setupSource ?? (params.setup ? 'explicit_request' : 'orchestration_default'))
|
||||
: 'existing_worktree'
|
||||
}
|
||||
|
||||
/** The setup a start reports before any setup ran: none for an existing worktree. */
|
||||
export function initialFederatedSetup(
|
||||
params: { setup?: WorkerSetupReceipt['requested'] },
|
||||
createsWorktree: boolean,
|
||||
source: WorkerSetupReceipt['source']
|
||||
): WorkerSetupReceipt {
|
||||
return {
|
||||
requested: createsWorktree ? (params.setup ?? 'run') : 'not_applicable',
|
||||
effective: createsWorktree ? (params.setup ?? 'run') : 'not_applicable',
|
||||
source,
|
||||
hookFound: false,
|
||||
startupPolicy: 'start-immediately',
|
||||
state: createsWorktree ? 'not_configured' : 'not_applicable'
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,14 @@
|
||||
import type { RuntimeTerminalCreate } from '../../../../../../shared/runtime-terminal-create-contract'
|
||||
import { waitForWorkerAgentReady } from '../../../../terminal/wait/launched-agent-composer-readiness'
|
||||
import { describeTerminalWaitBlockedReason } from '../../../../../../shared/terminal/runtime-policy/terminal-wait-blocked-reason-legacy-alias'
|
||||
import { buildDispatchPreamble } from '../../../../orchestration/workers/launch/worker-dispatch-preamble'
|
||||
import { sendAgentTurn } from '../../../../orchestration/turn-delivery/send-agent-turn'
|
||||
import { createWorkerBriefWriteGuard } from '../../../../terminal/wait/launched-agent-write-guard'
|
||||
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
|
||||
import { defineMethod } from '../../../core'
|
||||
import { assertOrchestrationWorktreeCreationSupported } from '../worker/folder-worktree-placement'
|
||||
import type { FederationEffect } from './federation-effects'
|
||||
import { isFederationEffectUnknown, type FederationEffect } from './federation-effects'
|
||||
import { launchFederatedWorkerAgent } from './federated-worker-agent-launch'
|
||||
import type { WorkerSetupReceipt } from '../worker/worker-topology'
|
||||
import {
|
||||
federatedSetupSource,
|
||||
initialFederatedSetup,
|
||||
monitorFederatedSetup,
|
||||
persistFederatedReadinessStage,
|
||||
persistFederatedSetupSpawnFailure,
|
||||
@@ -26,6 +24,18 @@ import {
|
||||
import { assertWorkerStartTaskSpecWithinPromptBudget } from '../worker/worker-start-prompt-budget'
|
||||
import { prepareFederatedAttachmentAuthority } from './federation-attachment-authority'
|
||||
import { resolveRpcWorkspaceCreatorProvenance } from '../../../workspace-creator-context'
|
||||
import {
|
||||
decideWorkerStartMode,
|
||||
readWorkerStartModeSettings,
|
||||
resolveWorkerStartModeOnHost
|
||||
} from '../structured-workers/orchestration-worker-start-mode'
|
||||
import { tearDownFailedWorkerStart } from '../worker/failed-worker-start-teardown'
|
||||
import {
|
||||
awaitFederatedWorkerReady,
|
||||
federatedWorkerAddress,
|
||||
sendFederatedWorkerBrief,
|
||||
type StructuredSession
|
||||
} from './federated-worker-brief'
|
||||
import { claimTerminalForRemoteAttach } from '../../../../orchestration/db/worker-dispatch/worker-terminal-claim'
|
||||
|
||||
export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
|
||||
@@ -96,17 +106,14 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
|
||||
let worktree
|
||||
let terminalHandle = params.terminal
|
||||
let terminalCreation: RuntimeTerminalCreate | undefined
|
||||
const setupSource = createsWorktree
|
||||
? (params.setupSource ?? (params.setup ? 'explicit_request' : 'orchestration_default'))
|
||||
: 'existing_worktree'
|
||||
let setup: WorkerSetupReceipt = {
|
||||
requested: createsWorktree ? (params.setup ?? 'run') : 'not_applicable',
|
||||
effective: createsWorktree ? (params.setup ?? 'run') : 'not_applicable',
|
||||
source: setupSource,
|
||||
hookFound: false,
|
||||
startupPolicy: 'start-immediately',
|
||||
state: createsWorktree ? 'not_configured' : 'not_applicable'
|
||||
}
|
||||
let structuredSession: StructuredSession | null = null
|
||||
// This host decides chat or terminal from its own settings, as a local start does.
|
||||
let mode = decideWorkerStartMode({
|
||||
params: { agent, terminal: params.terminal, worktree: params.worktree },
|
||||
settings: readWorkerStartModeSettings(runtime)
|
||||
})
|
||||
const setupSource = federatedSetupSource(params, createsWorktree)
|
||||
let setup = initialFederatedSetup(params, createsWorktree, setupSource)
|
||||
try {
|
||||
const launchAgent = (existing?: { id: string }) =>
|
||||
launchFederatedWorkerAgent({
|
||||
@@ -115,6 +122,7 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
|
||||
params,
|
||||
agent,
|
||||
launchPreferences: launch.preferences,
|
||||
mode,
|
||||
...(existing
|
||||
? { worktree: existing }
|
||||
: { creatorProvenance: resolveRpcWorkspaceCreatorProvenance(context) }),
|
||||
@@ -128,7 +136,8 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
|
||||
}
|
||||
})
|
||||
if (createsWorktree) {
|
||||
;({ worktree, terminalHandle, terminalCreation } = await launchAgent())
|
||||
;({ worktree, terminalHandle, terminalCreation, structuredSession, mode } =
|
||||
await launchAgent())
|
||||
} else {
|
||||
worktree = await runtime.showManagedTerminalWorkspace(params.worktree).catch(() => {
|
||||
throw new OrchestrationError(
|
||||
@@ -162,7 +171,9 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
|
||||
})
|
||||
} else {
|
||||
failedStage = 'terminal_create'
|
||||
;({ terminalHandle, terminalCreation } = await launchAgent(worktree))
|
||||
mode = await resolveWorkerStartModeOnHost(runtime, mode, worktree.id, agent)
|
||||
;({ terminalHandle, terminalCreation, structuredSession, mode } =
|
||||
await launchAgent(worktree))
|
||||
}
|
||||
}
|
||||
if (!worktree || !terminalHandle) {
|
||||
@@ -182,14 +193,21 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
|
||||
}
|
||||
persistFederatedReadinessStage(setupStage)
|
||||
failedStage = 'agent_readiness'
|
||||
const wait = await waitForWorkerAgentReady(runtime, terminalHandle, {
|
||||
const wait = await awaitFederatedWorkerReady({
|
||||
runtime,
|
||||
structuredSession,
|
||||
terminalHandle,
|
||||
agent,
|
||||
reusesTerminal: Boolean(params.terminal),
|
||||
creation: terminalCreation,
|
||||
setup,
|
||||
effects,
|
||||
timeoutMs: readinessTimeoutMs
|
||||
})
|
||||
persistFederatedSetupWaitOutcome({ ...setupStage, wait })
|
||||
if (!wait.satisfied) {
|
||||
if (wait) {
|
||||
persistFederatedSetupWaitOutcome({ ...setupStage, wait })
|
||||
}
|
||||
if (wait && !wait.satisfied) {
|
||||
if (setup.state === 'failed') {
|
||||
failedStage = 'setup_wait'
|
||||
}
|
||||
@@ -207,33 +225,32 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
|
||||
terminalHandle,
|
||||
setup,
|
||||
effects,
|
||||
reusesTerminal: Boolean(params.terminal)
|
||||
reusesTerminal: Boolean(params.terminal),
|
||||
...(structuredSession ? { identity: structuredSession.identity } : {})
|
||||
})
|
||||
failedStage = 'dispatch_input'
|
||||
// A shell back at its prompt also reads as ready, so the brief needs the agent found in front.
|
||||
const briefGuard = createWorkerBriefWriteGuard(runtime, agent, !params.terminal)
|
||||
const prompt = await sendAgentTurn({
|
||||
kind: 'terminal',
|
||||
const preamble = buildDispatchPreamble({
|
||||
taskId: params.taskId,
|
||||
dispatchId: params.dispatchId,
|
||||
taskSpec: params.taskSpec,
|
||||
coordinatorHandle: 'Run home (relayed by Orca)',
|
||||
workerHandle: federatedWorkerAddress(structuredSession, terminalHandle),
|
||||
devMode: params.devMode,
|
||||
// Enforce the execution host's nesting cap using its own setting.
|
||||
canDispatchSubWorkers: (params.depth ?? 1) < runtime.getNestedWorkerMaxDepth(),
|
||||
subWorkersInOwnRun: true,
|
||||
cliCommand: runtime.getTerminalOrchestrationCliCommand(terminalHandle)
|
||||
})
|
||||
const prompt = await sendFederatedWorkerBrief({
|
||||
runtime,
|
||||
handle: terminalHandle,
|
||||
...(briefGuard ? { beforeWrite: briefGuard.beforeWrite } : {}),
|
||||
turn: {
|
||||
purpose: 'dispatch-preamble',
|
||||
operationId: orchestrationMutation.requestId,
|
||||
body: buildDispatchPreamble({
|
||||
taskId: params.taskId,
|
||||
dispatchId: params.dispatchId,
|
||||
taskSpec: params.taskSpec,
|
||||
coordinatorHandle: 'Run home (relayed by Orca)',
|
||||
workerHandle: terminalHandle,
|
||||
devMode: params.devMode,
|
||||
// Enforce the execution host's nesting cap using its own setting.
|
||||
canDispatchSubWorkers: (params.depth ?? 1) < runtime.getNestedWorkerMaxDepth(),
|
||||
subWorkersInOwnRun: true,
|
||||
cliCommand: runtime.getTerminalOrchestrationCliCommand(terminalHandle)
|
||||
})
|
||||
}
|
||||
}).finally(() => briefGuard?.dispose())
|
||||
structuredSession,
|
||||
terminalHandle,
|
||||
agent,
|
||||
params,
|
||||
operationId: orchestrationMutation.requestId,
|
||||
preamble,
|
||||
budgetMs: readinessTimeoutMs
|
||||
})
|
||||
effects.push({
|
||||
kind: 'dispatch_input',
|
||||
role: 'agent',
|
||||
@@ -251,11 +268,21 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
|
||||
terminalHandle,
|
||||
setup,
|
||||
launch: launch.receipt,
|
||||
mode,
|
||||
effects,
|
||||
...(prompt.prompt ? { prompt: prompt.prompt } : {}),
|
||||
...(prompt?.prompt ? { prompt: prompt.prompt } : {}),
|
||||
residualResources: []
|
||||
}
|
||||
} catch (error) {
|
||||
// A chat that may hold its brief stays for the worker to report; a refused start retires it.
|
||||
if (structuredSession && !isFederationEffectUnknown(error, failedStage)) {
|
||||
await tearDownFailedWorkerStart({
|
||||
runtime,
|
||||
structuredSession,
|
||||
dispatchId: params.dispatchId,
|
||||
error
|
||||
})
|
||||
}
|
||||
return failFederatedAttachmentWithReceipt({
|
||||
db,
|
||||
dispatchId: params.dispatchId,
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import type { OrcaRuntimeService } from '../../../../orca-runtime'
|
||||
import type { WorkerStopReceipt } from '../../../../../../shared/orchestration/workers/orchestration-worker-stop-receipt'
|
||||
import type {
|
||||
WorkerTerminalArchiveRow,
|
||||
WorkerTerminalResourceRow
|
||||
} from '../../../../orchestration/terminal-authority/release/worker-terminal-ownership'
|
||||
import { remoteChatWorkerOf } from '../../../../orchestration/federation/remote-chat-worker'
|
||||
import type { StructuredWorkerIdentity } from '../../../../orchestration/terminal-authority/structured-sessions/structured-worker-identity'
|
||||
import type { WorkerTerminalReleaseRuntime } from '../../../../orchestration/terminal-authority/release/worker-terminal-release-runtime'
|
||||
import { stopStructuredWorker } from '../structured-workers/orchestration-structured-worker-lifecycle'
|
||||
import { stopStructuredWorkerForRelease } from '../worker/structured-worker-release-stop'
|
||||
import { captureWorkerReleaseArchive } from '../worker/worker-release-archive-capture'
|
||||
import {
|
||||
isStillReleasing,
|
||||
retainedReason,
|
||||
type WorkerReleaseReceipt
|
||||
} from '../worker/worker-release-completion'
|
||||
import { orchestrationTimestampToMs } from '../worker/worker-output'
|
||||
import { archiveSummary } from '../worker/worker-terminal-resource-presentation'
|
||||
import { summarizeWorkerOutputArchive } from '../../../../orchestration/workers/archives/worker-output-archive'
|
||||
import { yieldAutoReleaseToWorkingAgent } from '../../../../orchestration/terminal-authority/release/worker-terminal-auto-release-working-yield'
|
||||
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
|
||||
import type { RemoteDispatchAttachmentRow } from '../../../../orchestration/db/orchestration-records'
|
||||
|
||||
/** Close a chat worker the way a local stop does; only a proven close settles. */
|
||||
export async function stopRemoteChatWorker(
|
||||
runtime: OrcaRuntimeService,
|
||||
dispatchId: string,
|
||||
chat: NonNullable<ReturnType<typeof remoteChatWorkerOf>>
|
||||
): Promise<WorkerStopReceipt> {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const unknown = (reason: string, processAction: string): WorkerStopReceipt => {
|
||||
const attachment = db.markRemoteAttachmentStopUnknown(dispatchId, reason)
|
||||
return {
|
||||
dispatchId,
|
||||
state: attachment.state,
|
||||
alreadySettled: false,
|
||||
processAction,
|
||||
lastError: attachment.last_error
|
||||
}
|
||||
}
|
||||
if (!chat.exact) {
|
||||
return unknown(
|
||||
'The recorded chat worker is no longer this Dispatch; nothing was closed.',
|
||||
'none'
|
||||
)
|
||||
}
|
||||
// Install the chat host after a restart, or the close cannot reach the session.
|
||||
await runtime.ensureStructuredAgentSessionHost().catch(() => undefined)
|
||||
const stop = await stopStructuredWorker(chat.identity, runtime)
|
||||
if (!stop.stopped) {
|
||||
return unknown(
|
||||
stop.reason ?? 'The close was not proven.',
|
||||
stop.closeAttempted ? 'closed_agent_terminal' : 'none'
|
||||
)
|
||||
}
|
||||
const attachment = db.settleRemoteAttachmentStop(dispatchId)
|
||||
return {
|
||||
dispatchId,
|
||||
state: attachment.state,
|
||||
alreadySettled: false,
|
||||
processAction: 'closed_agent_terminal'
|
||||
}
|
||||
}
|
||||
|
||||
/** A chat releases the way a local chat worker does: its transcript is kept, the release commits,
|
||||
* a working agent defers an automatic one, and only then does its chat close. */
|
||||
export async function releaseRemoteChatWorker(
|
||||
runtime: WorkerTerminalReleaseRuntime,
|
||||
attachment: RemoteDispatchAttachmentRow,
|
||||
identity: StructuredWorkerIdentity,
|
||||
resource: WorkerTerminalResourceRow,
|
||||
storedArchive: WorkerTerminalArchiveRow | null
|
||||
): Promise<WorkerReleaseReceipt> {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const dispatchId = attachment.dispatch_id
|
||||
await runtime.ensureStructuredAgentSessionHost().catch(() => undefined)
|
||||
let archive: Pick<
|
||||
Parameters<OrchestrationDb['commitWorkerTerminalArchiveForRelease']>[0],
|
||||
'kind' | 'content' | 'archiveSource' | 'archiveStatus'
|
||||
>
|
||||
if (storedArchive) {
|
||||
const stored = summarizeWorkerOutputArchive(storedArchive)
|
||||
archive = {
|
||||
archiveSource: stored.source,
|
||||
archiveStatus: stored.status === 'empty' ? 'empty' : 'captured'
|
||||
}
|
||||
} else {
|
||||
const capture = await captureWorkerReleaseArchive({
|
||||
db,
|
||||
resource,
|
||||
runtime,
|
||||
dispatchId,
|
||||
terminalHandle: identity.handle,
|
||||
attachedAtMs: orchestrationTimestampToMs(attachment.created_at),
|
||||
structuredWorker: identity
|
||||
})
|
||||
if ('receipt' in capture) {
|
||||
return capture.receipt
|
||||
}
|
||||
const { captured } = capture
|
||||
archive = {
|
||||
kind: captured.kind,
|
||||
content: JSON.stringify(captured.content),
|
||||
archiveSource: captured.kind === 'terminal_tail' ? 'terminal' : 'transcript',
|
||||
archiveStatus: captured.status === 'empty' ? 'empty' : 'captured'
|
||||
}
|
||||
}
|
||||
const committed = db.commitWorkerTerminalArchiveForRelease({
|
||||
dispatchId,
|
||||
resourceId: resource.id,
|
||||
...archive
|
||||
})
|
||||
const releasing = !isStillReleasing(committed)
|
||||
? committed
|
||||
: !remoteChatWorkerOf(db, attachment)?.exact
|
||||
? db.revertWorkerTerminalReleaseToRetained(resource.id, 'identity_unproven')
|
||||
: yieldAutoReleaseToWorkingAgent(runtime, committed)
|
||||
if (!isStillReleasing(releasing)) {
|
||||
return {
|
||||
dispatchId,
|
||||
state: 'retained',
|
||||
reason: retainedReason(releasing),
|
||||
processAction: 'none',
|
||||
archive: archiveSummary(releasing)
|
||||
}
|
||||
}
|
||||
const released = await stopStructuredWorkerForRelease({
|
||||
structured: identity,
|
||||
dispatchId,
|
||||
resource: releasing,
|
||||
runtime,
|
||||
db,
|
||||
archiveSource: releasing.archive_source,
|
||||
archiveStatus: releasing.archive_status
|
||||
})
|
||||
if (released.state === 'released') {
|
||||
db.recordRemoteAttachmentStage({ dispatchId, stage: 'released' })
|
||||
}
|
||||
return released
|
||||
}
|
||||
@@ -271,7 +271,7 @@ export function releaseUnknownRecovery(dispatchId: string): string {
|
||||
return `Inspect with: orca orchestration worker-show --dispatch ${dispatchId} --json — then retry worker-release with a fresh request ID (omit --retry-request to let the CLI generate one). Reusing the prior request ID only replays this release_unknown receipt. Never substitute a broad terminal close.`
|
||||
}
|
||||
|
||||
function retainedReason(resource: WorkerTerminalResourceRow): WorkerTerminalRetainedReason {
|
||||
export function retainedReason(resource: WorkerTerminalResourceRow): WorkerTerminalRetainedReason {
|
||||
if (resource.retained_reason) {
|
||||
return resource.retained_reason as WorkerTerminalRetainedReason
|
||||
}
|
||||
@@ -281,6 +281,6 @@ function retainedReason(resource: WorkerTerminalResourceRow): WorkerTerminalReta
|
||||
return 'identity_unproven'
|
||||
}
|
||||
|
||||
function isStillReleasing(resource: WorkerTerminalResourceRow): boolean {
|
||||
export function isStillReleasing(resource: WorkerTerminalResourceRow): boolean {
|
||||
return resource.ownership_state === 'owned' && resource.release_state === 'releasing'
|
||||
}
|
||||
|
||||
@@ -171,7 +171,7 @@ export async function startWorker(
|
||||
settings: readWorkerStartModeSettings(runtime)
|
||||
})
|
||||
if (params.on) {
|
||||
// Remote workers are terminal-only; carry the mode receipt to explain structured-preference fallback.
|
||||
// The server decides chat or terminal; one that cannot run chats leaves this host's terminal receipt.
|
||||
const receipt = await startFederatedWorker({
|
||||
params,
|
||||
runtime,
|
||||
@@ -184,7 +184,7 @@ export async function startWorker(
|
||||
followUp
|
||||
})
|
||||
await stopHelperAfterStart(runtime, receipt, orchestrationMutation)
|
||||
return withRun(receipt && typeof receipt === 'object' ? { ...receipt, mode } : receipt)
|
||||
return withRun(receipt && typeof receipt === 'object' ? { mode, ...receipt } : receipt)
|
||||
}
|
||||
const receipt = await startLocalWorker({
|
||||
params: { ...params, timeoutMs: readinessTimeoutMs },
|
||||
|
||||
Reference in New Issue
Block a user