refactor(orchestration): isolate the legacy remote-worker code in two folders (#40624)

* refactor(orchestration): isolate the legacy remote-attachment code behind one entry per side

Move today's remote-attachment (federation) code, unchanged, into
orchestration/legacy-remote/ (worker server) and
orchestration/legacy-remote-home/ (Run home). Code outside each folder
imports it only through legacy-remote.ts / legacy-remote-home.ts; a test
pins that. No behaviour change.

* refactor(orchestration): keep the legacy entry modules out of import cycles

The DB layer reaches each legacy folder through a DB-only module, and the
federation method arrays are spread where they were before, so no module
loads differently from main when imported first.

* test(orchestration): only the DB layer may use the legacy DB-only modules

* refactor(orchestration): let the two legacy folders import each other directly

They are one frozen unit; going through the other folder's entry module
made the Run-home half load the whole worker-server half.

* refactor(orchestration): import the pinned-server lookup from its legacy sibling

* refactor(orchestration): remove the import cycles the Run-home entry module created

Keep the pinned-server lookup at its original path (shared worker code calls it)
and let the worker list import the fleet snapshot directly; import/no-cycle is
clean again.
This commit is contained in:
Jinwoo Hong
2026-11-10 00:22:03 -05:00
committed by GitHub
parent fc8da7f6c6
commit d8893ebc9b
86 changed files with 797 additions and 618 deletions
@@ -3,7 +3,7 @@ import {
dispatchAssigneeCoordinatorKey,
selectRunBoundDispatchRecipient
} from '../../rpc/methods/orchestration/messaging/recipient-routing'
import { federatedControlMailRefusal } from '../../rpc/methods/orchestration/messaging/send-control-mail'
import { federatedControlMailRefusal } from '../legacy-remote-home/legacy-remote-home'
import type { createLineageRunningSessionResolver } from '../addressing/session-lineage/structured-session-lineage'
import type { DispatchInput } from './crew-recipient-owner-queries'
import type { CrewRecipientTerminal } from './crew-recipient-inputs'
@@ -7,17 +7,9 @@ import { attachDispatchContextStore } from './dispatch-context/dispatch-context-
import { attachDispatchLookup } from './dispatch-context/dispatch-lookup'
import { attachDispatchDepth } from './dispatch-depth'
import { attachWorkerReportSettlement } from './dispatch-context/worker-report-settlement'
import { attachFederatedDispatchStore } from './federation/federated-dispatch-store'
import { attachFederatedDispatchObservationFence } from './federation/federated-dispatch-observation-fence'
import { attachFederationRelayAck } from './federation/federation-relay-ack'
import { attachFederationRelayEnqueue } from './federation/federation-relay-enqueue'
import { attachFederationRelayImport } from './federation/federation-relay-import'
import { attachFederationRelayItem } from './federation/federation-relay-item'
import { attachRemoteDispatchAttachmentAuthority } from './federation/remote-dispatch-attachment-authority'
import { attachRemoteDispatchAttachmentCreate } from './federation/remote-dispatch-attachment-create'
import { attachRemoteDispatchAttachmentRelease } from './federation/remote-dispatch-attachment-release'
import { attachRemoteDispatchAttachmentStop } from './federation/remote-dispatch-attachment-stop'
import { attachRemoteQuestionStore } from './federation/remote-question-store'
import { attachLegacyRemoteDbMethods } from '../legacy-remote/legacy-remote-db-methods'
import { attachLegacyRemoteHomeDbMethods } from '../legacy-remote-home/legacy-remote-home-db-methods'
import { attachPeerMailStore } from './federation/peer-mail-store'
import { attachLegacyAskOperation } from './legacy/legacy-ask-operation'
import { attachLegacyCompatibilityCandidates } from './legacy/legacy-compatibility-candidates'
@@ -54,7 +46,6 @@ import { attachSchemaMigrate } from './schema/migrate'
import { attachSchemaColumnProbes } from './schema/schema-column-probes'
import { attachTaskStore } from './tasks/task-store'
import { attachTaskStatusTransition } from './tasks/task-status-transition'
import { attachFederatedWorkerStartReconcile } from './worker-dispatch/federated-worker-start-reconcile'
import { attachWorkerDispatchAbandon } from './worker-dispatch/worker-dispatch-abandon'
import { attachWorkerDispatchAuthority } from './worker-dispatch/worker-dispatch-authority'
import { attachWorkerDispatchOutcome } from './worker-dispatch/worker-dispatch-outcome'
@@ -109,22 +100,13 @@ export function attachOrchestrationDbMethods(ctor: { prototype: object }): void
attachWorkerDispatchStage(ctor)
attachWorkerDispatchAuthority(ctor)
attachWorkerDispatchOutcome(ctor)
attachFederatedWorkerStartReconcile(ctor)
attachWorkerTerminalRecovery(ctor)
attachWorkerDispatchStop(ctor)
attachWorkerDispatchAbandon(ctor)
attachFederatedDispatchStore(ctor)
attachFederatedDispatchObservationFence(ctor)
attachRemoteDispatchAttachmentCreate(ctor)
attachRemoteDispatchAttachmentAuthority(ctor)
attachRemoteDispatchAttachmentRelease(ctor)
attachRemoteDispatchAttachmentStop(ctor)
attachFederationRelayEnqueue(ctor)
attachLegacyRemoteHomeDbMethods(ctor)
attachLegacyRemoteDbMethods(ctor)
attachFederationRelayAck(ctor)
attachFederationRelayImport(ctor)
attachRemoteQuestionStore(ctor)
attachPeerMailStore(ctor)
attachFederationRelayItem(ctor)
attachWorkerTerminalResourceStore(ctor)
attachWorkerTerminalTransfer(ctor)
attachWorkerTerminalRelease(ctor)
@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest'
import { parseFederatedWorkerReportOutcome } from './federated-worker-report-outcome'
import { parseFederatedWorkerReportOutcome } from '../legacy-remote/federated-worker-report-outcome'
describe('federated-worker-report-outcome', () => {
it('reads succeeded or failed from a nested report payload', () => {
@@ -1,36 +1,7 @@
import type {
WorkerReportOutcome,
FederationRelayDirection,
FederationRelayItemRow
} from '../orchestration-records'
import { OrchestrationError } from '../../orchestration-error'
import { ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION } from '../../../../../shared/orchestration/orchestration-runtime-capabilities'
import { parseFederatedWorkerReportOutcome } from '../federated-worker-report-outcome'
import type { FederationRelayDirection, FederationRelayItemRow } from '../orchestration-records'
import type { FederationRelayAcknowledgment } from '../../legacy-remote/legacy-remote-db-methods'
import type { OrchestrationDb } from '../orchestration-db'
export function listFederationRelay(
this: OrchestrationDb,
params: {
dispatchId: string
direction: FederationRelayDirection
afterSequence: number
limit?: number
}
): FederationRelayItemRow[] {
return this.db
.prepare(
`SELECT * FROM federation_relay_items
WHERE dispatch_id = ? AND direction = ? AND sequence > ?
ORDER BY sequence LIMIT ?`
)
.all(
params.dispatchId,
params.direction,
params.afterSequence,
Math.min(Math.max(params.limit ?? 50, 1), 50)
) as FederationRelayItemRow[]
}
export function listPendingFederationRelay(
this: OrchestrationDb,
dispatchId: string,
@@ -48,78 +19,11 @@ export function listPendingFederationRelay(
export function acknowledgeFederationRelay(
this: OrchestrationDb,
params: {
dispatchId: string
direction: FederationRelayDirection
throughSequence: number
settleRemoteReports?: { sequence: number; outcome?: WorkerReportOutcome }[]
// Only the home's acceptance proves settlement; arm after it, never at queue time.
armAutoRelease?: boolean
}
params: FederationRelayAcknowledgment
): void {
this.db.exec('BEGIN IMMEDIATE')
try {
const settledReports = params.settleRemoteReports ?? []
for (const settledReport of settledReports) {
const report = this.getFederationRelayItem(
params.dispatchId,
params.direction,
settledReport.sequence
)
if (
params.direction !== 'to_home' ||
settledReport.sequence > params.throughSequence ||
report?.kind !== 'worker_done' ||
(settledReport.outcome !== undefined &&
parseFederatedWorkerReportOutcome(report.payload) !== settledReport.outcome)
) {
throw new OrchestrationError(
'request_mismatch',
`Federation acknowledgment for ${params.dispatchId} does not match its queued worker_done.`
)
}
}
const attachment = this.getRemoteDispatchAttachment(params.dispatchId)
if (
params.direction === 'to_home' &&
attachment !== undefined &&
attachment.protocol_version >= ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION
) {
const acknowledgedReports = this.db
.prepare(
`SELECT sequence FROM federation_relay_items
WHERE dispatch_id = ? AND direction = 'to_home' AND kind = 'worker_done'
AND acked_at IS NULL AND sequence <= ?`
)
.all(params.dispatchId, params.throughSequence) as { sequence: number }[]
const settledSequences = new Set(settledReports.map((report) => report.sequence))
if (acknowledgedReports.some((report) => !settledSequences.has(report.sequence))) {
throw new OrchestrationError(
'request_mismatch',
`Federation acknowledgment for ${params.dispatchId} omits a worker_done settlement.`
)
}
}
const terminalOutcomes = new Set(
settledReports.flatMap((report) => (report.outcome ? [report.outcome] : []))
)
if (terminalOutcomes.size > 1) {
throw new OrchestrationError(
'request_mismatch',
`Federation acknowledgment for ${params.dispatchId} contains conflicting settlements.`
)
}
const terminalOutcome = settledReports.find((report) => report.outcome)?.outcome
if (terminalOutcome) {
this.settleRemoteAttachmentInRelayTransaction(
params.dispatchId,
terminalOutcome,
'worker_report_settled'
)
if (params.armAutoRelease) {
this.armWorkerTerminalAutoReleaseStatement(params.dispatchId)
}
}
this.settleRemoteReportsInRelayAcknowledgment(params)
this.db
.prepare(
`UPDATE federation_relay_items SET acked_at = COALESCE(acked_at, datetime('now'))
@@ -133,93 +37,14 @@ export function acknowledgeFederationRelay(
}
}
export function setFederatedHomeImportSequence(
this: OrchestrationDb,
dispatchId: string,
sequence: number
): void {
this.db
.prepare(
`UPDATE federated_dispatches
SET to_home_imported_sequence = ?, updated_at = datetime('now')
WHERE dispatch_id = ? AND to_home_imported_sequence < ?`
)
.run(sequence, dispatchId, sequence)
}
export function recordFederatedHomeAcknowledgment(
this: OrchestrationDb,
params: {
dispatchId: string
remoteRuntimeEpoch: string
sequence: number
}
): void {
const federated = this.getFederatedDispatch(params.dispatchId)
if (
!federated ||
!Number.isInteger(params.sequence) ||
params.sequence < 0 ||
params.sequence > federated.to_home_imported_sequence
) {
throw new OrchestrationError(
'request_mismatch',
`Federation acknowledgment for ${params.dispatchId} exceeds imported relay state.`
)
}
this.db
.prepare(
`UPDATE federated_dispatches
SET remote_runtime_epoch = ?,
to_home_acknowledged_sequence = CASE
WHEN remote_runtime_epoch = ?
THEN MAX(to_home_acknowledged_sequence, ?)
ELSE ?
END,
updated_at = datetime('now')
WHERE dispatch_id = ?`
)
.run(
params.remoteRuntimeEpoch,
params.remoteRuntimeEpoch,
params.sequence,
params.sequence,
params.dispatchId
)
}
/** A worker_done queued for the Run home that it has not acknowledged. */
export function hasPendingFederatedWorkerReport(
this: OrchestrationDb,
dispatchId: string
): boolean {
return (
this.db
.prepare(
`SELECT 1 FROM federation_relay_items
WHERE dispatch_id = ? AND direction = 'to_home' AND kind = 'worker_done'
AND acked_at IS NULL LIMIT 1`
)
.get(dispatchId) !== undefined
)
}
export type FederationRelayAckMethods = {
listFederationRelay: typeof listFederationRelay
listPendingFederationRelay: typeof listPendingFederationRelay
acknowledgeFederationRelay: typeof acknowledgeFederationRelay
setFederatedHomeImportSequence: typeof setFederatedHomeImportSequence
recordFederatedHomeAcknowledgment: typeof recordFederatedHomeAcknowledgment
hasPendingFederatedWorkerReport: typeof hasPendingFederatedWorkerReport
}
export function attachFederationRelayAck(ctor: { prototype: object }): void {
Object.assign(ctor.prototype, {
listFederationRelay,
listPendingFederationRelay,
acknowledgeFederationRelay,
setFederatedHomeImportSequence,
recordFederatedHomeAcknowledgment,
hasPendingFederatedWorkerReport
acknowledgeFederationRelay
})
}
@@ -6,17 +6,9 @@ import type { DispatchContextStoreMethods } from './dispatch-context/dispatch-co
import type { DispatchLookupMethods } from './dispatch-context/dispatch-lookup'
import type { DispatchDepthMethods } from './dispatch-depth'
import type { WorkerReportSettlementMethods } from './dispatch-context/worker-report-settlement'
import type { FederatedDispatchStoreMethods } from './federation/federated-dispatch-store'
import type { FederatedDispatchObservationFenceMethods } from './federation/federated-dispatch-observation-fence'
import type { FederationRelayAckMethods } from './federation/federation-relay-ack'
import type { FederationRelayEnqueueMethods } from './federation/federation-relay-enqueue'
import type { FederationRelayImportMethods } from './federation/federation-relay-import'
import type { FederationRelayItemMethods } from './federation/federation-relay-item'
import type { RemoteDispatchAttachmentAuthorityMethods } from './federation/remote-dispatch-attachment-authority'
import type { RemoteDispatchAttachmentCreateMethods } from './federation/remote-dispatch-attachment-create'
import type { RemoteDispatchAttachmentReleaseMethods } from './federation/remote-dispatch-attachment-release'
import type { RemoteDispatchAttachmentStopMethods } from './federation/remote-dispatch-attachment-stop'
import type { RemoteQuestionStoreMethods } from './federation/remote-question-store'
import type { LegacyRemoteDbMethods } from '../legacy-remote/legacy-remote-db-methods'
import type { LegacyRemoteHomeDbMethods } from '../legacy-remote-home/legacy-remote-home-db-methods'
import type { PeerMailStoreMethods } from './federation/peer-mail-store'
import type { LegacyAskOperationMethods } from './legacy/legacy-ask-operation'
import type { LegacyCompatibilityCandidatesMethods } from './legacy/legacy-compatibility-candidates'
@@ -52,7 +44,6 @@ import type { SchemaMigrateMethods } from './schema/migrate'
import type { SchemaColumnProbesMethods } from './schema/schema-column-probes'
import type { TaskStoreMethods } from './tasks/task-store'
import type { TaskStatusTransitionMethods } from './tasks/task-status-transition'
import type { FederatedWorkerStartReconcileMethods } from './worker-dispatch/federated-worker-start-reconcile'
import type { WorkerDispatchAbandonMethods } from './worker-dispatch/worker-dispatch-abandon'
import type { WorkerDispatchAuthorityMethods } from './worker-dispatch/worker-dispatch-authority'
import type { WorkerDispatchOutcomeMethods } from './worker-dispatch/worker-dispatch-outcome'
@@ -106,22 +97,13 @@ export type OrchestrationDbMethods = AttemptObservationStoreMethods &
WorkerDispatchStageMethods &
WorkerDispatchAuthorityMethods &
WorkerDispatchOutcomeMethods &
FederatedWorkerStartReconcileMethods &
WorkerTerminalRecoveryMethods &
WorkerDispatchStopMethods &
WorkerDispatchAbandonMethods &
FederatedDispatchStoreMethods &
FederatedDispatchObservationFenceMethods &
RemoteDispatchAttachmentCreateMethods &
RemoteDispatchAttachmentAuthorityMethods &
RemoteDispatchAttachmentReleaseMethods &
RemoteDispatchAttachmentStopMethods &
FederationRelayEnqueueMethods &
LegacyRemoteHomeDbMethods &
LegacyRemoteDbMethods &
FederationRelayAckMethods &
FederationRelayImportMethods &
RemoteQuestionStoreMethods &
PeerMailStoreMethods &
FederationRelayItemMethods &
StructuredMailSendStoreMethods &
WorkerTerminalResourceStoreMethods &
WorkerTerminalTransferMethods &
@@ -4,7 +4,7 @@ import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { OrchestrationDb } from '../orchestration-db'
import { SCHEMA_VERSION, federatedStubHomeRunId } from '../contract-constants'
import { importFederatedControlMessage } from '../../federation/relay/federation-control-message'
import { importFederatedControlMessage } from '../../legacy-remote/federation-control-message'
describe('federated home Run migration', () => {
let db: OrchestrationDb
@@ -174,72 +174,6 @@ export function admitWorkerChatReleaseClose(
return changes > 0
}
/** Only for a host that cannot release on its own; arms once, so a retain or release before settlement still wins. */
export function armFederatedHomeAutoRelease(
this: OrchestrationDb,
dispatchId: string,
nowMs = Date.now()
): void {
const { changes } = this.db
.prepare(
`UPDATE federated_dispatches
SET home_auto_release_at = ?, home_auto_release_state = 'armed', updated_at = datetime('now')
WHERE dispatch_id = ? AND home_auto_release_state IS NULL`
)
.run(toSqliteUtcTimestamp(nowMs + WORKER_TERMINAL_AUTO_RELEASE_GRACE_MS), dispatchId)
if (changes > 0) {
this.workerTerminalAutoReleaseListener?.()
}
}
/** A retain, an explicit release or the host's answer to an attempt ends the home deadline for good. */
export function endFederatedHomeAutoRelease(this: OrchestrationDb, dispatchId: string): void {
this.db
.prepare(
`UPDATE federated_dispatches
SET home_auto_release_at = NULL, home_auto_release_state = 'ended', updated_at = datetime('now')
WHERE dispatch_id = ? AND home_auto_release_state IS NOT 'ended'`
)
.run(dispatchId)
}
/** Claiming postpones the deadline instead of hiding it, so an attempt cut short by a crash or failure runs again. */
export function claimDueFederatedHomeAutoReleases(this: OrchestrationDb, nowMs: number): string[] {
if (this.workerAutoReleaseEnabled?.() === false) {
return []
}
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the SELECT names exactly this one column.
const due = this.db
.prepare(
`SELECT dispatch_id FROM federated_dispatches
WHERE home_auto_release_state = 'armed' AND home_auto_release_at <= ?`
)
.all(toSqliteUtcTimestamp(nowMs)) as { dispatch_id: string }[]
const claim = this.db.prepare(
`UPDATE federated_dispatches
SET home_auto_release_at = ?, updated_at = datetime('now')
WHERE dispatch_id = ? AND home_auto_release_state = 'armed' AND home_auto_release_at <= ?`
)
const retryAt = toSqliteUtcTimestamp(nowMs + WORKER_TERMINAL_AUTO_RELEASE_GRACE_MS)
return due.flatMap(({ dispatch_id }) =>
claim.run(retryAt, dispatch_id, toSqliteUtcTimestamp(nowMs)).changes > 0 ? [dispatch_id] : []
)
}
export function isFederatedHomeAutoReleaseArmed(
this: OrchestrationDb,
dispatchId: string
): boolean {
return (
this.db
.prepare(
`SELECT 1 FROM federated_dispatches
WHERE dispatch_id = ? AND home_auto_release_state = 'armed'`
)
.get(dispatchId) !== undefined
)
}
export function nextWorkerTerminalAutoReleaseAtMs(this: OrchestrationDb): number | null {
// Off: nothing is due, so no timer runs until the setting turns back on.
if (this.workerAutoReleaseEnabled?.() === false) {
@@ -276,10 +210,6 @@ export type WorkerTerminalAutoReleaseMethods = {
claimDueWorkerTerminalAutoReleases: typeof claimDueWorkerTerminalAutoReleases
admitWorkerChatReleaseClose: typeof admitWorkerChatReleaseClose
nextWorkerTerminalAutoReleaseAtMs: typeof nextWorkerTerminalAutoReleaseAtMs
armFederatedHomeAutoRelease: typeof armFederatedHomeAutoRelease
endFederatedHomeAutoRelease: typeof endFederatedHomeAutoRelease
claimDueFederatedHomeAutoReleases: typeof claimDueFederatedHomeAutoReleases
isFederatedHomeAutoReleaseArmed: typeof isFederatedHomeAutoReleaseArmed
}
export function attachWorkerTerminalAutoRelease(ctor: { prototype: object }): void {
@@ -291,10 +221,6 @@ export function attachWorkerTerminalAutoRelease(ctor: { prototype: object }): vo
deferWorkerTerminalAutoRelease,
claimDueWorkerTerminalAutoReleases,
admitWorkerChatReleaseClose,
nextWorkerTerminalAutoReleaseAtMs,
armFederatedHomeAutoRelease,
endFederatedHomeAutoRelease,
claimDueFederatedHomeAutoReleases,
isFederatedHomeAutoReleaseArmed
nextWorkerTerminalAutoReleaseAtMs
})
}
@@ -1,7 +1,7 @@
import type { RuntimeRequestPreparation } from '../../../../../shared/runtime-rpc-envelope'
import { REMOTE_RUNTIME_MAX_INBOUND_REQUEST_JSON_BYTES } from '../../../../../shared/remote-runtime/remote-runtime-memory-limits'
import { OrchestrationError } from '../../orchestration-error'
import type { federationRelayWireItem } from './federation-relay-wire'
import type { federationRelayWireItem } from '../../legacy-remote/legacy-remote'
/** Fit only this already-selected page, using the actual sending socket's complete serializer. */
export function federationRelayPrefix(
@@ -19,16 +19,16 @@ import type {
import { OrchestrationError } from '../orchestration-error'
import {
clearFederationAckCheckpoints,
releaseFederationAckCheckpoint
} from './synchronization/federation-ack-checkpoints'
import { syncFederatedDispatch } from './synchronization/federation-sync'
releaseFederationAckCheckpoint,
syncFederatedDispatch,
FederationAcknowledgmentRecovery
} from '../legacy-remote-home/legacy-remote-home'
import type { OrchestrationDb } from '../db/orchestration-db'
import type { FederationRuntime } from './federation-runtime'
import {
FederationRelayScheduler,
type FederationRelayTickOutcome
} from './relay/federation-relay-scheduler'
import { FederationAcknowledgmentRecovery } from './relay/federation-acknowledgment-recovery'
export class RuntimeOrchestrationFederation {
private readonly relays = new FederationRelayScheduler((dispatchId) => this.relayTick(dispatchId))
@@ -6,7 +6,7 @@ import {
getFederationAckedThrough,
recordFederationAckCheckpoint,
type FederationAckIdentity
} from './federation-ack-checkpoints'
} from '../../legacy-remote-home/federation-ack-checkpoints'
describe('federation acknowledgment checkpoints', () => {
it('matches checkpoints only to their exact remote identity and never moves backward', () => {
@@ -1,7 +1,7 @@
import { describe, expect, it } from 'vitest'
import { ORCHESTRATION_FEDERATION_RELEASE_ARCHIVE_RUNTIME_CAPABILITY } from '../../../../../shared/orchestration/orchestration-runtime-capabilities'
import { createIdleSyncHarness } from './federation-sync-test-harness'
import { armHomeAutoReleaseForOlderHost } from './federation-sync-capability'
import { armHomeAutoReleaseForOlderHost } from '../../legacy-remote-home/federation-sync-capability'
function releasableTerminalWorker() {
const harness = createIdleSyncHarness()
@@ -0,0 +1,89 @@
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'
import { dirname, join, relative, resolve, sep } from 'node:path'
import { describe, expect, it } from 'vitest'
const SRC = resolve(import.meta.dirname, '../../..')
// Each folder has one entry module, plus a DB-only one so the DB layer never loads rpc code.
// The two folders are one frozen unit and may import each other directly.
const DB_LAYER = 'main/runtime/orchestration/db/'
const LEGACY_ENTRIES_BY_FOLDER = {
'main/runtime/orchestration/legacy-remote/': {
entry: 'main/runtime/orchestration/legacy-remote/legacy-remote.ts',
dbEntry: 'main/runtime/orchestration/legacy-remote/legacy-remote-db-methods.ts'
},
'main/runtime/orchestration/legacy-remote-home/': {
entry: 'main/runtime/orchestration/legacy-remote-home/legacy-remote-home.ts',
dbEntry: 'main/runtime/orchestration/legacy-remote-home/legacy-remote-home-db-methods.ts'
}
}
// Legacy starts load the worker list, so it bypasses the entry module to stay out of an import cycle.
const DIRECT_IMPORTS = new Set([
'main/runtime/rpc/methods/orchestration/worker/worker-list-projection.ts -> main/runtime/orchestration/legacy-remote-home/federated-fleet-snapshot.ts'
])
const IMPORT_SPECIFIER =
/(?:from\s+|import\s*\(\s*|import\s+|require\(\s*)['"](\.{1,2}\/[^'"]+)['"]/g
function productionSources(dir: string, out: string[] = []): string[] {
for (const entry of readdirSync(dir, { withFileTypes: true })) {
const path = join(dir, entry.name)
if (entry.isDirectory()) {
if (entry.name !== 'node_modules') {
productionSources(path, out)
}
} else if (
/\.tsx?$/.test(entry.name) &&
!/\.(test|spec)\.|test-support|harness/.test(entry.name)
) {
out.push(path)
}
}
return out
}
function srcPath(file: string): string {
return relative(SRC, file).split(sep).join('/')
}
function resolveSpecifier(fromFile: string, specifier: string): string | undefined {
const base = resolve(dirname(fromFile), specifier)
return [base, `${base}.ts`, `${base}.tsx`, join(base, 'index.ts')].find(
(candidate) => existsSync(candidate) && statSync(candidate).isFile()
)
}
describe('legacy remote-attachment folders', () => {
it('are imported from outside only through their entry module', () => {
const violations: string[] = []
const entryImporters = new Set<string>()
for (const file of productionSources(SRC)) {
const from = srcPath(file)
for (const match of readFileSync(file, 'utf8').matchAll(IMPORT_SPECIFIER)) {
const target = resolveSpecifier(file, match[1])
if (!target) {
continue
}
const to = srcPath(target)
for (const [folder, { entry, dbEntry }] of Object.entries(LEGACY_ENTRIES_BY_FOLDER)) {
if (
to.startsWith(folder) &&
!Object.keys(LEGACY_ENTRIES_BY_FOLDER).some((f) => from.startsWith(f))
) {
if (to === entry || (to === dbEntry && from.startsWith(DB_LAYER))) {
entryImporters.add(`${from} -> ${to}`)
} else if (!DIRECT_IMPORTS.has(`${from} -> ${to}`)) {
violations.push(`${from} -> ${to}`)
}
}
}
}
}
expect(violations).toEqual([])
// The capability branch on each side: legacy worker starts and the legacy federation methods.
expect(entryImporters).toContain(
'main/runtime/rpc/methods/orchestration/worker/workers.ts -> main/runtime/orchestration/legacy-remote-home/legacy-remote-home.ts'
)
expect(entryImporters).toContain(
'main/runtime/rpc/methods/orchestration/federation/federation-methods.ts -> main/runtime/orchestration/legacy-remote/legacy-remote.ts'
)
})
})
@@ -1,4 +1,4 @@
import type { OrchestrationWorkerLaunchReceipt } from '../worker/worker-launch-preferences'
import type { OrchestrationWorkerLaunchReceipt } from '../../rpc/methods/orchestration/worker/worker-launch-preferences'
export type RemoteFederatedWorkerStartReceipt = {
dispatchId: string
@@ -1,6 +1,6 @@
import type { FederatedDispatchRow } from '../orchestration-records'
import { OrchestrationError } from '../../orchestration-error'
import type { OrchestrationDb } from '../orchestration-db'
import type { FederatedDispatchRow } from '../db/orchestration-records'
import { OrchestrationError } from '../orchestration-error'
import type { OrchestrationDb } from '../db/orchestration-db'
export function getFederatedDispatch(
this: OrchestrationDb,
@@ -125,6 +125,61 @@ export function updateFederatedDispatchRuntimeEpoch(
.run(remoteRuntimeEpoch, dispatchId, remoteRuntimeEpoch)
}
export function setFederatedHomeImportSequence(
this: OrchestrationDb,
dispatchId: string,
sequence: number
): void {
this.db
.prepare(
`UPDATE federated_dispatches
SET to_home_imported_sequence = ?, updated_at = datetime('now')
WHERE dispatch_id = ? AND to_home_imported_sequence < ?`
)
.run(sequence, dispatchId, sequence)
}
export function recordFederatedHomeAcknowledgment(
this: OrchestrationDb,
params: {
dispatchId: string
remoteRuntimeEpoch: string
sequence: number
}
): void {
const federated = this.getFederatedDispatch(params.dispatchId)
if (
!federated ||
!Number.isInteger(params.sequence) ||
params.sequence < 0 ||
params.sequence > federated.to_home_imported_sequence
) {
throw new OrchestrationError(
'request_mismatch',
`Federation acknowledgment for ${params.dispatchId} exceeds imported relay state.`
)
}
this.db
.prepare(
`UPDATE federated_dispatches
SET remote_runtime_epoch = ?,
to_home_acknowledged_sequence = CASE
WHEN remote_runtime_epoch = ?
THEN MAX(to_home_acknowledged_sequence, ?)
ELSE ?
END,
updated_at = datetime('now')
WHERE dispatch_id = ?`
)
.run(
params.remoteRuntimeEpoch,
params.remoteRuntimeEpoch,
params.sequence,
params.sequence,
params.dispatchId
)
}
export type FederatedDispatchStoreMethods = {
getFederatedDispatch: typeof getFederatedDispatch
listFederatedDispatchesByIds: typeof listFederatedDispatchesByIds
@@ -133,6 +188,8 @@ export type FederatedDispatchStoreMethods = {
isFederatedDispatchRelayEligible: typeof isFederatedDispatchRelayEligible
updateFederatedDispatchResources: typeof updateFederatedDispatchResources
updateFederatedDispatchRuntimeEpoch: typeof updateFederatedDispatchRuntimeEpoch
setFederatedHomeImportSequence: typeof setFederatedHomeImportSequence
recordFederatedHomeAcknowledgment: typeof recordFederatedHomeAcknowledgment
}
export function attachFederatedDispatchStore(ctor: { prototype: object }): void {
@@ -143,6 +200,8 @@ export function attachFederatedDispatchStore(ctor: { prototype: object }): void
findNextTerminalFederatedDispatchPendingAcknowledgment,
isFederatedDispatchRelayEligible,
updateFederatedDispatchResources,
updateFederatedDispatchRuntimeEpoch
updateFederatedDispatchRuntimeEpoch,
setFederatedHomeImportSequence,
recordFederatedHomeAcknowledgment
})
}
@@ -1,7 +1,7 @@
import { ORCHESTRATION_FLEET_PAGE_MAX } from '../../../../../../shared/orchestration/fleet/orchestration-fleet-projection'
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import type { FederatedDispatchRow } from '../../../../orchestration/db/orchestration-records'
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import { ORCHESTRATION_FLEET_PAGE_MAX } from '../../../../shared/orchestration/fleet/orchestration-fleet-projection'
import type { OrchestrationDb } from '../db/orchestration-db'
import type { FederatedDispatchRow } from '../db/orchestration-records'
import type { OrcaRuntimeService } from '../../orca-runtime'
type HostGroup = {
environmentId: string
@@ -1,18 +1,18 @@
import { groupFederatedDispatches } from './federated-fleet-host-groups'
import { mapWithConcurrency } from '../../../../../../shared/map-with-concurrency'
import { ORCHESTRATION_FEDERATION_FLEET_SNAPSHOT_RUNTIME_CAPABILITY } from '../../../../../../shared/orchestration/orchestration-runtime-capabilities'
import { mapWithConcurrency } from '../../../../shared/map-with-concurrency'
import { ORCHESTRATION_FEDERATION_FLEET_SNAPSHOT_RUNTIME_CAPABILITY } from '../../../../shared/orchestration/orchestration-runtime-capabilities'
import {
refreshOrchestrationFleetLivenessAttention,
type FleetDurableWorker,
type OrchestrationFleetPage
} from '../../../../../../shared/orchestration/fleet/orchestration-fleet-projection'
import { projectFleetNextAction } from '../../../../../../shared/orchestration/fleet/orchestration-fleet-worker-projection'
import { getOrchestrationPeerCapabilityCache } from '../../../../orchestration/federation/orchestration-peer-capability-cache'
import { liveFederatedHomeAutoReleaseAt } from '../../../../orchestration/terminal-authority/release/worker-terminal-ownership'
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import { resolvePinnedFederatedServer } from '../worker/pinned-federated-server'
} from '../../../../shared/orchestration/fleet/orchestration-fleet-projection'
import { projectFleetNextAction } from '../../../../shared/orchestration/fleet/orchestration-fleet-worker-projection'
import { getOrchestrationPeerCapabilityCache } from '../federation/orchestration-peer-capability-cache'
import { liveFederatedHomeAutoReleaseAt } from '../terminal-authority/release/worker-terminal-ownership'
import type { OrchestrationDb } from '../db/orchestration-db'
import { OrchestrationError } from '../orchestration-error'
import type { OrcaRuntimeService } from '../../orca-runtime'
import { resolvePinnedFederatedServer } from '../../rpc/methods/orchestration/worker/pinned-federated-server'
import { applyConfirmedFederatedReleaseHomeProjection } from './federated-worker-release'
const FLEET_HOST_CONCURRENCY = 4
@@ -0,0 +1,85 @@
import type { OrchestrationDb } from '../db/orchestration-db'
import { toSqliteUtcTimestamp } from '../db/utc-timestamp'
import { WORKER_TERMINAL_AUTO_RELEASE_GRACE_MS } from '../db/worker-terminal/worker-terminal-auto-release'
/** Only for a host that cannot release on its own; arms once, so a retain or release before settlement still wins. */
export function armFederatedHomeAutoRelease(
this: OrchestrationDb,
dispatchId: string,
nowMs = Date.now()
): void {
const { changes } = this.db
.prepare(
`UPDATE federated_dispatches
SET home_auto_release_at = ?, home_auto_release_state = 'armed', updated_at = datetime('now')
WHERE dispatch_id = ? AND home_auto_release_state IS NULL`
)
.run(toSqliteUtcTimestamp(nowMs + WORKER_TERMINAL_AUTO_RELEASE_GRACE_MS), dispatchId)
if (changes > 0) {
this.workerTerminalAutoReleaseListener?.()
}
}
/** A retain, an explicit release or the host's answer to an attempt ends the home deadline for good. */
export function endFederatedHomeAutoRelease(this: OrchestrationDb, dispatchId: string): void {
this.db
.prepare(
`UPDATE federated_dispatches
SET home_auto_release_at = NULL, home_auto_release_state = 'ended', updated_at = datetime('now')
WHERE dispatch_id = ? AND home_auto_release_state IS NOT 'ended'`
)
.run(dispatchId)
}
/** Claiming postpones the deadline instead of hiding it, so an attempt cut short by a crash or failure runs again. */
export function claimDueFederatedHomeAutoReleases(this: OrchestrationDb, nowMs: number): string[] {
if (this.workerAutoReleaseEnabled?.() === false) {
return []
}
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the SELECT names exactly this one column.
const due = this.db
.prepare(
`SELECT dispatch_id FROM federated_dispatches
WHERE home_auto_release_state = 'armed' AND home_auto_release_at <= ?`
)
.all(toSqliteUtcTimestamp(nowMs)) as { dispatch_id: string }[]
const claim = this.db.prepare(
`UPDATE federated_dispatches
SET home_auto_release_at = ?, updated_at = datetime('now')
WHERE dispatch_id = ? AND home_auto_release_state = 'armed' AND home_auto_release_at <= ?`
)
const retryAt = toSqliteUtcTimestamp(nowMs + WORKER_TERMINAL_AUTO_RELEASE_GRACE_MS)
return due.flatMap(({ dispatch_id }) =>
claim.run(retryAt, dispatch_id, toSqliteUtcTimestamp(nowMs)).changes > 0 ? [dispatch_id] : []
)
}
export function isFederatedHomeAutoReleaseArmed(
this: OrchestrationDb,
dispatchId: string
): boolean {
return (
this.db
.prepare(
`SELECT 1 FROM federated_dispatches
WHERE dispatch_id = ? AND home_auto_release_state = 'armed'`
)
.get(dispatchId) !== undefined
)
}
export type FederatedHomeAutoReleaseStoreMethods = {
armFederatedHomeAutoRelease: typeof armFederatedHomeAutoRelease
endFederatedHomeAutoRelease: typeof endFederatedHomeAutoRelease
claimDueFederatedHomeAutoReleases: typeof claimDueFederatedHomeAutoReleases
isFederatedHomeAutoReleaseArmed: typeof isFederatedHomeAutoReleaseArmed
}
export function attachFederatedHomeAutoReleaseStore(ctor: { prototype: object }): void {
Object.assign(ctor.prototype, {
armFederatedHomeAutoRelease,
endFederatedHomeAutoRelease,
claimDueFederatedHomeAutoReleases,
isFederatedHomeAutoReleaseArmed
})
}
@@ -1,11 +1,11 @@
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import type { OrchestrationSessionCaller } from '../../../../orchestration/addressing/orchestration-caller-identity'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import { readCallerStoreInstallationId } from '../../../../hosts/platform-probe/host-descriptor'
import { getCanonicalUserDataPath } from '../../../../../persistence/loading-store/user-data-path'
import type { CrossStoreParentRef } from '../../../../../../shared/worktree/lineage-types'
import { supportsCrossStoreLineage } from '../../../../../../shared/worktree-runtime-capabilities'
import { resolveDispatchCallerWorktreeId } from '../orchestration-caller-workspace'
import type { OrcaRuntimeService } from '../../orca-runtime'
import type { OrchestrationSessionCaller } from '../addressing/orchestration-caller-identity'
import { OrchestrationError } from '../orchestration-error'
import { readCallerStoreInstallationId } from '../../hosts/platform-probe/host-descriptor'
import { getCanonicalUserDataPath } from '../../../persistence/loading-store/user-data-path'
import type { CrossStoreParentRef } from '../../../../shared/worktree/lineage-types'
import { supportsCrossStoreLineage } from '../../../../shared/worktree-runtime-capabilities'
import { resolveDispatchCallerWorktreeId } from '../../rpc/methods/orchestration/orchestration-caller-workspace'
/**
* `--on <server> --worktree new-child`: the server's new worktree nests under the coordinator's
@@ -1,5 +1,5 @@
import type { WorkerTerminalReleaseRuntime } from '../../../../orchestration/terminal-authority/release/worker-terminal-release-runtime'
import { resolvePinnedFederatedServer } from '../worker/worker-observation'
import type { WorkerTerminalReleaseRuntime } from '../terminal-authority/release/worker-terminal-release-runtime'
import { resolvePinnedFederatedServer } from '../../rpc/methods/orchestration/worker/pinned-federated-server'
import { releaseFederatedWorker } from './federated-worker-release'
/** The home's deadline for a host that did not arm its own; the host's takeover and ownership checks still refuse it. */
@@ -1,15 +1,15 @@
import type {
ORCHESTRATION_WORKER_READ_SOURCES,
OrchestrationWorkerReadResult
} from '../../../../../../shared/orchestration/workers/orchestration-worker-output'
import { ORCHESTRATION_FEDERATION_STRUCTURED_READ_RUNTIME_CAPABILITY } from '../../../../../../shared/orchestration/orchestration-runtime-capabilities'
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import { getOrchestrationPeerCapabilityCache } from '../../../../orchestration/federation/orchestration-peer-capability-cache'
import type { FederatedDispatchRow } from '../../../../orchestration/db/orchestration-records'
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import { readLegacyFederatedTerminal } from '../worker/worker-legacy-federated-read'
import type { resolvePinnedFederatedServer } from '../worker/worker-observation'
} from '../../../../shared/orchestration/workers/orchestration-worker-output'
import { ORCHESTRATION_FEDERATION_STRUCTURED_READ_RUNTIME_CAPABILITY } from '../../../../shared/orchestration/orchestration-runtime-capabilities'
import type { OrchestrationDb } from '../db/orchestration-db'
import { OrchestrationError } from '../orchestration-error'
import { getOrchestrationPeerCapabilityCache } from '../federation/orchestration-peer-capability-cache'
import type { FederatedDispatchRow } from '../db/orchestration-records'
import type { OrcaRuntimeService } from '../../orca-runtime'
import { readLegacyFederatedTerminal } from './worker-legacy-federated-read'
import type { resolvePinnedFederatedServer } from '../../rpc/methods/orchestration/worker/pinned-federated-server'
export async function readFederatedWorkerOutput(args: {
runtime: OrcaRuntimeService
@@ -1,4 +1,4 @@
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import type { OrchestrationDb } from '../db/orchestration-db'
import type { ReadyRemoteFederatedWorkerStartReceipt } from './federated-attach-receipt'
/** Records where a ready remote worker lives and that the server accepted its input. */
@@ -1,17 +1,17 @@
import {
ORCHESTRATION_FEDERATION_RELEASE_ARCHIVE_RUNTIME_CAPABILITY,
ORCHESTRATION_WORKER_AUTO_RELEASE_RUNTIME_CAPABILITY
} from '../../../../../../shared/orchestration/orchestration-runtime-capabilities'
import type { RuntimeStatus } from '../../../../../../shared/runtime-session-contracts'
} from '../../../../shared/orchestration/orchestration-runtime-capabilities'
import type { RuntimeStatus } from '../../../../shared/runtime-session-contracts'
import { z } from 'zod'
import { getOrchestrationPeerCapabilityCache } from '../../../../orchestration/federation/orchestration-peer-capability-cache'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import type { FederatedDispatchRow } from '../../../../orchestration/db/orchestration-records'
import type { WorkerTerminalReleaseRuntime } from '../../../../orchestration/terminal-authority/release/worker-terminal-release-runtime'
import type { WorkerReleaseReceipt } from '../worker/worker-release-completion'
import { releaseUnknownRecovery } from '../worker/worker-release-unknown-recovery'
import type { resolvePinnedFederatedServer } from '../worker/worker-observation'
import { getOrchestrationPeerCapabilityCache } from '../federation/orchestration-peer-capability-cache'
import { OrchestrationError } from '../orchestration-error'
import type { OrchestrationDb } from '../db/orchestration-db'
import type { FederatedDispatchRow } from '../db/orchestration-records'
import type { WorkerTerminalReleaseRuntime } from '../terminal-authority/release/worker-terminal-release-runtime'
import type { WorkerReleaseReceipt } from '../../rpc/methods/orchestration/worker/worker-release-completion'
import { releaseUnknownRecovery } from '../../rpc/methods/orchestration/worker/worker-release-unknown-recovery'
import type { resolvePinnedFederatedServer } from '../../rpc/methods/orchestration/worker/pinned-federated-server'
type RemoteReleaseReceipt = Omit<WorkerReleaseReceipt, 'archive'> & {
archive?: WorkerReleaseReceipt['archive']
@@ -1,20 +1,17 @@
import type { OrchestrationFleetWorker } from '../../../../../../shared/orchestration/fleet/orchestration-fleet-projection'
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import { exposeDispatchContext } from '../worker/dispatch-context-exposure'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import { liveFederatedHomeAutoReleaseAt } from '../../../../orchestration/terminal-authority/release/worker-terminal-ownership'
import type {
DispatchContextRow,
FederatedDispatchRow
} from '../../../../orchestration/db/orchestration-records'
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import type { OrchestrationFleetWorker } from '../../../../shared/orchestration/fleet/orchestration-fleet-projection'
import type { OrchestrationDb } from '../db/orchestration-db'
import { exposeDispatchContext } from '../../rpc/methods/orchestration/worker/dispatch-context-exposure'
import { OrchestrationError } from '../orchestration-error'
import { liveFederatedHomeAutoReleaseAt } from '../terminal-authority/release/worker-terminal-ownership'
import type { DispatchContextRow, FederatedDispatchRow } from '../db/orchestration-records'
import type { OrcaRuntimeService } from '../../orca-runtime'
import {
callFederatedWorkerShow,
exposeFederatedWorkerObservation,
exposeWorker,
projectFleetWorkerPage,
resolvePinnedFederatedServer
} from '../worker/worker-observation'
projectFleetWorkerPage
} from '../../rpc/methods/orchestration/worker/worker-observation'
import { resolvePinnedFederatedServer } from '../../rpc/methods/orchestration/worker/pinned-federated-server'
import { applyFederatedFleetObservations } from './federated-fleet-snapshot'
import { applyConfirmedFederatedReleaseHomeProjection } from './federated-worker-release'
@@ -1,4 +1,4 @@
import type { OrchestrationWorkerLaunchReceipt } from '../worker/worker-launch-preferences'
import type { OrchestrationWorkerLaunchReceipt } from '../../rpc/methods/orchestration/worker/worker-launch-preferences'
export function isKnownRemoteStartFailure(code: string): boolean {
return [
@@ -1,13 +1,13 @@
import type { WorkerDispatchRow } from '../orchestration-records'
import { OrchestrationError } from '../../orchestration-error'
import type { OrchestrationDb } from '../orchestration-db'
import { reconcileTaskAfterDispatchInterruption } from '../dispatch-context/task-dispatch-reconciliation'
import type { WorkerDispatchRow } from '../db/orchestration-records'
import { OrchestrationError } from '../orchestration-error'
import type { OrchestrationDb } from '../db/orchestration-db'
import { reconcileTaskAfterDispatchInterruption } from '../db/dispatch-context/task-dispatch-reconciliation'
import {
beginLifecycleWriteTransaction,
commitLifecycleWriteTransaction,
rollbackLifecycleWriteTransaction,
transitionLifecycleWithDb
} from '../lifecycle-transition'
} from '../db/lifecycle-transition'
export function reconcileFederatedWorkerStart(
this: OrchestrationDb,
@@ -1,8 +1,8 @@
import type { OrchestrationSessionCaller } from '../../../../orchestration/addressing/orchestration-caller-identity'
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import type { WorkerStartInput } from '../worker/worker-start-schema'
import type { WorkerStartMutation } from '../worker/worker-start-mutation'
import type { OrchestrationSessionCaller } from '../addressing/orchestration-caller-identity'
import type { OrcaRuntimeService } from '../../orca-runtime'
import type { OrchestrationDb } from '../db/orchestration-db'
import type { WorkerStartInput } from '../../rpc/methods/orchestration/worker/worker-start-schema'
import type { WorkerStartMutation } from '../../rpc/methods/orchestration/worker/worker-start-mutation'
export type FederatedWorkerStartRequest = {
params: WorkerStartInput
@@ -1,6 +1,6 @@
import type { FederatedWorkerStartRequest } from './federated-worker-start-request'
import { isTuiAgent } from '../../../../../../shared/agent-launch/catalog/tui-agent-config'
import type { RuntimeStatus } from '../../../../../../shared/runtime-session-contracts'
import { isTuiAgent } from '../../../../shared/agent-launch/catalog/tui-agent-config'
import type { RuntimeStatus } from '../../../../shared/runtime-session-contracts'
import {
ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY,
ORCHESTRATION_FEDERATION_CONTROL_MAIL_PROTOCOL_VERSION,
@@ -9,37 +9,37 @@ import {
ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_RUNTIME_CAPABILITY,
ORCHESTRATION_FEDERATION_RUNTIME_CAPABILITY,
ORCHESTRATION_WORKER_TURN_STOP_REPORTS_RUNTIME_CAPABILITY
} from '../../../../../../shared/orchestration/orchestration-runtime-capabilities'
import { orchestrationMigrationData } from '../../../../../../shared/orchestration/orchestration-rpc-contract'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
} from '../../../../shared/orchestration/orchestration-runtime-capabilities'
import { orchestrationMigrationData } from '../../../../shared/orchestration/orchestration-rpc-contract'
import { OrchestrationError } from '../orchestration-error'
import {
assertWorkerLaunchPreferencesRuntimeSupported,
assertWorkerLaunchPreferencesCreateTerminal,
createPendingWorkerLaunchReceipt,
resolveFederatedWorkerLaunchReceipt
} from '../worker/worker-launch-preferences'
import { validateFederatedWorkerStartPlacement } from '../worker/worker-start-validation'
import { resolveFederatedWorkerStartBudgets } from '../worker/worker-start-budgets'
import { resolveDispatchCreator } from '../runs/dispatch-creator'
} from '../../rpc/methods/orchestration/worker/worker-launch-preferences'
import { validateFederatedWorkerStartPlacement } from '../../rpc/methods/orchestration/worker/worker-start-validation'
import { resolveFederatedWorkerStartBudgets } from '../../rpc/methods/orchestration/worker/worker-start-budgets'
import { resolveDispatchCreator } from '../../rpc/methods/orchestration/runs/dispatch-creator'
import {
isReadyRemoteFederatedWorkerStartReceipt,
parseRemoteFederatedWorkerStartReceipt
} from './federated-attach-receipt'
import { isWorkerStartTimeoutWithinTimerLimit } from '../../../../../../shared/orchestration/orchestration-timing-budgets'
import { isWorkerStartTimeoutWithinTimerLimit } from '../../../../shared/orchestration/orchestration-timing-budgets'
import {
federatedFailedReceipt,
federatedUnknownReceipt,
isKnownRemoteStartFailure
} from './federated-worker-start-receipts'
import { releaseRetriedWorkerTerminal } from '../worker/retried-worker-terminal-release'
import { withWorkerStartNext } from '../receipt-next'
import { releaseRetriedWorkerTerminal } from '../../rpc/methods/orchestration/worker/retried-worker-terminal-release'
import { withWorkerStartNext } from '../../rpc/methods/orchestration/receipt-next'
import { recordReadyFederatedDispatch } from './federated-worker-ready-record'
import { resolveFederatedChildParent } from './federated-worker-child-parent'
import { prepareWorkerStartCard } from '../../../../orchestration/workers/launch/worker-start-card'
import { prepareWorkerStartCard } from '../workers/launch/worker-start-card'
import {
recordWorkerLaunchObservation,
workerLaunchAgent
} from '../../../../orchestration/workers/launch/worker-launch-observation'
} from '../workers/launch/worker-launch-observation'
export async function startFederatedWorker(args: FederatedWorkerStartRequest): Promise<unknown> {
const { params, runtime, db, task, orchestrationMutation } = args
@@ -1,4 +1,4 @@
import { federationRelayRetryDelayMs } from './federation-relay-scheduler'
import { federationRelayRetryDelayMs } from '../federation/relay/federation-relay-scheduler'
type PendingAcknowledgmentRow = { dispatchId: string; rowId: number }
@@ -4,9 +4,9 @@ import type {
MessageRow,
WorkerReportOutcome,
WorkerReportSettlement
} from '../orchestration-records'
import { OrchestrationError } from '../../orchestration-error'
import type { OrchestrationDb } from '../orchestration-db'
} from '../db/orchestration-records'
import { OrchestrationError } from '../orchestration-error'
import type { OrchestrationDb } from '../db/orchestration-db'
export function importFederatedRelayItem(
this: OrchestrationDb,
@@ -1,17 +1,17 @@
import type { RuntimeStatus } from '../../../../../shared/runtime-session-contracts'
import '../../../../../shared/protocol-version'
import type { RuntimeStatus } from '../../../../shared/runtime-session-contracts'
import '../../../../shared/protocol-version'
import {
ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION,
ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_RUNTIME_CAPABILITY,
ORCHESTRATION_FEDERATION_RELEASE_ARCHIVE_RUNTIME_CAPABILITY,
ORCHESTRATION_WORKER_AUTO_RELEASE_RUNTIME_CAPABILITY
} from '../../../../../shared/orchestration/orchestration-runtime-capabilities'
import type { RuntimeCapability } from '../../../../../shared/protocol-version'
import { STRUCTURED_WORKER_HANDLE_PREFIX } from '../../../../../shared/orchestration/workers/structured-worker-handle'
import type { FederationRuntime } from '../federation-runtime'
import type { FederatedDispatchRow } from '../../db/orchestration-records'
import { getOrchestrationPeerCapabilityCache } from '../orchestration-peer-capability-cache'
import { WORKER_RELEASABLE_STATES } from '../../terminal-authority/release/worker-terminal-ownership'
} from '../../../../shared/orchestration/orchestration-runtime-capabilities'
import type { RuntimeCapability } from '../../../../shared/protocol-version'
import { STRUCTURED_WORKER_HANDLE_PREFIX } from '../../../../shared/orchestration/workers/structured-worker-handle'
import type { FederationRuntime } from '../federation/federation-runtime'
import type { FederatedDispatchRow } from '../db/orchestration-records'
import { getOrchestrationPeerCapabilityCache } from '../federation/orchestration-peer-capability-cache'
import { WORKER_RELEASABLE_STATES } from '../terminal-authority/release/worker-terminal-ownership'
export async function resolveFederatedLifecycleSettlementCapability(
runtime: FederationRuntime,
@@ -4,9 +4,9 @@ import {
type MessagePriority,
type MessageType,
type WorkerReportOutcome
} from '../../db/orchestration-records'
import { OrchestrationError } from '../../orchestration-error'
import { parseFederatedWorkerReportPayload } from '../relay/federation-worker-report-payload'
} from '../db/orchestration-records'
import { OrchestrationError } from '../orchestration-error'
import { parseFederatedWorkerReportPayload } from './federation-worker-report-payload'
export type RelayedMessage = {
from: string
@@ -1,21 +1,21 @@
import { federationRelayPrefix } from '../relay/federation-relay-prefix'
import { federationRelayWireItem } from '../relay/federation-relay-wire'
import type { FederationRuntime } from '../federation-runtime'
import type { FederatedLifecycleSettlement } from '../federation-lifecycle-settlement'
import '../../../../../shared/protocol-version'
import { ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION } from '../../../../../shared/orchestration/orchestration-runtime-capabilities'
import { OrchestrationError } from '../../orchestration-error'
import { federationRelayPrefix } from '../federation/relay/federation-relay-prefix'
import { federationRelayWireItem } from '../legacy-remote/federation-relay-wire'
import type { FederationRuntime } from '../federation/federation-runtime'
import type { FederatedLifecycleSettlement } from '../legacy-remote/federation-lifecycle-settlement'
import '../../../../shared/protocol-version'
import { ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION } from '../../../../shared/orchestration/orchestration-runtime-capabilities'
import { OrchestrationError } from '../orchestration-error'
import {
acquireFederationAckLease,
getFederationAckedThrough,
recordFederationAckCheckpoint
} from './federation-ack-checkpoints'
import { bindCoordinatorMutationPayload } from '../../workers/reports/dispatch-message-binding'
import { bindCoordinatorMutationPayload } from '../workers/reports/dispatch-message-binding'
import {
armHomeAutoReleaseForOlderHost,
resolveFederatedLifecycleSettlementCapability
} from './federation-sync-capability'
import { getOrchestrationPeerCapabilityCache } from '../orchestration-peer-capability-cache'
import { getOrchestrationPeerCapabilityCache } from '../federation/orchestration-peer-capability-cache'
import {
parseFederatedLifecycle,
parseRelayedMessage,
@@ -1,5 +1,5 @@
import type { WorkerReportOutcome } from '../../db/orchestration-records'
import { OrchestrationError } from '../../orchestration-error'
import type { WorkerReportOutcome } from '../db/orchestration-records'
import { OrchestrationError } from '../orchestration-error'
export function parseFederatedWorkerReportPayload(payload: string | null): {
taskId: string
@@ -0,0 +1,21 @@
// The DB layer's only way into this folder; keep it free of rpc and runtime imports.
import { attachFederatedDispatchStore } from './federated-dispatch-store'
import type { FederatedDispatchStoreMethods } from './federated-dispatch-store'
import { attachFederatedHomeAutoReleaseStore } from './federated-home-auto-release-store'
import type { FederatedHomeAutoReleaseStoreMethods } from './federated-home-auto-release-store'
import { attachFederatedWorkerStartReconcile } from './federated-worker-start-reconcile'
import type { FederatedWorkerStartReconcileMethods } from './federated-worker-start-reconcile'
import { attachFederationRelayImport } from './federation-relay-import'
import type { FederationRelayImportMethods } from './federation-relay-import'
export type LegacyRemoteHomeDbMethods = FederatedWorkerStartReconcileMethods &
FederatedDispatchStoreMethods &
FederationRelayImportMethods &
FederatedHomeAutoReleaseStoreMethods
export function attachLegacyRemoteHomeDbMethods(ctor: { prototype: object }): void {
attachFederatedWorkerStartReconcile(ctor)
attachFederatedDispatchStore(ctor)
attachFederationRelayImport(ctor)
attachFederatedHomeAutoReleaseStore(ctor)
}
@@ -0,0 +1,18 @@
// Run-home half of today's remote attachments (`worker-start --on` drives a worker on a paired Orca).
// Frozen: bug fixes only. Code outside this folder imports it only through this module,
// except the DB layer (legacy-remote-home-db-methods.ts) and worker-list-projection.ts, which
// legacy starts load, so it imports federated-fleet-snapshot.ts directly to avoid an import cycle.
/** The one place a worker start leaves the local path: `worker-start --on <server>`. */
export { startFederatedWorker } from './federated-worker-start'
export { releaseFederatedWorkerAtHomeDeadline } from './federated-worker-home-auto-release'
export { readFederatedWorkerOutput } from './federated-worker-read'
export { releaseFederatedWorker, retainFederatedWorker } from './federated-worker-release'
export { showFederatedWorker } from './federated-worker-show'
export {
clearFederationAckCheckpoints,
releaseFederationAckCheckpoint
} from './federation-ack-checkpoints'
export { FederationAcknowledgmentRecovery } from './federation-acknowledgment-recovery'
export { syncFederatedDispatch } from './federation-sync'
export { federatedControlMailRefusal, sendFederatedControlMail } from './send-control-mail'
@@ -1,15 +1,12 @@
import type {
MessagePriority,
MessageType
} from '../../../../orchestration/db/orchestration-records'
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import { encodeFederatedControlMessage } from '../../../../orchestration/federation/relay/federation-control-message'
import { ORCHESTRATION_FEDERATION_CONTROL_MAIL_PROTOCOL_VERSION } from '../../../../../../shared/orchestration/orchestration-runtime-capabilities'
import { recordReceiptBeforeNudge } from './mutation-replay-nudge'
import type { SendParams } from './send-params'
import type { SendRecipientWarning } from './recipient-routing'
import type { MessagePriority, MessageType } from '../db/orchestration-records'
import type { OrchestrationDb } from '../db/orchestration-db'
import type { OrcaRuntimeService } from '../../orca-runtime'
import { OrchestrationError } from '../orchestration-error'
import { encodeFederatedControlMessage } from '../legacy-remote/federation-control-message'
import { ORCHESTRATION_FEDERATION_CONTROL_MAIL_PROTOCOL_VERSION } from '../../../../shared/orchestration/orchestration-runtime-capabilities'
import { recordReceiptBeforeNudge } from '../../rpc/methods/orchestration/messaging/mutation-replay-nudge'
import type { SendParams } from '../../rpc/methods/orchestration/messaging/send-params'
import type { SendRecipientWarning } from '../../rpc/methods/orchestration/messaging/recipient-routing'
import type { z } from 'zod'
type SendParamsInput = z.infer<typeof SendParams>
@@ -1,12 +1,12 @@
import type { ORCHESTRATION_WORKER_READ_SOURCES } from '../../../../../../shared/orchestration/workers/orchestration-worker-output'
import type { RuntimeTerminalRead } from '../../../../../../shared/runtime-terminal-contracts'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import type { ORCHESTRATION_WORKER_READ_SOURCES } from '../../../../shared/orchestration/workers/orchestration-worker-output'
import type { RuntimeTerminalRead } from '../../../../shared/runtime-terminal-contracts'
import { OrchestrationError } from '../orchestration-error'
import {
createWorkerOutputSourceIdentity,
decodeWorkerOutputCursor,
encodeWorkerOutputCursor
} from '../../../../orchestration/workers/transcripts/worker-output-cursor'
import type { resolvePinnedFederatedServer } from './worker-observation'
} from '../workers/transcripts/worker-output-cursor'
import type { resolvePinnedFederatedServer } from '../../rpc/methods/orchestration/worker/pinned-federated-server'
// Old peers expose only terminal reads; retain fences and cursor scope instead of silently degrading transcript cursors.
export async function readLegacyFederatedTerminal(args: {
@@ -1,14 +1,14 @@
import type { z } from 'zod'
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import type { WorkerDispatchState } from '../../../../orchestration/db/orchestration-records'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import { clampOrchestrationAskTimeoutMs } from '../../../../../../shared/orchestration/mailbox/orchestration-ask-timeout'
import type { AskParams } from '../../../../../../shared/rpc-contract/orchestration-params'
import { assertWorkerCanReport } from '../../../../orchestration/workers/reports/worker-report-admission'
import type { OrcaRuntimeService } from '../../orca-runtime'
import type { WorkerDispatchState } from '../db/orchestration-records'
import { OrchestrationError } from '../orchestration-error'
import { clampOrchestrationAskTimeoutMs } from '../../../../shared/orchestration/mailbox/orchestration-ask-timeout'
import type { AskParams } from '../../../../shared/rpc-contract/orchestration-params'
import { assertWorkerCanReport } from '../workers/reports/worker-report-admission'
import {
askedQuestionReceipt,
waitForQuestionAnswer
} from '../../../../orchestration/mailbox/questions/question-answer-delivery'
} from '../mailbox/questions/question-answer-delivery'
export async function askRemoteRunHome(args: {
params: z.infer<typeof AskParams>
@@ -1,9 +1,9 @@
import type { OrchestrationDb } from '../orchestration-db'
import type { OrchestrationDb } from '../db/orchestration-db'
import {
beginLifecycleWriteTransaction,
commitLifecycleWriteTransaction,
rollbackLifecycleWriteTransaction
} from '../lifecycle-transition'
} from '../db/lifecycle-transition'
export type FederatedDispatchObservationFence = {
dispatch_id: string
@@ -1,22 +1,22 @@
import type { RuntimeTerminalCreate } from '../../../../../../shared/runtime-terminal-create-contract'
import type { RuntimeTerminalCreate } from '../../../../shared/runtime-terminal-create-contract'
/** Execute terminalOnly on the receiving host with federation's agent-first worktree/background-terminal factories. The dispatch brief is delivered later. */
import type { AgentLaunchPreferences } from '../../../../../../shared/agent-session/agent-session-host-authority'
import type { TuiAgent } from '../../../../../../shared/agent-launch/catalog/tui-agent'
import { executeAgentLaunch } from '../../../../../agent-launch/agent-launch-executor'
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import { resolveUnattendedSetupDecision } from '../../../../../workspace-hooks/effective-hook-config'
import { WORKER_START_VOCABULARY } from '../structured-workers/orchestration-worker-start-mode'
import type { WorkerSetupReceipt } from '../worker/worker-topology'
import { resolveWorkerWorktreeName } from '../worker/worker-worktree-creation'
import type { AgentLaunchPreferences } from '../../../../shared/agent-session/agent-session-host-authority'
import type { TuiAgent } from '../../../../shared/agent-launch/catalog/tui-agent'
import { executeAgentLaunch } from '../../../agent-launch/agent-launch-executor'
import type { OrcaRuntimeService } from '../../orca-runtime'
import type { OrchestrationDb } from '../db/orchestration-db'
import { resolveUnattendedSetupDecision } from '../../../workspace-hooks/effective-hook-config'
import { WORKER_START_VOCABULARY } from '../../rpc/methods/orchestration/structured-workers/orchestration-worker-start-mode'
import type { WorkerSetupReceipt } from '../../rpc/methods/orchestration/worker/worker-topology'
import { resolveWorkerWorktreeName } from '../../rpc/methods/orchestration/worker/worker-worktree-creation'
import {
appendFederationSetupEffect,
appendFederationTerminalEffects,
type FederationEffect
} from './federation-effects'
import type { FederationAttachStartInput } from '../../../../../../shared/rpc-contract/orchestration-federation-start-params'
import type { Worktree } from '../../../../../../shared/worktree/types'
import type { FederationAttachStartInput } from '../../../../shared/rpc-contract/orchestration-federation-start-params'
import type { Worktree } from '../../../../shared/worktree/types'
const TERMINAL_ONLY_MESSAGE = 'A federated worker starts a terminal agent only.'
@@ -1,29 +1,29 @@
import { describeUnconfirmedAgentStop } from '../../../../../../shared/terminal/pty/lifecycle/pty-liveness-verdict'
import type { RemoteDispatchAttachmentRow } from '../../../../orchestration/db/orchestration-records'
import { describeUnconfirmedAgentStop } from '../../../../shared/terminal/pty/lifecycle/pty-liveness-verdict'
import type { RemoteDispatchAttachmentRow } from '../db/orchestration-records'
import type {
WorkerTerminalArchiveRow,
WorkerTerminalResourceRow,
WorkerTerminalRetainedReason
} from '../../../../orchestration/terminal-authority/release/worker-terminal-ownership'
} from '../terminal-authority/release/worker-terminal-ownership'
import {
captureWorkerOutputArchive,
releaseCommitFieldsForCapturedArchive,
releaseCommitFieldsForStoredArchive,
summarizeWorkerOutputArchive
} from '../../../../orchestration/workers/archives/worker-output-archive'
import type { WorkerTerminalReleaseRuntime } from '../../../../orchestration/terminal-authority/release/worker-terminal-release-runtime'
} from '../workers/archives/worker-output-archive'
import type { WorkerTerminalReleaseRuntime } from '../terminal-authority/release/worker-terminal-release-runtime'
import { readRemoteAttachmentArchive } from './remote-attachment-archive-read'
import { WorkerTerminalReleaseSingleFlight } from '../worker/worker-terminal-release-single-flight'
import { yieldAutoReleaseToWorkingAgent } from '../../../../orchestration/terminal-authority/release/worker-terminal-auto-release-working-yield'
import { archiveSummary } from '../worker/worker-terminal-resource-presentation'
import type { WorkerReleaseReceipt } from '../worker/worker-release-completion'
import { releaseUnknownRecovery } from '../worker/worker-release-unknown-recovery'
import { orchestrationTimestampToMs } from '../worker/worker-output'
import { WorkerTerminalReleaseSingleFlight } from '../../rpc/methods/orchestration/worker/worker-terminal-release-single-flight'
import { yieldAutoReleaseToWorkingAgent } from '../terminal-authority/release/worker-terminal-auto-release-working-yield'
import { archiveSummary } from '../../rpc/methods/orchestration/worker/worker-terminal-resource-presentation'
import type { WorkerReleaseReceipt } from '../../rpc/methods/orchestration/worker/worker-release-completion'
import { releaseUnknownRecovery } from '../../rpc/methods/orchestration/worker/worker-release-unknown-recovery'
import { orchestrationTimestampToMs } from '../../rpc/methods/orchestration/worker/worker-output'
import type { inspectRemoteAttachment } from './federation-attachment-observation'
import {
classifyWorkerTerminalCloseError,
TRANSIENT_WORKER_RELEASE_RECOVERY
} from '../worker/worker-release-close-error'
} from '../../rpc/methods/orchestration/worker/worker-release-close-error'
type RemoteAttachmentReleaseArgs = {
runtime: WorkerTerminalReleaseRuntime
@@ -1,4 +1,4 @@
import type { WorkerReportOutcome } from './orchestration-records'
import type { WorkerReportOutcome } from '../db/orchestration-records'
export function parseFederatedWorkerReportOutcome(
payload: string
@@ -1,7 +1,7 @@
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import type { WorkerStopReceipt } from '../../../../../../shared/orchestration/workers/orchestration-worker-stop-receipt'
import { archiveWorkerOutputBeforeStop } from '../worker/worker-stop-archive'
import { describeUnconfirmedAgentStop } from '../../../../../../shared/terminal/pty/lifecycle/pty-liveness-verdict'
import type { OrcaRuntimeService } from '../../orca-runtime'
import type { WorkerStopReceipt } from '../../../../shared/orchestration/workers/orchestration-worker-stop-receipt'
import { archiveWorkerOutputBeforeStop } from '../../rpc/methods/orchestration/worker/worker-stop-archive'
import { describeUnconfirmedAgentStop } from '../../../../shared/terminal/pty/lifecycle/pty-liveness-verdict'
import { inspectRemoteAttachment } from './federation-attachment-observation'
/** Close the exact attached worker this host runs for its home. */
@@ -1,7 +1,7 @@
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import type { OrcaRuntimeService } from '../../orca-runtime'
import type { OrchestrationDb } from '../db/orchestration-db'
import type { FederationEffect } from './federation-effects'
import type { WorkerSetupReceipt } from '../worker/worker-topology'
import type { WorkerSetupReceipt } from '../../rpc/methods/orchestration/worker/worker-topology'
export function prepareFederatedAttachmentAuthority(args: {
runtime: OrcaRuntimeService
@@ -1,9 +1,9 @@
import type { RuntimeTerminalInteractiveWait } from '../../../../../../shared/runtime-terminal-contracts'
import type { WorkerTerminalReleaseRuntime } from '../../../../orchestration/terminal-authority/release/worker-terminal-release-runtime'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import { parseWorkerTerminalHostScope } from '../../../../../../shared/orchestration/workers/worker-terminal-host-scope'
import type { RemoteDispatchAttachmentRow } from '../../../../orchestration/db/orchestration-records'
import { orchestrationTimestampToMs } from '../worker/worker-output'
import type { RuntimeTerminalInteractiveWait } from '../../../../shared/runtime-terminal-contracts'
import type { WorkerTerminalReleaseRuntime } from '../terminal-authority/release/worker-terminal-release-runtime'
import { OrchestrationError } from '../orchestration-error'
import { parseWorkerTerminalHostScope } from '../../../../shared/orchestration/workers/worker-terminal-host-scope'
import type { RemoteDispatchAttachmentRow } from '../db/orchestration-records'
import { orchestrationTimestampToMs } from '../../rpc/methods/orchestration/worker/worker-output'
export function requireHomeAttachment(
runtime: WorkerTerminalReleaseRuntime,
@@ -1,10 +1,6 @@
import {
MESSAGE_TYPES,
type MessagePriority,
type MessageType
} from '../../db/orchestration-records'
import type { OrchestrationDb } from '../../db/orchestration-db'
import { OrchestrationError } from '../../orchestration-error'
import { MESSAGE_TYPES, type MessagePriority, type MessageType } from '../db/orchestration-records'
import type { OrchestrationDb } from '../db/orchestration-db'
import { OrchestrationError } from '../orchestration-error'
const MESSAGE_TYPE_SET = new Set<MessageType>(MESSAGE_TYPES)
@@ -1,11 +1,11 @@
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import type { RemoteDispatchAttachmentRow } from '../../../../orchestration/db/orchestration-records'
import { defineMethod } from '../../../core'
import { isUserKeptWorkerTerminalOwner } from '../../../../orchestration/db/worker-terminal/worker-terminal-user-keep'
import { mapWithConcurrency } from '../../../../../../shared/map-with-concurrency'
import { readExactWorkerOutput } from '../worker/worker-output'
import { exposeWorkerRetainReceipt } from '../worker/worker-terminal-resource-presentation'
import { withStoppedHelpers } from '../worker/worker-stop'
import { OrchestrationError } from '../orchestration-error'
import type { RemoteDispatchAttachmentRow } from '../db/orchestration-records'
import { defineMethod } from '../../rpc/core'
import { isUserKeptWorkerTerminalOwner } from '../db/worker-terminal/worker-terminal-user-keep'
import { mapWithConcurrency } from '../../../../shared/map-with-concurrency'
import { readExactWorkerOutput } from '../../rpc/methods/orchestration/worker/worker-output'
import { exposeWorkerRetainReceipt } from '../../rpc/methods/orchestration/worker/worker-terminal-resource-presentation'
import { withStoppedHelpers } from '../../rpc/methods/orchestration/worker/worker-stop'
import { stopRemoteAttachment } from './federated-worker-stop-host'
import {
inspectRemoteAttachment,
@@ -19,7 +19,7 @@ import {
FederationFleetSnapshotParams,
FederationOutputReadParams,
FederationReadParams
} from '../../../../../../shared/rpc-contract/orchestration-federation-control-params'
} from '../../../../shared/rpc-contract/orchestration-federation-control-params'
export const ORCHESTRATION_FEDERATION_CONTROL_METHODS = [
defineMethod({
@@ -1,16 +1,16 @@
import { readAnswerSenderRelayReport } from '../../mailbox/senders/answer-sender-relay-report'
import { readAnswerSenderRelayReport } from '../mailbox/senders/answer-sender-relay-report'
import type {
WorkerReportOutcome,
FederationRelayDirection,
FederationRelayItemRow
} from '../orchestration-records'
import { OrchestrationError } from '../../orchestration-error'
import { generateId } from '../generated-id'
import type { OrchestrationDb } from '../orchestration-db'
} from '../db/orchestration-records'
import { OrchestrationError } from '../orchestration-error'
import { generateId } from '../db/generated-id'
import type { OrchestrationDb } from '../db/orchestration-db'
import {
isWorkerStateIn,
SETTLEABLE_WORKER_STATES
} from '../../workers/reports/worker-report-admission'
} from '../workers/reports/worker-report-admission'
export function enqueueFederationRelay(
this: OrchestrationDb,
@@ -2,13 +2,13 @@ import type {
WorkerReportOutcome,
FederationRelayDirection,
FederationRelayItemRow
} from '../orchestration-records'
import { OrchestrationError } from '../../orchestration-error'
import type { OrchestrationDb } from '../orchestration-db'
} from '../db/orchestration-records'
import { OrchestrationError } from '../orchestration-error'
import type { OrchestrationDb } from '../db/orchestration-db'
import {
isWorkerStateIn,
SETTLEABLE_WORKER_STATES
} from '../../workers/reports/worker-report-admission'
} from '../workers/reports/worker-report-admission'
export function getFederationRelayItem(
this: OrchestrationDb,
@@ -0,0 +1,139 @@
import type {
WorkerReportOutcome,
FederationRelayDirection,
FederationRelayItemRow
} from '../db/orchestration-records'
import { OrchestrationError } from '../orchestration-error'
import type { OrchestrationDb } from '../db/orchestration-db'
import { ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION } from '../../../../shared/orchestration/orchestration-runtime-capabilities'
import { parseFederatedWorkerReportOutcome } from './federated-worker-report-outcome'
export function listFederationRelay(
this: OrchestrationDb,
params: {
dispatchId: string
direction: FederationRelayDirection
afterSequence: number
limit?: number
}
): FederationRelayItemRow[] {
return this.db
.prepare(
`SELECT * FROM federation_relay_items
WHERE dispatch_id = ? AND direction = ? AND sequence > ?
ORDER BY sequence LIMIT ?`
)
.all(
params.dispatchId,
params.direction,
params.afterSequence,
Math.min(Math.max(params.limit ?? 50, 1), 50)
) as FederationRelayItemRow[]
}
/** A worker_done queued for the Run home that it has not acknowledged. */
export function hasPendingFederatedWorkerReport(
this: OrchestrationDb,
dispatchId: string
): boolean {
return (
this.db
.prepare(
`SELECT 1 FROM federation_relay_items
WHERE dispatch_id = ? AND direction = 'to_home' AND kind = 'worker_done'
AND acked_at IS NULL LIMIT 1`
)
.get(dispatchId) !== undefined
)
}
export type FederationRelayAcknowledgment = {
dispatchId: string
direction: FederationRelayDirection
throughSequence: number
settleRemoteReports?: { sequence: number; outcome?: WorkerReportOutcome }[]
// Only the home's acceptance proves settlement; arm after it, never at queue time.
armAutoRelease?: boolean
}
/** Runs inside acknowledgeFederationRelay's transaction, before the items are marked acked. */
export function settleRemoteReportsInRelayAcknowledgment(
this: OrchestrationDb,
params: FederationRelayAcknowledgment
): void {
const settledReports = params.settleRemoteReports ?? []
for (const settledReport of settledReports) {
const report = this.getFederationRelayItem(
params.dispatchId,
params.direction,
settledReport.sequence
)
if (
params.direction !== 'to_home' ||
settledReport.sequence > params.throughSequence ||
report?.kind !== 'worker_done' ||
(settledReport.outcome !== undefined &&
parseFederatedWorkerReportOutcome(report.payload) !== settledReport.outcome)
) {
throw new OrchestrationError(
'request_mismatch',
`Federation acknowledgment for ${params.dispatchId} does not match its queued worker_done.`
)
}
}
const attachment = this.getRemoteDispatchAttachment(params.dispatchId)
if (
params.direction === 'to_home' &&
attachment !== undefined &&
attachment.protocol_version >= ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION
) {
const acknowledgedReports = this.db
.prepare(
`SELECT sequence FROM federation_relay_items
WHERE dispatch_id = ? AND direction = 'to_home' AND kind = 'worker_done'
AND acked_at IS NULL AND sequence <= ?`
)
.all(params.dispatchId, params.throughSequence) as { sequence: number }[]
const settledSequences = new Set(settledReports.map((report) => report.sequence))
if (acknowledgedReports.some((report) => !settledSequences.has(report.sequence))) {
throw new OrchestrationError(
'request_mismatch',
`Federation acknowledgment for ${params.dispatchId} omits a worker_done settlement.`
)
}
}
const terminalOutcomes = new Set(
settledReports.flatMap((report) => (report.outcome ? [report.outcome] : []))
)
if (terminalOutcomes.size > 1) {
throw new OrchestrationError(
'request_mismatch',
`Federation acknowledgment for ${params.dispatchId} contains conflicting settlements.`
)
}
const terminalOutcome = settledReports.find((report) => report.outcome)?.outcome
if (terminalOutcome) {
this.settleRemoteAttachmentInRelayTransaction(
params.dispatchId,
terminalOutcome,
'worker_report_settled'
)
if (params.armAutoRelease) {
this.armWorkerTerminalAutoReleaseStatement(params.dispatchId)
}
}
}
export type FederationRelayReportAcknowledgmentMethods = {
listFederationRelay: typeof listFederationRelay
hasPendingFederatedWorkerReport: typeof hasPendingFederatedWorkerReport
settleRemoteReportsInRelayAcknowledgment: typeof settleRemoteReportsInRelayAcknowledgment
}
export function attachFederationRelayReportAcknowledgment(ctor: { prototype: object }): void {
Object.assign(ctor.prototype, {
listFederationRelay,
hasPendingFederatedWorkerReport,
settleRemoteReportsInRelayAcknowledgment
})
}
@@ -1,5 +1,5 @@
import type { FederationRelayItemRow } from '../../db/orchestration-records'
import { readAnswerSenderRelayReport } from '../../mailbox/senders/answer-sender-relay-report'
import type { FederationRelayItemRow } from '../db/orchestration-records'
import { readAnswerSenderRelayReport } from '../mailbox/senders/answer-sender-relay-report'
export function federationRelayWireItem(row: FederationRelayItemRow) {
const answerSenderReport = readAnswerSenderRelayReport(row.answer_sender_report)
@@ -1,19 +1,19 @@
import { federationRelayWireItem } from '../../../../orchestration/federation/relay/federation-relay-wire'
import { ORCHESTRATION_FEDERATION_CONTROL_MAIL_PROTOCOL_VERSION } from '../../../../../../shared/orchestration/orchestration-runtime-capabilities'
import { deliverQuestionAnswer } from '../../../../orchestration/mailbox/questions/question-answer-delivery'
import { importFederatedControlMessage } from '../../../../orchestration/federation/relay/federation-control-message'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import { federationRelayWireItem } from './federation-relay-wire'
import { ORCHESTRATION_FEDERATION_CONTROL_MAIL_PROTOCOL_VERSION } from '../../../../shared/orchestration/orchestration-runtime-capabilities'
import { deliverQuestionAnswer } from '../mailbox/questions/question-answer-delivery'
import { importFederatedControlMessage } from './federation-control-message'
import { OrchestrationError } from '../orchestration-error'
import {
areFederatedLifecycleSettlementsEqual,
publishFederatedLifecycleSettlement,
type FederatedLifecycleSettlement
} from '../../../../orchestration/federation/federation-lifecycle-settlement'
import { defineMethod, type RpcMethod } from '../../../core'
} from './federation-lifecycle-settlement'
import { defineMethod, type RpcMethod } from '../../rpc/core'
import {
FederationAckParams,
FederationImportParams,
FederationPullParams
} from '../../../../../../shared/rpc-contract/orchestration-federation-relay-params'
} from '../../../../shared/rpc-contract/orchestration-federation-relay-params'
export const ORCHESTRATION_FEDERATION_RELAY_METHODS = [
defineMethod({
@@ -1,6 +1,9 @@
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import { applyWaitForSetupOutcome, type WorkerSetupReceipt } from '../worker/worker-topology'
import type { OrcaRuntimeService } from '../../orca-runtime'
import type { OrchestrationDb } from '../db/orchestration-db'
import {
applyWaitForSetupOutcome,
type WorkerSetupReceipt
} from '../../rpc/methods/orchestration/worker/worker-topology'
import { isFederationResidualEffect, type FederationEffect } from './federation-effects'
type FederationSetupStageArgs = {
@@ -1,8 +1,8 @@
import { formatTerminalStartFailure } from '../../../../../../shared/terminal/pty/lifecycle/terminal-start-failure'
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import { formatTerminalStartFailure } from '../../../../shared/terminal/pty/lifecycle/terminal-start-failure'
import type { OrchestrationDb } from '../db/orchestration-db'
import { isFederationEffectUnknown } from './federation-effects'
import type { WorkerSetupReceipt } from '../worker/worker-topology'
import type { OrchestrationWorkerLaunchReceipt } from '../worker/worker-launch-preferences'
import type { WorkerSetupReceipt } from '../../rpc/methods/orchestration/worker/worker-topology'
import type { OrchestrationWorkerLaunchReceipt } from '../../rpc/methods/orchestration/worker/worker-launch-preferences'
export function failFederatedAttachmentWithReceipt(args: {
db: OrchestrationDb
@@ -1,32 +1,32 @@
import type { RuntimeTerminalCreate } from '../../../../../../shared/runtime-terminal-create-contract'
import { waitForWorkerAgentReady } from '../../../../terminal/wait/launched-agent-composer-readiness'
import { describeTerminalWaitBlockedReason } from '../../../../../../shared/terminal/runtime-policy/terminal-wait-blocked-reason-legacy-alias'
import { buildDispatchPreamble } from '../../../../orchestration/workers/launch/worker-dispatch-preamble'
import { sendAgentTurn } from '../../../../orchestration/turn-delivery/send-agent-turn'
import { createWorkerBriefWriteGuard } from '../../../../terminal/wait/launched-agent-write-guard'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import { defineMethod } from '../../../core'
import { assertOrchestrationWorktreeCreationSupported } from '../worker/folder-worktree-placement'
import type { RuntimeTerminalCreate } from '../../../../shared/runtime-terminal-create-contract'
import { waitForWorkerAgentReady } from '../../terminal/wait/launched-agent-composer-readiness'
import { describeTerminalWaitBlockedReason } from '../../../../shared/terminal/runtime-policy/terminal-wait-blocked-reason-legacy-alias'
import { buildDispatchPreamble } from '../workers/launch/worker-dispatch-preamble'
import { sendAgentTurn } from '../turn-delivery/send-agent-turn'
import { createWorkerBriefWriteGuard } from '../../terminal/wait/launched-agent-write-guard'
import { OrchestrationError } from '../orchestration-error'
import { defineMethod } from '../../rpc/core'
import { assertOrchestrationWorktreeCreationSupported } from '../../rpc/methods/orchestration/worker/folder-worktree-placement'
import type { FederationEffect } from './federation-effects'
import { launchFederatedWorkerAgent } from './federated-worker-agent-launch'
import type { WorkerSetupReceipt } from '../worker/worker-topology'
import type { WorkerSetupReceipt } from '../../rpc/methods/orchestration/worker/worker-topology'
import {
monitorFederatedSetup,
persistFederatedReadinessStage,
persistFederatedSetupSpawnFailure,
persistFederatedSetupWaitOutcome
} from './federation-setup'
import { FederationAttachStartParams } from '../../../../../../shared/rpc-contract/orchestration-federation-start-params'
import { FederationAttachStartParams } from '../../../../shared/rpc-contract/orchestration-federation-start-params'
import { failFederatedAttachmentWithReceipt } from './federation-start-receipt'
import { prepareFederationWorkerLaunchOnHost } from '../worker/worker-opencode-model-preflight'
import { prepareFederationWorkerLaunchOnHost } from '../../rpc/methods/orchestration/worker/worker-opencode-model-preflight'
import {
isWorkerStartTimeoutWithinTimerLimit,
resolveWorkerStartReadinessTimeoutMs
} from '../../../../../../shared/orchestration/orchestration-timing-budgets'
import { assertWorkerStartTaskSpecWithinPromptBudget } from '../worker/worker-start-prompt-budget'
} from '../../../../shared/orchestration/orchestration-timing-budgets'
import { assertWorkerStartTaskSpecWithinPromptBudget } from '../../rpc/methods/orchestration/worker/worker-start-prompt-budget'
import { prepareFederatedAttachmentAuthority } from './federation-attachment-authority'
import { resolveRpcWorkspaceCreatorProvenance } from '../../../workspace-creator-context'
import { claimTerminalForRemoteAttach } from '../../../../orchestration/db/worker-dispatch/worker-terminal-claim'
import { resolveRpcWorkspaceCreatorProvenance } from '../../rpc/workspace-creator-context'
import { claimTerminalForRemoteAttach } from '../db/worker-dispatch/worker-terminal-claim'
export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [
defineMethod({
@@ -0,0 +1,43 @@
// The DB layer's only way into this folder; keep it free of rpc and runtime imports.
import { attachFederatedDispatchObservationFence } from './federated-dispatch-observation-fence'
import type { FederatedDispatchObservationFenceMethods } from './federated-dispatch-observation-fence'
import { attachFederationRelayEnqueue } from './federation-relay-enqueue'
import type { FederationRelayEnqueueMethods } from './federation-relay-enqueue'
import { attachFederationRelayItem } from './federation-relay-item'
import type { FederationRelayItemMethods } from './federation-relay-item'
import { attachFederationRelayReportAcknowledgment } from './federation-relay-report-acknowledgment'
import type { FederationRelayReportAcknowledgmentMethods } from './federation-relay-report-acknowledgment'
import { attachRemoteDispatchAttachmentAuthority } from './remote-dispatch-attachment-authority'
import type { RemoteDispatchAttachmentAuthorityMethods } from './remote-dispatch-attachment-authority'
import { attachRemoteDispatchAttachmentCreate } from './remote-dispatch-attachment-create'
import type { RemoteDispatchAttachmentCreateMethods } from './remote-dispatch-attachment-create'
import { attachRemoteDispatchAttachmentRelease } from './remote-dispatch-attachment-release'
import type { RemoteDispatchAttachmentReleaseMethods } from './remote-dispatch-attachment-release'
import { attachRemoteDispatchAttachmentStop } from './remote-dispatch-attachment-stop'
import type { RemoteDispatchAttachmentStopMethods } from './remote-dispatch-attachment-stop'
import { attachRemoteQuestionStore } from './remote-question-store'
import type { RemoteQuestionStoreMethods } from './remote-question-store'
export type { FederationRelayAcknowledgment } from './federation-relay-report-acknowledgment'
export type LegacyRemoteDbMethods = FederatedDispatchObservationFenceMethods &
RemoteDispatchAttachmentCreateMethods &
RemoteDispatchAttachmentAuthorityMethods &
RemoteDispatchAttachmentReleaseMethods &
RemoteDispatchAttachmentStopMethods &
FederationRelayEnqueueMethods &
RemoteQuestionStoreMethods &
FederationRelayItemMethods &
FederationRelayReportAcknowledgmentMethods
export function attachLegacyRemoteDbMethods(ctor: { prototype: object }): void {
attachFederatedDispatchObservationFence(ctor)
attachRemoteDispatchAttachmentCreate(ctor)
attachRemoteDispatchAttachmentAuthority(ctor)
attachRemoteDispatchAttachmentRelease(ctor)
attachRemoteDispatchAttachmentStop(ctor)
attachFederationRelayEnqueue(ctor)
attachRemoteQuestionStore(ctor)
attachFederationRelayItem(ctor)
attachFederationRelayReportAcknowledgment(ctor)
}
@@ -0,0 +1,14 @@
// Worker-server half of today's remote attachments (a paired Orca drives a worker here).
// Frozen: bug fixes only. Code outside this folder imports it only through this module,
// except the DB layer, which uses legacy-remote-db-methods.ts.
/** The `orchestration.federation*` methods a paired Run home calls on this server. */
export { ORCHESTRATION_FEDERATION_ATTACH_METHODS } from './federation'
export { ORCHESTRATION_FEDERATION_CONTROL_METHODS } from './federation-control'
export { ORCHESTRATION_FEDERATION_RELAY_METHODS } from './federation-relay'
export { askRemoteRunHome } from './ask-remote'
export { releaseRemoteAttachment } from './federated-worker-release-host'
export { inspectRemoteAttachment } from './federation-attachment-observation'
export { federationRelayWireItem } from './federation-relay-wire'
export { replyToRemoteRunHome } from './reply-remote'
export { sendRemoteMessage } from './send-remote'
@@ -1,6 +1,6 @@
import type { RemoteDispatchAttachmentRow } from '../../../../orchestration/db/orchestration-records'
import type { WorkerTerminalReleaseRuntime } from '../../../../orchestration/terminal-authority/release/worker-terminal-release-runtime'
import { readArchivedWorkerOutput } from '../worker/worker-archive-read'
import type { RemoteDispatchAttachmentRow } from '../db/orchestration-records'
import type { WorkerTerminalReleaseRuntime } from '../terminal-authority/release/worker-terminal-release-runtime'
import { readArchivedWorkerOutput } from '../../rpc/methods/orchestration/worker/worker-archive-read'
export async function readRemoteAttachmentArchive(args: {
runtime: WorkerTerminalReleaseRuntime
@@ -1,7 +1,7 @@
import type { RemoteDispatchAttachmentRow } from '../orchestration-records'
import { OrchestrationError } from '../../orchestration-error'
import { isEquivalentPaneKey } from '../pane-key-match'
import type { OrchestrationDb } from '../orchestration-db'
import type { RemoteDispatchAttachmentRow } from '../db/orchestration-records'
import { OrchestrationError } from '../orchestration-error'
import { isEquivalentPaneKey } from '../db/pane-key-match'
import type { OrchestrationDb } from '../db/orchestration-db'
export function prepareRemoteAttachmentAuthority(
this: OrchestrationDb,
@@ -1,10 +1,10 @@
import type { WorkerDispatchState, RemoteDispatchAttachmentRow } from '../orchestration-records'
import { OrchestrationError } from '../../orchestration-error'
import { ensureMutationReceiptCapacity } from '../mutation-receipts/mutation-receipt-capacity'
import type { OrchestrationDb } from '../orchestration-db'
import { federatedStubHomeRunId } from '../contract-constants'
import { insertRemoteDispatchAttachmentRow } from '../dispatch-row-writer'
import { REMOTE_DISPATCH_ATTACHMENT_COLUMN_LIST } from '../row-column-lists'
import type { WorkerDispatchState, RemoteDispatchAttachmentRow } from '../db/orchestration-records'
import { OrchestrationError } from '../orchestration-error'
import { ensureMutationReceiptCapacity } from '../db/mutation-receipts/mutation-receipt-capacity'
import type { OrchestrationDb } from '../db/orchestration-db'
import { federatedStubHomeRunId } from '../db/contract-constants'
import { insertRemoteDispatchAttachmentRow } from '../db/dispatch-row-writer'
import { REMOTE_DISPATCH_ATTACHMENT_COLUMN_LIST } from '../db/row-column-lists'
export function createRemoteDispatchAttachment(
this: OrchestrationDb,
@@ -4,9 +4,9 @@ import {
WORKER_TERMINAL_RELEASABLE_ROW_SQL,
type WorkerTerminalResourceRow,
type WorkerTerminalRetainedReason
} from '../../terminal-authority/release/worker-terminal-ownership'
import { OrchestrationError } from '../../orchestration-error'
import type { OrchestrationDb } from '../orchestration-db'
} from '../terminal-authority/release/worker-terminal-ownership'
import { OrchestrationError } from '../orchestration-error'
import type { OrchestrationDb } from '../db/orchestration-db'
export function requestRemoteAttachmentTerminalRelease(
this: OrchestrationDb,
@@ -1,11 +1,14 @@
import { parsePaneKey } from '../../../../../shared/workspace-layout/stable-pane-id'
import type { RemoteDispatchAttachmentRow } from '../orchestration-records'
import { OrchestrationError } from '../../orchestration-error'
import { paneKeyMatchSuffix, REMOTE_ATTACHMENT_PANE_KEY_MATCH_SUFFIX_SQL } from '../pane-key-match'
import type { OrchestrationDb } from '../orchestration-db'
import { potentiallyLiveRemoteAttachmentSql } from './remote-attachment-liveness'
import { REMOTE_DISPATCH_ATTACHMENT_COLUMN_LIST } from '../row-column-lists'
import { refuseStopWhileKeptOpen } from '../worker-terminal/worker-terminal-user-keep'
import { parsePaneKey } from '../../../../shared/workspace-layout/stable-pane-id'
import type { RemoteDispatchAttachmentRow } from '../db/orchestration-records'
import { OrchestrationError } from '../orchestration-error'
import {
paneKeyMatchSuffix,
REMOTE_ATTACHMENT_PANE_KEY_MATCH_SUFFIX_SQL
} from '../db/pane-key-match'
import type { OrchestrationDb } from '../db/orchestration-db'
import { potentiallyLiveRemoteAttachmentSql } from '../db/federation/remote-attachment-liveness'
import { REMOTE_DISPATCH_ATTACHMENT_COLUMN_LIST } from '../db/row-column-lists'
import { refuseStopWhileKeptOpen } from '../db/worker-terminal/worker-terminal-user-keep'
export function beginRemoteAttachmentStop(
this: OrchestrationDb,
@@ -1,6 +1,6 @@
import { importedAnswerSenderSnapshot } from '../../mailbox/senders/answer-sender-relay-report'
import { OrchestrationError } from '../../orchestration-error'
import type { OrchestrationDb } from '../orchestration-db'
import { importedAnswerSenderSnapshot } from '../mailbox/senders/answer-sender-relay-report'
import { OrchestrationError } from '../orchestration-error'
import type { OrchestrationDb } from '../db/orchestration-db'
export function getRemoteQuestion(
this: OrchestrationDb,
@@ -1,7 +1,7 @@
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import type { MessageRow } from '../../../../orchestration/db/orchestration-records'
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import { remoteRunHomeUnreachable } from '../routing'
import type { OrchestrationDb } from '../db/orchestration-db'
import type { MessageRow } from '../db/orchestration-records'
import type { OrcaRuntimeService } from '../../orca-runtime'
import { remoteRunHomeUnreachable } from '../../rpc/methods/orchestration/routing'
import { sendRemoteMessage } from './send-remote'
/** Reply to mirrored remote-Run mail only at its home, through this pane's own Dispatch link. */
@@ -1,20 +1,17 @@
import type {
MessageType,
WorkerDispatchState
} from '../../../../orchestration/db/orchestration-records'
import type { OrchestrationDb } from '../../../../orchestration/db/orchestration-db'
import type { MessageType, WorkerDispatchState } from '../db/orchestration-records'
import type { OrchestrationDb } from '../db/orchestration-db'
import type { OrcaRuntimeService } from '../../../../orca-runtime'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import { waitForFederatedLifecycleSettlement } from '../../../../orchestration/federation/federation-lifecycle-settlement'
import { bindCoordinatorMutationPayload } from '../../../../orchestration/workers/reports/dispatch-message-binding'
import { ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION } from '../../../../../../shared/orchestration/orchestration-runtime-capabilities'
import type { OrcaRuntimeService } from '../../orca-runtime'
import { OrchestrationError } from '../orchestration-error'
import { waitForFederatedLifecycleSettlement } from './federation-lifecycle-settlement'
import { bindCoordinatorMutationPayload } from '../workers/reports/dispatch-message-binding'
import { ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION } from '../../../../shared/orchestration/orchestration-runtime-capabilities'
import type { z } from 'zod'
import { parseRemoteWorkerPayload } from './message-payload'
import type { SendParams } from './send-params'
import { rejectFederatedExplicitTarget } from '../routing'
import { WORKER_DONE_RECORDED_NEXT } from '../receipt-next'
import { assertWorkerCanReport } from '../../../../orchestration/workers/reports/worker-report-admission'
import { parseRemoteWorkerPayload } from '../../rpc/methods/orchestration/messaging/message-payload'
import type { SendParams } from '../../rpc/methods/orchestration/messaging/send-params'
import { rejectFederatedExplicitTarget } from '../../rpc/methods/orchestration/routing'
import { WORKER_DONE_RECORDED_NEXT } from '../../rpc/methods/orchestration/receipt-next'
import { assertWorkerCanReport } from '../workers/reports/worker-report-admission'
type SendParamsInput = z.infer<typeof SendParams>
@@ -1,6 +1,6 @@
import type { WorkerTerminalReleaseRuntime } from './worker-terminal-release-runtime'
import { reconcileRequestedWorkerTerminalReleases } from './worker-terminal-release-reconciliation'
import { releaseFederatedWorkerAtHomeDeadline } from '../../../rpc/methods/orchestration/federation/federated-worker-home-auto-release'
import { releaseFederatedWorkerAtHomeDeadline } from '../../legacy-remote-home/legacy-remote-home'
const MAX_TIMER_DELAY_MS = 2_147_483_647
@@ -1,6 +1,5 @@
import type { WorkerTerminalReleaseRuntime } from './worker-terminal-release-runtime'
import { inspectRemoteAttachment } from '../../../rpc/methods/orchestration/federation/federation-attachment-observation'
import { releaseRemoteAttachment } from '../../../rpc/methods/orchestration/federation/federated-worker-release-host'
import { inspectRemoteAttachment, releaseRemoteAttachment } from '../../legacy-remote/legacy-remote'
import { completeWorkerTerminalRelease } from '../../../rpc/methods/orchestration/worker/worker-release-completion'
export type WorkerTerminalReleaseReconciliationResult = {
@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest'
import { parseRemoteFederatedWorkerStartReceipt } from './federated-attach-receipt'
import { parseRemoteFederatedWorkerStartReceipt } from '../../../../orchestration/legacy-remote-home/federated-attach-receipt'
describe('remote federated worker start receipt', () => {
it.each([
@@ -7,10 +7,10 @@ import { OrcaRuntimeService } from '../../../../orca-runtime'
import type { FederatedDispatchRow } from '../../../../orchestration/db/orchestration-records'
import './federated-worker-read'
import { releaseFederatedWorker } from './federated-worker-release'
import '../../../../orchestration/legacy-remote-home/federated-worker-read'
import { releaseFederatedWorker } from '../../../../orchestration/legacy-remote-home/federated-worker-release'
import '../worker/worker-observation'
import '../../../../orchestration/federation/synchronization/federation-sync'
import '../../../../orchestration/legacy-remote-home/federation-sync'
import { ORCHESTRATION_WORKER_STOP_METHODS } from '../worker/worker-stop'
const server = {
@@ -3,7 +3,7 @@ import {
appendFederationSetupEffect,
appendFederationTerminalEffects,
type FederationEffect
} from './federation-effects'
} from '../../../../orchestration/legacy-remote/federation-effects'
describe('orchestration federation effects', () => {
it('uses exact terminal handles instead of display titles for setup identity', () => {
@@ -12,7 +12,7 @@ import '../../../../orchestration/db/messages/mailbox-routing-page'
import '../../../../orchestration/db/dispatch-row-writer'
import '../../../../orchestration/db/attempt-outcome-projection'
import type { OrchestrationEnvironmentTransport } from '../../../../orchestration/federation/environment-transport'
import '../../../../orchestration/federation/federation-lifecycle-settlement'
import '../../../../orchestration/legacy-remote/federation-lifecycle-settlement'
import { RpcDispatcher } from '../../../dispatcher'
import { ORCHESTRATION_METHODS } from '../orchestration'
import { createFederationWorkerStartRequest as startRequest } from './federation-request.test-support'
@@ -1,6 +1,8 @@
import { ORCHESTRATION_FEDERATION_CONTROL_METHODS } from './federation-control'
import { ORCHESTRATION_FEDERATION_RELAY_METHODS } from './federation-relay'
import { ORCHESTRATION_FEDERATION_ATTACH_METHODS } from './federation'
import {
ORCHESTRATION_FEDERATION_ATTACH_METHODS,
ORCHESTRATION_FEDERATION_CONTROL_METHODS,
ORCHESTRATION_FEDERATION_RELAY_METHODS
} from '../../../../orchestration/legacy-remote/legacy-remote'
import { ORCHESTRATION_PEER_MAIL_METHODS } from './peer-mail-methods'
export const ORCHESTRATION_FEDERATION_METHODS = [
@@ -6,7 +6,7 @@ import '../../../../orchestration/db/messages/mailbox-routing-page'
import '../../../../orchestration/db/dispatch-row-writer'
import '../../../../orchestration/db/attempt-outcome-projection'
import { ORCHESTRATION_METHODS } from '../orchestration'
import { monitorFederatedSetup } from './federation-setup'
import { monitorFederatedSetup } from '../../../../orchestration/legacy-remote/federation-setup'
describe('orchestration federated setup evidence', () => {
const databases: OrchestrationDb[] = []
@@ -5,7 +5,7 @@ import { isGroupAddress } from '../../../../orchestration/addressing/groups/grou
import { dispatchReportMailbox } from '../../../../orchestration/workers/reports/dispatch-report-mailbox'
import { AskParams } from '../../../../../../shared/rpc-contract/orchestration-params'
import { rejectFederatedExplicitTarget } from '../routing'
import { askRemoteRunHome } from './ask-remote'
import { askRemoteRunHome } from '../../../../orchestration/legacy-remote/legacy-remote'
import {
mailboxAddressOf,
runCoordinatorKey
@@ -13,7 +13,7 @@ import { OrcaRuntimeService } from '../../../../orca-runtime'
import {
encodeFederatedControlMessage,
importFederatedControlMessage
} from '../../../../orchestration/federation/relay/federation-control-message'
} from '../../../../orchestration/legacy-remote/federation-control-message'
const DISPATCH_ID = 'ctx_federated_worker_1'
const WORKER_HANDLE = 'term_federated_worker'
@@ -13,7 +13,7 @@ import { exposeMessage } from './mailbox-message-receipt'
import { recordReceiptBeforeNudge, replayMutationNudge } from './mutation-replay-nudge'
import { resolveReplyRecipient } from './recipient-routing'
import { helperQuestionAnswerer } from './helper-question-answerer'
import { replyToRemoteRunHome } from './reply-remote'
import { replyToRemoteRunHome } from '../../../../orchestration/legacy-remote/legacy-remote'
import { replyOverPeerMail } from './peer-mail-reply'
import { withAnswerNext } from '../receipt-next'
import { ReplyParams } from '../../../../../../shared/rpc-contract/orchestration-params'
@@ -23,11 +23,11 @@ import {
} from '../../../orchestration-mutation-receipt'
import { replayMutationNudge } from './mutation-replay-nudge'
import { handBackReplayTargets } from '../../../../orchestration/db/messages/helper-mailbox-hand-back'
import { sendRemoteMessage } from './send-remote'
import { sendRemoteMessage } from '../../../../orchestration/legacy-remote/legacy-remote'
import { mayNameSession } from './session-recipient'
import { sendPointToPointMessage } from './send-point-to-point'
import { sendGroupMessage } from './send-group'
import { sendFederatedControlMail } from './send-control-mail'
import { sendFederatedControlMail } from '../../../../orchestration/legacy-remote-home/legacy-remote-home'
import { orchestrationCallerIdentity } from '../runs/run-scope'
import { assertLifecycleCallerIsNotAnotherParty } from './lifecycle-caller-fence'
import { planWorkerFollowUp, startWorkerFollowUp } from './worker-follow-up'
@@ -13,7 +13,7 @@ import '../../../../orchestration/db/attempt-outcome-projection'
import { RpcDispatcher } from '../../../dispatcher'
import { ORCHESTRATION_METHODS } from '../orchestration'
import { startFederatedWorker } from '../federation/federated-worker-start'
import { startFederatedWorker } from '../../../../orchestration/legacy-remote-home/federated-worker-start'
import { createRootDispatch } from '../../../../orchestration/db/root-dispatch-test-fixture'
describe('orchestration migration behavior', () => {
@@ -17,8 +17,10 @@ import { sessionIdFromStructuredWorkerIncarnation } from '../../../../orchestrat
import { readExactWorkerOutput } from './worker-output'
import { exposeWorkerTerminalResource } from './worker-terminal-resource-presentation'
import { liveWorkerTerminalAutoReleaseAt } from '../../../../orchestration/terminal-authority/release/worker-terminal-ownership'
import { readFederatedWorkerOutput } from '../federation/federated-worker-read'
import { showFederatedWorker } from '../federation/federated-worker-show'
import {
readFederatedWorkerOutput,
showFederatedWorker
} from '../../../../orchestration/legacy-remote-home/legacy-remote-home'
import {
WorkerDispatchParams,
WorkerReadParams
@@ -14,7 +14,7 @@ import { createSessionWorkerLivenessReader } from '../../../../orchestration/wor
import {
applyFederatedFleetObservations,
readFederatedFleetSnapshots
} from '../federation/federated-fleet-snapshot'
} from '../../../../orchestration/legacy-remote-home/federated-fleet-snapshot'
import { ensureSessionWorkerRecords } from './session-worker-observation'
export type WorkerListPageParams = {
@@ -3,7 +3,7 @@ import { defineMethod } from '../../../core'
import {
releaseFederatedWorker,
retainFederatedWorker
} from '../federation/federated-worker-release'
} from '../../../../orchestration/legacy-remote-home/legacy-remote-home'
import { ORCHESTRATION_WORKER_LIST_METHOD } from './worker-list-method'
import { resolvePinnedFederatedServer } from './worker-observation'
import { archiveSummary, exposeWorkerRetainReceipt } from './worker-terminal-resource-presentation'
@@ -2,7 +2,7 @@ import { stopHelperAfterStart } from './worker-stop'
import { OrchestrationError } from '../../../../orchestration/orchestration-error'
import type { z } from 'zod'
import { defineMethod, type RpcContext } from '../../../core'
import { startFederatedWorker } from '../federation/federated-worker-start'
import { startFederatedWorker } from '../../../../orchestration/legacy-remote-home/legacy-remote-home'
import { startLocalWorker } from './local-worker-start'
import type { WorkerFollowUp } from './worker-follow-up-placement'
import { withWorkerStartNext } from '../receipt-next'
@@ -18,7 +18,7 @@ import { OrcaRuntimeRpcServer } from './runtime-rpc'
import { DeviceRegistry } from './pairing/device-registry'
import { sendRequest, withCurrentOrchestrationContract } from './runtime-rpc-test-harness'
import { STATUS_METHODS } from '../methods/host-management/status'
import { ORCHESTRATION_FEDERATION_CONTROL_METHODS } from '../methods/orchestration/federation/federation-control'
import { ORCHESTRATION_FEDERATION_CONTROL_METHODS } from '../../orchestration/legacy-remote/federation-control'
import { ORCHESTRATION_RESET_METHODS } from '../methods/orchestration/runs/reset-methods'
import { SESSION_TAB_METHODS } from '../methods/session-tabs/session-tabs'
import { FILE_MUTATION_METHODS } from '../methods/files/files-mutation-methods'