mirror of
https://github.com/stablyai/orca.git
synced 2026-10-24 08:03:58 +00:00
fix(orchestration): fence migrated coordinator identity
This commit is contained in:
@@ -2446,6 +2446,7 @@ export type OrchestrationCompatibilityCallerAuthority = Readonly<{
|
||||
terminalHandle: string
|
||||
processIncarnation: string
|
||||
launchTokenHash: string
|
||||
terminalProvenance?: 'current_runtime' | 'restored'
|
||||
}>
|
||||
|
||||
type RestoredOrchestrationAuthorityReceipt = Readonly<{
|
||||
@@ -14531,7 +14532,8 @@ export class OrcaRuntimeService {
|
||||
terminal.processIncarnation,
|
||||
claimedPaneKey,
|
||||
terminalHandle,
|
||||
launchTokenHash
|
||||
launchTokenHash,
|
||||
terminalProvenance
|
||||
)
|
||||
}
|
||||
const attestation = this.attestAgentHookCompatibilityAuthorityFn?.({
|
||||
@@ -14548,7 +14550,8 @@ export class OrcaRuntimeService {
|
||||
terminal.processIncarnation,
|
||||
attestation.paneKey,
|
||||
terminalHandle,
|
||||
launchTokenHash
|
||||
launchTokenHash,
|
||||
terminalProvenance
|
||||
)
|
||||
}
|
||||
|
||||
@@ -14557,14 +14560,16 @@ export class OrcaRuntimeService {
|
||||
processIncarnation: string,
|
||||
paneKey: string,
|
||||
terminalHandle: string,
|
||||
launchTokenHash: string
|
||||
launchTokenHash: string,
|
||||
terminalProvenance: 'current_runtime' | 'restored'
|
||||
): OrchestrationCompatibilityCallerAuthority {
|
||||
return Object.freeze({
|
||||
hostScope: Object.freeze({ ...terminal.hostScope }),
|
||||
paneKey,
|
||||
terminalHandle,
|
||||
processIncarnation,
|
||||
launchTokenHash
|
||||
launchTokenHash,
|
||||
terminalProvenance
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -70,7 +70,8 @@ describe('orchestration compatibility runtime authority', () => {
|
||||
paneKey: PANE_KEY,
|
||||
terminalHandle: 'term-1',
|
||||
processIncarnation: 'incarnation-1',
|
||||
launchTokenHash: TOKEN_HASH
|
||||
launchTokenHash: TOKEN_HASH,
|
||||
terminalProvenance: 'current_runtime'
|
||||
})
|
||||
expect(JSON.stringify(authority)).not.toContain(TOKEN)
|
||||
expect(
|
||||
|
||||
@@ -52,6 +52,14 @@ describe('Run coordinator authority migration', () => {
|
||||
hostScope: JSON.stringify({ kind: 'local', hostId: 'local' })
|
||||
})
|
||||
).toBe(false)
|
||||
expect(
|
||||
isCurrentRunCoordinator(db.getRun(run.id)!, {
|
||||
handle: 'term_old',
|
||||
paneKey: 'tab_replacement:11111111-1111-4111-8111-111111111111',
|
||||
processIncarnation: 'replacement:incarnation-1',
|
||||
hostScope: JSON.stringify({ kind: 'local', hostId: 'local' })
|
||||
})
|
||||
).toBe(false)
|
||||
|
||||
const rebound = db.bindRun({
|
||||
runId: run.id,
|
||||
|
||||
@@ -9,6 +9,9 @@ export type RunCoordinatorIdentity = {
|
||||
}
|
||||
|
||||
export function isCurrentRunCoordinator(run: RunRow, identity: RunCoordinatorIdentity): boolean {
|
||||
if (run.coordinator_authority_revision < 0) {
|
||||
return false
|
||||
}
|
||||
const hasProcessAuthority = Boolean(
|
||||
run.coordinator_process_incarnation || run.coordinator_host_scope
|
||||
)
|
||||
|
||||
@@ -210,6 +210,111 @@ describe('orchestration RPC methods', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('does not grant a migrated binding to a replacement incarnation on the same handle', async () => {
|
||||
setup(false)
|
||||
const pane = 'tab_old:11111111-1111-4111-8111-111111111111'
|
||||
vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(pane)
|
||||
vi.spyOn(runtime, 'getLiveTerminalPaneKey').mockReturnValue(pane)
|
||||
vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockReturnValue({
|
||||
runtimeId: 'runtime_test',
|
||||
terminalHandle: 'term_old',
|
||||
ptyId: 'pty_replacement',
|
||||
worktreeId: 'folder:workspace',
|
||||
processIncarnation: 'pty_replacement:incarnation-2',
|
||||
paneKey: pane,
|
||||
launchTokenHash: 'replacement-launch',
|
||||
hostScope: { kind: 'local', hostId: 'local' }
|
||||
})
|
||||
const run = db.createRun({
|
||||
objective: 'Migrated same-handle owner',
|
||||
coordinatorHandle: 'term_old',
|
||||
coordinatorPaneKey: pane
|
||||
})
|
||||
db.db.prepare('UPDATE runs SET coordinator_authority_revision = -1 WHERE id = ?').run(run.id)
|
||||
|
||||
await expect(
|
||||
call('orchestration.runUse', { id: run.id, from: 'term_old' })
|
||||
).rejects.toMatchObject({
|
||||
code: 'consumer_fenced',
|
||||
data: { effectsApplied: false, coordinatorStatus: 'live' }
|
||||
})
|
||||
expect(db.getRun(run.id)).toMatchObject({
|
||||
coordinator_handle: 'term_old',
|
||||
coordinator_process_incarnation: null,
|
||||
consumer_generation: run.consumer_generation
|
||||
})
|
||||
})
|
||||
|
||||
it('backfills an exact migrated handle only from restored process authority', async () => {
|
||||
setup(false)
|
||||
const pane = 'tab_old:11111111-1111-4111-8111-111111111111'
|
||||
const processIncarnation = 'pty_retained:incarnation-1'
|
||||
vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(pane)
|
||||
vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockReturnValue({
|
||||
runtimeId: 'runtime_test',
|
||||
terminalHandle: 'term_old',
|
||||
ptyId: 'pty_retained',
|
||||
worktreeId: 'folder:workspace',
|
||||
processIncarnation,
|
||||
paneKey: pane,
|
||||
launchTokenHash: null,
|
||||
hostScope: { kind: 'local', hostId: 'local' }
|
||||
})
|
||||
ctx = {
|
||||
runtime,
|
||||
orchestrationCompatibilityEvidence: {
|
||||
terminalHandle: 'term_old',
|
||||
paneKey: pane,
|
||||
launchToken: 'retained-launch'
|
||||
}
|
||||
}
|
||||
let attestedProcessIncarnation = 'pty_retained:incarnation-stale'
|
||||
let verificationCount = 0
|
||||
let dropAttestationAfterFirstVerification = false
|
||||
vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation(() => {
|
||||
verificationCount += 1
|
||||
return dropAttestationAfterFirstVerification && verificationCount > 1
|
||||
? null
|
||||
: {
|
||||
terminalHandle: 'term_old',
|
||||
paneKey: pane,
|
||||
processIncarnation: attestedProcessIncarnation,
|
||||
launchTokenHash: 'retained-launch-hash',
|
||||
hostScope: { kind: 'local', hostId: 'local' },
|
||||
terminalProvenance: 'restored'
|
||||
}
|
||||
})
|
||||
const run = db.createRun({
|
||||
objective: 'Migrated retained owner',
|
||||
coordinatorHandle: 'term_old',
|
||||
coordinatorPaneKey: pane
|
||||
})
|
||||
db.db.prepare('UPDATE runs SET coordinator_authority_revision = -1 WHERE id = ?').run(run.id)
|
||||
|
||||
await expect(
|
||||
call('orchestration.runUse', { id: run.id, from: 'term_old' })
|
||||
).rejects.toMatchObject({ code: 'consumer_fenced' })
|
||||
attestedProcessIncarnation = processIncarnation
|
||||
verificationCount = 0
|
||||
dropAttestationAfterFirstVerification = true
|
||||
|
||||
await expect(
|
||||
call('orchestration.runUse', { id: run.id, from: 'term_old' })
|
||||
).rejects.toMatchObject({ code: 'consumer_fenced' })
|
||||
verificationCount = 0
|
||||
dropAttestationAfterFirstVerification = false
|
||||
|
||||
const rebound = (await call('orchestration.runUse', {
|
||||
id: run.id,
|
||||
from: 'term_old'
|
||||
})) as { run: { consumer_generation: number; coordinator_process_incarnation: string } }
|
||||
|
||||
expect(rebound.run).toMatchObject({
|
||||
consumer_generation: run.consumer_generation,
|
||||
coordinator_process_incarnation: processIncarnation
|
||||
})
|
||||
})
|
||||
|
||||
it('recovers a migrated binding after the resolved incumbent process is proven exited', async () => {
|
||||
setup(false)
|
||||
const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111'
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
isCurrentRunCoordinator,
|
||||
type RunCoordinatorIdentity
|
||||
} from '../../orchestration/run-coordinator-authority'
|
||||
import { isEquivalentPaneKey } from '../../orchestration/db/pane-key-match'
|
||||
import {
|
||||
isCallerCurrentRunCoordinator,
|
||||
resolveRunCoordinatorIdentity
|
||||
@@ -101,9 +102,13 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [
|
||||
runtime,
|
||||
legacyCoordinatorAuthority,
|
||||
orchestrationCompatibilityEvidence,
|
||||
orchestrationCompatibilityCallerAuthority: callerAuthority
|
||||
orchestrationCompatibilityCallerAuthority: preflightCallerAuthority
|
||||
}
|
||||
) => {
|
||||
const callerAuthority =
|
||||
preflightCallerAuthority ??
|
||||
runtime.verifyOrchestrationCompatibilityCaller(orchestrationCompatibilityEvidence) ??
|
||||
undefined
|
||||
const paneKey = resolveOrchestrationCaller(runtime, {
|
||||
callerTerminalHandle: params.from,
|
||||
callerEvidence: orchestrationCompatibilityEvidence,
|
||||
@@ -142,8 +147,20 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [
|
||||
incumbentIdentity.processIncarnation === identity.processIncarnation &&
|
||||
incumbentIdentity.hostScope === identity.hostScope
|
||||
)
|
||||
const restoredMigratedContinuity = Boolean(
|
||||
targetRun &&
|
||||
targetRun.coordinator_authority_revision < 0 &&
|
||||
callerAuthority?.terminalProvenance === 'restored' &&
|
||||
callerAuthority.processIncarnation === identity.processIncarnation &&
|
||||
JSON.stringify(callerAuthority.hostScope) === identity.hostScope &&
|
||||
targetRun.coordinator_handle === params.from &&
|
||||
targetRun.coordinator_pane_key &&
|
||||
isEquivalentPaneKey(targetRun.coordinator_pane_key, paneKey)
|
||||
)
|
||||
const sameAuthority = targetRun
|
||||
? isCurrentRunCoordinator(targetRun, identity) || dynamicProcessContinuity
|
||||
? isCurrentRunCoordinator(targetRun, identity) ||
|
||||
dynamicProcessContinuity ||
|
||||
restoredMigratedContinuity
|
||||
: false
|
||||
const incumbentObservation =
|
||||
targetRun && !sameAuthority
|
||||
@@ -164,6 +181,15 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [
|
||||
params.takeoverLegacy ? { currentRuntimeLaunchSufficient: true } : undefined
|
||||
)
|
||||
: null
|
||||
const restoredContinuityStillAttested =
|
||||
!restoredMigratedContinuity ||
|
||||
Boolean(
|
||||
revalidatedCaller?.terminalProvenance === 'restored' &&
|
||||
revalidatedCaller.terminalHandle === params.from &&
|
||||
revalidatedCaller.paneKey === paneKey &&
|
||||
revalidatedCaller.processIncarnation === currentIdentity.processIncarnation &&
|
||||
JSON.stringify(revalidatedCaller.hostScope) === currentIdentity.hostScope
|
||||
)
|
||||
const claimantStillLive = revalidatedCaller
|
||||
? revalidatedCaller.terminalHandle === params.from && revalidatedCaller.paneKey === paneKey
|
||||
: legacyCoordinatorAuthority || !orchestrationCompatibilityEvidence
|
||||
@@ -171,6 +197,7 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [
|
||||
: false
|
||||
if (
|
||||
(incumbentObservation && !claimantStillLive) ||
|
||||
!restoredContinuityStillAttested ||
|
||||
currentPaneKey !== paneKey ||
|
||||
currentIdentity.processIncarnation !== identity.processIncarnation ||
|
||||
currentIdentity.hostScope !== identity.hostScope
|
||||
@@ -194,7 +221,7 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [
|
||||
coordinatorPaneKey: paneKey,
|
||||
coordinatorProcessIncarnation: identity.processIncarnation,
|
||||
coordinatorHostScope: identity.hostScope,
|
||||
authorityContinuity: dynamicProcessContinuity,
|
||||
authorityContinuity: dynamicProcessContinuity || restoredMigratedContinuity,
|
||||
incumbentObservation,
|
||||
takeoverLegacy: params.takeoverLegacy,
|
||||
legacyCoordinatorAuthority
|
||||
|
||||
Reference in New Issue
Block a user