mirror of
https://github.com/stablyai/orca.git
synced 2026-09-30 16:02:56 +00:00
fix(orchestration): fence a worker settled while its tab is still open
This commit is contained in:
@@ -24,7 +24,10 @@ const FENCE_SWEEPING_METHOD_NAMES = new Set([
|
||||
'orchestration.workerRelease',
|
||||
'orchestration.workerRetain',
|
||||
'orchestration.workerStop',
|
||||
'orchestration.workerAbandon'
|
||||
'orchestration.workerAbandon',
|
||||
// Reusing a settled worker's pane for a new Dispatch drops the old row from the plan; without
|
||||
// this the stale fence stays on the pane it just relaunched into.
|
||||
'orchestration.workerStart'
|
||||
])
|
||||
|
||||
export function sweepingSettledWorkerResumeFences(method: RpcMethod): RpcMethod {
|
||||
|
||||
@@ -23,6 +23,10 @@ export class RuntimeLegacyWorkerTerminalRecoveryPersistence {
|
||||
private readonly notifyFenceChanged?: (paneKey: string, blocked: boolean) => void
|
||||
) {}
|
||||
|
||||
/** Panes announced as fenced before any sleeping record existed; the only place a lift for one
|
||||
* can come from, because `liftRetiredFences` can only see panes that already have a record. */
|
||||
private readonly announcedBlockedPaneKeys = new Set<string>()
|
||||
|
||||
prepare(): LegacyWorkerTerminalRecoveryPlan {
|
||||
const plan = this.getPlan()
|
||||
if (!plan) {
|
||||
@@ -45,6 +49,12 @@ export class RuntimeLegacyWorkerTerminalRecoveryPersistence {
|
||||
const changedHostIds = new Set<ExecutionHostId>()
|
||||
const fenceChanges: [string, boolean][] = []
|
||||
for (const blocked of plan.blockedPanes) {
|
||||
// A worker can settle while its tab is still open, so there is no sleeping record to stamp
|
||||
// yet. Tell the live renderer anyway: it mints the record on close and must fence it there.
|
||||
if (!this.announcedBlockedPaneKeys.has(blocked.paneKey)) {
|
||||
this.announcedBlockedPaneKeys.add(blocked.paneKey)
|
||||
fenceChanges.push([blocked.paneKey, true])
|
||||
}
|
||||
let hostIds: ExecutionHostId[]
|
||||
try {
|
||||
const hostId = this.getHostId(blocked.worktreeId)
|
||||
@@ -82,14 +92,10 @@ export class RuntimeLegacyWorkerTerminalRecoveryPersistence {
|
||||
[blocked.paneKey]: { ...record, automaticResumeBlockedBy: 'legacy-orchestration-worker' }
|
||||
}
|
||||
changedHostIds.add(hostId)
|
||||
fenceChanges.push([blocked.paneKey, true])
|
||||
}
|
||||
}
|
||||
this.liftRetiredFences(store, plan, sessions, changedHostIds, fenceChanges)
|
||||
const changed = [...sessions].filter(([hostId]) => changedHostIds.has(hostId))
|
||||
if (changed.length === 0) {
|
||||
return plan
|
||||
}
|
||||
try {
|
||||
for (const [hostId, state] of changed) {
|
||||
store.setWorkspaceSession(state.next, hostId)
|
||||
@@ -115,6 +121,12 @@ export class RuntimeLegacyWorkerTerminalRecoveryPersistence {
|
||||
fenceChanges: [string, boolean][]
|
||||
): void {
|
||||
const blockedPaneKeys = new Set(plan.blockedPanes.map((blocked) => blocked.paneKey))
|
||||
for (const paneKey of this.announcedBlockedPaneKeys) {
|
||||
if (!blockedPaneKeys.has(paneKey)) {
|
||||
this.announcedBlockedPaneKeys.delete(paneKey)
|
||||
fenceChanges.push([paneKey, false])
|
||||
}
|
||||
}
|
||||
for (const hostId of store.getWorkspaceSessionHostIds?.() ?? [LOCAL_EXECUTION_HOST_ID]) {
|
||||
const staged = sessions.get(hostId)
|
||||
const session = staged?.next ?? store.getWorkspaceSession?.(hostId)
|
||||
|
||||
@@ -36,7 +36,11 @@ describe('settled worker automatic-resume fence persistence', () => {
|
||||
|
||||
afterEach(() => db?.close())
|
||||
|
||||
function harness(onFenceChanged?: (paneKey: string, blocked: boolean) => void): {
|
||||
function harness(
|
||||
onFenceChanged?: (paneKey: string, blocked: boolean) => void,
|
||||
/** False models a worker that settles while its tab is still open: no record to stamp yet. */
|
||||
withSleepingRecord = true
|
||||
): {
|
||||
db: OrchestrationDb
|
||||
taskId: string
|
||||
dispatchId: string
|
||||
@@ -45,7 +49,9 @@ describe('settled worker automatic-resume fence persistence', () => {
|
||||
} {
|
||||
const orchestrationDb = new OrchestrationDb(':memory:')
|
||||
db = orchestrationDb
|
||||
let session = sessionWithSleepingWorker()
|
||||
let session = withSleepingRecord
|
||||
? sessionWithSleepingWorker()
|
||||
: (getDefaultWorkspaceSession() as WorkspaceSessionState)
|
||||
const store = {
|
||||
getWorkspaceSession: () => session,
|
||||
setWorkspaceSession: (next: WorkspaceSessionState) => {
|
||||
@@ -102,6 +108,25 @@ describe('settled worker automatic-resume fence persistence', () => {
|
||||
expect(fenceChanges).toEqual([[PANE_KEY, true]])
|
||||
})
|
||||
|
||||
it('announces the fence for a pane that has no sleeping record to stamp yet', () => {
|
||||
const fenceChanges: [string, boolean][] = []
|
||||
const h = harness((paneKey, blocked) => fenceChanges.push([paneKey, blocked]), false)
|
||||
settle(h.db, h.taskId, h.dispatchId)
|
||||
|
||||
h.persistence.prepare()
|
||||
expect(fenceChanges).toEqual([[PANE_KEY, true]])
|
||||
|
||||
const requested = h.db.requestWorkerTerminalRelease(h.dispatchId)
|
||||
h.db.settleWorkerTerminalRelease((requested as { resource: { id: string } }).resource.id)
|
||||
h.persistence.prepare()
|
||||
|
||||
// A fence the plan no longer claims must be lifted even with no record to read it from.
|
||||
expect(fenceChanges).toEqual([
|
||||
[PANE_KEY, true],
|
||||
[PANE_KEY, false]
|
||||
])
|
||||
})
|
||||
|
||||
// The STA-4577 repro: worker_done, no release, restart, open the worktree — the pane still
|
||||
// holds a resumable provider session and must not respawn `codex resume`.
|
||||
it('fences a settled worker pane whose terminal was never released', () => {
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { AgentStatusEntry } from '../../../../shared/agent-status-types'
|
||||
import type { AppState } from '../types'
|
||||
import { createTestStore, makeTab } from './store-test-helpers'
|
||||
|
||||
const NOW = 1_800_000_000_000
|
||||
const PANE_KEY = 'tab-1:leaf-1'
|
||||
|
||||
function liveWorkerEntry(): AgentStatusEntry {
|
||||
return {
|
||||
state: 'working',
|
||||
prompt: 'finish the task',
|
||||
updatedAt: NOW,
|
||||
stateStartedAt: NOW,
|
||||
stateHistory: [],
|
||||
agentType: 'codex',
|
||||
paneKey: PANE_KEY,
|
||||
tabId: 'tab-1',
|
||||
worktreeId: 'wt-1',
|
||||
providerSession: { key: 'session_id', id: 'session-1' }
|
||||
}
|
||||
}
|
||||
|
||||
// The worker settles while its tab is still open, so there is no sleeping record to stamp; the
|
||||
// record is minted on close and used to arrive unfenced, respawning settled work on reopen.
|
||||
describe('a resume fence that arrives before the sleeping record exists', () => {
|
||||
it('carries the block onto the record minted after the tab closes', () => {
|
||||
const store = createTestStore()
|
||||
store.setState({
|
||||
tabsByWorktree: { 'wt-1': [makeTab({ id: 'tab-1', worktreeId: 'wt-1' })] },
|
||||
agentStatusByPaneKey: { [PANE_KEY]: liveWorkerEntry() }
|
||||
} as Partial<AppState>)
|
||||
|
||||
store.getState().setSleepingAgentAutomaticResumeBlocked(PANE_KEY, true)
|
||||
expect(store.getState().sleepingAgentSessionsByPaneKey[PANE_KEY]).toBeUndefined()
|
||||
|
||||
store.getState().captureAllSleepingAgentSessions('quit')
|
||||
|
||||
expect(store.getState().sleepingAgentSessionsByPaneKey[PANE_KEY]).toMatchObject({
|
||||
paneKey: PANE_KEY,
|
||||
automaticResumeBlockedBy: 'legacy-orchestration-worker'
|
||||
})
|
||||
})
|
||||
|
||||
it('mints an unfenced record once the runtime lifts the block', () => {
|
||||
const store = createTestStore()
|
||||
store.setState({
|
||||
tabsByWorktree: { 'wt-1': [makeTab({ id: 'tab-1', worktreeId: 'wt-1' })] },
|
||||
agentStatusByPaneKey: { [PANE_KEY]: liveWorkerEntry() }
|
||||
} as Partial<AppState>)
|
||||
|
||||
store.getState().setSleepingAgentAutomaticResumeBlocked(PANE_KEY, true)
|
||||
store.getState().setSleepingAgentAutomaticResumeBlocked(PANE_KEY, false)
|
||||
store.getState().captureAllSleepingAgentSessions('quit')
|
||||
|
||||
expect(
|
||||
store.getState().sleepingAgentSessionsByPaneKey[PANE_KEY]?.automaticResumeBlockedBy
|
||||
).toBeUndefined()
|
||||
})
|
||||
})
|
||||
@@ -111,6 +111,18 @@ export function createAgentStatusRecoveryActions(
|
||||
|
||||
setSleepingAgentAutomaticResumeBlocked: (paneKey, blocked) => {
|
||||
set((s) => {
|
||||
// The pane key is tracked even with no record: a worker settled while its tab was open
|
||||
// is fenced before the record exists, and the record is only minted on close.
|
||||
const wasBlocked = s.automaticResumeBlockedPaneKeys[paneKey] === true
|
||||
let paneKeys = s.automaticResumeBlockedPaneKeys
|
||||
if (blocked !== wasBlocked) {
|
||||
paneKeys = { ...s.automaticResumeBlockedPaneKeys }
|
||||
if (blocked) {
|
||||
paneKeys[paneKey] = true
|
||||
} else {
|
||||
delete paneKeys[paneKey]
|
||||
}
|
||||
}
|
||||
const current = s.sleepingAgentSessionsByPaneKey[paneKey]
|
||||
if (
|
||||
!current ||
|
||||
@@ -118,7 +130,9 @@ export function createAgentStatusRecoveryActions(
|
||||
? current.automaticResumeBlockedBy === 'legacy-orchestration-worker'
|
||||
: current.automaticResumeBlockedBy === undefined)
|
||||
) {
|
||||
return s
|
||||
return paneKeys === s.automaticResumeBlockedPaneKeys
|
||||
? s
|
||||
: { automaticResumeBlockedPaneKeys: paneKeys }
|
||||
}
|
||||
const next = { ...current }
|
||||
if (blocked) {
|
||||
@@ -127,6 +141,7 @@ export function createAgentStatusRecoveryActions(
|
||||
delete next.automaticResumeBlockedBy
|
||||
}
|
||||
return {
|
||||
automaticResumeBlockedPaneKeys: paneKeys,
|
||||
sleepingAgentSessionsByPaneKey: {
|
||||
...s.sleepingAgentSessionsByPaneKey,
|
||||
[paneKey]: next
|
||||
|
||||
@@ -59,7 +59,11 @@ export function sleepingRecordFromEntry(args: {
|
||||
: {}),
|
||||
...(args.launchConfig ? { launchConfig: copyLaunchConfig(args.launchConfig) } : {}),
|
||||
...(args.entry.interrupted ? { interrupted: true } : {}),
|
||||
...(args.origin ? { origin: args.origin } : {})
|
||||
...(args.origin ? { origin: args.origin } : {}),
|
||||
// The worker can settle while the tab is open, so the fence arrives before this record exists.
|
||||
...(args.state.automaticResumeBlockedPaneKeys?.[args.entry.paneKey]
|
||||
? { automaticResumeBlockedBy: 'legacy-orchestration-worker' as const }
|
||||
: {})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -51,6 +51,10 @@ export type AgentStatusSlice = {
|
||||
/** Durable agent sessions captured on sleep (not live rows); power the one-click CLI resume on wake. */
|
||||
sleepingAgentSessionsByPaneKey: Record<string, SleepingAgentSessionRecord>
|
||||
|
||||
/** Panes the runtime fenced against automatic resume. Held separately because a worker can
|
||||
* settle while its tab is open, before the sleeping record the fence belongs on exists. */
|
||||
automaticResumeBlockedPaneKeys: Record<string, true>
|
||||
|
||||
/** Ephemeral launch snapshots keyed by pane; hook payloads lack Orca launch settings, so the renderer supplies them from startup. */
|
||||
agentLaunchConfigByPaneKey: Record<string, AgentLaunchConfigRegistryEntry>
|
||||
|
||||
|
||||
@@ -100,6 +100,7 @@ export const createAgentStatusSlice: StateCreator<AppState, [], [], AgentStatusS
|
||||
transientClearedAgentStatusConnectionIds: {},
|
||||
retainedAgentsByPaneKey: {},
|
||||
sleepingAgentSessionsByPaneKey: {},
|
||||
automaticResumeBlockedPaneKeys: {},
|
||||
agentLaunchConfigByPaneKey: {},
|
||||
retentionSuppressedPaneKeys: {},
|
||||
recentlyClosedAgentStatusTabIds: {},
|
||||
|
||||
Reference in New Issue
Block a user