fix(orchestration): fence a worker settled while its tab is still open

This commit is contained in:
Jinwoo-H
2026-09-04 04:27:26 -04:00
parent 2736a72edd
commit d59745a87b
8 changed files with 133 additions and 9 deletions
@@ -24,7 +24,10 @@ const FENCE_SWEEPING_METHOD_NAMES = new Set([
'orchestration.workerRelease',
'orchestration.workerRetain',
'orchestration.workerStop',
'orchestration.workerAbandon'
'orchestration.workerAbandon',
// Reusing a settled worker's pane for a new Dispatch drops the old row from the plan; without
// this the stale fence stays on the pane it just relaunched into.
'orchestration.workerStart'
])
export function sweepingSettledWorkerResumeFences(method: RpcMethod): RpcMethod {
@@ -23,6 +23,10 @@ export class RuntimeLegacyWorkerTerminalRecoveryPersistence {
private readonly notifyFenceChanged?: (paneKey: string, blocked: boolean) => void
) {}
/** Panes announced as fenced before any sleeping record existed; the only place a lift for one
* can come from, because `liftRetiredFences` can only see panes that already have a record. */
private readonly announcedBlockedPaneKeys = new Set<string>()
prepare(): LegacyWorkerTerminalRecoveryPlan {
const plan = this.getPlan()
if (!plan) {
@@ -45,6 +49,12 @@ export class RuntimeLegacyWorkerTerminalRecoveryPersistence {
const changedHostIds = new Set<ExecutionHostId>()
const fenceChanges: [string, boolean][] = []
for (const blocked of plan.blockedPanes) {
// A worker can settle while its tab is still open, so there is no sleeping record to stamp
// yet. Tell the live renderer anyway: it mints the record on close and must fence it there.
if (!this.announcedBlockedPaneKeys.has(blocked.paneKey)) {
this.announcedBlockedPaneKeys.add(blocked.paneKey)
fenceChanges.push([blocked.paneKey, true])
}
let hostIds: ExecutionHostId[]
try {
const hostId = this.getHostId(blocked.worktreeId)
@@ -82,14 +92,10 @@ export class RuntimeLegacyWorkerTerminalRecoveryPersistence {
[blocked.paneKey]: { ...record, automaticResumeBlockedBy: 'legacy-orchestration-worker' }
}
changedHostIds.add(hostId)
fenceChanges.push([blocked.paneKey, true])
}
}
this.liftRetiredFences(store, plan, sessions, changedHostIds, fenceChanges)
const changed = [...sessions].filter(([hostId]) => changedHostIds.has(hostId))
if (changed.length === 0) {
return plan
}
try {
for (const [hostId, state] of changed) {
store.setWorkspaceSession(state.next, hostId)
@@ -115,6 +121,12 @@ export class RuntimeLegacyWorkerTerminalRecoveryPersistence {
fenceChanges: [string, boolean][]
): void {
const blockedPaneKeys = new Set(plan.blockedPanes.map((blocked) => blocked.paneKey))
for (const paneKey of this.announcedBlockedPaneKeys) {
if (!blockedPaneKeys.has(paneKey)) {
this.announcedBlockedPaneKeys.delete(paneKey)
fenceChanges.push([paneKey, false])
}
}
for (const hostId of store.getWorkspaceSessionHostIds?.() ?? [LOCAL_EXECUTION_HOST_ID]) {
const staged = sessions.get(hostId)
const session = staged?.next ?? store.getWorkspaceSession?.(hostId)
@@ -36,7 +36,11 @@ describe('settled worker automatic-resume fence persistence', () => {
afterEach(() => db?.close())
function harness(onFenceChanged?: (paneKey: string, blocked: boolean) => void): {
function harness(
onFenceChanged?: (paneKey: string, blocked: boolean) => void,
/** False models a worker that settles while its tab is still open: no record to stamp yet. */
withSleepingRecord = true
): {
db: OrchestrationDb
taskId: string
dispatchId: string
@@ -45,7 +49,9 @@ describe('settled worker automatic-resume fence persistence', () => {
} {
const orchestrationDb = new OrchestrationDb(':memory:')
db = orchestrationDb
let session = sessionWithSleepingWorker()
let session = withSleepingRecord
? sessionWithSleepingWorker()
: (getDefaultWorkspaceSession() as WorkspaceSessionState)
const store = {
getWorkspaceSession: () => session,
setWorkspaceSession: (next: WorkspaceSessionState) => {
@@ -102,6 +108,25 @@ describe('settled worker automatic-resume fence persistence', () => {
expect(fenceChanges).toEqual([[PANE_KEY, true]])
})
it('announces the fence for a pane that has no sleeping record to stamp yet', () => {
const fenceChanges: [string, boolean][] = []
const h = harness((paneKey, blocked) => fenceChanges.push([paneKey, blocked]), false)
settle(h.db, h.taskId, h.dispatchId)
h.persistence.prepare()
expect(fenceChanges).toEqual([[PANE_KEY, true]])
const requested = h.db.requestWorkerTerminalRelease(h.dispatchId)
h.db.settleWorkerTerminalRelease((requested as { resource: { id: string } }).resource.id)
h.persistence.prepare()
// A fence the plan no longer claims must be lifted even with no record to read it from.
expect(fenceChanges).toEqual([
[PANE_KEY, true],
[PANE_KEY, false]
])
})
// The STA-4577 repro: worker_done, no release, restart, open the worktree — the pane still
// holds a resumable provider session and must not respawn `codex resume`.
it('fences a settled worker pane whose terminal was never released', () => {
@@ -0,0 +1,60 @@
import { describe, expect, it } from 'vitest'
import type { AgentStatusEntry } from '../../../../shared/agent-status-types'
import type { AppState } from '../types'
import { createTestStore, makeTab } from './store-test-helpers'
const NOW = 1_800_000_000_000
const PANE_KEY = 'tab-1:leaf-1'
function liveWorkerEntry(): AgentStatusEntry {
return {
state: 'working',
prompt: 'finish the task',
updatedAt: NOW,
stateStartedAt: NOW,
stateHistory: [],
agentType: 'codex',
paneKey: PANE_KEY,
tabId: 'tab-1',
worktreeId: 'wt-1',
providerSession: { key: 'session_id', id: 'session-1' }
}
}
// The worker settles while its tab is still open, so there is no sleeping record to stamp; the
// record is minted on close and used to arrive unfenced, respawning settled work on reopen.
describe('a resume fence that arrives before the sleeping record exists', () => {
it('carries the block onto the record minted after the tab closes', () => {
const store = createTestStore()
store.setState({
tabsByWorktree: { 'wt-1': [makeTab({ id: 'tab-1', worktreeId: 'wt-1' })] },
agentStatusByPaneKey: { [PANE_KEY]: liveWorkerEntry() }
} as Partial<AppState>)
store.getState().setSleepingAgentAutomaticResumeBlocked(PANE_KEY, true)
expect(store.getState().sleepingAgentSessionsByPaneKey[PANE_KEY]).toBeUndefined()
store.getState().captureAllSleepingAgentSessions('quit')
expect(store.getState().sleepingAgentSessionsByPaneKey[PANE_KEY]).toMatchObject({
paneKey: PANE_KEY,
automaticResumeBlockedBy: 'legacy-orchestration-worker'
})
})
it('mints an unfenced record once the runtime lifts the block', () => {
const store = createTestStore()
store.setState({
tabsByWorktree: { 'wt-1': [makeTab({ id: 'tab-1', worktreeId: 'wt-1' })] },
agentStatusByPaneKey: { [PANE_KEY]: liveWorkerEntry() }
} as Partial<AppState>)
store.getState().setSleepingAgentAutomaticResumeBlocked(PANE_KEY, true)
store.getState().setSleepingAgentAutomaticResumeBlocked(PANE_KEY, false)
store.getState().captureAllSleepingAgentSessions('quit')
expect(
store.getState().sleepingAgentSessionsByPaneKey[PANE_KEY]?.automaticResumeBlockedBy
).toBeUndefined()
})
})
@@ -111,6 +111,18 @@ export function createAgentStatusRecoveryActions(
setSleepingAgentAutomaticResumeBlocked: (paneKey, blocked) => {
set((s) => {
// The pane key is tracked even with no record: a worker settled while its tab was open
// is fenced before the record exists, and the record is only minted on close.
const wasBlocked = s.automaticResumeBlockedPaneKeys[paneKey] === true
let paneKeys = s.automaticResumeBlockedPaneKeys
if (blocked !== wasBlocked) {
paneKeys = { ...s.automaticResumeBlockedPaneKeys }
if (blocked) {
paneKeys[paneKey] = true
} else {
delete paneKeys[paneKey]
}
}
const current = s.sleepingAgentSessionsByPaneKey[paneKey]
if (
!current ||
@@ -118,7 +130,9 @@ export function createAgentStatusRecoveryActions(
? current.automaticResumeBlockedBy === 'legacy-orchestration-worker'
: current.automaticResumeBlockedBy === undefined)
) {
return s
return paneKeys === s.automaticResumeBlockedPaneKeys
? s
: { automaticResumeBlockedPaneKeys: paneKeys }
}
const next = { ...current }
if (blocked) {
@@ -127,6 +141,7 @@ export function createAgentStatusRecoveryActions(
delete next.automaticResumeBlockedBy
}
return {
automaticResumeBlockedPaneKeys: paneKeys,
sleepingAgentSessionsByPaneKey: {
...s.sleepingAgentSessionsByPaneKey,
[paneKey]: next
@@ -59,7 +59,11 @@ export function sleepingRecordFromEntry(args: {
: {}),
...(args.launchConfig ? { launchConfig: copyLaunchConfig(args.launchConfig) } : {}),
...(args.entry.interrupted ? { interrupted: true } : {}),
...(args.origin ? { origin: args.origin } : {})
...(args.origin ? { origin: args.origin } : {}),
// The worker can settle while the tab is open, so the fence arrives before this record exists.
...(args.state.automaticResumeBlockedPaneKeys?.[args.entry.paneKey]
? { automaticResumeBlockedBy: 'legacy-orchestration-worker' as const }
: {})
}
}
@@ -51,6 +51,10 @@ export type AgentStatusSlice = {
/** Durable agent sessions captured on sleep (not live rows); power the one-click CLI resume on wake. */
sleepingAgentSessionsByPaneKey: Record<string, SleepingAgentSessionRecord>
/** Panes the runtime fenced against automatic resume. Held separately because a worker can
* settle while its tab is open, before the sleeping record the fence belongs on exists. */
automaticResumeBlockedPaneKeys: Record<string, true>
/** Ephemeral launch snapshots keyed by pane; hook payloads lack Orca launch settings, so the renderer supplies them from startup. */
agentLaunchConfigByPaneKey: Record<string, AgentLaunchConfigRegistryEntry>
@@ -100,6 +100,7 @@ export const createAgentStatusSlice: StateCreator<AppState, [], [], AgentStatusS
transientClearedAgentStatusConnectionIds: {},
retainedAgentsByPaneKey: {},
sleepingAgentSessionsByPaneKey: {},
automaticResumeBlockedPaneKeys: {},
agentLaunchConfigByPaneKey: {},
retentionSuppressedPaneKeys: {},
recentlyClosedAgentStatusTabIds: {},