fix(runtime): make restart adoption session-exact

This commit is contained in:
Brennan Benson
2026-08-13 17:57:24 -07:00
parent fd45107ae8
commit dcbf5a0194
9 changed files with 306 additions and 61 deletions
+1
View File
@@ -1441,6 +1441,7 @@ export class DaemonPtyAdapter implements IPtyProvider {
admission.admit({
id: session.sessionId,
...(session.incarnationId ? { incarnationId: session.incarnationId } : {}),
...(session.pid ? { rootProcessId: session.pid } : {}),
// Why: OSC 7 may not arrive before cleanup; spawn cwd is authoritative until the daemon reports a live cwd.
cwd: session.cwd ?? this.initialCwds.get(session.sessionId) ?? '',
title: 'shell',
+2
View File
@@ -4,6 +4,8 @@ import type { PtyIncarnationId } from '../../shared/pty-incarnation'
export type PtyProcessInfo = {
id: string
incarnationId?: PtyIncarnationId
/** Root process owned by this exact PTY incarnation, when the provider can prove it. */
rootProcessId?: number
cwd: string
title: string
/** Owning worktree when the provider can report it authoritatively. */
@@ -60,6 +60,8 @@ export class PtyProcessListAdmission {
worktreeIdBytes === null ||
terminalHandleBytes === null ||
wslDistroBytes === null ||
(value.rootProcessId !== undefined &&
(!Number.isSafeInteger(value.rootProcessId) || value.rootProcessId <= 0)) ||
(value.incarnationId !== undefined && !isPtyIncarnationId(value.incarnationId)) ||
(value.agentSessionOwners !== undefined && !Array.isArray(value.agentSessionOwners))
) {
@@ -108,6 +110,7 @@ export class PtyProcessListAdmission {
cwd: value.cwd,
title: value.title,
...(value.incarnationId !== undefined ? { incarnationId: value.incarnationId } : {}),
...(value.rootProcessId !== undefined ? { rootProcessId: value.rootProcessId } : {}),
...(value.worktreeId !== undefined ? { worktreeId: value.worktreeId } : {}),
...(value.terminalHandle !== undefined ? { terminalHandle: value.terminalHandle } : {}),
...(value.wslDistro !== undefined ? { wslDistro: value.wslDistro } : {}),
@@ -1,4 +1,5 @@
import { describe, expect, it, vi } from 'vitest'
import { getDefaultWorkspaceSession } from '../../shared/constants'
import type { StructuredAgentSessionHandoffTransport } from '../native-chat/agent-session-wire/structured-agent-session-handoff-types'
import { createEphemeralAgentSessionClaimSigner } from './agent-session-claim-identity'
import { agentSessionPtyWriteGate } from './agent-session-pty-write-gate'
@@ -91,16 +92,50 @@ describe('structured TUI launch tab binding', () => {
})
const terminalHandle = 'term_current_runtime'
const leafId = '23013912-13f8-44e5-818f-d40a1ff4e8c5'
const conflictingLeafId = '33013912-13f8-44e5-818f-d40a1ff4e8c5'
resolvePinnedCodexRolloutProof.mockResolvedValue('/tmp/codex-home/sessions/thread-1.jsonl')
const writeAgentSessionProof = vi.fn(() => false)
const runtime = new OrcaRuntimeService(undefined, undefined, {
agentSessionClaimSigner: signer
})
const persistedSession = {
...getDefaultWorkspaceSession(),
tabsByWorktree: {
[WORKTREE_ID]: [
{
id: 'tab-cold-owner',
ptyId: 'pty-cold-owner',
worktreeId: WORKTREE_ID,
title: 'Codex',
customTitle: null,
color: null,
sortOrder: 0,
createdAt: 1
}
]
},
terminalLayoutsByTabId: {
'tab-cold-owner': {
root: { type: 'leaf' as const, leafId },
activeLeafId: leafId,
expandedLeafId: null,
ptyIdsByLeafId: { [leafId]: 'pty-cold-owner' }
}
},
terminalPtyIncarnationsByPaneKey: {
[`tab-cold-owner:${leafId}`]: 'incarnation-1'
}
}
const runtime = new OrcaRuntimeService(
{ getWorkspaceSession: () => persistedSession } as never,
undefined,
{
agentSessionClaimSigner: signer
}
)
runtime.setPtyController({
listProcesses: vi.fn(async () => [
{
id: 'pty-cold-owner',
incarnationId: 'incarnation-1',
rootProcessId: 31337,
cwd: '/tmp/structured-handoff',
title: 'codex',
worktreeId: WORKTREE_ID,
@@ -119,6 +154,29 @@ describe('structured TUI launch tab binding', () => {
}
}
]
},
{
id: 'pty-conflicting-owner',
incarnationId: 'incarnation-2',
rootProcessId: 41337,
cwd: '/tmp/structured-handoff',
title: 'codex',
worktreeId: WORKTREE_ID,
terminalHandle: 'term_conflicting_runtime',
agentSessionOwners: [
{
claim,
generation: 'generation-2',
phase: 'live' as const,
ptyId: 'pty-conflicting-owner',
surface: {
worktreeId: WORKTREE_ID,
tabId: 'tab-conflicting-owner',
leafId: conflictingLeafId,
terminalHandle: 'term_conflicting_runtime'
}
}
]
}
]),
write: () => true,
@@ -159,6 +217,12 @@ describe('structured TUI launch tab binding', () => {
outcome: 'identity-matched',
matchedOn: ['process-start-time']
})
readStructuredTuiProcessIdentity.mockResolvedValue({
hostId: 'local',
pid: 4243,
processStartTimeMs: 10,
spawnToken: 'spawn-token'
})
await internal.refreshMobileSessionPtyRecords()
const coldPty = internal.ptysById.get('pty-cold-owner')!
@@ -197,7 +261,11 @@ describe('structured TUI launch tab binding', () => {
'thread-1'
)
expect(writeAgentSessionProof).not.toHaveBeenCalled()
expect(readStructuredTuiProcessIdentity).toHaveBeenCalledWith(
expect.objectContaining({ rootPid: 31337, spawnToken: 'spawn-token' })
)
expect(agentSessionPtyWriteGate.boundSessionId('pty-cold-owner')).toBe('session-1')
expect(agentSessionPtyWriteGate.boundSessionId('pty-conflicting-owner')).toBeNull()
agentSessionPtyWriteGate.unbindPty('pty-cold-owner')
}
)
+48 -9
View File
@@ -91,7 +91,10 @@ import {
proveCodexTuiRollout,
resolvePinnedCodexRolloutProof
} from '../codex/codex-tui-rollout-proof'
import { probeAgentSessionProcessIdentity } from './agent-session-process-identity-probe'
import {
PROCESS_START_TIME_TOLERANCE_MS,
probeAgentSessionProcessIdentity
} from './agent-session-process-identity-probe'
import { waitForStructuredTuiExitProof } from './structured-tui-exit-proof'
import { readStructuredTuiProcessIdentity } from './structured-tui-process-identity'
import { hasStructuredTuiIdleEvidence } from './structured-tui-idle-evidence'
@@ -9273,23 +9276,29 @@ export class OrcaRuntimeService {
}),
canonicalWorktreeId: workspace.id
})
const recovered = [...this.ptysById.values()]
const recoveredCandidates = [...this.ptysById.values()]
.flatMap((pty) => pty.agentSessionOwners.map((owner) => ({ pty, owner })))
.find(
.filter(
({ pty, owner }) =>
pty.connected &&
owner.phase === 'live' &&
owner.ptyId === pty.ptyId &&
Boolean(pty.incarnationId) &&
runtimeWorktreeIdsEqual(pty.worktreeId, workspace.id) &&
runtimeWorktreeIdsEqual(owner.surface.worktreeId, workspace.id) &&
scopedAgentSessionClaimsEqual(owner.claim, claim)
scopedAgentSessionClaimsEqual(owner.claim, claim) &&
this.hasExactPersistedTerminalSurfaceIdentity({
worktreeId: owner.surface.worktreeId,
tabId: owner.surface.tabId,
leafId: owner.surface.leafId,
ptyId: owner.ptyId,
incarnationId: pty.incarnationId!
})
)
const processProof = recovered
? await probeAgentSessionProcessIdentity({ identity })
: null
const recovered = recoveredCandidates.length === 1 ? recoveredCandidates[0] : null
if (
!recovered ||
processProof?.outcome !== 'identity-matched' ||
processProof.matchedOn.length === 0
!(await this.proveRecoveredStructuredTuiPtyProcess(recovered.pty, identity))
) {
throw new Error('The owning agent terminal could not be recovered.')
}
@@ -9362,6 +9371,36 @@ export class OrcaRuntimeService {
}
}
private async proveRecoveredStructuredTuiPtyProcess(
pty: RuntimePtyWorktreeRecord,
identity: NonNullable<AgentSessionRecord['lease']['ownerProcess']>
): Promise<boolean> {
const listings = await this.ptyController?.listProcesses?.(pty.connectionId)
const listed = listings?.find(
(candidate) => candidate.id === pty.ptyId && candidate.incarnationId === pty.incarnationId
)
if (!listed?.rootProcessId || identity.processStartTimeMs === null) {
return false
}
try {
const observed = await readStructuredTuiProcessIdentity({
hostId: identity.hostId,
rootPid: listed.rootProcessId,
spawnToken: identity.spawnToken,
agent: 'codex'
})
return (
observed.hostId === identity.hostId &&
observed.pid === identity.pid &&
observed.processStartTimeMs !== null &&
Math.abs(observed.processStartTimeMs - identity.processStartTimeMs) <=
PROCESS_START_TIME_TOLERANCE_MS
)
} catch {
return false
}
}
private async closeStructuredTuiOwner(
owner: StructuredTuiOwner
): Promise<{ transcriptPath?: string }> {
@@ -1,8 +1,13 @@
import { describe, expect, it, vi } from 'vitest'
import { recoverLegacyWorkerTerminalsForRendererStartup } from './legacy-worker-renderer-recovery'
import {
FIRST_WINDOW_STARTUP_SERVICE_TIMEOUT_MS,
LOCAL_PTY_STARTUP_FAIL_OPEN_TIMEOUT_MS,
startFirstWindowStartupServices
} from './first-window-startup-services'
describe('legacy worker renderer recovery', () => {
it('hydrates after bounded barriers while provider startup remains pending', async () => {
it('waits for daemon adoption before renderer recovery can continue', async () => {
let resolveFirstWindow!: () => void
let resolveWslBarrier!: () => void
let resolveProvider!: () => void
@@ -29,14 +34,15 @@ describe('legacy worker renderer recovery', () => {
expect(reconcile).not.toHaveBeenCalled()
resolveWslBarrier()
await startup
expect(reconcile).toHaveBeenCalledTimes(1)
await Promise.resolve()
expect(reconcile).not.toHaveBeenCalled()
resolveProvider()
await vi.waitFor(() => expect(reconcile).toHaveBeenCalledTimes(2))
await startup
expect(reconcile).toHaveBeenCalledOnce()
})
it('retries after initial recovery when the provider is already ready', async () => {
it('recovers once when the provider is already ready', async () => {
const reconcile = vi.fn().mockResolvedValue(undefined)
await recoverLegacyWorkerTerminalsForRendererStartup({
@@ -47,7 +53,7 @@ describe('legacy worker renderer recovery', () => {
onDeferredRecoveryError: vi.fn()
})
await vi.waitFor(() => expect(reconcile).toHaveBeenCalledTimes(2))
expect(reconcile).toHaveBeenCalledOnce()
})
it('contains provider startup rejection after initial recovery', async () => {
@@ -67,12 +73,12 @@ describe('legacy worker renderer recovery', () => {
})
await expect(reportedError).resolves.toBe(providerError)
expect(reconcile).toHaveBeenCalledTimes(1)
expect(reconcile).not.toHaveBeenCalled()
})
it('contains deferred recovery rejection', async () => {
it('contains recovery rejection', async () => {
const recoveryError = new Error('recovery failed')
const reconcile = vi.fn().mockResolvedValueOnce(undefined).mockRejectedValueOnce(recoveryError)
const reconcile = vi.fn().mockRejectedValueOnce(recoveryError)
let reportError!: (error: unknown) => void
const reportedError = new Promise<unknown>((resolve) => {
reportError = resolve
@@ -87,31 +93,42 @@ describe('legacy worker renderer recovery', () => {
})
await expect(reportedError).resolves.toBe(recoveryError)
expect(reconcile).toHaveBeenCalledTimes(2)
expect(reconcile).toHaveBeenCalledOnce()
})
it('contains initial recovery rejection and still retries when the provider becomes ready', async () => {
const initialError = new Error('initial recovery failed')
let resolveProvider!: () => void
const providerReady = new Promise<void>((resolve) => {
resolveProvider = resolve
})
const reconcile = vi.fn().mockRejectedValueOnce(initialError).mockResolvedValueOnce(undefined)
const onDeferredRecoveryError = vi.fn()
await expect(
recoverLegacyWorkerTerminalsForRendererStartup({
firstWindowStartupServicesReady: Promise.resolve(),
it('fails open at the hard cap without allowing a premature recovery', async () => {
vi.useFakeTimers()
let daemonSignal: AbortSignal | undefined
try {
const services = startFirstWindowStartupServices({
startDaemonPtyProvider: (signal) => {
daemonSignal = signal
return new Promise<void>(() => {})
},
startAgentHookServer: () => Promise.resolve(),
onDaemonError: vi.fn(),
onAgentHookServerError: vi.fn()
})
const reconcile = vi.fn().mockResolvedValue(undefined)
const startup = recoverLegacyWorkerTerminalsForRendererStartup({
firstWindowStartupServicesReady: services.firstWindowReady,
managedWslCliStartupBarrierReady: Promise.resolve(),
localPtyProviderStartupReady: providerReady,
localPtyProviderStartupReady: services.localPtyProviderReady,
reconcile,
onDeferredRecoveryError
onDeferredRecoveryError: vi.fn()
})
).resolves.toBeUndefined()
expect(onDeferredRecoveryError).toHaveBeenCalledWith(initialError)
expect(reconcile).toHaveBeenCalledTimes(1)
resolveProvider()
await vi.waitFor(() => expect(reconcile).toHaveBeenCalledTimes(2))
await vi.advanceTimersByTimeAsync(FIRST_WINDOW_STARTUP_SERVICE_TIMEOUT_MS)
expect(reconcile).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(
LOCAL_PTY_STARTUP_FAIL_OPEN_TIMEOUT_MS - FIRST_WINDOW_STARTUP_SERVICE_TIMEOUT_MS
)
await startup
expect(reconcile).toHaveBeenCalledOnce()
expect(daemonSignal?.aborted).toBe(true)
} finally {
vi.useRealTimers()
}
})
})
@@ -13,18 +13,15 @@ export async function recoverLegacyWorkerTerminalsForRendererStartup(
() => ({ ok: true as const }),
(error: unknown) => ({ ok: false as const, error })
)
await Promise.all([
const [providerResult] = await Promise.all([
providerStartupResult,
options.firstWindowStartupServicesReady,
options.managedWslCliStartupBarrierReady
])
void providerStartupResult
.then(async (result) => {
if (!result.ok) {
throw result.error
}
await options.reconcile()
})
.catch(options.onDeferredRecoveryError)
if (!providerResult.ok) {
options.onDeferredRecoveryError(providerResult.error)
return
}
try {
await options.reconcile()
} catch (error) {
@@ -1,19 +1,63 @@
import { describe, expect, it, vi } from 'vitest'
import type { SleepingAgentSessionRecord } from '../../../shared/agent-session-resume'
import type { PtyListedSession } from '../../../shared/pty-listed-session'
import { runWorktreeAgentActivationGate } from './worktree-agent-activation-gate'
const WORKTREE_ID = 'repo::/worktree'
const LIVE_LEAF_ID = '11111111-1111-4111-8111-111111111111'
const DEAD_LEAF_ID = '22222222-2222-4222-8222-222222222222'
function listed(id: string): PtyListedSession {
return { id, cwd: '/worktree', title: 'Codex', agentOwnership: 'present' }
}
function testDeps(args: { sessions?: PtyListedSession[]; structured?: boolean }) {
function sleepingRecord(
tabId: string,
leafId: string,
providerSessionId: string
): SleepingAgentSessionRecord {
return {
paneKey: `${tabId}:${leafId}`,
tabId,
worktreeId: WORKTREE_ID,
agent: 'codex',
providerSession: { key: 'session_id', id: providerSessionId },
prompt: 'resume',
state: 'working',
capturedAt: 1,
updatedAt: 1
}
}
function testDeps(args: {
sessions?: PtyListedSession[]
sleeping?: SleepingAgentSessionRecord[]
structured?: boolean
resumeCount?: number
}) {
const createTab = vi.fn()
const resume = vi.fn(() => 1)
const resume = vi.fn(() => args.resumeCount ?? 1)
const sleeping = args.sleeping ?? []
const store = {
createTab,
ptyIdsByTabId: {},
sleepingAgentSessionsByPaneKey: Object.fromEntries(
sleeping.map((record) => [record.paneKey, record])
),
terminalLayoutsByTabId: Object.fromEntries(
sleeping.map((record) => {
const leafId = record.paneKey.slice(record.paneKey.indexOf(':') + 1)
return [
record.tabId!,
{
root: { type: 'leaf' as const, leafId },
activeLeafId: leafId,
expandedLeafId: null,
ptyIdsByLeafId: {}
}
]
})
),
unifiedTabsByWorktree: {
[WORKTREE_ID]: args.structured
? [
@@ -149,6 +193,34 @@ describe('worktree agent activation gate', () => {
expect(createTab).not.toHaveBeenCalled()
expect(resume).toHaveBeenCalledOnce()
expect(resume).toHaveBeenCalledWith(WORKTREE_ID)
expect(resume).toHaveBeenCalledWith(WORKTREE_ID, { skipClaimKeys: new Set() })
})
it('resumes a dead agent when the workspace only has a non-agent PTY', async () => {
const dead = sleepingRecord('tab-dead', DEAD_LEAF_ID, 'dead-session')
const plainPtyId = `${WORKTREE_ID}@@plain-shell`
const { deps, resume } = testDeps({
sessions: [{ ...listed(plainPtyId), title: 'zsh', agentOwnership: 'absent' }],
sleeping: [dead]
})
await expect(runWorktreeAgentActivationGate(WORKTREE_ID, deps)).resolves.toBe('resumed')
expect(resume).toHaveBeenCalledWith(WORKTREE_ID, { skipClaimKeys: new Set() })
})
it('suppresses only the exact live agent session while resuming a dead sibling', async () => {
const live = sleepingRecord('tab-live', LIVE_LEAF_ID, 'live-session')
const dead = sleepingRecord('tab-dead', DEAD_LEAF_ID, 'dead-session')
const livePtyId = `${WORKTREE_ID}@@live-agent`
const { deps, resume } = testDeps({ sessions: [listed(livePtyId)], sleeping: [live, dead] })
const store = deps.getState()
store.terminalLayoutsByTabId['tab-live']!.ptyIdsByLeafId[LIVE_LEAF_ID] = livePtyId
await expect(runWorktreeAgentActivationGate(WORKTREE_ID, deps)).resolves.toBe('resumed')
expect(resume).toHaveBeenCalledWith(WORKTREE_ID, {
skipClaimKeys: new Set([`${WORKTREE_ID}\0codex\0session_id\0live-session`])
})
})
})
@@ -2,12 +2,21 @@ import { useAppStore } from '@/store'
import type { PtyListedSession } from '../../../shared/pty-listed-session'
import { parsePtySessionId, PTY_SESSION_ID_SEPARATOR } from '../../../shared/pty-session-id-format'
import type { RuntimeMobileSessionTabsResult } from '../../../shared/runtime-types'
import { parsePaneKey } from '../../../shared/stable-pane-id'
import { parseWorkspaceKey } from '../../../shared/workspace-scope'
import { resumeSleepingAgentSessionsForWorktree } from './resume-sleeping-agent-session'
import {
resumeSleepingAgentSessionsForWorktree,
type ResumeSleepingAgentSessionsOptions
} from './resume-sleeping-agent-session'
import { getProviderSessionClaimKey } from './sleeping-agent-pane-ownership'
type ActivationStore = Pick<
ReturnType<typeof useAppStore.getState>,
'createTab' | 'ptyIdsByTabId' | 'unifiedTabsByWorktree'
| 'createTab'
| 'ptyIdsByTabId'
| 'sleepingAgentSessionsByPaneKey'
| 'terminalLayoutsByTabId'
| 'unifiedTabsByWorktree'
>
type ActivationGateDeps = {
@@ -15,7 +24,7 @@ type ActivationGateDeps = {
awaitReady?: () => Promise<boolean>
listSessions: () => Promise<PtyListedSession[]>
hasStructuredSession?: (worktreeId: string) => Promise<boolean>
resume: (worktreeId: string) => number
resume: (worktreeId: string, options?: ResumeSleepingAgentSessionsOptions) => number
}
export type WorktreeAgentActivationOutcome = 'adopted' | 'structured' | 'resumed' | 'blocked'
@@ -80,6 +89,30 @@ function sessionBelongsToWorkspace(sessionId: string, worktreeId: string): boole
)
}
function liveSleepingAgentClaimKeys(
store: ActivationStore,
worktreeId: string,
livePtyIds: ReadonlySet<string>
): Set<string> {
const keys = new Set<string>()
for (const record of Object.values(store.sleepingAgentSessionsByPaneKey)) {
if (record.worktreeId !== worktreeId) {
continue
}
const stable = parsePaneKey(record.paneKey)
const tabId = record.tabId ?? stable?.tabId
const persistedPtyId = stable
? store.terminalLayoutsByTabId[stable.tabId]?.ptyIdsByLeafId?.[stable.leafId]
: tabId && store.ptyIdsByTabId[tabId]?.length === 1
? store.ptyIdsByTabId[tabId][0]
: undefined
if (persistedPtyId && livePtyIds.has(persistedPtyId)) {
keys.add(getProviderSessionClaimKey(record))
}
}
return keys
}
export async function runWorktreeAgentActivationGate(
worktreeId: string,
deps: ActivationGateDeps
@@ -102,6 +135,7 @@ export async function runWorktreeAgentActivationGate(
const liveWorkspaceSessions = sessions.filter((session) =>
sessionBelongsToWorkspace(session.id, worktreeId)
)
const liveWorkspacePtyIds = new Set(liveWorkspaceSessions.map((session) => session.id))
if (liveWorkspaceSessions.length > 0) {
for (const session of liveWorkspaceSessions) {
const store = deps.getState()
@@ -114,21 +148,33 @@ export async function runWorktreeAgentActivationGate(
recordInteraction: false
})
}
return 'adopted'
if (!workspaceHasSleepingAgentSessions(deps.getState(), worktreeId)) {
return 'adopted'
}
}
let structured = false
try {
if (
structured = Boolean(
hasStructuredSession(deps.getState(), worktreeId) ||
(await deps.hasStructuredSession?.(worktreeId))
) {
)
if (structured && !workspaceHasSleepingAgentSessions(deps.getState(), worktreeId)) {
return 'structured'
}
} catch {
return 'blocked'
}
deps.resume(worktreeId)
return 'resumed'
const launched = deps.resume(worktreeId, {
skipClaimKeys: liveSleepingAgentClaimKeys(deps.getState(), worktreeId, liveWorkspacePtyIds)
})
return launched > 0
? 'resumed'
: liveWorkspaceSessions.length > 0
? 'adopted'
: structured
? 'structured'
: 'resumed'
}
export function gateWorktreeAgentActivation(