Files
orca/src/main/ipc/pty/runtime/spawn-commit.ts
T
Neil e42c60e8a3 fix(ssh): resolve a pane's binding from the target partition, not the stale local copy (#18546)
One SSH pane accumulated one extra reattachable lease per relay restart (2, 3, 4,
5, 6 across five), and every one of them costs a `pty.attach` round trip on every
later connect, forever. Nothing prunes `sshRemotePtyLeases`, so the fan-out only
grows.

`supersedeSiblingLeasesForPane` is fenced on the PTY the pane is durably bound to,
and `durablyBoundPtyIdForPane` read `state.workspaceSession` (local) before
`workspaceSessionsByHostId['ssh:<target>']`. But `persistPtyBinding(binding, hostId)`
updates ONLY the host partition:

  AFTER-PERSIST  local= ssh:t@@pty2:old:1   host= ssh:t@@pty2:new:1

So for the length of a reconnect the local copy still names the predecessor, the
fence resolved to it, supersession took an already-`expired` lease as its winner,
and returned having marked nothing. Both partitions agree again once the renderer
republishes its layout, which is why the settled store looks consistent and hid
this.

Read both partitions as an ordered list, target's own first, and test the fence by
membership rather than by equality with whichever was read first. Pick the winner
preferring a lease this client still has a route to, since the stale partition
names an expired one. Never retire a lease that is both bound and live, so a
partition disagreement can't strand a running remote process.

Superseded predecessors stay `expired` and are never `terminated`: losing a lease
is not evidence the shell died (docs/reference/ssh-execution-boundary.md). A pane
with no binding is skipped rather than pruned, so a genuine orphan stays askable.

Also re-runs supersession from the binding side after each spawn commit's binding
write, so the lease/binding order at a call site no longer decides, and reconciles
every pane for a target immediately before `reattachKnownPtys` reads the set it
feeds to `pty.attach` — that repairs stores which already accumulated these rows.

The guard suite could not catch this: every assertion bound the pane BEFORE
upserting the lease, an order no caller uses. Rewritten to the spawn commits' real
order (lease, then binding, then the binding-side trigger); it fails 8 assertions
without this change. Added a suite that drives the real `persistPtyIpcSpawnCommit`
rather than the store primitives, including the exact stale-partition state written
by production's own binding writer.

Verified on the Docker SSH lane: five `relay.js` SIGKILLs with recovery between
each, reattachable leases flat at one per pane.

Note: this bounds the reattach SET, not the store. `sshRemotePtyLeases` still has
no cap or TTL and rows still accumulate; pruning is left alone deliberately, since
an `expired` row without `supersededBy` is a genuine orphan and must not be dropped
on age.
2026-09-03 16:47:32 -07:00

304 lines
12 KiB
TypeScript

import { isValidTerminalTabId } from '../../../../shared/terminal-tab-id'
import { ptyOwnership, ptyIncarnationById, deletePtyOwnership } from '../provider/ownership-state'
import { ptySizes } from '../delivery/visibility-state'
import {
shouldSkipCodexHomeEnvForWindowsShell,
recordCodexPaneAccountForSpawn,
codexReattachedHomeRouteField
} from '../host-env/codex-home'
import { markClaudePtySpawned } from '../../../claude-accounts/live-pty-gate'
import { registerPty } from '../../../memory/pty-registry'
import { rememberPaneKeyForPty } from '../pane/key-state'
import {
pendingByPaneKey,
pendingPtyIdBySerializerGeneration,
rendererSerializerReadiness
} from '../pane/serializer-state'
import { seedTerminalRestoreRecordsFromSpawnResult } from '../pane/agent-session-owners'
import { track } from '../../../telemetry/client'
import { getCohortAtEmit } from '../../../telemetry/cohort-classifier'
import {
agentKindSchema,
launchSourceSchema,
requestKindSchema
} from '../../../../shared/telemetry-events'
import { persistAdmittedStablePaneBinding } from '../pane/stable-owner'
import { claimSshPaneLease } from '../pane/ssh-pane-lease-claim'
import {
isNativeWindowsLocalPtySpawn,
markNativeWindowsConptyPty
} from '../../../runtime/terminal-model-query-authority'
import { toSshExecutionHostId } from '../../../../shared/execution-host'
import { createTerminalSessionStateSaveFailureMessage } from '../../../../shared/terminal-session-state-save-failure'
import { clearProviderPtyState } from '../provider/state-cleanup'
import { resolvePaneSpawnReservation } from '../pane/spawn-reservation'
import { admitProviderReattachLaunchIdentity } from '../pane/launch-authority'
import type { RuntimePtySpawnState } from './spawn-state'
export async function commitRuntimePtySpawn(ctx: RuntimePtySpawnState) {
const args = ctx.args
const providerReattachLaunchIdentity = admitProviderReattachLaunchIdentity(ctx.result)
try {
ctx.stablePaneBindingPersisted = persistAdmittedStablePaneBinding({
store: ctx.hostSessionBinding?.store,
owner: ctx.stablePaneOwner,
result: ctx.result,
worktreeId: ctx.hostSessionBinding?.worktreeId,
startupCwd: ctx.cwd,
connectionId: args.connectionId
})
} catch (error) {
if (error instanceof Error && error.message === 'terminal_pane_owner_changed') {
throw error
}
console.error('[pty] failed to persist runtime PTY binding after attach:', error)
throw Object.assign(new Error(createTerminalSessionStateSaveFailureMessage()), {
agentSessionOperationOutcome: 'unknown' as const
})
}
if (ctx.result.agentSessionEnsure?.disposition === 'adopted') {
const owner = ctx.result.agentSessionEnsure.owner
ptyOwnership.set(ctx.result.id, args.connectionId ?? ptyOwnership.get(ctx.result.id) ?? null)
ctx.deps.runtime?.registerPreAllocatedHandleForPty(ctx.result.id, owner.surface.terminalHandle)
if (ctx.result.incarnationId) {
ptyIncarnationById.set(ctx.result.id, ctx.result.incarnationId)
}
ctx.deps.runtime?.registerPty(
ctx.result.id,
owner.surface.worktreeId,
args.connectionId ?? null,
{
tabId: owner.surface.tabId,
leafId: owner.surface.leafId,
terminalHandle: owner.surface.terminalHandle,
...(ctx.result.incarnationId ? { incarnationId: ctx.result.incarnationId } : {}),
...(providerReattachLaunchIdentity ? { providerReattachLaunchIdentity } : {})
}
)
if (!args.connectionId) {
ctx.deps.options?.onCodexHomePtySpawned?.({
id: ctx.result.id,
codexHomePath: ctx.selectedCodexHomePath,
reattached: true,
startedAt: ctx.codexHomeLaunchStartedAt,
startedSequence: ctx.codexHomeLaunchStartedSequence,
...codexReattachedHomeRouteField(ctx.reattachedCodexHomeRoutes, ctx.result.id, true),
...(ctx.env ? { launchEnv: ctx.env } : {})
})
}
// Why: the adopted branch returns before the normal settle site, so the
// reservation must be resolved here or every later spawn for this pane
// awaits a promise that never settles.
resolvePaneSpawnReservation(ctx.paneSpawnReservationKey, ctx.paneSpawnReservation, {
...ctx.result,
isReattach: true
})
return {
id: ctx.result.id,
...(ctx.result.incarnationId ? { incarnationId: ctx.result.incarnationId } : {}),
agentSessionEnsure: ctx.result.agentSessionEnsure
}
}
ptyOwnership.set(ctx.result.id, args.connectionId ?? null)
if (ctx.result.incarnationId) {
ptyIncarnationById.set(ctx.result.id, ctx.result.incarnationId)
}
// Why: record the native-Windows-local-PTY determination before any byte reaches the emulator, so its ConPTY DA1 override exists from byte zero.
if (
isNativeWindowsLocalPtySpawn({
connectionId: args.connectionId,
cwd: args.cwd,
shellOverride: ctx.daemonShellOverride
})
) {
markNativeWindowsConptyPty(ctx.result.id)
}
const persistSshLease = (): void =>
claimSshPaneLease({
store: ctx.deps.store,
connectionId: args.connectionId,
ptyId: ctx.result.id,
worktreeId: args.worktreeId,
tabId: args.tabId,
leafId: args.leafId
})
if (!ctx.hostSessionBinding) {
persistSshLease()
}
ptySizes.set(ctx.result.id, { cols: args.cols, rows: args.rows })
if (ctx.effectiveSessionAppId !== undefined && ctx.effectiveSessionAppId !== ctx.result.id) {
ptySizes.delete(ctx.effectiveSessionAppId)
}
recordCodexPaneAccountForSpawn({
ptyId: ctx.result.id,
isDaemonHostSpawn: ctx.isDaemonHostSpawn,
isReattach: ctx.result.isReattach === true,
pinnedByResume: ctx.codexResumeHomeSelected,
launchCodexHomePath: ctx.selectedCodexHomePath,
launchEnv: args.env,
target: ctx.codexSelectionTarget,
settings: ctx.deps.getSettings?.()
})
if (ctx.hostSessionBinding && !ctx.stablePaneBindingPersisted) {
try {
const binding = {
worktreeId: ctx.hostSessionBinding.worktreeId,
tabId: ctx.hostSessionBinding.tabId,
leafId: ctx.hostSessionBinding.leafId,
ptyId: ctx.result.id,
hostAdmittedMembership: true,
...(ctx.result.incarnationId ? { incarnationId: ctx.result.incarnationId } : {}),
...(ctx.cwd ? { startupCwd: ctx.cwd } : {}),
...(ctx.hostSessionBinding.expectedSourceBinding
? { expectedSourceBinding: ctx.hostSessionBinding.expectedSourceBinding }
: {})
}
const persisted = args.connectionId
? ctx.hostSessionBinding.store.persistPtyBinding(
binding,
toSshExecutionHostId(args.connectionId)
)
: ctx.hostSessionBinding.store.persistPtyBinding(binding)
if (persisted === false) {
throw new Error('terminal_split_source_not_found')
}
} catch (err) {
console.error('[pty] failed to persist runtime PTY binding after spawn:', err)
if (!ctx.result.isReattach) {
deletePtyOwnership(ctx.result.id)
try {
await ctx.provider.shutdown(ctx.result.id, { immediate: true })
} catch (shutdownErr) {
console.warn('[pty] failed to clean up PTY after persistence failure:', shutdownErr)
}
clearProviderPtyState(ctx.result.id)
}
if (err instanceof Error && err.message === 'terminal_split_source_not_found') {
throw err
}
throw Object.assign(new Error(createTerminalSessionStateSaveFailureMessage()), {
agentSessionOperationOutcome: 'unknown' as const
})
}
persistSshLease()
}
if (args.preAllocatedHandle && !ctx.stablePaneOwner?.handle) {
ctx.deps.runtime?.registerPreAllocatedHandleForPty(ctx.result.id, args.preAllocatedHandle)
}
if (args.worktreeId) {
ctx.deps.runtime?.registerPty(
ctx.result.id,
args.worktreeId,
args.connectionId ?? null,
// Why: thread validated pane identity so main can back a pending mobile create even if graph-sync stalls (#7587).
typeof args.tabId === 'string' &&
isValidTerminalTabId(args.tabId) &&
args.tabId.length <= 512 &&
ctx.metadataLeafId !== null
? {
tabId: args.tabId,
leafId: ctx.metadataLeafId,
...(args.preAllocatedHandle ? { terminalHandle: args.preAllocatedHandle } : {}),
...(ctx.result.incarnationId ? { incarnationId: ctx.result.incarnationId } : {}),
...(providerReattachLaunchIdentity ? { providerReattachLaunchIdentity } : {})
}
: undefined,
!args.connectionId
? shouldSkipCodexHomeEnvForWindowsShell(ctx.daemonShellOverride, ctx.cwd)
: undefined
)
} else {
// Why: non-worktree PTYs have no later surface-registration phase to clear admission intent.
ctx.deps.runtime?.cancelPendingPtyRegistration?.(ctx.result.id, ctx.result.incarnationId)
}
// Why: runtime-controller creates (headless serve, CLI, splits) adopt surviving daemon sessions too; without this seed their records stay blank.
seedTerminalRestoreRecordsFromSpawnResult(ctx.deps.runtime, ctx.result)
// Why: arms main's per-PTY Command Code output detector from the launch command (renderer startupCommand parity).
if (!ctx.stablePaneOwner) {
ctx.deps.runtime?.noteTerminalSpawnCommand?.(ctx.result.id, ctx.launchCommand ?? null)
}
if (ctx.isClaudeLaunch && !ctx.stablePaneOwner) {
markClaudePtySpawned(ctx.result.id)
}
if (args.telemetry && !ctx.stablePaneOwner) {
const agentKindParse = agentKindSchema.safeParse(args.telemetry.agent_kind)
const launchSourceParse = launchSourceSchema.safeParse(args.telemetry.launch_source)
const requestKindParse = requestKindSchema.safeParse(args.telemetry.request_kind)
if (agentKindParse.success && launchSourceParse.success && requestKindParse.success) {
track('agent_started', {
agent_kind: agentKindParse.data,
launch_source: launchSourceParse.data,
request_kind: requestKindParse.data,
...getCohortAtEmit()
})
}
}
// Why: runtime-owned CLI PTYs bypass the renderer pty:spawn handler; record paneKey here too since hook titles and cache cleanup need this reverse lookup.
const paneKey = rememberPaneKeyForPty(ctx.result.id, ctx.env?.ORCA_PANE_KEY)
const pendingSerializer = paneKey ? pendingByPaneKey.get(paneKey) : undefined
const inheritRendererReadiness =
ctx.result.isReattach === true &&
!pendingSerializer &&
rendererSerializerReadiness.has(ctx.result.id)
rendererSerializerReadiness.beginIncarnation(ctx.result.id, inheritRendererReadiness)
if (paneKey && pendingSerializer) {
pendingPtyIdBySerializerGeneration.set(pendingSerializer.gen, ctx.result.id)
}
if (!args.connectionId) {
registerPty({
ptyId: ctx.result.id,
worktreeId: args.worktreeId ?? null,
sessionId: ctx.sessionId ?? null,
paneKey,
pid:
typeof ctx.result.pid === 'number' && Number.isFinite(ctx.result.pid) && ctx.result.pid > 0
? ctx.result.pid
: null
})
}
// Why: runtime-owned/background spawns bypass mounted-pane state, so inventory consumers need an explicit signal.
ctx.deps.sendPtySpawnedToRenderer(ctx.result.id)
if (!args.connectionId) {
ctx.deps.options?.onCodexHomePtySpawned?.({
id: ctx.result.id,
codexHomePath: ctx.selectedCodexHomePath,
startedAt: ctx.codexHomeLaunchStartedAt,
startedSequence: ctx.codexHomeLaunchStartedSequence,
...codexReattachedHomeRouteField(
ctx.reattachedCodexHomeRoutes,
ctx.result.id,
ctx.result.isReattach === true
),
...(ctx.result.isReattach === true
? { reattached: true }
: ctx.env
? { launchEnv: ctx.env }
: {})
})
}
const response = {
id: ctx.result.id,
...(ctx.result.incarnationId ? { incarnationId: ctx.result.incarnationId } : {}),
...(ctx.stablePaneOwner && (ctx.stablePaneOwner.handle || args.preAllocatedHandle)
? {
stablePaneOwner: {
handle: ctx.stablePaneOwner.handle ?? args.preAllocatedHandle!,
tabId: ctx.stablePaneOwner.tabId,
leafId: ctx.stablePaneOwner.leafId
}
}
: {}),
...(ctx.result.agentSessionEnsure ? { agentSessionEnsure: ctx.result.agentSessionEnsure } : {})
}
resolvePaneSpawnReservation(ctx.paneSpawnReservationKey, ctx.paneSpawnReservation, {
...ctx.result,
...(typeof ctx.result.snapshotKittyKeyboardFlags === 'number' &&
ctx.reconciledSnapshotSeq !== null &&
ctx.snapshotKittyFlagsCoverReconciledSeq
? { snapshotSeq: ctx.reconciledSnapshotSeq }
: { snapshotKittyKeyboardFlags: undefined }),
isReattach: true
})
return response
}