mirror of
https://github.com/stablyai/orca.git
synced 2026-10-31 00:57:45 +00:00
fix(pty): an exit proof names its incarnation; a spawn is tracked from its reply (#30754)
* fix(pty): an exit proof names its incarnation, and a failed discard keeps ownership A same-id respawn could be retired by its predecessor's exit proof; finishPtyShutdown now refuses a proof whose incarnation is not the record's, and retireRejectedPty is scoped to the rejected spawn's incarnation. A spawn discarded after a failed save no longer drops ownership when its shutdown failed: it reads unverifiable until a later stop reaches it. Co-Authored-By: Claude <noreply@anthropic.com> * fix(pty): a failed discard leaves a same-id replacement's verdict alone Co-Authored-By: Claude <noreply@anthropic.com> * fix(pty): a lost discard reply does not overwrite an observed exit Co-Authored-By: Claude <noreply@anthropic.com> * fix(pty): a failed discard tracks the PTY it could not stop; test the incarnation refusal The discard runs before the spawn is published, so it records ownership and incarnation before stopping; a failed stop now leaves a handle. Every exit proof states its incarnation. Co-Authored-By: Claude <noreply@anthropic.com> * refactor(pty): track a spawn from its reply, so only an exit or a proven stop removes it Ownership and incarnation are recorded at commit start instead of at publish, before the binding save. The discard no longer re-records anything, so an exit observed during the save stays exited. Co-Authored-By: Claude <noreply@anthropic.com> * fix(pty): admit a same-id respawn despite a predecessor's verdict; retire an exit seen before commit on its fence Co-Authored-By: Claude <noreply@anthropic.com> * fix(pty): admit a fresh spawn only past the runtime's incarnation-scoped start fence Replaces the id-only verdict check, so a same-id respawn is tracked and an incarnation that exited before its commit is not resurrected; registration no longer has to retire it. Co-Authored-By: Claude <noreply@anthropic.com> * fix(pty): a proof that names no incarnation still retires; tidy kill-time capture and adopted re-record Co-Authored-By: Claude <noreply@anthropic.com> --------- Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -5,7 +5,6 @@
|
||||
|
||||
1 src/main/ipc/pty/ipc/renderer-kill.ts # SSH provider unregistered: R4-4 tab-close replay
|
||||
1 src/main/ipc/pty/pane/absent-pane-owner.ts # host-confirmed absence, untyped
|
||||
1 src/main/ipc/pty/pane/spawn-registration.ts # drops ownership even when the discard shutdown threw
|
||||
1 src/main/ipc/pty/provider/bind-listeners.ts # observed provider exit, untyped
|
||||
1 src/main/ipc/pty/provider/local-configure.ts # observed local exit, untyped
|
||||
1 src/main/ipc/pty/provider/ownership-state.ts # deletePtyOwnership, test-only caller
|
||||
|
||||
@@ -2,7 +2,12 @@ import { describe, expect, it, vi } from 'vitest'
|
||||
import { SSH_PROVIDER_UNREGISTERED_REASON } from '../../shared/pty-liveness-verdict'
|
||||
import { setupPtyIpcSuite } from './pty-ipc-test-harness'
|
||||
import { registerPtyHandlers } from './pty/register-handlers'
|
||||
import { deletePtyOwnership, setPtyOwnership } from './pty/provider/ownership-state'
|
||||
import {
|
||||
deletePtyOwnership,
|
||||
ptyIncarnationById,
|
||||
ptyOwnership,
|
||||
setPtyOwnership
|
||||
} from './pty/provider/ownership-state'
|
||||
import { getLocalPtyProvider } from './pty/provider/registry'
|
||||
|
||||
vi.mock('electron', () => import('./pty-ipc-mock-registry').then((m) => m.electronModuleMock()))
|
||||
@@ -58,7 +63,11 @@ describe('stopping a PTY whose SSH provider is unregistered', () => {
|
||||
controller: {
|
||||
kill: (ptyId: string) => boolean
|
||||
listProcesses: (hostId?: string) => Promise<{ id: string }[]>
|
||||
retireRejectedPty: (ptyId: string, stopConfirmed: boolean) => void
|
||||
retireRejectedPty: (
|
||||
ptyId: string,
|
||||
stopConfirmed: boolean,
|
||||
incarnationId: string | undefined
|
||||
) => void
|
||||
stopAndWait: (ptyId: string, opts?: { deadlineMs?: number }) => Promise<boolean>
|
||||
}
|
||||
runtime: {
|
||||
@@ -131,7 +140,7 @@ describe('stopping a PTY whose SSH provider is unregistered', () => {
|
||||
setPtyOwnership(ptyId, 'ssh:ssh-dropped')
|
||||
const { controller, runtime } = installController()
|
||||
try {
|
||||
controller.retireRejectedPty(ptyId, false)
|
||||
controller.retireRejectedPty(ptyId, false, undefined)
|
||||
|
||||
expect(runtime.markPtyLivenessUnverifiable).toHaveBeenCalledWith(
|
||||
ptyId,
|
||||
@@ -144,6 +153,26 @@ describe('stopping a PTY whose SSH provider is unregistered', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('leaves a same-id respawn alone when the rejected incarnation is retired', () => {
|
||||
const ptyId = 'ssh-respawned-split'
|
||||
// The rejected spawn's stop already retired its incarnation; a respawn took the id since.
|
||||
setPtyOwnership(ptyId, 'ssh:ssh-live')
|
||||
ptyIncarnationById.set(ptyId, 'inc-respawn')
|
||||
const { controller, runtime } = installController()
|
||||
try {
|
||||
controller.retireRejectedPty(ptyId, true, 'inc-rejected')
|
||||
controller.retireRejectedPty(ptyId, false, 'inc-rejected')
|
||||
|
||||
expect(ptyOwnership.get(ptyId)).toBe('ssh:ssh-live')
|
||||
expect(ptyIncarnationById.get(ptyId)).toBe('inc-respawn')
|
||||
expect(runtime.onPtyExit).not.toHaveBeenCalled()
|
||||
expect(runtime.markPtyLivenessUnverifiable).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
ptyIncarnationById.delete(ptyId)
|
||||
deletePtyOwnership(ptyId)
|
||||
}
|
||||
})
|
||||
|
||||
it('still confirms a stop the owning provider actually performed', async () => {
|
||||
const daemon = installObservableDaemonTestProvider()
|
||||
vi.spyOn(getLocalPtyProvider(), 'listProcesses').mockResolvedValue([])
|
||||
|
||||
@@ -106,19 +106,23 @@ async function stopRendererOwnedPtyProcess(
|
||||
return
|
||||
}
|
||||
const shutdownProvider = provider ?? getProviderForPty(args.id)
|
||||
const stoppingIncarnationId = ptyIncarnationById.get(args.id)
|
||||
let proof: PtyExitProof
|
||||
try {
|
||||
const providerExitObserved = await shutdownProviderAndDetectExit(shutdownProvider, args.id, {
|
||||
immediate: true,
|
||||
keepHistory: args.keepHistory ?? false
|
||||
})
|
||||
proof = PtyExitProof.from(providerExitObserved ? 'exit-observed' : 'shutdown-acknowledged')
|
||||
proof = PtyExitProof.from(
|
||||
providerExitObserved ? 'exit-observed' : 'shutdown-acknowledged',
|
||||
stoppingIncarnationId
|
||||
)
|
||||
} catch (err) {
|
||||
if (!isPtyAlreadyGoneError(err)) {
|
||||
// Why: a failed shutdown can leave the local daemon's process alive; keep ownership so a retry can reach it.
|
||||
throw err
|
||||
}
|
||||
proof = PtyExitProof.from('absent')
|
||||
proof = PtyExitProof.from('absent', stoppingIncarnationId)
|
||||
}
|
||||
// Why: some shutdown paths do not emit onExit through the provider listener.
|
||||
// Explicit cleanup is idempotent and covers already-dead PTYs.
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { LOCAL_EXECUTION_HOST_ID } from '../../../../shared/execution-host'
|
||||
import { closeStartupQueryAuthorityForPty } from '../provider/registry'
|
||||
import { createTerminalSessionStateSaveFailureMessage } from '../../../../shared/terminal-session-state-save-failure'
|
||||
import { recordCodexPaneAccountForSpawn } from '../host-env/codex-home'
|
||||
@@ -9,7 +8,6 @@ import {
|
||||
pendingPtyIdBySerializerGeneration,
|
||||
rendererSerializerReadiness
|
||||
} from '../pane/serializer-state'
|
||||
import { ptyOwnership, ptyIncarnationById } from '../provider/ownership-state'
|
||||
import { ptySizes } from '../delivery/visibility-state'
|
||||
import { resolveCommittedPtySize, type PtyGrid } from '../delivery/attached-pty-size'
|
||||
import { discardUnpersistedPtySpawn } from '../pane/spawn-registration'
|
||||
@@ -70,7 +68,7 @@ export async function persistPtyIpcSpawnCommit(ctx: PtyIpcSpawnState): Promise<P
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[pty] failed to persist PTY binding after spawn:', err)
|
||||
await discardUnpersistedPtySpawn(ctx.provider, ctx.result)
|
||||
await discardUnpersistedPtySpawn(ctx.provider, ctx.result, ctx.deps.runtime)
|
||||
if (err instanceof Error && err.message === 'terminal_pane_owner_changed') {
|
||||
throw err
|
||||
}
|
||||
@@ -98,10 +96,6 @@ export function publishPtyIpcSpawnCommit(ctx: PtyIpcSpawnState, committedSize: P
|
||||
target: ctx.codexSelectionTarget,
|
||||
settings: ctx.deps.getSettings?.()
|
||||
})
|
||||
ptyOwnership.set(ctx.result.id, LOCAL_EXECUTION_HOST_ID)
|
||||
if (ctx.result.incarnationId) {
|
||||
ptyIncarnationById.set(ctx.result.id, ctx.result.incarnationId)
|
||||
}
|
||||
if (ctx.initiallyHidden) {
|
||||
// Refresh the pre-spawn hidden mark only after this incarnation survives its save.
|
||||
ctx.deps.transitionSpawnHiddenRendererPtyDeliveryState(ctx.result.id, true)
|
||||
@@ -109,7 +103,7 @@ export function publishPtyIpcSpawnCommit(ctx: PtyIpcSpawnState, committedSize: P
|
||||
// Defense: never strand a mark on an id the provider renamed.
|
||||
ctx.deps.transitionSpawnHiddenRendererPtyDeliveryState(ctx.preSpawnHiddenMarkId, false)
|
||||
}
|
||||
// Why after ptyOwnership.set: provider lookup routes by ownership, and a hidden-spawned agent should be paceable from its first flood.
|
||||
// Why after ownership (recorded at commit start): provider lookup routes by it, and a hidden-spawned agent should be paceable from its first flood.
|
||||
ctx.deps.syncPtyBackgroundedDelivery(ctx.result.id, 'spawn')
|
||||
closeStartupQueryAuthorityForPty(ctx.result.id)
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ import {
|
||||
} from '../pane/launch-authority'
|
||||
import type { PtyIpcSpawnState } from './spawn-state'
|
||||
import { persistPtyIpcSpawnCommit, publishPtyIpcSpawnCommit } from './spawn-commit-persist'
|
||||
import { admitPtyReattachOwnership, registerPersistedPtySpawn } from '../pane/spawn-registration'
|
||||
import { admitPtySpawnOwnership, registerPersistedPtySpawn } from '../pane/spawn-registration'
|
||||
import { reflowHeadlessTerminalToCommittedGrid } from '../delivery/attached-pty-size'
|
||||
import { seedHeadlessTerminalFromSpawnResult } from '../pane/terminal-spawn-restore'
|
||||
import { markNativeWindowsConptyPty } from '../../../runtime/terminal-model-query-authority'
|
||||
@@ -30,7 +30,7 @@ export async function commitPtyIpcSpawn(ctx: PtyIpcSpawnState): Promise<PtySpawn
|
||||
|
||||
async function commitReservedPtyIpcSpawn(ctx: PtyIpcSpawnState): Promise<PtySpawnResult> {
|
||||
const args = ctx.args
|
||||
admitPtyReattachOwnership(ctx.deps.runtime, ctx.result)
|
||||
admitPtySpawnOwnership(ctx.deps.runtime, ctx.result)
|
||||
if (ctx.nativeWindowsConptySpawn) {
|
||||
markNativeWindowsConptyPty(ctx.result.id)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { LOCAL_EXECUTION_HOST_ID } from '../../../../shared/execution-host'
|
||||
import type { IPtyProvider, PtySpawnResult } from '../../../providers/pty-provider-contract'
|
||||
import { ptyIncarnationById, ptyOwnership } from '../provider/ownership-state'
|
||||
import { admitPtySpawnOwnership, discardUnpersistedPtySpawn } from './spawn-registration'
|
||||
|
||||
const PTY_ID = 'unpersisted-spawn'
|
||||
const result: PtySpawnResult = { id: PTY_ID, incarnationId: 'inc-1' }
|
||||
|
||||
function providerWhoseShutdown(shutdown: () => Promise<void>): IPtyProvider {
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: discard calls only shutdown.
|
||||
return { shutdown } as unknown as IPtyProvider
|
||||
}
|
||||
|
||||
// The commit records ownership at its start, before the save whose failure leads here.
|
||||
describe('discarding a spawn whose binding failed to save', () => {
|
||||
afterEach(() => {
|
||||
ptyOwnership.delete(PTY_ID)
|
||||
ptyIncarnationById.delete(PTY_ID)
|
||||
})
|
||||
|
||||
function discard(
|
||||
shutdown: () => Promise<void>,
|
||||
runtime = { markPtyLivenessUnverifiable: vi.fn() }
|
||||
): Promise<typeof runtime> {
|
||||
admitPtySpawnOwnership(undefined, result)
|
||||
return discardUnpersistedPtySpawn(providerWhoseShutdown(shutdown), result, runtime).then(
|
||||
() => runtime
|
||||
)
|
||||
}
|
||||
|
||||
it('keeps tracking the PTY and reads unverifiable when the shutdown fails', async () => {
|
||||
const runtime = await discard(() => Promise.reject(new Error('shutdown timed out')))
|
||||
|
||||
expect(ptyOwnership.get(PTY_ID)).toBe(LOCAL_EXECUTION_HOST_ID)
|
||||
expect(ptyIncarnationById.get(PTY_ID)).toBe('inc-1')
|
||||
expect(runtime.markPtyLivenessUnverifiable).toHaveBeenCalledWith(
|
||||
PTY_ID,
|
||||
expect.stringContaining('shutdown timed out')
|
||||
)
|
||||
})
|
||||
|
||||
it('retires the PTY once the host acknowledged the shutdown', async () => {
|
||||
const runtime = await discard(() => Promise.resolve())
|
||||
|
||||
expect(ptyOwnership.has(PTY_ID)).toBe(false)
|
||||
expect(runtime.markPtyLivenessUnverifiable).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('leaves a same-id replacement that arrived during the shutdown alone', async () => {
|
||||
await discard(async () => {
|
||||
ptyOwnership.set(PTY_ID, 'ssh:successor')
|
||||
ptyIncarnationById.set(PTY_ID, 'inc-2')
|
||||
})
|
||||
|
||||
expect(ptyOwnership.get(PTY_ID)).toBe('ssh:successor')
|
||||
expect(ptyIncarnationById.get(PTY_ID)).toBe('inc-2')
|
||||
})
|
||||
|
||||
it('keeps an exit observed during a shutdown whose reply was lost', async () => {
|
||||
const runtime = await discard(async () => {
|
||||
// The provider's exit listener retires the PTY before the RPC rejects.
|
||||
ptyOwnership.delete(PTY_ID)
|
||||
ptyIncarnationById.delete(PTY_ID)
|
||||
throw new Error('shutdown response lost')
|
||||
})
|
||||
|
||||
expect(ptyOwnership.has(PTY_ID)).toBe(false)
|
||||
expect(runtime.markPtyLivenessUnverifiable).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not mark a same-id replacement unverifiable when the predecessor shutdown fails', async () => {
|
||||
const runtime = await discard(async () => {
|
||||
ptyIncarnationById.set(PTY_ID, 'inc-2')
|
||||
throw new Error('shutdown response lost')
|
||||
})
|
||||
|
||||
expect(ptyIncarnationById.get(PTY_ID)).toBe('inc-2')
|
||||
expect(runtime.markPtyLivenessUnverifiable).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -5,29 +5,57 @@ import {
|
||||
isCurrentPtyExit,
|
||||
ptyIncarnationById,
|
||||
ptyOwnership,
|
||||
setAdoptedPtyOwnership
|
||||
setAdoptedPtyOwnership,
|
||||
setPtyOwnership
|
||||
} from '../provider/ownership-state'
|
||||
import { clearProviderPtyState } from '../provider/state-cleanup'
|
||||
import { LOCAL_EXECUTION_HOST_ID } from '../../../../shared/execution-host'
|
||||
import { finishPtyShutdown, isPtyAlreadyGoneError } from '../provider/liveness'
|
||||
import { PtyExitProof } from '../../../pty-exit-proof'
|
||||
import { errorMessage } from '../../../../shared/error-message'
|
||||
import { retirePersistedStablePaneOwner } from './stable-owner'
|
||||
|
||||
export function admitPtyReattachOwnership(
|
||||
/** Tracks a PTY from its spawn reply, before any save: only an observed exit, or a stop on proof,
|
||||
* removes it, so a spawn whose save or stop fails is never forgotten. */
|
||||
export function admitPtySpawnOwnership(
|
||||
runtime: OrcaRuntimeService | undefined,
|
||||
result: PtySpawnResult
|
||||
): void {
|
||||
if (!result.isReattach && result.agentSessionEnsure?.disposition !== 'adopted') {
|
||||
if (result.isReattach || result.agentSessionEnsure?.disposition === 'adopted') {
|
||||
runtime?.assertPtyRegistrationAllowed?.(result.id, result.incarnationId)
|
||||
// A failed local save must not strand a live process already admitted by its host.
|
||||
setAdoptedPtyOwnership(result.id)
|
||||
} else if (exitedBeforeAdmission(runtime, result)) {
|
||||
// Its exit listener already ran; tracking it now would resurrect it.
|
||||
return
|
||||
} else {
|
||||
setPtyOwnership(result.id, LOCAL_EXECUTION_HOST_ID)
|
||||
}
|
||||
runtime?.assertPtyRegistrationAllowed?.(result.id, result.incarnationId)
|
||||
// A failed local save must not strand a live process already admitted by its host.
|
||||
setAdoptedPtyOwnership(result.id)
|
||||
if (result.incarnationId) {
|
||||
ptyIncarnationById.set(result.id, result.incarnationId)
|
||||
}
|
||||
}
|
||||
|
||||
/** The runtime's incarnation-scoped start fence, read without failing the commit: registration
|
||||
* reports the early exit itself. */
|
||||
function exitedBeforeAdmission(
|
||||
runtime: OrcaRuntimeService | undefined,
|
||||
result: PtySpawnResult
|
||||
): boolean {
|
||||
try {
|
||||
runtime?.assertPtyRegistrationAllowed?.(result.id, result.incarnationId)
|
||||
return false
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === 'agent_session_exited_during_start') {
|
||||
return true
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export async function discardUnpersistedPtySpawn(
|
||||
provider: IPtyProvider,
|
||||
result: PtySpawnResult
|
||||
result: PtySpawnResult,
|
||||
runtime: Pick<OrcaRuntimeService, 'markPtyLivenessUnverifiable'> | undefined
|
||||
): Promise<void> {
|
||||
if (
|
||||
result.isReattach ||
|
||||
@@ -36,19 +64,35 @@ export async function discardUnpersistedPtySpawn(
|
||||
) {
|
||||
return
|
||||
}
|
||||
let proof: PtyExitProof
|
||||
try {
|
||||
await provider.shutdown(result.id, {
|
||||
immediate: true,
|
||||
...(result.incarnationId ? { expectedIncarnationId: result.incarnationId } : {})
|
||||
})
|
||||
proof = PtyExitProof.from('shutdown-acknowledged', result.incarnationId)
|
||||
} catch (error) {
|
||||
console.warn('[pty] failed to clean up PTY after persistence failure:', error)
|
||||
}
|
||||
// A replacement may arrive while the execution host finishes shutting down the predecessor.
|
||||
if (isCurrentPtyExit(result)) {
|
||||
clearProviderPtyState(result.id)
|
||||
ptyOwnership.delete(result.id)
|
||||
if (!isPtyAlreadyGoneError(error)) {
|
||||
console.warn('[pty] failed to clean up PTY after persistence failure:', error)
|
||||
// An observed exit, or a same-id replacement, outranks a lost control reply.
|
||||
if (
|
||||
!ptyOwnership.has(result.id) ||
|
||||
ptyIncarnationById.get(result.id) !== result.incarnationId
|
||||
) {
|
||||
return
|
||||
}
|
||||
// Keep ownership: the process may still run, so it reads unverifiable until it exits or an
|
||||
// orphan sweep reaps it, never forgotten.
|
||||
runtime?.markPtyLivenessUnverifiable?.(
|
||||
result.id,
|
||||
`discard shutdown failed: ${errorMessage(error)}`
|
||||
)
|
||||
return
|
||||
}
|
||||
proof = PtyExitProof.from('absent', result.incarnationId)
|
||||
}
|
||||
// The proof names this incarnation, so a replacement that arrived meanwhile keeps its own.
|
||||
finishPtyShutdown(result.id, null, undefined, proof)
|
||||
}
|
||||
|
||||
// Successful registration must not yield before the remaining spawn publication.
|
||||
|
||||
@@ -3,14 +3,14 @@ import { agentSessionOwnerBindingsEqual } from '../../../../shared/claimed-agent
|
||||
import { addNodePtyRecoveryHint } from '../../../daemon/node-pty-error-hints'
|
||||
import type { Store } from '../../../persistence'
|
||||
import type { IPtyProvider } from '../../../providers/pty-provider-contract'
|
||||
import { ptyIncarnationById, ptyOwnership } from './ownership-state'
|
||||
import { isCurrentPtyExit, ptyIncarnationById, ptyOwnership } from './ownership-state'
|
||||
import { getRelayPtyId } from './registry'
|
||||
import {
|
||||
KEEP_HISTORY_STOP_POLL_MS,
|
||||
KEEP_HISTORY_STOP_SETTLE_MS
|
||||
} from '../delivery/visibility-state'
|
||||
import { clearProviderPtyState } from './state-cleanup'
|
||||
import type { PtyRetirementEvidence } from '../../../pty-exit-proof'
|
||||
import { PtyExitProof, type PtyRetirementEvidence } from '../../../pty-exit-proof'
|
||||
|
||||
export function normalizeNodePtySpawnError(err: unknown): Error {
|
||||
const rawMessage = err instanceof Error ? err.message : String(err)
|
||||
@@ -100,12 +100,22 @@ export async function verifyPtyStopped(
|
||||
return true
|
||||
}
|
||||
|
||||
/** Returns the retired incarnation, or `null` when the proof names a different incarnation than
|
||||
* the record's: a same-id respawn owns it now, so nothing is retired. A proof naming none still
|
||||
* retires: owner reconciliation can stamp the same process's incarnation mid-stop. */
|
||||
export function finishPtyShutdown(
|
||||
id: string,
|
||||
connectionId: string | null | undefined,
|
||||
store: Store | undefined,
|
||||
evidence: PtyRetirementEvidence
|
||||
): string | undefined {
|
||||
): string | undefined | null {
|
||||
if (
|
||||
evidence instanceof PtyExitProof &&
|
||||
evidence.incarnationId !== undefined &&
|
||||
!isCurrentPtyExit({ id, incarnationId: evidence.incarnationId })
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const incarnationId = ptyIncarnationById.get(id)
|
||||
clearProviderPtyState(id)
|
||||
if (connectionId) {
|
||||
|
||||
@@ -62,8 +62,8 @@ export function installPtyRuntimeController(deps: PtyRuntimeControllerDeps): voi
|
||||
// attach. Attach-only and false-on-doubt: never creates or resizes.
|
||||
attach: (ptyId) => attachPtyFromRuntimeController(deps, ptyId),
|
||||
kill: (ptyId) => killPtyFromRuntimeController(deps, ptyId),
|
||||
retireRejectedPty: (ptyId, stopConfirmed) =>
|
||||
retireRejectedPtyFromRuntimeController(deps, ptyId, stopConfirmed),
|
||||
retireRejectedPty: (ptyId, stopConfirmed, incarnationId) =>
|
||||
retireRejectedPtyFromRuntimeController(deps, ptyId, stopConfirmed, incarnationId),
|
||||
stopAndWait: (ptyId, opts) => stopAndWaitPtyFromRuntimeController(deps, ptyId, opts),
|
||||
getForegroundProcess: (ptyId) => getForegroundProcessFromRuntimeController(ptyId),
|
||||
inspectProcess: (ptyId, options) => inspectProcessFromRuntimeController(ptyId, options),
|
||||
|
||||
@@ -36,7 +36,7 @@ describe('a synthetic PTY exit', () => {
|
||||
|
||||
it('leaves the liveness verdict to the exit when the host proved it', () => {
|
||||
const { calls, deps } = harness()
|
||||
publishSyntheticExit(deps, 'pty-1', 0, undefined, PtyExitProof.from('absent'))
|
||||
publishSyntheticExit(deps, 'pty-1', 0, undefined, PtyExitProof.from('absent', undefined))
|
||||
expect(calls).toEqual(['exit'])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -40,7 +40,7 @@ describe('a proven stop-and-wait of a terminal an older relay still runs', () =>
|
||||
ptyId,
|
||||
null,
|
||||
null,
|
||||
PtyExitProof.from('absent')
|
||||
PtyExitProof.from('absent', undefined)
|
||||
)
|
||||
expect(deps.runtime.onPtyExit).toHaveBeenCalledWith(ptyId, 0, undefined)
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { LOCAL_EXECUTION_HOST_ID } from '../../../../shared/execution-host'
|
||||
import type { IPtyProvider } from '../../../providers/pty-provider-contract'
|
||||
import { stopRendererOwnedPty, type PtyKillIpcDeps } from '../ipc/renderer-kill'
|
||||
import { finishPtyShutdown } from '../provider/liveness'
|
||||
import { ptyIncarnationById, ptyOwnership } from '../provider/ownership-state'
|
||||
import { getLocalPtyProvider, setLocalPtyProvider } from '../provider/registry'
|
||||
import type { PtyRuntimeControllerDeps } from './controller-deps'
|
||||
import { killPtyFromRuntimeController, stopAndWaitPtyFromRuntimeController } from './kill'
|
||||
|
||||
const PTY_ID = 'respawned-pty'
|
||||
|
||||
// The shutdown acknowledges only after `duringShutdown` ran; `listed` is the host's inventory then.
|
||||
function providerWhose(duringShutdown: () => void, listed: { id: string }[]): IPtyProvider {
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the stops call only shutdown and listProcesses.
|
||||
return {
|
||||
shutdown: async () => duringShutdown(),
|
||||
listProcesses: async () => listed
|
||||
} as unknown as IPtyProvider
|
||||
}
|
||||
|
||||
function harness() {
|
||||
const runtime = {
|
||||
markPtyStopRequested: vi.fn(),
|
||||
markPtyLivenessUnverifiable: vi.fn(),
|
||||
markPtyLivenessLive: vi.fn(),
|
||||
onPtyExit: vi.fn()
|
||||
}
|
||||
const ports = {
|
||||
runtime,
|
||||
getLocalPtyProviderStartupPromise: () => undefined,
|
||||
shutdownProviderAndDetectExit: async (provider: IPtyProvider, id: string) => {
|
||||
await provider.shutdown(id, { immediate: true })
|
||||
return false
|
||||
},
|
||||
rememberSyntheticKillExit: vi.fn(),
|
||||
sendPtyExitToRenderer: vi.fn(),
|
||||
finishPtyShutdown,
|
||||
retiredRejectedPtyIds: new Map<string, NodeJS.Timeout>()
|
||||
}
|
||||
return { runtime, ports }
|
||||
}
|
||||
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the stops read only these ports and an optional store.
|
||||
const asControllerDeps = (ports: unknown) => ports as PtyRuntimeControllerDeps
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the tab-close stop reads only these ports and an optional store.
|
||||
const asKillIpcDeps = (ports: unknown) => ports as PtyKillIpcDeps
|
||||
|
||||
async function settleKill(): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, 0))
|
||||
}
|
||||
|
||||
const original = getLocalPtyProvider()
|
||||
afterEach(() => {
|
||||
setLocalPtyProvider(original)
|
||||
ptyOwnership.delete(PTY_ID)
|
||||
ptyIncarnationById.delete(PTY_ID)
|
||||
})
|
||||
|
||||
describe('a stop whose shutdown lands after a same-id respawn', () => {
|
||||
function respawn(): void {
|
||||
ptyOwnership.set(PTY_ID, LOCAL_EXECUTION_HOST_ID)
|
||||
ptyIncarnationById.set(PTY_ID, 'inc-respawn')
|
||||
}
|
||||
function start(): ReturnType<typeof harness> {
|
||||
setLocalPtyProvider(providerWhose(respawn, [{ id: PTY_ID }]))
|
||||
ptyOwnership.set(PTY_ID, LOCAL_EXECUTION_HOST_ID)
|
||||
ptyIncarnationById.set(PTY_ID, 'inc-stopped')
|
||||
return harness()
|
||||
}
|
||||
|
||||
function expectRespawnUntouched({ runtime, ports }: ReturnType<typeof harness>): void {
|
||||
expect(ptyOwnership.get(PTY_ID)).toBe(LOCAL_EXECUTION_HOST_ID)
|
||||
expect(ptyIncarnationById.get(PTY_ID)).toBe('inc-respawn')
|
||||
expect(runtime.onPtyExit).not.toHaveBeenCalled()
|
||||
expect(ports.sendPtyExitToRenderer).not.toHaveBeenCalled()
|
||||
}
|
||||
|
||||
it('leaves the respawn alone after a runtime kill', async () => {
|
||||
const h = start()
|
||||
killPtyFromRuntimeController(asControllerDeps(h.ports), PTY_ID)
|
||||
await settleKill()
|
||||
expectRespawnUntouched(h)
|
||||
})
|
||||
|
||||
it('reports the listed respawn live after a stop-and-wait', async () => {
|
||||
const h = start()
|
||||
await expect(
|
||||
stopAndWaitPtyFromRuntimeController(asControllerDeps(h.ports), PTY_ID)
|
||||
).resolves.toBe(false)
|
||||
expect(h.runtime.markPtyLivenessLive).toHaveBeenCalledWith(PTY_ID)
|
||||
expectRespawnUntouched(h)
|
||||
})
|
||||
|
||||
it('leaves the respawn alone after a renderer tab close', async () => {
|
||||
const h = start()
|
||||
await stopRendererOwnedPty(asKillIpcDeps(h.ports), { id: PTY_ID })
|
||||
expectRespawnUntouched(h)
|
||||
})
|
||||
})
|
||||
|
||||
describe('a stop of a PTY whose incarnation was stamped mid-stop', () => {
|
||||
// Owner reconciliation records the same process's incarnation while its shutdown is in flight.
|
||||
function start(): ReturnType<typeof harness> {
|
||||
setLocalPtyProvider(providerWhose(() => ptyIncarnationById.set(PTY_ID, 'inc-same'), []))
|
||||
ptyOwnership.set(PTY_ID, LOCAL_EXECUTION_HOST_ID)
|
||||
return harness()
|
||||
}
|
||||
|
||||
function expectRetired({ ports }: ReturnType<typeof harness>): void {
|
||||
expect(ptyOwnership.has(PTY_ID)).toBe(false)
|
||||
expect(ports.sendPtyExitToRenderer).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: PTY_ID })
|
||||
)
|
||||
}
|
||||
|
||||
it('retires it after a renderer tab close', async () => {
|
||||
const h = start()
|
||||
await stopRendererOwnedPty(asKillIpcDeps(h.ports), { id: PTY_ID })
|
||||
expectRetired(h)
|
||||
})
|
||||
|
||||
it('retires it after a runtime kill', async () => {
|
||||
const h = start()
|
||||
killPtyFromRuntimeController(asControllerDeps(h.ports), PTY_ID)
|
||||
await settleKill()
|
||||
expectRetired(h)
|
||||
})
|
||||
})
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { IPtyProvider } from '../../../providers/pty-provider-contract'
|
||||
import { SSH_PROVIDER_UNREGISTERED_REASON } from '../../../../shared/pty-liveness-verdict'
|
||||
import { ptyOwnership, ptyIncarnationById } from '../provider/ownership-state'
|
||||
import { isCurrentPtyExit, ptyOwnership, ptyIncarnationById } from '../provider/ownership-state'
|
||||
import {
|
||||
getProviderForPty,
|
||||
getPtySshConnectionId,
|
||||
@@ -25,16 +25,20 @@ import {
|
||||
const OBSERVED_EXIT_RECORD_WAIT_MS = 10_000
|
||||
|
||||
/** Tells the runtime and renderer a PTY ended that no provider exit reported. Without proof its
|
||||
* liveness reads `unverifiable`, never `exited`. */
|
||||
* liveness reads `unverifiable`, never `exited`. A `null` incarnation (a proof
|
||||
* `finishPtyShutdown` refused) publishes nothing: the id belongs to a respawn now. */
|
||||
export function publishSyntheticExit(
|
||||
deps: Pick<PtyRuntimeControllerDeps, 'rememberSyntheticKillExit' | 'sendPtyExitToRenderer'> & {
|
||||
runtime?: Pick<OrcaRuntimeService, 'onPtyExit' | 'markPtyLivenessUnverifiable'>
|
||||
},
|
||||
ptyId: string,
|
||||
code: number,
|
||||
incarnationId: string | undefined,
|
||||
incarnationId: string | undefined | null,
|
||||
evidence: PtyRetirementEvidence
|
||||
): void {
|
||||
if (incarnationId === null) {
|
||||
return
|
||||
}
|
||||
if (evidence instanceof PtyRetirementWithoutProof) {
|
||||
deps.runtime?.markPtyLivenessUnverifiable?.(ptyId, evidence.unverifiableReason)
|
||||
}
|
||||
@@ -81,13 +85,13 @@ export function killPtyFromRuntimeController(
|
||||
}
|
||||
// Why: close runtime tails without clearing provider ownership, so
|
||||
// a retry can still target a PTY that survived the failed shutdown.
|
||||
const reportStopFailure = (err: unknown): void => {
|
||||
const reportStopFailure = (err: unknown, incarnationId: string | undefined): void => {
|
||||
const msg = errorMessage(err)
|
||||
console.warn(`[pty] Failed to stop PTY ${ptyId}: ${msg}`)
|
||||
if (retiredRejectedPtyIds.has(ptyId)) {
|
||||
return
|
||||
}
|
||||
runtime?.onPtyExit(ptyId, -1, ptyIncarnationById.get(ptyId))
|
||||
runtime?.onPtyExit(ptyId, -1, incarnationId)
|
||||
}
|
||||
const killWithCurrentProvider = (): boolean => {
|
||||
let provider: IPtyProvider
|
||||
@@ -104,12 +108,14 @@ export function killPtyFromRuntimeController(
|
||||
}
|
||||
return false
|
||||
}
|
||||
const stoppingIncarnationId = ptyIncarnationById.get(ptyId)
|
||||
// Why: controller is synchronous, but keep ownership until async shutdown proves whether the provider emitted an exit.
|
||||
void shutdownProviderAndDetectExit(provider, ptyId, { immediate: false })
|
||||
.then((providerExitObserved) => {
|
||||
const retired = retiredRejectedPtyIds.has(ptyId)
|
||||
const proof = PtyExitProof.from(
|
||||
providerExitObserved ? 'exit-observed' : 'shutdown-acknowledged'
|
||||
providerExitObserved ? 'exit-observed' : 'shutdown-acknowledged',
|
||||
stoppingIncarnationId
|
||||
)
|
||||
const incarnationId = finishPtyShutdown(ptyId, connectionId, store, proof)
|
||||
if (!providerExitObserved && !retired) {
|
||||
@@ -119,21 +125,24 @@ export function killPtyFromRuntimeController(
|
||||
.catch((err) => {
|
||||
const retired = retiredRejectedPtyIds.has(ptyId)
|
||||
if (isPtyAlreadyGoneError(err)) {
|
||||
const proof = PtyExitProof.from('absent')
|
||||
const proof = PtyExitProof.from('absent', stoppingIncarnationId)
|
||||
const incarnationId = finishPtyShutdown(ptyId, connectionId, store, proof)
|
||||
if (!retired) {
|
||||
publishSyntheticExit(deps, ptyId, -1, incarnationId, proof)
|
||||
}
|
||||
return
|
||||
}
|
||||
reportStopFailure(err)
|
||||
reportStopFailure(err, stoppingIncarnationId)
|
||||
})
|
||||
return true
|
||||
}
|
||||
const killedIncarnationId = ptyIncarnationById.get(ptyId)
|
||||
const startupPromise = getLocalPtyProviderStartupPromise(resolvePtyExecutionHost(ptyId))
|
||||
if (startupPromise) {
|
||||
// Why: select the provider after the daemon swap; the fallback first can report success while orphaning a daemon PTY.
|
||||
void startupPromise.then(killWithCurrentProvider).catch(reportStopFailure)
|
||||
void startupPromise
|
||||
.then(killWithCurrentProvider)
|
||||
.catch((err) => reportStopFailure(err, killedIncarnationId))
|
||||
return true
|
||||
}
|
||||
return killWithCurrentProvider()
|
||||
@@ -142,9 +151,14 @@ export function killPtyFromRuntimeController(
|
||||
export function retireRejectedPtyFromRuntimeController(
|
||||
deps: PtyRuntimeControllerDeps,
|
||||
ptyId: string,
|
||||
stopConfirmed: boolean
|
||||
stopConfirmed: boolean,
|
||||
incarnationId: string | undefined
|
||||
): void {
|
||||
const { runtime, store, rememberRetiredRejectedPty, finishPtyShutdown } = deps
|
||||
// A same-id respawn owns the record now; this verdict is about the rejected incarnation only.
|
||||
if (!isCurrentPtyExit({ id: ptyId, incarnationId })) {
|
||||
return
|
||||
}
|
||||
rememberRetiredRejectedPty(ptyId)
|
||||
if (!stopConfirmed) {
|
||||
const unproven = PtyRetirementWithoutProof.because(
|
||||
@@ -154,17 +168,17 @@ export function retireRejectedPtyFromRuntimeController(
|
||||
runtime?.markPtyLivenessUnverifiable?.(ptyId, unproven.unverifiableReason)
|
||||
return
|
||||
}
|
||||
publishSyntheticExit(deps, ptyId, -1, ptyIncarnationById.get(ptyId), unproven)
|
||||
publishSyntheticExit(deps, ptyId, -1, incarnationId, unproven)
|
||||
return
|
||||
}
|
||||
// Why: a completed stop already cleared provider state, tombstoned the lease and told the
|
||||
// renderer; repeating that double-fires the exit IPC. The runtime still needs code 0 so an
|
||||
// SSH pane retires for good instead of staying preserved by the stop's negative exit.
|
||||
if (!ptyOwnership.has(ptyId)) {
|
||||
runtime?.onPtyExit(ptyId, 0, ptyIncarnationById.get(ptyId))
|
||||
runtime?.onPtyExit(ptyId, 0, incarnationId)
|
||||
return
|
||||
}
|
||||
const proof = PtyExitProof.from('stop-confirmed')
|
||||
const proof = PtyExitProof.from('stop-confirmed', incarnationId)
|
||||
const connectionId = getPtySshConnectionId(ptyId)
|
||||
publishSyntheticExit(deps, ptyId, 0, finishPtyShutdown(ptyId, connectionId, store, proof), proof)
|
||||
}
|
||||
@@ -197,6 +211,7 @@ export async function stopAndWaitPtyFromRuntimeController(
|
||||
if (opts?.keepHistory) {
|
||||
return false
|
||||
}
|
||||
const stoppingIncarnationId = ptyIncarnationById.get(ptyId)
|
||||
const deadline = opts?.deadlineMs ?? Date.now() + OBSERVED_EXIT_RECORD_WAIT_MS
|
||||
const stopped = await Promise.race([
|
||||
stopLegacyRelayPaneAndWait(ptyId, deadline),
|
||||
@@ -205,7 +220,7 @@ export async function stopAndWaitPtyFromRuntimeController(
|
||||
if (stopped) {
|
||||
// The bridge retired the lease and holds no pane: the relay reported the exit, or the
|
||||
// listing lacks it. The runtime learns it here.
|
||||
const proof = PtyExitProof.from('absent')
|
||||
const proof = PtyExitProof.from('absent', stoppingIncarnationId)
|
||||
publishSyntheticExit(deps, ptyId, 0, finishPtyShutdown(ptyId, null, store, proof), proof)
|
||||
}
|
||||
return stopped
|
||||
@@ -250,6 +265,7 @@ export async function stopAndWaitPtyFromRuntimeController(
|
||||
}
|
||||
return false
|
||||
}
|
||||
const stoppingIncarnationId = ptyIncarnationById.get(ptyId)
|
||||
let providerExitObserved = false
|
||||
try {
|
||||
providerExitObserved = await shutdownProviderAndDetectExit(provider, ptyId, {
|
||||
@@ -272,7 +288,10 @@ export async function stopAndWaitPtyFromRuntimeController(
|
||||
console.warn(`[pty] Failed to verify PTY ${ptyId} stopped: ${errorMessage(err)}`)
|
||||
return false
|
||||
}
|
||||
const proof = PtyExitProof.from(providerExitObserved ? 'exit-observed' : 'absent')
|
||||
const proof = PtyExitProof.from(
|
||||
providerExitObserved ? 'exit-observed' : 'absent',
|
||||
stoppingIncarnationId
|
||||
)
|
||||
const incarnationId = finishPtyShutdown(ptyId, connectionId, store, proof)
|
||||
if (!providerExitObserved) {
|
||||
// The owning provider's fresh inventory observed absence, so this is a
|
||||
|
||||
@@ -1,9 +1,4 @@
|
||||
import { isValidTerminalTabId } from '../../../../shared/terminal-tab-id'
|
||||
import {
|
||||
ptyOwnership,
|
||||
ptyIncarnationById,
|
||||
setAdoptedPtyOwnership
|
||||
} from '../provider/ownership-state'
|
||||
import { ptySizes } from '../delivery/visibility-state'
|
||||
import { commitRuntimePtySize } from './spawn-commit-pty-size'
|
||||
import {
|
||||
@@ -27,7 +22,6 @@ import {
|
||||
isNativeWindowsLocalPtySpawn,
|
||||
markNativeWindowsConptyPty
|
||||
} from '../../../runtime/terminal-model-query-authority'
|
||||
import { LOCAL_EXECUTION_HOST_ID } from '../../../../shared/execution-host'
|
||||
import { createTerminalSessionStateSaveFailureMessage } from '../../../../shared/terminal-session-state-save-failure'
|
||||
import { resolvePaneSpawnReservation } from '../pane/spawn-reservation'
|
||||
import { admitProviderReattachLaunchIdentity } from '../pane/launch-authority'
|
||||
@@ -35,7 +29,7 @@ import { spawnCommitBindingOrigin } from '../../../persistence/loading-store/pty
|
||||
import type { RuntimePtySpawnState } from './spawn-state'
|
||||
import { commitPtyWithOpenCodePromptIntent } from '../../../opencode/opencode-startup-prompt-owner'
|
||||
import {
|
||||
admitPtyReattachOwnership,
|
||||
admitPtySpawnOwnership,
|
||||
discardUnpersistedPtySpawn,
|
||||
registerPersistedPtySpawn
|
||||
} from '../pane/spawn-registration'
|
||||
@@ -46,7 +40,7 @@ export async function commitRuntimePtySpawn(ctx: RuntimePtySpawnState) {
|
||||
|
||||
async function commitReservedRuntimePtySpawn(ctx: RuntimePtySpawnState) {
|
||||
const args = ctx.args
|
||||
admitPtyReattachOwnership(ctx.deps.runtime, ctx.result)
|
||||
admitPtySpawnOwnership(ctx.deps.runtime, ctx.result)
|
||||
const providerReattachLaunchIdentity = admitProviderReattachLaunchIdentity(ctx.result)
|
||||
if (
|
||||
isNativeWindowsLocalPtySpawn({
|
||||
@@ -104,11 +98,7 @@ async function commitReservedRuntimePtySpawn(ctx: RuntimePtySpawnState) {
|
||||
ctx.result,
|
||||
ctx.hostSessionBinding?.expectedSourceBinding
|
||||
)
|
||||
setAdoptedPtyOwnership(ctx.result.id)
|
||||
ctx.deps.runtime?.registerPreAllocatedHandleForPty(ctx.result.id, owner.surface.terminalHandle)
|
||||
if (ctx.result.incarnationId) {
|
||||
ptyIncarnationById.set(ctx.result.id, ctx.result.incarnationId)
|
||||
}
|
||||
ctx.deps.options?.onCodexHomePtySpawned?.({
|
||||
id: ctx.result.id,
|
||||
codexHomePath: ctx.selectedCodexHomePath,
|
||||
@@ -158,7 +148,7 @@ async function commitReservedRuntimePtySpawn(ctx: RuntimePtySpawnState) {
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[pty] failed to persist runtime PTY binding after spawn:', err)
|
||||
await discardUnpersistedPtySpawn(ctx.provider, ctx.result)
|
||||
await discardUnpersistedPtySpawn(ctx.provider, ctx.result, ctx.deps.runtime)
|
||||
if (err instanceof Error && err.message === 'terminal_split_source_not_found') {
|
||||
throw err
|
||||
}
|
||||
@@ -205,10 +195,6 @@ async function commitReservedRuntimePtySpawn(ctx: RuntimePtySpawnState) {
|
||||
if (args.preAllocatedHandle && !ctx.stablePaneOwner?.handle) {
|
||||
ctx.deps.runtime?.registerPreAllocatedHandleForPty(ctx.result.id, args.preAllocatedHandle)
|
||||
}
|
||||
ptyOwnership.set(ctx.result.id, LOCAL_EXECUTION_HOST_ID)
|
||||
if (ctx.result.incarnationId) {
|
||||
ptyIncarnationById.set(ctx.result.id, ctx.result.incarnationId)
|
||||
}
|
||||
|
||||
commitRuntimePtySize(ctx, ctx.result)
|
||||
if (ctx.effectiveSessionAppId !== undefined && ctx.effectiveSessionAppId !== ctx.result.id) {
|
||||
|
||||
@@ -155,7 +155,7 @@ describe('Store SSH remote PTY bindings across host partitions', () => {
|
||||
store.setWorkspaceSession(makeBoundHostSession(ptyId), 'local')
|
||||
store.setWorkspaceSession(makeBoundHostSession(ptyId), 'ssh:ssh-1')
|
||||
|
||||
store.markSshRemotePtyLease('ssh-1', ptyId, PtyExitProof.from('exit-observed'))
|
||||
store.markSshRemotePtyLease('ssh-1', ptyId, PtyExitProof.from('exit-observed', undefined))
|
||||
|
||||
for (const hostId of ['local', 'ssh:ssh-1']) {
|
||||
const session = store.getWorkspaceSession(hostId)
|
||||
|
||||
@@ -80,7 +80,7 @@ describe('operator-closed SSH lease tombstones', () => {
|
||||
store.markSshRemotePtyLease(
|
||||
'ssh-1',
|
||||
'ssh:ssh-1@@remote-pty',
|
||||
PtyExitProof.from('exit-observed')
|
||||
PtyExitProof.from('exit-observed', undefined)
|
||||
)
|
||||
|
||||
// `isRestorablePtyBinding` still consults this row to refuse replaying that binding.
|
||||
@@ -101,7 +101,7 @@ describe('operator-closed SSH lease tombstones', () => {
|
||||
store.markSshRemotePtyLease(
|
||||
'ssh-1',
|
||||
'ssh:ssh-1@@remote-pty',
|
||||
PtyExitProof.from('exit-observed')
|
||||
PtyExitProof.from('exit-observed', undefined)
|
||||
)
|
||||
|
||||
expect(store.getSshRemotePtyLeases('ssh-1')).toEqual([
|
||||
@@ -136,7 +136,7 @@ describe('operator-closed SSH lease tombstones', () => {
|
||||
store.markSshRemotePtyLease(
|
||||
'ssh-1',
|
||||
'ssh:ssh-1@@remote-pty-3',
|
||||
PtyExitProof.from('exit-observed')
|
||||
PtyExitProof.from('exit-observed', undefined)
|
||||
)
|
||||
|
||||
expect(store.getSshRemotePtyLeases('ssh-1')).toEqual([
|
||||
@@ -163,7 +163,7 @@ describe('operator-closed SSH lease tombstones', () => {
|
||||
store.markSshRemotePtyLease(
|
||||
'ssh-1',
|
||||
'ssh:ssh-1@@remote-pty-1',
|
||||
PtyExitProof.from('exit-observed')
|
||||
PtyExitProof.from('exit-observed', undefined)
|
||||
)
|
||||
|
||||
// Other targets are untouched: the pass is scoped to the one whose bindings were just scrubbed.
|
||||
|
||||
@@ -553,7 +553,7 @@ describe('Store', () => {
|
||||
store.markSshRemotePtyLease(
|
||||
'ssh-1',
|
||||
'ssh:ssh-2@@remote-pty',
|
||||
PtyExitProof.from('exit-observed')
|
||||
PtyExitProof.from('exit-observed', undefined)
|
||||
)
|
||||
).toThrow('belongs to SSH connection "ssh-2"')
|
||||
})
|
||||
@@ -570,7 +570,7 @@ describe('Store', () => {
|
||||
store.markSshRemotePtyLease(
|
||||
'ssh-1',
|
||||
'ssh:ssh-1@@remote-pty',
|
||||
PtyExitProof.from('exit-observed')
|
||||
PtyExitProof.from('exit-observed', undefined)
|
||||
)
|
||||
|
||||
// An unresolved id would have left the lease `attached`; this unbound row is retired instead.
|
||||
@@ -583,7 +583,7 @@ describe('Store', () => {
|
||||
store.markSshRemotePtyLease(
|
||||
'ssh-1',
|
||||
'ssh:ssh-1@@remote-pty',
|
||||
PtyExitProof.from('exit-observed')
|
||||
PtyExitProof.from('exit-observed', undefined)
|
||||
)
|
||||
|
||||
const session = store.getWorkspaceSession()
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { isTerminalSessionStorageCapacityFailure } from '../../../shared/terminal-session-state-save-failure'
|
||||
import { afterEach, expect, it, vi } from 'vitest'
|
||||
import { LOCAL_EXECUTION_HOST_ID } from '../../../shared/execution-host'
|
||||
import { deferred, fixture, admitRuntimeSpawnPane } from './profile-state-delayed-authority-fixture'
|
||||
import { OrcaRuntimeService } from '../../runtime/orca-runtime'
|
||||
import { commitPtyIpcSpawn } from '../../ipc/pty/ipc/spawn-commit'
|
||||
@@ -172,3 +173,140 @@ it('does not label an IPC binding ownership refusal as a storage failure', async
|
||||
vi.spyOn(store, 'persistPtyBinding').mockResolvedValue(false)
|
||||
await expect(commitPtyIpcSpawn(ctx)).rejects.toThrow('terminal_pane_owner_changed')
|
||||
})
|
||||
|
||||
it.each(
|
||||
['ipc', 'runtime'].flatMap((controller) =>
|
||||
[false, true].map((exitsDuringSave) => ({ controller, exitsDuringSave }))
|
||||
)
|
||||
)(
|
||||
'after a failed $controller save and a failed discard shutdown (exit during save: $exitsDuringSave)',
|
||||
async ({ controller, exitsDuringSave }) => {
|
||||
const { store } = await fixture()
|
||||
const runtime = new OrcaRuntimeService(store)
|
||||
runtime.onPtySpawned(binding.ptyId, binding.incarnationId)
|
||||
const deps = createPtySpawnCommitDependencies(runtime, store)
|
||||
vi.spyOn(store, 'persistPtyBinding').mockImplementation(async () => {
|
||||
if (exitsDuringSave) {
|
||||
// The provider's exit listener, as local-configure wires it.
|
||||
clearProviderPtyState(binding.ptyId)
|
||||
ptyOwnership.delete(binding.ptyId)
|
||||
await runtime.onPtyExit(binding.ptyId, 0, binding.incarnationId, {
|
||||
providerExitObserved: true
|
||||
})
|
||||
}
|
||||
throw new Error('disk full')
|
||||
})
|
||||
let commit: () => Promise<unknown>
|
||||
let provider: { shutdown: (...args: never[]) => Promise<void> }
|
||||
if (controller === 'ipc') {
|
||||
const ctx = createPtyIpcSpawnState(deps, { ...binding, cols: 80, rows: 24 })
|
||||
ctx.result = { id: binding.ptyId, incarnationId: binding.incarnationId }
|
||||
ctx.validatedLeafId = binding.leafId
|
||||
provider = ctx.provider
|
||||
commit = () => commitPtyIpcSpawn(ctx)
|
||||
} else {
|
||||
const ctx = createRuntimePtySpawnState(deps, { ...binding, cols: 80, rows: 24 })
|
||||
ctx.result = { id: binding.ptyId, incarnationId: binding.incarnationId }
|
||||
ctx.metadataLeafId = binding.leafId
|
||||
await admitRuntimeSpawnPane(store, binding)
|
||||
ctx.hostSessionBinding = { store, ...binding }
|
||||
provider = ctx.provider
|
||||
commit = () => commitRuntimePtySpawn(ctx)
|
||||
}
|
||||
vi.spyOn(provider, 'shutdown').mockRejectedValue(new Error('shutdown timed out'))
|
||||
|
||||
await expect(commit()).rejects.toThrow()
|
||||
|
||||
if (exitsDuringSave) {
|
||||
expect(ptyOwnership.has(binding.ptyId)).toBe(false)
|
||||
expect(runtime.getPtyLivenessVerdict(binding.ptyId)?.status).toBe('exited')
|
||||
} else {
|
||||
expect(ptyOwnership.get(binding.ptyId)).toBe(LOCAL_EXECUTION_HOST_ID)
|
||||
expect(ptyIncarnationById.get(binding.ptyId)).toBe(binding.incarnationId)
|
||||
expect(runtime.getPtyLivenessVerdict(binding.ptyId)?.status).toBe('unverifiable')
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
it.each(
|
||||
['ipc', 'runtime'].flatMap((controller) =>
|
||||
[false, true].map((saveAndShutdownFail) => ({ controller, saveAndShutdownFail }))
|
||||
)
|
||||
)(
|
||||
'leaves a $controller spawn that exited before its commit began exited (save and shutdown fail: $saveAndShutdownFail)',
|
||||
async ({ controller, saveAndShutdownFail }) => {
|
||||
const { store } = await fixture()
|
||||
const runtime = new OrcaRuntimeService(store)
|
||||
runtime.onPtySpawned(binding.ptyId, binding.incarnationId)
|
||||
// The provider's exit listener ran before the commit recorded ownership.
|
||||
await runtime.onPtyExit(binding.ptyId, 0, binding.incarnationId, { providerExitObserved: true })
|
||||
const deps = createPtySpawnCommitDependencies(runtime, store)
|
||||
if (saveAndShutdownFail) {
|
||||
vi.spyOn(store, 'persistPtyBinding').mockRejectedValue(new Error('disk full'))
|
||||
}
|
||||
let commit: () => Promise<unknown>
|
||||
let provider: { shutdown: (...args: never[]) => Promise<void> }
|
||||
if (controller === 'ipc') {
|
||||
const ctx = createPtyIpcSpawnState(deps, { ...binding, cols: 80, rows: 24 })
|
||||
ctx.result = { id: binding.ptyId, incarnationId: binding.incarnationId }
|
||||
ctx.metadataLeafId = binding.leafId
|
||||
ctx.validatedLeafId = binding.leafId
|
||||
provider = ctx.provider
|
||||
commit = () => commitPtyIpcSpawn(ctx)
|
||||
} else {
|
||||
const ctx = createRuntimePtySpawnState(deps, { ...binding, cols: 80, rows: 24 })
|
||||
ctx.result = { id: binding.ptyId, incarnationId: binding.incarnationId }
|
||||
ctx.metadataLeafId = binding.leafId
|
||||
await admitRuntimeSpawnPane(store, binding)
|
||||
ctx.hostSessionBinding = { store, ...binding }
|
||||
provider = ctx.provider
|
||||
commit = () => commitRuntimePtySpawn(ctx)
|
||||
}
|
||||
if (saveAndShutdownFail) {
|
||||
vi.spyOn(provider, 'shutdown').mockRejectedValue(new Error('shutdown timed out'))
|
||||
}
|
||||
|
||||
await commit().catch(() => undefined)
|
||||
|
||||
expect(ptyOwnership.has(binding.ptyId)).toBe(false)
|
||||
expect(ptyIncarnationById.has(binding.ptyId)).toBe(false)
|
||||
expect(runtime.getPtyLivenessVerdict(binding.ptyId)?.status).toBe('exited')
|
||||
}
|
||||
)
|
||||
|
||||
it.each(['ipc', 'runtime'])(
|
||||
'tracks a %s same-id respawn whose predecessor left an exited verdict',
|
||||
async (controller) => {
|
||||
const { store } = await fixture()
|
||||
const runtime = new OrcaRuntimeService(store)
|
||||
runtime.onPtySpawned(binding.ptyId, replacementIncarnation)
|
||||
await runtime.onPtyExit(binding.ptyId, 0, replacementIncarnation, {
|
||||
providerExitObserved: true
|
||||
})
|
||||
// A daemon respawn under the same id does not reset that id-keyed verdict.
|
||||
const deps = createPtySpawnCommitDependencies(runtime, store)
|
||||
let commit: () => Promise<unknown>
|
||||
if (controller === 'ipc') {
|
||||
const ctx = createPtyIpcSpawnState(deps, { ...binding, cols: 80, rows: 24 })
|
||||
ctx.result = { id: binding.ptyId, incarnationId: binding.incarnationId }
|
||||
ctx.metadataLeafId = binding.leafId
|
||||
ctx.validatedLeafId = binding.leafId
|
||||
commit = () => commitPtyIpcSpawn(ctx)
|
||||
} else {
|
||||
const ctx = createRuntimePtySpawnState(deps, { ...binding, cols: 80, rows: 24 })
|
||||
ctx.result = { id: binding.ptyId, incarnationId: binding.incarnationId }
|
||||
ctx.metadataLeafId = binding.leafId
|
||||
await admitRuntimeSpawnPane(store, binding)
|
||||
ctx.hostSessionBinding = { store, ...binding }
|
||||
commit = () => commitRuntimePtySpawn(ctx)
|
||||
}
|
||||
|
||||
await commit().catch(() => undefined)
|
||||
|
||||
expect(ptyOwnership.get(binding.ptyId)).toBe(LOCAL_EXECUTION_HOST_ID)
|
||||
expect(ptyIncarnationById.get(binding.ptyId)).toBe(binding.incarnationId)
|
||||
await runtime.onPtyExit(binding.ptyId, 0, binding.incarnationId, {
|
||||
providerExitObserved: true
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
@@ -19,10 +19,15 @@ export type PtyExitProofSource =
|
||||
| 'stop-confirmed'
|
||||
|
||||
export class PtyExitProof {
|
||||
private constructor(readonly source: PtyExitProofSource) {}
|
||||
private constructor(
|
||||
readonly source: PtyExitProofSource,
|
||||
readonly incarnationId: string | undefined
|
||||
) {}
|
||||
|
||||
static from(source: PtyExitProofSource): PtyExitProof {
|
||||
return new PtyExitProof(source)
|
||||
/** `incarnationId` is the one the evidence was about (`undefined` only for records that carry
|
||||
* none, like relay leases): a same-id respawn must not inherit it. */
|
||||
static from(source: PtyExitProofSource, incarnationId: string | undefined): PtyExitProof {
|
||||
return new PtyExitProof(source, incarnationId)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ const TAB_ID = 'tab-remote'
|
||||
const SOURCE_LEAF_ID = '11111111-1111-4111-8111-111111111111'
|
||||
const SOURCE_PTY_ID = 'pty-source'
|
||||
const SPLIT_PTY_ID = 'pty-split'
|
||||
const SPLIT_INCARNATION_ID = 'pty-split-inc'
|
||||
|
||||
function sourceLayout(): TerminalLayoutSnapshot {
|
||||
return {
|
||||
@@ -109,11 +110,11 @@ function createHarness(
|
||||
},
|
||||
persistPtyBinding: () => true
|
||||
})
|
||||
let resolveSpawn: ((result: { id: string }) => void) | undefined
|
||||
let resolveSpawn: ((result: { id: string; incarnationId: string }) => void) | undefined
|
||||
const spawn = options.deferSpawn
|
||||
? vi.fn(
|
||||
() =>
|
||||
new Promise<{ id: string }>((resolve) => {
|
||||
new Promise<{ id: string; incarnationId?: string }>((resolve) => {
|
||||
resolveSpawn = resolve
|
||||
})
|
||||
)
|
||||
@@ -222,7 +223,7 @@ function createHarness(
|
||||
}
|
||||
},
|
||||
resolveReveal: () => resolveReveal?.({ tabId: TAB_ID }),
|
||||
resolveSpawn: () => resolveSpawn?.({ id: SPLIT_PTY_ID })
|
||||
resolveSpawn: () => resolveSpawn?.({ id: SPLIT_PTY_ID, incarnationId: SPLIT_INCARNATION_ID })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -461,7 +462,7 @@ describe('remote runtime terminal split authority', () => {
|
||||
expect.objectContaining({ deadlineMs: expect.any(Number) })
|
||||
)
|
||||
expect(harness.kill).not.toHaveBeenCalled()
|
||||
expect(harness.retireRejectedPty).toHaveBeenCalledWith(SPLIT_PTY_ID, true)
|
||||
expect(harness.retireRejectedPty).toHaveBeenCalledWith(SPLIT_PTY_ID, true, SPLIT_INCARNATION_ID)
|
||||
})
|
||||
|
||||
it('refuses a split whose source is only projected, before spawning', async () => {
|
||||
@@ -495,7 +496,11 @@ describe('remote runtime terminal split authority', () => {
|
||||
|
||||
await expect(split).rejects.toThrow('terminal_split_source_not_found')
|
||||
expect(harness.kill).toHaveBeenCalledWith(SPLIT_PTY_ID)
|
||||
expect(harness.retireRejectedPty).toHaveBeenCalledWith(SPLIT_PTY_ID, false)
|
||||
expect(harness.retireRejectedPty).toHaveBeenCalledWith(
|
||||
SPLIT_PTY_ID,
|
||||
false,
|
||||
SPLIT_INCARNATION_ID
|
||||
)
|
||||
expect(harness.getSession().terminalLayoutsByTabId[TAB_ID]?.root).toEqual({
|
||||
type: 'leaf',
|
||||
leafId: SOURCE_LEAF_ID
|
||||
|
||||
@@ -130,7 +130,8 @@ export type RuntimePtyController = {
|
||||
* False on doubt (absent session, SSH-scoped id, non-daemon provider). */
|
||||
attach?(ptyId: string): Promise<boolean>
|
||||
kill(ptyId: string): boolean
|
||||
retireRejectedPty?(ptyId: string, stopConfirmed: boolean): void
|
||||
/** `incarnationId` is the rejected spawn's; a same-id respawn since then is left alone. */
|
||||
retireRejectedPty?(ptyId: string, stopConfirmed: boolean, incarnationId: string | undefined): void
|
||||
stopAndWait?(
|
||||
ptyId: string,
|
||||
opts?: { keepHistory?: boolean; deadlineMs?: number }
|
||||
|
||||
@@ -188,7 +188,7 @@ export abstract class OrcaRuntimeWithSplitPtyBackedTerminal extends OrcaRuntimeW
|
||||
}
|
||||
}
|
||||
try {
|
||||
this.ptyController.retireRejectedPty?.(result.id, stopped)
|
||||
this.ptyController.retireRejectedPty?.(result.id, stopped, result.incarnationId)
|
||||
} catch {
|
||||
// Best-effort cleanup; preserve the original split authority error.
|
||||
}
|
||||
|
||||
@@ -88,7 +88,7 @@ describe('a pane whose SSH lease expired can still be re-adopted', () => {
|
||||
it('drops the persisted owner after the lease is terminated', () => {
|
||||
const store = storeWithBoundRemotePane()
|
||||
|
||||
store.markSshRemotePtyLease(TARGET, APP_PTY_ID, PtyExitProof.from('exit-observed'))
|
||||
store.markSshRemotePtyLease(TARGET, APP_PTY_ID, PtyExitProof.from('exit-observed', undefined))
|
||||
|
||||
expect(
|
||||
resolvePersistedStablePaneOwner(store, makePaneKey(TAB, TEST_LEAF_1), WORKTREE, TARGET)
|
||||
|
||||
@@ -119,7 +119,7 @@ function bridgeFor(targetId: string): LegacyRelayTargetBridge | null {
|
||||
targetId,
|
||||
leases: () => installed.leases(targetId),
|
||||
retireLease: (relayPtyId, proof) =>
|
||||
installed.retireLease(targetId, relayPtyId, PtyExitProof.from(proof)),
|
||||
installed.retireLease(targetId, relayPtyId, PtyExitProof.from(proof, undefined)),
|
||||
listEndpoints: async (signal) => {
|
||||
const conn = installed.connection(targetId)
|
||||
return conn ? listLegacyRelayEndpoints(conn, targetId, signal) : { kind: 'unverifiable' }
|
||||
|
||||
@@ -127,9 +127,9 @@ describe('acting on a terminal proof', () => {
|
||||
provenPtyIds: ['att', 'det', 'exp', 'ter']
|
||||
})
|
||||
expect(markSshRemotePtyLease.mock.calls).toEqual([
|
||||
['ssh-1', 'att', PtyExitProof.from('host-census')],
|
||||
['ssh-1', 'det', PtyExitProof.from('host-census')],
|
||||
['ssh-1', 'exp', PtyExitProof.from('host-census')]
|
||||
['ssh-1', 'att', PtyExitProof.from('host-census', undefined)],
|
||||
['ssh-1', 'det', PtyExitProof.from('host-census', undefined)],
|
||||
['ssh-1', 'exp', PtyExitProof.from('host-census', undefined)]
|
||||
])
|
||||
})
|
||||
|
||||
|
||||
@@ -118,7 +118,7 @@ export function retireProvenExitedLeases(
|
||||
return
|
||||
}
|
||||
const proven = new Set(proof.provenPtyIds)
|
||||
const census = PtyExitProof.from('host-census')
|
||||
const census = PtyExitProof.from('host-census', undefined)
|
||||
for (const lease of store.getSshRemotePtyLeases(targetId)) {
|
||||
if (lease.state !== 'terminated' && proven.has(lease.ptyId)) {
|
||||
store.markSshRemotePtyLease(targetId, lease.ptyId, census)
|
||||
|
||||
Reference in New Issue
Block a user