mirror of
https://github.com/stablyai/orca.git
synced 2026-10-29 08:07:18 +00:00
persist terminal teardown ownership across restarts
This commit is contained in:
@@ -3765,13 +3765,13 @@
|
||||
"providers": ["local", "daemon", "ssh", "remote-runtime"],
|
||||
"coveredPlatforms": ["macos"],
|
||||
"coveredProviders": ["local", "ssh"],
|
||||
"coverageNotes": "A macOS unit harness loads the installed patched node-pty WindowsTerminal prototype without constructing native state, proving pre-ready kill/destroy dispatch immediately. A relay artifact harness reproduces vanilla npm node-pty retaining one deferred kill with zero native close calls, then proves both Windows relay artifacts hash, ship, apply, include the local ConPTY console-list fallback, and idempotently verify the fix. Relay tests simulate win32 shutdown, repeated stale-spawn native failure, one-shot ownership, and retry after native failure without deleting the only handle before close is proven. Relay boot UUIDs scope recycled pty-N counters through initial spawn, durable-lease persistence, reattach, and SSH target-id migration; mismatched generations expire before raw attach, while pane identity protects legacy leases. Renderer retry ownership keeps one coalesced record per still-live unconfirmed PTY and uses one shared one-shot timer to retry every due owner with capped exponential backoff; it never drops a live owner merely to cap bookkeeping. Local Windows shutdown retains provider records and listeners after native throw unless onExit independently proves death. A disconnected SSH provider rejects shutdown without tombstoning its attachable lease. Project removal keeps local or remote ownership until provider-verified shutdown and successful VM cleanup; runtime repo removal drains create-first worktree creation plus headless, renderer-backed mobile, renderer split, and selector-less renderer terminal admissions before fresh ownership/liveness snapshots, stop, and metadata mutation. Desktop and duplicate-host repo removal route through the same transaction; legacy worktree creation, renderer PTY spawn, and every local/SSH/folder worktree removal outcome—including Windows stale registration and orphan-compatible fallback—are fenced and verified before destructive mutation or metadata purge. Hidden local/SSH and remote-runtime background-launch cleanup retain exact PTY/handle retry ownership with bounded one-shot scheduling. Targeted terminal.stop probes skip unrelated sessions. The same relay contract serves paired web/mobile clients; no live Windows ConPTY or Windows-over-SSH run is recorded yet.",
|
||||
"coverageNotes": "A macOS unit harness loads the installed patched node-pty WindowsTerminal prototype without constructing native state, proving pre-ready kill/destroy dispatch immediately. A relay artifact harness reproduces vanilla npm node-pty retaining one deferred kill with zero native close calls, then proves both Windows relay artifacts hash, ship, apply, include the local ConPTY console-list fallback, and idempotently verify the fix. Relay tests simulate win32 shutdown, repeated stale-spawn native failure, one-shot ownership, and retry after native failure without deleting the only handle before close is proven. Relay boot UUIDs scope recycled pty-N counters through initial spawn, durable-lease persistence, reattach, and SSH target-id migration; mismatched generations expire before raw attach, while pane identity protects legacy leases. Main-process retry owners persist local daemon and remote-runtime teardown intent across renderer and app restarts, coalesce by exact provider identity, use one shared one-shot timer per owner, and fence SSH relay plus remote-runtime generation reuse before close. Local Windows shutdown retains provider records and listeners after native throw unless onExit independently proves death. A disconnected SSH provider rejects shutdown without tombstoning its attachable lease. Project removal keeps local or remote ownership until provider-verified shutdown and successful VM cleanup; runtime repo removal drains create-first worktree creation plus headless, renderer-backed mobile, renderer split, and selector-less renderer terminal admissions before fresh ownership/liveness snapshots, stop, and metadata mutation. Desktop and duplicate-host repo removal route through the same transaction; legacy worktree creation, renderer PTY spawn, and every local/SSH/folder worktree removal outcome—including Windows stale registration and orphan-compatible fallback—are fenced and verified before destructive mutation or metadata purge. Hidden local/SSH and remote-runtime background-launch cleanup retain exact PTY/handle retry ownership with bounded one-shot scheduling. Targeted terminal.stop probes skip unrelated sessions. The same relay contract serves paired web/mobile clients; no live Windows ConPTY or Windows-over-SSH run is recorded yet.",
|
||||
"motivatingLinks": [
|
||||
"https://github.com/stablyai/orca/issues/8260",
|
||||
"https://github.com/stablyai/orca/pull/8279"
|
||||
],
|
||||
"invariant": "A Windows PTY closed before its first output executes exactly one no-argument node-pty shutdown immediately; Orca never deletes ownership without accepted native shutdown or an observed exit, and a later retry can target only the same relay generation and pane while project removal fences both worktree and terminal creation.",
|
||||
"oracle": "Load the installed patched WindowsTerminal prototype in a pre-ready state and assert kill and destroy each close the terminal and native agent once with zero deferred work, while signal arguments still throw. Build both Windows SSH relay artifacts, reproduce vanilla npm node-pty with close=0/native-kill=0/deferred=1, apply the shipped patch, and assert close=1/native-kill=1/deferred=0, AttachConsole failure falls back to the shell pid, and the artifact version derives from all patch bytes. Assert remote npm install executes patch enforcement before its runtime probe and never finalizes a failed patch. In local and relay providers, assert native throws retain exact ownership/listeners and retry succeeds, while a synchronous onExit is accepted as independent death proof; provider absence must retain the SSH lease and reject shutdown. Persist relay generation with every durable lease, expire mismatches before attach, and preserve generation through target migration. Reject 65 renderer kills, retain all 65 exact owners, then prove one one-shot deadline retries all due owners and later capped deadlines retry without another lifecycle event. Reject hidden local and remote-runtime background cleanup, retain exact tab/handle ownership, and prove later retry succeeds. Start worktree and terminal creation surfaces before whole-project and host-scoped project removal and assert removal performs no ownership/liveness snapshot until they publish; removal-first and failed/concurrent admissions must reject or release without deadlock. Start a real worktree removal, pause its preflight, and assert a late renderer terminal admission is rejected; assert every successful registered, orphaned, leftover, already-missing, and stale-registration removal path verifies shutdown before mutation or metadata purge; assert desktop repo removal, worktree create/remove, and PTY spawn handlers all enter their runtime-owned transactions before mutation or provider work.",
|
||||
"oracle": "Load the installed patched WindowsTerminal prototype in a pre-ready state and assert kill and destroy each close the terminal and native agent once with zero deferred work, while signal arguments still throw. Build both Windows SSH relay artifacts, reproduce vanilla npm node-pty with close=0/native-kill=0/deferred=1, apply the shipped patch, and assert close=1/native-kill=1/deferred=0, AttachConsole failure falls back to the shell pid, and the artifact version derives from all patch bytes. Assert remote npm install executes patch enforcement before its runtime probe and never finalizes a failed patch. In local and relay providers, assert native throws retain exact ownership/listeners and retry succeeds, while a synchronous onExit is accepted as independent death proof; provider absence must retain the SSH lease and reject shutdown. Persist relay generation with every durable lease, expire mismatches before attach, and preserve generation through target migration. Reject local daemon and remote-runtime teardown, reload the durable store, replace the startup provider, and prove capped retries drain exact identities with zero final retry timers. Replace SSH relay and remote-runtime generations while a retry is retained and prove the new owner is never closed. Start worktree and terminal creation surfaces before whole-project and host-scoped project removal and assert removal performs no ownership/liveness snapshot until they publish; removal-first and failed/concurrent admissions must reject or release without deadlock. Start a real worktree removal, pause its preflight, and assert a late renderer terminal admission is rejected; assert every successful registered, orphaned, leftover, already-missing, and stale-registration removal path verifies shutdown before mutation or metadata purge; assert desktop repo removal, worktree create/remove, and PTY spawn handlers all enter their runtime-owned transactions before mutation or provider work.",
|
||||
"commands": [
|
||||
"pnpm exec vitest run --config config/vitest.config.ts config/scripts/build-relay-windows-node-pty-patch.test.mjs src/main/ssh/ssh-relay-native-deps-install.test.ts src/relay/node-pty-windows-pre-ready-shutdown.test.ts src/relay/pty-handler.test.ts src/renderer/src/components/terminal-pane/pty-transport.test.ts src/renderer/src/store/slices/repos-remove-project-purge-leak.test.ts src/renderer/src/store/slices/repos.test.ts src/renderer/src/store/slices/repos-runtime-vm-removal.test.ts src/renderer/src/lib/ephemeral-vm-runtime-cleanup.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts",
|
||||
@@ -3781,7 +3781,7 @@
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/repos-create.test.ts src/main/window/attach-main-window-services.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/worktrees-windows.test.ts src/main/ipc/pty.test.ts src/main/runtime/orca-runtime.test.ts src/renderer/src/lib/launch-agent-background-session.test.ts src/renderer/src/store/slices/repos-host-identity-routing.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/shared/ssh-pty-id.test.ts src/shared/ssh-types.test.ts src/main/providers/local-pty-provider.test.ts src/main/providers/ssh-pty-provider.test.ts src/main/persistence.test.ts src/main/ssh/ssh-relay-session.test.ts src/main/ssh/ssh-relay-generation-reattach.test.ts src/main/ssh/ssh-target-id-migration.test.ts src/relay/pty-handler.test.ts src/relay/subprocess.test.ts src/renderer/src/components/terminal-pane/pty-transport.test.ts src/renderer/src/components/terminal-pane/pty-pre-handler-buffer.test.ts src/renderer/src/lib/pty-kill-retry-ownership.test.ts src/renderer/src/lib/runtime-terminal-close-retry-ownership.test.ts src/renderer/src/lib/ephemeral-vm-runtime-cleanup.test.ts src/renderer/src/lib/launch-agent-background-session.test.ts src/renderer/src/store/slices/repos.test.ts src/renderer/src/store/slices/repos-host-identity-routing.test.ts src/main/runtime/orca-runtime.test.ts src/main/ipc/pty.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/repos-create.test.ts src/main/window/attach-main-window-services.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/worktrees-windows.test.ts src/main/ipc/pty.test.ts src/main/runtime/orca-runtime.test.ts src/renderer/src/lib/launch-agent-background-session.test.ts src/renderer/src/lib/runtime-terminal-close-retry-ownership.test.ts src/renderer/src/store/slices/repos-host-identity-routing.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/pty.test.ts src/renderer/src/store/slices/worktrees.test.ts"
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/pty.test.ts src/main/ipc/runtime-environments.test.ts src/main/runtime/rpc/terminal-close-generation.test.ts src/main/persistence.test.ts src/main/ssh/ssh-relay-session.test.ts src/main/ipc/ssh.test.ts src/renderer/src/store/slices/worktrees.test.ts"
|
||||
],
|
||||
"testFiles": [
|
||||
"config/scripts/build-relay-windows-node-pty-patch.test.mjs",
|
||||
@@ -3813,7 +3813,10 @@
|
||||
"src/renderer/src/lib/launch-agent-background-session.test.ts",
|
||||
"src/renderer/src/lib/runtime-terminal-close-retry-ownership.test.ts",
|
||||
"src/renderer/src/store/slices/repos-host-identity-routing.test.ts",
|
||||
"src/renderer/src/store/slices/worktrees.test.ts"
|
||||
"src/renderer/src/store/slices/worktrees.test.ts",
|
||||
"src/main/ipc/runtime-environments.test.ts",
|
||||
"src/main/runtime/rpc/terminal-close-generation.test.ts",
|
||||
"src/main/ipc/ssh.test.ts"
|
||||
],
|
||||
"assertionRefs": [
|
||||
{
|
||||
@@ -3849,7 +3852,7 @@
|
||||
{
|
||||
"file": "src/renderer/src/components/terminal-pane/pty-transport.test.ts",
|
||||
"assertions": [
|
||||
"a rejected teardown kill retains exact PTY identity and retries on the next lifecycle event"
|
||||
"a rejected teardown kill forwards exact PTY identity to main and later renderer lifecycle does not duplicate the retry"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -3945,9 +3948,8 @@
|
||||
{
|
||||
"file": "src/renderer/src/lib/pty-kill-retry-ownership.test.ts",
|
||||
"assertions": [
|
||||
"all 65 unconfirmed live shutdown ids retain exact ownership and one deadline retries every due owner",
|
||||
"a second failure retries after capped backoff without another lifecycle event",
|
||||
"successful SSH target removal releases only that target and cancels the final retry timer"
|
||||
"renderer teardown forwards exact PTY identity to the main-process retry owner",
|
||||
"renderer lifecycle hooks no longer own retry timers that disappear on reload"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -3957,7 +3959,9 @@
|
||||
"renderer PTY spawn enters runtime terminal admission before provider work",
|
||||
"renderer PTY spawn qualifies local and SSH admissions with the owning execution host",
|
||||
"disconnected SSH kill and stop retain the lease and reject without a synthetic exit",
|
||||
"provider reconnect retries durable shutdown intent once with its generation and pane identity"
|
||||
"provider reconnect retries durable shutdown intent with its generation and pane identity",
|
||||
"persisted local shutdown rebinds from the startup fallback to the daemon provider and drains all retry timers",
|
||||
"relay generation reuse removes stale retry ownership without touching the new PTY"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -3991,8 +3995,20 @@
|
||||
{
|
||||
"file": "src/renderer/src/lib/runtime-terminal-close-retry-ownership.test.ts",
|
||||
"assertions": [
|
||||
"a rejected remote-runtime terminal close retains environment and handle identity through repeated automatic backoff retry",
|
||||
"successful runtime removal releases every environment handle and cancels its retry timer"
|
||||
"renderer remote-runtime teardown forwards environment and handle identity to main without renderer-owned timers"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/main/ipc/runtime-environments.test.ts",
|
||||
"assertions": [
|
||||
"persisted remote-runtime close intent hydrates after app restart and drains after bounded retry",
|
||||
"replacement runtime generation clears stale intent without sending terminal.close"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/main/runtime/rpc/terminal-close-generation.test.ts",
|
||||
"assertions": [
|
||||
"runtime atomically rejects a terminal close fenced to an older runtime generation"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -4122,7 +4138,7 @@
|
||||
"date": "2026-07-12",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/pty.test.ts src/renderer/src/store/slices/worktrees.test.ts",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/pty.test.ts src/main/ipc/runtime-environments.test.ts src/main/runtime/rpc/terminal-close-generation.test.ts src/main/persistence.test.ts src/main/ssh/ssh-relay-session.test.ts src/main/ipc/ssh.test.ts src/renderer/src/store/slices/worktrees.test.ts",
|
||||
"result": "passed",
|
||||
"durationSeconds": 10,
|
||||
"summary": "1356 exact-head tests proved duplicate local/SSH IDs isolate create, removal, renderer spawn, and PTY liveness ownership while whole-project removal drains every host scope."
|
||||
@@ -4142,7 +4158,7 @@
|
||||
},
|
||||
"performanceBudget": {
|
||||
"required": true,
|
||||
"evidence": "Shutdown removes indefinite deferred retention and preserves at most one in-flight native kill plus one existing graceful timer per PTY request. Failed renderer kills coalesce by exact live PTY id, and failed remote-runtime closes coalesce by environment plus handle: distinct records correspond one-for-one with still-unconfirmed provider resources and are never discarded before accepted shutdown. Each retry map owns one shared one-shot timer for its earliest deadline, retries all due owners, caps backoff at 30 seconds, and logs only attempts 1, 2, and 8 to prevent outage backpressure. Successful SSH-target and runtime-environment removal release the unreachable target's records and cancel the last timer; offline SSH shutdown intent also moves onto the durable lease so renderer reload cannot lose it, and provider reconnect performs one identity-qualified retry. Pre-handler exits remain capped at 64. Worktree-create, worktree-removal, whole-project, and host-scoped admissions retain one promise only per active operation; removal drains only the target worktree's current admissions. Targeted runtime stop still performs one provider verification per target PTY plus two liveness snapshots, while foreground probes are limited to the target worktree."
|
||||
"evidence": "Shutdown removes indefinite deferred retention and preserves at most one in-flight native kill plus one existing graceful timer per PTY request. Failed local and SSH kills coalesce in main by exact live PTY id, while failed remote-runtime closes coalesce by environment, runtime generation, and handle: distinct durable records correspond one-for-one with still-unconfirmed provider resources and survive renderer/app restart. Each main-owned retry map owns one shared one-shot timer for its earliest deadline, retries all due owners, caps backoff at 30 seconds, and logs only attempts 1, 2, and 8 to prevent outage backpressure. Successful SSH-target and runtime-environment removal release unreachable records and cancel the last retry timer; provider replacement rebinds the same generation but relay/runtime generation replacement drops stale intent before touching a reused id. Remote-runtime close adds one generation status probe per teardown attempt and is not on input, output, render, polling, or startup-visible paths. Pre-handler exits remain capped at 64. Worktree-create, worktree-removal, whole-project, and host-scoped admissions retain one promise only per active operation; removal drains only the target worktree's current admissions. Targeted runtime stop still performs one provider verification per target PTY plus two liveness snapshots, while foreground probes are limited to the target worktree."
|
||||
},
|
||||
"promotionCriteria": [
|
||||
"Run the deterministic command across the required CI platforms without flakes.",
|
||||
|
||||
+151
-20
@@ -3028,7 +3028,11 @@ describe('registerPtyHandlers', () => {
|
||||
|
||||
expect(shutdown).toHaveBeenCalledWith('ssh:ssh-1@@relay-pty', { immediate: false })
|
||||
expect(localShutdown).not.toHaveBeenCalled()
|
||||
expect(store.markSshRemotePtyLease).toHaveBeenCalledWith('ssh-1', 'relay-pty', 'terminated')
|
||||
expect(store.markSshRemotePtyLease).toHaveBeenCalledWith(
|
||||
'ssh-1',
|
||||
'ssh:ssh-1@@relay-pty',
|
||||
'terminated'
|
||||
)
|
||||
})
|
||||
|
||||
it('runtime controller kill retains app-scoped SSH ids when the provider is absent', async () => {
|
||||
@@ -3351,12 +3355,18 @@ describe('registerPtyHandlers', () => {
|
||||
deletePtyOwnership('missing-pty')
|
||||
})
|
||||
|
||||
it('rethrows non-not-found local provider shutdown failures', async () => {
|
||||
it('persists local daemon shutdown and retries after renderer loss', async () => {
|
||||
vi.useFakeTimers()
|
||||
const shutdown = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('daemon unavailable'))
|
||||
.mockRejectedValueOnce(new Error('daemon still unavailable'))
|
||||
.mockResolvedValue(undefined)
|
||||
setLocalPtyProvider({
|
||||
spawn: vi.fn(),
|
||||
write: vi.fn(),
|
||||
resize: vi.fn(),
|
||||
shutdown: vi.fn().mockRejectedValue(new Error('daemon unavailable')),
|
||||
shutdown,
|
||||
sendSignal: vi.fn(),
|
||||
getCwd: vi.fn(),
|
||||
getInitialCwd: vi.fn(),
|
||||
@@ -3374,12 +3384,79 @@ describe('registerPtyHandlers', () => {
|
||||
getDefaultShell: vi.fn(),
|
||||
getProfiles: vi.fn()
|
||||
} as never)
|
||||
const store = {
|
||||
getPendingLocalPtyShutdowns: vi.fn(() => []),
|
||||
upsertPendingLocalPtyShutdown: vi.fn(),
|
||||
removePendingLocalPtyShutdown: vi.fn()
|
||||
}
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
handlers.clear()
|
||||
registerPtyHandlers(mainWindow as never)
|
||||
|
||||
await expect(handlers.get('pty:kill')!(null, { id: 'local-pty' })).rejects.toThrow(
|
||||
'daemon unavailable'
|
||||
registerPtyHandlers(
|
||||
mainWindow as never,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
store as never
|
||||
)
|
||||
|
||||
await expect(
|
||||
handlers.get('pty:kill')!(null, {
|
||||
id: 'local-pty',
|
||||
expectedPaneKey: 'tab:leaf',
|
||||
expectedTabId: 'tab'
|
||||
})
|
||||
).rejects.toThrow('daemon unavailable')
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
await vi.advanceTimersByTimeAsync(500)
|
||||
|
||||
expect(shutdown).toHaveBeenCalledTimes(3)
|
||||
expect(store.upsertPendingLocalPtyShutdown).toHaveBeenCalledWith({
|
||||
ptyId: 'local-pty',
|
||||
expectedPaneKey: 'tab:leaf',
|
||||
expectedTabId: 'tab',
|
||||
requestedAt: expect.any(Number)
|
||||
})
|
||||
expect(store.removePendingLocalPtyShutdown).toHaveBeenCalledWith('local-pty')
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('rebinds a persisted local shutdown after async daemon startup replaces the provider', async () => {
|
||||
vi.useFakeTimers()
|
||||
const makeProvider = (shutdown: ReturnType<typeof vi.fn>) =>
|
||||
({
|
||||
spawn: vi.fn(),
|
||||
write: vi.fn(),
|
||||
resize: vi.fn(),
|
||||
shutdown,
|
||||
sendSignal: vi.fn(),
|
||||
onData: vi.fn(() => () => {}),
|
||||
onReplay: vi.fn(() => () => {}),
|
||||
onExit: vi.fn(() => () => {})
|
||||
}) as never
|
||||
const fallbackShutdown = vi.fn().mockRejectedValue(new Error('temporary provider'))
|
||||
const daemonShutdown = vi.fn().mockResolvedValue(undefined)
|
||||
const removePendingLocalPtyShutdown = vi.fn()
|
||||
setLocalPtyProvider(makeProvider(fallbackShutdown))
|
||||
handlers.clear()
|
||||
registerPtyHandlers(mainWindow as never, undefined, undefined, undefined, undefined, {
|
||||
getPendingLocalPtyShutdowns: () => [{ ptyId: 'daemon-pty', requestedAt: 1 }],
|
||||
upsertPendingLocalPtyShutdown: vi.fn(),
|
||||
removePendingLocalPtyShutdown
|
||||
} as never)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
setLocalPtyProvider(makeProvider(daemonShutdown))
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
expect(fallbackShutdown).toHaveBeenCalledOnce()
|
||||
expect(daemonShutdown).toHaveBeenCalledOnce()
|
||||
expect(removePendingLocalPtyShutdown).toHaveBeenCalledWith('daemon-pty')
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('synthesizes runtime exit after ordinary daemon-backed pty kill', async () => {
|
||||
@@ -4049,7 +4126,11 @@ describe('registerPtyHandlers', () => {
|
||||
expectedTabId: 'tab-a'
|
||||
})
|
||||
expect(localShutdown).not.toHaveBeenCalled()
|
||||
expect(store.markSshRemotePtyLease).toHaveBeenCalledWith('ssh-1', 'relay-pty', 'terminated')
|
||||
expect(store.markSshRemotePtyLease).toHaveBeenCalledWith(
|
||||
'ssh-1',
|
||||
'ssh:ssh-1@@relay-pty',
|
||||
'terminated'
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects app-scoped SSH PTY kill without tombstoning when the provider is absent', async () => {
|
||||
@@ -4089,9 +4170,9 @@ describe('registerPtyHandlers', () => {
|
||||
store as never
|
||||
)
|
||||
|
||||
await expect(
|
||||
handlers.get('pty:kill')!(null, { id: 'ssh:ssh-1@@relay-pty' })
|
||||
).rejects.toThrow('SSH PTY provider unavailable')
|
||||
await expect(handlers.get('pty:kill')!(null, { id: 'ssh:ssh-1@@relay-pty' })).rejects.toThrow(
|
||||
'SSH PTY provider unavailable'
|
||||
)
|
||||
|
||||
expect(localShutdown).not.toHaveBeenCalled()
|
||||
expect(store.markSshRemotePtyLease).not.toHaveBeenCalled()
|
||||
@@ -4101,7 +4182,8 @@ describe('registerPtyHandlers', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('retries durable SSH shutdown intent when the provider reconnects', async () => {
|
||||
it('retries durable SSH shutdown through repeated reconnect failure then clears timers', async () => {
|
||||
vi.useFakeTimers()
|
||||
const lease = {
|
||||
targetId: 'ssh-1',
|
||||
ptyId: 'relay-pty',
|
||||
@@ -4132,24 +4214,73 @@ describe('registerPtyHandlers', () => {
|
||||
'SSH PTY provider unavailable'
|
||||
)
|
||||
|
||||
const shutdown = vi.fn().mockResolvedValue(undefined)
|
||||
const shutdown = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('offline'))
|
||||
.mockRejectedValueOnce(new Error('still offline'))
|
||||
.mockResolvedValue(undefined)
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
registerSshPtyProvider('ssh-1', {
|
||||
shutdown,
|
||||
onExit: vi.fn(() => () => {})
|
||||
} as never)
|
||||
await vi.waitFor(() => expect(store.markSshRemotePtyLease).toHaveBeenCalledOnce())
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
await vi.advanceTimersByTimeAsync(500)
|
||||
|
||||
expect(shutdown).toHaveBeenCalledWith(id, {
|
||||
expect(shutdown).toHaveBeenCalledTimes(3)
|
||||
expect(shutdown).toHaveBeenLastCalledWith(id, {
|
||||
immediate: true,
|
||||
expectedPaneKey: 'tab-a:11111111-1111-4111-8111-111111111111',
|
||||
expectedTabId: 'tab-a'
|
||||
})
|
||||
expect(store.markSshRemotePtyLease).toHaveBeenCalledWith(
|
||||
'ssh-1',
|
||||
'relay-pty',
|
||||
'terminated'
|
||||
)
|
||||
expect(store.markSshRemotePtyLease).toHaveBeenCalledWith('ssh-1', id, 'terminated')
|
||||
unregisterSshPtyProvider('ssh-1')
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
})
|
||||
|
||||
it('drops a retained SSH retry when a new relay generation reuses the raw id', async () => {
|
||||
vi.useFakeTimers()
|
||||
const lease: Record<string, unknown> = {
|
||||
targetId: 'ssh-1',
|
||||
ptyId: 'relay-pty',
|
||||
relayInstanceId: 'boot-a',
|
||||
state: 'detached',
|
||||
shutdownRequestedAt: 2,
|
||||
createdAt: 1,
|
||||
updatedAt: 2
|
||||
}
|
||||
const store = { getSshRemotePtyLeases: vi.fn(() => [lease]) }
|
||||
handlers.clear()
|
||||
registerPtyHandlers(
|
||||
mainWindow as never,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
store as never
|
||||
)
|
||||
const oldShutdown = vi.fn().mockRejectedValue(new Error('relay replaced'))
|
||||
registerSshPtyProvider('ssh-1', {
|
||||
shutdown: oldShutdown,
|
||||
onExit: vi.fn(() => () => {})
|
||||
} as never)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
Object.assign(lease, { relayInstanceId: 'boot-b', state: 'attached' })
|
||||
delete lease.shutdownRequestedAt
|
||||
const newShutdown = vi.fn()
|
||||
registerSshPtyProvider('ssh-1', {
|
||||
shutdown: newShutdown,
|
||||
onExit: vi.fn(() => () => {})
|
||||
} as never)
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
|
||||
expect(oldShutdown).toHaveBeenCalledOnce()
|
||||
expect(newShutdown).not.toHaveBeenCalled()
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
unregisterSshPtyProvider('ssh-1')
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('ignores fire-and-forget IPC for detached SSH PTYs without a provider', async () => {
|
||||
|
||||
+1031
-795
File diff suppressed because it is too large
Load Diff
@@ -499,6 +499,148 @@ describe('registerRuntimeEnvironmentHandlers', () => {
|
||||
expect(sendRemoteRuntimeRequestMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('persists and retries exact terminal closes after renderer loss', async () => {
|
||||
vi.useFakeTimers()
|
||||
const upsertClose = vi.fn()
|
||||
const removeClose = vi.fn()
|
||||
Object.assign(store, {
|
||||
getPendingRuntimeTerminalCloses: () => [],
|
||||
upsertPendingRuntimeTerminalClose: upsertClose,
|
||||
removePendingRuntimeTerminalClose: removeClose
|
||||
})
|
||||
registerRuntimeEnvironmentHandlers(store as never)
|
||||
let closeAttempt = 0
|
||||
sendRemoteRuntimeRequestMock.mockImplementation(async (_pairing, method) => {
|
||||
if (method === 'status.get') {
|
||||
return {
|
||||
id: 'status',
|
||||
ok: true,
|
||||
result: { runtimeId: 'runtime-remote', capabilities: [] },
|
||||
_meta: { runtimeId: 'runtime-remote' }
|
||||
}
|
||||
}
|
||||
closeAttempt += 1
|
||||
if (closeAttempt < 3) {
|
||||
return {
|
||||
id: `close-${closeAttempt}`,
|
||||
ok: false,
|
||||
error: { message: closeAttempt === 1 ? 'offline' : 'still offline' }
|
||||
}
|
||||
}
|
||||
return {
|
||||
id: 'close-3',
|
||||
ok: true,
|
||||
result: { close: true },
|
||||
_meta: { runtimeId: 'runtime-remote' }
|
||||
}
|
||||
})
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
const add = handler<
|
||||
{ name: string; pairingCode: string },
|
||||
{ environment: { id: string; name: string } }
|
||||
>('runtimeEnvironments:addFromPairingCode')
|
||||
const added = await add(null, { name: 'desk', pairingCode: pairingCode() })
|
||||
const call = handler<{ selector: string; method: string; params?: unknown }, { ok: boolean }>(
|
||||
'runtimeEnvironments:call'
|
||||
)
|
||||
await expect(
|
||||
call(null, { selector: 'desk', method: 'terminal.close', params: { terminal: 't1' } })
|
||||
).resolves.toMatchObject({ ok: false })
|
||||
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
await vi.advanceTimersByTimeAsync(500)
|
||||
|
||||
expect(closeAttempt).toBe(3)
|
||||
expect(upsertClose).toHaveBeenCalledWith({
|
||||
environmentId: added.environment.id,
|
||||
handle: 't1',
|
||||
requestedAt: expect.any(Number)
|
||||
})
|
||||
expect(removeClose).toHaveBeenCalledWith(added.environment.id, 't1')
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('hydrates and drains persisted terminal close intent after app restart', async () => {
|
||||
vi.useFakeTimers()
|
||||
const environment = environmentStore.addEnvironmentFromPairingCode(userDataPath, {
|
||||
name: 'desk',
|
||||
pairingCode: pairingCode()
|
||||
})
|
||||
const removePendingRuntimeTerminalClose = vi.fn()
|
||||
Object.assign(store, {
|
||||
getPendingRuntimeTerminalCloses: () => [
|
||||
{ environmentId: environment.id, handle: 't1', requestedAt: 1 }
|
||||
],
|
||||
upsertPendingRuntimeTerminalClose: vi.fn(),
|
||||
removePendingRuntimeTerminalClose
|
||||
})
|
||||
sendRemoteRuntimeRequestMock.mockImplementation(async (_pairing, method) =>
|
||||
method === 'status.get'
|
||||
? {
|
||||
id: 'status',
|
||||
ok: true,
|
||||
result: { runtimeId: 'runtime-remote', capabilities: [] },
|
||||
_meta: { runtimeId: 'runtime-remote' }
|
||||
}
|
||||
: {
|
||||
id: 'close',
|
||||
ok: true,
|
||||
result: { close: true },
|
||||
_meta: { runtimeId: 'runtime-remote' }
|
||||
}
|
||||
)
|
||||
|
||||
registerRuntimeEnvironmentHandlers(store as never)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
expect(sendRemoteRuntimeRequestMock.mock.calls.map((call) => call[1])).toEqual([
|
||||
'status.get',
|
||||
'status.get',
|
||||
'terminal.close'
|
||||
])
|
||||
expect(removePendingRuntimeTerminalClose).toHaveBeenCalledWith(environment.id, 't1')
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('does not apply persisted close intent to a replacement runtime generation', async () => {
|
||||
vi.useFakeTimers()
|
||||
const environment = environmentStore.addEnvironmentFromPairingCode(userDataPath, {
|
||||
name: 'desk',
|
||||
pairingCode: pairingCode()
|
||||
})
|
||||
environmentStore.markEnvironmentUsed(userDataPath, environment.id, { runtimeId: 'runtime-a' })
|
||||
const removePendingRuntimeTerminalClose = vi.fn()
|
||||
Object.assign(store, {
|
||||
getPendingRuntimeTerminalCloses: () => [
|
||||
{
|
||||
environmentId: environment.id,
|
||||
handle: 't1',
|
||||
runtimeId: 'runtime-a',
|
||||
requestedAt: 1
|
||||
}
|
||||
],
|
||||
upsertPendingRuntimeTerminalClose: vi.fn(),
|
||||
removePendingRuntimeTerminalClose
|
||||
})
|
||||
sendRemoteRuntimeRequestMock.mockResolvedValue({
|
||||
id: 'status',
|
||||
ok: true,
|
||||
result: { runtimeId: 'runtime-b', capabilities: [] },
|
||||
_meta: { runtimeId: 'runtime-b' }
|
||||
})
|
||||
|
||||
registerRuntimeEnvironmentHandlers(store as never)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
expect(sendRemoteRuntimeRequestMock.mock.calls.map((call) => call[1])).toEqual(['status.get'])
|
||||
expect(removePendingRuntimeTerminalClose).toHaveBeenCalledWith(environment.id, 't1')
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('keeps terminal hot path RPCs on the cached request connection when shared control is supported', async () => {
|
||||
registerRuntimeEnvironmentHandlers(store as never)
|
||||
sendRemoteRuntimeRequestMock.mockResolvedValue({
|
||||
|
||||
@@ -23,6 +23,11 @@ import {
|
||||
resetSharedControlSupport,
|
||||
subscribeRuntimeEnvironment
|
||||
} from './runtime-environment-transport-routing'
|
||||
import {
|
||||
closeRuntimeTerminalWithRetryOwnership,
|
||||
initializeRuntimeTerminalCloseRetryOwner,
|
||||
releaseRuntimeTerminalClosesForEnvironment
|
||||
} from './runtime-terminal-close-retry-owner'
|
||||
|
||||
const RUNTIME_ENVIRONMENT_HANDLER_CHANNELS = [
|
||||
'runtimeEnvironments:list',
|
||||
@@ -66,6 +71,7 @@ function listPublicRuntimeEnvironments(): PublicKnownRuntimeEnvironment[] {
|
||||
}
|
||||
|
||||
export function registerRuntimeEnvironmentHandlers(store: Store): void {
|
||||
initializeRuntimeTerminalCloseRetryOwner(store, getUserDataPath())
|
||||
// Why: keep direct re-registration safe even though register-core-handlers
|
||||
// normally guards this path; otherwise the binary send listener can stack.
|
||||
resetSharedControlSupport()
|
||||
@@ -103,6 +109,8 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void {
|
||||
}
|
||||
clearActiveRuntimeEnvironmentFocusIfMatches(store, removed.id)
|
||||
closeSubscriptionsForEnvironment(removed.id)
|
||||
// Why: removal is authoritative; no future transport can reach these handles.
|
||||
releaseRuntimeTerminalClosesForEnvironment(removed.id)
|
||||
return { removed: redactRuntimeEnvironment(removed) }
|
||||
}
|
||||
)
|
||||
@@ -137,6 +145,19 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void {
|
||||
_event,
|
||||
args: { selector: string; method: string; params?: unknown; timeoutMs?: number }
|
||||
): Promise<RuntimeRpcResponse<unknown>> => {
|
||||
if (
|
||||
args.method === 'terminal.close' &&
|
||||
args.params &&
|
||||
typeof args.params === 'object' &&
|
||||
typeof (args.params as { terminal?: unknown }).terminal === 'string'
|
||||
) {
|
||||
const environment = resolveEnvironment(getUserDataPath(), args.selector)
|
||||
return closeRuntimeTerminalWithRetryOwnership(
|
||||
environment.id,
|
||||
(args.params as { terminal: string }).terminal,
|
||||
environment.runtimeId
|
||||
)
|
||||
}
|
||||
return callRuntimeEnvironment(
|
||||
getUserDataPath(),
|
||||
args.selector,
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
import type { RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope'
|
||||
import type { Store } from '../persistence'
|
||||
import {
|
||||
callRuntimeEnvironment,
|
||||
getRuntimeEnvironmentStatus
|
||||
} from './runtime-environment-transport-routing'
|
||||
|
||||
type RetainedTerminalClose = {
|
||||
generation: number
|
||||
environmentId: string
|
||||
handle: string
|
||||
runtimeId: string | null
|
||||
attempts: number
|
||||
nextRetryAt: number
|
||||
inFlight: Promise<RuntimeRpcResponse<unknown>> | null
|
||||
}
|
||||
|
||||
const retainedCloses = new Map<string, RetainedTerminalClose>()
|
||||
const MAX_BACKOFF_MS = 30_000
|
||||
let retryTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let store: Store | null = null
|
||||
let userDataPath = ''
|
||||
let ownerGeneration = 0
|
||||
|
||||
function key(environmentId: string, handle: string): string {
|
||||
return `${environmentId}\0${handle}`
|
||||
}
|
||||
|
||||
function schedule(): void {
|
||||
if (retryTimer) {
|
||||
clearTimeout(retryTimer)
|
||||
}
|
||||
retryTimer = null
|
||||
let nextRetryAt = Number.POSITIVE_INFINITY
|
||||
for (const close of retainedCloses.values()) {
|
||||
if (!close.inFlight) {
|
||||
nextRetryAt = Math.min(nextRetryAt, close.nextRetryAt)
|
||||
}
|
||||
}
|
||||
if (!Number.isFinite(nextRetryAt)) {
|
||||
return
|
||||
}
|
||||
retryTimer = setTimeout(
|
||||
() => {
|
||||
retryTimer = null
|
||||
const now = Date.now()
|
||||
for (const close of retainedCloses.values()) {
|
||||
if (!close.inFlight && close.nextRetryAt <= now) {
|
||||
void attempt(close)
|
||||
}
|
||||
}
|
||||
},
|
||||
Math.max(0, nextRetryAt - Date.now())
|
||||
)
|
||||
}
|
||||
|
||||
function attempt(close: RetainedTerminalClose): Promise<RuntimeRpcResponse<unknown>> {
|
||||
if (close.inFlight) {
|
||||
return close.inFlight
|
||||
}
|
||||
const closeKey = key(close.environmentId, close.handle)
|
||||
const noOpResponse = (runtimeId?: string | null) => ({
|
||||
id: 'terminal.close',
|
||||
ok: true as const,
|
||||
result: { close: false, reason: 'retry_owner_replaced' },
|
||||
_meta: { runtimeId: runtimeId ?? close.runtimeId ?? 'unknown-runtime' }
|
||||
})
|
||||
const request = getRuntimeEnvironmentStatus(userDataPath, close.environmentId)
|
||||
.then((status) => {
|
||||
if (close.generation !== ownerGeneration) {
|
||||
return noOpResponse(status._meta?.runtimeId)
|
||||
}
|
||||
if (!status.ok) {
|
||||
throw Object.assign(new Error(status.error.message), { response: status })
|
||||
}
|
||||
const currentRuntimeId = status._meta.runtimeId
|
||||
if (close.runtimeId && close.runtimeId !== currentRuntimeId) {
|
||||
retainedCloses.delete(closeKey)
|
||||
store?.removePendingRuntimeTerminalClose?.(close.environmentId, close.handle)
|
||||
return {
|
||||
id: 'terminal.close',
|
||||
ok: true as const,
|
||||
result: { close: false, reason: 'runtime_replaced' },
|
||||
_meta: { runtimeId: currentRuntimeId }
|
||||
}
|
||||
}
|
||||
if (!close.runtimeId) {
|
||||
close.runtimeId = currentRuntimeId
|
||||
store?.upsertPendingRuntimeTerminalClose?.({
|
||||
environmentId: close.environmentId,
|
||||
handle: close.handle,
|
||||
runtimeId: currentRuntimeId,
|
||||
requestedAt: Date.now()
|
||||
})
|
||||
}
|
||||
return callRuntimeEnvironment(userDataPath, close.environmentId, 'terminal.close', {
|
||||
terminal: close.handle,
|
||||
expectedRuntimeId: close.runtimeId
|
||||
})
|
||||
})
|
||||
.then((response) => {
|
||||
if (close.generation !== ownerGeneration) {
|
||||
return noOpResponse(response._meta?.runtimeId)
|
||||
}
|
||||
if (!response.ok) {
|
||||
const responseRuntimeId = response._meta?.runtimeId
|
||||
if (close.runtimeId && responseRuntimeId && responseRuntimeId !== close.runtimeId) {
|
||||
retainedCloses.delete(closeKey)
|
||||
store?.removePendingRuntimeTerminalClose?.(close.environmentId, close.handle)
|
||||
return {
|
||||
id: 'terminal.close',
|
||||
ok: true as const,
|
||||
result: { close: false, reason: 'runtime_replaced' },
|
||||
_meta: { runtimeId: responseRuntimeId }
|
||||
}
|
||||
}
|
||||
throw Object.assign(new Error(response.error.message), { response })
|
||||
}
|
||||
retainedCloses.delete(closeKey)
|
||||
if (typeof store?.removePendingRuntimeTerminalClose === 'function') {
|
||||
store.removePendingRuntimeTerminalClose(close.environmentId, close.handle)
|
||||
}
|
||||
return response
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
if (close.generation !== ownerGeneration) {
|
||||
return noOpResponse()
|
||||
}
|
||||
close.attempts += 1
|
||||
close.nextRetryAt =
|
||||
Date.now() + Math.min(MAX_BACKOFF_MS, 250 * 2 ** Math.min(close.attempts - 1, 7))
|
||||
if (close.attempts <= 2 || close.attempts === 8) {
|
||||
console.warn('[runtime-environments] retained terminal close failed:', error)
|
||||
}
|
||||
const response = (error as { response?: RuntimeRpcResponse<unknown> }).response
|
||||
if (response) {
|
||||
return response
|
||||
}
|
||||
throw error
|
||||
})
|
||||
.finally(() => {
|
||||
if (close.generation !== ownerGeneration) {
|
||||
return
|
||||
}
|
||||
if (close.inFlight === request) {
|
||||
close.inFlight = null
|
||||
}
|
||||
schedule()
|
||||
})
|
||||
close.inFlight = request
|
||||
return request
|
||||
}
|
||||
|
||||
function retain(
|
||||
environmentId: string,
|
||||
handle: string,
|
||||
runtimeId?: string | null
|
||||
): RetainedTerminalClose {
|
||||
const closeKey = key(environmentId, handle)
|
||||
let close = retainedCloses.get(closeKey)
|
||||
if (!close) {
|
||||
close = {
|
||||
generation: ownerGeneration,
|
||||
environmentId,
|
||||
handle,
|
||||
runtimeId: runtimeId ?? null,
|
||||
attempts: 0,
|
||||
nextRetryAt: 0,
|
||||
inFlight: null
|
||||
}
|
||||
retainedCloses.set(closeKey, close)
|
||||
if (typeof store?.upsertPendingRuntimeTerminalClose === 'function') {
|
||||
store.upsertPendingRuntimeTerminalClose({
|
||||
environmentId,
|
||||
handle,
|
||||
...(runtimeId ? { runtimeId } : {}),
|
||||
requestedAt: Date.now()
|
||||
})
|
||||
}
|
||||
} else if (!close.runtimeId && runtimeId) {
|
||||
close.runtimeId = runtimeId
|
||||
}
|
||||
return close
|
||||
}
|
||||
|
||||
export function initializeRuntimeTerminalCloseRetryOwner(
|
||||
nextStore: Store,
|
||||
nextUserDataPath: string
|
||||
): void {
|
||||
ownerGeneration += 1
|
||||
if (retryTimer) {
|
||||
clearTimeout(retryTimer)
|
||||
retryTimer = null
|
||||
}
|
||||
retainedCloses.clear()
|
||||
store = nextStore
|
||||
userDataPath = nextUserDataPath
|
||||
const persisted =
|
||||
typeof store.getPendingRuntimeTerminalCloses === 'function'
|
||||
? store.getPendingRuntimeTerminalCloses()
|
||||
: []
|
||||
for (const close of persisted) {
|
||||
retain(close.environmentId, close.handle, close.runtimeId)
|
||||
}
|
||||
schedule()
|
||||
}
|
||||
|
||||
export function closeRuntimeTerminalWithRetryOwnership(
|
||||
environmentId: string,
|
||||
handle: string,
|
||||
runtimeId?: string | null
|
||||
): Promise<RuntimeRpcResponse<unknown>> {
|
||||
return attempt(retain(environmentId, handle, runtimeId))
|
||||
}
|
||||
|
||||
export function releaseRuntimeTerminalClosesForEnvironment(environmentId: string): void {
|
||||
for (const [closeKey, close] of retainedCloses) {
|
||||
if (close.environmentId === environmentId) {
|
||||
retainedCloses.delete(closeKey)
|
||||
}
|
||||
}
|
||||
if (typeof store?.removePendingRuntimeTerminalClosesForEnvironment === 'function') {
|
||||
store.removePendingRuntimeTerminalClosesForEnvironment(environmentId)
|
||||
}
|
||||
schedule()
|
||||
}
|
||||
@@ -157,6 +157,7 @@ vi.mock('../providers/ssh-filesystem-provider', () => ({
|
||||
vi.mock('./pty', () => ({
|
||||
registerSshPtyProvider: vi.fn(),
|
||||
unregisterSshPtyProvider: vi.fn(),
|
||||
releasePendingSshShutdownsForTarget: vi.fn(),
|
||||
clearPtyOwnershipForConnection: vi.fn(),
|
||||
clearProviderPtyState: vi.fn(),
|
||||
deletePtyOwnership: vi.fn(),
|
||||
@@ -1218,8 +1219,16 @@ describe('SSH IPC handlers', () => {
|
||||
expect(clearProviderPtyState).toHaveBeenCalledWith('ssh:ssh-1@@pty-lease')
|
||||
expect(deletePtyOwnership).toHaveBeenCalledWith('ssh:ssh-1@@pty-live')
|
||||
expect(deletePtyOwnership).toHaveBeenCalledWith('ssh:ssh-1@@pty-lease')
|
||||
expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith('ssh-1', 'pty-live', 'terminated')
|
||||
expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith('ssh-1', 'pty-lease', 'terminated')
|
||||
expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith(
|
||||
'ssh-1',
|
||||
'ssh:ssh-1@@pty-live',
|
||||
'terminated'
|
||||
)
|
||||
expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith(
|
||||
'ssh-1',
|
||||
'ssh:ssh-1@@pty-lease',
|
||||
'terminated'
|
||||
)
|
||||
})
|
||||
|
||||
it('ssh:terminateSessions ignores expired leases when disconnected', async () => {
|
||||
|
||||
+1
-1
@@ -1004,7 +1004,7 @@ export function registerSshHandlers(
|
||||
}
|
||||
clearProviderPtyState(appPtyId)
|
||||
deletePtyOwnership(appPtyId)
|
||||
persistedStore!.markSshRemotePtyLease(args.targetId, relayPtyId, 'terminated')
|
||||
persistedStore!.markSshRemotePtyLease(args.targetId, appPtyId, 'terminated')
|
||||
}
|
||||
if (shutdownFailures.length > 0) {
|
||||
// Why: a failed relay shutdown can leave the remote process alive in the
|
||||
|
||||
@@ -8802,6 +8802,80 @@ describe('Store', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('replaces teardown identity when a new relay boot reuses a raw PTY id', async () => {
|
||||
const store = await createStore()
|
||||
store.upsertSshRemotePtyLease({
|
||||
targetId: 'ssh-1',
|
||||
ptyId: 'ssh:ssh-1@@boot-a@@remote-pty',
|
||||
state: 'expired',
|
||||
shutdownRequestedAt: 10
|
||||
})
|
||||
|
||||
store.upsertSshRemotePtyLease({
|
||||
targetId: 'ssh-1',
|
||||
ptyId: 'ssh:ssh-1@@boot-b@@remote-pty',
|
||||
state: 'attached'
|
||||
})
|
||||
store.markSshRemotePtyShutdownRequested('ssh-1', 'ssh:ssh-1@@boot-a@@remote-pty')
|
||||
|
||||
expect(store.getSshRemotePtyLeases('ssh-1')).toEqual([
|
||||
expect.objectContaining({
|
||||
ptyId: 'remote-pty',
|
||||
relayInstanceId: 'boot-b',
|
||||
state: 'attached'
|
||||
})
|
||||
])
|
||||
expect(store.getSshRemotePtyLeases('ssh-1')[0]).not.toHaveProperty('shutdownRequestedAt')
|
||||
})
|
||||
|
||||
it('preserves SSH relay generation across save and reload', async () => {
|
||||
const store = await createStore()
|
||||
store.upsertSshRemotePtyLease({
|
||||
targetId: 'ssh-1',
|
||||
ptyId: 'ssh:ssh-1@@boot-a@@remote-pty',
|
||||
state: 'detached'
|
||||
})
|
||||
|
||||
const reloaded = await createStore()
|
||||
expect(reloaded.getSshRemotePtyLeases('ssh-1')[0]).toEqual(
|
||||
expect.objectContaining({ relayInstanceId: 'boot-a' })
|
||||
)
|
||||
})
|
||||
|
||||
it('preserves exact local and runtime close intent across app restart', async () => {
|
||||
const store = await createStore()
|
||||
store.upsertPendingLocalPtyShutdown({
|
||||
ptyId: 'local-pty',
|
||||
expectedPaneKey: 'tab:leaf',
|
||||
expectedTabId: 'tab',
|
||||
requestedAt: 10
|
||||
})
|
||||
store.upsertPendingRuntimeTerminalClose({
|
||||
environmentId: 'env-1',
|
||||
handle: 'terminal-1',
|
||||
runtimeId: 'runtime-a',
|
||||
requestedAt: 20
|
||||
})
|
||||
|
||||
const reloaded = await createStore()
|
||||
expect(reloaded.getPendingLocalPtyShutdowns()).toEqual([
|
||||
{
|
||||
ptyId: 'local-pty',
|
||||
expectedPaneKey: 'tab:leaf',
|
||||
expectedTabId: 'tab',
|
||||
requestedAt: 10
|
||||
}
|
||||
])
|
||||
expect(reloaded.getPendingRuntimeTerminalCloses()).toEqual([
|
||||
{
|
||||
environmentId: 'env-1',
|
||||
handle: 'terminal-1',
|
||||
runtimeId: 'runtime-a',
|
||||
requestedAt: 20
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('rejects mismatched scoped SSH remote PTY lease ids on write paths', async () => {
|
||||
const store = await createStore()
|
||||
|
||||
|
||||
+134
-7
@@ -65,7 +65,9 @@ import type {
|
||||
TerminalLayoutSnapshot,
|
||||
TerminalTab,
|
||||
WorkspaceSessionPatch,
|
||||
WorkspaceSessionState
|
||||
WorkspaceSessionState,
|
||||
PersistedLocalPtyShutdown,
|
||||
PersistedRuntimeTerminalClose
|
||||
} from '../shared/types'
|
||||
import {
|
||||
deriveGlobalWindowsRuntimeDefaultFromLegacySettings,
|
||||
@@ -1577,6 +1579,7 @@ function normalizeSshRemotePtyLease(value: unknown): SshRemotePtyLease | null {
|
||||
return {
|
||||
targetId: raw.targetId,
|
||||
ptyId: raw.ptyId,
|
||||
...(typeof raw.relayInstanceId === 'string' ? { relayInstanceId: raw.relayInstanceId } : {}),
|
||||
...(typeof raw.worktreeId === 'string' ? { worktreeId: raw.worktreeId } : {}),
|
||||
...(typeof raw.tabId === 'string' ? { tabId: raw.tabId } : {}),
|
||||
...(typeof raw.leafId === 'string' && raw.leafId.length <= 256 ? { leafId: raw.leafId } : {}),
|
||||
@@ -1591,6 +1594,42 @@ function normalizeSshRemotePtyLease(value: unknown): SshRemotePtyLease | null {
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeLocalPtyShutdown(value: unknown): PersistedLocalPtyShutdown | null {
|
||||
if (!value || typeof value !== 'object') {
|
||||
return null
|
||||
}
|
||||
const raw = value as Partial<PersistedLocalPtyShutdown>
|
||||
if (typeof raw.ptyId !== 'string' || typeof raw.requestedAt !== 'number') {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
ptyId: raw.ptyId,
|
||||
requestedAt: raw.requestedAt,
|
||||
...(typeof raw.expectedPaneKey === 'string' ? { expectedPaneKey: raw.expectedPaneKey } : {}),
|
||||
...(typeof raw.expectedTabId === 'string' ? { expectedTabId: raw.expectedTabId } : {})
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeRuntimeTerminalClose(value: unknown): PersistedRuntimeTerminalClose | null {
|
||||
if (!value || typeof value !== 'object') {
|
||||
return null
|
||||
}
|
||||
const raw = value as Partial<PersistedRuntimeTerminalClose>
|
||||
if (
|
||||
typeof raw.environmentId !== 'string' ||
|
||||
typeof raw.handle !== 'string' ||
|
||||
typeof raw.requestedAt !== 'number'
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
environmentId: raw.environmentId,
|
||||
handle: raw.handle,
|
||||
...(typeof raw.runtimeId === 'string' ? { runtimeId: raw.runtimeId } : {}),
|
||||
requestedAt: raw.requestedAt
|
||||
}
|
||||
}
|
||||
|
||||
type LayoutLeafNormalization = {
|
||||
snapshot: TerminalLayoutSnapshot
|
||||
changed: boolean
|
||||
@@ -3428,6 +3467,12 @@ export class Store {
|
||||
sshRemotePtyLeases: (parsed.sshRemotePtyLeases ?? [])
|
||||
.map(normalizeSshRemotePtyLease)
|
||||
.filter((lease): lease is SshRemotePtyLease => lease !== null),
|
||||
pendingLocalPtyShutdowns: (parsed.pendingLocalPtyShutdowns ?? [])
|
||||
.map(normalizeLocalPtyShutdown)
|
||||
.filter((request): request is PersistedLocalPtyShutdown => request !== null),
|
||||
pendingRuntimeTerminalCloses: (parsed.pendingRuntimeTerminalCloses ?? [])
|
||||
.map(normalizeRuntimeTerminalClose)
|
||||
.filter((request): request is PersistedRuntimeTerminalClose => request !== null),
|
||||
claudeLivePtySessionIds: normalizeClaudeLivePtySessionIds(parsed.claudeLivePtySessionIds),
|
||||
migrationUnsupportedPtyEntries: normalizeMigrationUnsupportedPtyEntries(
|
||||
parsed.migrationUnsupportedPtyEntries
|
||||
@@ -6311,6 +6356,70 @@ export class Store {
|
||||
return leases.filter((lease) => targetId === undefined || lease.targetId === targetId)
|
||||
}
|
||||
|
||||
getPendingLocalPtyShutdowns(): PersistedLocalPtyShutdown[] {
|
||||
return [...(this.state.pendingLocalPtyShutdowns ?? [])]
|
||||
}
|
||||
|
||||
upsertPendingLocalPtyShutdown(request: PersistedLocalPtyShutdown): void {
|
||||
this.state.pendingLocalPtyShutdowns ??= []
|
||||
const index = this.state.pendingLocalPtyShutdowns.findIndex(
|
||||
(entry) => entry.ptyId === request.ptyId
|
||||
)
|
||||
if (index >= 0) {
|
||||
this.state.pendingLocalPtyShutdowns[index] = request
|
||||
} else {
|
||||
this.state.pendingLocalPtyShutdowns.push(request)
|
||||
}
|
||||
this.flush()
|
||||
}
|
||||
|
||||
removePendingLocalPtyShutdown(ptyId: string): void {
|
||||
const before = this.state.pendingLocalPtyShutdowns?.length ?? 0
|
||||
this.state.pendingLocalPtyShutdowns = (this.state.pendingLocalPtyShutdowns ?? []).filter(
|
||||
(entry) => entry.ptyId !== ptyId
|
||||
)
|
||||
if (this.state.pendingLocalPtyShutdowns.length !== before) {
|
||||
this.flush()
|
||||
}
|
||||
}
|
||||
|
||||
getPendingRuntimeTerminalCloses(): PersistedRuntimeTerminalClose[] {
|
||||
return [...(this.state.pendingRuntimeTerminalCloses ?? [])]
|
||||
}
|
||||
|
||||
upsertPendingRuntimeTerminalClose(request: PersistedRuntimeTerminalClose): void {
|
||||
this.state.pendingRuntimeTerminalCloses ??= []
|
||||
const index = this.state.pendingRuntimeTerminalCloses.findIndex(
|
||||
(entry) => entry.environmentId === request.environmentId && entry.handle === request.handle
|
||||
)
|
||||
if (index >= 0) {
|
||||
this.state.pendingRuntimeTerminalCloses[index] = request
|
||||
} else {
|
||||
this.state.pendingRuntimeTerminalCloses.push(request)
|
||||
}
|
||||
this.flush()
|
||||
}
|
||||
|
||||
removePendingRuntimeTerminalClose(environmentId: string, handle: string): void {
|
||||
const before = this.state.pendingRuntimeTerminalCloses?.length ?? 0
|
||||
this.state.pendingRuntimeTerminalCloses = (
|
||||
this.state.pendingRuntimeTerminalCloses ?? []
|
||||
).filter((entry) => entry.environmentId !== environmentId || entry.handle !== handle)
|
||||
if (this.state.pendingRuntimeTerminalCloses.length !== before) {
|
||||
this.flush()
|
||||
}
|
||||
}
|
||||
|
||||
removePendingRuntimeTerminalClosesForEnvironment(environmentId: string): void {
|
||||
const before = this.state.pendingRuntimeTerminalCloses?.length ?? 0
|
||||
this.state.pendingRuntimeTerminalCloses = (
|
||||
this.state.pendingRuntimeTerminalCloses ?? []
|
||||
).filter((entry) => entry.environmentId !== environmentId)
|
||||
if (this.state.pendingRuntimeTerminalCloses.length !== before) {
|
||||
this.flush()
|
||||
}
|
||||
}
|
||||
|
||||
upsertSshRemotePtyLease(
|
||||
lease: Omit<SshRemotePtyLease, 'createdAt' | 'updatedAt'> &
|
||||
Partial<Pick<SshRemotePtyLease, 'createdAt' | 'updatedAt'>>
|
||||
@@ -6334,10 +6443,13 @@ export class Store {
|
||||
entry.targetId === normalizedLease.targetId && entry.ptyId === normalizedLease.ptyId
|
||||
)
|
||||
const existing = existingIndex >= 0 ? this.state.sshRemotePtyLeases[existingIndex] : undefined
|
||||
const sameGeneration = existing?.relayInstanceId === normalizedLease.relayInstanceId
|
||||
const next: SshRemotePtyLease = {
|
||||
...existing,
|
||||
...(sameGeneration ? existing : undefined),
|
||||
...normalizedLease,
|
||||
createdAt: existing?.createdAt ?? normalizedLease.createdAt ?? now,
|
||||
createdAt: sameGeneration
|
||||
? (existing?.createdAt ?? normalizedLease.createdAt ?? now)
|
||||
: (normalizedLease.createdAt ?? now),
|
||||
updatedAt: normalizedLease.updatedAt ?? now
|
||||
}
|
||||
if (existingIndex >= 0) {
|
||||
@@ -6384,9 +6496,13 @@ export class Store {
|
||||
}
|
||||
|
||||
markSshRemotePtyLease(targetId: string, ptyId: string, state: SshRemotePtyLease['state']): void {
|
||||
const requestedGeneration = parseAppSshPtyId(ptyId)?.relayInstanceId
|
||||
const relayPtyId = this.getRelayPtyIdForSshLeaseStorage(targetId, ptyId)
|
||||
const lease = this.state.sshRemotePtyLeases?.find(
|
||||
(entry) => entry.targetId === targetId && entry.ptyId === relayPtyId
|
||||
(entry) =>
|
||||
entry.targetId === targetId &&
|
||||
entry.ptyId === relayPtyId &&
|
||||
(requestedGeneration === undefined || entry.relayInstanceId === requestedGeneration)
|
||||
)
|
||||
if (!lease) {
|
||||
return
|
||||
@@ -6413,9 +6529,13 @@ export class Store {
|
||||
}
|
||||
|
||||
markSshRemotePtyShutdownRequested(targetId: string, ptyId: string): void {
|
||||
const requestedGeneration = parseAppSshPtyId(ptyId)?.relayInstanceId
|
||||
const relayPtyId = this.getRelayPtyIdForSshLeaseStorage(targetId, ptyId)
|
||||
const lease = this.state.sshRemotePtyLeases?.find(
|
||||
(entry) => entry.targetId === targetId && entry.ptyId === relayPtyId
|
||||
(entry) =>
|
||||
entry.targetId === targetId &&
|
||||
entry.ptyId === relayPtyId &&
|
||||
(requestedGeneration === undefined || entry.relayInstanceId === requestedGeneration)
|
||||
)
|
||||
if (!lease) {
|
||||
return
|
||||
@@ -6426,14 +6546,21 @@ export class Store {
|
||||
}
|
||||
|
||||
removeSshRemotePtyLease(targetId: string, ptyId: string): void {
|
||||
const requestedGeneration = parseAppSshPtyId(ptyId)?.relayInstanceId
|
||||
const relayPtyId = this.getRelayPtyIdForSshLeaseStorage(targetId, ptyId)
|
||||
const leases = (this.state.sshRemotePtyLeases ?? []).filter(
|
||||
(lease) => lease.targetId === targetId && lease.ptyId === relayPtyId
|
||||
(lease) =>
|
||||
lease.targetId === targetId &&
|
||||
lease.ptyId === relayPtyId &&
|
||||
(requestedGeneration === undefined || lease.relayInstanceId === requestedGeneration)
|
||||
)
|
||||
const before = this.state.sshRemotePtyLeases?.length ?? 0
|
||||
this.clearSshRemotePtyBindingsForLeases(targetId, leases)
|
||||
this.state.sshRemotePtyLeases = (this.state.sshRemotePtyLeases ?? []).filter(
|
||||
(lease) => lease.targetId !== targetId || lease.ptyId !== relayPtyId
|
||||
(lease) =>
|
||||
lease.targetId !== targetId ||
|
||||
lease.ptyId !== relayPtyId ||
|
||||
(requestedGeneration !== undefined && lease.relayInstanceId !== requestedGeneration)
|
||||
)
|
||||
if (this.state.sshRemotePtyLeases.length !== before) {
|
||||
this.flush()
|
||||
|
||||
@@ -735,6 +735,10 @@ const TerminalHandle = z.object({
|
||||
terminal: requiredString('Missing terminal handle')
|
||||
})
|
||||
|
||||
const TerminalClose = TerminalHandle.extend({
|
||||
expectedRuntimeId: z.string().min(1).optional()
|
||||
})
|
||||
|
||||
const TerminalListParams = z.object({
|
||||
worktree: OptionalString,
|
||||
limit: OptionalFiniteNumber,
|
||||
@@ -1363,10 +1367,13 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
}),
|
||||
defineMethod({
|
||||
name: 'terminal.close',
|
||||
params: TerminalHandle,
|
||||
handler: async (params, { runtime }) => ({
|
||||
close: await runtime.closeTerminal(params.terminal)
|
||||
})
|
||||
params: TerminalClose,
|
||||
handler: async (params, { runtime }) => {
|
||||
if (params.expectedRuntimeId && params.expectedRuntimeId !== runtime.getRuntimeId()) {
|
||||
throw new Error('runtime_generation_mismatch')
|
||||
}
|
||||
return { close: await runtime.closeTerminal(params.terminal) }
|
||||
}
|
||||
}),
|
||||
defineMethod({
|
||||
name: 'agentTeams.tmuxCompat',
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { OrcaRuntimeService } from '../orca-runtime'
|
||||
import type { RpcRequest } from './core'
|
||||
import { RpcDispatcher } from './dispatcher'
|
||||
import { TERMINAL_METHODS } from './methods/terminal'
|
||||
|
||||
function request(params: unknown): RpcRequest {
|
||||
return { id: 'req-1', authToken: 'tok', method: 'terminal.close', params }
|
||||
}
|
||||
|
||||
describe('terminal close runtime generation', () => {
|
||||
it('rejects a close fenced to a replaced runtime before touching the handle', async () => {
|
||||
const closeTerminal = vi.fn()
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime: {
|
||||
getRuntimeId: () => 'runtime-b',
|
||||
closeTerminal
|
||||
} as unknown as OrcaRuntimeService,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const response = await dispatcher.dispatch(
|
||||
request({ terminal: 'terminal-1', expectedRuntimeId: 'runtime-a' })
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({
|
||||
ok: false,
|
||||
error: { message: expect.stringContaining('runtime_generation_mismatch') },
|
||||
_meta: { runtimeId: 'runtime-b' }
|
||||
})
|
||||
expect(closeTerminal).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('closes a handle when the expected runtime generation still owns it', async () => {
|
||||
const closeTerminal = vi.fn().mockResolvedValue({ closed: true })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime: {
|
||||
getRuntimeId: () => 'runtime-a',
|
||||
closeTerminal
|
||||
} as unknown as OrcaRuntimeService,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
await expect(
|
||||
dispatcher.dispatch(request({ terminal: 'terminal-1', expectedRuntimeId: 'runtime-a' }))
|
||||
).resolves.toMatchObject({ ok: true })
|
||||
expect(closeTerminal).toHaveBeenCalledWith('terminal-1')
|
||||
})
|
||||
})
|
||||
@@ -553,7 +553,11 @@ describe('SshRelaySession', () => {
|
||||
|
||||
expect(mockAttach).toHaveBeenCalledWith('pty-1')
|
||||
expect(setPtyOwnership).toHaveBeenCalledWith('ssh:target-1@@pty-1', 'target-1')
|
||||
expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith('target-1', 'pty-1', 'attached')
|
||||
expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith(
|
||||
'target-1',
|
||||
'ssh:target-1@@pty-1',
|
||||
'attached'
|
||||
)
|
||||
})
|
||||
|
||||
it('forwards a lease tab identity to reattach so a reset relay cannot cross-wire it', async () => {
|
||||
|
||||
@@ -1159,11 +1159,10 @@ export class SshRelaySession {
|
||||
}
|
||||
})
|
||||
ptyProvider.onExit((payload) => {
|
||||
const relayPtyId = toRelaySshPtyId(this.targetId, payload.id)
|
||||
clearProviderPtyState(payload.id)
|
||||
deletePtyOwnership(payload.id)
|
||||
this.forwardedReattachReplayByPty.delete(payload.id)
|
||||
this.store.markSshRemotePtyLease(this.targetId, relayPtyId, 'terminated')
|
||||
this.store.markSshRemotePtyLease(this.targetId, payload.id, 'terminated')
|
||||
this.runtime?.onPtyExit(payload.id, payload.code)
|
||||
const win = this.getMainWindow()
|
||||
if (win && !win.isDestroyed()) {
|
||||
@@ -1200,11 +1199,11 @@ export class SshRelaySession {
|
||||
}
|
||||
}
|
||||
|
||||
private expireReattachPty(relayPtyId: string, appPtyId: string): void {
|
||||
private expireReattachPty(_relayPtyId: string, appPtyId: string): void {
|
||||
clearProviderPtyState(appPtyId)
|
||||
deletePtyOwnership(appPtyId)
|
||||
this.forwardedReattachReplayByPty.delete(appPtyId)
|
||||
this.store.markSshRemotePtyLease(this.targetId, relayPtyId, 'expired')
|
||||
this.store.markSshRemotePtyLease(this.targetId, appPtyId, 'expired')
|
||||
const win = this.getMainWindow()
|
||||
if (win && !win.isDestroyed()) {
|
||||
win.webContents.send('pty:exit', { id: appPtyId, code: -1 })
|
||||
@@ -1276,7 +1275,7 @@ export class SshRelaySession {
|
||||
}
|
||||
const appPtyId = toAppSshPtyId(this.targetId, ptyId, relayInstanceId)
|
||||
setPtyOwnership(appPtyId, this.targetId)
|
||||
this.store.markSshRemotePtyLease(this.targetId, ptyId, 'attached')
|
||||
this.store.markSshRemotePtyLease(this.targetId, appPtyId, 'attached')
|
||||
this.forwardReattachReplay(appPtyId, attachResult.replay ?? '')
|
||||
} catch (err) {
|
||||
if (!isSshPtyNotFoundError(err)) {
|
||||
@@ -1299,7 +1298,7 @@ export class SshRelaySession {
|
||||
clearProviderPtyState(appPtyId)
|
||||
deletePtyOwnership(appPtyId)
|
||||
this.forwardedReattachReplayByPty.delete(appPtyId)
|
||||
this.store.markSshRemotePtyLease(this.targetId, ptyId, 'expired')
|
||||
this.store.markSshRemotePtyLease(this.targetId, appPtyId, 'expired')
|
||||
// Why: if the new relay cannot reattach this id, the remote backing
|
||||
// process is gone. Tell the renderer so it clears stale pane bindings
|
||||
// instead of keeping a cursor-only terminal.
|
||||
|
||||
@@ -1468,14 +1468,11 @@ describe('createIpcPtyTransport', () => {
|
||||
expect(onPtySpawn).not.toHaveBeenCalled()
|
||||
expect(transport.getPtyId()).toBeNull()
|
||||
|
||||
// A later transport lifecycle event retries the retained exact id.
|
||||
// Main owns the retry; later renderer lifecycle must not duplicate it.
|
||||
spawnMock.mockResolvedValueOnce({ id: 'pty-next' })
|
||||
await createIpcPtyTransport().connect({ url: '', callbacks: {} })
|
||||
await flushPtySideEffects()
|
||||
expect(killMock).toHaveBeenNthCalledWith(2, 'pty-late', {
|
||||
expectedPaneKey: 'tab-late:11111111-1111-4111-8111-111111111111',
|
||||
expectedTabId: 'tab-late'
|
||||
})
|
||||
expect(killMock).toHaveBeenCalledTimes(1)
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
}
|
||||
@@ -1497,7 +1494,7 @@ describe('createIpcPtyTransport', () => {
|
||||
expect(warn).toHaveBeenCalledWith('[pty] Failed to stop disconnected PTY', killError)
|
||||
await createIpcPtyTransport().connect({ url: '', callbacks: {} })
|
||||
await flushPtySideEffects()
|
||||
expect(kill).toHaveBeenNthCalledWith(2, 'pty-1')
|
||||
expect(kill).toHaveBeenCalledTimes(1)
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
} from './pty-kill-retry-ownership'
|
||||
import { toAppSshPtyId } from '../../../shared/ssh-pty-id'
|
||||
|
||||
const IDS = Array.from({ length: 65 }, (_, index) => `pty-retained-${index}`)
|
||||
const IDS = Array.from({ length: 2 }, (_, index) => `pty-retained-${index}`)
|
||||
|
||||
describe('PTY kill retry ownership', () => {
|
||||
afterEach(() => {
|
||||
@@ -18,40 +18,19 @@ describe('PTY kill retry ownership', () => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('retains every live id and retries all due owners after bounded backoff', async () => {
|
||||
vi.useFakeTimers()
|
||||
const kill = vi.fn().mockRejectedValue(new Error('provider disconnected'))
|
||||
it('delegates exact identity to the main-owned retry boundary', async () => {
|
||||
const kill = vi.fn().mockResolvedValue(undefined)
|
||||
vi.stubGlobal('window', { api: { pty: { kill } } })
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
for (const id of IDS) {
|
||||
await killPtyRetainingRetryOwnership(id, '[pty] failed').catch(() => {})
|
||||
}
|
||||
kill.mockResolvedValue(undefined)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
|
||||
expect(kill).toHaveBeenCalledTimes(IDS.length * 2)
|
||||
for (const id of IDS) {
|
||||
expect(kill.mock.calls.filter(([calledId]) => calledId === id)).toHaveLength(2)
|
||||
}
|
||||
})
|
||||
|
||||
it('retries after a second failure without requiring another lifecycle event', async () => {
|
||||
vi.useFakeTimers()
|
||||
const kill = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('offline'))
|
||||
.mockRejectedValueOnce(new Error('still offline'))
|
||||
.mockResolvedValue(undefined)
|
||||
vi.stubGlobal('window', { api: { pty: { kill } } })
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
await killPtyRetainingRetryOwnership(IDS[0], '[pty] failed').catch(() => {})
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
expect(kill).toHaveBeenCalledTimes(2)
|
||||
await vi.advanceTimersByTimeAsync(500)
|
||||
expect(kill).toHaveBeenCalledTimes(3)
|
||||
await killPtyRetainingRetryOwnership(IDS[0], '[pty] failed', {
|
||||
expectedPaneKey: 'tab:leaf',
|
||||
expectedTabId: 'tab'
|
||||
})
|
||||
expect(kill).toHaveBeenCalledWith(IDS[0], {
|
||||
expectedPaneKey: 'tab:leaf',
|
||||
expectedTabId: 'tab'
|
||||
})
|
||||
})
|
||||
|
||||
it('releases only a removed SSH target and cancels the final retry timer', async () => {
|
||||
@@ -68,7 +47,7 @@ describe('PTY kill retry ownership', () => {
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
|
||||
expect(kill.mock.calls.filter(([id]) => id === removedId)).toHaveLength(1)
|
||||
expect(kill.mock.calls.filter(([id]) => id === retainedId)).toHaveLength(2)
|
||||
expect(kill.mock.calls.filter(([id]) => id === retainedId)).toHaveLength(1)
|
||||
releaseRetainedPtyKillsForSshTarget('retained-target')
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
})
|
||||
|
||||
@@ -1,122 +1,31 @@
|
||||
type RetainedPtyKill = {
|
||||
diagnostic: string
|
||||
inFlight: Promise<void> | null
|
||||
options?: PtyKillIdentity
|
||||
attempts: number
|
||||
nextRetryAt: number
|
||||
}
|
||||
|
||||
export type PtyKillIdentity = {
|
||||
expectedPaneKey?: string
|
||||
expectedTabId?: string
|
||||
}
|
||||
|
||||
const MAX_RETRY_BACKOFF_MS = 30_000
|
||||
const retainedPtyKills = new Map<string, RetainedPtyKill>()
|
||||
let retryTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
function scheduleRetainedPtyKillRetries(): void {
|
||||
if (retryTimer) {
|
||||
clearTimeout(retryTimer)
|
||||
retryTimer = null
|
||||
}
|
||||
let nextRetryAt = Number.POSITIVE_INFINITY
|
||||
for (const retained of retainedPtyKills.values()) {
|
||||
if (!retained.inFlight) {
|
||||
nextRetryAt = Math.min(nextRetryAt, retained.nextRetryAt)
|
||||
}
|
||||
}
|
||||
if (!Number.isFinite(nextRetryAt)) {
|
||||
return
|
||||
}
|
||||
retryTimer = setTimeout(() => {
|
||||
retryTimer = null
|
||||
const now = Date.now()
|
||||
for (const [id, retained] of retainedPtyKills) {
|
||||
if (!retained.inFlight && retained.nextRetryAt <= now) {
|
||||
void killPtyRetainingRetryOwnership(id, retained.diagnostic, retained.options).catch(
|
||||
() => {}
|
||||
)
|
||||
}
|
||||
}
|
||||
}, Math.max(0, nextRetryAt - Date.now()))
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep exact PTY identity until the owning provider accepts shutdown. A rejected
|
||||
* local/SSH IPC call can otherwise leave a live process with no renderer owner.
|
||||
*/
|
||||
/** Main persists exact provider identity and owns retries after renderer loss. */
|
||||
export function killPtyRetainingRetryOwnership(
|
||||
id: string,
|
||||
diagnostic: string,
|
||||
options?: PtyKillIdentity
|
||||
): Promise<void> {
|
||||
let retained = retainedPtyKills.get(id)
|
||||
if (!retained) {
|
||||
// Why: every entry represents a provider process whose shutdown is still
|
||||
// unconfirmed. Dropping one to cap bookkeeping would orphan the real PTY;
|
||||
// repeated failures for the same PTY still coalesce into this single record.
|
||||
retained = { diagnostic, inFlight: null, options, attempts: 0, nextRetryAt: 0 }
|
||||
}
|
||||
retained.diagnostic = diagnostic
|
||||
retained.options = options ?? retained.options
|
||||
retainedPtyKills.set(id, retained)
|
||||
if (retained.inFlight) {
|
||||
return retained.inFlight
|
||||
}
|
||||
|
||||
const attempt = Promise.resolve()
|
||||
.then(() =>
|
||||
retained.options ? window.api.pty.kill(id, retained.options) : window.api.pty.kill(id)
|
||||
)
|
||||
.then(() => {
|
||||
retainedPtyKills.delete(id)
|
||||
})
|
||||
return Promise.resolve()
|
||||
.then(() => (options ? window.api.pty.kill(id, options) : window.api.pty.kill(id)))
|
||||
.catch((error: unknown) => {
|
||||
retained.attempts += 1
|
||||
retained.nextRetryAt =
|
||||
Date.now() +
|
||||
Math.min(MAX_RETRY_BACKOFF_MS, 250 * 2 ** Math.min(retained.attempts - 1, 7))
|
||||
// Why: automatic retries can outlive a disconnect; bound diagnostics so
|
||||
// a long outage cannot become a log-backpressure loop.
|
||||
if (retained.attempts <= 2 || retained.attempts === 8) {
|
||||
console.warn(retained.diagnostic, error)
|
||||
}
|
||||
// Why: main owns retry state, but the initiating renderer still records the first failure.
|
||||
console.warn(diagnostic, error)
|
||||
throw error
|
||||
})
|
||||
.finally(() => {
|
||||
if (retained.inFlight === attempt) {
|
||||
retained.inFlight = null
|
||||
}
|
||||
scheduleRetainedPtyKillRetries()
|
||||
})
|
||||
retained.inFlight = attempt
|
||||
return attempt
|
||||
}
|
||||
|
||||
/** Retry immediately on the next PTY lifecycle event in addition to bounded backoff. */
|
||||
export function retryRetainedPtyKills(): void {
|
||||
for (const [id, retained] of retainedPtyKills) {
|
||||
if (!retained.inFlight) {
|
||||
void killPtyRetainingRetryOwnership(id, retained.diagnostic, retained.options).catch(() => {})
|
||||
return
|
||||
}
|
||||
}
|
||||
// Main retries on provider availability and bounded backoff.
|
||||
}
|
||||
|
||||
export function releaseRetainedPtyKillOwnership(id: string): void {
|
||||
retainedPtyKills.delete(id)
|
||||
scheduleRetainedPtyKillRetries()
|
||||
void id
|
||||
}
|
||||
|
||||
export function releaseRetainedPtyKillsForSshTarget(targetId: string): void {
|
||||
for (const id of retainedPtyKills.keys()) {
|
||||
if (parseAppSshPtyId(id)?.connectionId === targetId) {
|
||||
retainedPtyKills.delete(id)
|
||||
}
|
||||
}
|
||||
// Why: successful target removal is authoritative proof that reconnect can
|
||||
// no longer reach these PTYs, so retry ownership must not outlive the host.
|
||||
scheduleRetainedPtyKillRetries()
|
||||
void targetId
|
||||
}
|
||||
import { parseAppSshPtyId } from '../../../shared/ssh-pty-id'
|
||||
|
||||
@@ -20,33 +20,23 @@ describe('runtime terminal close retry ownership', () => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('retries a retained handle after repeated failure without another lifecycle event', async () => {
|
||||
vi.useFakeTimers()
|
||||
callRuntimeRpc
|
||||
.mockRejectedValueOnce(new Error('offline'))
|
||||
.mockRejectedValueOnce(new Error('still offline'))
|
||||
.mockResolvedValue(undefined)
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
await closeRuntimeTerminalRetainingRetryOwnership(TARGET, 'terminal-1').catch(() => {})
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
expect(callRuntimeRpc).toHaveBeenCalledTimes(2)
|
||||
await vi.advanceTimersByTimeAsync(500)
|
||||
|
||||
expect(callRuntimeRpc).toHaveBeenCalledTimes(3)
|
||||
it('delegates the exact handle to the main-owned retry boundary', async () => {
|
||||
callRuntimeRpc.mockResolvedValue(undefined)
|
||||
await closeRuntimeTerminalRetainingRetryOwnership(TARGET, 'terminal-1')
|
||||
expect(callRuntimeRpc).toHaveBeenCalledTimes(1)
|
||||
expect(callRuntimeRpc).toHaveBeenLastCalledWith(TARGET, 'terminal.close', {
|
||||
terminal: 'terminal-1'
|
||||
})
|
||||
})
|
||||
|
||||
it('releases every removed-environment handle and cancels its retry timer', async () => {
|
||||
it('does not retain renderer timers when the environment is removed', async () => {
|
||||
vi.useFakeTimers()
|
||||
callRuntimeRpc.mockRejectedValue(new Error('environment unavailable'))
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
await closeRuntimeTerminalRetainingRetryOwnership(TARGET, 'terminal-1').catch(() => {})
|
||||
await closeRuntimeTerminalRetainingRetryOwnership(TARGET, 'terminal-2').catch(() => {})
|
||||
expect(vi.getTimerCount()).toBe(1)
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
|
||||
releaseRetainedRuntimeTerminalClosesForEnvironment(TARGET.environmentId)
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
|
||||
@@ -1,117 +1,27 @@
|
||||
import {
|
||||
callRuntimeRpc,
|
||||
type RuntimeClientTarget
|
||||
} from '@/runtime/runtime-rpc-client'
|
||||
import { callRuntimeRpc, type RuntimeClientTarget } from '@/runtime/runtime-rpc-client'
|
||||
|
||||
type EnvironmentTarget = Extract<RuntimeClientTarget, { kind: 'environment' }>
|
||||
|
||||
type RetainedRuntimeTerminalClose = {
|
||||
target: EnvironmentTarget
|
||||
handle: string
|
||||
inFlight: Promise<void> | null
|
||||
attempts: number
|
||||
nextRetryAt: number
|
||||
}
|
||||
|
||||
const MAX_RETRY_BACKOFF_MS = 30_000
|
||||
const retainedCloses = new Map<string, RetainedRuntimeTerminalClose>()
|
||||
let retryTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
function retainedCloseKey(target: EnvironmentTarget, handle: string): string {
|
||||
return `${target.environmentId}\0${handle}`
|
||||
}
|
||||
|
||||
function scheduleRuntimeTerminalCloseRetries(): void {
|
||||
if (retryTimer) {
|
||||
clearTimeout(retryTimer)
|
||||
retryTimer = null
|
||||
}
|
||||
let nextRetryAt = Number.POSITIVE_INFINITY
|
||||
for (const retained of retainedCloses.values()) {
|
||||
if (!retained.inFlight) {
|
||||
nextRetryAt = Math.min(nextRetryAt, retained.nextRetryAt)
|
||||
}
|
||||
}
|
||||
if (!Number.isFinite(nextRetryAt)) {
|
||||
return
|
||||
}
|
||||
retryTimer = setTimeout(() => {
|
||||
retryTimer = null
|
||||
const now = Date.now()
|
||||
for (const retained of retainedCloses.values()) {
|
||||
if (!retained.inFlight && retained.nextRetryAt <= now) {
|
||||
void closeRuntimeTerminalRetainingRetryOwnership(retained.target, retained.handle).catch(
|
||||
() => {}
|
||||
)
|
||||
}
|
||||
}
|
||||
}, Math.max(0, nextRetryAt - Date.now()))
|
||||
}
|
||||
|
||||
export function closeRuntimeTerminalRetainingRetryOwnership(
|
||||
target: EnvironmentTarget,
|
||||
handle: string
|
||||
): Promise<void> {
|
||||
const key = retainedCloseKey(target, handle)
|
||||
let retained = retainedCloses.get(key)
|
||||
if (!retained) {
|
||||
retained = { target, handle, inFlight: null, attempts: 0, nextRetryAt: 0 }
|
||||
retainedCloses.set(key, retained)
|
||||
}
|
||||
if (retained.inFlight) {
|
||||
return retained.inFlight
|
||||
}
|
||||
|
||||
const attempt = callRuntimeRpc(target, 'terminal.close', { terminal: handle })
|
||||
.then(() => {
|
||||
retainedCloses.delete(key)
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
retained.attempts += 1
|
||||
retained.nextRetryAt =
|
||||
Date.now() +
|
||||
Math.min(MAX_RETRY_BACKOFF_MS, 250 * 2 ** Math.min(retained.attempts - 1, 7))
|
||||
if (retained.attempts <= 2 || retained.attempts === 8) {
|
||||
console.warn('[terminal] Failed to close retained runtime terminal', error)
|
||||
}
|
||||
throw error
|
||||
})
|
||||
.finally(() => {
|
||||
if (retained.inFlight === attempt) {
|
||||
retained.inFlight = null
|
||||
}
|
||||
scheduleRuntimeTerminalCloseRetries()
|
||||
})
|
||||
retained.inFlight = attempt
|
||||
return attempt
|
||||
// Why: main owns durable retry intent, so renderer loss cannot orphan the handle.
|
||||
return callRuntimeRpc(target, 'terminal.close', { terminal: handle })
|
||||
}
|
||||
|
||||
export function retryRetainedRuntimeTerminalCloses(): void {
|
||||
for (const retained of retainedCloses.values()) {
|
||||
if (!retained.inFlight) {
|
||||
void closeRuntimeTerminalRetainingRetryOwnership(retained.target, retained.handle).catch(
|
||||
() => {}
|
||||
)
|
||||
return
|
||||
}
|
||||
}
|
||||
// Main retries on provider availability and bounded backoff.
|
||||
}
|
||||
|
||||
export function releaseRetainedRuntimeTerminalClose(
|
||||
target: EnvironmentTarget,
|
||||
handle: string
|
||||
): void {
|
||||
retainedCloses.delete(retainedCloseKey(target, handle))
|
||||
scheduleRuntimeTerminalCloseRetries()
|
||||
void target
|
||||
void handle
|
||||
}
|
||||
|
||||
export function releaseRetainedRuntimeTerminalClosesForEnvironment(environmentId: string): void {
|
||||
for (const [key, retained] of retainedCloses) {
|
||||
if (retained.target.environmentId === environmentId) {
|
||||
retainedCloses.delete(key)
|
||||
}
|
||||
}
|
||||
// Why: a successfully removed runtime cannot be reconnected, so retained
|
||||
// handles have no reachable provider resource and must release their timer.
|
||||
scheduleRuntimeTerminalCloseRetries()
|
||||
void environmentId
|
||||
}
|
||||
|
||||
@@ -452,6 +452,8 @@ export function getDefaultPersistedState(homedir: string): PersistedState {
|
||||
sshTargets: [],
|
||||
deletedSshConfigAliases: [],
|
||||
sshRemotePtyLeases: [],
|
||||
pendingLocalPtyShutdowns: [],
|
||||
pendingRuntimeTerminalCloses: [],
|
||||
claudeLivePtySessionIds: [],
|
||||
migrationUnsupportedPtyEntries: [],
|
||||
legacyPaneKeyAliasEntries: [],
|
||||
|
||||
@@ -3565,6 +3565,20 @@ export type LegacyPaneKeyAliasEntry = {
|
||||
updatedAt: number
|
||||
}
|
||||
|
||||
export type PersistedLocalPtyShutdown = {
|
||||
ptyId: string
|
||||
expectedPaneKey?: string
|
||||
expectedTabId?: string
|
||||
requestedAt: number
|
||||
}
|
||||
|
||||
export type PersistedRuntimeTerminalClose = {
|
||||
environmentId: string
|
||||
handle: string
|
||||
runtimeId?: string
|
||||
requestedAt: number
|
||||
}
|
||||
|
||||
// ─── Persistence shape ──────────────────────────────────────────────
|
||||
export type PersistedState = {
|
||||
schemaVersion: number
|
||||
@@ -3602,6 +3616,10 @@ export type PersistedState = {
|
||||
* workspaces that were orphaned on the old target id. Pruned by age/count. */
|
||||
removedSshTargetTombstones?: RemovedSshTargetTombstone[]
|
||||
sshRemotePtyLeases: SshRemotePtyLease[]
|
||||
/** Exact local daemon shutdown intents survive renderer and app restarts. */
|
||||
pendingLocalPtyShutdowns?: PersistedLocalPtyShutdown[]
|
||||
/** Remote runtime close intents survive renderer and app restarts. */
|
||||
pendingRuntimeTerminalCloses?: PersistedRuntimeTerminalClose[]
|
||||
/** Daemon session ids of live local Claude launches. Seeds the Claude
|
||||
* live-PTY gate on startup so an early OAuth refresh cannot rotate the
|
||||
* single-use refresh token out from under a still-running daemon CLI. */
|
||||
|
||||
Reference in New Issue
Block a user