From c124ea9f2a83eee17534140b1c486276f9038ec4 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Sun, 23 Aug 2026 20:11:33 -0400 Subject: [PATCH] fix: fence terminal session owner authority --- PTY_DAEMON_IDENTITY_ARCHITECTURE_DESIGN.md | 487 ----------- TERMINAL_SESSION_DISAPPEARANCE_HANDOFF.md | 242 ------ ...SSION_DISAPPEARANCE_INCIDENT_2026-08-23.md | 783 ------------------ config/reliability-gates.jsonc | 215 +++++ .../RECOVERY_CATALOG.md | 243 ------ .../POST_CUTOVER_RECOVERY_STATUS.md | 170 ---- .../RECOVERY_V2_CUTOVER_STATUS.md | 220 ----- .../restored-subagent-liveness-sweep.test.ts | 58 +- .../restored-subagent-liveness-sweep.ts | 27 +- src/main/daemon/daemon-errors.ts | 7 + ...n-foreground-confirmation-protocol.test.ts | 2 +- src/main/daemon/daemon-protocol-version.ts | 7 +- ...pty-adapter-protocol-compatibility.test.ts | 37 +- ...aemon-pty-adapter-session-adoption.test.ts | 54 ++ .../daemon-pty-adapter-subscription-fanout.ts | 4 +- src/main/daemon/daemon-pty-adapter.test.ts | 26 + src/main/daemon/daemon-pty-adapter.ts | 141 +++- .../daemon-pty-router-owner-fencing.test.ts | 69 ++ src/main/daemon/daemon-pty-router.test.ts | 39 +- src/main/daemon/daemon-pty-router.ts | 26 +- src/main/daemon/daemon-server.ts | 3 + ...mon-session-owner-authority.oracle.test.ts | 53 ++ .../daemon-session-owner-resolution.test.ts | 64 +- .../daemon/daemon-session-owner-resolution.ts | 94 ++- .../degraded-daemon-pty-provider.test.ts | 23 +- .../daemon/degraded-daemon-pty-provider.ts | 4 +- .../daemon/terminal-host-create-contract.ts | 2 + .../terminal-host-incarnation-fence.test.ts | 115 +++ .../daemon/terminal-host-session-create.ts | 7 + src/main/daemon/types.ts | 4 + src/main/index.ts | 5 +- src/main/ipc/pty-dead-owner-respawn.test.ts | 7 +- .../pty-persisted-incarnation-repair.test.ts | 108 +-- .../pty-startup-barrier-and-listing.test.ts | 76 ++ src/main/ipc/pty.ts | 118 ++- src/main/providers/pty-provider-contract.ts | 5 +- ...-pty-provider-reattach-incarnation.test.ts | 63 +- .../providers/ssh-pty-provider-spawn.test.ts | 8 +- .../providers/ssh-pty-session-reattach.ts | 38 +- src/preload/api/pty-api.ts | 2 +- src/preload/index.ts | 5 +- src/relay/pty-handler-attach-replay.test.ts | 49 +- src/relay/pty-handler.ts | 12 +- ...nnection-direct-ssh-reattach-retry.test.ts | 78 ++ .../pty-connection-reattach-binding.test.ts | 111 ++- .../pty-connection-session-liveness.test.ts | 6 +- .../pty-connection/deferred-session-attach.ts | 10 +- .../deferred-session-reattach-connect.ts | 25 +- .../direct-ssh-reattach-recovery.test.ts | 8 +- .../pty-connection/reattach-result-handler.ts | 24 +- .../session-reconcile-dispose.ts | 7 +- ...y.ts => unverifiable-reattach-recovery.ts} | 2 +- .../pty-transport-spawn-errors.test.ts | 2 +- .../terminal-pane/pty-transport-types.ts | 1 + .../components/terminal-pane/pty-transport.ts | 33 +- .../remote-runtime-pty-transport.ts | 12 +- .../terminal-dead-session-reconcile.ts | 5 +- src/shared/terminal-pane-owner-verdict.ts | 12 + 58 files changed, 1482 insertions(+), 2576 deletions(-) delete mode 100644 PTY_DAEMON_IDENTITY_ARCHITECTURE_DESIGN.md delete mode 100644 TERMINAL_SESSION_DISAPPEARANCE_HANDOFF.md delete mode 100644 TERMINAL_SESSION_DISAPPEARANCE_INCIDENT_2026-08-23.md delete mode 100644 recovery/2026-08-23-session-disappearance/RECOVERY_CATALOG.md delete mode 100644 recovery/2026-08-23-session-disappearance/cutover-20260823T201851Z/POST_CUTOVER_RECOVERY_STATUS.md delete mode 100644 recovery/2026-08-23-session-disappearance/cutover-v2-20260823T140110MST/RECOVERY_V2_CUTOVER_STATUS.md create mode 100644 src/main/daemon/daemon-pty-router-owner-fencing.test.ts create mode 100644 src/main/daemon/daemon-session-owner-authority.oracle.test.ts create mode 100644 src/main/daemon/terminal-host-incarnation-fence.test.ts rename src/renderer/src/components/terminal-pane/pty-connection/{direct-ssh-reattach-recovery.ts => unverifiable-reattach-recovery.ts} (92%) create mode 100644 src/shared/terminal-pane-owner-verdict.ts diff --git a/PTY_DAEMON_IDENTITY_ARCHITECTURE_DESIGN.md b/PTY_DAEMON_IDENTITY_ARCHITECTURE_DESIGN.md deleted file mode 100644 index 02b51922a02..00000000000 --- a/PTY_DAEMON_IDENTITY_ARCHITECTURE_DESIGN.md +++ /dev/null @@ -1,487 +0,0 @@ -# PTY daemon identity and lifetime architecture - -## Status - -Discussion document. No architecture is selected yet. - -This document compares two permanent fixes for PTY ownership across Orca app and daemon -restarts: - -1. incarnation-addressed daemons; -2. one stable PTY host separated from the replaceable control daemon. - -It is grounded in the -[2026-08-23 terminal-session incident](./TERMINAL_SESSION_DISAPPEARANCE_INCIDENT_2026-08-23.md) -and the -[recovery-v2 control case](./recovery/2026-08-23-session-disappearance/cutover-v2-20260823T140110MST/RECOVERY_V2_CUTOVER_STATUS.md). - -## Problem - -Orca currently addresses a local daemon primarily through a protocol-version endpoint such as -`daemon-v36.sock`. Protocol version answers whether two processes can communicate; it is not the -identity of the process that owns a PTY. - -The current code already has useful process- and session-incarnation checks. However, the exact -owner route is not a durable, self-addressing part of every persisted terminal record, and the -endpoint namespace still permits only one current owner per protocol version. On restart, routing -can fall back to asking the currently registered adapters which one recognizes a session. - -That produces an invalid inference: - -```text -session not found in registered adapters - -> exact owner route is unavailable - -> historically collapsed to exited -``` - -The only valid exit authority is the execution host and exact owner/session incarnation: - -```text -matching owner reports matching session incarnation exited - -> raw PTY exited -``` - -The execution-host contract remains -[`live` / `unverifiable` / `exited`](./docs/reference/ssh-execution-boundary.md). Transport loss, -registry loss, socket absence, or an answer from a different daemon incarnation can never -establish `exited`. - -## Goals - -- Route every persisted PTY to its exact owner without scanning unrelated adapters. -- Allow an app update to reuse a compatible live owner. -- Prevent a new same-protocol process from taking over the identity of an old owner. -- Reconstruct routes after app/control-plane restart. -- Accept exit evidence only from the matching execution host and owner/session incarnation. -- Bound idle CPU, memory, process, socket, and file-descriptor overhead. -- Support macOS, Linux, Windows, WSL, direct SSH, paired runtimes, git worktrees, and folder - workspaces. -- Preserve mixed-version compatibility during a staged rollout. -- Keep logical provider recovery distinct from raw PTY reattachment. - -## Non-goals - -- Proving that a detached descendant process exited when its PTY owner died. -- Reconstructing arbitrary shell state after the process holding the PTY master is gone. -- Treating worktree existence, UI topology, or provider-session presence as process-liveness - evidence. -- Solving the separate missing-worktree teardown defect. -- Changing update-window activation behavior. - -## Shared identity model - -Both approaches need to separate logical identity, process ownership, compatibility, and -incarnation: - -```ts -interface PersistedPtyOwnerRef { - executionHostId: string - ownerKind: 'daemon' | 'pty-host' - ownerIncarnationId: string - sessionIncarnationId: string - protocolVersion: number -} -``` - -The existing logical PTY ID remains stable when product behavior requires an in-place provider -continuation. A replacement raw PTY must always receive a new `sessionIncarnationId`, even if it -reuses the logical PTY ID and visual tab. - -| Field | Meaning | Must not mean | -|---|---|---| -| Logical PTY ID | Stable product/tab/provider recovery identity | A particular OS process | -| Session incarnation | One raw PTY lifetime | The latest session with the same logical ID | -| Owner incarnation | Exact PTY-owning process lifetime | Protocol version or PID | -| Execution host | Host with process authority | Client currently displaying the terminal | -| Protocol version | Compatibility contract | Owner identity | -| PID | Diagnostic observation | Durable identity; PIDs are reused | - -Endpoint paths must be derived from validated incarnation IDs inside a host-controlled runtime -directory. Persisted state must not contain an arbitrary socket or named-pipe path. - -## Shared semantic invariants - -Both approaches must enforce these rules: - -1. Only the exact owner may report `live` or `exited` for a session incarnation. -2. An unreachable or undiscoverable owner produces `unverifiable`. -3. A different compatible owner reporting absence is irrelevant. -4. A late event for a superseded session incarnation cannot close its replacement. -5. UI and persisted topology survive every `unverifiable` verdict. -6. Provider continuation creates a new raw session incarnation; it is not reattachment. -7. New sessions may use a current-owner pointer, but existing sessions never do. -8. Owner routing is execution-host state. A client disconnect cannot change the verdict to - `exited`. - -## Approach A: incarnation-addressed daemons - -### Design - -Every daemon process gets a random incarnation UUID and a unique endpoint: - -```text -runtime/ - daemons/ - / - endpoint - metadata.json - token -``` - -On Windows, the host derives an equivalent named-pipe name from the UUID. On Unix, the runtime -directory and abbreviated/encoded UUID must stay within socket-path length limits. - -The host maintains a small current-daemon registry used only for new spawns: - -```text -compatibility/capability requirement -> current daemon incarnation -``` - -Persisted sessions route directly through `ownerIncarnationId`. A new v36 daemon cannot replace -the endpoint belonging to an older v36 daemon. Multiple daemons speaking v36 may coexist while -they own different sessions. - -```text -new PTY -> compatible current daemon -old PTY -> recorded daemon incarnation -> unique endpoint -``` - -### Lifecycle - -#### Spawn - -1. Select or start a compatible current daemon. -2. The daemon creates the PTY and returns both logical and session-incarnation IDs plus its daemon - incarnation ID. -3. Persist the owner reference before exposing the terminal as recoverable. -4. Route all later operations directly to that owner. - -#### App restart or update - -1. Read persisted owner references. -2. Connect to each referenced daemon endpoint directly. -3. Reuse a compatible live daemon for new sessions where policy permits. -4. Do not create one daemon per app build. - -#### Protocol or capability transition - -Start a new current daemon only when the existing one cannot satisfy a required capability. The -old daemon continues serving only its existing sessions and retires after its authoritative live -session count reaches zero. - -#### Owner unavailable - -Return `unverifiable`. Keep topology and the exact owner reference. A host-side proof that the -owner process incarnation is gone establishes loss of the raw PTY owner, but it does not prove -that detached descendants exited. - -#### Retirement and garbage collection - -A daemon may remove its endpoint after reporting zero sessions and completing shutdown. A later -collector may remove stale endpoint metadata only after proving the exact process incarnation is -gone. Endpoint absence by itself is not deletion authority for session records. - -### Performance model - -```text -full daemon processes = live incompatible/independent daemon generations -session state = active sessions -``` - -Compatible app builds reuse one daemon, as recovery-v2 reused PID `26391`. Accumulation occurs -only when long-lived sessions span protocol/capability transitions or an old daemon remains -unverifiable. Idle daemons must be event-driven and should consume no recurring polling CPU, but -each generation still retains a process, runtime, socket, credentials, logs, and baseline memory. - -### Advantages - -- Smallest change from the current architecture. -- Directly fixes same-protocol endpoint collision and adapter-scan ownership inference. -- Reuses existing daemon, adapter, protocol, and process-incarnation machinery. -- Allows independent retirement of drained generations. -- Limits migration risk because PTY ownership stays where it is today. -- A failure in one old daemon affects only sessions owned by that daemon. -- Compatible builds normally keep one daemon, avoiding per-build accumulation. - -### Disadvantages - -- Long-lived sessions can retain multiple full daemon generations. -- Old daemon code, including security-sensitive code, may remain alive indefinitely. -- The old bundle may be removed while its daemon still runs; binaries and required resources must - live in a stable versioned support directory. -- Lifecycle, endpoint garbage collection, and security retirement policy become explicit product - responsibilities. -- A daemon crash still destroys every PTY master held by that daemon. -- A wedged daemon may retain resources while remaining `unverifiable`. -- Existing records without an owner incarnation require a conservative legacy migration path. - -### Migration - -- Add owner fields as optional persisted and wire fields first. -- For legacy records, retain adapter discovery only as a non-destructive compatibility path. -- Backfill an owner only when exactly one adapter positively proves ownership and returns its - incarnation. Never backfill from a negative scan or protocol number. -- New hosts publish optional owner metadata; old clients ignore it. If behavior depends on a new - terminal-stream operation rather than an optional field, capability-negotiate it according to - [remote wire compatibility](./docs/reference/remote-wire-compatibility.md). -- Remove legacy discovery only after the oldest supported persisted state is migrated or retired. - -## Approach B: stable PTY host plus replaceable control daemon - -### Design - -Split the current daemon into two lifetime domains: - -```text -Orca app - -> replaceable control daemon - -> stable PTY host - -> PTY masters and child processes -``` - -The PTY host contains only the minimum long-lived data-plane behavior: - -- spawn and attach; -- read and write; -- resize; -- signal and terminate; -- bounded replay/history handoff; -- process inspection required for terminal correctness; -- session-incarnation and exit reporting. - -Agent orchestration, provider policy, Git/worktree behavior, update logic, application routing, -and fast-moving product features remain in the replaceable control daemon. - -All local sessions point to the stable host incarnation. App and control-daemon updates reconnect -to the same host and do not change PTY ownership. - -### Lifecycle - -#### Spawn - -1. The control daemon requests a spawn from the stable host. -2. The host creates the PTY and returns a session incarnation. -3. Persist the stable-host and session-incarnation identity. -4. The control daemon may restart without changing either identity. - -#### App or control-daemon update - -Disconnect the control plane, replace it, and reconnect to the existing host. The host's API must -remain backward compatible across the supported mixed-version window. - -#### PTY-host update - -The stable host cannot be casually replaced while it owns sessions. Available policies are: - -1. defer replacement until it drains; -2. run old and new stable-host generations side by side temporarily; -3. require explicit recovery/termination for a mandatory security update. - -Policy 2 temporarily reintroduces Approach A at the PTY-host layer. - -#### Host unavailable - -Transport loss is `unverifiable`. If the execution host proves that the exact PTY-host process -incarnation exited, all raw PTYs it owned are gone. Detached descendants may still exist and must -not be described as exited without their own evidence. - -### Performance model - -```text -full PTY-owning host processes = 1 in steady state -control daemons = 1 current process -session state = active sessions -``` - -This hard-bounds daemon-generation accumulation during ordinary app/control-daemon updates. -Session memory, PTY handles, replay buffers, and child processes still grow with active sessions; -that cost is inherent. The split adds one IPC hop between the control daemon and PTY host and may -duplicate some connection, serialization, and buffering work. - -### Advantages - -- One PTY-owning process in steady state. -- App and control-daemon replacement cannot sever raw PTYs. -- Separates stable terminal lifetime from rapidly changing product logic. -- Removes most reasons to retain old full-feature daemon generations. -- Gives one place to enforce exact session-incarnation and exit authority. -- Makes control-daemon crash recovery materially stronger than Approach A. - -### Disadvantages - -- Larger initial refactor and a new internal protocol boundary. -- The stable host is a single failure domain: if it crashes, every PTY master it owns is lost. -- The host API must remain deliberately small and backward compatible; feature leakage recreates - the current versioning problem. -- Updating the stable host while sessions are live remains difficult. -- An additional IPC hop can affect input latency, output throughput, backpressure, replay, and - resize ordering. -- Cross-platform extraction is substantial, especially Windows ConPTY, WSL, macOS process - attribution, and SSH relay ownership. -- Debugging spans app, control daemon, host, and child-process logs. -- Security-sensitive host defects can remain resident until all sessions drain. - -### Migration - -- First put the current daemon's PTY operations behind an internal host interface without changing - process placement. -- Add a capability-negotiated external host transport and run it in shadow/verification mode. -- Move new local sessions to the stable host while existing daemon-owned sessions keep their old - owners. -- During transition, route by `ownerKind` and exact owner incarnation; never guess based on logical - PTY ID. -- Retire daemon-owned sessions naturally. Do not attempt cross-process PTY transfer as part of the - first migration. -- Keep remote execution ownership on the remote host. A desktop client must never address a remote - host socket directly. - -## Side-by-side comparison - -| Criterion | Approach A: incarnation-addressed daemons | Approach B: stable PTY host | -|---|---|---| -| Fundamental identity ambiguity | Eliminated | Eliminated | -| Same-protocol owner collision | Eliminated with unique endpoints | Eliminated because ownership stays in the stable host | -| App restart survival | Yes, if owning daemon remains live | Yes | -| Control-daemon crash survival | No; it owns the PTYs | Yes; PTY host owns them | -| PTY-owner crash survival | No | No; stable host is the PTY owner | -| Steady full-process count | One per live daemon generation | One host plus one current control daemon | -| Generation accumulation | Possible across compatibility transitions | Avoided for control-daemon updates; possible only when the host itself must version | -| Additional steady IPC hop | No | Yes | -| Initial implementation scope | Moderate | Large | -| Migration risk | Lower | Higher | -| Old security-sensitive code lifetime | Per retained daemon generation | Stable host may remain old until drain | -| Failure blast radius | Sessions in one daemon generation | All sessions in the stable host | -| Protocol evolution | New daemon generation | Stable host API must remain backward compatible | -| Best fit | Correctness retrofit with minimal disruption | Hard process bound and control-plane restart survival | - -## Neither approach provides PTY-owner crash survival - -This distinction must remain explicit: - -```text -app/control process dies -> Approach B preserves PTYs -PTY-owning process dies -> both approaches lose its PTY masters -``` - -Surviving the PTY-owning process itself requires a third architecture: one independently -supervised host per session, or an OS facility that can transfer/recover PTY ownership. A -per-session host reduces the failure blast radius to one terminal but adds a process and endpoint -per active session. Cross-platform transfer of live Unix PTY and Windows ConPTY state is not a -simple alternative and is outside this comparison. - -## Cross-platform and execution-host requirements - -### Local Unix - -- Derive socket paths with platform path utilities. -- Keep paths below Unix-domain socket limits. -- Restrict directory and token permissions to the user. -- Prove process incarnation with PID plus platform start identity, never PID alone. - -### Windows - -- Derive named-pipe names from validated incarnation IDs. -- Preserve `windowsHide` and the shared child-process boundary. -- Treat ConPTY ownership and handle lifetime as host-local implementation details. -- Do not assume Unix descriptor-transfer mechanisms exist. - -### WSL - -- Keep native Windows, each WSL distribution, and any relay as distinct execution hosts. -- Never route a WSL PTY through a native endpoint merely because the client can see both. -- Preserve existing `wsl.exe --exec` and captured-login-shell requirements. - -### Direct SSH and paired runtimes - -- Store and resolve owner identities on the execution host. -- A client or relay disconnect always produces `unverifiable`. -- The host publishes optional opaque ownership evidence; it does not expose a connectable local - filesystem path to the client. -- Mixed client/host versions are normal. New optional fields must remain optional; new operations - require capability negotiation. - -### Worktrees and folder workspaces - -PTY ownership must not depend on Git identity. Worktree or folder workspace IDs are grouping and -display metadata, not execution authority. Deleting or failing to discover a workspace cannot -change an owner verdict without a separate, host-authoritative stop operation. - -## Security and operational policy - -Both approaches require: - -- authenticated endpoints with per-owner credentials; -- endpoint derivation inside a protected runtime directory; -- process-incarnation verification before trusting metadata or deleting stale endpoints; -- bounded logs and replay buffers; -- explicit maximum supported age for security-critical owners; -- observable owner/session counts and retirement reasons; -- no automatic destructive fallback when metadata is corrupt or incomplete; -- a user-visible recovery choice when a security update cannot preserve an old owner. - -## Acceptance criteria shared by both approaches - -1. Two same-protocol owners can each host a session without either claiming the other's ID. -2. App restart reconstructs exact routes without broad `listSessions` ownership inference. -3. Removing an adapter, endpoint, or transport returns `unverifiable` and preserves all topology. -4. An exact-owner exit closes only the matching session incarnation. -5. A delayed exit for an old incarnation cannot close a replacement using the same logical PTY - ID. -6. Ten compatible app updates do not create ten PTY-owning processes. -7. A protocol transition retains the old owner only while it has sessions and retires it after - drain. -8. SSH disconnect and relay loss remain `unverifiable`. -9. Folder workspaces pass the same lifecycle tests as Git worktrees. -10. Current client/old host and old client/current host pass the relevant cross-version suites. -11. Windows ConPTY, WSL, macOS, Linux, and remote hosts pass owner-isolation tests. -12. Provider continuation preserves the logical tab but creates a new session incarnation. - -## Approach-specific performance gates - -### Approach A - -- Measure baseline memory and idle CPU for one current and multiple drained-but-live daemons. -- Require no periodic polling proportional to daemon generations or persisted sessions. -- Verify compatible app updates reuse one daemon. -- Bound endpoint discovery and startup work by referenced owners, not every historical protocol. - -### Approach B - -- Measure input latency and output throughput across the added IPC hop. -- Verify backpressure, replay, resize, pause/resume, and exit ordering under dense output. -- Measure stable-host memory per active session and control-daemon restart time. -- Prove ordinary control-daemon updates never create another PTY host. - -No architecture should be selected without production-shaped measurements for hundreds of -sessions and at least two simultaneously supported host/client versions. - -## Decision framework - -Choose Approach A when the priority is the smallest safe correction to ownership identity and the -product can tolerate a small number of draining daemon generations. It fixes the incident class -without moving PTY ownership across a new process boundary. - -Choose Approach B when the product requires a hard steady-state bound on PTY-owning daemons and -raw terminals must survive replacement or crashes of the feature/control daemon. Accept the -larger refactor, extra IPC hop, and stable host's larger failure blast radius. - -If the requirement is that a raw terminal survive failure of the process that owns its PTY master, -neither approach is sufficient. Decide that requirement separately before treating Approach B as -complete crash continuity. - -## Implementation touchpoints - -The eventual implementation is expected to affect at least: - -- `src/main/daemon/daemon-init.ts` and daemon endpoint path construction; -- `src/main/daemon/daemon-pty-router.ts`; -- `src/main/daemon/daemon-session-owner-resolution.ts`; -- `src/main/daemon/daemon-pty-adapter.ts`; -- `src/main/ipc/pty.ts`; -- persisted terminal and pane ownership records; -- local/SSH/paired-runtime terminal publications; -- daemon/relay launch and retirement telemetry; -- cross-version terminal-wire and restart recovery tests. - -Before implementation, the selected approach needs a concrete schema, endpoint authentication -contract, migration state machine, rollback plan, and measured resource budget. diff --git a/TERMINAL_SESSION_DISAPPEARANCE_HANDOFF.md b/TERMINAL_SESSION_DISAPPEARANCE_HANDOFF.md deleted file mode 100644 index d7eb97ad5f9..00000000000 --- a/TERMINAL_SESSION_DISAPPEARANCE_HANDOFF.md +++ /dev/null @@ -1,242 +0,0 @@ -# Terminal session disappearance: engineering handoff - -## Start here - -This PR hands off the investigation and immediate safety fixes for the 2026-08-23 terminal-session -disappearance incident. It also records the unresolved architectural question: how a persisted PTY -should identify the exact process that owns it across Orca restarts and updates. - -The short version is: - -> Orca sometimes treated “I cannot find the daemon that owns this terminal” as “the terminal -> exited.” It then deleted the terminal's saved UI state. The current branch prevents that data -> loss, but it does not yet replace the underlying daemon-identity architecture. - -## Background - -During a failed application update, Orca restarted several times. Protocol-v36 daemon PID `6835` -became unavailable for a reason the retained evidence cannot identify, and PID `26391` became the -new v36 daemon. - -Starting `26391` was reasonable. A new compatible daemon is needed for new terminals when the old -endpoint cannot be used. The problem was what happened to persisted terminals whose exact owner -could no longer be reached. - -Orca's daemon endpoint is primarily named by protocol version, for example `daemon-v36.sock`. -Protocol 36 is like a language both processes speak; it is not the identity or address of one -specific daemon process. Once a different v36 daemon occupied the endpoint, the app could not -reliably reconstruct which exact daemon incarnation owned every historical PTY. - -## User impact - -The stable before/after profile comparison found: - -| Measure | Result | -|---|---:| -| Removed terminal rows | 158 | -| Affected worktrees | 61 | -| Represented pane PTYs | 191 | -| Primary PTYs with explicit exit/kill evidence | 20 | -| Primary PTYs with no authoritative exit evidence | 138 | -| Unverifiable primary PTYs with surviving history | 138 | - -At least 41 detached agent processes were still alive in the post-incident process observation. -That observation does not prove every affected process survived, but it proves the incident was -not simply “all terminals exited.” Much of the damage was deletion of persisted topology and UI -records. - -## What actually failed - -There were two destructive software paths and one separate physical-kill path. - -### 1. Main converted missing routing authority into `false` - -On terminal visibility resume, the renderer asked whether a saved PTY still existed. The old path -used boolean-oriented `hasPty()` behavior. When no currently registered daemon adapter recognized -the PTY, the router returned `false`. - -That answer only established: - -```text -the current adapter registry cannot locate the session -``` - -The renderer interpreted it as: - -```text -the owning host proved that the session exited -``` - -It then ran the PTY-exit close path and persisted the terminal row and unified tab as deleted. - -### 2. Renderer cleared a binding after an owner-unverified reattach - -The first guarded build fixed tri-state liveness in main, but live recovery exposed another path. -Main correctly reported `terminal_pane_owner_unverified`; the renderer then received no reattach -result, cleared the saved binding, and could start a replacement shell. The displayed toast said -the old session was left untouched even though the binding was not preserved. - -The second branch fix preserves the old binding and refuses to fresh-spawn solely because owner -authority is unavailable. - -### 3. Missing-worktree cleanup killed a separate subset - -A burst of 15 graceful daemon kills strongly matches a successful-but-partial worktree scan being -used as destructive authority. Those are real terminal stops and need an independent fix. They do -not account for the bulk of the 158 removed rows. - -## Why PID `6835` is not the root cause of the bulk loss - -PID `6835` becoming unavailable could strand raw PTYs that it actually owned. It cannot explain -the entire incident set: - -| Frozen primary-PTY verdict | Latest event from `6835` | Latest event from another daemon PID | -|---|---:|---:| -| `unverifiable` | 5 | 133 | -| `exited` | 15 | 5 | - -Historical event provenance does not prove current liveness. It does show that loss of one daemon -cannot be used as exit evidence for the 138-session unverifiable set. - -Recovery-v2 supplied a useful control case: a newer app retained the still-live hourly daemon PID -`26391`, despite its older build metadata, and preserved all 131 complete affected topologies that -remained in tracked worktrees. Same-protocol app/daemon version skew is therefore supported and is -not inherently destructive. - -## Confirmed versus unknown - -### Confirmed - -- The intended ad hoc update did not install during the incident. -- The installed and intended bundles both used daemon protocol 36 and state schema 1. -- PID `6835` stopped being reachable and PID `26391` became the current v36 daemon. -- The boolean liveness path could convert missing adapter registration into an exit decision. -- PTY-exit handling could delete both persisted terminal models without killing the underlying - descendant process. -- The renderer had a second owner-unverified binding-clearing defect. -- Only 20 primary PTYs have explicit latest exit/kill evidence in the frozen catalog. -- Recovery-v2 preserved topology and successfully continued one saved Codex provider session in - place under a new raw session incarnation. - -### Unknown - -- What stopped PID `6835`. -- Whether every one of the 158 deletions traversed the same renderer close callback. -- Which historical raw PTYs or descendants remained live at each later observation. -- Whether a missing route meant the owner was dead, unreachable, omitted from registration, or - hidden behind same-protocol endpoint replacement for each individual session. - -Unknown must remain `unverifiable`; it must not be rewritten as `exited`. - -## Immediate fixes included on this branch - -### `25ae8e8edf`: preserve unverifiable terminal sessions - -- Waits for local PTY-provider startup before answering liveness. -- Prefers tri-state `probePtyLiveness()` over boolean `hasPty()`. -- Returns `null` when the provider or route cannot authoritatively answer. -- Allows destructive reconciliation only on literal authoritative `false`. - -### `bd09097708`: preserve owner-unverified terminal bindings - -- Recognizes `terminal_pane_owner_unverified` during reattach. -- Preserves the saved pane/tab PTY binding. -- Prevents the missing reattach result from being treated as a stale/dead session. -- Avoids starting a replacement shell over an uncertain old owner. - -These commits are immediate data-loss guards. They are not the permanent daemon-identity design. - -## Fundamental problem - -The current architecture partially conflates three different concepts: - -```text -protocol compatibility: “can we speak?” -endpoint slot: “where is the current v36 daemon?” -owner identity: “which exact process owns this PTY?” -``` - -A persisted session needs the third answer. Asking every currently registered compatible adapter -whether it recognizes an ID is not a durable ownership model. - -The permanent invariant should be: - -> Only the matching execution host, owner incarnation, and session incarnation may establish that -> a raw PTY is live or exited. Missing routing/contact is `unverifiable`. - -## Rough permanent solutions - -The architecture document compares two approaches in detail. - -### Approach A: incarnation-addressed daemons - -Give every daemon process a UUID and unique endpoint. Persist that daemon incarnation with every -session. New sessions use a current compatible daemon; existing sessions always route to their -recorded owner. - -```text -logical PTY -> session incarnation -> daemon incarnation -> unique endpoint -``` - -This is the smaller retrofit. It eliminates protocol-slot identity collisions and adapter-scan -ownership inference, but long-lived sessions may retain multiple daemon generations. If the -owning daemon crashes, its PTY masters are still lost. - -### Approach B: stable PTY host plus replaceable control daemon - -Move PTY ownership into one deliberately small, long-lived host. Keep fast-moving Orca behavior in -a replaceable control daemon. - -```text -app -> replaceable control daemon -> stable PTY host -> PTYs -``` - -This bounds PTY-owning daemon accumulation and lets raw terminals survive app/control-daemon -replacement. It is a larger cross-platform refactor, adds an IPC hop, and makes the stable host a -single failure domain. If the stable host itself crashes, its PTY masters are still lost. - -### Decision that remains - -- Choose Approach A for the smallest safe ownership correction. -- Choose Approach B if a hard steady-state process bound and control-daemon crash survival justify - the larger refactor. -- If PTYs must survive failure of the process that owns the PTY master, neither approach is enough; - that requires per-session hosts or another independently surviving owner. - -## Suggested next steps for the new owner - -1. Review the immediate fixes independently from the permanent architecture decision. -2. Confirm whether product requirements include raw PTY survival across control-daemon failure, - PTY-owner failure, or only app updates. -3. Measure the existing daemon's idle memory/CPU and the expected number of simultaneous protocol - generations. -4. Select Approach A or B using the architecture document's comparison and performance gates. -5. Define the persisted owner schema, endpoint authentication, migration state machine, and - mixed-version behavior before implementation. -6. Keep the legacy discovery path non-destructive during migration. -7. Fix missing-worktree teardown independently; do not couple it to daemon identity. - -## Reading order and PR artifacts - -1. [`TERMINAL_SESSION_DISAPPEARANCE_HANDOFF.md`](TERMINAL_SESSION_DISAPPEARANCE_HANDOFF.md) — this - plain-language entry point. -2. [`PTY_DAEMON_IDENTITY_ARCHITECTURE_DESIGN.md`](PTY_DAEMON_IDENTITY_ARCHITECTURE_DESIGN.md) — - permanent approaches, tradeoffs, migration, and acceptance criteria. -3. [`TERMINAL_SESSION_DISAPPEARANCE_INCIDENT_2026-08-23.md`](TERMINAL_SESSION_DISAPPEARANCE_INCIDENT_2026-08-23.md) - — full incident timeline, causal analysis, and immediate fixes. -4. [`RECOVERY_CATALOG.md`](recovery/2026-08-23-session-disappearance/RECOVERY_CATALOG.md) — frozen - per-session classifications and limitations. -5. [`POST_CUTOVER_RECOVERY_STATUS.md`](recovery/2026-08-23-session-disappearance/cutover-20260823T201851Z/POST_CUTOVER_RECOVERY_STATUS.md) - — first guarded cutover, live accounting, and discovery of the renderer-side binding defect. -6. [`RECOVERY_V2_CUTOVER_STATUS.md`](recovery/2026-08-23-session-disappearance/cutover-v2-20260823T140110MST/RECOVERY_V2_CUTOVER_STATUS.md) - — controlled upgrade and provider-continuation validation. - -The PR intentionally excludes machine-specific app bundles, DMGs, live profiles, terminal-history -payloads, and large raw logs. The included documents retain the relevant counts, hashes, -limitations, and evidence locations without making the PR unsafe or impractically large. - -## Scope boundary - -Preventing application activation during an update window is intentionally out of scope for this -handoff. It is recorded in the incident timeline but should be handled separately from PTY owner -identity and liveness authority. diff --git a/TERMINAL_SESSION_DISAPPEARANCE_INCIDENT_2026-08-23.md b/TERMINAL_SESSION_DISAPPEARANCE_INCIDENT_2026-08-23.md deleted file mode 100644 index e0b54d8a865..00000000000 --- a/TERMINAL_SESSION_DISAPPEARANCE_INCIDENT_2026-08-23.md +++ /dev/null @@ -1,783 +0,0 @@ -# Terminal session disappearance after failed ad hoc update — 2026-08-23 - -## Status - -- Investigation status: root cause identified with high confidence; several supporting facts are proven directly by persisted state and logs. -- Investigation mode: read-only. No Orca restart, update retry, pane reopen, session kill, state restore, or profile edit was performed as part of this investigation. -- Incident machine: macOS/Darwin 26.5.1, arm64. -- Shell reported by the affected pane: `/opt/homebrew/bin/bash`. -- Local timezone used below: America/Phoenix (`MST`, UTC-07:00). -- Installed application during the incident: `1.4.189-hourly.202608231007`, commit `c3a1694b1d30`. -- Intended update: `1.4.189-adhoc.20260823180729`, commit `91a8c0977a39`. -- Both bundles use state schema 1 and daemon protocol 36. -- Source worktree HEAD during investigation: `e3327c2f3169`. - -## Executive conclusion - -This was a cascade of separate failures: - -1. The ad hoc update did not install. During both ShipIt installation windows, a separate `/usr/bin/open /Applications/Orca.app` activation relaunched the still-installed hourly bundle. ShipIt then aborted with `App Still Running Error`. -2. Protocol-v36 daemon PID `6835` became unavailable for an unknown reason, and PID `26391` legitimately became the new current v36 daemon. That transition could strand PTYs actually owned by `6835`, but it cannot explain the bulk loss: only 5 of the 138 `unverifiable` primary PTYs have their latest frozen daemon event from `6835`; the other 133 point to other daemon PIDs. -3. The repeated app/daemon transitions exposed persisted PTY bindings whose exact owning daemon route was no longer reachable or registered. On terminal-pane visibility resume, the renderer called the boolean-oriented `pty.hasPty` path. For an old unmapped PTY, `DaemonPtyRouter.hasPty()` returned `false` after the owning adapter was omitted. That answer meant only that the current set of registered adapters did not contain the PTY, but the renderer treated it as authoritative **exited**. -4. The renderer's exit path cleared the pane binding and called `closeTerminalTab(..., { reason: 'pty-exit' })`, deleting the persisted terminal row and its unified tab without necessarily stopping the underlying process. -5. Worktree activation reconciled the two tab models. This made remaining tab shells disappear as affected worktrees were clicked. -6. Separately, a burst of 15 graceful daemon kills strongly matches false "missing worktree" cleanup. Those kills physically stopped a subset of sessions and worsened the incident, but they do not explain the bulk of the 158 removed terminal rows. - -The user-visible message—"Orca couldn't confirm whether this terminal's previous session is still running, so it left the session untouched"—is the correct **unverifiable** behavior from a different reattach path. The destructive visibility-resume path incorrectly collapsed the same class of uncertainty into **exited**. - -This was not a successful ad hoc installation followed by rollback, and it is not evidence that every agent process exited. The app lost a large amount of persisted terminal topology/UI state; many process and history artifacts survived. - -The concise causal distinction is: - -```text -daemon unavailable -> replacement daemon is expected -session absent from registered adapters -> route/authority is missing -matching owner reports exit -> session is exited -``` - -The incident incorrectly treated the second condition as the third. - -## Confidence labels - -- **Proven**: directly supported by immutable/stable persisted-state comparison, ShipIt logs, bundle metadata, or explicit daemon events. -- **High confidence**: the code path exactly produces the observed state transition and the timing/shape match, but per-mutation renderer logging is missing. -- **Unknown**: the retained evidence cannot attribute a caller or exact initiating event. - -## Detailed timeline - -Times are local MST unless a UTC timestamp is explicitly shown. - -| Time | Event | Evidence/status | -| --- | --- | --- | -| 11:09:17 | Pre-incident state snapshot later stored as `orca-data.json.bak.1`. | Proven by file mtime and state contents. | -| 11:18:21 | Main PID `15010` records `updater_update_downloaded` for `1.4.189-adhoc.20260823180729`. | Proven in `main.trace.ndjson`. | -| 11:18:24 | PID `15010` invokes native quit/install. Quit breadcrumb records `daemonTeardown: "disconnect"`. ShipIt PID `78338` starts the install. | Proven. The updater shutdown intentionally disconnected from daemons rather than stopping their PTYs. | -| 11:18:51 | `/usr/bin/open` PID `79643` activates `/Applications/Orca.app`; launchd starts hourly Orca PID `79644`. | Proven from macOS unified logs collected during investigation. Exact caller of `open` is unknown. | -| 11:24:23 | ShipIt PID `78338` aborts because one target-app instance is running. | Proven: `SQRLInstallerErrorDomain Code=-9`, `App Still Running Error`. | -| 11:25:50 | Relaunched hourly PID `79644` records the ad hoc update downloaded again. | Proven in main trace. | -| 11:25:57 | PID `79644` invokes native quit/install; quit again records `daemonTeardown: "disconnect"`. ShipIt PID `72258` starts the second install. | Proven. | -| 11:26:22–11:26:23 | `/usr/bin/open` PID `77840` activates Orca; hourly PID `77841` starts. | Proven from macOS unified logs. Caller unknown. | -| 11:29:01 | PID `77841` is SIGKILLed. | Proven from macOS process logs; source of the signal is not retained. | -| 11:29:08–11:29:09 | `/usr/bin/open` PID `91473` activates Orca; hourly PID `91476` starts. | Proven from macOS unified logs. Caller unknown. | -| 11:30:11 | One `add-starting-agent-from-tab-e2e` PTY receives `session-killed`, `immediate:true`. | Proven daemon event; separate from the later graceful burst. | -| 11:31:31 | One `improve-cmd-j-ranking-with-recency` PTY receives `session-killed`, `immediate:true`. | Proven daemon event; separate from the graceful burst. | -| 11:31:54–11:32:45 | Fifteen PTYs across nine worktrees receive `session-killed`, `immediate:false`. | Proven events. Attribution to false missing-worktree reconciliation is high confidence. | -| 11:33:05 | New protocol-v36 daemon PID `26391` starts and becomes ready on `daemon-v36.sock`, replacing the prior protocol-v36 generation PID `6835`. | Proven. Why PID `6835` became unavailable is not recorded in `daemon.log`. | -| 11:36:16 | ShipIt PID `72258` aborts because one target-app instance is running. | Proven: second `App Still Running Error`. | -| 11:38–11:39 | Screenshots capture the app still on hourly and the missing-session UI. | User-provided evidence. | -| 12:10:11 | Post-incident state snapshot later stored as `orca-data.json.bak.0`. It reproduces the original state-loss measurement. | Proven by file mtime, hash, and state contents. | - -The exact wall-clock moment of each renderer row deletion is not logged. The state snapshots bound the mutation window, while the restart, visibility, daemon, and kill events locate the likely trigger period. - -## Finding 1: why the update failed - -### Proven facts - -The staged ad hoc bundle is valid and remains in ShipIt's cache: - -```text -/Users/jinjingliang/Library/Caches/com.stablyai.orca.ShipIt/update.2pCRJKe/Orca.app -version: 1.4.189-adhoc.20260823180729 -commit: 91a8c0977a39 -daemonProtocolVersion: 36 -``` - -The installed bundle was never replaced: - -```text -/Applications/Orca.app -version: 1.4.189-hourly.202608231007 -commit: c3a1694b1d30 -daemonProtocolVersion: 36 -``` - -ShipIt recorded two failed attempts: - -```text -2026-08-23 11:18:25.850 Beginning installation -2026-08-23 11:24:23.461 Aborting update attempt because there are 1 running instances of the target app -2026-08-23 11:24:23.576 Installation cancelled: ... "App Still Running Error" - -2026-08-23 11:26:00.620 Beginning installation -2026-08-23 11:36:16.660 Aborting update attempt because there are 1 running instances of the target app -2026-08-23 11:36:16.821 Installation cancelled: ... "App Still Running Error" -``` - -The three intervening activations were LaunchServices `/usr/bin/open` processes, not ShipIt's post-install success relaunch. Each activation occurred before the corresponding installation could complete, and each launched the old installed hourly bundle. - -Both updater quits recorded: - -```json -{"breadcrumb.name":"updater_will_quit_cleanup_started","breadcrumb.data":{"daemonTeardown":"disconnect"}} -``` - -Therefore the updater's normal quit path did not intentionally kill daemon-backed PTYs. - -### Unknown caller of `/usr/bin/open` - -The short-lived `open` process parents/callers were not retained in the available unified logs. Plausible sources include a manual Dock/Finder activation, a script, or an Orca CLI `open` operation, but none is proven. - -`orca status` is ruled out as the source in current/incident-equivalent source: [`status`](src/cli/handlers/core.ts#L136) calls only `getCliStatus()`. [`openOrca()`](src/cli/runtime/client.ts#L208) is the path that calls [`launchOrcaApp()`](src/cli/runtime/launch.ts#L23), which can spawn `open` on macOS. This does not identify which caller ran during the incident. - -### Ruled out - -- Bad or incompatible ad hoc artifact: both bundles advertise the same state schema and daemon protocol. -- Successful install followed by rollback: installed bundle metadata never changed to the ad hoc build. -- ShipIt success relaunch: ShipIt logged cancellation, not installation completion, for both attempts. -- Updater-requested PTY teardown: both quit breadcrumbs say `disconnect`. - -## Finding 2: daemon topology became unverifiable - -Orca retained multiple daemon generations so older PTYs could remain attachable: - -| Protocol | PID | Start time (UTC) | Notes | -| --- | ---: | --- | --- | -| 33 | `98955` | 2026-08-14T09:33:45.910Z | Legacy daemon; owns several incident-adjacent sessions. | -| 34 | `67978` | 2026-08-19T01:50:14.609Z | Legacy daemon. | -| 35 | `75742` | 2026-08-18T19:00:15.281Z | Legacy daemon. | -| 36 | `6835` | 2026-08-20T18:13:32.036Z | Current generation before incident replacement; emitted several kills. | -| 36 | `26391` | 2026-08-23T18:33:05.635Z | New current generation after restart/recovery. | - -Additional persisted bindings referenced still older protocols such as v23/v26. Subsequent worktree-removal traces contain `ECONNREFUSED` for `daemon-v23.sock`, confirming that some saved ownership routes pointed at unavailable daemons. - -Legacy adapter creation probes each old socket. If the socket probe fails, that adapter is omitted in [`createLegacyDaemonAdapters()`](src/main/daemon/daemon-init.ts#L1379). Loss of contact is therefore represented partly as absence from the router's adapter collection. - -Per the execution-boundary contract, a missing/unreachable adapter makes process state **unverifiable**. It is not proof that the process **exited**. - -### What PID `6835` can and cannot explain - -PID `6835` becoming unavailable is an incident trigger, not a sufficient cause for the 158-row loss. Starting PID `26391` as the new v36 daemon was correct once the old v36 endpoint could not be used. The retained evidence cannot identify who or what stopped `6835`. - -The frozen recovery catalog makes the scope mismatch explicit: - -| Primary PTY classification | Latest event from PID `6835` | Latest event from another daemon PID | Total | -|---|---:|---:|---:| -| `unverifiable` | 5 | 133 | 138 | -| `exited` | 15 | 5 | 20 | - -Historical event provenance does not prove present liveness. It does prove that the catalog cannot support a single-daemon explanation for the bulk loss. The 20 proven exits are backed by explicit exit/kill events, including sessions from the separate kill burst; the remaining 138 rows required an independent UI/persistence deletion mechanism. - -The later recovery-v2 cutover supplies a control case. A newer app retained the still-live hourly daemon PID `26391`, despite its older bundle metadata, and preserved all 131 complete affected topologies in tracked worktrees. Same-protocol app/daemon version skew is therefore not itself a reason to replace a live daemon or delete its sessions. - -"Not found in registered daemon adapters" is an adapter-lookup result. It can mean that the exact owner route was never reconstructed, its socket is unreachable, its adapter was omitted, or a same-protocol incarnation now occupies the fixed endpoint. It does not identify which condition occurred and does not prove process death. - -## Finding 3: the destructive liveness misclassification - -### Code path - -On a hidden-to-visible terminal-pane transition, [`use-terminal-pane-lifecycle.ts`](src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.ts#L2044) calls: - -```text -reconcileMissingSessions({ - bindings: panePtyBindingsRef.current.values(), - hasPty: window.api.pty.hasPty -}) -``` - -The IPC handler in [`src/main/ipc/pty.ts`](src/main/ipc/pty.ts#L7856) selects a provider and calls synchronous `provider.hasPty(id)`. The handler can return `null` when no provider exists or a call throws, but it does not use the daemon router's asynchronous tri-state `probePtyLiveness()`. - -For a daemon-backed provider, [`DaemonPtyRouter.hasPty()`](src/main/daemon/daemon-pty-router.ts#L80) does this: - -```text -if a session-specific adapter route exists: - return routed.hasPty(id) -otherwise: - return current.hasPty(id) || any registered legacy adapter.hasPty(id) -``` - -If the actual owning daemon adapter is unreachable and omitted, the PTY is absent from every remaining adapter. The method returns `false`. That is a fabricated authoritative answer: it proves only "not present in the currently registered adapters," not "the process exited." - -The same router already has `probePtyLiveness(id): Promise`, which is the correct tri-state operation. The visibility-resume path bypasses it. - -The renderer is correctly conservative only after it receives a tri-state value: [`shouldReconcileMissingSession()`](src/renderer/src/components/terminal-pane/terminal-dead-session-reconcile.ts#L66) acts only on literal `false`. Because main fabricated `false`, [`reconcileIfSessionMissing()`](src/renderer/src/components/terminal-pane/pty-connection.ts#L9497) calls `onExit(currentPtyId)`. - -The exit cascade is: - -1. Clear pane/tab PTY bindings in [`pty-connection.ts`](src/renderer/src/components/terminal-pane/pty-connection.ts#L2713). -2. `Terminal.handlePtyExit()` calls `closeTerminalTab(tabId, { reason: 'pty-exit' })` in [`Terminal.tsx`](src/renderer/src/components/Terminal.tsx#L1801). -3. `closeTab()` removes the terminal row from every `tabsByWorktree` array in [`terminals.ts`](src/renderer/src/store/slices/terminals.ts#L1618). -4. The same operation removes a matching unified terminal tab in [`terminals.ts`](src/renderer/src/store/slices/terminals.ts#L1831). -5. State persistence writes the reduced models to `orca-data.json`. - -Crucially, `closeTab()` treats only reasons `user` and `cleanup` as physical session retirement. The `pty-exit` reason removes UI/persistence state without sending a provider kill. This exactly explains why many terminal records disappeared with no daemon `session-exited` or `session-killed` event and why detached agent processes could remain live. - -### Confidence - -This is the **high-confidence root cause of the bulk persisted row loss**. - -It is not possible to prove that every one of the 158 rows traversed this exact function because renderer mutation logging does not record each `closeTerminalTab` call, liveness verdict, or reason with tab/PTY identity. However: - -- The code path exactly produces both observed deletions: runtime terminal row plus unified terminal tab. -- It runs on the visibility transitions involved in clicking/reopening affected worktrees. -- The necessary daemon ownership uncertainty existed. -- 164 of 191 incident-referenced PTYs checked against daemon history had no corresponding exit/kill event. -- The state loss is much larger than the daemon kill count. - -No alternative inspected path matches the state delta and missing kill events as closely. - -## Finding 4: why tabs appeared to vanish worktree-by-worktree - -Orca persists two related terminal-tab models: - -- `workspaceSession.tabsByWorktree`: legacy/runtime terminal rows with the tab ID and top-level PTY ID. -- `workspaceSession.unifiedTabs`: the visible unified tab shell referencing the terminal row by `entityId`. - -During an intermediate live-state observation, 138 of the 158 removed runtime rows still had unified tab shells. This transient state was later overwritten as the app continued persisting. It demonstrates that row loss and visible-tab cleanup occurred in stages. - -When a worktree is activated, [`setActiveWorktree()`](src/renderer/src/store/slices/worktrees/session/set-active-worktree.ts#L50) invokes [`projectWorktreeTabModelReconciliation()`](src/renderer/src/store/slices/tabs.ts#L623). Once the runtime terminal row and reconnectable binding are gone, the unified terminal is considered orphaned and is removed. This makes sessions appear to disappear as each worktree is clicked. - -By the stable 12:10 post-incident snapshot, none of the 158 removed runtime rows retained a matching unified terminal entry. - -## Finding 5: the reattach failure and toast are related but not the row-deletion mechanism alone - -The observed toast says: - -> Orca couldn't confirm whether this terminal's previous session is still running, so it left the session untouched. Reopen this pane to retry. If this persists, please file an issue. - -That wording explicitly represents an **unverifiable** verdict. It does not say or prove that the process exited. - -The failed/empty reattach branch in [`pty-connection.ts`](src/renderer/src/components/terminal-pane/pty-connection.ts#L8280) can clear saved pane/tab PTY bindings when no PTY ID is returned. This makes later recovery harder, but that branch does not by itself remove the runtime terminal row. It therefore cannot explain the full 158-row deletion without the exit/close path or later reconciliation. - -The existence of this safe toast path alongside the destructive boolean liveness path is the central semantic inconsistency: - -```text -connection/reattach uncertainty -> unverifiable -> leave session untouched -visibility-resume adapter omission -> false -> exited -> delete terminal state -``` - -## Finding 6: separate false missing-worktree teardown - -### Observed graceful kills - -The daemon recorded 15 `session-killed` events with `immediate:false` from 11:31:54 through 11:32:45: - -| Local time | Worktree | PTY suffix | Daemon PID | Client ID | -| --- | --- | --- | ---: | --- | -| 11:31:54.324 | `orca-top-level-cleanup` | `fa471831` | `98955` | `d8d97b7c-...` | -| 11:31:59.874 | `1.4.183-p0-orca-crashed-all-agent-sessions` | `8804d1a3` | `98955` | `d8d97b7c-...` | -| 11:32:06.374 | `add-more-e2e-tests` | `1f313236` | `98955` | `d8d97b7c-...` | -| 11:32:06.377 | `add-more-e2e-tests` | `79e61569` | `6835` | `985cf608-...` | -| 11:32:07.330 | `add-more-e2e-tests` | `ba88e6c5` | `6835` | `985cf608-...` | -| 11:32:13.651 | `add-starting-agent-from-tab-e2e` | `4ea40b01` | `6835` | `985cf608-...` | -| 11:32:13.652 | `add-starting-agent-from-tab-e2e` | `31c4cb12` | `6835` | `985cf608-...` | -| 11:32:13.652 | `add-starting-agent-from-tab-e2e` | `43f5e12a` | `6835` | `985cf608-...` | -| 11:32:13.652 | `add-starting-agent-from-tab-e2e` | `e7da1734` | `6835` | `985cf608-...` | -| 11:32:28.276 | `allow-create-automations-with-ai` | `e97a31de` | `6835` | `985cf608-...` | -| 11:32:28.277 | `allow-create-automations-with-ai` | `49fbac2f` | `6835` | `985cf608-...` | -| 11:32:41.658 | `analytics` | `5fb657da` | `98955` | `d8d97b7c-...` | -| 11:32:41.816 | `analytics` | `20adb901` | `98955` | `d8d97b7c-...` | -| 11:32:44.539 | `audit-default-settings` | `55fddb5f` | `6835` | `985cf608-...` | -| 11:32:45.638 | `audit-default-settings` | `048ff989` | `6835` | `985cf608-...` | - -The sequence follows Git worktree inventory ordering and crosses daemon generations while preserving the same main-client grouping. The worktree directories survived and remained registered. This strongly matches [`stopMissingWorktreeTerminals()`](src/main/runtime/missing-worktree-terminal-reconciliation.ts#L68), called through `OrcaRuntimeService.teardownMissingManagedWorktreeTerminals()`. - -That code computes `knownWorktreeIds - detectedWorktreeIds`, then calls `killAllProcessesForWorktree()` for each allegedly missing worktree. A partial or incorrect detected scan can therefore perform real graceful shutdowns against worktrees that still exist. - -### Attribution caveat - -Attribution is high confidence, not absolute. `daemon.log` records the kill, `immediate` flag, session, daemon PID, and client ID, but not the RPC name, reconciliation reason, scan generation, or callsite. No `terminal.close` spans were present during the graceful burst. The bulk pattern does not match the renderer manual-sleep path, which uses immediate kills. - -### Two separate immediate kills - -These should not be grouped with missing-worktree reconciliation: - -| Local time | Worktree | PTY suffix | `immediate` | -| --- | --- | --- | --- | -| 11:30:11.893 | `add-starting-agent-from-tab-e2e` | `8972c618` | `true` | -| 11:31:31.967 | `improve-cmd-j-ranking-with-recency` | `c9fdd621` | `true` | - -They match renderer `pty:kill`/transport-disconnect behavior but do not explain the later 15-session graceful sequence or the 158-row state loss. - -## Persisted-state impact - -### Stable comparison pair - -The original incident measurement is reproducible using: - -```text -before: /Users/jinjingliang/Library/Application Support/orca/profiles/local-default/orca-data.json.bak.1 -mtime: 2026-08-23 11:09:17 -0700 - -after: /Users/jinjingliang/Library/Application Support/orca/profiles/local-default/orca-data.json.bak.0 -mtime: 2026-08-23 12:10:11 -0700 -``` - -| Measure | Before | After | Detail | -| --- | ---: | ---: | --- | -| Runtime/legacy terminal rows | 978 | 838 | 158 old rows removed; 18 new rows added; net -140. | -| Unified terminal tabs | 596 | 447 | 166 old entries removed; 17 new entries added; net -149. | -| Unique PTY references across terminal-bearing state | 1,188 | 1,009 | 199 old PTY references lost; additions account for the different net delta. | -| Affected worktrees | — | 61 | 48 ended with zero terminal rows. | - -Every one of the 158 removed rows: - -- had a non-null PTY ID before the incident; -- had a matching unified terminal entry before the incident; -- belonged to a worktree listed in `activeWorktreeIdsOnShutdown`; -- was absent from `tabsByWorktree` in the post-incident snapshot; -- had no matching unified terminal entry left by the post-incident snapshot. - -At the first filesystem check, 58 of the 61 affected paths still existed. Later, 57 of 60 main-repo worktrees were still registered by Git. The three absent worktrees were subsequent intentional removals: - -- `create-scanning-issues` -- `sta-4062-folder-note-rollback` -- `sta-4064-phantom-folder-note` - -The disappearance cannot be explained as ordinary cleanup of 61 deleted worktrees. - -### Volatility warning - -The running app continued to mutate `orca-data.json` and rotate backups during the investigation. A later comparison against the live file produced different totals because some records were recreated and additional tabs were opened/closed. Use the two hashed backup files above for incident analysis, not the current live state. - -An intermediate observation that found 138 unified shells still present is no longer directly reproducible from the rotated files. It is retained here because it explains the staged, click-by-click symptom; the stable 12:10 snapshot captures the end state after reconciliation. - -## Surviving process and history evidence - -Read-only inspection after the incident found: - -- At least 41 detached agent processes still live with PPID 1. -- 3,721 terminal `output.log` files still present. -- 164 of 191 incident PTYs examined had no daemon `session-exited` or `session-killed` event. - -These are point-in-time observations, not a guarantee that the same processes remain live now. They prove that the incident was not uniformly "all agents exited." A large part of the loss was persisted topology/UI record deletion. - -Do not equate missing UI records, failed socket contact, or the toast with process death. The correct verdict vocabulary is **live**, **unverifiable**, or **exited**. - -## Near-match prior fix already present - -Installed commit `c3a1694b1d30` includes: - -```text -da6b9d80658168c3d94789d26a7a302295d6fce9 -fix(terminal): stop orphaning live agent terminals across host restarts and graph syncs (#15644) -``` - -That change added host-admitted membership/runtime ownership preservation and tests for the case where graph sync prunes a live CLI terminal while its agent keeps running. Its test contract explicitly says assertions are made only for PTYs recorded as connected or exited—never **unverifiable**. - -The current incident is adjacent but not fully covered: - -- older pre-existing terminals may lack the newer ownership signal; -- the visibility-resume `pty.hasPty` path still fabricates `false` when an owning daemon adapter is unavailable; -- missing-worktree cleanup can physically stop sessions before graph preservation helps; -- failed reattach can clear the binding needed for later recovery. - -The relevant liveness files are unchanged between installed commit `c3a1694b1d30` and the investigation worktree HEAD. This is not explained by inspecting only a newer source version. - -## Evidence inventory - -### Primary artifacts - -```text -Current/live state: -/Users/jinjingliang/Library/Application Support/orca/profiles/local-default/orca-data.json - -Stable pre-incident state: -/Users/jinjingliang/Library/Application Support/orca/profiles/local-default/orca-data.json.bak.1 - -Stable post-incident state: -/Users/jinjingliang/Library/Application Support/orca/profiles/local-default/orca-data.json.bak.0 - -Main trace: -/Users/jinjingliang/Library/Application Support/orca/logs/main.trace.ndjson - -Daemon logs: -/Users/jinjingliang/Library/Application Support/orca/logs/daemon.log -/Users/jinjingliang/Library/Application Support/orca/logs/daemon.log.1 -/Users/jinjingliang/Library/Application Support/orca/logs/daemon.log.2 - -ShipIt log: -/Users/jinjingliang/Library/Caches/com.stablyai.orca.ShipIt/ShipIt_stderr.log - -Installed metadata: -/Applications/Orca.app/Contents/Resources/orca-local-build.json - -Staged ad hoc metadata: -/Users/jinjingliang/Library/Caches/com.stablyai.orca.ShipIt/update.2pCRJKe/Orca.app/Contents/Resources/orca-local-build.json - -Installed source bundle: -/Applications/Orca.app/Contents/Resources/app.asar -``` - -### User screenshots - -```text -/var/folders/69/kktjrbf5569d0qvcyxjgmllm0000gn/T/TemporaryItems/NSIRD_screencaptureui_HMz6vr/Screenshot 2026-08-23 at 11.38.22 AM.png -/var/folders/69/kktjrbf5569d0qvcyxjgmllm0000gn/T/TemporaryItems/NSIRD_screencaptureui_KuOi0a/Screenshot 2026-08-23 at 11.39.19 AM.png -``` - -These are in a temporary directory and should be copied before macOS removes them if they are needed for a bug report. - -### SHA-256 values captured at 2026-08-23 12:45:36 MST - -| Artifact | SHA-256 | -| --- | --- | -| `orca-data.json.bak.1` | `34709f2dc87db86cecd13dfb7726f753c136419ce6ef0ce40cce7bc4c0df8fbe` | -| `orca-data.json.bak.0` | `383d229dc447523ed55ee836d1d6bf570edf7a372d8bcb604c6722caa4ef2de3` | -| `ShipIt_stderr.log` | `1900b6ab8acfc47f0c17ee71c1d2b64ef1e81c50e5ab64bba3fbc762c189c79e` | -| Installed `orca-local-build.json` | `eed31c47e4ea02599c5d6513a481bf2bf40c2c1e0f67aa70d5012ca205ea435a` | -| Staged `orca-local-build.json` | `89b86c5a4c79facdfcb12384df63b08c328e496454869782f3ec48c55a43de0c` | -| Screenshot 11:38:22 | `104f2cc5f0be0a34528073c8bdede235f4eaa18a691c3a048b2cb241bdf276f8` | -| Screenshot 11:39:19 | `1cccae152d01368c506fb689abba5a8d75b43784ff303fb4d26efa722b6dba2d` | - -`main.trace.ndjson`, `daemon.log`, and the live state were still changing, so hashes of those live files are not durable evidence unless copies are made first. - -## Affected worktree and PTY inventory - -All PTY IDs share the local repo/worktree prefix. The suffixes below are sufficient to correlate against `daemon.log` and terminal history directories. Counts and suffixes come from the stable `bak.1 -> bak.0` comparison. - -| Worktree | Removed rows | PTY suffixes | -| --- | ---: | --- | -| `orca-top-level-cleanup` | 1 | `fa471831` | -| `1.4.183-p0-orca-crashed-all-agent-sessions` | 1 | `8804d1a3` | -| `add-more-e2e-tests` | 1 | `ba88e6c5` | -| `add-starting-agent-from-tab-e2e` | 4 | `4ea40b01`, `8972c618`, `31c4cb12`, `43f5e12a` | -| `allow-create-automations-with-ai` | 1 | `49fbac2f` | -| `analytics` | 1 | `5fb657da` | -| `audit-default-settings` | 1 | `55fddb5f` | -| `auto-auto-pr-assignment-run-86-20260814T1645` | 2 | `b9b726f2`, `074c00d3` | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260814T1500` | 1 | `2c0c1f2c` | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260814T2300` | 1 | `72fbd9e4` | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260815T1900` | 3 | `ca76565c`, `c88a58ad`, `34e82ad0` | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T1900` | 3 | `3867ab5d`, `614960b9`, `343c97dc` | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | 24 | `fa5e2406`, `593e9a00`, `1a8a174c`, `1bc85db0`, `eb3b9fe2`, `3921d7fb`, `edf568c4`, `fac0c631`, `e6430688`, `efca7e8b`, `6bae4f90`, `d5016fa5`, `d6499bd5`, `837371b4`, `7f7126e5`, `b0fcff2e`, `6bc4cece`, `23423419`, `59e2e93c`, `4885350c`, `8298b45c`, `2727bdb1`, `1a32dbcd`, `0232124a` | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260819T0300` | 4 | `0a40db48`, `9f7d856b`, `19ac9fc3`, `b7feadd3` | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260819T1500` | 1 | `c3e9dcbc` | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260820T1900` | 13 | `c79d5dbc`, `e4ac2526`, `a0c3f640`, `505bd5f9`, `22e20c9f`, `ede1ab8a`, `701b0a50`, `39848d1b`, `04e88198`, `c7849e0a`, `d84ba2c4`, `290ed235`, `863608d1` | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260821T0500` | 1 | `66a64cb2` | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260821T1622` | 7 | `a29ba124`, `501dddb0`, `02d0f9bb`, `5334f289`, `8b0aa695`, `6187ebba`, `2d2f4519` | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260821T2300` | 2 | `f2e1c169`, `13b2d8a3` | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260822T0300` | 1 | `0396191b` | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260822T2300` | 1 | `e9af8bc5` | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260823T0300` | 1 | `a2dbb054` | -| `auto-daily-agents-hot-topics-digest-gtm-run-7-20260814T1700` | 1 | `37f05f5c` | -| `auto-daily-prod-release-scan-run-11-20260815T1754` | 1 | `59091a79` | -| `auto-daily-prod-release-scan-run-13-20260817T1740` | 4 | `1a405155`, `2bf19ea3`, `2363fd5a`, `9ea3c2e7` | -| `auto-daily-prod-release-scan-run-14-20260818T1740` | 1 | `06c6bab4` | -| `auto-daily-prod-release-scan-run-16-20260820T1708` | 6 | `845a4da1`, `d19e8f14`, `c2329e22`, `6b298232`, `65b864bf`, `21553287` | -| `auto-daily-prod-release-scan-run-18-20260821T1740` | 3 | `a313bcea`, `04230ccf`, `782038ec` | -| `auto-daily-prod-release-scan-run-19-20260822T1740` | 1 | `6613c1d9` | -| `auto-daily-prod-release-scan-run-20-20260823T1740` | 1 | `57026aac` | -| `auto-daily-prod-release-scan-run-6-20260811T1700` | 2 | `2acfc72b`, `69aa6e2f` | -| `auto-daily-prod-release-scan-run-7-20260812T1700` | 3 | `34de7420`, `9fb50f61`, `7ec7c820` | -| `auto-daily-prod-release-scan-run-8-20260813T1700` | 3 | `64f6db4a`, `0da91944`, `db45412e` | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-10-20260823T0200` | 2 | `98a89cf0`, `0997b509` | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-11-20260823T0700` | 2 | `dd1678cf`, `824b1859` | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-2-20260819T0200` | 4 | `b5d80904`, `de00197d`, `3a7d2133`, `0befdc75` | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-5-20260820T0700` | 3 | `35cc8ecd`, `1f90f984`, `6c4f82c3` | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-6-20260821T0200` | 2 | `e4d686e8`, `de5a3cf2` | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-7-20260821T0700` | 2 | `8263332e`, `eedea3aa` | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-8-20260822T0200` | 1 | `3d9b1dac` | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-9-20260822T0700` | 1 | `a95190fc` | -| `being-able-to-select-parent-worktree-when-creating-worktree` | 2 | `e57e2dd1`, `a8342e80` | -| `close-tab-on-mobile-back-to-prev` | 2 | `91aa4322`, `6684a1fc` | -| `commit-now-takes-a-long-time` | 3 | `1d81bbec`, `f01fe425`, `23acb531` | -| `create-scanning-issues` | 1 | `fe3a15cb` | -| `create-ssh-worktree-error` | 3 | `b70d565b`, `a2eba1ca`, `9cc25c3c` | -| `custom-agents` | 2 | `2c152915`, `85065bfc` | -| `daemon-downgrade-recovery` | 1 | `07c92549` | -| `default-child-workspace` | 1 | `752b0330` | -| `deleting-skill` | 1 | `b3d49597` | -| `discord-support` | 1 | `70675bd9` | -| `display-failed-automations` | 1 | `e1547367` | -| `e2e-automation` | 2 | `a93a2bba`, `bde9cf3c` | -| `enterprise-governance` | 1 | `7ee5502d` | -| `extension-planning` | 1 | `0cb31d71` | -| `fix-notes-send-name-send-targets-after-their-tab` | 3 | `3b9fd0c0`, `1b189bf4`, `6e7c347f` | -| `pure-extract-orca-runtime` | 3 | `48533547`, `bc7e0f51`, `bb0ce93b` | -| `split-pty-connection` | 2 | `f0265ed6`, `183c573b` | -| `split-task-page` | 4 | `75ea0008`, `7c525cae`, `4042761c`, `d3cfd89c` | -| `sta-4062-folder-note-rollback` | 3 | `11818262`, `41a4a575`, `98c13247` | -| `sta-4064-phantom-folder-note` | 3 | `c4d72e5c`, `4b2345f4`, `2effc0c4` | - -## Read-only reproduction commands - -Use copies of the artifacts when possible. The following commands do not mutate Orca state. - -### Confirm bundle versions - -```bash -jq . '/Applications/Orca.app/Contents/Resources/orca-local-build.json' -jq . '/Users/jinjingliang/Library/Caches/com.stablyai.orca.ShipIt/update.2pCRJKe/Orca.app/Contents/Resources/orca-local-build.json' -``` - -### Confirm ShipIt failure - -```bash -rg -n '11:18:|11:24:|11:25:|11:26:|11:36:|App Still Running Error|Aborting update attempt' \ - '/Users/jinjingliang/Library/Caches/com.stablyai.orca.ShipIt/ShipIt_stderr.log' -``` - -### Count terminal models in a snapshot - -```bash -jq '{ - terminalRows: ([.workspaceSession.tabsByWorktree[] | .[]] | length), - unifiedTerminalTabs: ([.workspaceSession.unifiedTabs[] | .[] | select(.contentType == "terminal")] | length) -}' '/Users/jinjingliang/Library/Application Support/orca/profiles/local-default/orca-data.json.bak.1' -``` - -Run the same command against `orca-data.json.bak.0` for the post-incident totals. - -### Extract incident-adjacent daemon kills - -```bash -jq -rc ' - select( - .ts >= "2026-08-23T18:29:00Z" and - .ts <= "2026-08-23T18:34:30Z" and - (.event == "session-killed" or .event == "session-exited") - ) | - [.ts, .pid, .event, .sessionId, .immediate, .code, .cause, .clientId] | - @tsv -' '/Users/jinjingliang/Library/Application Support/orca/logs/daemon.log' -``` - -### Locate stale daemon socket failures - -```bash -rg -n 'ECONNREFUSED|daemon-v[0-9]+\.sock' \ - '/Users/jinjingliang/Library/Application Support/orca/logs' -``` - -## Recommended code fixes - -The permanent PTY-owner alternatives, their performance models, migration paths, and tradeoffs are -specified in -[`PTY_DAEMON_IDENTITY_ARCHITECTURE_DESIGN.md`](PTY_DAEMON_IDENTITY_ARCHITECTURE_DESIGN.md). - -### 1. Make visibility-resume liveness tri-state end to end - -- Do not use synchronous `DaemonPtyRouter.hasPty()` for destructive reconciliation. -- Route `pty:hasPty` through `probePtyLiveness()` or rename the IPC to make its verdict semantics explicit. -- Return `false` only when the owning execution host/daemon authoritatively reports absence or exit. -- Return `null`/`unverifiable` when the PTY has no established route, an expected legacy adapter is missing, the socket is unreachable, the provider is unregistered, or discovery is incomplete. -- Keep `reconcileMissingSessions()` restricted to literal authoritative `false`. - -Required regression test: - -1. Persist a PTY owned by an older daemon generation. -2. Start a new current daemon while making the old socket unreachable/omitted. -3. Trigger terminal visibility resume. -4. Assert the verdict is `null`, `onExit` is not called, PTY bindings remain, and both terminal models remain persisted. - -Cover local, folder workspace, SSH, and mixed client/host version cases. For SSH, loss of contact must remain **unverifiable**. - -### 2. Preserve bindings on unverifiable reattach - -- Distinguish `exitedBeforeAttach` from transport failure or declined unverifiable resume. -- Do not clear `ptyIdsByLeafId`, tab `ptyId`, or ownership records on `ECONNREFUSED` alone. -- Store retry metadata separately so reopening the pane can attempt the same route again. -- Ensure a failed retry cannot create a fresh shell over a still-live detached agent without an explicit recovery choice. - -The first guarded build fixed main-process tri-state liveness but exposed a second renderer defect: `terminal_pane_owner_unverified` produced an undefined reattach result, after which the connection layer cleared the saved binding and started a replacement shell while displaying a toast that claimed the binding was untouched. Recovery-v2 adds the required renderer guard. Both layers must remain covered because either one can turn uncertainty into destructive state replacement. - -### 3. Persist exact owner and incarnation identity - -- Treat protocol version as compatibility, never daemon identity. -- Persist execution host, daemon incarnation, session incarnation, and logical PTY identity separately. -- Rebuild routes to the exact recorded owner after app restart; do not infer ownership from whichever daemon currently occupies `daemon-v.sock`. -- Accept an authoritative exit only from the matching execution host and daemon/session incarnation. -- Permit a new PTY to reuse a logical recovery identity only with a new incarnation and an explicit authoritative-missing or user-approved recovery decision. -- Keep provider continuation distinct from raw PTY reattachment. Resuming a saved Codex/Claude conversation creates a new process; it does not prove that the old raw process exited. - -If raw terminals must survive a true daemon crash, their PTY master must be held by a per-session broker or another independently surviving process. A replacement control daemon cannot reconstruct a PTY master file descriptor that died with its owner. - -### 4. Harden missing-worktree teardown - -- Treat a detected worktree list as destructive authority only when scan completeness is proven, not merely when the Git command exits successfully. -- Revalidate every allegedly missing path/worktree immediately before shutdown. -- Consider requiring two consistent authoritative scans separated by a graph/scan generation boundary before killing live PTYs. -- Include folder workspaces and execution-host scope in the reconciliation contract. -- Never infer process exit from SSH or relay contact loss. -- Abort the teardown if the detected set collapses suspiciously relative to the prior inventory. - -Required regression test: - -1. Return a successful but partial worktree scan that omits registered, existing worktrees. -2. Ensure no PTYs for omitted survivors are stopped. -3. Cover sessions distributed across current and legacy daemon adapters. - -### 5. Protect ShipIt's installation exclusivity window - -- Record an update-in-progress/handoff sentinel before quitting. -- On LaunchServices activation during that window, defer desktop startup or hand control back to ShipIt instead of starting the old installed bundle. -- Ensure CLI `open` honors the sentinel. -- Log activation provenance where possible: parent PID, responsible process, CLI operation, handoff token, and update attempt ID. -- Test repeated external activation while ShipIt is waiting for the target app to stay closed. - -### 6. Add forensic logging - -The absence of these fields prevented absolute attribution: - -- Every terminal row removal: tab ID, PTY ID, worktree ID hash/shape, close reason, liveness verdict, verdict source, provider route, daemon protocol, and renderer action. -- Every daemon kill: RPC method, caller subsystem, reason, worktree-reconciliation attempt ID, scan generation, and destructive/non-destructive classification. -- Missing-worktree reconciliation: known count, detected count, authoritative flag, scan source, omitted-ID hashes, cache generation, and per-item revalidation result. -- Daemon adapter discovery: expected generation, socket probe result, PID identity result, adapter omitted/retained decision, and PTY route count. -- Update activation: attempt ID plus the process responsible for reopening the app. - -Avoid logging raw absolute remote paths where existing privacy conventions require path shapes or hashes. - -## Open questions - -1. What invoked the three short-lived `/usr/bin/open` processes? -2. Why was hourly PID `77841` SIGKILLed at 11:29:01, and by whom? -3. Why did protocol-v36 daemon PID `6835` become unavailable and get replaced by PID `26391` at 11:33:05? -4. What caused an apparently successful/authoritative Git worktree scan to omit existing registered worktrees before the 15 graceful kills? -5. Did every one of the 158 row deletions traverse visibility-resume reconciliation, or did another exit callback contribute? Current logs cannot resolve individual mutations. -6. Which of the old detached agents are still live now, and which retained output/history is recoverable? -7. Did older pre-`da6b9d8065` terminal records lack ownership metadata needed by the near-match graph-sync fix? - -## Recovery and evidence-preservation cautions - -- Do not test fixes or reproduction steps against the only copy of the affected profile. -- Do not treat pane reopening, app restart, failed attach, or worktree activation as read-only; each can persist reconciliation results and rotate backups. -- Preserve `orca-data.json.bak.1`, `orca-data.json.bak.0`, all `daemon.log*`, `main.trace.ndjson`, `ShipIt_stderr.log`, the staged ad hoc bundle metadata, terminal history directories, and the temporary screenshots before further debugging. -- Inventory candidate processes and output logs before attempting to reconstruct tabs. -- A missing daemon route is **unverifiable**, not evidence that a process exited. -- Recovery should rebuild UI records only after matching PTY/process identity and execution host. Never attach a local record to an SSH/relay or different daemon session based only on a reused short suffix. - -No live Orca profile, PTY, daemon, or process was mutated in producing this report or the offline recovery package below. - -## Recovery package and cutover status - -The frozen evidence now has a deterministic recovery package under -`recovery/2026-08-23-session-disappearance/`: - -- `RECOVERY_CATALOG.md` and `RECOVERY_CATALOG.json`: 158 incident tabs across 61 worktrees, including every pane PTY, latest daemon event, history evidence, profile presence, and recommended recovery action. -- `orca-data.recovery-candidate.json`: an offline merge based on the frozen current profile. -- `RECOVERY_MERGE_VALIDATION.json`: topology, preservation, uniqueness, group-reachability, exclusion, count, and SHA-256 gates. -- `generate-recovery-artifacts.mjs`: deterministic generator. Pass `--current ` and `--output-dir ` to rebase the candidate on a consistent post-shutdown live profile. - -Classification and candidate totals: - -| Item | Count | -|---|---:| -| Incident tabs | 158 | -| Worktrees | 61 | -| Pane PTYs | 191 | -| Primary PTYs `unverifiable` | 138 | -| Primary PTYs `exited` | 20 | -| Missing layouts added to the frozen-base candidate | 135 | -| Missing legacy tabs added | 135 | -| Missing unified tabs added | 9 | -| Missing tab groups added | 3 | - -All 138 `unverifiable` primary PTYs have surviving history directories. None of the 20 `exited` primary PTYs does. The candidate never overwrites current layout/tab records and does not add missing records for the 20 proven-exited sessions. - -### Guarded build - -The liveness fix is committed on `debug-session-disappear`: - -- Fix commit: `25ae8e8edf` -- Current guarded branch head after merging `origin/main`: `05a27fb4274ad23c8485bb71af42fde9a8c67028` -- Signed/notarized ad hoc workflow: - -The IPC now waits for local PTY-provider startup, prefers `probePtyLiveness()`, preserves `null` as `unverifiable`, and uses legacy `hasPty()` only when no tri-state probe exists. Focused validation passed 35 tests plus the Node typecheck. - -### Additional restart hazard: stale ShipIt state - -`~/Library/Caches/com.stablyai.orca.ShipIt/ShipItState.plist` remains active and records: - -```text -bundleIdentifier = com.stablyai.orca -launchAfterInstallation = true -targetBundleURL = file:///Applications/Orca.app/ -updateBundleURL = file:///Users/jinjingliang/Library/Caches/com.stablyai.orca.ShipIt/update.2pCRJKe/Orca.app/ -``` - -Therefore a plain restart is unsafe even after preparing a candidate: ShipIt could later overwrite the guarded app with the stale unguarded ad hoc bundle. After the normal GUI process exits and only after confirming no ShipIt process is running, move the entire ShipIt cache into the cutover evidence directory. Do not delete it. - -### Controlled cutover gates - -1. Validate the signed/notarized arm64 app before downtime: exact commit/version metadata, bundle ID, architecture, readable state schema, attachable daemon protocol, strict code signature, Gatekeeper acceptance, and notarization staple. -2. Capture pre-cutover `orca status`, full terminal topology, and worktree process inventory. -3. Quit the GUI normally and wait for the main process to exit. Do not kill or restart the daemon; daemon survival is expected. A local in-process fallback PTY remains a residual restart risk. -4. Copy the full profile directory and profile index after shutdown. Quarantine the stale ShipIt cache. -5. Regenerate the recovery candidate against that exact post-quit `orca-data.json`. Require the validation base SHA-256 to equal the post-quit live SHA-256. -6. While `/Applications/Orca.app` is absent, atomically activate the rebased candidate and guarded app. This blocks the unknown external `open` actor from relaunching the hourly build during the swap. -7. Launch the exact app path. Before clicking through worktrees, require a new runtime ID, guarded version, ready runtime/graph, no degraded or stale-bundle errors, and no loss of pre-cutover PTY IDs. -8. Roll back the profile and app bundle together if any gate fails. Never run the old hourly app against the guarded recovery candidate alone. - -## Post-cutover outcome: preservation succeeded, automatic reattachment did not - -The controlled cutover completed with guarded app version -`1.4.189-adhoc.20260823201001`, runtime ID -`f73694d1-31b5-4abb-bc86-3123aae61783`, and surviving hourly daemon PID `26391`. -The stale ShipIt cache is quarantined under the cutover evidence directory. - -The initial post-launch merge validation found all 138 `unverifiable` incident topologies -reachable in the profile. Subsequent live inspection established that this was a persistence -result, not a runtime-recovery result: - -- 48 of 51 affected `unverifiable` worktrees are still tracked by Orca. -- Those 48 worktrees contain 131 incident sessions. -- 130 incident sessions still have their exact original PTY topology in the live profile. -- Three original incident panes, all in one worktree, are directly listed, connected, writable, - and readable. -- 73 tracked sessions have saved provider-session records; all 131 have preserved terminal - history. -- Seven records belong to three worktrees whose directories no longer exist. Their unified-tab - reachability was later removed by missing-worktree cleanup, while frozen evidence remained. -- The remaining non-exact record, `deleting-skill`, had already been rebound to blank replacement - shells before the guarded cutover. - -The guarded IPC fix prevents `null` liveness from being misclassified as `exited`; it does not -recreate a missing legacy daemon route. Live verification found a second defect: main correctly -throws `terminal_pane_owner_unverified` without retiring persistence, but the renderer transport -returns an undefined reattach result after reporting that marker. The connection layer treats the -undefined result as stale/dead, clears the leaf/tab PTY bindings, and starts a fresh shell. The -toast saying the session was left untouched is therefore false. A second renderer guard is -required before affected panes can be opened safely. - -A representative provider fallback succeeded for `close-tab-on-mobile-back-to-prev`: Codex -session `01a0068d-2be5-77b1-8a9e-cc7cda073fd2` resumed into a new live Orca tab without receiving -a new prompt. This establishes that provider continuation is technically available, but the new -tab must remain idle while the original raw PTY is `unverifiable`; otherwise two surviving agents -could be driven concurrently. Provider resume is safe after an authoritative `exited` verdict or -an explicit user decision to abandon the uncertain raw route. Plain shell tabs require -terminal-history export and cannot be assumed to retain process state. - -Full live accounting, exact identifiers, rollback locations, and the safe recovery sequence are -recorded in -`recovery/2026-08-23-session-disappearance/cutover-20260823T201851Z/POST_CUTOVER_RECOVERY_STATUS.md`. - -## Recovery-v2 cutover and first successful controlled recovery - -The second renderer guard was committed as `bd09097708` and shipped in signed/notarized arm64 build -`1.4.189-adhoc.20260823204916`. The installed app is now that build, runtime ID -`f5c43687-8851-4f0b-8222-1b4292caa125`, with runtime and graph both `ready`. Surviving hourly daemon -PID `26391` remained alive throughout the upgrade. - -The external-launch race also reproduced during this cutover. After v1 GUI PID `65542` exited -normally, the still-installed v1 bundle reopened as PID `96491` at 14:04:49 MST before the swap. -The second attempt normally quit that process and immediately moved the old bundle out of -`/Applications/Orca.app`, removing the relaunch target before the final profile snapshot and v2 -activation. This directly reinforces the original finding that an external activation can defeat a -ShipIt-style installation window even when Orca itself quits correctly. - -The final isolated pre-v2 profile SHA-256 was -`9bdd64cbfa764021149d281b8390797ed214916064628035db83b950c172d347`. Deterministic -validation before and after v2 launch preserved 1001 layouts and all 131 complete incident -topologies in tracked worktrees. ShipIt remains quarantined. - -One affected worktree was then activated as a controlled recovery probe: - -```text -auto-e2e-tests-autofix-scheduled-ci-1h-run-9-20260822T0700 -tab d6d43b87-9d6b-4c48-87cc-641719ecab8e -leaf 4d1848a5-e6ae-4108-b59e-e049791b70d9 -PTY @@a95190fc -Codex session 01a02a6a-01fc-74c2-b77a-1d1f674d66e0 -``` - -The current daemon authoritatively lacked that old raw session, so Orca created one replacement -daemon session under the same logical PTY ID and resumed the saved Codex provider session in the -same tab. Core layout, legacy-tab, unified-tab, leaf, and PTY binding data were byte-identical before -and after. The new runtime handle is `term_88f38ceb-d943-4c3c-8b36-b844616ad603`, incarnation -`761a724d-5d78-42c2-946b-8dd962f5a217`; it is connected, writable, and TUI-idle. No prompt was sent. - -This live probe exercised authoritative-missing provider recovery, not the owner-unverified branch. -The latter is covered by the focused 10-test renderer suite and must preserve bindings without -creating a new PTY. Full build verification, cutover evidence, the exact distinction between raw -reattach and provider continuation, rollback paths, and safe next steps are recorded in -`recovery/2026-08-23-session-disappearance/cutover-v2-20260823T140110MST/RECOVERY_V2_CUTOVER_STATUS.md`. diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 12b937b187f..099554287c7 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -14089,6 +14089,221 @@ "The legacy remote-uploads root is intentionally not reclaimed while a mixed-version relay may still own bytes there; eventual legacy-root retirement requires a separate compatibility decision." ], "demotionRule": "Demote if one live owner can remove another owner's bytes, cleanup exceeds 64 inspected entries per attempt, a lifecycle path retains its exact owned archive after settlement, or the process oracle flakes without an identified harness or product defect." + }, + { + "id": "terminal-session.owner-incarnation-authority", + "title": "Only the exact PTY owner and session can report liveness", + "maturity": "experimental", + "protection": "partial", + "owner": "terminal-runtime", + "layer": "provider-main-renderer-lifecycle", + "surfaces": [ + "daemon generation handoff", + "renderer visibility reconciliation", + "persisted pane reattach", + "direct SSH guarded recovery", + "logical PTY id reuse" + ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local-daemon", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos", "linux"], + "coveredProviders": ["local-daemon", "ssh"], + "coverageNotes": "Deterministic production-boundary tests cover local daemon routing, pre-side-effect daemon attach fencing, SSH provider/relay incarnation checks, exact relay exit proof, renderer preservation, and guarded recovery. A real macOS multi-generation daemon journey and rendered Docker SSH reconnect journey passed. Paired headed/headless was blocked by missing web-client URL; targeted A/B update handoff, WSL, ConPTY, and physical Windows remain gaps.", + "motivatingLinks": [ + "https://linear.app/stably/issue/STA-5269", + "https://github.com/stablyai/orca/pull/16152" + ], + "invariant": "Only the exact execution host, owner incarnation, and session incarnation may report live or exited. Missing routes, adapters, metadata, or contact are unverifiable and preserve every PTY binding, terminal row, pane, tab, and retry lease.", + "oracle": "Persist a PTY owned by daemon A, replace the compatible current slot with daemon B, and have B report the logical id absent. Require an unverifiable verdict without querying B as the owner, without broad fallback, deletion, fresh spawn, or unrelated stop. Exact A plus the exact session may report live or exited; a mismatched reused incarnation, provider loss, and owner-unverified result or throw preserve topology and settle guarded recovery.", + "commands": [ + "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/daemon-session-owner-authority.oracle.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/daemon-session-owner-resolution.test.ts src/main/daemon/daemon-pty-router.test.ts src/main/daemon/daemon-pty-router-owner-fencing.test.ts src/main/daemon/daemon-pty-adapter-protocol-compatibility.test.ts src/main/daemon/daemon-pty-adapter-session-adoption.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/terminal-host-incarnation-fence.test.ts src/main/daemon/degraded-daemon-pty-provider.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/pty-startup-barrier-and-listing.test.ts src/main/ipc/pty-persisted-incarnation-repair.test.ts src/main/ipc/pty-dead-owner-respawn.test.ts src/main/ipc/pty-controller-spawn-admission.test.ts src/main/agent-hooks/restored-subagent-liveness-sweep.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-provider-spawn.test.ts src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts src/relay/pty-handler-attach-replay.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts src/renderer/src/components/terminal-pane/pty-transport-reattach-admission.test.ts src/renderer/src/components/terminal-pane/pty-connection-session-liveness.test.ts src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts --reporter=dot", + "pnpm exec playwright test tests/e2e/daemon-generation-reconnect-safety.spec.ts tests/e2e/daemon-generation-legacy-close-safety.spec.ts --config tests/playwright.config.ts --project=electron-headless --workers=1", + "ORCA_E2E_SSH_DOCKER=1 pnpm exec playwright test tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts --config tests/playwright.config.ts --project=electron-headless --workers=1", + "pnpm exec playwright test tests/e2e/paired-remote-terminal-probe-gap-recovery.spec.ts tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts --config tests/playwright.config.ts --project=electron-headful --workers=1" + ], + "testFiles": [ + "src/main/daemon/daemon-session-owner-authority.oracle.test.ts", + "src/main/daemon/daemon-session-owner-resolution.test.ts", + "src/main/daemon/daemon-pty-router.test.ts", + "src/main/daemon/daemon-pty-router-owner-fencing.test.ts", + "src/main/daemon/daemon-pty-adapter-protocol-compatibility.test.ts", + "src/main/daemon/daemon-pty-adapter-session-adoption.test.ts", + "src/main/daemon/terminal-host.test.ts", + "src/main/daemon/terminal-host-incarnation-fence.test.ts", + "src/main/daemon/degraded-daemon-pty-provider.test.ts", + "src/main/ipc/pty-startup-barrier-and-listing.test.ts", + "src/main/ipc/pty-persisted-incarnation-repair.test.ts", + "src/main/ipc/pty-dead-owner-respawn.test.ts", + "src/main/ipc/pty-controller-spawn-admission.test.ts", + "src/main/agent-hooks/restored-subagent-liveness-sweep.test.ts", + "src/main/providers/ssh-pty-provider-spawn.test.ts", + "src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts", + "src/relay/pty-handler-attach-replay.test.ts", + "src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts", + "src/renderer/src/components/terminal-pane/pty-transport-reattach-admission.test.ts", + "src/renderer/src/components/terminal-pane/pty-connection-session-liveness.test.ts", + "src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts", + "src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts", + "src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts", + "tests/e2e/daemon-generation-reconnect-safety.spec.ts", + "tests/e2e/daemon-generation-legacy-close-safety.spec.ts", + "tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts", + "tests/e2e/paired-remote-terminal-probe-gap-recovery.spec.ts", + "tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts" + ], + "assertionRefs": [ + { + "file": "src/main/daemon/daemon-session-owner-resolution.test.ts", + "assertions": [ + "a replacement daemon cannot report a historical owner session exited", + "only an exact routed owner and session can confirm absence" + ] + }, + { + "file": "src/main/daemon/terminal-host-incarnation-fence.test.ts", + "assertions": [ + "a mismatched incarnation rejects before snapshot, detach, attach, or output ownership changes" + ] + }, + { + "file": "src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts", + "assertions": [ + "relay absence and missing replay continuity remain unverifiable while exact matched process death is exited" + ] + }, + { + "file": "src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts", + "assertions": [ + "result and thrown owner-unverified outcomes settle the exact retry and preserve topology" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-08-23", + "runner": "local", + "platform": "macos", + "result": "passed", + "command": "pnpm exec playwright test tests/e2e/daemon-generation-reconnect-safety.spec.ts tests/e2e/daemon-generation-legacy-close-safety.spec.ts --config tests/playwright.config.ts --project=electron-headless --workers=1", + "durationSeconds": 26.5, + "summary": "The macOS multi-generation close-intent journey passed; two Windows-specific reconnect cases skipped on macOS." + }, + { + "date": "2026-08-23", + "runner": "local", + "platform": "linux", + "result": "passed", + "command": "ORCA_E2E_SSH_DOCKER=1 pnpm exec playwright test tests/e2e/ssh-docker-reconnect-pane-restore.spec.ts --config tests/playwright.config.ts --project=electron-headless --workers=1", + "durationSeconds": 60, + "summary": "Rendered Docker OpenSSH/relay reconnect preserved the exact scrollback marker and TUI frame and produced a usable new tab." + }, + { + "date": "2026-08-23", + "runner": "local", + "platform": "macos", + "result": "failed", + "command": "pnpm exec playwright test tests/e2e/paired-remote-terminal-probe-gap-recovery.spec.ts tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts --config tests/playwright.config.ts --project=electron-headful --workers=1", + "durationSeconds": 20, + "summary": "Infrastructure-blocked before terminal exercise: both headed-host and isolated-headless-host cases lacked the required paired web-client URL." + }, + { + "date": "2026-08-23", + "runner": "local", + "platform": "macos", + "result": "passed", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/daemon-session-owner-authority.oracle.test.ts --reporter=dot", + "durationSeconds": 0.38, + "summary": "The candidate passed the SHA-1 d72d4911e461876041406594a5682b6e2b9324a0 byte-identical A/B topology oracle." + }, + { + "date": "2026-08-23", + "runner": "local", + "platform": "macos", + "result": "passed", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/daemon-session-owner-resolution.test.ts src/main/daemon/daemon-pty-router.test.ts src/main/daemon/daemon-pty-router-owner-fencing.test.ts src/main/daemon/daemon-pty-adapter-protocol-compatibility.test.ts src/main/daemon/daemon-pty-adapter-session-adoption.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/terminal-host-incarnation-fence.test.ts src/main/daemon/degraded-daemon-pty-provider.test.ts --reporter=dot", + "durationSeconds": 3.2, + "summary": "Eight daemon authority files passed 200 routing, mixed-protocol, pre-side-effect fencing, and provider tests." + }, + { + "date": "2026-08-23", + "runner": "local", + "platform": "macos", + "result": "passed", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/pty-startup-barrier-and-listing.test.ts src/main/ipc/pty-persisted-incarnation-repair.test.ts src/main/ipc/pty-dead-owner-respawn.test.ts src/main/ipc/pty-controller-spawn-admission.test.ts src/main/agent-hooks/restored-subagent-liveness-sweep.test.ts --reporter=dot", + "durationSeconds": 10.52, + "summary": "Five IPC and lifecycle files passed 53 startup, persistence, delayed-verdict, cleanup, and spawn-admission tests." + }, + { + "date": "2026-08-23", + "runner": "local", + "platform": "macos", + "result": "passed", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-provider-spawn.test.ts src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts src/relay/pty-handler-attach-replay.test.ts --reporter=dot", + "durationSeconds": 0.63, + "summary": "Three SSH provider and relay files passed 54 attach, incarnation, source-recovery, and exact-exit tests." + }, + { + "date": "2026-08-23", + "runner": "local", + "platform": "macos", + "result": "passed", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts src/renderer/src/components/terminal-pane/pty-transport-reattach-admission.test.ts src/renderer/src/components/terminal-pane/pty-connection-session-liveness.test.ts src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts --reporter=dot", + "durationSeconds": 4.07, + "summary": "Six renderer files passed 65 liveness, typed-result, topology-preservation, and bounded recovery tests." + }, + { + "date": "2026-08-23", + "runner": "local", + "platform": "macos", + "result": "failed", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/daemon-session-owner-authority.oracle.test.ts --reporter=dot", + "durationSeconds": 0.45, + "summary": "Expected red: both latest main and PR #16152 returned false from replacement B, activating modeled row/tab/pane deletion and fresh spawn." + }, + { + "date": "2026-08-23", + "runner": "local", + "platform": "macos", + "result": "failed", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/daemon-session-owner-authority.oracle.test.ts --reporter=dot", + "durationSeconds": 2.08, + "summary": "Expected red: disabling the exact-route guard returned false from B and reactivated the destructive topology branch." + } + ], + "runtimeBudget": { + "p95Seconds": 30, + "scope": "focused owner resolver, provider, IPC, and renderer lifecycle contracts" + }, + "flakeHistory": { + "status": "not-started", + "evidence": "The deterministic contracts use in-memory inventories and explicit promise barriers; CI and soak history have not started." + }, + "redGreenEvidence": { + "status": "complete", + "evidence": "The SHA-1 d72d4911e461876041406594a5682b6e2b9324a0 byte-identical A/B oracle failed on latest main, PR #16152, and the candidate with exact-route guarding disabled: each returned false from replacement B and activated modeled topology deletion/fresh spawn. The candidate returned unverifiable without querying B and preserved row, tab, pane, and live-marker state." + }, + "performanceBudget": { + "required": true, + "evidence": "Exact routed probes remain O(1); unmapped negative probes do not scan or query a replacement daemon. Existing coalesced inventories remain bounded and cached for positive route discovery." + }, + "promotionCriteria": [ + "Pass Docker SSH/relay and paired headed/headless mixed-version journeys.", + "Collect physical Windows ConPTY/WSL and Linux owner-incarnation evidence.", + "Collect 100 consecutive focused CI passes or 14 days without an unexplained flake." + ], + "knownGaps": [ + "Persisted daemon owner-incarnation identity is not yet implemented; the immediate guard uses exact in-process routes plus session incarnations and fails closed when the route is missing.", + "A current client intentionally refuses authoritative reattach to a surviving pre-v37 daemon because that owner cannot fence before attach side effects; topology survives but the terminal can remain inaccessible for that PTY's lifetime.", + "A pre-fence SSH relay can ignore the optional expected incarnation and mutate delivery before a current client rejects its response; capability negotiation remains permanent-boundary work.", + "The real macOS and Docker journeys did not recreate the exact A-unregistered/B-compatible-slot incident topology.", + "Paired headed/headless could not start because the local runtime exposed no paired web-client URL; WSL, ConPTY, and physical Windows remain untested.", + "No design can recover a PTY master after its actual owning process dies." + ], + "demotionRule": "Demote if any missing owner, route, adapter, session incarnation, or transport is interpreted as exit; if a reused logical id is retired by an older incarnation; or if guarded recovery clears topology or spawns a shell solely because ownership is unverifiable." } ] } diff --git a/recovery/2026-08-23-session-disappearance/RECOVERY_CATALOG.md b/recovery/2026-08-23-session-disappearance/RECOVERY_CATALOG.md deleted file mode 100644 index edb1102a9be..00000000000 --- a/recovery/2026-08-23-session-disappearance/RECOVERY_CATALOG.md +++ /dev/null @@ -1,243 +0,0 @@ -# Terminal session recovery catalog - -> Live recovery update, 2026-08-23 14:12 MST: recovery-v2 -> `1.4.189-adhoc.20260823204916` is installed, all 131 complete affected topologies in tracked -> worktrees survived the cutover, and one provider-backed affected Codex tab was recovered in -> place. This catalog intentionally remains a frozen-evidence artifact; current runtime results are -> recorded in -> `cutover-v2-20260823T140110MST/RECOVERY_V2_CUTOVER_STATUS.md`. - -This catalog is derived only from the frozen evidence in `evidence/`. It did not query or modify the live Orca app, profile, runtime, PTYs, or process table. - -## Recovery summary - -| Measure | Count | -|---|---:| -| Incident terminal tabs | 158 | -| Worktrees | 61 | -| Pane PTYs represented by those tabs | 191 | -| Primary PTYs: unverifiable | 138 | -| Primary PTYs: exited | 20 | -| Surviving primary history directories | 138 | -| History bytes | 5100637 | -| Frozen-current unverifiable topologies already complete | 3 | -| Missing unverifiable layouts added to candidate | 135 | -| Missing unverifiable legacy tabs added | 135 | -| Missing unverifiable unified tabs added | 9 | -| Missing tab groups added | 3 | -| Exited terminal tabs receiving no restored records | 20 | - -The incident set is the 158 terminal layout IDs present in `orca-data.json.bak.1` and absent from `orca-data.json.bak.0`. A primary PTY is **exited** only if its latest event, ordered by timestamp and then frozen log order, is `session-exited` or `session-killed`; all other outcomes are **unverifiable**. Loss of contact is never interpreted as exit. - -## Offline merge policy - -The candidate starts from the frozen current `evidence/profile/orca-data.json`. For each unverifiable primary PTY it adds only missing before-profile records: 135 layouts, 135 legacy tabs, 9 unified tabs, and 3 tab groups. Existing current layouts and tab records are never overwritten; existing group state stays authoritative and receives only missing tab IDs. No missing records are added for the 20 proven-exited primary PTYs, though any records already present in the frozen current base remain untouched. - -Validation result: **PASS**. The candidate grows from 863 to 998 layouts. Full machine-readable checks and hashes are in `RECOVERY_MERGE_VALIDATION.json`. - -The candidate is not ready for direct use while the affected hourly build is running. Before use: perform a controlled Orca shutdown, capture a fresh live-profile backup, and start a build with tri-state liveness protection. Only then compare the fresh shutdown profile with this frozen-base candidate and decide whether to regenerate or substitute it. - -## Evidence limitations - -- No frozen `orca terminal list` output is present, so the JSON catalog leaves runtime handles null. A profile PTY ID is recorded separately and is not claimed to be a live runtime handle. -- No frozen `ps` snapshot is present, so process-candidate arrays are empty. Daemon log `pid` values are daemon emitter PIDs, not agent-child PIDs. -- A historical `session-created` or `session-attached` event does not prove current liveness; it therefore remains unverifiable. -- Of the 138 unverifiable primary PTYs, only 5 have their latest frozen daemon event from PID `6835`; 133 have their latest event from other daemon PIDs. This provenance does not prove those sessions remained live, but it rules out treating loss of PID `6835` as sufficient evidence that the whole unverifiable set exited. -- `Not found in registered daemon adapters` is routing evidence, not an exit verdict. The catalog requires a latest `session-exited` or `session-killed` event before classifying a primary PTY as exited. -- Each JSON session includes all pane PTYs, latest daemon event provenance, terminal-history metadata and hashes, provider recovery metadata, current-profile presence, candidate disposition, and a recommended recovery action. - -## Per-session index - -| Worktree | Tab ID | Primary PTY suffix | Panes | Verdict | History | Provider records | Current layout | Candidate action | -|---|---|---|---:|---|---|---:|---|---| -| `orca-top-level-cleanup` | `7e00a32c-a6cb-4f1e-9f27-777f10ba2163` | `fa471831` | 1 | **exited** | no | 0 | missing | preserve-current-exited-records | -| `1.4.183-p0-orca-crashed-all-agent-sessions` | `5839876f-859d-48fa-ad8a-4aede2ea95b0` | `8804d1a3` | 1 | **exited** | no | 1 | missing | preserve-current-exited-records | -| `add-more-e2e-tests` | `a3440ae6-882a-4362-95f5-c58d5016ddbe` | `ba88e6c5` | 1 | **exited** | no | 1 | missing | preserve-current-exited-records | -| `add-starting-agent-from-tab-e2e` | `2a3a2338-1445-45ec-8598-13c019467109` | `31c4cb12` | 1 | **exited** | no | 1 | present | preserve-current-exited-records | -| `add-starting-agent-from-tab-e2e` | `46e4f9a1-0867-4dfe-be66-e683bdb5cdde` | `43f5e12a` | 1 | **exited** | no | 1 | present | preserve-current-exited-records | -| `add-starting-agent-from-tab-e2e` | `c8cc6933-c4f3-473f-a544-70bd3f43ee78` | `4ea40b01` | 1 | **exited** | no | 1 | present | preserve-current-exited-records | -| `add-starting-agent-from-tab-e2e` | `f853e0e5-15d7-4888-af9b-390e1d819fe9` | `8972c618` | 1 | **exited** | no | 0 | missing | exclude-proven-exited | -| `allow-create-automations-with-ai` | `271d789d-6db1-423b-a714-d74e91a111db` | `49fbac2f` | 2 | **exited** | no | 0 | missing | preserve-current-exited-records | -| `analytics` | `ae9b57fa-3351-4327-a726-94af04868e77` | `5fb657da` | 2 | **exited** | no | 1 | missing | preserve-current-exited-records | -| `audit-default-settings` | `fd0a28c0-ce62-438a-b13f-08dfc9a868de` | `55fddb5f` | 2 | **exited** | no | 1 | missing | preserve-current-exited-records | -| `auto-auto-pr-assignment-run-86-20260814T1645` | `bc077d77-7800-4aad-91c6-3ab0471b56de` | `b9b726f2` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-auto-pr-assignment-run-86-20260814T1645` | `cac7e5ca-96c8-4dd3-bd26-1c42e539940d` | `074c00d3` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260814T1500` | `bd3fce23-8552-44cf-bd6a-b6e0aeeb4f42` | `2c0c1f2c` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260814T2300` | `3c80f8b3-cded-44dd-9236-25a657dab47b` | `72fbd9e4` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260815T1900` | `1b7b2a04-0627-43c7-92b1-d66861b28455` | `ca76565c` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260815T1900` | `b2e2fccd-9b81-4632-b91c-067dd671b611` | `34e82ad0` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260815T1900` | `f7aaf80d-9522-4c93-b641-d9c0e34516e5` | `c88a58ad` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T1900` | `0fd88ccc-9dde-43e9-aff9-6301aa75d9b4` | `3867ab5d` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T1900` | `c7178099-e489-4bc2-b6e6-f866723a03f9` | `343c97dc` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T1900` | `eb2b6d01-3018-4733-a9ab-69357c1795bd` | `614960b9` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `09d7a488-4e20-4229-a9fb-a176f71a9311` | `6bc4cece` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `138ce7f3-cad3-40de-afa1-e6279977fbcc` | `fac0c631` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `149d3580-bd7a-4815-af23-9906587bb716` | `593e9a00` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `1f55d417-be78-4ea2-b9c6-d8d1348aea84` | `8298b45c` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `3bd7cf1c-bf7a-4900-89d3-ab6d549a514f` | `2727bdb1` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `45fb20d7-093e-4bc8-880f-852fe488f983` | `eb3b9fe2` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `4cbc1a76-0265-42c0-8cb1-1742d512738e` | `e6430688` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `50b5be53-de68-4a4a-bb66-0b41e63d2b83` | `d6499bd5` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `5437e8b5-3003-4f18-8027-80e8b94d6634` | `1a32dbcd` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `56a02156-5f07-4008-b32f-0abb086630fd` | `0232124a` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `8630026e-0525-4c18-a8b5-5eb8b4c92f80` | `7f7126e5` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `8d1c66d8-4f5f-4b4d-84f2-f0c9c30f006f` | `3921d7fb` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `9f78616e-1a5c-4688-b5b8-2f2b76f768fd` | `1bc85db0` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `a76c55c9-e879-4f92-ba24-906058b011e7` | `efca7e8b` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `acba7889-fc0f-406a-b2ef-215a5ecd436f` | `b0fcff2e` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `b019ccdd-f5cb-4468-a3cc-daf60a1cf439` | `fa5e2406` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `bd0f896f-6c69-4668-a72b-1dc927f4bbc1` | `1a8a174c` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `bee2ce09-9103-43a0-8802-e19b3d607eea` | `23423419` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `c340f891-0215-441f-be38-d2bb8498fea3` | `4885350c` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `c3b6fbef-2f24-4057-a148-8d2f8b0cf777` | `59e2e93c` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `cc84b271-6b9b-4f9d-82af-02fc7fc48797` | `edf568c4` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `d61ce87a-b9c9-4ffe-93b8-23d4eb0bccc1` | `837371b4` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `e7b9457a-4759-44e3-9058-fa375e61ff92` | `6bae4f90` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260818T2300` | `fa56e660-455d-4ac5-a367-8481c1ce351c` | `d5016fa5` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260819T0300` | `05701a5e-d2a9-4002-bddf-e24a6b248e88` | `19ac9fc3` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260819T0300` | `63ac1f4e-4596-4858-bbfd-5e60ccc8217c` | `0a40db48` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260819T0300` | `8692ed46-447c-47b5-a68a-f3d7c9dcf6d9` | `9f7d856b` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260819T0300` | `ce89a3d7-5d76-4e0c-9696-42a63b61e82c` | `b7feadd3` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260819T1500` | `e522f804-bf30-42bc-a1ff-ae99cb1d9b36` | `c3e9dcbc` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260820T1900` | `036086d6-fdc4-42fd-92cb-78231b572075` | `22e20c9f` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260820T1900` | `064fe2f7-0030-4a3a-b998-d6a66660fd03` | `290ed235` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260820T1900` | `3325dda4-085e-45d2-a803-9aa406cf9db9` | `863608d1` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260820T1900` | `339f2a7a-5b4b-4f8e-b136-408792de069c` | `d84ba2c4` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260820T1900` | `33e38b02-b71f-437e-b10a-21d546275ff8` | `c79d5dbc` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260820T1900` | `44a6cdaf-eb1b-4b3d-babe-93939175930d` | `505bd5f9` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260820T1900` | `450e8103-9607-41b9-8dd6-ee033a36e9c5` | `e4ac2526` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260820T1900` | `5b025b35-c98d-4b66-b233-478b5df9437a` | `ede1ab8a` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260820T1900` | `689bec64-1482-44dc-a4f6-633bc4ed4283` | `c7849e0a` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260820T1900` | `90726d94-0fc5-4adc-b321-1dfd4f5f9027` | `701b0a50` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260820T1900` | `b5286573-3ac6-4724-9baa-374b2ceb6563` | `04e88198` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260820T1900` | `bae2dfc1-6d54-446e-9488-759aa8e3a670` | `a0c3f640` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260820T1900` | `eb7ae053-ba4f-48bb-a064-7a8ec67ee139` | `39848d1b` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260821T0500` | `2be24a99-bd69-4eea-aef6-e5f91a1b2976` | `66a64cb2` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260821T1622` | `4280f25e-44f9-46f7-a3b8-866b6056f20d` | `5334f289` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260821T1622` | `4c79f3fd-72c7-4842-9225-edfe1e66ac2e` | `02d0f9bb` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260821T1622` | `53d48771-8101-4e97-ad39-9ae23f20f6a9` | `8b0aa695` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260821T1622` | `5b859e30-dca0-43ad-9246-53c8338ae6c6` | `501dddb0` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260821T1622` | `a53310e5-268f-498a-b7ed-52dd7cb8fbcb` | `2d2f4519` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260821T1622` | `b28a6a9f-407a-4669-be9e-d145e0b68945` | `6187ebba` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260821T1622` | `c10c91f1-76e5-4170-a0cc-141cb97bb8a8` | `a29ba124` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260821T2300` | `2910a21a-7c02-44e2-b0ca-661c39866298` | `13b2d8a3` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260821T2300` | `6b19be54-3d68-42e6-8876-f962ae952c12` | `f2e1c169` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260822T0300` | `8958317c-91dd-4349-9bcd-3a98ec1c7ca1` | `0396191b` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260822T2300` | `26726cbb-f7b5-4035-8d07-6c9b41990e6b` | `e9af8bc5` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-continuous-readiness-checklist-8a-12p-4p-20260823T0300` | `dcddf2fb-0971-4d43-ad03-fb5ce5a2cd61` | `a2dbb054` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-daily-agents-hot-topics-digest-gtm-run-7-20260814T1700` | `428bc398-11c0-4781-9b79-2ab03054dc81` | `37f05f5c` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-11-20260815T1754` | `ba9d00f8-3f5b-493f-9319-94037dc10ee6` | `59091a79` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-13-20260817T1740` | `59761048-ed6b-4590-b6ec-8417e7fd77a6` | `2bf19ea3` | 2 | **unverifiable** | yes | 2 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-13-20260817T1740` | `c5e12afc-4531-4d54-8035-68ed6d2e8026` | `2363fd5a` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-13-20260817T1740` | `db740737-57bb-4700-8497-0780b54fb043` | `9ea3c2e7` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-13-20260817T1740` | `ee01f0b4-a157-48d1-a305-53e7f48fecf1` | `1a405155` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-14-20260818T1740` | `e35397a5-5cee-441d-baf0-57c014f14790` | `06c6bab4` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-16-20260820T1708` | `6af9a99f-12f5-443d-90ab-134b8baebe48` | `845a4da1` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-16-20260820T1708` | `c06ef3bb-2944-45bf-b8aa-64e49a514109` | `c2329e22` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-16-20260820T1708` | `d169ec37-3ec4-4540-8af2-a89dfd02e506` | `d19e8f14` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-16-20260820T1708` | `db39371f-27cb-4c08-a175-f43ae51aa53f` | `21553287` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-16-20260820T1708` | `eaa5210e-0174-46fe-bbdf-94b5929b41f4` | `6b298232` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-16-20260820T1708` | `f913ab34-7dbb-4582-8d75-e2822ccda52d` | `65b864bf` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-18-20260821T1740` | `8478a4b5-4d01-4b0f-a1dc-0bf55993719e` | `782038ec` | 1 | **unverifiable** | yes | 1 | present | preserve-current-topology | -| `auto-daily-prod-release-scan-run-18-20260821T1740` | `bc221df1-fac5-48fa-a35b-0d3f2375945b` | `a313bcea` | 2 | **exited** | no | 0 | missing | exclude-proven-exited | -| `auto-daily-prod-release-scan-run-18-20260821T1740` | `d919ab75-fb15-4f21-9425-21e6c16794be` | `04230ccf` | 2 | **unverifiable** | yes | 1 | present | preserve-current-topology | -| `auto-daily-prod-release-scan-run-19-20260822T1740` | `237117d8-b14d-4edb-a56b-399d6f4ab0c9` | `6613c1d9` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-20-20260823T1740` | `00e08298-f6f3-425b-9ee6-22fe7132f229` | `57026aac` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-6-20260811T1700` | `190bcbca-c988-4b37-bcca-d1c8b6d7d7cd` | `69aa6e2f` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-6-20260811T1700` | `2864af3e-6155-4901-b663-afb55ba43064` | `2acfc72b` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-7-20260812T1700` | `869e32e7-6566-4ba5-adf2-9eda7b4494f8` | `34de7420` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-7-20260812T1700` | `9728013f-aff4-4294-92ce-36afea103bac` | `9fb50f61` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-7-20260812T1700` | `ab920a81-8c20-46cf-a32d-9bdcd5e2d9ae` | `7ec7c820` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-8-20260813T1700` | `53484cef-73d2-4f33-a2ea-96e44aee2c72` | `0da91944` | 3 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-8-20260813T1700` | `7c9afce8-f410-45c2-b3ea-f169c5304fb1` | `64f6db4a` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-daily-prod-release-scan-run-8-20260813T1700` | `f8e25869-400b-478f-9ad7-563cd165830a` | `db45412e` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-10-20260823T0200` | `d1d27580-d7be-437f-8fa5-951368151478` | `98a89cf0` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-10-20260823T0200` | `fae01383-5948-44e1-8e23-d7f5a800d1e8` | `0997b509` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-11-20260823T0700` | `19fd8026-8d74-4241-84f0-8feec88c2f93` | `824b1859` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-11-20260823T0700` | `2668697d-3b06-45a7-ada2-ff7772ac764e` | `dd1678cf` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-2-20260819T0200` | `05a1ca5c-d5e6-4a6a-910e-13c79b9ad769` | `3a7d2133` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-2-20260819T0200` | `8d2caa53-ba05-4495-bb5f-3cc00fa9d15d` | `0befdc75` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-2-20260819T0200` | `937c8488-57f4-4bbe-8264-b028e6c8bd1c` | `b5d80904` | 2 | **unverifiable** | yes | 2 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-2-20260819T0200` | `9820d9bb-f4a9-4e20-86a0-92e583e9ce5a` | `de00197d` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-5-20260820T0700` | `29652a5a-2fd5-47ca-a286-a2eb9e90cca6` | `35cc8ecd` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-5-20260820T0700` | `3b15826a-a649-42e4-bb39-2cb3a66df945` | `1f90f984` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-5-20260820T0700` | `e15e1c2e-b031-47aa-af53-f686cbb21efd` | `6c4f82c3` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-6-20260821T0200` | `90b910a9-55d2-4912-a155-51d281aec183` | `e4d686e8` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-6-20260821T0200` | `be872a35-6422-4b3e-8a0f-9b4b4cff1a6b` | `de5a3cf2` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-7-20260821T0700` | `9d8a5da9-0189-45f2-bc3b-832384bdf0a7` | `eedea3aa` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-7-20260821T0700` | `e87ae76c-6fe3-40f9-9dd5-8f146b0971dd` | `8263332e` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-8-20260822T0200` | `6ed35091-30c4-4088-b1be-d6869408aa33` | `3d9b1dac` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `auto-e2e-tests-autofix-scheduled-ci-1h-run-9-20260822T0700` | `d6d43b87-9d6b-4c48-87cc-641719ecab8e` | `a95190fc` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `being-able-to-select-parent-worktree-when-creating-worktree` | `d3df7002-ba01-4e10-ab50-5553480ad974` | `a8342e80` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `being-able-to-select-parent-worktree-when-creating-worktree` | `d7db5e13-d101-444e-bde3-513e689a05c5` | `e57e2dd1` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `close-tab-on-mobile-back-to-prev` | `96c97b5f-1301-426f-9873-beda6bbdd657` | `6684a1fc` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `close-tab-on-mobile-back-to-prev` | `bef02734-4073-4ad1-aee8-22b9b256e919` | `91aa4322` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `commit-now-takes-a-long-time` | `335c5789-42bb-4380-b164-7fc2bd60a21a` | `f01fe425` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `commit-now-takes-a-long-time` | `3b7657d0-ac7a-46bd-a85b-5c6fd25cf91e` | `23acb531` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `commit-now-takes-a-long-time` | `da97fd74-8b48-4a49-9327-4b2955c20a15` | `1d81bbec` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `create-scanning-issues` | `bd4aa329-3e5c-40e6-bdb1-d55a77b27552` | `fe3a15cb` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `create-ssh-worktree-error` | `242f0327-915a-4694-bf94-91c0b4cdea2b` | `9cc25c3c` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `create-ssh-worktree-error` | `24f63cd7-cd4a-4b81-a339-4f2eda115bac` | `b70d565b` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `create-ssh-worktree-error` | `7b261f3b-fc27-40b4-a0fe-573a1f76b79a` | `a2eba1ca` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `custom-agents` | `097c89c9-3ce5-4ec9-a6e2-744c33dddf0c` | `2c152915` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `custom-agents` | `8e5359bf-aa35-4930-a8d8-3ee82e308832` | `85065bfc` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `daemon-downgrade-recovery` | `fe005739-e157-4086-88d2-5867637e29b3` | `07c92549` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `default-child-workspace` | `02d3c458-c11d-4374-bab5-5f54de21dba6` | `752b0330` | 2 | **unverifiable** | yes | 2 | missing | restore-missing-topology | -| `deleting-skill` | `77ed8cdf-1731-40bf-a1cc-26a5b45a737b` | `b3d49597` | 2 | **unverifiable** | yes | 0 | present | preserve-current-topology | -| `discord-support` | `fc907521-5566-41d7-acfa-e166b1350974` | `70675bd9` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `display-failed-automations` | `5f407300-a1d8-4d13-9ab0-d9f32ae7ee66` | `e1547367` | 2 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `e2e-automation` | `4425945e-9c22-437c-9629-b8443d0fcfad` | `a93a2bba` | 2 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `e2e-automation` | `744bf23a-d4be-4570-92b2-87214a887c00` | `bde9cf3c` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `enterprise-governance` | `51d7aea7-3927-4703-9380-7ed8274e34d0` | `7ee5502d` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `extension-planning` | `8af94eca-4174-4c23-9182-c7581479f741` | `0cb31d71` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `fix-notes-send-name-send-targets-after-their-tab` | `5eb4ce7d-491a-44cb-9109-feeebe956a49` | `1b189bf4` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `fix-notes-send-name-send-targets-after-their-tab` | `96a2653c-ab9d-44ee-bf68-beb04f9d0c13` | `3b9fd0c0` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `fix-notes-send-name-send-targets-after-their-tab` | `dc663961-f9e6-46b5-8e84-4d6897914348` | `6e7c347f` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `pure-extract-orca-runtime` | `b42751ef-5273-440c-871e-ba6d1331111a` | `bb0ce93b` | 1 | **exited** | no | 0 | missing | exclude-proven-exited | -| `pure-extract-orca-runtime` | `cca690e8-4b3e-4d3b-8230-85117385425e` | `bc7e0f51` | 1 | **exited** | no | 1 | missing | exclude-proven-exited | -| `pure-extract-orca-runtime` | `e0f9711c-fa77-495c-b239-a58276d50142` | `48533547` | 1 | **exited** | no | 0 | missing | exclude-proven-exited | -| `split-pty-connection` | `d782e028-017b-4288-bd6a-b6c0b66fd086` | `183c573b` | 1 | **exited** | no | 1 | missing | exclude-proven-exited | -| `split-pty-connection` | `fe1b69c8-41d3-469a-b699-42a27729d5a4` | `f0265ed6` | 1 | **exited** | no | 1 | missing | exclude-proven-exited | -| `split-task-page` | `1f120487-cfc4-4d3e-ac64-08e6411b1ab0` | `7c525cae` | 1 | **exited** | no | 1 | missing | exclude-proven-exited | -| `split-task-page` | `a37b588d-1b79-4cb4-b87d-c03ece690c45` | `75ea0008` | 1 | **exited** | no | 0 | missing | exclude-proven-exited | -| `split-task-page` | `dd0f15df-e203-4e51-a43f-9be0d1117e52` | `d3cfd89c` | 1 | **exited** | no | 0 | missing | exclude-proven-exited | -| `split-task-page` | `f275d8ca-12a1-4044-9c86-37ed6de988d8` | `4042761c` | 1 | **exited** | no | 1 | missing | exclude-proven-exited | -| `sta-4062-folder-note-rollback` | `1a7bd257-37b1-4362-af23-a183518414e7` | `11818262` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `sta-4062-folder-note-rollback` | `e7dfc74c-f23f-48b5-9221-a19393ed24c4` | `98c13247` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `sta-4062-folder-note-rollback` | `f66fd01a-e071-4482-9bc3-b675039f3330` | `41a4a575` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `sta-4064-phantom-folder-note` | `33dd7abb-2862-4771-be6b-9708205323f6` | `c4d72e5c` | 2 | **unverifiable** | yes | 0 | missing | restore-missing-topology | -| `sta-4064-phantom-folder-note` | `aa128462-b739-4657-a9bd-931e51640b2c` | `4b2345f4` | 1 | **unverifiable** | yes | 1 | missing | restore-missing-topology | -| `sta-4064-phantom-folder-note` | `dd6cbd36-106b-4a97-85f0-8afbe458bed6` | `2effc0c4` | 1 | **unverifiable** | yes | 0 | missing | restore-missing-topology | - -## Proven-exited sessions - -These 20 primary PTYs receive no restored records in the offline candidate. Records already present in the frozen current base remain untouched. Do not attempt raw PTY attachment; use a recorded provider session when available. - -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca-top-level-cleanup@@fa471831` — no provider recovery metadata in frozen evidence -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/1.4.183-p0-orca-crashed-all-agent-sessions@@8804d1a3` — provider recovery metadata available -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/add-more-e2e-tests@@ba88e6c5` — provider recovery metadata available -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/add-starting-agent-from-tab-e2e@@31c4cb12` — provider recovery metadata available -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/add-starting-agent-from-tab-e2e@@43f5e12a` — provider recovery metadata available -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/add-starting-agent-from-tab-e2e@@4ea40b01` — provider recovery metadata available -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/add-starting-agent-from-tab-e2e@@8972c618` — no provider recovery metadata in frozen evidence -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/allow-create-automations-with-ai@@49fbac2f` — no provider recovery metadata in frozen evidence -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/analytics@@5fb657da` — provider recovery metadata available -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/audit-default-settings@@55fddb5f` — provider recovery metadata available -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/auto-daily-prod-release-scan-run-18-20260821T1740@@a313bcea` — no provider recovery metadata in frozen evidence -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/pure-extract-orca-runtime@@bb0ce93b` — no provider recovery metadata in frozen evidence -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/pure-extract-orca-runtime@@bc7e0f51` — provider recovery metadata available -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/pure-extract-orca-runtime@@48533547` — no provider recovery metadata in frozen evidence -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/split-pty-connection@@183c573b` — provider recovery metadata available -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/split-pty-connection@@f0265ed6` — provider recovery metadata available -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/split-task-page@@7c525cae` — provider recovery metadata available -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/split-task-page@@75ea0008` — no provider recovery metadata in frozen evidence -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/split-task-page@@d3cfd89c` — no provider recovery metadata in frozen evidence -- `48e366bb-2436-46a9-bb37-10ace25ada5f::/Users/jinjingliang/Documents/projects/orca/split-task-page@@4042761c` — provider recovery metadata available - -## Artifact map - -- `RECOVERY_CATALOG.json`: authoritative per-session evidence and recommendations. -- `RECOVERY_CATALOG.md`: human-readable index and merge explanation. -- `orca-data.recovery-candidate.json`: offline candidate based on the frozen current profile. -- `RECOVERY_MERGE_VALIDATION.json`: merge invariants, counts, hashes, and cutover safety gate. -- `generate-recovery-artifacts.mjs`: deterministic generator; it reads only `evidence/`. diff --git a/recovery/2026-08-23-session-disappearance/cutover-20260823T201851Z/POST_CUTOVER_RECOVERY_STATUS.md b/recovery/2026-08-23-session-disappearance/cutover-20260823T201851Z/POST_CUTOVER_RECOVERY_STATUS.md deleted file mode 100644 index 642dfb88717..00000000000 --- a/recovery/2026-08-23-session-disappearance/cutover-20260823T201851Z/POST_CUTOVER_RECOVERY_STATUS.md +++ /dev/null @@ -1,170 +0,0 @@ -# Post-cutover recovery status - -> Superseded on 2026-08-23 14:12 MST by recovery-v2. The v1 status below is retained as -> historical evidence. Recovery-v2 `1.4.189-adhoc.20260823204916` is installed, daemon PID -> `26391` survived, all 131 complete tracked incident topologies remained intact, and one -> provider-backed affected session was recovered in place without changing its core tab/layout/PTY -> binding. See -> `../cutover-v2-20260823T140110MST/RECOVERY_V2_CUTOVER_STATUS.md` for the current status. - -Historical v1 snapshot updated: 2026-08-23 13:37 MST - -## Bottom line - -The guarded build prevents one false-exit path, but the terminal-session recovery is only partial -and a second destructive path remains. The renderer can clear an old PTY binding after an -owner-unverified attach error and then request a fresh shell, despite displaying a message that -the session was left untouched. Most affected panes remain `unverifiable`; do not open them until -the renderer reattach guard is installed. - -Do not roll back the app, restart or kill daemon PID `26391`, restore the quarantined ShipIt -cache, or open affected panes. The installed guarded build is safe for the liveness probe but is -not yet safe for an owner-unverified reattach attempt. - -## Live runtime - -| Item | Observation | -|---|---| -| App | `1.4.189-adhoc.20260823201001` | -| App PID | `65542` | -| Runtime ID | `f73694d1-31b5-4abb-bc86-3123aae61783` | -| Runtime / graph | `ready` / `ready` | -| Surviving daemon | PID `26391`, started 2026-08-23 11:32:59 MST | -| Daemon build metadata | `1.4.189-hourly.202608231007`, protocol v36 | -| Dynamic terminal inventory at last check | 62 panes | - -The terminal inventory count is dynamic and is not a recovery score. At the last check it still -omitted all paired/SSH hosts except `local`, and absence from the list is not evidence of exit. - -## Incident recovery accounting - -The frozen incident contains 138 primary sessions classified `unverifiable`, spanning 51 -worktrees and 167 pane PTY IDs. - -| Recovery state | Count | -|---|---:| -| Incident worktrees still tracked by Orca | 48 of 51 | -| Incident sessions in those tracked worktrees | 131 of 138 | -| Exact original PTY topology still present in the live profile | 130 of 138 | -| Original incident panes directly listed, connected, and readable | 3 panes in 1 worktree | -| Tracked sessions with a saved provider-session recovery record | 73 of 131 | -| Tracked sessions with preserved terminal history | 131 of 131 | -| Provider resume attempts performed | 1 | -| Successful provider resumes | 1 | - -The three worktrees no longer tracked by Orca are: - -- `create-scanning-issues` -- `sta-4062-folder-note-rollback` -- `sta-4064-phantom-folder-note` - -They account for seven of the eight non-exact records. Their directories are absent, so removal -of their unified-tab reachability is consistent with missing-worktree cleanup rather than a new -false-exit verdict. Their frozen recovery evidence remains intact. - -The eighth record is `deleting-skill`. Its two incident PTY bindings had already been replaced by -new blank-shell PTYs at 2026-08-23 12:13 MST, before the guarded cutover. The frozen catalog and -history still preserve its original primary PTY `@@b3d49597`; the current blank shells are not a -successful recovery of that session. - -## Directly live incident panes - -All three directly listed incident panes belong to -`auto-daily-prod-release-scan-run-18-20260821T1740`: - -- `@@04230ccf` -- `@@fde1083b` -- `@@782038ec` - -Each is connected, writable, and returned its original terminal screen. This proves that some -raw PTYs survived and reattached successfully, but it does not establish the status of the other -incident PTYs. - -## Representative provider-session recovery - -The tracked worktree `close-tab-on-mobile-back-to-prev` had two preserved incident tabs but zero -terminals in the live runtime graph. One tab had a saved Codex provider session: - -```text -01a0068d-2be5-77b1-8a9e-cc7cda073fd2 -``` - -The transcript still existed at its recorded path. A new Orca terminal was created in the same -worktree with: - -```text -codex resume 01a0068d-2be5-77b1-8a9e-cc7cda073fd2 -``` - -Result: - -- New tab ID: `999334bc-5707-4631-909a-29c433d54414` -- New PTY: `@@1f02bf06` -- Runtime handle: `term_044c404a-b786-4ffe-93d8-98f39405b924` -- State: `live`, connected, writable, TUI idle -- Original prompt and conversation were visible -- No new prompt was sent - -This is a successful provider continuation test. It does not revive the old raw PTY; it resumes -the provider conversation in a new terminal while leaving the uncertain old record preserved. -Because the raw PTY is still `unverifiable`, the new tab must not receive work until the old route -is proven `exited` or the user explicitly chooses to abandon that route. The test tab is idle and -received no new prompt. - -## Why clicking the worktrees was still destructive - -The installed patch changes only the liveness verdict used before visibility-resume -reconciliation: `null` remains `unverifiable` instead of becoming `false`/`exited`. When the old -daemon route cannot prove ownership, stable-pane attach throws -`terminal_pane_owner_unverified`. Main correctly avoids retiring persistence and the renderer -shows the retry message. - -The renderer transport then converts that error into `onError(marker)` plus an undefined spawn -result. The connection layer treats every undefined reattach result as stale/dead, clears the leaf -binding and tab PTY ID, and starts a fresh spawn. Those fields are persisted. A later reopen may -therefore create a blank shell because the exact old ID has already been removed. - -The required second guard is an explicit owner-unverified reattach outcome that reports the -toast but does not clear `terminalLayoutsByTabId`, `ptyIdsByTabId`, or the tab PTY ID and does not -fresh-spawn. Generic undefined reattach failures must keep their current behavior. - -Relevant source boundaries at the guarded branch head: - -- `src/main/ipc/pty.ts:878`: stable owner attach uses `attachOnly: true`. -- `src/main/ipc/pty.ts:900`: owner-unverified is translated without retiring persistence. -- `src/renderer/src/components/terminal-pane/pty-transport.ts:946`: spawn rejection becomes - `onError(...)` plus an undefined result. -- `src/renderer/src/components/terminal-pane/pty-connection.ts:8273`: undefined reattach is treated - as stale, clears bindings, and starts fresh spawn. -- `src/renderer/src/components/terminal-pane/TerminalPane.tsx:1133`: leaf layout binding clear. -- `src/renderer/src/store/slices/terminals.ts:2474`: tab PTY/live-index clear. -- `src/renderer/src/store/session-write-subscriber.ts:180`: the changed session model is persisted. - -## Safe next steps - -1. Keep the current app and daemon PID `26391` running, but do not open affected panes. -2. Install a second guarded build that preserves bindings on - `terminal_pane_owner_unverified`; validate it against a copied profile before resuming recovery. -3. Handle one provider claim at a time. Let guarded in-place attach win when it returns `live`. - Resume into a new tab only after the old PTY is authoritatively `exited`, or after an explicit - user decision to abandon the `unverifiable` raw route. Never send prompts automatically. -4. Export terminal checkpoints for the 58 tracked sessions without provider records. These can - recover screen/history but cannot reconstruct arbitrary shell process state. -5. Recreate a deleted worktree only when its Git identity and desired base are independently - known; never infer that identity from a PTY suffix. -6. Do not batch-resume `unverifiable` sessions. Before any approved recovery batch, create another - consistent profile/evidence snapshot and use an idempotent ledger so retries do not duplicate - resumed sessions. - -## Rollback and preserved evidence - -| Item | Location | -|---|---| -| Previous hourly app | `/Applications/.Orca.pre-recovery-20260823T201851Z.bundle` | -| Quarantined ShipIt state | `ShipIt.disabled/` in this cutover directory | -| Pre-cutover profile | `profiles.pre-cutover/local-default/` | -| Post-launch validation | `post-launch-profile-validation/RECOVERY_MERGE_VALIDATION.json` | -| Frozen incident catalog | `../../RECOVERY_CATALOG.json` | - -Do not restore the old app and profile independently. Any rollback must treat the app bundle, -profile, and ShipIt state as one controlled operation. diff --git a/recovery/2026-08-23-session-disappearance/cutover-v2-20260823T140110MST/RECOVERY_V2_CUTOVER_STATUS.md b/recovery/2026-08-23-session-disappearance/cutover-v2-20260823T140110MST/RECOVERY_V2_CUTOVER_STATUS.md deleted file mode 100644 index b2a12ced600..00000000000 --- a/recovery/2026-08-23-session-disappearance/cutover-v2-20260823T140110MST/RECOVERY_V2_CUTOVER_STATUS.md +++ /dev/null @@ -1,220 +0,0 @@ -# Recovery-v2 cutover and first controlled recovery - -Updated: 2026-08-23 14:12 MST - -## Bottom line - -Recovery-v2 is installed and running. The controlled upgrade preserved the surviving hourly -daemon, all 131 complete affected topologies that remain in tracked worktrees, and the exact -state-schema/protocol boundary. One affected local Codex session was then recovered in place: its -tab, layout, leaf, and logical PTY ID stayed unchanged, while Orca created one new daemon session -under that identity and resumed the saved provider conversation. The terminal is connected, -writable, and TUI-idle. No prompt was sent. - -This proves that provider-backed recovery works on v2 for this incident shape. It does not prove -that every raw PTY survived, and it does not reconstruct arbitrary shell process state. Sessions -without provider metadata still depend on their preserved terminal history when the raw PTY is -authoritatively `exited`. - -## Root-cause implications - -This cutover is a control case against attributing the bulk incident to daemon PID `6835` stopping. -The newer v2 app retained the still-live hourly daemon PID `26391` across the bundle replacement, -despite the daemon's older build metadata, while preserving all 131 complete affected topologies -in tracked worktrees. Same-protocol app/daemon version skew is therefore supported and is not by -itself destructive. - -PID `6835` becoming unavailable could strand raw PTYs that it actually owned, but absence from the -replacement daemon or from the current adapter registry is only a routing/authority failure. It -does not prove that another daemon generation's session exited. The frozen catalog reinforces this: -only 5 of 138 unverifiable primary PTYs have their latest event from `6835`; the other 133 have -their latest event from other daemon PIDs. - -The first controlled recovery also separates logical recovery from raw-process survival. Orca -kept the old tab, leaf, layout, and logical PTY ID, but created a new session incarnation and -resumed the saved provider conversation. That is successful provider continuation, not -reattachment to or resurrection of the old PTY. Durable raw-session continuity still requires -exact daemon/session-incarnation routing and, across a true owner crash, a surviving holder of the -PTY master. - -## Current live state - -| Item | Observation | -|---|---| -| Installed app | `1.4.189-adhoc.20260823204916` | -| Build commit | `bd09097708c9` | -| Build ID | `1.4.189-adhoc.20260823204916-bd09097708c9-arm64` | -| App PID | `27511` | -| Runtime ID | `f5c43687-8851-4f0b-8222-1b4292caa125` | -| Runtime / graph | `ready` / `ready` | -| Surviving daemon | PID `26391`, unchanged since 2026-08-23 11:32:59 MST | -| Daemon build metadata | `1.4.189-hourly.202608231007`, protocol v36 | -| ShipIt | Still quarantined; no active `com.stablyai.orca.ShipIt` cache | -| Dynamic worktree inventory | 200 tracked worktrees, 72 live panes, 21 attached worktrees | - -The live pane count is diagnostic only. Its change from 75 before cutover, to 71 immediately after -launch, to 72 after the controlled recovery is not a terminal verdict. - -## Build verification - -The arm64 release asset was downloaded from -`v1.4.189-adhoc.20260823204916` and independently verified before installation. - -| Gate | Result | -|---|---| -| DMG SHA-256 | `b20fdafd6d4c17fdce8fa10ed6e9059786c454eeb956acb1b8e107ad94f9944a` | -| GitHub asset digest match | Pass | -| Version / commit | `1.4.189-adhoc.20260823204916` / `bd09097708c9` | -| Architecture | Thin arm64 Mach-O | -| Strict deep code-sign verification | Pass | -| Signing identity | Developer ID Application: Lovecast LLC | -| Team ID | `6CX3WHS9HZ` | -| Gatekeeper | Accepted, Notarized Developer ID | -| Notarization staple | Valid | -| State schema | 1; readable versions `[1]` | -| Daemon protocol | 36; attachable versions 1 through 36 | - -The DMG container itself is unsigned, matching the earlier v1 recovery DMG. The app inside is the -signed, notarized, stapled trust boundary and passed every application gate. - -## Cutover sequence - -1. Captured app status, all worktree process summaries, the live profile, its SHA-256, and a - deterministic recovery merge validation. -2. Staged the verified app at - `/Applications/.Orca.recovery-v2-20260823T140110MST.bundle` and re-ran strict code-sign - verification on the staged copy. -3. Normally quit v1 GUI PID `65542`. Daemon PID `26391` survived. -4. Before the bundle swap, the same external-launch race from the incident reproduced: the old v1 - bundle reopened as PID `96491` at 14:04:49 MST. No app swap had occurred. -5. Recorded that relaunch, normally quit PID `96491`, and immediately moved the old bundle out of - `/Applications/Orca.app`. This removed the external actor's old-version launch target. -6. Captured a second, isolated post-quit profile. Its SHA-256 was - `9bdd64cbfa764021149d281b8390797ed214916064628035db83b950c172d347`; the recovery - validation used that exact hash and passed. -7. Atomically moved v2 into `/Applications/Orca.app` and launched the exact path. -8. Required the new version/runtime ID, `ready` runtime and graph, surviving daemon PID `26391`, - unchanged incident topology counts, and continued ShipIt quarantine before recovery testing. - -The prior v1 app remains recoverable at: - -```text -/Applications/.Orca.pre-recovery-v2-20260823T140110MST.bundle -``` - -Do not run that backup against the current profile. Any rollback must restore the app and matching -profile snapshot together, with ShipIt still quarantined. - -## Profile preservation result - -The deterministic validation returned the same result immediately before and after v2 launch: - -| Check | Before v2 launch | After v2 launch | -|---|---:|---:| -| Validation | pass | pass | -| Current layouts | 1001 | 1001 | -| Complete current incident topologies | 131 | 131 | -| Incident layouts restored by the candidate | 0 | 0 | -| Exited records retained because current state still references them | 3 | 3 | -| Proven-exited records excluded | 17 | 17 | -| Missing-worktree unified tabs represented only in the offline candidate | 7 | 7 | - -The seven candidate-only records belong to the three previously removed worktrees. The offline -candidate was validated but was not applied; re-adding those rows while their directories are absent -would only invite missing-worktree cleanup to remove them again. - -## First controlled recovery - -### Target - -| Item | Value | -|---|---| -| Worktree | `auto-e2e-tests-autofix-scheduled-ci-1h-run-9-20260822T0700` | -| Tab ID | `d6d43b87-9d6b-4c48-87cc-641719ecab8e` | -| Leaf ID | `4d1848a5-e6ae-4108-b59e-e049791b70d9` | -| Logical PTY ID | `@@a95190fc` under the full worktree prefix | -| Saved provider | Codex session `01a02a6a-01fc-74c2-b77a-1d1f674d66e0` | -| Historical verdict | `unverifiable`; latest frozen event was `session-created` on 2026-08-22 | -| Pre-activation runtime inventory | Zero terminal handles for this worktree | - -The worktree had one affected pane, complete topology, a 69,916-byte frozen terminal-history record, -and a saved Codex provider session. It was selected to keep the live test isolated to exactly one -affected session. - -### What happened - -Activating the worktree caused Orca's normal cold-wake path to recover the provider session in the -preserved pane. The current daemon did not contain the old raw session, so it created one new session -under the same logical PTY identity and ran the recorded Codex resume command. - -Evidence: - -- Daemon `session-created` at `2026-08-23T21:09:32.690Z` for the exact old logical PTY ID. -- New process PID `88082`, parent daemon PID `26391`. -- New incarnation ID `761a724d-5d78-42c2-946b-8dd962f5a217`. -- Runtime handle `term_88f38ceb-d943-4c3c-8b36-b844616ad603`. -- `connected: true`, `writable: true`, and `terminal wait --for tui-idle` satisfied. -- The original conversation appeared and the provider session resumed. -- No prompt or terminal input was sent. - -The core persistence object—layout, legacy tab, unified tab, leaf binding, and logical PTY ID—was -byte-identical before and after activation. The only expected persistence change in the captured -object was removal of the consumed sleeping/provider record. This was not a blank replacement tab. - -### Important interpretation - -This probe exercised authoritative-missing cold recovery, not the -`terminal_pane_owner_unverified` error branch. A fresh daemon session was correct for this verdict -because the current daemon authoritatively lacked the old session and a provider continuation was -available. The v2 owner-unverified branch remains covered by the focused 10-test renderer suite: it -keeps the old binding and does not fresh-spawn when ownership cannot be decided. - -Do not conflate these outcomes: - -- `live`: attach the existing raw PTY. -- `exited`: recover via provider continuation or preserved history; creating a new PTY is expected. -- `unverifiable`: keep the binding untouched and do not start a replacement solely from loss of - contact. - -The frozen incident verdict remains `unverifiable` because historical logs cannot prove what -happened to the old process. The live current-daemon recovery decision had stronger present-tense -evidence: that daemon did not own the session and could safely cold-recover it. - -## What recovery can and cannot restore - -- Provider-backed tabs can restore the conversation in the same visual tab, as this probe did. -- A provider resume does not resurrect the old raw PTY process; it starts a new PTY and asks the - provider CLI to continue its saved session. -- Plain shell tabs with no provider record can restore scrollback/history, but arbitrary shell - process state cannot be reconstructed after an authoritative exit. -- An owner-unverified pane must remain untouched. v2 now enforces that rule in the renderer as well - as main. -- Remote and SSH panes require execution-host authority. Loss of contact is never evidence of exit. - -## Safe continuation - -1. Recover one worktree at a time. Do not open dozens simultaneously; each provider continuation - consumes a PTY and may resume a previously active coordination session. -2. After opening a worktree, verify the old tab remains present and classify the result: - existing raw attachment, provider continuation, preserved-but-unverifiable, or history-only. -3. Do not send a prompt automatically. Leave recovered agents idle until their old ownership and - orchestration context have been reviewed. -4. If Orca shows the owner-unverified message, close/reopen only after recording the tab/leaf/PTY - identity. On v2 the binding should remain unchanged and no blank shell should replace it. -5. Keep daemon PID `26391` and the ShipIt quarantine intact while recovery continues. -6. Do not batch-resume frozen provider claims. The live provider record is a recovery capability, - not proof that its old raw process exited. - -## Evidence index - -| Evidence | Path | -|---|---| -| Pre-v2 status/worktrees/profile | `pre-status.json`, `pre-worktrees.json`, `orca-data.pre-v2.json` | -| Unexpected v1 relaunch | `unexpected-v1-relaunch-status.json`, `unexpected-v1-relaunch-process.txt` | -| Final isolated profile | `orca-data.post-second-quit.json`, `post-second-quit-profile.sha256` | -| Isolated merge validation | `post-second-quit-profile-validation/` | -| Post-v2 launch state | `post-v2-status.json`, `post-v2-worktrees.json`, `orca-data.post-v2-launch.json` | -| Post-v2 merge validation | `post-v2-profile-validation/` | -| Controlled recovery probe | `owner-unverified-probe/` | -| State after first recovery | `final-status.json`, `final-worktrees.json`, `orca-data.after-first-recovery.json` | -| Signed build artifact | `../guarded-build-v2/orca-macos-arm64.dmg` | diff --git a/src/main/agent-hooks/restored-subagent-liveness-sweep.test.ts b/src/main/agent-hooks/restored-subagent-liveness-sweep.test.ts index 9811773eb07..7a9f55000cb 100644 --- a/src/main/agent-hooks/restored-subagent-liveness-sweep.test.ts +++ b/src/main/agent-hooks/restored-subagent-liveness-sweep.test.ts @@ -81,21 +81,32 @@ async function restartWithInFlightSubagent(options?: { function sweepWith( server: AgentHookServer, overrides: { - probeLiveLocalPty?: (ptyId: string) => boolean | null | Promise + probeLiveLocalPty?: ( + ptyId: string, + expectedIncarnationId: string + ) => boolean | null | Promise executionHostId?: string | null boundPtyIdByPaneKey?: Record persistedPtyIdByPaneKey?: Record + persistedPtyIncarnationByPaneKey?: Record } = {} ): Promise { return sweepRestoredSubagentsWithoutLiveAgent({ - probeLiveLocalPty: async (ptyId) => - overrides.probeLiveLocalPty ? await overrides.probeLiveLocalPty(ptyId) : false, + probeLiveLocalPty: async (ptyId, expectedIncarnationId) => + overrides.probeLiveLocalPty + ? await overrides.probeLiveLocalPty(ptyId, expectedIncarnationId) + : false, isLocalExecutionHost: () => isLocalExecutionHost( overrides.executionHostId === undefined ? 'local' : overrides.executionHostId ), getBoundPtyIdForPaneKey: (paneKey) => overrides.boundPtyIdByPaneKey?.[paneKey], getPersistedPtyIdForPaneKey: (paneKey) => overrides.persistedPtyIdByPaneKey?.[paneKey], + getPersistedPtyIncarnationForPaneKey: (paneKey) => + overrides.persistedPtyIncarnationByPaneKey?.[paneKey] ?? + (overrides.persistedPtyIdByPaneKey?.[paneKey] || overrides.boundPtyIdByPaneKey?.[paneKey] + ? 'incarnation-1' + : undefined), reap: (isLocalHost, isLocalPaneAgentLive, isLocalPaneLivenessEvidenceCurrent) => server.reapRestoredClaudeSubagentsWithoutLiveAgent( isLocalHost, @@ -186,7 +197,7 @@ describe('restored subagent liveness sweep', () => { }) ).toBe(0) - expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY) + expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY, 'incarnation-1') expect(paneStatus(server).state).toBe('working') } finally { server.stop() @@ -205,7 +216,7 @@ describe('restored subagent liveness sweep', () => { }) ).toBe(2) - expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY) + expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY, 'incarnation-1') } finally { server.stop() } @@ -373,7 +384,9 @@ describe('restored subagent liveness sweep', () => { ) try { const sweep = sweepWith(server, { probeLiveLocalPty, boundPtyIdByPaneKey }) - await vi.waitFor(() => expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY)) + await vi.waitFor(() => + expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY, 'incarnation-1') + ) boundPtyIdByPaneKey[PANE] = 'wt-1__pty-new' resolveProbe(false) @@ -400,7 +413,9 @@ describe('restored subagent liveness sweep', () => { boundPtyIdByPaneKey, persistedPtyIdByPaneKey: { [PANE]: PTY } }) - await vi.waitFor(() => expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY)) + await vi.waitFor(() => + expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY, 'incarnation-1') + ) boundPtyIdByPaneKey[PANE] = PTY resolveProbe(false) @@ -411,6 +426,35 @@ describe('restored subagent liveness sweep', () => { } }) + it('keeps a same-id replacement whose persisted incarnation changes during the probe', async () => { + const server = await restartWithInFlightSubagent() + const persistedPtyIncarnationByPaneKey = { [PANE]: 'incarnation-a' } + let resolveProbe!: (live: boolean | null) => void + const probeLiveLocalPty = vi.fn( + () => + new Promise((resolve) => { + resolveProbe = resolve + }) + ) + try { + const sweep = sweepWith(server, { + probeLiveLocalPty, + persistedPtyIdByPaneKey: { [PANE]: PTY }, + persistedPtyIncarnationByPaneKey + }) + await vi.waitFor(() => + expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY, 'incarnation-a') + ) + persistedPtyIncarnationByPaneKey[PANE] = 'incarnation-b' + resolveProbe(false) + + expect(await sweep).toBe(0) + expect(paneStatus(server).state).toBe('working') + } finally { + server.stop() + } + }) + it('keeps a same-id rebind that lands after its probe but before the batch settles', async () => { const otherPane = makePaneKey('tab-2', LEAF) const otherPty = 'wt-1__pty-2' diff --git a/src/main/agent-hooks/restored-subagent-liveness-sweep.ts b/src/main/agent-hooks/restored-subagent-liveness-sweep.ts index 3402ab7bf5b..8b1c5d558e6 100644 --- a/src/main/agent-hooks/restored-subagent-liveness-sweep.ts +++ b/src/main/agent-hooks/restored-subagent-liveness-sweep.ts @@ -11,13 +11,14 @@ import { parseWorkspaceKey } from '../../shared/workspace-scope' export type RestoredSubagentLivenessSweepDeps = { /** Targeted provider liveness, or null when the provider cannot prove either state. */ - probeLiveLocalPty: (ptyId: string) => Promise + probeLiveLocalPty: (ptyId: string, expectedIncarnationId: string) => Promise isLocalExecutionHost: (worktreeId: string | undefined) => boolean /** PTY bound to this pane in the current session, if it has one. */ getBoundPtyIdForPaneKey: (paneKey: string) => string | undefined /** PTY this pane was bound to when the session was last persisted; covers panes * whose surviving daemon session has not been reattached yet. */ getPersistedPtyIdForPaneKey: (paneKey: string) => string | undefined + getPersistedPtyIncarnationForPaneKey: (paneKey: string) => string | undefined reap: ( isLocalExecutionHost: (worktreeId: string | undefined) => boolean, isLocalPaneAgentLive: (paneKey: string) => Promise, @@ -30,34 +31,44 @@ export type RestoredSubagentLivenessSweepDeps = { export async function sweepRestoredSubagentsWithoutLiveAgent( deps: RestoredSubagentLivenessSweepDeps ): Promise { - const probesByPtyId = new Map>() + const probesByOwner = new Map>() const boundPtyIdAtProbeByPaneKey = new Map() + const incarnationAtProbeByPaneKey = new Map() return await deps.reap( (worktreeId) => deps.isLocalExecutionHost(worktreeId), async (paneKey) => { const boundPtyId = deps.getBoundPtyIdForPaneKey(paneKey) boundPtyIdAtProbeByPaneKey.set(paneKey, boundPtyId) const ptyId = boundPtyId ?? deps.getPersistedPtyIdForPaneKey(paneKey) - if (!ptyId) { + const incarnationId = deps.getPersistedPtyIncarnationForPaneKey(paneKey) + incarnationAtProbeByPaneKey.set(paneKey, incarnationId) + if (!ptyId || !incarnationId) { return true } try { - let probe = probesByPtyId.get(ptyId) + const ownerKey = `${ptyId}\0${incarnationId}` + let probe = probesByOwner.get(ownerKey) if (!probe) { - probe = deps.probeLiveLocalPty(ptyId) - probesByPtyId.set(ptyId, probe) + probe = deps.probeLiveLocalPty(ptyId, incarnationId) + probesByOwner.set(ownerKey, probe) } const live = await probe const currentBoundPtyId = deps.getBoundPtyIdForPaneKey(paneKey) // Why: cold restore can rebind the persisted id while its absence probe is in flight. - return currentBoundPtyId !== boundPtyId || live !== false + return ( + currentBoundPtyId !== boundPtyId || + deps.getPersistedPtyIncarnationForPaneKey(paneKey) !== incarnationId || + live !== false + ) } catch { return true } }, (paneKey) => !boundPtyIdAtProbeByPaneKey.has(paneKey) || - deps.getBoundPtyIdForPaneKey(paneKey) === boundPtyIdAtProbeByPaneKey.get(paneKey) + (deps.getBoundPtyIdForPaneKey(paneKey) === boundPtyIdAtProbeByPaneKey.get(paneKey) && + deps.getPersistedPtyIncarnationForPaneKey(paneKey) === + incarnationAtProbeByPaneKey.get(paneKey)) ) } diff --git a/src/main/daemon/daemon-errors.ts b/src/main/daemon/daemon-errors.ts index 12136d37a5b..22e815b8b72 100644 --- a/src/main/daemon/daemon-errors.ts +++ b/src/main/daemon/daemon-errors.ts @@ -68,6 +68,13 @@ export class TerminalSessionOwnerUnverifiedError extends Error { } } +export class TerminalSessionExitedError extends Error { + constructor(sessionId: string) { + super(`Terminal session exited: ${sessionId}`) + this.name = 'TerminalSessionExitedError' + } +} + export class TerminalHostGoneError extends Error { constructor() { super('terminal_host_gone') diff --git a/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts b/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts index 142d1706f9c..82ce0e1c423 100644 --- a/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts +++ b/src/main/daemon/daemon-foreground-confirmation-protocol.test.ts @@ -3,7 +3,7 @@ import { PREVIOUS_DAEMON_PROTOCOL_VERSIONS, PROTOCOL_VERSION } from './types' describe('foreground-confirmation daemon protocol', () => { it('rejects daemons from before the fresh-confirmation RPC', () => { - expect(PROTOCOL_VERSION).toBe(36) + expect(PROTOCOL_VERSION).toBe(37) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(19) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(22) expect(PREVIOUS_DAEMON_PROTOCOL_VERSIONS).toContain(23) diff --git a/src/main/daemon/daemon-protocol-version.ts b/src/main/daemon/daemon-protocol-version.ts index bae2ae972dc..28bcfa0f70a 100644 --- a/src/main/daemon/daemon-protocol-version.ts +++ b/src/main/daemon/daemon-protocol-version.ts @@ -1,6 +1,7 @@ // Why: daemons survive app updates, so wire behavior must be version-gated. -// v36 launches shells from content-addressed wrapper trees; older owners stay attachable. -export const PROTOCOL_VERSION = 36 +// v37 fences attach-only requests by exact session incarnation before side effects. +export const PROTOCOL_VERSION = 37 +export const SESSION_INCARNATION_ATTACH_FENCE_DAEMON_PROTOCOL_VERSION = 37 export const CONTENT_ADDRESSED_SHELL_WRAPPER_DAEMON_PROTOCOL_VERSION = 36 export const ASYNC_CWD_VALIDATION_DAEMON_PROTOCOL_VERSION = 35 export const CODEX_SHELL_LAUNCH_PREFLIGHT_DAEMON_PROTOCOL_VERSION = 34 @@ -30,7 +31,7 @@ export const CLEAN_DISCONNECT_PROTOCOL_VERSION = 24 export const MODE_2031_UNSUBSCRIBE_FACT_PROTOCOL_VERSION = 29 export const PREVIOUS_DAEMON_PROTOCOL_VERSIONS = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, - 28, 29, 30, 31, 32, 33, 34, 35 + 28, 29, 30, 31, 32, 33, 34, 35, 36 ] as const export function supportsPtyStartupIngress(protocolVersion: number): boolean { diff --git a/src/main/daemon/daemon-pty-adapter-protocol-compatibility.test.ts b/src/main/daemon/daemon-pty-adapter-protocol-compatibility.test.ts index 0ee95478743..ad964da1c85 100644 --- a/src/main/daemon/daemon-pty-adapter-protocol-compatibility.test.ts +++ b/src/main/daemon/daemon-pty-adapter-protocol-compatibility.test.ts @@ -464,7 +464,7 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { expect(adapter.hasPty(id)).toBe(false) await expect(adapter.probePtyLiveness(id)).resolves.toBe(true) - await expect(adapter.probePtyLiveness('missing-session')).resolves.toBe(false) + await expect(adapter.probePtyLiveness('missing-session')).resolves.toBeNull() }) it('returns unknown when the daemon cannot answer', async () => { @@ -478,16 +478,27 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { await expect(adapter.probePtyLiveness('session')).resolves.toBeNull() }) - function createProbeAdapter( - protocolVersion: number, - request: ReturnType - ): DaemonPtyAdapter { + type DaemonRequest = ( + type: string, + payload: unknown, + timeoutMs?: number, + signal?: AbortSignal + ) => Promise + + function createProbeAdapter(protocolVersion: number, request: DaemonRequest): DaemonPtyAdapter { const probeAdapter = new DaemonPtyAdapter({ socketPath, tokenPath, protocolVersion }) - ;( - probeAdapter as unknown as { - client: { request: ReturnType; disconnect: ReturnType } - } - ).client = { request, disconnect: vi.fn() } + const client = (probeAdapter as unknown as { client: DaemonClient }).client + vi.spyOn(client, 'request').mockImplementation(request) + vi.spyOn( + probeAdapter as unknown as { ensureConnected: () => Promise }, + 'ensureConnected' + ).mockImplementation(async () => { + ;( + probeAdapter as unknown as { + lastAuthenticatedIdentity: { pid: number; startedAtMs: number; launchNonce: string } + } + ).lastAuthenticatedIdentity = { pid: 42, startedAtMs: 1, launchNonce: 'probe-owner' } + }) return probeAdapter } @@ -510,8 +521,8 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { const legacy = createProbeAdapter(GET_SIZE_PROTOCOL_VERSION - 1, request) await expect(legacy.probePtyLiveness('legacy-live')).resolves.toBe(true) - await expect(legacy.probePtyLiveness('legacy-exited')).resolves.toBe(false) - await expect(legacy.probePtyLiveness('never-existed')).resolves.toBe(false) + await expect(legacy.probePtyLiveness('legacy-exited')).resolves.toBeNull() + await expect(legacy.probePtyLiveness('never-existed')).resolves.toBeNull() expect(request).toHaveBeenCalledWith('listSessions', undefined, LIVENESS_PROBE_TIMEOUT_MS) expect(request).not.toHaveBeenCalledWith('getSize', expect.anything()) @@ -543,7 +554,7 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { const ambiguous = createProbeAdapter(GET_SIZE_PROTOCOL_VERSION, request) await expect(ambiguous.probePtyLiveness('ambiguous-live')).resolves.toBe(true) - await expect(ambiguous.probePtyLiveness('never-existed')).resolves.toBe(false) + await expect(ambiguous.probePtyLiveness('never-existed')).resolves.toBeNull() // The rejection is remembered, so later probes skip the round trip that cannot work. expect(request.mock.calls.filter(([type]) => type === 'getSize')).toHaveLength(1) diff --git a/src/main/daemon/daemon-pty-adapter-session-adoption.test.ts b/src/main/daemon/daemon-pty-adapter-session-adoption.test.ts index 8cff91f1907..1993fc0150b 100644 --- a/src/main/daemon/daemon-pty-adapter-session-adoption.test.ts +++ b/src/main/daemon/daemon-pty-adapter-session-adoption.test.ts @@ -173,6 +173,60 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { expect(lastSubprocess).toBe(subprocessBeforeAttach) }) + it('forwards an authoritative expected incarnation to a current daemon', async () => { + const created = await adapter.spawn({ + cols: 80, + rows: 24, + sessionId: 'incarnation-fenced-session' + }) + const request = vi.spyOn(DaemonClient.prototype, 'request') + + await expect( + adapter.spawn({ + cols: 80, + rows: 24, + sessionId: created.id, + attachOnly: true, + expectedIncarnationId: created.incarnationId, + expectedIncarnationIsAuthoritative: true + }) + ).resolves.toMatchObject({ + id: created.id, + incarnationId: created.incarnationId, + isReattach: true + }) + + expect(request).toHaveBeenCalledWith( + 'createOrAttach', + expect.objectContaining({ + attachOnly: true, + expectedIncarnationId: created.incarnationId + }) + ) + request.mockRestore() + }) + + it('does not send an incarnation-fenced attach to a pre-fence daemon', async () => { + const request = vi.spyOn(DaemonClient.prototype, 'request') + const legacy = new DaemonPtyAdapter({ socketPath, tokenPath, protocolVersion: 36 }) + try { + await expect( + legacy.spawn({ + cols: 80, + rows: 24, + sessionId: 'legacy-owner-session', + attachOnly: true, + expectedIncarnationId: 'expected-incarnation', + expectedIncarnationIsAuthoritative: true + }) + ).rejects.toMatchObject({ name: 'TerminalSessionOwnerUnverifiedError' }) + expect(request).not.toHaveBeenCalled() + } finally { + legacy.dispose() + request.mockRestore() + } + }) + it('does not inspect cold history for attach-only ownership checks', async () => { const historyDir = join(dir, 'attach-only-history') const historyAdapter = new DaemonPtyAdapter({ diff --git a/src/main/daemon/daemon-pty-adapter-subscription-fanout.ts b/src/main/daemon/daemon-pty-adapter-subscription-fanout.ts index 1d82ef70b7b..b4ac69efe37 100644 --- a/src/main/daemon/daemon-pty-adapter-subscription-fanout.ts +++ b/src/main/daemon/daemon-pty-adapter-subscription-fanout.ts @@ -10,7 +10,7 @@ export class DaemonPtyAdapterSubscriptionFanout { constructor( private readonly adapters: readonly DaemonPtyAdapter[], - onAdapterExit: (id: string) => void, + onAdapterExit: (adapter: DaemonPtyAdapter, id: string) => void, onAdapterIdentityChanged?: (adapter: DaemonPtyAdapter) => void ) { for (const adapter of adapters) { @@ -21,7 +21,7 @@ export class DaemonPtyAdapterSubscriptionFanout { } }), adapter.onExit((payload) => { - onAdapterExit(payload.id) + onAdapterExit(adapter, payload.id) for (const listener of this.exitListeners) { listener(payload) } diff --git a/src/main/daemon/daemon-pty-adapter.test.ts b/src/main/daemon/daemon-pty-adapter.test.ts index ef5fa687586..6c73892813a 100644 --- a/src/main/daemon/daemon-pty-adapter.test.ts +++ b/src/main/daemon/daemon-pty-adapter.test.ts @@ -712,6 +712,32 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { cause: { kind: 'exited', exitCode: 42 } }) }) + + it('ignores an incarnation-less exit for an incarnation-fenced session', async () => { + const exits: { id: string; code: number; incarnationId?: string }[] = [] + adapter.onExit((payload) => exits.push(payload)) + const spawned = await adapter.spawn({ cols: 80, rows: 24 }) + const client = (adapter as unknown as { client: DaemonClient }).client + const eventListeners = ( + client as unknown as { + eventListeners: { + each: (visit: (listener: (event: unknown) => void) => void) => void + } + } + ).eventListeners + + eventListeners.each((listener) => + listener({ + type: 'event', + event: 'exit', + sessionId: spawned.id, + payload: { code: 0 } + }) + ) + + expect(exits).toEqual([]) + expect(adapter.hasPty(spawned.id)).toBe(true) + }) }) describe('serialize / revive', () => { diff --git a/src/main/daemon/daemon-pty-adapter.ts b/src/main/daemon/daemon-pty-adapter.ts index 56bad83e39e..5f81ce11df9 100644 --- a/src/main/daemon/daemon-pty-adapter.ts +++ b/src/main/daemon/daemon-pty-adapter.ts @@ -41,7 +41,8 @@ import { GET_SIZE_PROTOCOL_VERSION, HISTORY_SEED_TRANSFER_PROTOCOL_VERSION, SNAPSHOT_SERIALIZER_FIDELITY_DAEMON_PROTOCOL_VERSION, - STABLE_PANE_ATTACH_ONLY_DAEMON_PROTOCOL_VERSION + STABLE_PANE_ATTACH_ONLY_DAEMON_PROTOCOL_VERSION, + SESSION_INCARNATION_ATTACH_FENCE_DAEMON_PROTOCOL_VERSION } from './daemon-protocol-version' import { isAgentSessionClaimedSpawnResult, @@ -95,6 +96,7 @@ import type { DaemonEvidenceSource, ExactDaemonIncarnation } from './daemon-inca import { createDaemonAuditEligibilityTracker } from './daemon-audit-eligibility-event' import { normalizeDesktopTerminalSnapshotRows } from '../../shared/terminal-scrollback-policy' import type { TerminalExitCause } from '../../shared/terminal-exit-cause' +import { TerminalSessionOwnerUnverifiedError } from './daemon-errors' type PendingDaemonSpawnOperation = { exitsBySessionId: Map @@ -283,6 +285,8 @@ export class DaemonPtyAdapter implements IPtyProvider { // A replacement daemon has none of the old PTYs; only createOrAttach can make their bindings writable again. private sessionsAwaitingDaemonRecovery = new Set() private sessionIncarnations = new Map() + private sessionOwnerIdentities = new Map() + private livenessInventoryInFlight: Promise | null = null private pendingSpawnOperationsBySessionId = new Map>() private pendingClaimSpawnOperations = new Set() private historySpawnLocks = new Map>() @@ -519,6 +523,14 @@ export class DaemonPtyAdapter implements IPtyProvider { const requestedSessionId = opts.sessionId! // Why: v30 daemons survive upgrades; reject their accidental create result before publication. const attachOnly = opts.attachOnly === true + if ( + attachOnly && + opts.expectedIncarnationIsAuthoritative === true && + opts.expectedIncarnationId !== undefined && + this.protocolVersion < SESSION_INCARNATION_ATTACH_FENCE_DAEMON_PROTOCOL_VERSION + ) { + throw new TerminalSessionOwnerUnverifiedError(requestedSessionId) + } const emulateLegacyAttachOnly = attachOnly && this.protocolVersion < STABLE_PANE_ATTACH_ONLY_DAEMON_PROTOCOL_VERSION let sessionId = requestedSessionId @@ -646,6 +658,11 @@ export class DaemonPtyAdapter implements IPtyProvider { startupCommandDelivery: attachOnly ? undefined : opts.startupCommandDelivery, launchAgent: attachOnly ? undefined : opts.launchAgent, ...(attachOnly && !emulateLegacyAttachOnly ? { attachOnly: true } : {}), + ...(attachOnly && + opts.expectedIncarnationIsAuthoritative === true && + opts.expectedIncarnationId !== undefined + ? { expectedIncarnationId: opts.expectedIncarnationId } + : {}), // Why: without forwarding the override, the daemon falls back to cmd.exe/PowerShell, ignoring the shell the renderer chose; this matches LocalPtyProvider. shellOverride: attachOnly ? undefined : opts.shellOverride, terminalWindowsWslDistro: attachOnly ? undefined : opts.terminalWindowsWslDistro, @@ -770,9 +787,14 @@ export class DaemonPtyAdapter implements IPtyProvider { if (exitedResult) { return exitedResult } - if (result.incarnationId) { - this.sessionIncarnations.set(sessionId, result.incarnationId) + if ( + opts.attachOnly && + opts.expectedIncarnationIsAuthoritative && + (!result.incarnationId || result.incarnationId !== opts.expectedIncarnationId) + ) { + throw new TerminalSessionOwnerUnverifiedError(sessionId) } + this.recordSessionAuthority(sessionId, result.incarnationId) const claimResult = (): Pick | Record => result.agentSessionEnsure ? { agentSessionEnsure: result.agentSessionEnsure } : {} const incarnationResult = (): Pick | Record => @@ -842,9 +864,7 @@ export class DaemonPtyAdapter implements IPtyProvider { if (exitedRetryResult) { return exitedRetryResult } - if (result.incarnationId) { - this.sessionIncarnations.set(sessionId, result.incarnationId) - } + this.recordSessionAuthority(sessionId, result.incarnationId) providerWslDistro = result.wslDistro === undefined ? wslDistro : result.wslDistro wslDistro = providerWslDistro ?? undefined if (wslDistro) { @@ -1085,16 +1105,81 @@ export class DaemonPtyAdapter implements IPtyProvider { return this.activeSessionIds.has(id) } - async probePtyLiveness(id: string): Promise { + private recordSessionAuthority(id: string, incarnationId?: PtyIncarnationId): void { + if (!incarnationId || !this.lastAuthenticatedIdentity) { + return + } + this.sessionIncarnations.set(id, incarnationId) + this.sessionOwnerIdentities.set(id, { ...this.lastAuthenticatedIdentity }) + } + + private hasExactSessionAuthority(id: string, expectedIncarnationId?: PtyIncarnationId): boolean { + const sessionIncarnationId = this.sessionIncarnations.get(id) + const ownerIdentity = this.sessionOwnerIdentities.get(id) + const currentIdentity = this.lastAuthenticatedIdentity + return Boolean( + sessionIncarnationId && + ownerIdentity && + currentIdentity && + sameEndpointIdentity(ownerIdentity, currentIdentity) && + (expectedIncarnationId === undefined || expectedIncarnationId === sessionIncarnationId) + ) + } + + private listSessionsForLiveness(): Promise { + if (this.livenessInventoryInFlight) { + return this.livenessInventoryInFlight + } + const request = this.client.request( + 'listSessions', + undefined, + LIVENESS_PROBE_TIMEOUT_MS + ) + this.livenessInventoryInFlight = request + void request.then( + () => { + if (this.livenessInventoryInFlight === request) { + this.livenessInventoryInFlight = null + } + }, + () => { + if (this.livenessInventoryInFlight === request) { + this.livenessInventoryInFlight = null + } + } + ) + return request + } + + async probePtyLiveness( + id: string, + expectedIncarnationId?: PtyIncarnationId + ): Promise { try { - if (!this.getSizeUnsupported && this.protocolVersion >= GET_SIZE_PROTOCOL_VERSION) { + await this.ensureConnected(Date.now() + LIVENESS_PROBE_TIMEOUT_MS) + const ownerIdentityAtProbe = this.lastAuthenticatedIdentity + ? { ...this.lastAuthenticatedIdentity } + : null + const canConfirmAbsence = this.hasExactSessionAuthority(id, expectedIncarnationId) + if ( + (expectedIncarnationId === undefined || canConfirmAbsence) && + !this.getSizeUnsupported && + this.protocolVersion >= GET_SIZE_PROTOCOL_VERSION + ) { try { const result = await this.client.request<{ size: { cols: number; rows: number } | null }>( 'getSize', { sessionId: id }, LIVENESS_PROBE_TIMEOUT_MS ) - return result.size !== null + if ( + !ownerIdentityAtProbe || + !this.lastAuthenticatedIdentity || + !sameEndpointIdentity(ownerIdentityAtProbe, this.lastAuthenticatedIdentity) + ) { + return null + } + return result.size !== null ? true : canConfirmAbsence ? false : null } catch (error) { // Why the capability probe rather than the version alone: `getSize` shipped into an // already-released protocol without a bump, so a daemon can report a version that @@ -1112,12 +1197,26 @@ export class DaemonPtyAdapter implements IPtyProvider { // the first daemon protocol. Requested directly rather than through `listProcesses` so a // liveness probe does not publish inventory audit observations as a side effect; both // rethrow on failure, so either way a dead socket stays `null` instead of reading absent. - const { sessions } = await this.client.request( - 'listSessions', - undefined, - LIVENESS_PROBE_TIMEOUT_MS - ) - return sessions.some((session) => session.sessionId === id && session.isAlive) + const { sessions } = await this.listSessionsForLiveness() + if ( + !ownerIdentityAtProbe || + !this.lastAuthenticatedIdentity || + !sameEndpointIdentity(ownerIdentityAtProbe, this.lastAuthenticatedIdentity) + ) { + return null + } + const liveSession = sessions.find((session) => session.sessionId === id && session.isAlive) + if (liveSession) { + this.recordSessionAuthority(id, liveSession.incarnationId) + if ( + expectedIncarnationId !== undefined && + liveSession.incarnationId !== expectedIncarnationId + ) { + return canConfirmAbsence ? false : null + } + return true + } + return canConfirmAbsence ? false : null } catch { return null } @@ -1288,6 +1387,8 @@ export class DaemonPtyAdapter implements IPtyProvider { remainingRequestTimeoutMs(opts.deadlineMs) ) this.activeSessionIds.delete(id) + this.sessionIncarnations.delete(id) + this.sessionOwnerIdentities.delete(id) this.clearSessionAwaitingDaemonRecovery(id) this.dirtySessionVersions.delete(id) if (!opts.keepHistory) { @@ -1757,6 +1858,7 @@ export class DaemonPtyAdapter implements IPtyProvider { continue } aliveSessionIds.add(session.sessionId) + this.recordSessionAuthority(session.sessionId, session.incarnationId) const { worktreeId } = parsePtySessionId(session.sessionId) processes.push( admission.admit({ @@ -1778,6 +1880,8 @@ export class DaemonPtyAdapter implements IPtyProvider { for (const id of preRequestActiveIds) { if (!aliveSessionIds.has(id)) { this.activeSessionIds.delete(id) + this.sessionIncarnations.delete(id) + this.sessionOwnerIdentities.delete(id) } } this.publishAuditObservation( @@ -1871,6 +1975,7 @@ export class DaemonPtyAdapter implements IPtyProvider { }) } this.sessionIncarnations.delete(id) + this.sessionOwnerIdentities.delete(id) } } @@ -2962,9 +3067,8 @@ export class DaemonPtyAdapter implements IPtyProvider { } const currentIncarnationId = this.sessionIncarnations.get(event.sessionId) if ( - event.payload.incarnationId && - currentIncarnationId && - event.payload.incarnationId !== currentIncarnationId + !event.payload.incarnationId || + (currentIncarnationId && event.payload.incarnationId !== currentIncarnationId) ) { return } @@ -2996,6 +3100,7 @@ export class DaemonPtyAdapter implements IPtyProvider { this.initialCwds.delete(event.sessionId) this.wslDistrosBySessionId.delete(event.sessionId) this.sessionIncarnations.delete(event.sessionId) + this.sessionOwnerIdentities.delete(event.sessionId) // oxlint-disable-next-line unicorn/no-useless-spread -- copy-safe: listeners may unsubscribe during iteration for (const listener of [...this.exitListeners]) { listener({ diff --git a/src/main/daemon/daemon-pty-router-owner-fencing.test.ts b/src/main/daemon/daemon-pty-router-owner-fencing.test.ts new file mode 100644 index 00000000000..3cd6909542e --- /dev/null +++ b/src/main/daemon/daemon-pty-router-owner-fencing.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it, vi } from 'vitest' +import type { DaemonPtyAdapter } from './daemon-pty-adapter' +import { DaemonPtyRouter } from './daemon-pty-router' +import type { PtySpawnOptions, PtySpawnResult } from '../providers/types' + +type FenceAdapter = DaemonPtyAdapter & { + emitExit: (id: string, code: number, incarnationId: string) => void + emitIdentityChange: () => void +} + +function createFenceAdapter(label: string, initialSessions: string[] = []): FenceAdapter { + const sessions = [...initialSessions] + let exitListener: ((event: { id: string; code: number; incarnationId?: string }) => void) | null = + null + let identityChangeListener: (() => void) | null = null + const noopSubscription = vi.fn(() => () => {}) + return { + spawn: vi.fn(async (options: PtySpawnOptions): Promise => { + const id = options.sessionId ?? `${label}-new` + sessions.push(id) + return { id, incarnationId: `${label}:${id}` } + }), + listProcesses: vi.fn(async () => + sessions.map((id) => ({ id, incarnationId: `${label}:${id}`, cwd: '', title: label })) + ), + write: vi.fn(), + shutdown: vi.fn(), + onData: noopSubscription, + onBackgroundStreamEvent: noopSubscription, + onWriteUnavailable: noopSubscription, + onExit: vi.fn((listener) => { + exitListener = listener + return () => { + exitListener = null + } + }), + onDaemonIdentityChanged: vi.fn((listener) => { + identityChangeListener = listener + return () => { + identityChangeListener = null + } + }), + emitExit: (id, code, incarnationId) => exitListener?.({ id, code, incarnationId }), + emitIdentityChange: () => identityChangeListener?.() + } as unknown as FenceAdapter +} + +describe('DaemonPtyRouter owner fencing', () => { + it('keeps replacement B routed after a delayed exit from owner A', async () => { + const current = createFenceAdapter('current') + const legacy = createFenceAdapter('legacy', ['reused-session']) + const router = new DaemonPtyRouter({ current, legacy: [legacy] }) + await router.discoverLegacySessions() + + legacy.emitIdentityChange() + await router.spawn({ cols: 80, rows: 24, sessionId: 'reused-session' }) + const routes = (router as unknown as { sessionAdapters: Map }) + .sessionAdapters + expect(routes.get('reused-session')).toBe(current) + + legacy.emitExit('reused-session', 0, 'legacy:reused-session') + expect(routes.get('reused-session')).toBe(current) + router.write('reused-session', 'replacement-marker') + + expect(current.write).toHaveBeenCalledWith('reused-session', 'replacement-marker') + expect(legacy.write).not.toHaveBeenCalled() + expect(current.shutdown).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/daemon/daemon-pty-router.test.ts b/src/main/daemon/daemon-pty-router.test.ts index 66364aa184e..46e90d49141 100644 --- a/src/main/daemon/daemon-pty-router.test.ts +++ b/src/main/daemon/daemon-pty-router.test.ts @@ -62,11 +62,12 @@ function createAdapter( spawn: vi.fn(async (opts: PtySpawnOptions): Promise => { const id = opts.sessionId ?? `${label}-new` sessions.push(id) - return { id } + return { id, incarnationId: `${label}:${id}` } }), listProcesses: vi.fn(async () => sessions.map((id) => ({ id, + incarnationId: `${label}:${id}`, cwd: '', title: label })) @@ -555,14 +556,14 @@ describe('DaemonPtyRouter', () => { expect(current.hasPty).not.toHaveBeenCalledWith('legacy-session') }) - it('discovers an unmapped live session from one coalesced inventory', async () => { + it('does not scan adapters to discover an unmapped liveness owner', async () => { const current = createAdapter('current') const legacy = createAdapter('legacy', ['surviving-session']) const router = new DaemonPtyRouter({ current, legacy: [legacy] }) - await expect(router.probePtyLiveness('surviving-session')).resolves.toBe(true) - expect(current.listProcesses).toHaveBeenCalledOnce() - expect(legacy.listProcesses).toHaveBeenCalledOnce() + await expect(router.probePtyLiveness('surviving-session')).resolves.toBeNull() + expect(current.listProcesses).not.toHaveBeenCalled() + expect(legacy.listProcesses).not.toHaveBeenCalled() expect(current.probePtyLiveness).not.toHaveBeenCalled() expect(legacy.probePtyLiveness).not.toHaveBeenCalled() }) @@ -574,16 +575,18 @@ describe('DaemonPtyRouter', () => { const router = new DaemonPtyRouter({ current, legacy: [legacy] }) await expect(router.probePtyLiveness('unknown-session')).resolves.toBeNull() + expect(current.listProcesses).not.toHaveBeenCalled() + expect(legacy.listProcesses).not.toHaveBeenCalled() }) - it('reports absence after every possible daemon owner returns a complete inventory', async () => { + it('does not treat complete current-daemon inventory as proof about a missing owner', async () => { const current = createAdapter('current') const legacy = createAdapter('legacy') const router = new DaemonPtyRouter({ current, legacy: [legacy] }) - await expect(router.probePtyLiveness('missing-session')).resolves.toBe(false) - expect(current.listProcesses).toHaveBeenCalledOnce() - expect(legacy.listProcesses).toHaveBeenCalledOnce() + await expect(router.probePtyLiveness('missing-session')).resolves.toBeNull() + expect(current.listProcesses).not.toHaveBeenCalled() + expect(legacy.listProcesses).not.toHaveBeenCalled() expect(current.spawn).not.toHaveBeenCalled() expect(legacy.spawn).not.toHaveBeenCalled() }) @@ -619,10 +622,13 @@ describe('DaemonPtyRouter', () => { await router.discoverLegacySessions() await expect(router.probePtyLiveness('legacy-session')).resolves.toBe(true) - expect(legacy.probePtyLiveness).toHaveBeenCalledExactlyOnceWith('legacy-session') + expect(legacy.probePtyLiveness).toHaveBeenCalledExactlyOnceWith( + 'legacy-session', + 'legacy:legacy-session' + ) }) - it('keeps consulting a legacy daemon whose inventory listing failed', async () => { + it('does not rescan a failed legacy discovery for an unmapped probe', async () => { const current = createAdapter('current') const legacy = createAdapter('legacy', ['legacy-session']) vi.mocked(legacy.listProcesses).mockRejectedValue(new Error('wedged')) @@ -630,9 +636,8 @@ describe('DaemonPtyRouter', () => { const router = new DaemonPtyRouter({ current, legacy: [legacy] }) await router.discoverLegacySessions() - // Without an inventory nothing proves the legacy daemon doesn't own this id. await expect(router.probePtyLiveness('unknown-session')).resolves.toBeNull() - expect(legacy.listProcesses).toHaveBeenCalledTimes(2) + expect(legacy.listProcesses).toHaveBeenCalledOnce() expect(legacy.probePtyLiveness).not.toHaveBeenCalled() }) @@ -752,7 +757,7 @@ describe('DaemonPtyRouter', () => { expect(current.listProcesses).toHaveBeenCalledTimes(3) }) - it('pins colliding unmapped legacy ids falling through to the current daemon', async () => { + it('never falls an unmapped colliding id through to the current daemon', async () => { const sessionId = 'cross-generation-collision' const current = createAdapter('current', [sessionId]) const legacy = createAdapter('legacy', [sessionId]) @@ -761,9 +766,11 @@ describe('DaemonPtyRouter', () => { const router = new DaemonPtyRouter({ current, legacy: [legacy] }) await router.discoverLegacySessions() - router.write(sessionId, 'misrouted\n') + expect(() => router.write(sessionId, 'misrouted\n')).toThrow( + TerminalSessionOwnerUnverifiedError + ) - expect(current.write).toHaveBeenCalledWith(sessionId, 'misrouted\n') + expect(current.write).not.toHaveBeenCalled() expect(legacy.write).not.toHaveBeenCalled() warn.mockRestore() }) diff --git a/src/main/daemon/daemon-pty-router.ts b/src/main/daemon/daemon-pty-router.ts index 0da991b6931..04003d42c00 100644 --- a/src/main/daemon/daemon-pty-router.ts +++ b/src/main/daemon/daemon-pty-router.ts @@ -12,6 +12,7 @@ import type { PtyProcessInspection } from '../providers/pty-process-inspection' import { shouldHandoffDaemonHistory } from './daemon-history-handoff' import type { DaemonPtyRouterDataEvent, DaemonPtyRouterExitEvent } from './daemon-pty-router-events' import { DaemonSessionOwnerResolver } from './daemon-session-owner-resolution' +import { TerminalSessionOwnerUnverifiedError } from './daemon-errors' export class DaemonPtyRouter implements IPtyProvider { private current: DaemonPtyAdapter @@ -26,8 +27,8 @@ export class DaemonPtyRouter implements IPtyProvider { this.ownerResolver = new DaemonSessionOwnerResolver(this.allAdapters(), this.sessionAdapters) this.subscriptions = new DaemonPtyAdapterSubscriptionFanout( this.allAdapters(), - (id) => { - this.ownerResolver.forgetRoute(id) + (adapter, id) => { + this.ownerResolver.forgetRoute(id, adapter) }, (adapter) => this.ownerResolver.invalidateProvider(adapter) ) @@ -78,15 +79,11 @@ export class DaemonPtyRouter implements IPtyProvider { } hasPty(id: string): boolean { - const routed = this.sessionAdapters.get(id) - if (routed) { - return routed.hasPty(id) - } - return this.current.hasPty(id) || this.legacy.some((adapter) => adapter.hasPty(id)) + return this.adapterFor(id).hasPty(id) } - async probePtyLiveness(id: string): Promise { - return await this.ownerResolver.probe(id) + async probePtyLiveness(id: string, expectedIncarnationId?: string): Promise { + return await this.ownerResolver.probe(id, expectedIncarnationId) } write(id: string, data: string): boolean { @@ -314,17 +311,18 @@ export class DaemonPtyRouter implements IPtyProvider { } private adapterFor(sessionId: string): DaemonPtyAdapter { - return this.sessionAdapters.get(sessionId) ?? this.current + const adapter = this.sessionAdapters.get(sessionId) + if (!adapter) { + throw new TerminalSessionOwnerUnverifiedError(sessionId) + } + return adapter } private adapterForInspection(sessionId: string): DaemonPtyAdapter { - const adapter = - this.sessionAdapters.get(sessionId) ?? - this.allAdapters().find((candidate) => candidate.hasPty(sessionId)) + const adapter = this.sessionAdapters.get(sessionId) if (!adapter) { throw new Error('terminal_gone') } - this.sessionAdapters.set(sessionId, adapter) return adapter } diff --git a/src/main/daemon/daemon-server.ts b/src/main/daemon/daemon-server.ts index b276b3cb0ba..7568314f3e5 100644 --- a/src/main/daemon/daemon-server.ts +++ b/src/main/daemon/daemon-server.ts @@ -1133,6 +1133,9 @@ export class DaemonServer { command: p.command, startupCommandDelivery: p.startupCommandDelivery, ...(attachOnly ? { attachOnly: true } : {}), + ...(p.expectedIncarnationId !== undefined + ? { expectedIncarnationId: p.expectedIncarnationId } + : {}), // Why: RPC payloads are untrusted JSON; persist only the allowlisted routing enum, never arbitrary identity. ...(isTuiAgent(p.launchAgent) ? { launchAgent: p.launchAgent } : {}), shellOverride: p.shellOverride, diff --git a/src/main/daemon/daemon-session-owner-authority.oracle.test.ts b/src/main/daemon/daemon-session-owner-authority.oracle.test.ts new file mode 100644 index 00000000000..a1b397ec352 --- /dev/null +++ b/src/main/daemon/daemon-session-owner-authority.oracle.test.ts @@ -0,0 +1,53 @@ +import { expect, it, vi } from 'vitest' +import type { IPtyProvider } from '../providers/types' +import { DaemonSessionOwnerResolver } from './daemon-session-owner-resolution' + +it('preserves A topology when compatible replacement B reports A absent', async () => { + const persistedOwner = Object.freeze({ + executionHostId: 'local', + ownerIncarnationId: 'daemon-a', + sessionIncarnationId: 'session-a', + logicalPtyId: 'logical-pty' + }) + const ownerA = Object.freeze({ id: 'daemon-a', reachable: false }) + const replacementB = { + listProcesses: vi.fn(async () => []), + probePtyLiveness: vi.fn(async () => false) + } as unknown as IPtyProvider + const resolver = new DaemonSessionOwnerResolver([replacementB], new Map()) + const topology: { + savedRows: string[] + tabBindings: Record + paneBindings: Record + } = { + savedRows: [persistedOwner.logicalPtyId], + tabBindings: { 'tab-1': persistedOwner.logicalPtyId }, + paneBindings: { 'tab-1:leaf-1': persistedOwner.logicalPtyId } + } + const liveMarkerIo = ['agent-a-still-running'] + let freshShells = 0 + + const verdict = await ( + resolver as unknown as { + probe: (id: string, expectedIncarnationId?: string) => Promise + } + ).probe(persistedOwner.logicalPtyId, persistedOwner.sessionIncarnationId) + if (verdict === false) { + topology.savedRows = [] + topology.tabBindings = {} + topology.paneBindings = {} + freshShells += 1 + } + + expect(ownerA.reachable).toBe(false) + expect(verdict).toBeNull() + expect(replacementB.probePtyLiveness).not.toHaveBeenCalled() + expect(replacementB.listProcesses).not.toHaveBeenCalled() + expect(topology).toEqual({ + savedRows: ['logical-pty'], + tabBindings: { 'tab-1': 'logical-pty' }, + paneBindings: { 'tab-1:leaf-1': 'logical-pty' } + }) + expect(liveMarkerIo).toEqual(['agent-a-still-running']) + expect(freshShells).toBe(0) +}) diff --git a/src/main/daemon/daemon-session-owner-resolution.test.ts b/src/main/daemon/daemon-session-owner-resolution.test.ts index 6b02c4a9d2d..c38e760d5e4 100644 --- a/src/main/daemon/daemon-session-owner-resolution.test.ts +++ b/src/main/daemon/daemon-session-owner-resolution.test.ts @@ -5,7 +5,11 @@ import type { PtySpawnOptions, PtySpawnResult } from '../providers/types' -import { SessionNotFoundError, TerminalSessionOwnerUnverifiedError } from './daemon-errors' +import { + SessionNotFoundError, + TerminalSessionExitedError, + TerminalSessionOwnerUnverifiedError +} from './daemon-errors' import { DaemonSessionOwnerResolver } from './daemon-session-owner-resolution' function provider( @@ -21,7 +25,7 @@ function provider( } describe('DaemonSessionOwnerResolver', () => { - it('coalesces complete multi-provider absence without dispatching an attach', async () => { + it('coalesces complete inventory without treating unmapped absence as exit', async () => { let releaseFallback!: (processes: PtyProcessInfo[]) => void let releaseCurrent!: (processes: PtyProcessInfo[]) => void let releaseLegacy!: (processes: PtyProcessInfo[]) => void @@ -88,11 +92,11 @@ describe('DaemonSessionOwnerResolver', () => { expect(results).toEqual([ expect.objectContaining({ status: 'rejected', - reason: expect.any(SessionNotFoundError) + reason: expect.any(TerminalSessionOwnerUnverifiedError) }), expect.objectContaining({ status: 'rejected', - reason: expect.any(SessionNotFoundError) + reason: expect.any(TerminalSessionOwnerUnverifiedError) }) ]) expect(settled).toBe(2) @@ -111,10 +115,10 @@ describe('DaemonSessionOwnerResolver', () => { cols: 80, rows: 24 }) - ).rejects.toBeInstanceOf(SessionNotFoundError) - expect(fallbackInventory).toHaveBeenCalledTimes(2) - expect(currentInventory).toHaveBeenCalledTimes(2) - expect(legacyInventory).toHaveBeenCalledTimes(2) + ).rejects.toBeInstanceOf(TerminalSessionOwnerUnverifiedError) + expect(fallbackInventory).toHaveBeenCalledOnce() + expect(currentInventory).toHaveBeenCalledOnce() + expect(legacyInventory).toHaveBeenCalledOnce() expect(fallback.spawn).not.toHaveBeenCalled() expect(current.spawn).not.toHaveBeenCalled() expect(legacy.spawn).not.toHaveBeenCalled() @@ -364,18 +368,19 @@ describe('DaemonSessionOwnerResolver', () => { await expect(resolver.resolve('session', 'persisted')).resolves.toEqual({ kind: 'unknown' }) }) - it('accepts positive liveness proof from an incomplete inventory', async () => { + it('does not inventory possible owners for an unmapped liveness probe', async () => { + const firstInventory = vi.fn(async () => [{ id: 'session', cwd: '', title: 'live' }]) + const secondInventory = vi.fn(async () => { + throw new Error('offline') + }) const resolver = new DaemonSessionOwnerResolver( - [ - provider(async () => [{ id: 'session', cwd: '', title: 'live' }]), - provider(async () => { - throw new Error('offline') - }) - ], + [provider(firstInventory), provider(secondInventory)], new Map() ) - await expect(resolver.probe('session')).resolves.toBe(true) + await expect(resolver.probe('session')).resolves.toBeNull() + expect(firstInventory).not.toHaveBeenCalled() + expect(secondInventory).not.toHaveBeenCalled() }) it('does not let a replacement daemon report a historical owner session exited', async () => { @@ -384,9 +389,7 @@ describe('DaemonSessionOwnerResolver', () => { const resolver = new DaemonSessionOwnerResolver([replacement], new Map()) await expect(resolver.probe('session-owned-by-unregistered-daemon')).resolves.toBe(null) - expect(replacement.probePtyLiveness).toHaveBeenCalledExactlyOnceWith( - 'session-owned-by-unregistered-daemon' - ) + expect(replacement.probePtyLiveness).not.toHaveBeenCalled() }) it('fails closed when duplicate owners cannot be disambiguated', async () => { @@ -401,18 +404,27 @@ describe('DaemonSessionOwnerResolver', () => { expect(second.spawn).not.toHaveBeenCalled() }) - it('reports confirmed absence from a sole owner', async () => { + it('reports confirmed absence only from the exact routed owner and session', async () => { const owner = provider( async () => [], async () => { throw new SessionNotFoundError('missing') } ) - const resolver = new DaemonSessionOwnerResolver([owner], new Map()) + const routes = new Map([['missing', owner]]) + const resolver = new DaemonSessionOwnerResolver([owner], routes) + resolver.recordRoute('missing', owner, 'missing-incarnation') await expect( - resolver.spawnAttachOnly({ sessionId: 'missing', attachOnly: true, cols: 80, rows: 24 }) - ).rejects.toBeInstanceOf(SessionNotFoundError) + resolver.spawnAttachOnly({ + sessionId: 'missing', + expectedIncarnationId: 'missing-incarnation', + expectedIncarnationIsAuthoritative: true, + attachOnly: true, + cols: 80, + rows: 24 + }) + ).rejects.toBeInstanceOf(TerminalSessionExitedError) expect(owner.spawn).toHaveBeenCalledOnce() expect(owner.listProcesses).not.toHaveBeenCalled() }) @@ -520,7 +532,7 @@ describe('DaemonSessionOwnerResolver', () => { expect(inventory).toHaveBeenCalledTimes(2) }) - it('fails closed across identity replacement then confirms absence on retry', async () => { + it('keeps absence unverified after identity replacement and retry', async () => { let releaseCurrent!: (processes: PtyProcessInfo[]) => void let releaseLegacy!: (processes: PtyProcessInfo[]) => void const currentGate = new Promise((resolve) => { @@ -559,7 +571,9 @@ describe('DaemonSessionOwnerResolver', () => { expect(current.spawn).not.toHaveBeenCalled() expect(legacy.spawn).not.toHaveBeenCalled() - await expect(resolver.spawnAttachOnly(attach)).rejects.toBeInstanceOf(SessionNotFoundError) + await expect(resolver.spawnAttachOnly(attach)).rejects.toBeInstanceOf( + TerminalSessionOwnerUnverifiedError + ) expect(currentInventory).toHaveBeenCalledTimes(2) expect(legacyInventory).toHaveBeenCalledTimes(2) expect(current.spawn).not.toHaveBeenCalled() diff --git a/src/main/daemon/daemon-session-owner-resolution.ts b/src/main/daemon/daemon-session-owner-resolution.ts index 1906ebfb35b..090fceac70c 100644 --- a/src/main/daemon/daemon-session-owner-resolution.ts +++ b/src/main/daemon/daemon-session-owner-resolution.ts @@ -4,7 +4,11 @@ import type { PtySpawnOptions, PtySpawnResult } from '../providers/types' -import { SessionNotFoundError, TerminalSessionOwnerUnverifiedError } from './daemon-errors' +import { + SessionNotFoundError, + TerminalSessionExitedError, + TerminalSessionOwnerUnverifiedError +} from './daemon-errors' export type DaemonSessionOwnerResolution = | { kind: 'owner'; provider: T } @@ -30,6 +34,7 @@ function assertClientConnected(signal: PtySpawnOptions['signal']): void { } export class DaemonSessionOwnerResolver { + private readonly providerSet: ReadonlySet private inventoryInFlight: Promise> | null = null private cachedInventory: { value: OwnerInventory; expiresAt: number } | null = null private readonly failedProviderCooldowns = new Map() @@ -39,7 +44,9 @@ export class DaemonSessionOwnerResolver { constructor( private readonly providers: readonly T[], private readonly routes: Map - ) {} + ) { + this.providerSet = new Set(providers) + } invalidateProvider(provider: T): void { this.epoch += 1 @@ -56,8 +63,9 @@ export class DaemonSessionOwnerResolver { async spawnAttachOnly(opts: PtySpawnOptions & { sessionId: string }): Promise { assertClientConnected(opts.signal) - const routed = this.providers.find((provider) => provider === this.routes.get(opts.sessionId)) - const direct = routed ?? (this.providers.length === 1 ? this.providers[0] : undefined) + const route = this.routes.get(opts.sessionId) + const routed = route && this.providerSet.has(route) ? (route as T) : undefined + const direct = routed const routedIncarnation = this.routeIncarnations.get(opts.sessionId) const routeNeedsAuthoritativeResolution = direct && @@ -79,8 +87,14 @@ export class DaemonSessionOwnerResolver { if (!(error instanceof SessionNotFoundError)) { throw error } - if (this.providers.length === 1) { - throw error + if ( + this.providers.length === 1 && + opts.expectedIncarnationIsAuthoritative && + routedIncarnation === opts.expectedIncarnationId && + this.routes.get(opts.sessionId) === routed && + this.routeIncarnations.get(opts.sessionId) === routedIncarnation + ) { + throw new TerminalSessionExitedError(opts.sessionId) } if (routed && this.routes.get(opts.sessionId) === routed) { this.forgetRoute(opts.sessionId, routed) @@ -99,7 +113,7 @@ export class DaemonSessionOwnerResolver { throw new TerminalSessionOwnerUnverifiedError(opts.sessionId) } if (resolution.kind === 'absent') { - throw new SessionNotFoundError(opts.sessionId) + throw new TerminalSessionExitedError(opts.sessionId) } try { const result = await resolution.provider.spawn(opts) @@ -112,40 +126,43 @@ export class DaemonSessionOwnerResolver { } return result } catch (error) { - if (error instanceof SessionNotFoundError && this.providers.length > 1) { + if (error instanceof SessionNotFoundError) { + if ( + opts.expectedIncarnationIsAuthoritative && + this.routes.get(opts.sessionId) === resolution.provider && + this.routeIncarnations.get(opts.sessionId) === opts.expectedIncarnationId + ) { + throw new TerminalSessionExitedError(opts.sessionId) + } throw new TerminalSessionOwnerUnverifiedError(opts.sessionId) } throw error } } - async probe(sessionId: string): Promise { - const routed = this.providers.find((provider) => provider === this.routes.get(sessionId)) - const direct = routed ?? (this.providers.length === 1 ? this.providers[0] : undefined) - if (direct) { - const verdict = direct.probePtyLiveness - ? await direct.probePtyLiveness(sessionId) - : (direct.hasPty?.(sessionId) ?? null) - if (verdict !== false || this.providers.length === 1) { - return verdict + async probe(sessionId: string, expectedIncarnationId?: string): Promise { + const route = this.routes.get(sessionId) + const routed = route && this.providerSet.has(route) ? (route as T) : undefined + if (routed) { + const routedIncarnation = this.routeIncarnations.get(sessionId) + if ( + !routedIncarnation || + (expectedIncarnationId !== undefined && routedIncarnation !== expectedIncarnationId) + ) { + return null } - if (this.routes.get(sessionId) === routed) { - this.routes.delete(sessionId) - this.routeIncarnations.delete(sessionId) + const verdict = routed.probePtyLiveness + ? await routed.probePtyLiveness(sessionId, routedIncarnation) + : (routed.hasPty?.(sessionId) ?? null) + if ( + this.routes.get(sessionId) !== routed || + this.routeIncarnations.get(sessionId) !== routedIncarnation + ) { + return null } + return verdict } - const inventory = await this.inventory(false) - if (inventory.epoch !== this.epoch) { - return null - } - if ((inventory.candidatesBySessionId.get(sessionId)?.length ?? 0) > 0) { - return true - } - const resolution = this.resolveInventory(inventory, sessionId) - if (resolution.kind === 'owner') { - return resolution.provider === routed ? null : true - } - return resolution.kind === 'absent' ? false : null + return null } async resolve( @@ -214,27 +231,24 @@ export class DaemonSessionOwnerResolver { : undefined const soleProvider = providers.size === 1 ? providers.values().next().value : undefined const provider = - (exactProvider && (inventory.complete || expectedIncarnationIsAuthoritative) + exactProvider && (inventory.complete || expectedIncarnationIsAuthoritative) ? exactProvider - : undefined) ?? - (!expectedIncarnationIsAuthoritative && inventory.complete ? soleProvider : undefined) + : !expectedIncarnationIsAuthoritative && inventory.complete + ? soleProvider + : undefined if (provider) { const process = candidates.find((candidate) => candidate.provider === provider)?.process this.recordRoute(sessionId, provider, process?.incarnationId) return { kind: 'owner', provider } } - if (inventory.complete && providers.size === 0 && this.providers.length > 0) { - return { kind: 'absent' } - } return { kind: 'unknown' } } - private inventory(allowCached = true): Promise> { + private inventory(): Promise> { if (this.inventoryInFlight) { return this.inventoryInFlight } if ( - allowCached && this.cachedInventory?.value.epoch === this.epoch && this.cachedInventory.expiresAt > Date.now() ) { diff --git a/src/main/daemon/degraded-daemon-pty-provider.test.ts b/src/main/daemon/degraded-daemon-pty-provider.test.ts index ccf3def0b69..c766fbb9484 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.test.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.test.ts @@ -4,7 +4,7 @@ import { DEGRADED_DAEMON_RECOVERY_RETRY_MS } from './degraded-daemon-fresh-spawn import type { DaemonPtyAdapter } from './daemon-pty-adapter' import type { IPtyProvider, PtySpawnOptions, PtySpawnResult } from '../providers/types' import type { PtyProcessInspection } from '../providers/pty-process-inspection' -import { SessionNotFoundError, TerminalSessionOwnerUnverifiedError } from './daemon-errors' +import { TerminalSessionOwnerUnverifiedError } from './daemon-errors' type ProviderMock = IPtyProvider & { probePtyLiveness: (id: string) => Promise @@ -201,7 +201,7 @@ it('keeps an attach unresolved when a legacy inventory listing fails', async () expect(current.spawn).not.toHaveBeenCalled() }) -it('confirms repeated stale-binding absence without falling back or spawning', async () => { +it('keeps repeated stale-binding absence unverifiable without fallback spawn', async () => { const current = createDaemonAdapter('daemon-current') const legacy = createDaemonAdapter('daemon-legacy') const fallback = createProvider('local-fallback') @@ -223,12 +223,12 @@ it('confirms repeated stale-binding absence without falling back or spawning', a rows: 24 } as const - await expect(provider.spawn(attach)).rejects.toBeInstanceOf(SessionNotFoundError) - await expect(provider.spawn(attach)).rejects.toBeInstanceOf(SessionNotFoundError) + await expect(provider.spawn(attach)).rejects.toBeInstanceOf(TerminalSessionOwnerUnverifiedError) + await expect(provider.spawn(attach)).rejects.toBeInstanceOf(TerminalSessionOwnerUnverifiedError) - expect(fallback.listProcesses).toHaveBeenCalledTimes(2) - expect(current.listProcesses).toHaveBeenCalledTimes(2) - expect(legacy.listProcesses).toHaveBeenCalledTimes(2) + expect(fallback.listProcesses).toHaveBeenCalledOnce() + expect(current.listProcesses).toHaveBeenCalledOnce() + expect(legacy.listProcesses).toHaveBeenCalledOnce() expect(fallback.spawn).not.toHaveBeenCalled() expect(current.spawn).not.toHaveBeenCalled() expect(legacy.spawn).not.toHaveBeenCalled() @@ -256,9 +256,9 @@ it('confirms repeated stale-binding absence without falling back or spawning', a }) expect(current.spawn).not.toHaveBeenCalled() expect(legacy.spawn).not.toHaveBeenCalled() - expect(fallback.listProcesses).toHaveBeenCalledTimes(2) - expect(current.listProcesses).toHaveBeenCalledTimes(2) - expect(legacy.listProcesses).toHaveBeenCalledTimes(2) + expect(fallback.listProcesses).toHaveBeenCalledOnce() + expect(current.listProcesses).toHaveBeenCalledOnce() + expect(legacy.listProcesses).toHaveBeenCalledOnce() }) it('rejects completion inspection instead of borrowing the fallback provider', async () => { @@ -510,7 +510,8 @@ describe('DegradedDaemonPtyProvider', () => { expect(fallback.probePtyLiveness).not.toHaveBeenCalled() currentSessions.push('unknown-session') - await expect(provider.probePtyLiveness('unknown-session')).resolves.toBe(true) + await expect(provider.probePtyLiveness('unknown-session')).resolves.toBeNull() + expect(current.listProcesses).not.toHaveBeenCalled() }) it('reports a routed fallback PTY live before probing daemon absence', async () => { diff --git a/src/main/daemon/degraded-daemon-pty-provider.ts b/src/main/daemon/degraded-daemon-pty-provider.ts index afd603c3943..f2297d1a8bd 100644 --- a/src/main/daemon/degraded-daemon-pty-provider.ts +++ b/src/main/daemon/degraded-daemon-pty-provider.ts @@ -59,7 +59,9 @@ export class DegradedDaemonPtyProvider implements IPtyProvider { this.unsubscribers.push( provider.onData((payload) => this.dataListeners.forEach((listener) => listener(payload))), provider.onExit((payload) => { - this.ownerRecovery.forgetRoute(payload.id) + if (this.sessionProviders.get(payload.id) === provider) { + this.ownerRecovery.forgetRoute(payload.id) + } this.exitListeners.forEach((listener) => listener(payload)) }) ) diff --git a/src/main/daemon/terminal-host-create-contract.ts b/src/main/daemon/terminal-host-create-contract.ts index aaaffaeb8e8..7a2cc900143 100644 --- a/src/main/daemon/terminal-host-create-contract.ts +++ b/src/main/daemon/terminal-host-create-contract.ts @@ -22,6 +22,8 @@ export type CreateOrAttachOptions = { launchAgent?: TuiAgent /** Missing ownership is not permission to create during stable-pane adoption. */ attachOnly?: boolean + /** Exact session incarnation required before attach mutates client ownership. */ + expectedIncarnationId?: PtyIncarnationId /** Explicit shell the renderer asked for, forwarded to the subprocess. */ shellOverride?: string terminalWindowsWslDistro?: string | null diff --git a/src/main/daemon/terminal-host-incarnation-fence.test.ts b/src/main/daemon/terminal-host-incarnation-fence.test.ts new file mode 100644 index 00000000000..45f9ad1005e --- /dev/null +++ b/src/main/daemon/terminal-host-incarnation-fence.test.ts @@ -0,0 +1,115 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { TuiAgent } from '../../shared/tui-agent' +import type { Session } from './session' +import type { SubprocessHandle } from './session-subprocess-handle' +import { TerminalHost } from './terminal-host' + +type TestSubprocess = SubprocessHandle & { + emitData: (data: string) => void + emitExit: (code: number) => void +} + +function createSubprocess(): TestSubprocess { + let onData: ((data: string) => void) | null = null + let onExit: ((code: number) => void) | null = null + return { + pid: 99999, + getForegroundProcess: vi.fn(() => null), + write: vi.fn(), + resize: vi.fn(), + kill: vi.fn(() => onExit?.(0)), + terminateOwnedTree: () => 'unavailable', + forceKill: vi.fn(() => onExit?.(137)), + signal: vi.fn(), + onData: (callback) => { + onData = callback + }, + onExit: (callback) => { + onExit = callback + }, + dispose: vi.fn(), + emitData: (data) => onData?.(data), + emitExit: (code) => onExit?.(code) + } as TestSubprocess +} + +type SpawnSubprocess = (options: { + sessionId: string + cols: number + rows: number + cwd?: string + env?: Record + command?: string + launchAgent?: TuiAgent +}) => SubprocessHandle + +describe('TerminalHost incarnation fencing', () => { + let host: TerminalHost + let subprocess: TestSubprocess + + beforeEach(() => { + subprocess = createSubprocess() + const spawnSubprocess = vi.fn(() => subprocess) as SpawnSubprocess + host = new TerminalHost({ spawnSubprocess }) + }) + + afterEach(async () => { + await host.dispose() + }) + + it('attaches only when the expected session incarnation matches', async () => { + const created = await host.createOrAttach({ + sessionId: 'session-1', + cols: 80, + rows: 24, + streamClient: { onData: vi.fn(), onExit: vi.fn() } + }) + + const attached = await host.createOrAttach({ + sessionId: 'session-1', + cols: 80, + rows: 24, + attachOnly: true, + expectedIncarnationId: created.incarnationId, + streamClient: { onData: vi.fn(), onExit: vi.fn() } + }) + + expect(attached.incarnationId).toBe(created.incarnationId) + expect(attached.isNew).toBe(false) + }) + + it('rejects stale identity before snapshot or client attachment side effects', async () => { + const initialClient = { onData: vi.fn(), onExit: vi.fn() } + await host.createOrAttach({ + sessionId: 'session-1', + cols: 80, + rows: 24, + streamClient: initialClient + }) + const session = (host as unknown as { sessions: Map }).sessions.get( + 'session-1' + )! + const getSnapshot = vi.spyOn(session, 'getSnapshot') + const detachAllClients = vi.spyOn(session, 'detachAllClients') + const attachClient = vi.spyOn(session, 'attachClient') + const staleClient = { onData: vi.fn(), onExit: vi.fn() } + + await expect( + host.createOrAttach({ + sessionId: 'session-1', + cols: 80, + rows: 24, + attachOnly: true, + expectedIncarnationId: 'stale-incarnation', + streamClient: staleClient + }) + ).rejects.toThrow('Session not found: session-1') + + expect(getSnapshot).not.toHaveBeenCalled() + expect(detachAllClients).not.toHaveBeenCalled() + expect(attachClient).not.toHaveBeenCalled() + subprocess.emitData('still-owned-by-current-client') + expect(initialClient.onData).toHaveBeenCalledWith('still-owned-by-current-client') + expect(staleClient.onData).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/daemon/terminal-host-session-create.ts b/src/main/daemon/terminal-host-session-create.ts index 1cb2a75f232..5e46e8fbb72 100644 --- a/src/main/daemon/terminal-host-session-create.ts +++ b/src/main/daemon/terminal-host-session-create.ts @@ -38,6 +38,13 @@ export async function createOrAttachTerminalSession( throw new SessionNotFoundError(opts.sessionId) } + if ( + opts.expectedIncarnationId !== undefined && + existing?.incarnationId !== opts.expectedIncarnationId + ) { + throw new SessionNotFoundError(opts.sessionId) + } + if (existing && existing.isAlive && !existing.isTerminating) { const snapshot = existing.getSnapshot() existing.detachAllClients() diff --git a/src/main/daemon/types.ts b/src/main/daemon/types.ts index 2f1817d14c7..9230d872f16 100644 --- a/src/main/daemon/types.ts +++ b/src/main/daemon/types.ts @@ -14,6 +14,7 @@ export type { import type { StartupCommandDelivery } from '../../shared/codex-startup-delivery' import type { TuiAgent } from '../../shared/tui-agent' import type { PtyStartupIngressIntent } from '../../shared/pty-startup-ingress' +import type { PtyIncarnationId } from '../../shared/pty-incarnation' import type { AgentSessionExecutionClaim, AgentSessionOwnerBinding, @@ -35,6 +36,7 @@ export { PROTOCOL_VERSION, PTY_STARTUP_INGRESS_PROTOCOL_VERSION, MODE_2031_UNSUBSCRIBE_FACT_PROTOCOL_VERSION, + SESSION_INCARNATION_ATTACH_FENCE_DAEMON_PROTOCOL_VERSION, supportsMode2031UnsubscribeFact, supportsPtyStartupIngress } from './daemon-protocol-version' @@ -71,6 +73,8 @@ export type CreateOrAttachRequest = { launchAgent?: TuiAgent /** Rejects an absent session instead of interpreting mount uncertainty as create permission. */ attachOnly?: boolean + /** Exact session incarnation required for a side-effect-free attach. */ + expectedIncarnationId?: PtyIncarnationId /** Explicit Windows shell override selected by the user (e.g. 'wsl.exe'). * The daemon forwards this to its subprocess spawner so each tab honors * the shell picked in the "+" menu or the persisted default-shell setting, diff --git a/src/main/index.ts b/src/main/index.ts index ae8fed9981d..13e712b73bc 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -1013,7 +1013,8 @@ async function reapRestoredSubagentsWithoutLiveAgent(): Promise { currentStore.getWorkspaceSession().terminalLayoutsByTabId ?? {} ) await sweepRestoredSubagentsWithoutLiveAgent({ - probeLiveLocalPty: (ptyId) => provider.probePtyLiveness(ptyId), + probeLiveLocalPty: (ptyId, expectedIncarnationId) => + provider.probePtyLiveness(ptyId, expectedIncarnationId), isLocalExecutionHost: (worktreeId) => isLocalExecutionHost( resolveAgentWorkspaceExecutionHostId(worktreeId, { @@ -1026,6 +1027,8 @@ async function reapRestoredSubagentsWithoutLiveAgent(): Promise { ), getBoundPtyIdForPaneKey: getPtyIdForPaneKey, getPersistedPtyIdForPaneKey: (paneKey) => persistedPtyIdByPaneKey.get(paneKey), + getPersistedPtyIncarnationForPaneKey: (paneKey) => + currentStore.getWorkspaceSession().terminalPtyIncarnationsByPaneKey?.[paneKey], reap: (isLocalHost, isLocalPaneAgentLive, isLocalPaneLivenessEvidenceCurrent) => agentHookServer.reapRestoredClaudeSubagentsWithoutLiveAgent( isLocalHost, diff --git a/src/main/ipc/pty-dead-owner-respawn.test.ts b/src/main/ipc/pty-dead-owner-respawn.test.ts index 8f47fa6fcdf..c5d9514cdcb 100644 --- a/src/main/ipc/pty-dead-owner-respawn.test.ts +++ b/src/main/ipc/pty-dead-owner-respawn.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest' import { setupPtyIpcSuite } from './pty-ipc-test-harness' import { makePaneKey } from '../../shared/stable-pane-id' import { registerPtyHandlers, setLocalPtyProvider } from './pty' +import { TerminalSessionExitedError } from '../daemon/daemon-errors' vi.mock('electron', () => import('./pty-ipc-mock-registry').then((m) => m.electronModuleMock())) vi.mock('fs', () => import('./pty-ipc-mock-registry').then((m) => m.fsModuleMock())) @@ -59,7 +60,7 @@ describe('registerPtyHandlers', () => { const providerSpawn = vi.fn( async (options: { attachOnly?: boolean; command?: string; sessionId?: string }) => { if (options.attachOnly) { - throw new Error('Session not found: pty-proven-absent-owner') + throw new TerminalSessionExitedError('pty-proven-absent-owner') } return { id: 'pty-fresh-proven', incarnationId: 'inc-fresh-proven' } } @@ -178,7 +179,7 @@ describe('registerPtyHandlers', () => { const providerSpawn = vi.fn( async (options: { attachOnly?: boolean; command?: string; sessionId?: string }) => { if (options.attachOnly) { - throw new Error('Session not found: pty-probe-blip-owner') + throw new TerminalSessionExitedError('pty-probe-blip-owner') } return { id: 'pty-fresh-probe-blip', incarnationId: 'inc-fresh-probe-blip' } } @@ -299,7 +300,7 @@ describe('registerPtyHandlers', () => { const providerSpawn = vi.fn( async (options: { attachOnly?: boolean; command?: string; sessionId?: string }) => { if (options.attachOnly) { - throw new Error('Session not found: pty-already-retired-owner') + throw new TerminalSessionExitedError('pty-already-retired-owner') } return { id: 'pty-fresh-already-retired', incarnationId: 'inc-fresh-already-retired' } } diff --git a/src/main/ipc/pty-persisted-incarnation-repair.test.ts b/src/main/ipc/pty-persisted-incarnation-repair.test.ts index 8f565f1c08e..0a472ccd348 100644 --- a/src/main/ipc/pty-persisted-incarnation-repair.test.ts +++ b/src/main/ipc/pty-persisted-incarnation-repair.test.ts @@ -52,7 +52,7 @@ vi.mock('../codex/codex-state-db-backfill-recovery', () => describe('registerPtyHandlers', () => { const { handlers, mainWindow } = setupPtyIpcSuite() - it('repairs a stale persisted incarnation after exact same-id reattach', async () => { + it('preserves a stale persisted incarnation when the same id belongs to another session', async () => { const tabId = 'tab-persisted-owner' const leafId = '88888888-8888-4888-8888-888888888888' const paneKey = makePaneKey(tabId, leafId) @@ -140,72 +140,26 @@ describe('registerPtyHandlers', () => { } } - const mounted = await handlers.get('pty:spawn')!(null, spawnArgs) - - expect(mounted).toMatchObject({ - id: 'pty-persisted-owner', - incarnationId: 'inc-live-owner', - isReattach: true - }) + await expect(handlers.get('pty:spawn')!(null, spawnArgs)).rejects.toThrow( + 'terminal_pane_owner_unverified' + ) expect(providerSpawn).toHaveBeenNthCalledWith( 1, expect.objectContaining({ attachOnly: true, sessionId: 'pty-persisted-owner', expectedIncarnationId: 'inc-stale-owner', - expectedIncarnationIsAuthoritative: false, - command: undefined - }) - ) - expect(runtime.registerPreAllocatedHandleForPty).toHaveBeenCalledWith( - 'pty-persisted-owner', - 'term-rebuilt-owner' - ) - expect(runtime.noteTerminalSpawnCommand).not.toHaveBeenCalled() - expect(store.persistPtyBinding).toHaveBeenCalledOnce() - expect(store.persistPtyBinding).toHaveBeenCalledWith( - expect.objectContaining({ - worktreeId, - tabId, - leafId, - ptyId: 'pty-persisted-owner', - incarnationId: 'inc-live-owner', - expectedBinding: { - ptyId: 'pty-persisted-owner', - incarnationId: 'inc-stale-owner' - } - }), - undefined - ) - expect( - mainWindow.webContents.send.mock.calls.filter(([channel]) => channel === 'pty:spawned') - ).toHaveLength(1) - expect(runtime.onPtyExit).not.toHaveBeenCalled() - - store.persistPtyBinding.mockClear() - await expect(handlers.get('pty:spawn')!(null, spawnArgs)).rejects.toThrow( - 'terminal_pane_owner_changed' - ) - expect(store.persistPtyBinding).not.toHaveBeenCalled() - - runtime.assertPtyRegistrationAllowed.mockImplementationOnce(() => { - throw new Error('agent_session_exited_during_start') - }) - await expect(handlers.get('pty:spawn')!(null, spawnArgs)).rejects.toThrow( - 'agent_session_exited_during_start' - ) - expect(store.persistPtyBinding).not.toHaveBeenCalled() - expect(providerSpawn).toHaveBeenCalledTimes(3) - expect(providerSpawn).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ - attachOnly: true, - sessionId: 'pty-persisted-owner', - expectedIncarnationId: 'inc-live-owner', expectedIncarnationIsAuthoritative: true, command: undefined }) ) + expect(runtime.registerPreAllocatedHandleForPty).not.toHaveBeenCalled() + expect(runtime.noteTerminalSpawnCommand).not.toHaveBeenCalled() + expect(store.persistPtyBinding).not.toHaveBeenCalled() + expect( + mainWindow.webContents.send.mock.calls.filter(([channel]) => channel === 'pty:spawned') + ).toHaveLength(0) + expect(runtime.onPtyExit).not.toHaveBeenCalled() clearProviderPtyState('pty-persisted-owner') }) it.each([ @@ -222,7 +176,7 @@ describe('registerPtyHandlers', () => { folderMissing: true } ])( - 'retires a persistence-only dead owner before fresh recovery ($label)', + 'preserves a persistence-only owner after unqualified absence ($label)', async ({ worktreeId, cwd, folderMissing }) => { const tabId = 'tab-dead-persisted-owner' const leafId = '12121212-1212-4212-8212-121212121212' @@ -341,44 +295,16 @@ describe('registerPtyHandlers', () => { } }) - if (folderMissing) { - await expect(mountedPromise).rejects.toThrow(`folder_workspace_path_missing:${cwd}`) - expect(providerSpawn).toHaveBeenCalledOnce() - expect(providerSpawn.mock.calls[0]?.[0]).toMatchObject({ - attachOnly: true, - sessionId: 'pty-dead-persisted-owner', - command: undefined - }) - expect(store.setWorkspaceSession).toHaveBeenCalledOnce() - expect(runtime.onPtyExit).toHaveBeenCalledWith( - 'pty-dead-persisted-owner', - 0, - 'inc-dead-persisted-owner' - ) - return - } - const mounted = await mountedPromise - - expect(mounted).toMatchObject({ - id: 'pty-fresh-recovery', - incarnationId: 'inc-fresh-recovery' - }) - expect(providerSpawn).toHaveBeenCalledTimes(2) + await expect(mountedPromise).rejects.toThrow('terminal_pane_owner_unverified') + expect(providerSpawn).toHaveBeenCalledOnce() expect(providerSpawn.mock.calls[0]?.[0]).toMatchObject({ attachOnly: true, sessionId: 'pty-dead-persisted-owner', command: undefined }) - expect(providerSpawn.mock.calls[1]?.[0]).toMatchObject({ - command: 'codex resume exact-dead-provider-session' - }) - expect(store.setWorkspaceSession).toHaveBeenCalledOnce() - expect(store.flushOrThrow).toHaveBeenCalledOnce() - expect(runtime.onPtyExit).toHaveBeenCalledWith( - 'pty-dead-persisted-owner', - 0, - 'inc-dead-persisted-owner' - ) + expect(store.setWorkspaceSession).not.toHaveBeenCalled() + expect(store.flushOrThrow).not.toHaveBeenCalled() + expect(runtime.onPtyExit).not.toHaveBeenCalled() } ) it('keeps a persisted owner when daemon routing is unresolved', async () => { diff --git a/src/main/ipc/pty-startup-barrier-and-listing.test.ts b/src/main/ipc/pty-startup-barrier-and-listing.test.ts index 41aca430405..267bb3ca0ce 100644 --- a/src/main/ipc/pty-startup-barrier-and-listing.test.ts +++ b/src/main/ipc/pty-startup-barrier-and-listing.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest' import { spawnMock } from './pty-ipc-mock-registry' import { makeDeferred } from './pty-ipc-test-constants' +import { makePaneKey } from '../../shared/stable-pane-id' import { setupPtyIpcSuite } from './pty-ipc-test-harness' import { registerPtyHandlers, @@ -463,6 +464,81 @@ describe('registerPtyHandlers', () => { await expect(pending).resolves.toBe(true) expect(probePtyLiveness).toHaveBeenCalledWith('restored-local-pty') }) + it('qualifies renderer liveness by its persisted session incarnation', async () => { + const worktreeId = 'repo::/workspace' + const exactLeafId = '11111111-1111-4111-8111-111111111111' + const legacyLeafId = '22222222-2222-4222-8222-222222222222' + const exactPaneKey = makePaneKey('tab-exact', exactLeafId) + const legacyPaneKey = makePaneKey('tab-legacy', legacyLeafId) + const probePtyLiveness = vi.fn(async () => false) + installDaemonTestProvider({ probePtyLiveness }) + registerPtyHandlers(mainWindow as never, undefined, undefined, undefined, undefined, { + getWorkspaceSession: () => ({ + tabsByWorktree: { + [worktreeId]: [ + { id: 'tab-exact', worktreeId }, + { id: 'tab-legacy', worktreeId } + ] + }, + terminalLayoutsByTabId: { + 'tab-exact': { ptyIdsByLeafId: { [exactLeafId]: 'pty-exact' } }, + 'tab-legacy': { ptyIdsByLeafId: { [legacyLeafId]: 'pty-legacy' } } + }, + terminalPtyIncarnationsByPaneKey: { [exactPaneKey]: 'incarnation-exact' } + }) + } as never) + + await expect( + handlers.get('pty:hasPty')!(null, { + id: 'pty-exact', + paneKey: exactPaneKey, + worktreeId + }) + ).resolves.toBe(false) + await expect( + handlers.get('pty:hasPty')!(null, { + id: 'pty-legacy', + paneKey: legacyPaneKey, + worktreeId + }) + ).resolves.toBeNull() + + expect(probePtyLiveness).toHaveBeenCalledExactlyOnceWith('pty-exact', 'incarnation-exact') + }) + it('discards a delayed liveness verdict after the persisted incarnation changes', async () => { + const worktreeId = 'repo::/workspace' + const leafId = '11111111-1111-4111-8111-111111111111' + const paneKey = makePaneKey('tab-exact', leafId) + const probe = makeDeferred() + const probePtyLiveness = vi.fn(async () => { + await probe.promise + return false + }) + const workspaceSession = { + tabsByWorktree: { [worktreeId]: [{ id: 'tab-exact', worktreeId }] }, + terminalLayoutsByTabId: { + 'tab-exact': { ptyIdsByLeafId: { [leafId]: 'reused-pty' } } + }, + terminalPtyIncarnationsByPaneKey: { [paneKey]: 'incarnation-a' } + } + installDaemonTestProvider({ probePtyLiveness }) + registerPtyHandlers(mainWindow as never, undefined, undefined, undefined, undefined, { + getWorkspaceSession: () => workspaceSession + } as never) + + const pending = handlers.get('pty:hasPty')!(null, { + id: 'reused-pty', + paneKey, + worktreeId + }) + await vi.waitFor(() => + expect(probePtyLiveness).toHaveBeenCalledWith('reused-pty', 'incarnation-a') + ) + workspaceSession.terminalPtyIncarnationsByPaneKey[paneKey] = 'incarnation-b' + probe.resolve() + + await expect(pending).resolves.toBeNull() + }) it('treats unsupported or failed single-PTY liveness as unknown', async () => { setLocalPtyProvider({ spawn: vi.fn(), diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 6086bd8ca77..e9219b04b43 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -11,6 +11,7 @@ import type { OrcaRuntimeService } from '../runtime/orca-runtime' import type { PtyBindingSourceExpectation, Store } from '../persistence' import { retireTerminalSurfaceFromPersistence } from '../runtime/mobile-session-terminal-persistence-retirement' import { SSH_PROVIDER_UNREGISTERED_REASON } from '../../shared/pty-liveness-verdict' +import { TERMINAL_PANE_OWNER_UNVERIFIED } from '../../shared/terminal-pane-owner-verdict' import type { GlobalSettings } from '../../shared/global-settings-types' import type { TuiAgent } from '../../shared/tui-agent' import { @@ -103,6 +104,7 @@ import { recordDaemonStreamBacklogEvent } from '../daemon/daemon-stream-backlog- import { isDaemonEndpointGoneError, TerminalHostGoneError, + TerminalSessionExitedError, TerminalSessionOwnerUnverifiedError } from '../daemon/daemon-errors' import type { ClaudeRuntimeAuthPreparation } from '../claude-accounts/runtime-auth-service' @@ -311,7 +313,7 @@ const ptyIncarnationById = new Map() export function isCurrentPtyExit(payload: { id: string; incarnationId?: string }): boolean { const current = ptyIncarnationById.get(payload.id) - return !current || payload.incarnationId === current + return payload.incarnationId !== undefined && (!current || payload.incarnationId === current) } // Why: mobile clients must mirror desktop PTY geometry even before the renderer can provide an xterm snapshot (e.g. right after tab creation). const ptySizes = new Map() @@ -881,12 +883,20 @@ async function attachStablePaneOwner( const { owner, provider, runtime, spawnOptions } = args let result: PtySpawnResult try { + if ( + owner.hasPersistedBinding && + owner.runtimeIncarnationId === undefined && + owner.persistedIncarnationId === undefined + ) { + throw new TerminalSessionOwnerUnverifiedError(owner.ptyId) + } result = await provider.spawn({ ...spawnOptions, sessionId: owner.ptyId, attachOnly: true, expectedIncarnationId: owner.runtimeIncarnationId ?? owner.persistedIncarnationId, - expectedIncarnationIsAuthoritative: owner.runtimeIncarnationId !== undefined, + expectedIncarnationIsAuthoritative: + owner.runtimeIncarnationId !== undefined || owner.hasPersistedBinding === true, isNewSession: undefined, command: undefined, commandDelivery: undefined, @@ -897,15 +907,25 @@ async function attachStablePaneOwner( agentSessionCreateOperationId: undefined, onPtySpawnCommitted: undefined }) + if ( + owner.hasPersistedBinding && + owner.persistedIncarnationId !== undefined && + result.incarnationId !== owner.persistedIncarnationId + ) { + throw new TerminalSessionOwnerUnverifiedError(owner.ptyId) + } } catch (error) { if (error instanceof TerminalSessionOwnerUnverifiedError) { - throw new Error('terminal_pane_owner_unverified') + throw new Error(TERMINAL_PANE_OWNER_UNVERIFIED) } // Why: translate before paired-runtime RPC strips the socket error's code and syscall. if (isDaemonEndpointGoneError(error)) { throw new TerminalHostGoneError() } - if (!isPtyAlreadyGoneError(error)) { + if (!(error instanceof TerminalSessionExitedError)) { + if (isPtyAlreadyGoneError(error)) { + throw new Error(TERMINAL_PANE_OWNER_UNVERIFIED) + } throw error } const ownerBeforeRetire = args.resolveOwner?.() @@ -7853,38 +7873,72 @@ export function registerPtyHandlers( } ) - ipc.handle('pty:hasPty', async (_event, args: { id: string }): Promise => { - if (typeof args?.id !== 'string' || args.id.startsWith('remote:')) { - // Why: same routing hazard pty:kill guards against — ptyOwnership never holds - // a runtime terminal handle and parseAppSshPtyId ignores it, so the lookup - // falls through to the local provider and its "not in my table" reads as an - // authoritative dead. That is a fabricated answer about another host's PTY. - return null - } - const ownedConnectionId = ptyOwnership.get(args.id) - const parsedSshId = ownedConnectionId === undefined ? parseAppSshPtyId(args.id) : null - const connectionId = ownedConnectionId ?? parsedSshId?.connectionId - const startupPromise = getLocalPtyProviderStartupPromise(connectionId) - if (startupPromise) { - await startupPromise - } - const provider = connectionId ? sshProviders.get(connectionId) : tryGetProviderForPty(args.id) - if (!provider) { - return null - } - try { - if (provider.probePtyLiveness) { - return await provider.probePtyLiveness(args.id) + ipc.handle( + 'pty:hasPty', + async ( + _event, + args: { id: string; paneKey?: string; worktreeId?: string } + ): Promise => { + if (typeof args?.id !== 'string' || args.id.startsWith('remote:')) { + // Why: same routing hazard pty:kill guards against — ptyOwnership never holds + // a runtime terminal handle and parseAppSshPtyId ignores it, so the lookup + // falls through to the local provider and its "not in my table" reads as an + // authoritative dead. That is a fabricated answer about another host's PTY. + return null } - if (!provider.hasPty) { + const ownedConnectionId = ptyOwnership.get(args.id) + const parsedSshId = ownedConnectionId === undefined ? parseAppSshPtyId(args.id) : null + const connectionId = ownedConnectionId ?? parsedSshId?.connectionId + const startupPromise = getLocalPtyProviderStartupPromise(connectionId) + if (startupPromise) { + await startupPromise + } + let expectedIncarnationId: string | undefined + if (args.paneKey !== undefined || args.worktreeId !== undefined) { + if (!args.paneKey || !args.worktreeId || connectionId) { + return null + } + const owner = resolvePersistedStablePaneOwner(store, args.paneKey, args.worktreeId, null) + if (!owner?.incarnationId || owner.ptyId !== args.id) { + return null + } + expectedIncarnationId = owner.incarnationId + } + const provider = connectionId ? sshProviders.get(connectionId) : tryGetProviderForPty(args.id) + if (!provider) { + return null + } + try { + if (provider.probePtyLiveness) { + const verdict = expectedIncarnationId + ? await provider.probePtyLiveness(args.id, expectedIncarnationId) + : await provider.probePtyLiveness(args.id) + if (expectedIncarnationId) { + const currentOwner = resolvePersistedStablePaneOwner( + store, + args.paneKey!, + args.worktreeId!, + null + ) + if ( + currentOwner?.ptyId !== args.id || + currentOwner.incarnationId !== expectedIncarnationId + ) { + return null + } + } + return verdict + } + if (!provider.hasPty || expectedIncarnationId) { + return null + } + return provider.hasPty(args.id) + } catch { + // Why: liveness is only allowed to close panes on an authoritative false. return null } - return provider.hasPty(args.id) - } catch { - // Why: liveness is only allowed to close panes on an authoritative false. - return null } - }) + ) ipc.handle('pty:hasChildProcesses', async (_event, args: { id: string }): Promise => { if (!hasPtyProviderForInspection(args.id)) { diff --git a/src/main/providers/pty-provider-contract.ts b/src/main/providers/pty-provider-contract.ts index 333d6fcc62c..5ac62bd4c4a 100644 --- a/src/main/providers/pty-provider-contract.ts +++ b/src/main/providers/pty-provider-contract.ts @@ -135,7 +135,10 @@ export type IPtyProvider = { attach(id: string): Promise | void> hasPty?: (id: string) => boolean /** Exact provider readback: false only when the provider answered that the PTY is absent. */ - probePtyLiveness?: (id: string) => Promise + probePtyLiveness?: ( + id: string, + expectedIncarnationId?: PtyIncarnationId + ) => Promise write(id: string, data: string): boolean | void writeWithSettlement?: (id: string, data: string) => Promise resize(id: string, cols: number, rows: number): void diff --git a/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts b/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts index 74989dc34e5..b3e0af53c37 100644 --- a/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts +++ b/src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest' -import { SSH_SESSION_EXPIRED_ERROR } from './ssh-pty-errors' import { SshPtyProvider } from './ssh-pty-provider' +import { TerminalSessionOwnerUnverifiedError } from '../daemon/daemon-errors' +import { TerminalSessionExitedError } from '../daemon/daemon-errors' describe('SSH PTY provider session reattach incarnation', () => { it('remembers the authoritative incarnation before a legacy exit arrives', async () => { @@ -30,7 +31,7 @@ describe('SSH PTY provider session reattach incarnation', () => { ) }) - it('fails closed when generic reattach requires source restoration', async () => { + it('preserves a live PTY when generic reattach cannot restore source continuity', async () => { const mux = { request: vi.fn().mockResolvedValue({ incarnationId: 'incarnation-reattached', @@ -44,8 +45,62 @@ describe('SSH PTY provider session reattach incarnation', () => { } const provider = new SshPtyProvider('conn-1', mux as never) - await expect(provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })).rejects.toThrow( - `${SSH_SESSION_EXPIRED_ERROR}: pty-old` + await expect( + provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' }) + ).rejects.toBeInstanceOf(TerminalSessionOwnerUnverifiedError) + }) + + it('rejects a replacement incarnation before accepting the attach', async () => { + const mux = { + request: vi.fn().mockResolvedValue({ incarnationId: 'incarnation-replacement' }), + notify: vi.fn(), + onNotification: vi.fn().mockReturnValue(vi.fn()) + } + const provider = new SshPtyProvider('conn-1', mux as never) + + await expect( + provider.spawn({ + cols: 80, + rows: 24, + sessionId: 'pty-old', + expectedIncarnationId: 'incarnation-original' + }) + ).rejects.toBeInstanceOf(TerminalSessionOwnerUnverifiedError) + expect(mux.request).toHaveBeenCalledWith( + 'pty.attach', + expect.objectContaining({ expectedIncarnationId: 'incarnation-original' }), + expect.any(Object) ) }) + + it('keeps relay absence owner-unverified', async () => { + const mux = { + request: vi.fn().mockRejectedValue(new Error('PTY "pty-old" not found')), + notify: vi.fn(), + onNotification: vi.fn().mockReturnValue(vi.fn()) + } + const provider = new SshPtyProvider('conn-1', mux as never) + + await expect( + provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' }) + ).rejects.toBeInstanceOf(TerminalSessionOwnerUnverifiedError) + }) + + it('accepts exact relay process-exit proof as authoritative', async () => { + const mux = { + request: vi.fn().mockRejectedValue(new Error('terminal_session_exited: pty-old')), + notify: vi.fn(), + onNotification: vi.fn().mockReturnValue(vi.fn()) + } + const provider = new SshPtyProvider('conn-1', mux as never) + + await expect( + provider.spawn({ + cols: 80, + rows: 24, + sessionId: 'pty-old', + expectedIncarnationId: 'incarnation-original' + }) + ).rejects.toBeInstanceOf(TerminalSessionExitedError) + }) }) diff --git a/src/main/providers/ssh-pty-provider-spawn.test.ts b/src/main/providers/ssh-pty-provider-spawn.test.ts index 06eb32d586f..072333c3f4a 100644 --- a/src/main/providers/ssh-pty-provider-spawn.test.ts +++ b/src/main/providers/ssh-pty-provider-spawn.test.ts @@ -623,12 +623,12 @@ describe('spawn', () => { }) }) - it('does not fresh-spawn over an expired reattach session', async () => { + it('keeps an unqualified missing relay session owner-unverified', async () => { mux.request.mockRejectedValueOnce(new Error('PTY "pty-old" not found')) - await expect(provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })).rejects.toThrow( - 'SSH_SESSION_EXPIRED: pty-old' - ) + await expect( + provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' }) + ).rejects.toMatchObject({ name: 'TerminalSessionOwnerUnverifiedError' }) expect(mux.request).toHaveBeenNthCalledWith( 1, diff --git a/src/main/providers/ssh-pty-session-reattach.ts b/src/main/providers/ssh-pty-session-reattach.ts index 3a7b8b15c79..9ec7d6037e8 100644 --- a/src/main/providers/ssh-pty-session-reattach.ts +++ b/src/main/providers/ssh-pty-session-reattach.ts @@ -1,11 +1,6 @@ import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' import { isPtyIncarnationId, type PtyIncarnationId } from '../../shared/pty-incarnation' -import { - SSH_PTY_IDENTITY_MISMATCH_ERROR, - SSH_SESSION_EXPIRED_ERROR, - isSshPtyIdentityMismatchError, - isSshPtyNotFoundError -} from './ssh-pty-errors' +import { isSshPtyNotFoundError } from './ssh-pty-errors' import { toAppSshPtyId, toRelaySshPtyId } from './ssh-pty-id' import type { PtySpawnOptions, PtySpawnResult } from './types' import type { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race' @@ -18,6 +13,11 @@ import { type PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation' import type { SshPtyReceivingActivationLease } from './ssh-pty-notification-routing' +import { + TerminalSessionExitedError, + TerminalSessionOwnerUnverifiedError +} from '../daemon/daemon-errors' +import { isTerminalSessionExited } from '../../shared/terminal-pane-owner-verdict' export type SshPtyAttachResult = { replay?: string @@ -83,9 +83,16 @@ export async function requestSshPtyAttach(args: { activation: PtySourceReceivingActivation ) => SshPtyReceivingActivationLease rememberPtyIncarnation?: (relayPtyId: string, incarnationId: unknown) => void + expectedIncarnationId?: PtyIncarnationId }): Promise { let activationLease: SshPtyReceivingActivationLease | undefined const installFromResult = (result: SshPtyAttachResult): void => { + if ( + args.expectedIncarnationId !== undefined && + result.incarnationId !== args.expectedIncarnationId + ) { + throw new TerminalSessionOwnerUnverifiedError(args.relayPtyId) + } if (!activationLease && result.sourceActivation && args.installSourceActivation) { activationLease = args.installSourceActivation(args.relayPtyId, result.sourceActivation) } @@ -202,8 +209,12 @@ export async function reattachSshPtySession(args: { // answers "you already have this", and the pane stays blank until new output arrives. requireReplay: true, ...(expectedPaneKey ? { expectedPaneKey } : {}), - ...(expectedTabId ? { expectedTabId } : {}) + ...(expectedTabId ? { expectedTabId } : {}), + ...(args.options.expectedIncarnationId + ? { expectedIncarnationId: args.options.expectedIncarnationId } + : {}) }, + expectedIncarnationId: args.options.expectedIncarnationId, installSourceActivation: args.installSourceActivation, rememberPtyIncarnation: args.rememberPtyIncarnation }) @@ -224,11 +235,11 @@ export async function reattachSshPtySession(args: { } catch (error) { // Why: an expired relay lease must be surfaced distinctly so the renderer clears its binding. console.warn(`[ssh-pty] pty.attach FAILED for ${args.sessionId}:`, error) + if (isTerminalSessionExited(error)) { + throw new TerminalSessionExitedError(relaySessionId) + } if (isSshPtyNotFoundError(error)) { - const mismatchMarker = isSshPtyIdentityMismatchError(error) - ? ` ${SSH_PTY_IDENTITY_MISMATCH_ERROR}` - : '' - throw new Error(`${SSH_SESSION_EXPIRED_ERROR}: ${relaySessionId}${mismatchMarker}`) + throw new TerminalSessionOwnerUnverifiedError(relaySessionId) } throw error } @@ -279,9 +290,8 @@ export async function reattachSshPtySessionForSpawn( try { result = await reattachSshPtySessionWithExitFence(args) if (result.sourceRecovery?.status === 'restoreRequired') { - throw new Error( - `${SSH_SESSION_EXPIRED_ERROR}: ${toRelaySshPtyId(args.connectionId, result.id)}` - ) + // The exact PTY attached successfully; missing replay continuity cannot prove process exit. + throw new TerminalSessionOwnerUnverifiedError(toRelaySshPtyId(args.connectionId, result.id)) } args.acceptLivePty(result.id) result.sourceActivationLease?.commit() diff --git a/src/preload/api/pty-api.ts b/src/preload/api/pty-api.ts index d69d906047d..73a6f5c07f4 100644 --- a/src/preload/api/pty-api.ts +++ b/src/preload/api/pty-api.ts @@ -118,7 +118,7 @@ export type PtyApi = { getAuthoritativeBufferSnapshotCapabilities?: ( ids: string[] ) => Promise<{ id: string; authoritative: boolean | null }[]> - hasPty: (id: string) => Promise + hasPty: (id: string, owner?: { paneKey: string; worktreeId: string }) => Promise getMainBufferSnapshot: ( id: string, opts?: { scrollbackRows?: number } diff --git a/src/preload/index.ts b/src/preload/index.ts index b8a00c6d647..67f7da5d9a3 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -1143,7 +1143,10 @@ const api = { ids: string[] ): Promise<{ id: string; authoritative: boolean | null }[]> => ipcRenderer.invoke('pty:getAuthoritativeBufferSnapshotCapabilities', { ids }), - hasPty: (id: string): Promise => ipcRenderer.invoke('pty:hasPty', { id }), + hasPty: ( + id: string, + owner?: { paneKey: string; worktreeId: string } + ): Promise => ipcRenderer.invoke('pty:hasPty', { id, ...owner }), getMainBufferSnapshot: ( id: string, diff --git a/src/relay/pty-handler-attach-replay.test.ts b/src/relay/pty-handler-attach-replay.test.ts index 7de5242f2ae..9ac19aea74c 100644 --- a/src/relay/pty-handler-attach-replay.test.ts +++ b/src/relay/pty-handler-attach-replay.test.ts @@ -60,11 +60,7 @@ describe('PtyHandler', () => { await endPtyHandlerTest(handler, originalPlatform) }) - it('reports not-found and reaps a reattach whose backing shell is dead', async () => { - // Why: a lingering managed entry whose child died without an onExit would - // otherwise attach-succeed with an empty replay and strand the pane on a - // black shell. A provably-dead pid must surface as not-found so the SSH - // provider maps it to SSH_SESSION_EXPIRED and the pane respawns fresh. + it('reports exact exit and reaps a reattach whose matched backing shell is dead', async () => { let onExitCb: ((evt: { exitCode: number }) => void) | undefined mockPtySpawn.mockReturnValue({ ...mockPtyInstance, @@ -76,15 +72,21 @@ describe('PtyHandler', () => { const exits: { id: string; paneKey?: string }[] = [] handler.setExitListener((evt) => exits.push(evt)) - await dispatcher.callRequest('pty.spawn', { env: { ORCA_PANE_KEY: 'tab-dead:0' } }) + const spawned = (await dispatcher.callRequest('pty.spawn', { + env: { ORCA_PANE_KEY: 'tab-dead:0' } + })) as { incarnationId: string } expect(handler.activePtyCount).toBe(1) expect(onExitCb).toBeDefined() const aliveSpy = vi.spyOn(ptyShellUtils, 'isProcessAlive').mockReturnValue(false) try { await expect( - dispatcher.callRequest('pty.attach', { id: 'pty-1', suppressReplayNotification: true }) - ).rejects.toThrow('PTY "pty-1" not found') + dispatcher.callRequest('pty.attach', { + id: 'pty-1', + suppressReplayNotification: true, + expectedIncarnationId: spawned.incarnationId + }) + ).rejects.toThrow('terminal_session_exited: pty-1') } finally { aliveSpy.mockRestore() } @@ -413,6 +415,37 @@ describe('PtyHandler', () => { ) }) + it('rejects a mismatched session incarnation before consuming replay', async () => { + let dataCallback: ((data: string) => void) | undefined + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + onData: vi.fn((cb: (data: string) => void) => { + dataCallback = cb + }), + onExit: vi.fn() + }) + const spawn = await spawnPty() + dataCallback!('identity-fenced-output') + + await expect( + attachPty({ + id: 'pty-1', + expectedIncarnationId: 'different-incarnation', + suppressReplayNotification: true + }) + ).rejects.toThrow('incarnation mismatch') + await expect( + attachPty({ + id: 'pty-1', + expectedIncarnationId: spawn.incarnationId, + suppressReplayNotification: true + }) + ).resolves.toEqual({ + incarnationId: spawn.incarnationId, + replay: 'identity-fenced-output' + }) + }) + it('attach preserves buffer so repeated attaches return the same data plus new output', async () => { let dataCallback: ((data: string) => void) | undefined mockPtySpawn.mockReturnValue({ diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index a9fd21a4fe2..4c36a7d54b0 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -74,6 +74,7 @@ import type { PtySourceRecoveryRequest, PtySourceRecoveryResult } from '../shared/pty-source-recovery-contract' +import { TERMINAL_SESSION_EXITED } from '../shared/terminal-pane-owner-verdict' import type { PtySourceReceivingActivation } from '../shared/pty-source-receiving-activation' import { AGENT_SESSION_CREATE_OPERATION_PROTOCOL_VERSION, @@ -1775,9 +1776,16 @@ export class PtyHandler { if (!managed || managed.disposed) { throw new Error(`PTY "${id}" not found`) } + if ( + typeof params.expectedIncarnationId === 'string' && + params.expectedIncarnationId !== managed.incarnationId + ) { + throw new Error(`PTY "${id}" not found (incarnation mismatch)`) + } // Why: verify liveness because shells can exit without node-pty onExit. if (managed.pty.pid && !isProcessAlive(managed.pty.pid)) { + const exactIncarnationMatched = params.expectedIncarnationId === managed.incarnationId managed.physicalExit?.markExited() this.releaseRelayIngress(managed) this.flushPtyOutput(id) @@ -1786,7 +1794,9 @@ export class PtyHandler { disposeManagedPty(managed) this.removePty(id) this.clearPtyFlowState(id) - throw new Error(`PTY "${id}" not found`) + throw new Error( + exactIncarnationMatched ? `${TERMINAL_SESSION_EXITED}: ${id}` : `PTY "${id}" not found` + ) } // Why: generation resets can reuse PTY IDs; reject conflicting identities. diff --git a/src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts b/src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts index ff55695c857..0d7f384d9c8 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection-direct-ssh-reattach-retry.test.ts @@ -220,6 +220,84 @@ describe('connectPanePty', () => { await flushAsyncTicks(12) }) + it.each(['result', 'throw'] as const)( + 'settles an owner-unverifiable direct SSH retry without changing topology (%s)', + async (outcome) => { + const { connectPanePty } = await import('./pty-connection') + const restoredPtyId = toAppSshPtyId('target-a', 'pty-owner-unverifiable') + const transport = createMockTransport() + transport.connect.mockImplementation(async () => { + if (outcome === 'throw') { + throw new Error('terminal_pane_owner_unverified') + } + return { id: restoredPtyId, ownerUnverifiable: true } + }) + transportFactoryQueue.push(transport) + const pendingRetry = { + attemptId: `attempt-owner-unverifiable-${outcome}`, + authority: { + targetId: 'target-a', + providerEpoch: 'epoch-1', + connectionGeneration: 3 + }, + tabGeneration: 7, + startedAt: 1 + } + const settleDirectSshPaneRetry = vi.fn() + mockStoreState = { + ...mockStoreState, + tabsByWorktree: { + 'wt-1': [{ id: 'tab-1', ptyId: restoredPtyId, generation: 7 }] + }, + ptyIdsByTabId: { 'tab-1': [restoredPtyId] }, + terminalLayoutsByTabId: { + 'tab-1': { + root: { type: 'leaf', leafId: LEAF_1 }, + activeLeafId: LEAF_1, + expandedLeafId: null, + ptyIdsByLeafId: { [LEAF_1]: restoredPtyId } + } + }, + repos: [{ id: 'repo1', connectionId: 'target-a', displayName: 'orca' }], + sshConnectionStates: new Map([ + [ + 'target-a', + { + targetId: 'target-a', + status: 'connected', + providerEpoch: 'epoch-1', + connectionGeneration: 3 + } + ] + ]), + directSshPaneRetryByTabId: { 'tab-1': pendingRetry }, + settleDirectSshPaneRetry + } + const deps = createDeps({ + restoredLeafId: LEAF_1, + restoredPtyIdByLeafId: { [LEAF_1]: restoredPtyId } + }) + + connectPanePty(createPane(1) as never, createManager(1) as never, deps as never) + await flushAsyncTicks(12) + + expect(transport.connect).toHaveBeenCalledTimes(1) + expect(deps.clearExitedPanePtyLayoutBinding).not.toHaveBeenCalled() + expect(deps.clearTabPtyId).not.toHaveBeenCalled() + expect(settleDirectSshPaneRetry).toHaveBeenCalledExactlyOnceWith({ + status: 'failed', + tabId: 'tab-1', + attemptId: pendingRetry.attemptId, + authority: pendingRetry.authority, + tabGeneration: pendingRetry.tabGeneration + }) + expect(mockStoreState.tabsByWorktree['wt-1']?.[0]?.ptyId).toBe(restoredPtyId) + expect(mockStoreState.terminalLayoutsByTabId?.['tab-1']?.ptyIdsByLeafId).toEqual({ + [LEAF_1]: restoredPtyId + }) + } + ) + it('commits a successful direct SSH reattach with its exact retry lease', async () => { const { connectPanePty } = await import('./pty-connection') const restoredPtyId = toAppSshPtyId('target-a', 'pty-restored') diff --git a/src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts b/src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts index e5ba0d40d6f..5759b6298bd 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection-reattach-binding.test.ts @@ -368,18 +368,14 @@ describe('connectPanePty', () => { expect(deps.updateTabPtyId).toHaveBeenCalledWith('tab-1', 'stray-shell-pty') }) - it('spawns a fresh PTY when a restored daemon split session cannot reattach', async () => { + it('preserves a restored binding when reattach returns no owner verdict', async () => { const { connectPanePty } = await import('./pty-connection') const transport = createMockTransport() transport.connect.mockImplementation(async (opts: { sessionId?: string }) => { if (opts.sessionId) { return undefined } - const onPtySpawn = createdTransportOptions[0]?.onPtySpawn as - | ((ptyId: string) => void) - | undefined - onPtySpawn?.('fresh-pty') - return 'fresh-pty' + return undefined }) transportFactoryQueue.push(transport) mockStoreState = { @@ -399,18 +395,14 @@ describe('connectPanePty', () => { await Promise.resolve() await Promise.resolve() - expect(transport.connect).toHaveBeenNthCalledWith( - 1, + expect(transport.connect).toHaveBeenCalledOnce() + expect(transport.connect).toHaveBeenCalledWith( expect.objectContaining({ sessionId: 'stale-pty' }) ) - expect(transport.connect).toHaveBeenNthCalledWith( - 2, - expect.not.objectContaining({ sessionId: expect.any(String) }) - ) - expect(deps.clearExitedPanePtyLayoutBinding).toHaveBeenCalledWith(2, 'stale-pty') - expect(deps.clearTabPtyId).toHaveBeenCalledWith('tab-1', 'stale-pty') - expect(deps.syncPanePtyLayoutBinding).toHaveBeenCalledWith(2, 'fresh-pty') - expect(deps.updateTabPtyId).toHaveBeenCalledWith('tab-1', 'fresh-pty') + expect(deps.clearExitedPanePtyLayoutBinding).not.toHaveBeenCalled() + expect(deps.clearTabPtyId).not.toHaveBeenCalled() + expect(deps.syncPanePtyLayoutBinding).not.toHaveBeenCalled() + expect(deps.updateTabPtyId).not.toHaveBeenCalled() }) it.each([ @@ -484,50 +476,55 @@ describe('connectPanePty', () => { expect(mockStoreState.deferredSshSessionIdsByTabId['tab-1']).toBe(restoredPtyId) } ) - it('preserves a restored pane binding when its daemon owner is unverifiable', async () => { - const { connectPanePty } = await import('./pty-connection') - const transport = createMockTransport() - transport.connect.mockImplementation( - async (opts: { sessionId?: string; callbacks?: ConnectCallbacks }) => { - opts.callbacks?.onError?.('terminal_pane_owner_unverified') - return undefined - } - ) - transportFactoryQueue.push(transport) - mockStoreState = { - ...mockStoreState, - tabsByWorktree: { 'wt-1': [{ id: 'tab-1', ptyId: 'unverifiable-pty' }] }, - ptyIdsByTabId: { 'tab-1': ['unverifiable-pty'] }, - terminalLayoutsByTabId: { - 'tab-1': { - root: { type: 'leaf', leafId: LEAF_2 }, - activeLeafId: LEAF_2, - expandedLeafId: null, - ptyIdsByLeafId: { [LEAF_2]: 'unverifiable-pty' } + it.each(['result', 'throw'] as const)( + 'preserves a restored pane binding when its daemon owner is unverifiable (%s)', + async (outcome) => { + const { connectPanePty } = await import('./pty-connection') + const transport = createMockTransport() + transport.connect.mockImplementation( + async (opts: { sessionId?: string; callbacks?: ConnectCallbacks }) => { + if (outcome === 'throw') { + throw new Error('terminal_pane_owner_unverified') + } + return { id: opts.sessionId!, ownerUnverifiable: true } } - } - } as StoreState - const deps = createDeps({ - restoredLeafId: LEAF_2, - restoredPtyIdByLeafId: { [LEAF_2]: 'unverifiable-pty' } - }) + ) + transportFactoryQueue.push(transport) + mockStoreState = { + ...mockStoreState, + tabsByWorktree: { 'wt-1': [{ id: 'tab-1', ptyId: 'unverifiable-pty' }] }, + ptyIdsByTabId: { 'tab-1': ['unverifiable-pty'] }, + terminalLayoutsByTabId: { + 'tab-1': { + root: { type: 'leaf', leafId: LEAF_2 }, + activeLeafId: LEAF_2, + expandedLeafId: null, + ptyIdsByLeafId: { [LEAF_2]: 'unverifiable-pty' } + } + } + } as StoreState + const deps = createDeps({ + restoredLeafId: LEAF_2, + restoredPtyIdByLeafId: { [LEAF_2]: 'unverifiable-pty' } + }) - connectPanePty(createPane(2) as never, createManager(2) as never, deps as never) - await flushAsyncTicks() + connectPanePty(createPane(2) as never, createManager(2) as never, deps as never) + await flushAsyncTicks() - expect(transport.connect).toHaveBeenCalledTimes(1) - expect(transport.connect).toHaveBeenCalledWith( - expect.objectContaining({ sessionId: 'unverifiable-pty' }) - ) - expect(deps.clearExitedPanePtyLayoutBinding).not.toHaveBeenCalled() - expect(deps.clearTabPtyId).not.toHaveBeenCalled() - expect(deps.syncPanePtyLayoutBinding).not.toHaveBeenCalled() - expect(mockStoreState.tabsByWorktree['wt-1'][0]?.ptyId).toBe('unverifiable-pty') - expect(mockStoreState.ptyIdsByTabId?.['tab-1']).toEqual(['unverifiable-pty']) - expect(mockStoreState.terminalLayoutsByTabId?.['tab-1']?.ptyIdsByLeafId).toEqual({ - [LEAF_2]: 'unverifiable-pty' - }) - }) + expect(transport.connect).toHaveBeenCalledTimes(1) + expect(transport.connect).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: 'unverifiable-pty' }) + ) + expect(deps.clearExitedPanePtyLayoutBinding).not.toHaveBeenCalled() + expect(deps.clearTabPtyId).not.toHaveBeenCalled() + expect(deps.syncPanePtyLayoutBinding).not.toHaveBeenCalled() + expect(mockStoreState.tabsByWorktree['wt-1'][0]?.ptyId).toBe('unverifiable-pty') + expect(mockStoreState.ptyIdsByTabId?.['tab-1']).toEqual(['unverifiable-pty']) + expect(mockStoreState.terminalLayoutsByTabId?.['tab-1']?.ptyIdsByLeafId).toEqual({ + [LEAF_2]: 'unverifiable-pty' + }) + } + ) it('reattaches via the tab-level SSH pty id when deferred bookkeeping missed the tab', async () => { // Why: restore can miss the deferred maps; the tab's SSH pty id must still drive connect-then-reattach, not a fresh spawn into a missing provider. diff --git a/src/renderer/src/components/terminal-pane/pty-connection-session-liveness.test.ts b/src/renderer/src/components/terminal-pane/pty-connection-session-liveness.test.ts index 1d9da05fdcc..e993242a2f5 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection-session-liveness.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection-session-liveness.test.ts @@ -1,5 +1,6 @@ import type * as React from 'react' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { makePaneKey } from '../../../../shared/stable-pane-id' import { flushAsyncTicks } from './pty-connection-test-async' import { sendTerminalInputThroughPane } from './pty-connection-test-dom' import { @@ -203,7 +204,10 @@ describe('connectPanePty', () => { binding.reconcileIfSessionMissing(hasPty) await flushAsyncTicks() - expect(hasPty).toHaveBeenCalledWith('pty-pane-2') + expect(hasPty).toHaveBeenCalledWith('pty-pane-2', { + paneKey: makePaneKey('tab-1', LEAF_2), + worktreeId: 'wt-1' + }) expect(manager.closePane).toHaveBeenCalledWith(2) }) diff --git a/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-attach.ts b/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-attach.ts index 534bf88e41e..1e343280ebe 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-attach.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-attach.ts @@ -14,7 +14,7 @@ import { toProcessExitStartup } from './process-exit-startup' import type { ConnectPanePtySession } from './connect-pane-pty-session' import { runDeferredSessionReattachChoice } from './deferred-session-reattach-choice' -import { recoverUnverifiableDirectSshReattach } from './direct-ssh-reattach-recovery' +import { recoverUnverifiableReattach } from './unverifiable-reattach-recovery' export function runDeferredSessionAttach(session: ConnectPanePtySession): void { // Why: trigger the deferred SSH connect per-tab (not per-target) so multiple tabs for one target reattach independently. @@ -205,6 +205,12 @@ export function runDeferredSessionAttach(session: ConnectPanePtySession): void { } : 'undefined' ) + if (result && typeof result === 'object' && result.ownerUnverifiable) { + session.finishReattachLiveDataDeferral(false, outputCallbacks.generation) + await clearPreSignaledSerializer() + recoverUnverifiableReattach(session, pendingSessionId) + return + } if (!result && expiredReattachError) { session.finishReattachLiveDataDeferral(false, outputCallbacks.generation) await clearPreSignaledSerializer() @@ -286,7 +292,7 @@ export function runDeferredSessionAttach(session: ConnectPanePtySession): void { return } session.reportError(err instanceof Error ? err.message : String(err)) - recoverUnverifiableDirectSshReattach(session, pendingSessionId) + recoverUnverifiableReattach(session, pendingSessionId) }) session.armDirectSshPaneRetryTimeout( trackedReattachPromise, diff --git a/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-connect.ts b/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-connect.ts index 90d8e9f3eed..eb87d12fb6d 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-connect.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-connect.ts @@ -3,10 +3,9 @@ import { recordPtyConnectDiagnostic } from './pty-connect-limits' import { isSshSessionExpiredError } from './ssh-session-connect' import { isRemoteRuntimePtyId } from './paired-parked-terminal-restore' import { toProcessExitStartup } from './process-exit-startup' -import { recoverUnverifiableDirectSshReattach } from './direct-ssh-reattach-recovery' +import { recoverUnverifiableReattach } from './unverifiable-reattach-recovery' import type { ConnectPanePtySession } from './connect-pane-pty-session' - -const PANE_OWNER_UNVERIFIED_ERROR = 'terminal_pane_owner_unverified' +import { isTerminalPaneOwnerUnverified } from '../../../../../shared/terminal-pane-owner-verdict' export function startDeferredSessionReattach( session: ConnectPanePtySession, @@ -24,7 +23,6 @@ export function startDeferredSessionReattach( : window.api.pty.declarePendingPaneSerializer(session.cacheKey).catch(() => null) let expiredReattachError = false - let paneOwnerUnverified = false const coldRestoreStartup = session.buildColdRestoreAgentResumeStartup() const outputCallbacks = session.captureTransportOutputCallbacks( (message) => { @@ -32,9 +30,6 @@ export function startDeferredSessionReattach( expiredReattachError = true return } - if (message.includes(PANE_OWNER_UNVERIFIED_ERROR)) { - paneOwnerUnverified = true - } if (!session.isCapturedDirectSshReattachCurrent(deferredReattachSessionId)) { return } @@ -81,12 +76,13 @@ export function startDeferredSessionReattach( } return } - if (!result && paneOwnerUnverified) { + if (result && typeof result === 'object' && result.ownerUnverifiable) { session.finishReattachLiveDataDeferral(false, outputCallbacks.generation) const gen = await preSignalPromise if (typeof gen === 'number') { void window.api.pty.clearPendingPaneSerializer(session.cacheKey, gen).catch(() => {}) } + recoverUnverifiableReattach(session, deferredReattachSessionId) return } if (!result && expiredReattachError) { @@ -147,8 +143,9 @@ export function startDeferredSessionReattach( if (session.rejectObsoleteDirectSshReattach(deferredReattachSessionId)) { return } - if (message.includes(PANE_OWNER_UNVERIFIED_ERROR)) { + if (isTerminalPaneOwnerUnverified(message)) { session.reportError(message) + recoverUnverifiableReattach(session, deferredReattachSessionId) return } warnTerminalLifecycleAnomaly('restored PTY reattach threw', { @@ -168,15 +165,7 @@ export function startDeferredSessionReattach( return } session.reportError(message) - if (session.connectionId) { - recoverUnverifiableDirectSshReattach(session, deferredReattachSessionId) - return - } - session.deps.clearExitedPanePtyLayoutBinding(session.pane.id, deferredReattachSessionId) - session.deps.clearTabPtyId(session.deps.tabId, deferredReattachSessionId) - session.startFreshColdRestoreAgentResume(coldRestoreStartup, { - forceBlankRestoredViewport: true - }) + recoverUnverifiableReattach(session, deferredReattachSessionId) }) session.armDirectSshPaneRetryTimeout(trackedReattachPromise, session.directSshRetryAttempt) } diff --git a/src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts b/src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts index 03f3da2f158..b23f51c297f 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts @@ -1,12 +1,12 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { requestTerminalPaneRecovery } from '../terminal-pane-recovery' -import { recoverUnverifiableDirectSshReattach } from './direct-ssh-reattach-recovery' +import { recoverUnverifiableReattach } from './unverifiable-reattach-recovery' vi.mock('../terminal-pane-recovery', () => ({ requestTerminalPaneRecovery: vi.fn() })) -describe('recoverUnverifiableDirectSshReattach', () => { +describe('recoverUnverifiableReattach', () => { beforeEach(() => { vi.clearAllMocks() }) @@ -15,7 +15,7 @@ describe('recoverUnverifiableDirectSshReattach', () => { const attempt = { attemptId: 'attempt-1' } const settleDirectSshPaneRetryAttempt = vi.fn() - recoverUnverifiableDirectSshReattach( + recoverUnverifiableReattach( { directSshRetryAttempt: attempt, settleDirectSshPaneRetryAttempt } as never, 'ssh:target@@pty-1' ) @@ -25,7 +25,7 @@ describe('recoverUnverifiableDirectSshReattach', () => { }) it('remounts over the preserved PTY when no retry lease exists', () => { - recoverUnverifiableDirectSshReattach( + recoverUnverifiableReattach( { directSshRetryAttempt: undefined, deps: { tabId: 'tab-1' }, diff --git a/src/renderer/src/components/terminal-pane/pty-connection/reattach-result-handler.ts b/src/renderer/src/components/terminal-pane/pty-connection/reattach-result-handler.ts index 028e33d9992..5650fbbaba7 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/reattach-result-handler.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/reattach-result-handler.ts @@ -17,7 +17,7 @@ import type { ConnectPanePtySession } from './connect-pane-pty-session' import type { ReattachPayloadContext } from './reattach-payload-context' import { createReattachPayloadHandlers } from './apply-reattach-payload' import type { ReattachPayloadSession } from './reattach-payload-session' -import { recoverUnverifiableDirectSshReattach } from './direct-ssh-reattach-recovery' +import { recoverUnverifiableReattach } from './unverifiable-reattach-recovery' type ReattachResultSession = ReattachPayloadSession & Pick< @@ -68,6 +68,11 @@ export function bindHandleReattachResult(sessionBag: ConnectPanePtySession): voi const connectResult = result && typeof result === 'object' && 'id' in result ? (result as PtyConnectResult) : null + if (connectResult?.ownerUnverifiable) { + recoverUnverifiableReattach(sessionBag, staleSessionId) + return false + } + if (connectResult?.exitedBeforeAttach) { // Why: the transport already delivered the dead session's final frame + exit; treat as terminal state, not a failed reattach. return true @@ -90,22 +95,7 @@ export function bindHandleReattachResult(sessionBag: ConnectPanePtySession): voi paneId: session.pane.id, ptyId: staleSessionId ?? null }) - if (session.connectionId) { - recoverUnverifiableDirectSshReattach(sessionBag, staleSessionId) - return false - } - // Why: a stale restored session can fail reattach after mount; don't leave xterm alive without a backing PTY. - if (staleSessionId) { - session.deps.clearExitedPanePtyLayoutBinding(session.pane.id, staleSessionId) - } else { - session.deps.syncPanePtyLayoutBinding(session.pane.id, null) - } - if (staleSessionId) { - session.deps.clearTabPtyId(session.deps.tabId, staleSessionId) - } - session.startFreshColdRestoreAgentResume(coldRestoreStartup, { - forceBlankRestoredViewport: true - }) + recoverUnverifiableReattach(sessionBag, staleSessionId) return false } session.registerEffectiveLaunchConfig(connectResult?.launchConfig, { diff --git a/src/renderer/src/components/terminal-pane/pty-connection/session-reconcile-dispose.ts b/src/renderer/src/components/terminal-pane/pty-connection/session-reconcile-dispose.ts index 657a01169bc..3d296f918f2 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/session-reconcile-dispose.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/session-reconcile-dispose.ts @@ -61,7 +61,12 @@ export function installSessionReconcileDispose(session: ConnectPanePtySession): let livenessPromise: Promise try { - livenessPromise = Promise.resolve(hasPty(requestedPtyId)) + livenessPromise = Promise.resolve( + hasPty(requestedPtyId, { + paneKey: session.cacheKey, + worktreeId: session.deps.worktreeId + }) + ) } catch { return } diff --git a/src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.ts b/src/renderer/src/components/terminal-pane/pty-connection/unverifiable-reattach-recovery.ts similarity index 92% rename from src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.ts rename to src/renderer/src/components/terminal-pane/pty-connection/unverifiable-reattach-recovery.ts index f32896fc254..a08fdd4a56c 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/unverifiable-reattach-recovery.ts @@ -1,7 +1,7 @@ import { requestTerminalPaneRecovery } from '../terminal-pane-recovery' import type { ConnectPanePtySession } from './connect-pane-pty-session' -export function recoverUnverifiableDirectSshReattach( +export function recoverUnverifiableReattach( session: ConnectPanePtySession, ptyId: string | null | undefined ): void { diff --git a/src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts b/src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts index 32038c130ba..a8eb42109ce 100644 --- a/src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts @@ -52,7 +52,7 @@ describe('createIpcPtyTransport', () => { }) expect(onError).not.toHaveBeenCalled() - expect(result).toBeUndefined() + expect(result).toEqual({ id: 'pty-dead', exitedBeforeAttach: true }) }) it('still surfaces non-kill spawn errors via onError', async () => { diff --git a/src/renderer/src/components/terminal-pane/pty-transport-types.ts b/src/renderer/src/components/terminal-pane/pty-transport-types.ts index d8bdc63f62a..78c9c842c54 100644 --- a/src/renderer/src/components/terminal-pane/pty-transport-types.ts +++ b/src/renderer/src/components/terminal-pane/pty-transport-types.ts @@ -65,6 +65,7 @@ export type LocalPtySessionMetadata = { export type PtyConnectResult = { id: string + ownerUnverifiable?: true /** The requested session exited while it had no primary pane handler. Its * buffered final data/exit were delivered, so callers must not fresh-spawn. */ exitedBeforeAttach?: boolean diff --git a/src/renderer/src/components/terminal-pane/pty-transport.ts b/src/renderer/src/components/terminal-pane/pty-transport.ts index f713ee6c2f9..6072abaf4c8 100644 --- a/src/renderer/src/components/terminal-pane/pty-transport.ts +++ b/src/renderer/src/components/terminal-pane/pty-transport.ts @@ -50,6 +50,7 @@ import { type PtySideEffectGauge } from './pty-side-effect-pending-census' import { isTuiAgent } from '../../../../shared/tui-agent-config' +import { isTerminalPaneOwnerUnverified } from '../../../../shared/terminal-pane-owner-verdict' // Re-export public API so existing consumers keep working. export { @@ -856,10 +857,16 @@ export function createIpcPtyTransport(opts: IpcPtyTransportOptions = {}): PtyTra const resultLaunchAgent = isTuiAgent(spawnResult.launchAgent) ? spawnResult.launchAgent : undefined - const retireFreshSpawn = async (): Promise => { + const retireFreshSpawn = async (): Promise => { if (!spawnResult.isReattach && !spawnResult.coldRestore) { - await window.api.pty.kill(spawnResult.id) + try { + await window.api.pty.kill(spawnResult.id) + } catch (error) { + storedCallbacks.onError?.(error instanceof Error ? error.message : String(error)) + return false + } } + return true } // Why: on destroy mid-connect, kill only a fresh spawn — killing a reattached session (owned by the tab lifecycle) loses a live shell. @@ -870,8 +877,7 @@ export function createIpcPtyTransport(opts: IpcPtyTransportOptions = {}): PtyTra if (options.admitPtyId && !options.admitPtyId(spawnResult.id)) { // Why: a rejected session-expired fallback has no owner to retire its newly created process. - await retireFreshSpawn() - return spawnResult + return (await retireFreshSpawn()) ? spawnResult : undefined } if (spawnResult.isReattach && !admittedSessionId) { @@ -945,6 +951,13 @@ export function createIpcPtyTransport(opts: IpcPtyTransportOptions = {}): PtyTra return spawnResult.id } catch (err) { const msg = extractIpcErrorMessage(err, err instanceof Error ? err.message : String(err)) + if (options.sessionId && isTerminalPaneOwnerUnverified(msg)) { + storedCallbacks.onError?.(msg) + return { + id: options.sessionId, + ownerUnverifiable: true + } satisfies PtyConnectResult + } if ( connectionId && options.sessionId && @@ -956,9 +969,17 @@ export function createIpcPtyTransport(opts: IpcPtyTransportOptions = {}): PtyTra sessionExpired: true } satisfies PtyConnectResult } - // Why: re-spawning a Kill-All'd session throws TerminalKilledError; swallow it (pane still shows "Process exited"), don't toast (src/main/daemon/daemon-pty-adapter.ts). if (msg.includes('was explicitly killed')) { - return undefined + return options.sessionId + ? ({ id: options.sessionId, exitedBeforeAttach: true } satisfies PtyConnectResult) + : undefined + } + if (options.sessionId) { + storedCallbacks.onError?.(msg) + return { + id: options.sessionId, + ownerUnverifiable: true + } satisfies PtyConnectResult } // Why: on cold start the SSH provider isn't registered yet, so pty:spawn throws a raw IPC error; replace with a friendly message. if (connectionId && msg.includes('No PTY provider for connection')) { diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts index 7d50b9f48de..c182c96762e 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts @@ -2189,10 +2189,18 @@ export function createRemoteRuntimePtyTransport( ...(createdTerminal.isReattach === true ? { isReattach: true } : {}) } satisfies PtyConnectResult } catch (error) { + let ownerUnverifiableResult: PtyConnectResult | undefined if (!destroyed && lifecycleEpoch === connectLifecycleEpoch) { connecting = false const message = runtimeTerminalErrorMessage(error) - if (isRemoteTerminalGoneMessage(message)) { + if (options.sessionId) { + recovery.markDisconnected() + surfaceErrorMessage(message) + ownerUnverifiableResult = { + id: options.sessionId, + ownerUnverifiable: true + } + } else if (isRemoteTerminalGoneMessage(message)) { recovery.cancel() handleRemoteTerminalError(error) } else if ( @@ -2205,7 +2213,7 @@ export function createRemoteRuntimePtyTransport( surfaceErrorMessage(message) } } - return undefined + return ownerUnverifiableResult } }, diff --git a/src/renderer/src/components/terminal-pane/terminal-dead-session-reconcile.ts b/src/renderer/src/components/terminal-pane/terminal-dead-session-reconcile.ts index 7c9fe8a23fc..f6b71ac3855 100644 --- a/src/renderer/src/components/terminal-pane/terminal-dead-session-reconcile.ts +++ b/src/renderer/src/components/terminal-pane/terminal-dead-session-reconcile.ts @@ -16,7 +16,10 @@ export type ReconcilableBinding = { reconcileIfSessionMissing?: (hasPty: HasPty, livenessRequestedAt?: number) => void } -export type HasPty = (ptyId: string) => Promise +export type HasPty = ( + ptyId: string, + owner?: { paneKey: string; worktreeId: string } +) => Promise /** * PURE decision: should the pane bound to `ptyId` be reconciled (torn down) diff --git a/src/shared/terminal-pane-owner-verdict.ts b/src/shared/terminal-pane-owner-verdict.ts new file mode 100644 index 00000000000..2d301bf4187 --- /dev/null +++ b/src/shared/terminal-pane-owner-verdict.ts @@ -0,0 +1,12 @@ +export const TERMINAL_PANE_OWNER_UNVERIFIED = 'terminal_pane_owner_unverified' +export const TERMINAL_SESSION_EXITED = 'terminal_session_exited' + +export function isTerminalPaneOwnerUnverified(value: unknown): boolean { + const message = value instanceof Error ? value.message : String(value) + return message.includes(TERMINAL_PANE_OWNER_UNVERIFIED) +} + +export function isTerminalSessionExited(value: unknown): boolean { + const message = value instanceof Error ? value.message : String(value) + return message.includes(TERMINAL_SESSION_EXITED) +}