diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 5307bc71292..774b7526112 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -4124,63 +4124,199 @@ "remote-runtime", "ssh" ], - "coverageNotes": "Deterministic tests cover a four-child healthy runtime pool, a one-child healthy SSH relay pool, and at most four bounded quarantine children, with one quarantine attempt per watch lifetime, queue-aware cancellation, bounded initial and crash-resubscribe crawls, physical desktop-install abort, stale-generation rejection, per-child stat bounds, final-overflow RPC delivery, and renderer eviction before terminal callbacks can retry. The relay policy test maps a standard repository's base, Git common directory, and worktree roots to one healthy supervisor, then proves a shared-child failure recovers those roots in separate quarantine supervisors with overflow and resumed events. A built-entry macOS harness drives the real supervisor, kills its child with SIGSEGV, then requires automatic resubscription and a later event. SSH bundles the same child boundary beside relay.js, requires both artifacts for install completeness, and preserves registration-owned same-root cancellation. A built-relay harness kills that child and requires the original relay PID and existing PTY to survive before overflow and precise events resume.", + "coverageNotes": "Deterministic tests cover one shared healthy child per runtime process plus at most four bounded fault-quarantine children, native desktop, paired-runtime, and WSL snapshot processes sharing the same eight-physical-child reservation with typed serialized event-driven capacity recovery, including recursive re-wait when crash recovery reclaims an announced slot, one quarantine attempt per watch lifetime, generation-scoped cancellation, bounded termination with removable deadline waiters, replacement-crawl errors that cannot be blessed by a late readiness ack, pre-ready paired-web cancellation registered before an unbounded capacity wait, already-resolved and late physical-exit retry of a rejected paired-web teardown, physical desktop-install and local/SSH PTY teardown, stale-generation rejection, per-child stat bounds, final-overflow RPC delivery, and renderer eviction before terminal callbacks can retry. Destructive local and SSH removal fences both ID-derived and resolved-cwd terminal roots, including sibling-root/cwd combinations with reverse admission rollback, closes descendant relay watches before parent deletion, enumerates authoritative provider/cwd ownership, falls back to daemon spawn cwd before OSC 7, and keeps daemon/relay immediate shutdown pending until native exit or a bounded fail-closed error; a dead relay PID discovered during attach also settles concurrent shutdown before its stale entry is reaped, while Windows runtime watcher deadlines retain one removable physical-close waiter, treat Node's error-close path as positive physical-exit proof, and clear root ownership on late close. The relay policy test maps a standard repository's base, Git common directory, and worktree roots to one healthy supervisor, then proves a shared-child failure recovers those roots in separate quarantine supervisors with overflow and resumed events. Abortable capacity, quarantine, runtime-root, relay, SSH, and relay pre-install setup waits attach one reaction to each shared promise and explicitly remove 10,000 cancelled caller closures while one anchor remains. Built-entry macOS harnesses kill the desktop/runtime and SSH relay watcher children, then require automatic resubscription and later events while the host process and relay PTY survive. SSH bundles the child boundary beside relay.js, requires both artifacts for install completeness, and preserves registration-owned same-root cancellation.", "motivatingLinks": [ "https://github.com/stablyai/orca/issues/5308", "https://github.com/stablyai/orca/issues/7547", "https://github.com/stablyai/orca/issues/8212" ], - "invariant": "Local runtime roots use at most four shared watcher children while healthy, while each SSH relay process uses at most one; either pool uses at most four additional quarantine children after shard failure. Cancelling queued setup removes only that request; cancelling an active crawl restarts one shard and restores its healthy records. Setup deadlines start only when the child owns the crawl; the runtime RPC transport owns the normal 15-second initial budget, while a 60-second recovery deadline quarantines a root that would otherwise pin its shard. A crash-fused shared shard moves affected roots into the bounded quarantine pool, and a second fused quarantine ends those watch lifetimes instead of spawning replacements indefinitely. Every interrupted setup emits overflow after recovery, event delivery stays bounded, and each SSH target's relay, PTYs, Git, and other filesystem operations remain isolated from other targets and never load the watcher native addon or share its fault boundary.", - "oracle": "Fill the four-child healthy runtime pool with multiple roots, and map a standard SSH repository's base, Git common directory, and worktree roots to one relay supervisor. Inject IPC disconnect-before-exit, queued and active setup cancellation, initial and crash-resubscribe timeouts, and shard-level terminal failure, then assert healthy roots are restored, affected roots spread across no more than four quarantine supervisors, a second quarantine fuse creates no replacement, stale generations stop delivering, and existing same-root clients receive overflow plus later events without error/end. Abort a pending desktop install and require worktree close and app shutdown to settle before the native crawl resolves. Verify relay build hashes and remote install probes require both relay.js and relay-watcher.js. Launch the built relay with a real PTY and Parcel watch, kill only the watcher child, and require the same relay PID and PTY to answer before a replacement child delivers overflow and a post-crash event.", + "invariant": "Local runtime and SSH relay roots share one watcher child per host process while healthy and use at most four quarantine children after shard failure; native desktop, paired-runtime, and direct WSL snapshot children consume the same process-wide eight-child reservation and retry transient exhaustion without permanently disabling a root. Cancellation remains owned by its child generation and settles only after acknowledgement or bounded physical exit; paired-web retains cleanup ownership and can cancel before readiness even while capacity is exhausted, retries a failed unwatch after already-resolved or late physical exit without duplicating stream termination, replacements never overlap a still-retiring generation, and aborted or timed-out callers detach immediately even while queued behind same-root relay teardown. Destructive local and SSH cleanup holds one containment-aware watcher/terminal removal fence through descendant watcher and PTY physical exit plus parent Git/filesystem deletion; daemon and relay kill requests retain ownership until the actual subprocess exit callback or an equivalent positive dead-PID proof and propagate rejected signals or a bounded failure otherwise. A crash-fused shared shard moves affected roots into the bounded quarantine pool, a second fused quarantine ends those watch lifetimes, interrupted setup emits overflow after recovery, metadata work stays bounded, and the SSH relay, PTYs, Git, and other filesystem operations never load the watcher native addon or share its fault boundary.", + "oracle": "Fill the global watcher-child reservation, require native desktop, paired-runtime, and WSL setup to retain typed transient capacity and retry after one serialized release, then cancel pending desktop and paired-client watches and require no retry. Hold one live anchor while aborting 10,000 callers at the capacity, quarantine-assignment, runtime-root, relay-setup, and SSH-registration layers; require exactly one explicit waiter and one base-promise reaction to remain, then zero after settlement. Hold an old relay unsubscribe, queue one same-root setup leader and one anchor, then abort 10,000 joiners; require every cancellation to settle before teardown releases, exactly one anchor waiter, and only one later physical install. Advance the former paired-web callback deadline while capacity remains full, require cancellation ownership to stay live, publish the server token before ready, and require files.unwatch to abort setup before a later slot release can install a native root. Reject the first ready-watch unwatch with a physical-exit promise, resolve that exit later, and require a fresh physical unwatch attempt, registry release, and exactly one stream end. Fill one healthy shared child with multiple roots, map a standard SSH repository's base, Git common directory, and worktree roots to one relay supervisor, then inject IPC disconnect-before-exit, queued and active cancellation, operations during termination, kill-without-exit, repeated unkillable generations, initial and recovery timeouts, and shard failure. During local and SSH deletion, attempt a late terminal spawn whose worktree identity and cwd name different roots, enumerate a non-prefix remote PTY by authoritative worktree ownership, withhold daemon/relay exit callbacks, make attach positively prove a concurrent-shutdown PID dead, and make the provider unavailable; require every acquired admission to roll back on rejection, dead-PID proof to settle physical exit before reaping, timeout to retain ownership, and Git/filesystem deletion never to start. Require bounded watcher process count, physical-exit settlement, healthy-root recovery, stale-generation rejection, overflow plus later events, and fail-closed destructive cleanup. Verify relay build hashes and remote install probes require both relay.js and relay-watcher.js. Launch the built relay with a real PTY and Parcel watch, kill only the watcher child, and require the same relay PID and PTY to answer before a replacement child delivers overflow and a post-crash event.", "commands": [ - "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/parcel-watcher-process.test.ts src/main/ipc/runtime-watcher-process-pool.test.ts src/main/ipc/parcel-watcher-process-entry.test.ts src/main/ipc/parcel-watcher-event-delivery.test.ts src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts src/main/ipc/filesystem-watcher-wsl.test.ts src/main/runtime/file-watcher-host.test.ts src/main/runtime/rpc/methods/files.test.ts src/main/providers/ssh-filesystem-provider.test.ts src/renderer/src/runtime/runtime-file-client.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/rpc/methods/files-watch-cleanup.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-watcher-pending-setup-waiters.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-watcher-parent-removal.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/parcel-watcher-unsubscribe-timeout.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/filesystem-watcher.test.ts src/main/ipc/filesystem-watcher-native-capacity.test.ts src/main/ipc/parcel-watcher-child-registry.test.ts src/main/ipc/parcel-watcher-child-launch.test.ts src/main/ipc/parcel-watcher-process.test.ts src/main/ipc/parcel-watcher-supervisor-capacity-wait.test.ts src/main/ipc/parcel-watcher-supervisor-capacity.test.ts src/main/ipc/parcel-watcher-disconnect-termination.test.ts src/main/ipc/runtime-watcher-pending-assignment.test.ts src/main/ipc/runtime-watcher-process-pool.test.ts src/main/ipc/parcel-watcher-process-entry.test.ts src/main/ipc/parcel-watcher-event-delivery.test.ts src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts src/main/ipc/filesystem-watcher-wsl.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/watcher-removal-gate.test.ts src/main/runtime/file-watcher-host.test.ts src/main/runtime/orca-runtime-files-watch.test.ts src/main/runtime/worktree-teardown.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/files.test.ts src/main/runtime/rpc/methods/files-watch-cancellation.test.ts src/main/providers/ssh-filesystem-provider-watch-waiters.test.ts src/main/providers/ssh-filesystem-provider.test.ts src/renderer/src/runtime/runtime-file-client.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/cross-platform-path.test.ts src/shared/physical-exit-tracker.test.ts src/shared/promise-settlement-waiters.test.ts src/main/ipc/pty.test.ts src/main/providers/local-pty-provider.test.ts src/main/daemon/daemon-pty-adapter.test.ts src/main/daemon/pty-subprocess.test.ts src/main/daemon/session.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/terminal-host-session-reaping-leak.test.ts src/main/providers/ssh-pty-provider.test.ts src/relay/relay-watcher-setup-wait.test.ts src/relay/pty-handler.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/filesystem-watcher.test.ts src/main/ipc/filesystem-watcher-native-capacity.test.ts src/main/ipc/parcel-watcher-child-registry.test.ts src/main/ipc/parcel-watcher-child-launch.test.ts src/main/ipc/parcel-watcher-process.test.ts src/main/ipc/parcel-watcher-supervisor-capacity.test.ts src/main/ipc/parcel-watcher-disconnect-termination.test.ts src/main/ipc/runtime-watcher-process-pool.test.ts src/main/ipc/parcel-watcher-process-entry.test.ts src/main/ipc/parcel-watcher-event-delivery.test.ts src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts src/main/ipc/filesystem-watcher-wsl.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/watcher-removal-gate.test.ts src/main/runtime/file-watcher-host.test.ts src/main/runtime/orca-runtime-files-watch.test.ts src/main/runtime/worktree-teardown.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/files.test.ts src/main/runtime/rpc/methods/files-watch-cancellation.test.ts src/main/providers/ssh-filesystem-provider.test.ts src/renderer/src/runtime/runtime-file-client.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/cross-platform-path.test.ts src/shared/physical-exit-tracker.test.ts src/main/ipc/pty.test.ts src/main/providers/local-pty-provider.test.ts src/main/daemon/daemon-pty-adapter.test.ts src/main/daemon/pty-subprocess.test.ts src/main/daemon/session.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/terminal-host-session-reaping-leak.test.ts src/main/providers/ssh-pty-provider.test.ts src/relay/pty-handler.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/filesystem-watcher.test.ts src/main/ipc/parcel-watcher-process.test.ts src/main/ipc/parcel-watcher-disconnect-termination.test.ts src/main/ipc/runtime-watcher-process-pool.test.ts src/main/ipc/parcel-watcher-process-entry.test.ts src/main/ipc/parcel-watcher-event-delivery.test.ts src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts src/main/ipc/filesystem-watcher-wsl.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/watcher-removal-gate.test.ts src/main/runtime/file-watcher-host.test.ts src/main/runtime/orca-runtime-files-watch.test.ts src/main/runtime/worktree-teardown.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/files.test.ts src/main/providers/ssh-filesystem-provider.test.ts src/renderer/src/runtime/runtime-file-client.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/cross-platform-path.test.ts src/main/ipc/pty.test.ts src/main/providers/local-pty-provider.test.ts src/main/daemon/pty-subprocess.test.ts src/main/daemon/session.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/terminal-host-session-reaping-leak.test.ts src/main/providers/ssh-pty-provider.test.ts src/relay/pty-handler.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/parcel-watcher-process.test.ts src/main/ipc/parcel-watcher-disconnect-termination.test.ts src/main/ipc/runtime-watcher-process-pool.test.ts src/main/ipc/parcel-watcher-process-entry.test.ts src/main/ipc/parcel-watcher-event-delivery.test.ts src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts src/main/ipc/filesystem-watcher-wsl.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/watcher-removal-gate.test.ts src/main/runtime/file-watcher-host.test.ts src/main/runtime/orca-runtime-files-watch.test.ts src/main/runtime/worktree-teardown.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/files.test.ts src/main/providers/ssh-filesystem-provider.test.ts src/renderer/src/runtime/runtime-file-client.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/cross-platform-path.test.ts src/main/ipc/pty.test.ts src/main/providers/local-pty-provider.test.ts src/main/daemon/pty-subprocess.test.ts src/main/daemon/session.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/terminal-host-session-reaping-leak.test.ts src/main/providers/ssh-pty-provider.test.ts src/relay/pty-handler.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/parcel-watcher-process.test.ts src/main/ipc/parcel-watcher-disconnect-termination.test.ts src/main/ipc/runtime-watcher-process-pool.test.ts src/main/ipc/parcel-watcher-process-entry.test.ts src/main/ipc/parcel-watcher-event-delivery.test.ts src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts src/main/ipc/filesystem-watcher-wsl.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/watcher-removal-gate.test.ts src/main/runtime/file-watcher-host.test.ts src/main/runtime/orca-runtime-files-watch.test.ts src/main/runtime/worktree-teardown.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/files.test.ts src/main/providers/ssh-filesystem-provider.test.ts src/renderer/src/runtime/runtime-file-client.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/cross-platform-path.test.ts src/main/ipc/pty.test.ts src/main/providers/local-pty-provider.test.ts src/main/daemon/session.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/terminal-host-session-reaping-leak.test.ts src/main/providers/ssh-pty-provider.test.ts src/relay/pty-handler.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/parcel-watcher-process.test.ts src/main/ipc/parcel-watcher-disconnect-termination.test.ts src/main/ipc/runtime-watcher-process-pool.test.ts src/main/ipc/parcel-watcher-process-entry.test.ts src/main/ipc/parcel-watcher-event-delivery.test.ts src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts src/main/ipc/filesystem-watcher-wsl.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/watcher-removal-gate.test.ts src/main/runtime/file-watcher-host.test.ts src/main/runtime/orca-runtime-files-watch.test.ts src/main/runtime/worktree-teardown.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/files.test.ts src/main/providers/ssh-filesystem-provider.test.ts src/renderer/src/runtime/runtime-file-client.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/cross-platform-path.test.ts src/main/ipc/pty.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/terminal-host-session-reaping-leak.test.ts src/main/providers/ssh-pty-provider.test.ts src/relay/pty-handler.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/parcel-watcher-process.test.ts src/main/ipc/parcel-watcher-disconnect-termination.test.ts src/main/ipc/runtime-watcher-process-pool.test.ts src/main/ipc/parcel-watcher-process-entry.test.ts src/main/ipc/parcel-watcher-event-delivery.test.ts src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts src/main/ipc/filesystem-watcher-wsl.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/watcher-removal-gate.test.ts src/main/runtime/file-watcher-host.test.ts src/main/runtime/orca-runtime-files-watch.test.ts src/main/runtime/worktree-teardown.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/files.test.ts src/main/providers/ssh-filesystem-provider.test.ts src/renderer/src/runtime/runtime-file-client.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/cross-platform-path.test.ts", "pnpm run build:electron-vite", + "node config/scripts/runtime-file-watcher-resource-probe.mjs", "node config/scripts/runtime-file-watcher-fault-harness.mjs", "ELECTRON_RUN_AS_NODE=1 pnpm exec electron config/scripts/runtime-file-watcher-fault-harness.mjs", - "pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-filesystem-watch-registry.test.ts src/relay/fs-handler.test.ts src/relay/subprocess.test.ts src/main/ssh/ssh-relay-versioned-install.test.ts config/scripts/package-electron-runtime-contract.test.mjs", + "pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-watcher-setup-wait.test.ts src/relay/relay-filesystem-watch-registry.test.ts src/relay/fs-handler.test.ts src/relay/git-handler.test.ts src/relay/subprocess.test.ts src/main/ssh/ssh-relay-versioned-install.test.ts config/scripts/package-electron-runtime-contract.test.mjs", + "pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-filesystem-watch-registry.test.ts src/relay/fs-handler.test.ts src/relay/git-handler.test.ts src/relay/subprocess.test.ts src/main/ssh/ssh-relay-versioned-install.test.ts config/scripts/package-electron-runtime-contract.test.mjs", "pnpm run build:relay", "node config/scripts/relay-watcher-fault-harness.mjs" ], "testFiles": [ + "src/main/daemon/daemon-pty-adapter.test.ts", + "src/main/ipc/filesystem-watcher.test.ts", + "src/main/ipc/filesystem-watcher-native-capacity.test.ts", + "src/main/ipc/parcel-watcher-child-registry.test.ts", + "src/main/ipc/parcel-watcher-child-launch.test.ts", "src/main/ipc/parcel-watcher-process.test.ts", + "src/main/ipc/parcel-watcher-unsubscribe-timeout.test.ts", + "src/main/ipc/parcel-watcher-supervisor-capacity-wait.test.ts", + "src/main/ipc/parcel-watcher-supervisor-capacity.test.ts", + "src/main/ipc/parcel-watcher-disconnect-termination.test.ts", + "src/main/ipc/runtime-watcher-pending-assignment.test.ts", "src/main/ipc/runtime-watcher-process-pool.test.ts", "src/main/ipc/parcel-watcher-process-entry.test.ts", "src/main/ipc/parcel-watcher-event-delivery.test.ts", "src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts", "src/main/ipc/filesystem-watcher-wsl.test.ts", + "src/main/ipc/worktrees.test.ts", + "src/main/ipc/watcher-removal-gate.test.ts", + "src/main/ipc/pty.test.ts", + "src/main/providers/local-pty-provider.test.ts", + "src/main/daemon/pty-subprocess.test.ts", + "src/main/daemon/session.test.ts", + "src/main/daemon/terminal-host.test.ts", + "src/main/daemon/terminal-host-session-reaping-leak.test.ts", "src/main/runtime/file-watcher-host.test.ts", + "src/main/runtime/orca-runtime-files-watch.test.ts", + "src/main/runtime/worktree-teardown.test.ts", + "src/main/runtime/orca-runtime.test.ts", "src/main/runtime/rpc/methods/files.test.ts", + "src/main/runtime/rpc/methods/files-watch-cancellation.test.ts", + "src/main/runtime/rpc/methods/files-watch-cleanup.test.ts", "src/main/providers/ssh-filesystem-provider.test.ts", + "src/main/providers/ssh-filesystem-provider-watch-waiters.test.ts", + "src/main/providers/ssh-pty-provider.test.ts", "src/renderer/src/runtime/runtime-file-client.test.ts", + "src/renderer/src/web/web-runtime-client.test.ts", + "src/shared/cross-platform-path.test.ts", + "src/shared/physical-exit-tracker.test.ts", + "src/shared/promise-settlement-waiters.test.ts", "config/scripts/runtime-file-watcher-fault-harness.mjs", + "config/scripts/runtime-file-watcher-resource-probe.mjs", "src/relay/relay-filesystem-watch-registry.test.ts", + "src/relay/relay-watcher-pending-setup-waiters.test.ts", + "src/relay/relay-watcher-parent-removal.test.ts", + "src/relay/relay-watcher-setup-wait.test.ts", "src/relay/fs-handler.test.ts", + "src/relay/pty-handler.test.ts", + "src/relay/git-handler.test.ts", "src/relay/subprocess.test.ts", "src/main/ssh/ssh-relay-versioned-install.test.ts", "config/scripts/package-electron-runtime-contract.test.mjs", "config/scripts/relay-watcher-fault-harness.mjs" ], "assertionRefs": [ + { + "file": "src/main/ipc/filesystem-watcher.test.ts", + "assertions": [ + "a transient WSL child-capacity rejection automatically retries on release and renderer unwatch cancels the wait" + ] + }, + { + "file": "src/main/ipc/filesystem-watcher-native-capacity.test.ts", + "assertions": [ + "typed native child-capacity exhaustion automatically retries after a serialized slot release", + "renderer unwatch cancels a pending native capacity retry before subscription restarts" + ] + }, + { + "file": "src/main/ipc/parcel-watcher-child-registry.test.ts", + "assertions": [ + "capacity notification closes the release-before-listen race, supports cancellation, fires at most once, and serializes reserving waiters per released slot" + ] + }, + { + "file": "src/main/ipc/parcel-watcher-child-launch.test.ts", + "assertions": [ + "native launcher reports typed transient capacity exhaustion without forking another child" + ] + }, + { + "file": "src/shared/physical-exit-tracker.test.ts", + "assertions": [ + "timed-out physical-exit waiters detach while active and later retries still settle on exact exit" + ] + }, + { + "file": "src/shared/promise-settlement-waiters.test.ts", + "assertions": [ + "one shared-promise reaction serves an anchor plus 10,000 abortable callers, and all cancelled closures detach immediately", + "timeout and already-aborted paths remove timers, listeners, and ownership while preserving the active anchor" + ] + }, + { + "file": "src/main/daemon/daemon-pty-adapter.test.ts", + "assertions": [ + "daemon process enumeration falls back to spawn cwd before OSC 7 reports a live cwd", + "force-killed keep-history sleep remains cold-restorable across repeated wake cycles after physical exit" + ] + }, { "file": "src/main/ipc/parcel-watcher-process.test.ts", "assertions": [ "initial-subscribe crashes complete with an interruption refresh", + "watch errors before readiness reject setup and physically cancel the native crawl", + "watch errors during replacement crawl terminate recovery and a late readiness ack cannot revive the failed root", "IPC disconnect recovery restores every existing root before accepting new setup", "queued abort removes one request without restarting healthy roots, while active abort or crawl-scoped timeout restores the shard", - "a crash-resubscribe crawl deadline ends only the stuck root and tears down its shard process", + "a crash-resubscribe crawl deadline and concurrent unsubscribe settle only after physical shard exit, or report physical-exit failure at the termination deadline", + "a close-only asynchronous spawn failure settles physical termination without waiting for the exit deadline", + "production runtime-pool disposal rejects later subscriptions without forking an overlapping child", + "Vitest cleanup resets rather than re-disposes the reusable runtime pool", + "native child launch preserves typed transient capacity exhaustion instead of reporting permanent process unavailability", "missing built entries fail closed and unavailable canary storage remains optional", "fault-harness PID reporting cannot clobber an existing filesystem path" ] }, + { + "file": "src/main/ipc/parcel-watcher-unsubscribe-timeout.test.ts", + "assertions": [ + "a canary-independent native-unsubscribe deadline restarts the shard and restores healthy siblings after physical child exit", + "an unkillable child rejects the native unsubscribe with its retained physical-exit promise only at the bounded exit deadline" + ] + }, + { + "file": "src/main/ipc/parcel-watcher-supervisor-capacity-wait.test.ts", + "assertions": [ + "10,000 cancelled capacity callers leave exactly one live anchor and zero waiters after the released slot settles" + ] + }, + { + "file": "src/main/ipc/parcel-watcher-supervisor-capacity.test.ts", + "assertions": [ + "one released physical slot launches one shared supervisor child for every waiting root, while partial/all cancellation and disposal remove the shared capacity barrier", + "a waiter recursively re-enters the event-driven barrier when crash recovery reclaims its announced slot" + ] + }, + { + "file": "src/main/ipc/parcel-watcher-disconnect-termination.test.ts", + "assertions": [ + "pending-crawl cancellation after disconnect remains pending until the exact child exits", + "ready and pending-crawl cancellation force disconnected-child exit even while a healthy sibling root remains", + "a cancellation deadline joins active disconnect termination without duplicate sibling resubscription", + "an exit-deadline failure replaces the early cancellation and remains visible to synchronous unsubscribe until physical exit" + ] + }, + { + "file": "src/main/ipc/runtime-watcher-pending-assignment.test.ts", + "assertions": [ + "10,000 cancelled quarantine-assignment callers leave exactly one live anchor and zero waiters after grant", + "an already-aborted first caller synchronously releases pending assignment ownership" + ] + }, { "file": "src/main/ipc/runtime-watcher-process-pool.test.ts", "assertions": [ - "healthy roots spread across at most four shared supervisors before sharing", + "healthy roots share one supervisor until a fault requires quarantine", "a supervisor-scoped failure spreads affected roots across at most four quarantine supervisors", "a second fused quarantine creates no further replacement for that watch lifetime", "a root-specific failure preserves healthy shard assignments", "a live root whose crash-resubscribe times out moves to quarantine", - "an older same-root teardown cannot erase a newer watch lease's assignment" + "an older same-root teardown cannot erase a newer watch lease's assignment", + "test reset explicitly reopens the production-disposed pool", + "a zero-waiter aborted assignment cannot poison a same-turn replacement subscribe" ] }, { @@ -4205,15 +4341,121 @@ "file": "src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts", "assertions": [ "worktree deletion physically aborts pending native setup and settles before its crawl resolves", - "app shutdown physically aborts pending native setup and releases a late native handle" + "app shutdown physically aborts pending native setup and releases a late native handle", + "a cancelled pending install retains physical-exit failure across an immediate second destructive close", + "desktop terminal failure remains root-owned even when the cleared subscription later unsubscribes cleanly", + "Windows drive and UNC watcher ownership is found across path casing and separator variants", + "Windows event payloads retain the physical root spelling instead of the comparison key" + ] + }, + { + "file": "src/main/ipc/filesystem-watcher-wsl.test.ts", + "assertions": [ + "unsubscribe and startup abort remain pending until the WSL watcher process physically closes", + "a termination timeout or rejected kill retains the WSL watcher owner for later physical-exit proof", + "WSL watcher processes consume the shared eight-child reservation until physical close, including synchronous startup-write cleanup" + ] + }, + { + "file": "src/main/ipc/worktrees.test.ts", + "assertions": [ + "desktop IPC deletion never starts Git removal when physical PTY teardown cannot be proven", + "normal and orphan/leftover deletion paths perform strict PTY teardown before local or SSH Git/filesystem removal", + "SSH deletion fails closed before Git when the remote PTY provider is unavailable" + ] + }, + { + "file": "src/main/ipc/watcher-removal-gate.test.ts", + "assertions": [ + "Windows-equivalent spellings share one removal fence while distinct POSIX backslash roots do not", + "desktop and runtime removal entry points cannot own overlapping destructive roots concurrently", + "parent removal waits descendant terminal spawns and rejects later descendant installs" + ] + }, + { + "file": "src/main/ipc/pty.test.ts", + "assertions": [ + "renderer PTY spawn is rejected before provider creation while destructive worktree removal holds the shared terminal gate", + "renderer and runtime spawns fence both the worktree-ID root and resolved cwd, including cross-worktree cwd ownership with rollback", + "manual and runtime-controller local PTY shutdown retain listener and provider ownership until physical exit", + "did-finish-load orphan cleanup disposes data delivery but retains the onExit listener and tracker through physical exit", + "a failed local PTY kill retains retryable provider ownership until a later physical exit" + ] + }, + { + "file": "src/main/providers/local-pty-provider.test.ts", + "assertions": [ + "local shutdown remains pending and owned until node-pty reports physical exit", + "local process enumeration retains the spawn cwd for destructive ownership fallback", + "destructive cleanup escalates an existing graceful shutdown without losing its shared exit proof", + "POSIX graceful shutdown escalates to one bounded SIGKILL when the child ignores SIGTERM", + "Windows graceful-to-destructive cleanup treats the first ConPTY kill as final and never double-closes the handle", + "orphan cleanup racing immediate shutdown shares one kill and remains owned until the queued physical exit", + "timeout and kill failure preserve ownership for a later successful retry" + ] + }, + { + "file": "src/main/daemon/pty-subprocess.test.ts", + "assertions": [ + "rejected graceful and force kills propagate without marking the wrapper dead so retries can signal the same child" + ] + }, + { + "file": "src/main/daemon/session.test.ts", + "assertions": [ + "graceful daemon termination force-kills after its deadline but retains session ownership and late output until physical exit", + "rejected graceful and force kills reopen termination state for a later retry" + ] + }, + { + "file": "src/main/daemon/terminal-host.test.ts", + "assertions": [ + "immediate daemon kill remains pending and listed until the subprocess exit callback", + "an unconfirmed immediate kill rejects at the bounded deadline, retains ownership, and blocks same-id replacement" + ] + }, + { + "file": "src/main/daemon/terminal-host-session-reaping-leak.test.ts", + "assertions": [ + "immediate daemon kill reaps the emulator and session only after physical exit" ] }, { "file": "src/main/runtime/file-watcher-host.test.ts", "assertions": [ "same-root clients share one physical watch and one client abort cannot cancel another owner's setup", + "concurrent pre-ready cancellations join one native root release before an immediate replacement subscribes", + "10,000 cancelled runtime-root callers detach while one setup anchor remains", + "root-scoped destructive cleanup joins an in-progress subscriber teardown", "shard failure preserves existing subscribers through isolated recovery and rejects stale-generation events", - "only failed isolated recovery ends subscribers, after a conservative overflow" + "only failed isolated recovery ends subscribers, after a conservative overflow", + "initial setup failure remains discoverable by root-scoped destructive cleanup until physical exit" + ] + }, + { + "file": "src/main/runtime/orca-runtime-files-watch.test.ts", + "assertions": [ + "Windows runtime cleanup waits for the native close event, fails closed at a bounded deadline, and clears retained ownership on late close without retry listeners", + "a Windows watcher error before cleanup uses Node's proven native error-close as physical-exit proof, emits final overflow and terminal error, and releases deletion ownership", + "pre-publication local runtime setup failure remains owned through destructive cleanup" + ] + }, + { + "file": "src/main/runtime/worktree-teardown.test.ts", + "assertions": [ + "overlapping runtime, provider, and registry sweeps stay behind one physical PTY-stop promise", + "destructive teardown rejects instead of releasing deletion when physical stop misses its deadline", + "cwd-owned and floating PTYs are swept while legacy rows without cwd retain prefix ownership", + "remote provider enumeration matches authoritative worktree ownership without touching the local-only registry" + ] + }, + { + "file": "src/main/runtime/orca-runtime.test.ts", + "assertions": [ + "destructive worktree teardown awaits controller stopAndWait instead of treating fire-and-forget kill as physical exit", + "runtime worktree deletion never starts Git removal when a known PTY cannot be physically stopped", + "SSH runtime deletion shuts down an authoritatively owned remote PTY before remote Git removal", + "a delayed physical-exit retry remains scoped to its registered cleanup owner and cannot tear down a same-id replacement" ] }, { @@ -4223,12 +4465,43 @@ "terminal watcher loss flushes a final overflow change before error/end and unregisters cleanup" ] }, + { + "file": "src/main/runtime/rpc/methods/files-watch-cancellation.test.ts", + "assertions": [ + "the server publishes a cleanup token before ready and files.unwatch aborts pending setup without delivering queued crawl events" + ] + }, + { + "file": "src/main/runtime/rpc/methods/files-watch-cleanup.test.ts", + "assertions": [ + "a failed paired-web unwatch retries after already-resolved or late physical exit and emits stream end exactly once" + ] + }, + { + "file": "src/main/providers/ssh-filesystem-provider-watch-waiters.test.ts", + "assertions": [ + "10,000 cancelled SSH callers detach while one registration-owned setup anchor remains" + ] + }, { "file": "src/main/providers/ssh-filesystem-provider.test.ts", "assertions": [ "an SSH caller abort rejects promptly without cancelling another same-root subscriber's shared setup", "the physical relay request is cancelled when the last pending subscriber leaves", - "provider disposal cancels pending setup and sends one remote unwatch" + "provider disposal cancels pending setup and sends one remote unwatch", + "equivalent Windows spellings share one provider generation and terminal failure reaches the surviving owner" + ] + }, + { + "file": "src/main/providers/ssh-pty-provider.test.ts", + "assertions": [ + "remote process enumeration preserves authoritative worktree ownership while scoping relay PTY ids to the SSH connection" + ] + }, + { + "file": "src/shared/cross-platform-path.test.ts", + "assertions": [ + "comparison keys preserve literal POSIX backslashes while folding Windows drive and UNC spellings" ] }, { @@ -4240,6 +4513,25 @@ "a later same-root listener opens a fresh server subscription instead of reusing a dead stream" ] }, + { + "file": "src/relay/relay-watcher-parent-removal.test.ts", + "assertions": [ + "POSIX and Windows parent deletion closes and terminally notifies descendant watches before removal while preserving path siblings", + "parent deletion waits for descendant setup publication and already-retiring descendant teardown before removal" + ] + }, + { + "file": "src/relay/relay-watcher-pending-setup-waiters.test.ts", + "assertions": [ + "10,000 aborted relay setup callers detach while one same-root teardown anchor remains and only one replacement installs" + ] + }, + { + "file": "src/relay/relay-watcher-setup-wait.test.ts", + "assertions": [ + "10,000 cancelled relay setup callers leave exactly one explicit anchor and one base-promise reaction" + ] + }, { "file": "src/relay/relay-filesystem-watch-registry.test.ts", "assertions": [ @@ -4248,19 +4540,47 @@ "replacement emits overflow before later ordered events", "terminal shard failure preserves same-root clients through recovery", "pending setup aborts only after the last same-root client leaves", + "relay-side removal sends every client a terminal failure, closes every watch, and fences new same-root watcher or PTY setup even when deletion fails", "a missing child fails closed even if the relay inherits a VITEST environment" ] }, { "file": "src/relay/fs-handler.test.ts", "assertions": [ - "shared watches retain the twenty-root budget and release after every client leaves" + "shared watches retain the twenty-root budget and release after every client leaves", + "recursive relay deletion closes and terminally notifies same-root watches before removing the directory" + ] + }, + { + "file": "src/relay/pty-handler.test.ts", + "assertions": [ + "immediate relay shutdown stays pending with the PTY map entry retained until onExit", + "a missing exit rejects at the bounded deadline, preserves the physical owner, and a retry joins without duplicate SIGKILL", + "Windows relay graceful, fallback, immediate, stale-spawn, exit-dispose, and handler-dispose paths issue at most one ConPTY kill", + "relay PTY admission succeeds when post-Node-18 array copy methods are unavailable", + "attach dead-PID proof settles a concurrent immediate shutdown before the stale PTY entry is reaped", + "concurrent spawn and revive admission is atomic, capped at fifty physical PTYs, and same-id revive is deduplicated", + "worktree removal waits explicit and default-cwd PTY creation, rejects later creation, and physically stops only authoritative matching PTYs before deletion", + "relay spawn and revive fence distinct worktree-ID and cwd roots, rolling back earlier admissions in reverse when either root is removing" + ] + }, + { + "file": "src/renderer/src/web/web-runtime-client.test.ts", + "assertions": [ + "a stopped pre-ready paired-web watch retains its callback beyond the former deadline and uses the early server token to unwatch" + ] + }, + { + "file": "src/relay/git-handler.test.ts", + "assertions": [ + "remote Git worktree removal executes inside the relay filesystem watcher fence" ] }, { "file": "src/relay/subprocess.test.ts", "assertions": [ - "the real relay bundle serves PTY, filesystem, and Git requests with its adjacent watcher child" + "the real relay bundle serves PTY, filesystem, and Git requests with its adjacent watcher child", + "the Node-18-targeted relay bundle contains no unsupported array copy methods" ] }, { @@ -4275,6 +4595,15 @@ "every release platform packages and gates the hashed watcher child" ] }, + { + "file": "config/scripts/runtime-file-watcher-resource-probe.mjs", + "assertions": [ + "five healthy roots share one exact watcher PID within the per-child RSS budget", + "four quarantine roots use exactly four child PIDs and remain within the global eight-child reservation cap", + "CPU and RSS are sampled for every exact child PID and every PID exits during cleanup", + "per-child and aggregate RSS and sampled CPU ceilings fail the probe before the physical child cap can hide excess resource use" + ] + }, { "file": "config/scripts/runtime-file-watcher-fault-harness.mjs", "assertions": [ @@ -4293,13 +4622,202 @@ ], "evidenceRuns": [ { - "date": "2026-07-12", + "date": "2026-07-13", "runner": "local", "platform": "macos", - "command": "pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-filesystem-watch-registry.test.ts src/relay/fs-handler.test.ts src/relay/subprocess.test.ts src/main/ssh/ssh-relay-versioned-install.test.ts config/scripts/package-electron-runtime-contract.test.mjs", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/filesystem-watcher.test.ts src/main/ipc/filesystem-watcher-native-capacity.test.ts src/main/ipc/parcel-watcher-child-registry.test.ts src/main/ipc/parcel-watcher-child-launch.test.ts src/main/ipc/parcel-watcher-process.test.ts src/main/ipc/parcel-watcher-supervisor-capacity-wait.test.ts src/main/ipc/parcel-watcher-supervisor-capacity.test.ts src/main/ipc/parcel-watcher-disconnect-termination.test.ts src/main/ipc/runtime-watcher-pending-assignment.test.ts src/main/ipc/runtime-watcher-process-pool.test.ts src/main/ipc/parcel-watcher-process-entry.test.ts src/main/ipc/parcel-watcher-event-delivery.test.ts src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts src/main/ipc/filesystem-watcher-wsl.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/watcher-removal-gate.test.ts src/main/runtime/file-watcher-host.test.ts src/main/runtime/orca-runtime-files-watch.test.ts src/main/runtime/worktree-teardown.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/files.test.ts src/main/runtime/rpc/methods/files-watch-cancellation.test.ts src/main/providers/ssh-filesystem-provider-watch-waiters.test.ts src/main/providers/ssh-filesystem-provider.test.ts src/renderer/src/runtime/runtime-file-client.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/cross-platform-path.test.ts src/shared/physical-exit-tracker.test.ts src/shared/promise-settlement-waiters.test.ts src/main/ipc/pty.test.ts src/main/providers/local-pty-provider.test.ts src/main/daemon/daemon-pty-adapter.test.ts src/main/daemon/pty-subprocess.test.ts src/main/daemon/session.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/terminal-host-session-reaping-leak.test.ts src/main/providers/ssh-pty-provider.test.ts src/relay/relay-watcher-setup-wait.test.ts src/relay/pty-handler.test.ts --reporter=dot", "result": "passed", - "durationSeconds": 4.5, - "summary": "Five relay registry, handler, bundled subprocess, install, and package contract files passed with 114 tests." + "durationSeconds": 11.98, + "summary": "Thirty-nine watcher, cancellation, desktop/serve, paired-web, WSL, SSH/relay, removal, and PTY physical-exit files passed with 2,008 tests. Capacity, quarantine, runtime-root, relay, and SSH setup layers each prove that 10,000 aborted callers detach while one anchor remains and zero waiters remain after settlement." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/filesystem-watcher.test.ts src/main/ipc/filesystem-watcher-native-capacity.test.ts src/main/ipc/parcel-watcher-child-registry.test.ts src/main/ipc/parcel-watcher-child-launch.test.ts src/main/ipc/parcel-watcher-process.test.ts src/main/ipc/parcel-watcher-supervisor-capacity.test.ts src/main/ipc/parcel-watcher-disconnect-termination.test.ts src/main/ipc/runtime-watcher-process-pool.test.ts src/main/ipc/parcel-watcher-process-entry.test.ts src/main/ipc/parcel-watcher-event-delivery.test.ts src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts src/main/ipc/filesystem-watcher-wsl.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/watcher-removal-gate.test.ts src/main/runtime/file-watcher-host.test.ts src/main/runtime/orca-runtime-files-watch.test.ts src/main/runtime/worktree-teardown.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/files.test.ts src/main/runtime/rpc/methods/files-watch-cancellation.test.ts src/main/providers/ssh-filesystem-provider.test.ts src/renderer/src/runtime/runtime-file-client.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/cross-platform-path.test.ts src/shared/physical-exit-tracker.test.ts src/main/ipc/pty.test.ts src/main/providers/local-pty-provider.test.ts src/main/daemon/daemon-pty-adapter.test.ts src/main/daemon/pty-subprocess.test.ts src/main/daemon/session.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/terminal-host-session-reaping-leak.test.ts src/main/providers/ssh-pty-provider.test.ts src/relay/pty-handler.test.ts --reporter=dot", + "result": "passed", + "durationSeconds": 11.59, + "summary": "Thirty-four watcher, cancellation, desktop/serve, paired-web, WSL, removal, PTY physical-exit, provider, renderer, and path files passed with 1,999 tests; paired-web cancellation now owns a server token before ready and aborts pending setup without a late native install." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "node config/scripts/runtime-file-watcher-resource-probe.mjs", + "result": "passed", + "durationSeconds": 0.58, + "summary": "Five healthy roots shared one child at 50,688 KiB RSS and 4.9% sampled CPU; four quarantine children totaled 202,368 KiB RSS and 29.5% sampled CPU, all budgets and the eight-child cap passed, and exact-PID cleanup was true." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/filesystem-watcher.test.ts src/main/ipc/parcel-watcher-process.test.ts src/main/ipc/parcel-watcher-disconnect-termination.test.ts src/main/ipc/runtime-watcher-process-pool.test.ts src/main/ipc/parcel-watcher-process-entry.test.ts src/main/ipc/parcel-watcher-event-delivery.test.ts src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts src/main/ipc/filesystem-watcher-wsl.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/watcher-removal-gate.test.ts src/main/runtime/file-watcher-host.test.ts src/main/runtime/orca-runtime-files-watch.test.ts src/main/runtime/worktree-teardown.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/files.test.ts src/main/providers/ssh-filesystem-provider.test.ts src/renderer/src/runtime/runtime-file-client.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/cross-platform-path.test.ts src/main/ipc/pty.test.ts src/main/providers/local-pty-provider.test.ts src/main/daemon/pty-subprocess.test.ts src/main/daemon/session.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/terminal-host-session-reaping-leak.test.ts src/main/providers/ssh-pty-provider.test.ts src/relay/pty-handler.test.ts --reporter=dot", + "result": "passed", + "durationSeconds": 10.01, + "summary": "Twenty-seven watcher, WSL, desktop/serve removal, local/daemon/SSH PTY physical-exit, provider ownership, RPC, renderer, relay PTY admission, and cross-platform path files passed with 1,886 tests. The suite includes nested removal fencing, renderer/runtime cwd-only admission, legacy provider rows, rejected kill retries, transient WSL capacity, and synchronous-startup reservation cleanup." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "node config/scripts/runtime-file-watcher-resource-probe.mjs", + "result": "passed", + "durationSeconds": 0.56, + "summary": "Five healthy roots shared one child at 50,176 KiB RSS and 6.0% sampled CPU; four quarantine children totaled 200,112 KiB RSS and 18.0% sampled CPU. The global eight-child cap and all per-child/aggregate budgets passed, and exact-PID cleanup was true." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-filesystem-watch-registry.test.ts src/relay/fs-handler.test.ts src/relay/git-handler.test.ts src/relay/subprocess.test.ts src/main/ssh/ssh-relay-versioned-install.test.ts config/scripts/package-electron-runtime-contract.test.mjs", + "result": "passed", + "durationSeconds": 10.01, + "summary": "Six relay registry, filesystem/Git handler, bundled subprocess, install, and package contract files passed with 226 tests." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "pnpm run build:electron-vite", + "result": "passed", + "durationSeconds": 24.41, + "summary": "The production Electron main, preload, renderer, daemon, and watcher child entries built successfully." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "node config/scripts/runtime-file-watcher-fault-harness.mjs", + "result": "passed", + "durationSeconds": 0.1, + "summary": "The Node host survived watcher-child SIGSEGV, replaced and resubscribed the child, and delivered a post-crash event." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "ELECTRON_RUN_AS_NODE=1 pnpm exec electron config/scripts/runtime-file-watcher-fault-harness.mjs", + "result": "passed", + "durationSeconds": 0.46, + "summary": "The Electron-as-Node host survived watcher-child SIGSEGV, replaced and resubscribed the child, and delivered a post-crash event." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "pnpm run build:relay", + "result": "passed", + "durationSeconds": 0.38, + "summary": "Relay and adjacent watcher artifacts built for Linux, macOS, and Windows on x64 and arm64, plus the WSL hook relay." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "node config/scripts/relay-watcher-fault-harness.mjs", + "result": "passed", + "durationSeconds": 0.12, + "summary": "The relay and existing PTY survived watcher-child SIGSEGV; replacement delivered overflow and a precise post-crash event." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/parcel-watcher-process.test.ts src/main/ipc/parcel-watcher-disconnect-termination.test.ts src/main/ipc/runtime-watcher-process-pool.test.ts src/main/ipc/parcel-watcher-process-entry.test.ts src/main/ipc/parcel-watcher-event-delivery.test.ts src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts src/main/ipc/filesystem-watcher-wsl.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/watcher-removal-gate.test.ts src/main/runtime/file-watcher-host.test.ts src/main/runtime/orca-runtime-files-watch.test.ts src/main/runtime/worktree-teardown.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/files.test.ts src/main/providers/ssh-filesystem-provider.test.ts src/renderer/src/runtime/runtime-file-client.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/cross-platform-path.test.ts src/main/ipc/pty.test.ts src/main/providers/local-pty-provider.test.ts src/main/daemon/session.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/terminal-host-session-reaping-leak.test.ts src/main/providers/ssh-pty-provider.test.ts src/relay/pty-handler.test.ts", + "result": "passed", + "durationSeconds": 9.12, + "summary": "Twenty-five watcher, WSL, desktop/serve removal, local/daemon/SSH PTY physical-exit, provider ownership, RPC, renderer, relay PTY admission, and cross-platform path files passed with 1,762 tests. The suite covers operations joining in-flight termination, kill failures retaining ownership, bounded timeouts, atomic relay spawn/revive caps, and PTY-removal admission." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "node config/scripts/runtime-file-watcher-resource-probe.mjs", + "result": "passed", + "durationSeconds": 0.56, + "summary": "Five healthy roots shared one child at 50,096 KiB RSS and 3.4% sampled CPU; four quarantine children totaled 200,576 KiB RSS and 25.0% sampled CPU. The eight-child physical cap and every per-child/aggregate CPU and RSS budget passed, and exact-PID cleanup was true." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-filesystem-watch-registry.test.ts src/relay/fs-handler.test.ts src/relay/git-handler.test.ts src/relay/subprocess.test.ts src/main/ssh/ssh-relay-versioned-install.test.ts config/scripts/package-electron-runtime-contract.test.mjs", + "result": "passed", + "durationSeconds": 10.69, + "summary": "Six relay registry, filesystem/Git handler, bundled subprocess, install, and package contract files passed with 226 tests, including watcher/PTY removal fencing and recursive deletion failure behavior." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "pnpm run build:electron-vite", + "result": "passed", + "durationSeconds": 24, + "summary": "The production Electron main, preload, renderer, daemon, and watcher child entries built successfully." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "node config/scripts/runtime-file-watcher-fault-harness.mjs", + "result": "passed", + "durationSeconds": 0.01, + "summary": "The Node host survived watcher-child SIGSEGV, replaced and resubscribed the child, and delivered a post-crash event." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "ELECTRON_RUN_AS_NODE=1 pnpm exec electron config/scripts/runtime-file-watcher-fault-harness.mjs", + "result": "passed", + "durationSeconds": 0.46, + "summary": "The Electron-as-Node host survived watcher-child SIGSEGV, replaced and resubscribed the child, and delivered a post-crash event." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "pnpm run build:relay", + "result": "passed", + "durationSeconds": 0.39, + "summary": "Relay and adjacent watcher artifacts built for Linux, macOS, and Windows on x64 and arm64, plus the WSL hook relay." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "node config/scripts/relay-watcher-fault-harness.mjs", + "result": "passed", + "durationSeconds": 0.12, + "summary": "The relay and existing PTY survived watcher-child SIGSEGV; replacement delivered overflow and a precise post-crash event." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/parcel-watcher-process.test.ts src/main/ipc/parcel-watcher-disconnect-termination.test.ts src/main/ipc/runtime-watcher-process-pool.test.ts src/main/ipc/parcel-watcher-process-entry.test.ts src/main/ipc/parcel-watcher-event-delivery.test.ts src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts src/main/ipc/filesystem-watcher-wsl.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/watcher-removal-gate.test.ts src/main/runtime/file-watcher-host.test.ts src/main/runtime/orca-runtime-files-watch.test.ts src/main/runtime/worktree-teardown.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/files.test.ts src/main/providers/ssh-filesystem-provider.test.ts src/renderer/src/runtime/runtime-file-client.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/cross-platform-path.test.ts src/main/ipc/pty.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/terminal-host-session-reaping-leak.test.ts src/main/providers/ssh-pty-provider.test.ts src/relay/pty-handler.test.ts", + "result": "passed", + "durationSeconds": 9.03, + "summary": "Twenty-three watcher, desktop/serve removal, terminal-spawn gate, daemon/SSH physical PTY exit, provider ownership, RPC, renderer, and cross-platform path files passed with 1,630 tests. The suite includes same-turn watcher replacement, authoritative SSH teardown before Git, bounded kill-without-exit failures that retain ownership, and local/SSH deletion fail-closed behavior." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "node config/scripts/runtime-file-watcher-resource-probe.mjs", + "result": "passed", + "durationSeconds": 0.55, + "summary": "Five healthy roots shared one child at 50,160 KiB RSS and 5.6% sampled CPU; four quarantine children totaled 199,968 KiB RSS and 16.7% sampled CPU, hard per-child and aggregate RSS/CPU budgets passed, the eight-child reservation cap rejected the next launch, and every exact PID exited during cleanup." + }, + { + "date": "2026-07-13", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/relay/relay-filesystem-watch-registry.test.ts src/relay/fs-handler.test.ts src/relay/git-handler.test.ts src/relay/subprocess.test.ts src/main/ssh/ssh-relay-versioned-install.test.ts config/scripts/package-electron-runtime-contract.test.mjs", + "result": "passed", + "durationSeconds": 9.79, + "summary": "Six relay registry, filesystem/Git handler, bundled subprocess, install, and package contract files passed with 225 tests, including terminal removal notification and recursive deletion fencing." }, { "date": "2026-07-12", @@ -4311,13 +4829,13 @@ "summary": "The relay PID and existing PTY survived watcher-child SIGSEGV; overflow and precise events resumed from a replacement child." }, { - "date": "2026-07-12", + "date": "2026-07-13", "runner": "local", "platform": "macos", - "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/parcel-watcher-process.test.ts src/main/ipc/runtime-watcher-process-pool.test.ts src/main/ipc/parcel-watcher-process-entry.test.ts src/main/ipc/parcel-watcher-event-delivery.test.ts src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts src/main/ipc/filesystem-watcher-wsl.test.ts src/main/runtime/file-watcher-host.test.ts src/main/runtime/rpc/methods/files.test.ts src/main/providers/ssh-filesystem-provider.test.ts src/renderer/src/runtime/runtime-file-client.test.ts", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/parcel-watcher-process.test.ts src/main/ipc/parcel-watcher-disconnect-termination.test.ts src/main/ipc/runtime-watcher-process-pool.test.ts src/main/ipc/parcel-watcher-process-entry.test.ts src/main/ipc/parcel-watcher-event-delivery.test.ts src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts src/main/ipc/filesystem-watcher-wsl.test.ts src/main/ipc/worktrees.test.ts src/main/ipc/watcher-removal-gate.test.ts src/main/runtime/file-watcher-host.test.ts src/main/runtime/orca-runtime-files-watch.test.ts src/main/runtime/worktree-teardown.test.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/methods/files.test.ts src/main/providers/ssh-filesystem-provider.test.ts src/renderer/src/runtime/runtime-file-client.test.ts src/renderer/src/web/web-runtime-client.test.ts src/shared/cross-platform-path.test.ts", "result": "passed", - "durationSeconds": 1.7, - "summary": "10 focused supervisor, pool, child-entry, delivery, desktop/WSL cleanup, runtime-host, RPC, SSH ownership, and renderer-client test files passed with 223 tests." + "durationSeconds": 8.85, + "summary": "18 focused supervisor, disconnect termination, pool, child-entry, delivery, desktop/WSL cleanup and IPC removal, removal-gate, runtime-host/commands and physical PTY teardown, RPC, SSH ownership, renderer/paired-web clients, and cross-platform path files passed with 1,193 tests." }, { "date": "2026-07-12", @@ -4361,7 +4879,7 @@ }, "performanceBudget": { "required": true, - "evidence": "Local runtime operation remains capped at four shared child processes, while each SSH relay is capped at one healthy child instead of spreading a standard repository's three roots across three children. Based on the measured 51.9 MiB per child, normal SSH watcher overhead is bounded near 51.9 MiB rather than 155.7 MiB before relay and PTY overhead. Degraded operation adds at most four quarantine children, distributes affected roots by least load, disposes empty slots, and permits no replacement after a quarantine shard itself fuses. Each child caps directory stats at eight across its assigned roots (at most 32 healthy or 64 total logical stat operations for local runtime; 8 healthy or 40 total per SSH relay), each subscription retains at most one active plus one bounded pending batch, and no polling or serve-process stat fanout is added." + "evidence": "The committed resource probe reproducibly measures five healthy roots sharing one child (50,160 KiB RSS, 5.6% sampled CPU) and four quarantine children (199,968 KiB total RSS, 16.7% sampled CPU), enforces 128 MiB/50% per-child and 512 MiB/100% quarantine aggregate budgets, derives a 1 GiB/400% ceiling across the global eight-physical-child cap, and waits for every exact PID during cleanup. A relay policy test separately proves a standard SSH repository's three roots share one healthy child. Shared pending setup attaches one base-promise reaction per generation; quantitative owner tests abort 10,000 callers at each affected layer and require exactly one anchor, then zero explicit waiters after settlement. The shared supervisor capacity barrier is event-driven, one-shot, abortable, and serialized per released physical slot, waking every same-child desktop, paired-runtime, or SSH relay root without consuming crash-recovery attempts; direct WSL capacity retains the same event-driven desktop retry. Relay PTY admission adds at most two deduplicated path checks per spawn/revive and releases both without polling or retained listeners, while dead-PID reaping settles an existing tracker without allocating another timer. Fault quarantine distributes affected roots by least load, disposes empty slots, and permits no replacement after a quarantine shard itself fuses. Each child caps directory stats at eight across its assigned roots (at most eight healthy or 32 quarantined operations), each subscription retains at most one active plus one bounded pending batch, and no additional polling or serve-process stat fanout is added." }, "promotionCriteria": [ "Run the built-entry SIGSEGV harness in soak for at least 100 consecutive passes or 14 days on macOS.", @@ -4370,11 +4888,11 @@ ], "knownGaps": [ "The fault harness bundles the real source supervisor and drives the built child, but it does not launch a packaged orca serve pairing session or exercise app.asar path resolution.", - "The one-child SSH healthy and four-child quarantine memory ceilings are inferred from the earlier one-child measurement; direct degraded-state RSS plus sustained six-writer event-loop-delay and CPU soak evidence is still absent.", + "The committed resource probe covers the one-child healthy pool, four-child fault-quarantine ceiling, RSS/CPU sampling, global physical-child cap, and exact-PID cleanup, but sustained six-writer event-loop-delay and CPU soak evidence is still absent.", "Linux native watcher fault containment is source-equivalent but not live fault-injected in this evidence run.", - "Windows runtime watches bypass this path; SSH registration ownership is deterministic-contract tested but not live-relay fault-injected." + "Windows native runtime watches bypass this child path; WSL reservation/release is deterministic-contract tested but not live fault-injected, and SSH registration ownership is not live-relay fault-injected." ], - "demotionRule": "Demote or quarantine if the fault harness flakes without a product or harness bug, if healthy operation exceeds four local runtime watcher children or one child in any SSH relay, if total healthy plus quarantine operation exceeds eight locally or five in an SSH relay, if quarantine children outlive their roots or repeat after fusing, if event delivery becomes unbounded, or if metadata/stat work returns to the serve process." + "demotionRule": "Demote or quarantine if the fault harness flakes without a product or harness bug, if healthy operation exceeds one runtime watcher child, if total physical operation exceeds eight children including retiring generations, if quarantine children outlive their roots or repeat after fusing, if event delivery becomes unbounded, or if metadata/stat work returns to the serve process." } ] } diff --git a/config/scripts/runtime-file-watcher-resource-probe.mjs b/config/scripts/runtime-file-watcher-resource-probe.mjs new file mode 100644 index 00000000000..67d2df02a63 --- /dev/null +++ b/config/scripts/runtime-file-watcher-resource-probe.mjs @@ -0,0 +1,257 @@ +import { execFile } from 'node:child_process' +import { existsSync } from 'node:fs' +import { mkdtemp, realpath, rm } from 'node:fs/promises' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { promisify } from 'node:util' +import { build } from 'esbuild' + +const ENTRY_PATH = resolve('out/main/parcel-watcher-process-entry.js') +const POOL_SOURCE = resolve('src/main/ipc/runtime-watcher-process-pool.ts') +const FAILURE_SOURCE = resolve('src/main/ipc/parcel-watcher-process-failure.ts') +const REGISTRY_SOURCE = resolve('src/main/ipc/parcel-watcher-child-registry.ts') +const MAX_CHILD_RSS_KIB = 128 * 1024 +const MAX_CHILD_CPU_PERCENT = 50 +const MAX_QUARANTINE_RSS_KIB = 512 * 1024 +const MAX_QUARANTINE_CPU_PERCENT = 100 +const PHYSICAL_CHILD_CAP = 8 +const WAIT_TIMEOUT_MS = 15_000 +const execFileAsync = promisify(execFile) +const require = createRequire(import.meta.url) + +async function main() { + if (!existsSync(ENTRY_PATH)) { + throw new Error(`Missing ${ENTRY_PATH}; run pnpm run build:electron-vite first`) + } + + const createdRoots = [] + const exactPids = new Set() + let bundleDir + let pool + let result + try { + bundleDir = await mkdtemp(join(tmpdir(), 'orca-runtime-watcher-resource-')) + const { RuntimeWatcherProcessPool, WatcherProcessFailure, reserveWatcherChild } = + await loadProbe(bundleDir) + const roots = [] + for (let index = 0; index < 5; index++) { + const createdRoot = await mkdtemp(join(tmpdir(), `orca-watcher-resource-${index}-`)) + createdRoots.push(createdRoot) + roots.push(await realpath(createdRoot)) + } + + pool = new RuntimeWatcherProcessPool() + await Promise.all(roots.map((rootPath) => pool.subscribe(rootPath, () => undefined, {}, {}))) + const healthyPids = activeChildPids(pool) + assertPidCount(healthyPids, 1, 'five healthy roots') + healthyPids.forEach((pid) => exactPids.add(pid)) + const healthyResources = await sampleResources(healthyPids) + assertResourceBudget(healthyResources, 'healthy pool', { + maxTotalRssKiB: MAX_CHILD_RSS_KIB, + maxTotalCpuPercent: MAX_CHILD_CPU_PERCENT + }) + assertRegistryCapacity(reserveWatcherChild, 7) + + const sharedSlot = [...pool.activeSlots][0] + const failure = new WatcherProcessFailure( + 'resource probe synthetic shared-shard fault', + 'supervisor', + 'supervisor_crash_fuse' + ) + for (const record of sharedSlot.supervisor.records.values()) { + record.hooks.onTerminalError?.(failure) + } + await waitFor(() => !isPidAlive(healthyPids[0]), 'healthy child exit') + + await Promise.all( + roots.slice(0, 4).map((rootPath) => pool.subscribe(rootPath, () => undefined, {}, {})) + ) + const quarantinePids = activeChildPids(pool) + assertPidCount(quarantinePids, 4, 'four fault-quarantine roots') + quarantinePids.forEach((pid) => exactPids.add(pid)) + const quarantineResources = await sampleResources(quarantinePids) + assertResourceBudget(quarantineResources, 'quarantine pool', { + maxTotalRssKiB: MAX_QUARANTINE_RSS_KIB, + maxTotalCpuPercent: MAX_QUARANTINE_CPU_PERCENT + }) + // Four live quarantine children leave four reservations before the global + // eight-physical-child ceiling rejects the next launch. + assertRegistryCapacity(reserveWatcherChild, 4) + + result = { + hostPid: process.pid, + healthy: summarizeResources(healthyResources), + quarantine: summarizeResources(quarantineResources), + physicalChildCap: PHYSICAL_CHILD_CAP, + budgets: { + maxChildRssKiB: MAX_CHILD_RSS_KIB, + maxChildCpuPercent: MAX_CHILD_CPU_PERCENT, + maxQuarantineRssKiB: MAX_QUARANTINE_RSS_KIB, + maxQuarantineCpuPercent: MAX_QUARANTINE_CPU_PERCENT, + maxPhysicalRssKiB: MAX_CHILD_RSS_KIB * PHYSICAL_CHILD_CAP, + maxPhysicalCpuPercent: MAX_CHILD_CPU_PERCENT * PHYSICAL_CHILD_CAP + }, + exactPidCleanup: true + } + } finally { + pool?.dispose() + await Promise.all( + [...exactPids].map((pid) => waitFor(() => !isPidAlive(pid), `watcher PID ${pid} cleanup`)) + ) + await Promise.all([ + ...createdRoots.map((rootPath) => rm(rootPath, { recursive: true, force: true })), + bundleDir ? rm(bundleDir, { recursive: true, force: true }) : Promise.resolve() + ]) + } + console.log(JSON.stringify(result)) +} + +async function loadProbe(bundleDir) { + const outfile = join(bundleDir, 'watcher-resource-probe.cjs') + await build({ + stdin: { + contents: [ + `export { RuntimeWatcherProcessPool } from ${JSON.stringify(POOL_SOURCE)}`, + `export { WatcherProcessFailure } from ${JSON.stringify(FAILURE_SOURCE)}`, + `export { reserveWatcherChild } from ${JSON.stringify(REGISTRY_SOURCE)}` + ].join('\n'), + resolveDir: process.cwd(), + sourcefile: 'runtime-file-watcher-resource-probe-entry.ts', + loader: 'ts' + }, + bundle: true, + platform: 'node', + format: 'cjs', + outfile, + external: ['@parcel/watcher', 'electron'], + logLevel: 'silent' + }) + return require(outfile) +} + +function activeChildPids(pool) { + return [...pool.activeSlots] + .map((slot) => slot.supervisor.child?.pid) + .filter((pid) => Number.isInteger(pid)) +} + +function assertPidCount(pids, expected, label) { + if (new Set(pids).size !== expected || pids.some((pid) => !isPidAlive(pid))) { + throw new Error( + `${label} used ${new Set(pids).size} live watcher children; expected ${expected}` + ) + } +} + +function assertRegistryCapacity(reserveWatcherChild, availableReservations) { + const releases = [] + try { + for (let index = 0; index < availableReservations; index++) { + const release = reserveWatcherChild() + if (!release) { + throw new Error(`Watcher child registry rejected reservation ${index + 1}`) + } + releases.push(release) + } + const unexpectedReservation = reserveWatcherChild() + if (unexpectedReservation) { + unexpectedReservation() + throw new Error('Watcher child registry exceeded its global physical-process cap') + } + } finally { + releases.forEach((release) => release()) + } +} + +async function sampleResources(pids) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 250)) + return Promise.all(pids.map((pid) => sampleProcess(pid))) +} + +async function sampleProcess(pid) { + if (process.platform === 'win32') { + const command = + `$p=Get-Process -Id ${pid};` + + `[pscustomobject]@{rssKiB=[math]::Round($p.WorkingSet64/1KB);cpuSeconds=$p.CPU}|ConvertTo-Json -Compress` + const first = JSON.parse( + (await execFileAsync('powershell.exe', ['-NoProfile', '-Command', command])).stdout.trim() + ) + await new Promise((resolveDelay) => setTimeout(resolveDelay, 250)) + const second = JSON.parse( + (await execFileAsync('powershell.exe', ['-NoProfile', '-Command', command])).stdout.trim() + ) + return { + pid, + rssKiB: second.rssKiB, + cpuPercent: Math.max(0, (second.cpuSeconds - first.cpuSeconds) * 400) + } + } + const { stdout } = await execFileAsync('ps', ['-o', 'rss=', '-o', '%cpu=', '-p', String(pid)], { + env: { ...process.env, LC_ALL: 'C' } + }) + const [rssKiB, cpuPercent] = stdout.trim().split(/\s+/).map(Number) + return { pid, rssKiB, cpuPercent } +} + +function assertResourceBudget(resources, label, { maxTotalRssKiB, maxTotalCpuPercent }) { + for (const resource of resources) { + if (!Number.isFinite(resource.rssKiB) || resource.rssKiB <= 0) { + throw new Error(`Missing RSS evidence for watcher PID ${resource.pid}`) + } + if (resource.rssKiB > MAX_CHILD_RSS_KIB) { + throw new Error( + `Watcher PID ${resource.pid} RSS ${resource.rssKiB} KiB exceeded ${MAX_CHILD_RSS_KIB} KiB` + ) + } + if (!Number.isFinite(resource.cpuPercent) || resource.cpuPercent < 0) { + throw new Error(`Missing CPU evidence for watcher PID ${resource.pid}`) + } + if (resource.cpuPercent > MAX_CHILD_CPU_PERCENT) { + throw new Error( + `Watcher PID ${resource.pid} CPU ${resource.cpuPercent}% exceeded ${MAX_CHILD_CPU_PERCENT}%` + ) + } + } + const summary = summarizeResources(resources) + if (summary.totalRssKiB > maxTotalRssKiB) { + throw new Error( + `${label} RSS ${summary.totalRssKiB} KiB exceeded aggregate budget ${maxTotalRssKiB} KiB` + ) + } + if (summary.sampledCpuPercent > maxTotalCpuPercent) { + throw new Error( + `${label} CPU ${summary.sampledCpuPercent}% exceeded aggregate budget ${maxTotalCpuPercent}%` + ) + } +} + +function summarizeResources(resources) { + return { + childCount: resources.length, + pids: resources.map(({ pid }) => pid), + totalRssKiB: resources.reduce((total, { rssKiB }) => total + rssKiB, 0), + sampledCpuPercent: resources.reduce((total, { cpuPercent }) => total + cpuPercent, 0) + } +} + +function isPidAlive(pid) { + try { + process.kill(pid, 0) + return true + } catch { + return false + } +} + +async function waitFor(predicate, label) { + const deadline = Date.now() + WAIT_TIMEOUT_MS + while (!predicate()) { + if (Date.now() >= deadline) { + throw new Error(`Timed out waiting for ${label}`) + } + await new Promise((resolveDelay) => setTimeout(resolveDelay, 25)) + } +} + +await main() diff --git a/src/main/daemon/daemon-pty-adapter.test.ts b/src/main/daemon/daemon-pty-adapter.test.ts index 06a77789f4b..b3083d0ea4e 100644 --- a/src/main/daemon/daemon-pty-adapter.test.ts +++ b/src/main/daemon/daemon-pty-adapter.test.ts @@ -49,7 +49,7 @@ function createMockSubprocess(dataOnSubscribe?: string): SubprocessHandle & { pause: vi.fn<() => void>(), resume: vi.fn<() => void>(), kill: vi.fn(() => setTimeout(() => onExitCb?.(0), 5)), - forceKill: vi.fn(), + forceKill: vi.fn(() => setTimeout(() => onExitCb?.(137), 5)), signal: vi.fn(), onData(cb) { onDataCb = cb @@ -551,7 +551,7 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { describe('listProcesses', () => { it('returns active sessions', async () => { - await adapter.spawn({ cols: 80, rows: 24 }) + await adapter.spawn({ cols: 80, rows: 24, cwd: '/repo/owned-before-osc7' }) await adapter.spawn({ cols: 80, rows: 24 }) const procs = await adapter.listProcesses() @@ -559,6 +559,7 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => { expect(procs[0]).toHaveProperty('id') expect(procs[0]).toHaveProperty('cwd') expect(procs[0]).toHaveProperty('title') + expect(procs[0].cwd).toBe('/repo/owned-before-osc7') }) }) diff --git a/src/main/daemon/daemon-pty-adapter.ts b/src/main/daemon/daemon-pty-adapter.ts index e0d5ec598ca..ae867eeb1ff 100644 --- a/src/main/daemon/daemon-pty-adapter.ts +++ b/src/main/daemon/daemon-pty-adapter.ts @@ -498,6 +498,9 @@ export class DaemonPtyAdapter implements IPtyProvider { if (coldRestore) { this.coldRestoreCache.set(id, coldRestore) this.sleepRestoreSessionIds.add(id) + // Why: physical exit must not mark intentional sleep as a clean end; + // the final checkpoint remains the wake-time recovery authority. + this.historyManager?.suspendSession(id) } } await this.client.request('kill', { sessionId: id, immediate: opts.immediate ?? false }) @@ -750,7 +753,9 @@ export class DaemonPtyAdapter implements IPtyProvider { .filter((s) => s.isAlive) .map((s) => ({ id: s.sessionId, - cwd: s.cwd ?? '', + // Why: OSC 7 may not arrive before destructive cleanup. Spawn cwd is + // still authoritative ownership until the daemon reports a live cwd. + cwd: s.cwd ?? this.initialCwds.get(s.sessionId) ?? '', title: 'shell', ...(s.terminalHandle ? { terminalHandle: s.terminalHandle } : {}) })) diff --git a/src/main/daemon/daemon-server.test.ts b/src/main/daemon/daemon-server.test.ts index 3760ee89de2..45736e2ec0c 100644 --- a/src/main/daemon/daemon-server.test.ts +++ b/src/main/daemon/daemon-server.test.ts @@ -29,7 +29,7 @@ function createMockSubprocess(): SubprocessHandle & { write: vi.fn(), resize: vi.fn(), kill: vi.fn(() => setTimeout(() => onExitCb?.(0), 5)), - forceKill: vi.fn(), + forceKill: vi.fn(() => onExitCb?.(137)), signal: vi.fn(), onData(cb) { onDataCb = cb diff --git a/src/main/daemon/daemon-server.ts b/src/main/daemon/daemon-server.ts index fe87022e5c5..1b1128816e5 100644 --- a/src/main/daemon/daemon-server.ts +++ b/src/main/daemon/daemon-server.ts @@ -149,7 +149,7 @@ export class DaemonServer { async shutdown(): Promise { this.stopStreamBacklogProbe() this.transientFactRelay.dispose() - this.host.dispose() + await this.host.dispose() this.streamDataBatcher.clear() for (const [, client] of this.clients) { @@ -498,7 +498,7 @@ export class DaemonServer { sessionId: request.payload.sessionId, immediate: request.payload.immediate === true }) - this.host.kill(request.payload.sessionId, { immediate: request.payload.immediate }) + await this.host.kill(request.payload.sessionId, { immediate: request.payload.immediate }) return {} case 'signal': @@ -580,7 +580,7 @@ export class DaemonServer { killSessions: request.payload.killSessions === true }) if (request.payload.killSessions) { - this.host.dispose() + await this.host.dispose() } process.nextTick(() => this.shutdown()) return {} diff --git a/src/main/daemon/pty-subprocess.test.ts b/src/main/daemon/pty-subprocess.test.ts index 99a35047743..addb240b9e7 100644 --- a/src/main/daemon/pty-subprocess.test.ts +++ b/src/main/daemon/pty-subprocess.test.ts @@ -1239,6 +1239,46 @@ describe('createPtySubprocess', () => { expect(proc.kill).toHaveBeenCalled() }) + it('propagates rejected graceful kills without marking the wrapper dead', () => { + const proc = mockPtyProcess() + proc.kill.mockImplementationOnce(() => { + throw new Error('native kill rejected') + }) + spawnMock.mockReturnValue(proc) + + const handle = createPtySubprocess({ sessionId: 'test', cols: 80, rows: 24 }) + expect(() => handle.kill()).toThrow('native kill rejected') + handle.write('still owned') + expect(() => handle.kill()).not.toThrow() + + expect(proc.write).toHaveBeenCalledWith('still owned') + expect(proc.kill).toHaveBeenCalledTimes(2) + }) + + it('propagates rejected force kills so the owner can retry', () => { + const proc = mockPtyProcess(77) + proc.kill.mockImplementation(() => { + throw new Error('native fallback rejected') + }) + spawnMock.mockReturnValue(proc) + const killSpy = vi + .spyOn(process, 'kill') + .mockImplementationOnce(() => { + throw new Error('SIGKILL rejected') + }) + .mockImplementationOnce(() => true) + + try { + const handle = createPtySubprocess({ sessionId: 'test', cols: 80, rows: 24 }) + expect(() => handle.forceKill()).toThrow('SIGKILL rejected') + expect(() => handle.forceKill()).not.toThrow() + expect(killSpy).toHaveBeenCalledTimes(2) + expect(proc.kill).toHaveBeenCalledOnce() + } finally { + killSpy.mockRestore() + } + }) + it('forceKill sends SIGKILL to the child pid', () => { const proc = mockPtyProcess(77) spawnMock.mockReturnValue(proc) diff --git a/src/main/daemon/pty-subprocess.ts b/src/main/daemon/pty-subprocess.ts index ce1b1faf35c..62ab012d208 100644 --- a/src/main/daemon/pty-subprocess.ts +++ b/src/main/daemon/pty-subprocess.ts @@ -1176,11 +1176,14 @@ export function createPtySubprocess(opts: PtySubprocessOptions): SubprocessHandl if (dead) { return } + nodePtyKillIssued = true try { - nodePtyKillIssued = true proc.kill() - } catch { - dead = true + } catch (error) { + // Why: a rejected native kill is not proof of exit. Keep the wrapper + // live and let Session retain/retry the physical owner. + nodePtyKillIssued = false + throw error } }, forceKill: () => { @@ -1195,12 +1198,14 @@ export function createPtySubprocess(opts: PtySubprocessOptions): SubprocessHandl } try { process.kill(proc.pid, 'SIGKILL') - } catch { + } catch (signalError) { try { - nodePtyKillIssued = true proc.kill() + nodePtyKillIssued = true } catch { - // Process may already be dead + nodePtyKillIssued = false + // Keep the original OS failure so callers can retry the same owner. + throw signalError } } }, diff --git a/src/main/daemon/session.test.ts b/src/main/daemon/session.test.ts index ade8b18fe15..c57312f80b1 100644 --- a/src/main/daemon/session.test.ts +++ b/src/main/daemon/session.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { PRODUCER_PAUSE_FAILSAFE_MS, Session } from './session' +import { PRODUCER_PAUSE_FAILSAFE_MS, SESSION_FORCE_KILL_RETRY_MS, Session } from './session' import type { SessionState, ShellReadyState } from './types' // Stub the subprocess — Session talks to it via an interface, not child_process directly. @@ -353,13 +353,15 @@ describe('Session', () => { expect(taken?.snapshot).toBeTruthy() }) - it('cancels the post-ready flush gate when force-disposing the subprocess', () => { + it('cancels the post-ready flush gate when force-disposing the subprocess', async () => { createSession({ shellReadySupported: true }) session.write('codex\n') subprocess.simulateData('\x1b]777;orca-shell-ready\x07') expect(session.shellState).toBe('ready' satisfies ShellReadyState) - session.forceKillAndDisposeSubprocess() + const dispose = session.forceKillAndDisposeSubprocess() + subprocess.simulateExit(137) + await dispose vi.advanceTimersByTime(500) expect(subprocess.written).toEqual([]) @@ -406,6 +408,24 @@ describe('Session', () => { expect(session.isTerminating).toBe(true) }) + it('allows a graceful retry when the first kill is rejected', () => { + let attempts = 0 + subprocess.kill = () => { + attempts++ + if (attempts === 1) { + throw new Error('graceful kill rejected') + } + } + createSession() + + expect(() => session.kill()).toThrow('graceful kill rejected') + expect(session.isTerminating).toBe(false) + expect(() => session.kill()).not.toThrow() + + expect(attempts).toBe(2) + expect(session.isTerminating).toBe(true) + }) + it('notifies attached clients on exit after kill', async () => { vi.useRealTimers() createSession() @@ -422,7 +442,7 @@ describe('Session', () => { expect(exitCodes).toEqual([0]) }) - it('force-disposes after 5s if subprocess does not exit', () => { + it('force-kills after 5s but retains ownership until subprocess exit', () => { createSession() // Override kill to NOT trigger exit subprocess.kill = () => {} @@ -432,11 +452,58 @@ describe('Session', () => { expect(session.state).not.toBe('exited') vi.advanceTimersByTime(5_000) - expect(session.state).toBe('exited') expect(forceKillSpy).toHaveBeenCalled() + expect(session.state).toBe('running') + expect(session.isTerminating).toBe(true) + + subprocess.simulateExit(137) + expect(session.state).toBe('exited') }) - it('ignores late data and exit after force-dispose', () => { + it('retries a rejected destructive force kill while retaining ownership', async () => { + let forceKillAttempts = 0 + subprocess.forceKill = () => { + forceKillAttempts++ + if (forceKillAttempts === 1) { + throw new Error('force kill rejected') + } + } + createSession() + + const shutdown = session.forceKillAndWaitForExit() + expect(forceKillAttempts).toBe(1) + await vi.advanceTimersByTimeAsync(SESSION_FORCE_KILL_RETRY_MS) + expect(forceKillAttempts).toBe(2) + subprocess.simulateExit(137) + await shutdown + + expect(session.state).toBe('exited') + }) + + it('retries a rejected graceful-deadline force kill', async () => { + subprocess.kill = () => {} + let forceKillAttempts = 0 + subprocess.forceKill = () => { + forceKillAttempts++ + if (forceKillAttempts === 1) { + throw new Error('transient graceful fallback failure') + } + } + createSession() + + session.kill() + await vi.advanceTimersByTimeAsync(5_000) + expect(forceKillAttempts).toBe(1) + expect(session.isTerminating).toBe(true) + + await vi.advanceTimersByTimeAsync(SESSION_FORCE_KILL_RETRY_MS) + expect(forceKillAttempts).toBe(2) + subprocess.simulateExit(137) + expect(session.state).toBe('exited') + expect(vi.getTimerCount()).toBe(0) + }) + + it('keeps late data and the real exit code until physical exit', () => { createSession() subprocess.kill = () => {} const onData = vi.fn() @@ -449,10 +516,10 @@ describe('Session', () => { subprocess.simulateData('late output') subprocess.simulateExit(23) - expect(onData).not.toHaveBeenCalled() + expect(onData).toHaveBeenCalledWith('late output') expect(onExit).toHaveBeenCalledTimes(1) - expect(onExit).toHaveBeenCalledWith(-1) - expect(session.exitCode).toBe(-1) + expect(onExit).toHaveBeenCalledWith(23) + expect(session.exitCode).toBe(23) }) }) diff --git a/src/main/daemon/session.ts b/src/main/daemon/session.ts index b9c82a48174..085ffc47fdd 100644 --- a/src/main/daemon/session.ts +++ b/src/main/daemon/session.ts @@ -10,6 +10,7 @@ import { } from '../shell-ready-marker-scanner' import { isPowerShellProcess } from '../../shared/shell-process-detection' import type { TuiAgent } from '../../shared/types' +import { PhysicalExitTracker } from '../../shared/physical-exit-tracker' import type { PendingOutputRecord, SessionState, @@ -23,6 +24,9 @@ const SHELL_READY_TIMEOUT_MS = 15_000 // older daemon/local paths that still report shell-ready support for Codex. export const CODEX_SHELL_READY_TIMEOUT_MS = 300 const KILL_TIMEOUT_MS = 5_000 +export const IMMEDIATE_KILL_PHYSICAL_EXIT_TIMEOUT_MS = 8_000 +export const SESSION_FORCE_KILL_RETRY_MS = 250 +const SESSION_FORCE_KILL_MAX_ATTEMPTS = 2 // Why: pending records exist so the 5s checkpoint can persist increments // instead of re-serializing the whole buffer. If no client drains them (main // process gone, history disabled), memory must stay bounded — past the cap we @@ -125,6 +129,8 @@ export class Session { private producerPaused = false private producerPauseFailsafeTimer: ReturnType | null = null private readonly _historySeeded: boolean | undefined + private forceKillSent = false + private readonly physicalExit = new PhysicalExitTracker() constructor(opts: SessionOptions) { this.sessionId = opts.sessionId @@ -267,13 +273,50 @@ export class Session { // Why: a paused child can be blocked inside write(); resume before // signalling so it can run signal handlers and actually exit. this.releaseProducerPause({ resume: true }) - this.subprocess.kill() + try { + this.subprocess.kill() + } catch (error) { + // Why: rejected signalling is not termination. Reopen the session so a + // later graceful or destructive retry can still target the live child. + this._isTerminating = false + throw error + } + this.armForceKillFallback(KILL_TIMEOUT_MS, SESSION_FORCE_KILL_MAX_ATTEMPTS) + } + + private armForceKillFallback(delayMs: number, attemptsRemaining: number): void { this.killTimer = setTimeout(() => { + this.killTimer = null if (this._state !== 'exited') { - this.forceDispose() + try { + this.requestForceKill() + } catch (error) { + console.warn('[Session] failed to force-kill terminating subprocess:', error) + // Why: a transient SIGKILL rejection must not consume the only + // fallback owner after graceful shutdown has already returned. + if (attemptsRemaining > 1) { + this.armForceKillFallback(SESSION_FORCE_KILL_RETRY_MS, attemptsRemaining - 1) + } + } } - }, KILL_TIMEOUT_MS) + }, delayMs) + } + + async forceKillAndWaitForExit( + timeoutMs = IMMEDIATE_KILL_PHYSICAL_EXIT_TIMEOUT_MS + ): Promise { + if (this._state === 'exited') { + return + } + if (!this._isTerminating) { + this._isTerminating = true + this.releaseProducerPause({ resume: true }) + } + // Why: destructive cleanup joins a graceful termination but escalates it + // now; waiting for the 5s timer would spend most of the physical-exit budget. + await this.requestForceKillWithRetry() + await this.waitForPhysicalExit(timeoutMs) } signal(sig: string): void { @@ -475,26 +518,14 @@ export class Session { * out onExit to attached clients — renderer reconnects cold after daemon * exit. Callers MUST check isAlive first; see disposeSubprocess() for the * already-exited case. */ - forceKillAndDisposeSubprocess(): void { - // Why: forceKill before #teardownSubprocess. The helper's subprocess.dispose() - // neutralizes node-pty's proc.kill on POSIX (to kill the SIGHUP-to-recycled-pid - // hazard). subprocess.forceKill uses process.kill(pid, 'SIGKILL') directly - // (pty-subprocess.ts) — unaffected by the neutralization, because it does not - // go through proc.kill. SIGKILL is not ignorable; any child that would have - // survived the 5s timer is reaped immediately. - try { - this.subprocess.forceKill() - } catch { - /* swallow — child may already be gone */ - } - this.#teardownSubprocess() - this._state = 'exited' - // Why: free the headless emulator's scrollback here too (this path skips - // dispose()). Matches forceDispose(); reaping just drops the map entry. - this.emulator.dispose() + async forceKillAndDisposeSubprocess(): Promise { + // Why: daemon exit cannot neutralize the native handle until a bounded + // retry is accepted and onExit proves the child was physically reaped. + await this.forceKillAndWaitForExit() + this.dispose() } - /** Private: shared teardown helper called by dispose(), forceDispose(), and + /** Private: shared teardown helper called by dispose() and * forceKillAndDisposeSubprocess(). Flips `_disposed`, clears pending timers, * and forwards to subprocess.dispose() exactly once. Does NOT set `_state` — * the caller owns the state transition AFTER capturing any invariants that @@ -588,12 +619,14 @@ export class Session { } private handleSubprocessExit(code: number): void { + this.physicalExit.markExited() if (this._disposed) { return } this._exitCode = code this._state = 'exited' + this._isTerminating = false // Why resume:false — the child is reaped, so there is nothing to unblock; // only the failsafe timer must not outlive the session. this.releaseProducerPause({ resume: false }) @@ -676,40 +709,49 @@ export class Session { } } - private forceDispose(): void { - if (this._state === 'exited') { + private requestForceKill(): void { + if (this._state === 'exited' || this.forceKillSent) { return } - // Why: forceKill BEFORE #teardownSubprocess. Order is load-bearing — the - // helper's subprocess.dispose() neutralizes proc.kill on POSIX (to defuse - // the SIGHUP-to-recycled-pid hazard inside node-pty). forceKill uses - // process.kill(pid, 'SIGKILL') directly and is unaffected by that - // neutralization. Must NOT flip `_disposed` here before #teardownSubprocess - // runs, or the helper would early-return and skip subprocess.dispose() — - // the ptmx fd would leak on every kill-timeout (this whole doc's target). + this.forceKillSent = true try { this.subprocess.forceKill() - } catch { - /* already dead */ + } catch (error) { + this.forceKillSent = false + throw error } - this._exitCode = -1 - this._isTerminating = false + } - this.#teardownSubprocess() - this._state = 'exited' - - const clients = this.attachedClients - this.attachedClients = [] - this.preReadyStdinQueue = [] - this.postReadyFlushGate.clear() - this.emulator.dispose() - - for (const client of clients) { - client.onExit(-1) + private async requestForceKillWithRetry(): Promise { + let lastError: unknown + for (let attempt = 0; attempt < SESSION_FORCE_KILL_MAX_ATTEMPTS; attempt++) { + try { + this.requestForceKill() + return + } catch (error) { + lastError = error + } + if (attempt + 1 < SESSION_FORCE_KILL_MAX_ATTEMPTS) { + try { + await this.physicalExit.waitForExit( + SESSION_FORCE_KILL_RETRY_MS, + () => new Error(`Retrying force-kill for PTY ${this.sessionId}`) + ) + return + } catch { + // The bounded waiter detached; retry the still-owned subprocess. + } + } } + throw lastError + } - // Why: reap from the host map on the kill-timeout path too (emulator already - // disposed above; reapSession's dispose() call is a no-op and just drops it). - this.onSessionExit?.(-1) + private waitForPhysicalExit(timeoutMs: number): Promise { + // Why: timed-out destructive retries must detach from an unkillable child; + // otherwise every retry stays retained until the process eventually exits. + return this.physicalExit.waitForExit( + timeoutMs, + () => new Error(`Timed out waiting for PTY process exit: ${this.sessionId}`) + ) } } diff --git a/src/main/daemon/terminal-host-session-reaping-leak.test.ts b/src/main/daemon/terminal-host-session-reaping-leak.test.ts index 30d1736e42d..221a5581fe7 100644 --- a/src/main/daemon/terminal-host-session-reaping-leak.test.ts +++ b/src/main/daemon/terminal-host-session-reaping-leak.test.ts @@ -28,7 +28,7 @@ function createMockSubprocess(): SubprocessHandle & { kill: vi.fn(() => { setTimeout(() => onExitCb?.(0), 5) }), - forceKill: vi.fn(), + forceKill: vi.fn(() => onExitCb?.(137)), signal: vi.fn(), onData(cb) { onDataCb = cb @@ -60,8 +60,8 @@ describe('TerminalHost dead-session reaping (leak regression)', () => { host = new TerminalHost({ spawnSubprocess: spawnFn }) }) - afterEach(() => { - host.dispose() + afterEach(async () => { + await host.dispose() emulatorDispose.mockRestore() }) @@ -113,23 +113,32 @@ describe('TerminalHost dead-session reaping (leak regression)', () => { rows: 24, streamClient: streamClient() }) + lastSubprocess.forceKill = vi.fn() - host.kill('session-1', { immediate: true }) + const killed = host.kill('session-1', { immediate: true }) + + expect(emulatorDispose).not.toHaveBeenCalled() + expect(host.listSessions()).toHaveLength(1) + lastSubprocess._onExitCb?.(137) + await killed // Emulator freed and session dropped from the map (no lingering dead entry). expect(emulatorDispose).toHaveBeenCalledTimes(1) expect(host.listSessions()).toHaveLength(0) }) - it('reaps a session whose graceful kill times out (forceDispose path)', async () => { + it('retains a graceful-timeout session until the forced child physically exits', async () => { vi.useFakeTimers() try { + let stubbornSubprocess: ReturnType | undefined const stubbornHost = new TerminalHost({ spawnSubprocess: () => { const sub = createMockSubprocess() // Stubborn child: ignores graceful kill, so the KILL_TIMEOUT_MS timer // must force-dispose it. sub.kill = vi.fn() + sub.forceKill = vi.fn() + stubbornSubprocess = sub return sub } }) @@ -144,13 +153,16 @@ describe('TerminalHost dead-session reaping (leak regression)', () => { stubbornHost.kill('stubborn') expect(emulatorDispose).not.toHaveBeenCalled() - // The 5s KILL_TIMEOUT_MS fallback fires forceDispose, which disposes the - // emulator and reaps the session via the onExit hook. + // The 5s fallback sends SIGKILL but cannot claim physical cleanup yet. vi.advanceTimersByTime(5000) + expect(emulatorDispose).not.toHaveBeenCalled() + expect(stubbornHost.listSessions()).toHaveLength(1) + + stubbornSubprocess?._onExitCb?.(137) expect(emulatorDispose).toHaveBeenCalledTimes(1) expect(stubbornHost.listSessions()).toHaveLength(0) - stubbornHost.dispose() + await stubbornHost.dispose() } finally { vi.useRealTimers() } diff --git a/src/main/daemon/terminal-host-session-shutdown.ts b/src/main/daemon/terminal-host-session-shutdown.ts new file mode 100644 index 00000000000..0051522f8e7 --- /dev/null +++ b/src/main/daemon/terminal-host-session-shutdown.ts @@ -0,0 +1,22 @@ +import type { Session } from './session' + +export async function disposeTerminalHostSessions(sessions: Iterable): Promise { + const results = await Promise.allSettled( + [...sessions].map(async (session) => { + session.detachAllClients() + // Why: live children retain native ownership until physical exit, while + // exited children must release handles without signalling a recycled pid. + if (session.isAlive) { + await session.forceKillAndDisposeSubprocess() + } else { + session.disposeSubprocess() + } + }) + ) + const rejected = results.find( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ) + if (rejected) { + throw rejected.reason + } +} diff --git a/src/main/daemon/terminal-host.test.ts b/src/main/daemon/terminal-host.test.ts index 46e495e5f2e..ac1186cb36b 100644 --- a/src/main/daemon/terminal-host.test.ts +++ b/src/main/daemon/terminal-host.test.ts @@ -1,5 +1,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { Session, type SubprocessHandle } from './session' +import { + IMMEDIATE_KILL_PHYSICAL_EXIT_TIMEOUT_MS, + SESSION_FORCE_KILL_RETRY_MS, + Session, + type SubprocessHandle +} from './session' import { TerminalHost } from './terminal-host' function createMockSubprocess( @@ -19,7 +24,7 @@ function createMockSubprocess( kill: vi.fn(() => { setTimeout(() => onExitCb?.(0), 5) }), - forceKill: vi.fn(), + forceKill: vi.fn(() => onExitCb?.(137)), signal: vi.fn(), onData(cb) { onDataCb = cb @@ -67,8 +72,8 @@ describe('TerminalHost', () => { host = new TerminalHost({ spawnSubprocess: spawnFn as MockSpawnFn }) }) - afterEach(() => { - host.dispose() + afterEach(async () => { + await host.dispose() }) describe('createOrAttach', () => { @@ -205,7 +210,7 @@ describe('TerminalHost', () => { lastSubprocess = sub return sub }) - host.dispose() + await host.dispose() host = new TerminalHost({ spawnSubprocess: spawnFn as MockSpawnFn }) await host.createOrAttach({ @@ -233,7 +238,7 @@ describe('TerminalHost', () => { lastSubprocess = sub return sub }) - host.dispose() + await host.dispose() host = new TerminalHost({ spawnSubprocess: spawnFn as MockSpawnFn }) await host.createOrAttach({ @@ -261,7 +266,7 @@ describe('TerminalHost', () => { lastSubprocess = sub return sub }) - host.dispose() + await host.dispose() host = new TerminalHost({ spawnSubprocess: spawnFn as MockSpawnFn }) await host.createOrAttach({ @@ -287,7 +292,7 @@ describe('TerminalHost', () => { lastSubprocess = sub return sub }) - host.dispose() + await host.dispose() host = new TerminalHost({ spawnSubprocess: spawnFn as MockSpawnFn }) await host.createOrAttach({ @@ -381,15 +386,85 @@ describe('TerminalHost', () => { rows: 24, streamClient: { onData: vi.fn(), onExit: vi.fn() } }) + lastSubprocess.forceKill = vi.fn() - host.kill('session-1', { immediate: true }) + const killed = host.kill('session-1', { immediate: true }) expect(lastSubprocess.kill).not.toHaveBeenCalled() expect(lastSubprocess.forceKill).toHaveBeenCalled() + expect(lastSubprocess.dispose).not.toHaveBeenCalled() + expect(host.listSessions()).toHaveLength(1) + + lastSubprocess._onExitCb?.(137) + await killed + expect(lastSubprocess.dispose).toHaveBeenCalled() + expect(host.listSessions()).toHaveLength(0) expect(host.isKilled('session-1')).toBe(true) }) + it('escalates an already-graceful termination and joins its physical exit', async () => { + await host.createOrAttach({ + sessionId: 'session-1', + cols: 80, + rows: 24, + streamClient: { onData: vi.fn(), onExit: vi.fn() } + }) + lastSubprocess.forceKill = vi.fn() + + await host.kill('session-1') + const immediate = host.kill('session-1', { immediate: true }) + let settled = false + void immediate.then(() => { + settled = true + }) + await Promise.resolve() + + expect(lastSubprocess.kill).toHaveBeenCalledTimes(1) + expect(lastSubprocess.forceKill).toHaveBeenCalledTimes(1) + expect(settled).toBe(false) + expect(host.listSessions()).toHaveLength(1) + + lastSubprocess._onExitCb?.(137) + await immediate + expect(host.listSessions()).toHaveLength(0) + }) + + it('retains an immediate-kill session when physical exit times out', async () => { + vi.useFakeTimers() + try { + await host.createOrAttach({ + sessionId: 'session-1', + cols: 80, + rows: 24, + streamClient: { onData: vi.fn(), onExit: vi.fn() } + }) + lastSubprocess.forceKill = vi.fn() + + const killed = host.kill('session-1', { immediate: true }) + const rejected = expect(killed).rejects.toThrow('Timed out waiting for PTY process exit') + await vi.advanceTimersByTimeAsync(IMMEDIATE_KILL_PHYSICAL_EXIT_TIMEOUT_MS) + await rejected + + expect(lastSubprocess.forceKill).toHaveBeenCalledTimes(1) + expect(lastSubprocess.dispose).not.toHaveBeenCalled() + expect(host.listSessions()).toHaveLength(1) + await expect( + host.createOrAttach({ + sessionId: 'session-1', + cols: 80, + rows: 24, + streamClient: { onData: vi.fn(), onExit: vi.fn() } + }) + ).rejects.toThrow('is terminating') + + lastSubprocess._onExitCb?.(137) + expect(host.listSessions()).toHaveLength(0) + } finally { + vi.useRealTimers() + } + }) + it('throws for non-existent session', () => { expect(() => host.kill('missing')).toThrow('Session not found') }) @@ -474,7 +549,7 @@ describe('TerminalHost', () => { describe('tombstones', () => { it('caps tombstones at limit', async () => { - host.dispose() + await host.dispose() host = new TerminalHost({ spawnSubprocess: spawnFn as MockSpawnFn, maxTombstones: 3 }) for (let i = 0; i < 5; i++) { @@ -502,10 +577,9 @@ describe('TerminalHost', () => { streamClient: { onData: vi.fn(), onExit: vi.fn() } }) - host.dispose() - // Why: for LIVE sessions, dispose() calls session.forceKillAndDisposeSubprocess() - // which sends SIGKILL (forceKill) and releases the ptmx fd (subprocess.dispose) - // synchronously — no longer relies on the 5s KILL_TIMEOUT_MS fallback. + await host.dispose() + // Why: live sessions retain the native owner until force-kill is accepted + // and physical exit proves the child can no longer hold the ptmx fd. // Exited sessions take the disposeSubprocess() path instead (see the test // below). See docs/fix-pty-fd-leak.md. expect(lastSubprocess.forceKill).toHaveBeenCalled() @@ -513,7 +587,7 @@ describe('TerminalHost', () => { }) it('releases held shell-ready marker prefixes before final checkpoint', async () => { - host.dispose() + await host.dispose() const onFinalCheckpoint = vi.fn() host = new TerminalHost({ spawnSubprocess: spawnFn as MockSpawnFn, @@ -528,13 +602,54 @@ describe('TerminalHost', () => { }) lastSubprocess._onDataCb?.('\x1b]777;orca-shell-ready') - host.dispose() + await host.dispose() expect(onFinalCheckpoint).toHaveBeenCalledWith('session-1', expect.any(Object), [ { kind: 'output', data: '\x1b]777;orca-shell-ready' } ]) }) + it('fences creation and retries a rejected force kill before dropping ownership', async () => { + vi.useFakeTimers() + try { + await host.createOrAttach({ + sessionId: 'session-1', + cols: 80, + rows: 24, + streamClient: { onData: vi.fn(), onExit: vi.fn() } + }) + let attempts = 0 + const forceKill = vi.fn(() => { + attempts++ + if (attempts === 1) { + throw new Error('transient daemon dispose kill failure') + } + lastSubprocess._onExitCb?.(137) + }) + lastSubprocess.forceKill = forceKill + + const dispose = host.dispose() + expect(forceKill).toHaveBeenCalledTimes(1) + expect(host.dispose()).toBe(dispose) + await expect( + host.createOrAttach({ + sessionId: 'late-session', + cols: 80, + rows: 24, + streamClient: { onData: vi.fn(), onExit: vi.fn() } + }) + ).rejects.toThrow('Terminal host is shutting down') + + await vi.advanceTimersByTimeAsync(SESSION_FORCE_KILL_RETRY_MS) + await dispose + expect(forceKill).toHaveBeenCalledTimes(2) + expect(lastSubprocess.dispose).toHaveBeenCalled() + expect(host.listSessions()).toEqual([]) + } finally { + vi.useRealTimers() + } + }) + it('does not list exited sessions', async () => { await host.createOrAttach({ sessionId: 'session-1', @@ -577,7 +692,7 @@ describe('TerminalHost', () => { }) const liveSub = lastSubprocess - host.dispose() + await host.dispose() expect(exitedSub.forceKill).not.toHaveBeenCalled() expect(exitedSub.dispose).toHaveBeenCalled() diff --git a/src/main/daemon/terminal-host.ts b/src/main/daemon/terminal-host.ts index b15420fed14..e548e06e26e 100644 --- a/src/main/daemon/terminal-host.ts +++ b/src/main/daemon/terminal-host.ts @@ -7,6 +7,7 @@ import { buildStartupCommandSubmission } from '../../shared/startup-command-subm import type { SessionInfo, TakePendingOutputResult, TerminalSnapshot } from './types' import { SessionNotFoundError } from './types' import type { CreateOrAttachOptions, CreateOrAttachResult } from './terminal-host-create-contract' +import { disposeTerminalHostSessions } from './terminal-host-session-shutdown' export type { CreateOrAttachOptions, CreateOrAttachResult } from './terminal-host-create-contract' @@ -45,6 +46,8 @@ export class TerminalHost { private spawnSubprocess: TerminalHostOptions['spawnSubprocess'] private onFinalCheckpoint: TerminalHostOptions['onFinalCheckpoint'] private maxTombstones: number + private creationFenced = false + private disposePromise: Promise | null = null constructor(opts: TerminalHostOptions) { this.spawnSubprocess = opts.spawnSubprocess @@ -59,6 +62,9 @@ export class TerminalHost { * already deliver them through shell launch arguments. */ async createOrAttach(opts: CreateOrAttachOptions): Promise { + if (this.creationFenced) { + throw new Error('Terminal host is shutting down') + } const existing = this.sessions.get(opts.sessionId) // Why: a session that has been asked to terminate (kill() called but the @@ -81,6 +87,12 @@ export class TerminalHost { } } + if (existing?.isAlive && existing.isTerminating) { + // Why: replacing a SIGKILLed-but-unreaped child would lose ownership of + // its native handles and let the same session id hide two generations. + throw new Error(`Session "${opts.sessionId}" is terminating`) + } + // Clean up dead session if present if (existing) { existing.dispose() @@ -193,25 +205,20 @@ export class TerminalHost { this.sessions.get(sessionId)?.resumeProducer() } - kill(sessionId: string, opts: { immediate?: boolean } = {}): void { + kill(sessionId: string, opts: { immediate?: boolean } = {}): Promise { const session = this.getAliveSession(sessionId) this.recordTombstone(sessionId) if (opts.immediate) { - session.forceKillAndDisposeSubprocess() - // Why: the immediate path tears down synchronously without firing the - // session's onExit hook, so reap it here. The graceful path below funnels - // through Session.handleSubprocessExit -> onExit -> reapSession. - this.reapSession(sessionId) - return + return session.forceKillAndWaitForExit() } session.kill() + return Promise.resolve() } // Why: dispose a dead session's headless emulator and drop it from the map so // exited terminals don't pin ~5000 rows of scrollback for the daemon's life. // No-ops on live sessions (a live session must never be disposed here) and on - // already-reaped/unknown ids. Wired as the Session onExit hook and also called - // on the immediate-kill path. + // already-reaped/unknown ids. Wired as the Session onExit hook. private reapSession(sessionId: string): void { const session = this.sessions.get(sessionId) if (!session || session.isAlive) { @@ -340,7 +347,23 @@ export class TerminalHost { return result } - dispose(): void { + dispose(): Promise { + this.creationFenced = true + if (this.disposePromise) { + return this.disposePromise + } + const disposePromise = this.disposeSessions() + this.disposePromise = disposePromise + void disposePromise.catch(() => { + // Why: keep failed native owners retryable on a later shutdown request. + if (this.disposePromise === disposePromise) { + this.disposePromise = null + } + }) + return disposePromise + } + + private async disposeSessions(): Promise { // Why: write final checkpoints before killing sessions so graceful shutdown // has zero data loss. The checkpoint callback writes synchronously to disk. if (this.onFinalCheckpoint) { @@ -359,22 +382,7 @@ export class TerminalHost { } } - for (const [, session] of this.sessions) { - session.detachAllClients() - // Why: live-vs-exited is load-bearing. For LIVE sessions we use - // forceKillAndDisposeSubprocess (SIGKILL + destroy) to reap stubborn - // children AND release the ptmx fd on the same tick, bypassing the 5s - // KILL_TIMEOUT_MS fallback that would otherwise outlive the daemon - // process. For sessions that have already exited but are still in the - // map, SIGKILL would target a reaped pid — on POSIX that pid can be - // recycled to an unrelated process, so we MUST only release the fd via - // disposeSubprocess() (destroy without kill). See docs/fix-pty-fd-leak.md. - if (session.isAlive) { - session.forceKillAndDisposeSubprocess() - } else { - session.disposeSubprocess() - } - } + await disposeTerminalHostSessions(this.sessions.values()) this.sessions.clear() this.killedTombstones.clear() } diff --git a/src/main/git/remove-worktree.test.ts b/src/main/git/remove-worktree.test.ts index 4330a3be9e5..1329ba77595 100644 --- a/src/main/git/remove-worktree.test.ts +++ b/src/main/git/remove-worktree.test.ts @@ -40,7 +40,8 @@ import { assertWorktreeCleanForRemoval, forceDeleteLocalBranch, listWorktrees, - removeWorktree + removeWorktree, + WORKTREE_REMOVAL_PREFLIGHT_TIMEOUT_MS } from './worktree' beforeEach(() => { @@ -748,7 +749,7 @@ describe('assertWorktreeCleanForRemoval', () => { expect(gitExecFileAsyncMock).toHaveBeenCalledWith( ['status', '--porcelain', '--untracked-files=all'], - { cwd: '/repo-feature' } + { cwd: '/repo-feature', timeout: WORKTREE_REMOVAL_PREFLIGHT_TIMEOUT_MS } ) }) @@ -761,6 +762,47 @@ describe('assertWorktreeCleanForRemoval', () => { }) }) + it('ignores configured linked paths without mutating them before preflight', async () => { + gitExecFileAsyncMock.mockResolvedValueOnce({ stdout: '?? node_modules\0', stderr: '' }) + + await expect( + assertWorktreeCleanForRemoval('/repo-feature', false, { + ignoredUntrackedPaths: ['node_modules'] + }) + ).resolves.toBeUndefined() + + expect(gitExecFileAsyncMock).toHaveBeenCalledWith( + ['status', '--porcelain', '-z', '--untracked-files=all'], + { cwd: '/repo-feature', timeout: WORKTREE_REMOVAL_PREFLIGHT_TIMEOUT_MS } + ) + }) + + it('allows a shorter caller deadline for destructive preflight', async () => { + gitExecFileAsyncMock.mockRejectedValueOnce(new Error('git timed out.')) + + await expect( + assertWorktreeCleanForRemoval('/repo-feature', false, { timeout: 25 }) + ).rejects.toThrow('git timed out.') + + expect(gitExecFileAsyncMock).toHaveBeenCalledWith( + ['status', '--porcelain', '--untracked-files=all'], + { cwd: '/repo-feature', timeout: 25 } + ) + }) + + it('still rejects real changes when configured linked paths are ignored', async () => { + gitExecFileAsyncMock.mockResolvedValueOnce({ + stdout: '?? node_modules\0 M src/index.ts\0', + stderr: '' + }) + + await expect( + assertWorktreeCleanForRemoval('/repo-feature', false, { + ignoredUntrackedPaths: ['node_modules'] + }) + ).rejects.toMatchObject({ message: 'Worktree has uncommitted or untracked changes.' }) + }) + it('rethrows preflight subprocess failures as-is', async () => { const error = Object.assign(new Error('fatal: not a git repository'), { stderr: 'fatal: not a git repository (or any of the parent directories): .git\n' diff --git a/src/main/git/worktree.ts b/src/main/git/worktree.ts index c9175b3dcb4..97a731fff8f 100644 --- a/src/main/git/worktree.ts +++ b/src/main/git/worktree.ts @@ -39,6 +39,11 @@ type SparseWorktreeCreateError = Error & { export type GitWorktreeExecOptions = { wslDistro?: string signal?: AbortSignal + timeout?: number +} + +type WorktreeRemovalPreflightOptions = GitWorktreeExecOptions & { + ignoredUntrackedPaths?: readonly string[] } export type AddWorktreeOptions = GitWorktreeExecOptions & { @@ -81,15 +86,17 @@ const SPARSE_CHECKOUT_DETECTION_CONCURRENCY = 8 // fast instead of spinning forever. Generous enough not to kill a legit large // checkout (mirrors the SYNC runner's cloud-placeholder floor, #7225). export const WORKTREE_ADD_TIMEOUT_MS = 180_000 +export const WORKTREE_REMOVAL_PREFLIGHT_TIMEOUT_MS = 30_000 function gitExecOptions( cwd: string, options: GitWorktreeExecOptions = {} -): { cwd: string; wslDistro?: string; signal?: AbortSignal } { +): { cwd: string; wslDistro?: string; signal?: AbortSignal; timeout?: number } { return { cwd, ...(options.wslDistro ? { wslDistro: options.wslDistro } : {}), - ...(options.signal ? { signal: options.signal } : {}) + ...(options.signal ? { signal: options.signal } : {}), + ...(options.timeout ? { timeout: options.timeout } : {}) } } @@ -1348,16 +1355,26 @@ async function isLocalBranchCheckedOut( export async function assertWorktreeCleanForRemoval( worktreePath: string, force = false, - options: GitWorktreeExecOptions = {} + options: WorktreeRemovalPreflightOptions = {} ): Promise { if (force) { return } - const { stdout } = await gitExecFileAsync(['status', '--porcelain', '--untracked-files=all'], { - ...gitExecOptions(worktreePath, options) - }) - if (!stdout.trim()) { + const { ignoredUntrackedPaths = [], ...gitOptions } = options + const useNullTerminatedStatus = ignoredUntrackedPaths.length > 0 + const { stdout } = await gitExecFileAsync( + ['status', '--porcelain', ...(useNullTerminatedStatus ? ['-z'] : []), '--untracked-files=all'], + { + ...gitExecOptions(worktreePath, gitOptions), + timeout: gitOptions.timeout ?? WORKTREE_REMOVAL_PREFLIGHT_TIMEOUT_MS + } + ) + if ( + useNullTerminatedStatus + ? hasOnlyIgnoredUntrackedStatus(stdout, ignoredUntrackedPaths) + : !stdout.trim() + ) { return } @@ -1366,6 +1383,26 @@ export async function assertWorktreeCleanForRemoval( throw error } +function hasOnlyIgnoredUntrackedStatus( + status: string, + ignoredUntrackedPaths: readonly string[] +): boolean { + const ignored = new Set( + ignoredUntrackedPaths + .map((entry) => + entry + .trim() + .replace(/^[\\/]+/, '') + .replace(/\\/g, '/') + ) + .filter((entry) => entry && !entry.split('/').includes('..')) + ) + return status + .split('\0') + .filter(Boolean) + .every((entry) => entry.startsWith('?? ') && ignored.has(entry.slice(3).replace(/\\/g, '/'))) +} + function translateWorktreePath( worktreePath: string, repoPath: string, diff --git a/src/main/index.ts b/src/main/index.ts index eb2ac4a7668..5dc3adfbd7c 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -147,6 +147,7 @@ import { getPtyIdForPaneKey, registerPaneKeyTeardownListener, getLocalPtyProvider, + getSshPtyProvider, registerHeadlessPtyRuntime } from './ipc/pty' import { AgentBrowserBridge } from './browser/agent-browser-bridge' @@ -1809,6 +1810,9 @@ app.whenReady().then(async () => { // provider reference eagerly here would freeze the pre-daemon LocalPtyProvider // and defeat the teardown helper's prefix sweep (design §4.3 wire-up). getLocalProvider: () => getLocalPtyProvider(), + // Why: SSH relay providers are registered after runtime construction and + // may reconnect; destructive cleanup must resolve the current generation. + getSshProvider: (connectionId) => getSshPtyProvider(connectionId), onPtyStopped: clearProviderPtyState, onTerminalAgentStatus: (event) => { agentHookServer.ingestTerminalStatus(event) diff --git a/src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts b/src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts index a566bf178c6..e9a82723cd5 100644 --- a/src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts +++ b/src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type * as ParcelWatcherProcess from './parcel-watcher-process' const { handleMock } = vi.hoisted(() => ({ handleMock: vi.fn() @@ -22,6 +23,14 @@ vi.mock('./filesystem-watcher-wsl', () => ({ createWslWatcher: vi.fn() })) +vi.mock('./parcel-watcher-process', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + subscribeViaWatcherProcess: vi.fn(actual.subscribeViaWatcherProcess) + } +}) + vi.mock('../providers/ssh-filesystem-dispatch', () => ({ getSshFilesystemProvider: vi.fn() })) @@ -33,6 +42,8 @@ import { } from './filesystem-watcher' import { stat } from 'node:fs/promises' import { subscribe as subscribeParcelWatcher } from '@parcel/watcher' +import { subscribeViaWatcherProcess } from './parcel-watcher-process' +import { WatcherProcessFailure } from './parcel-watcher-process-failure' type HandlerMap = Record Promise | unknown> @@ -43,6 +54,7 @@ describe('local filesystem watcher unsubscribe cleanup', () => { handleMock.mockReset() vi.mocked(stat).mockReset() vi.mocked(subscribeParcelWatcher).mockReset() + vi.mocked(subscribeViaWatcherProcess).mockClear() for (const key of Object.keys(handlers)) { delete handlers[key] } @@ -273,6 +285,214 @@ describe('local filesystem watcher unsubscribe cleanup', () => { expect(unsubscribeMock).toHaveBeenCalledTimes(1) }) + it.each([ + ['C:\\Repo', 'c:/repo'], + ['\\\\Server\\Share\\Repo', '//server/share/repo'] + ])('matches Windows watcher cleanup across path casing for %s', async (watchPath, closePath) => { + vi.mocked(stat).mockResolvedValue({ isDirectory: () => true } as never) + const unsubscribeMock = vi.fn() + vi.mocked(subscribeParcelWatcher).mockResolvedValue({ unsubscribe: unsubscribeMock } as never) + const sender = { + isDestroyed: () => false, + send: vi.fn(), + once: vi.fn(), + id: 1 + } + + await handlers['fs:watchWorktree']({ sender }, { worktreePath: watchPath }) + await closeLocalWatcherForWorktreePath(closePath) + + expect(unsubscribeMock).toHaveBeenCalledTimes(1) + }) + + it.each([ + ['C:\\Repo', 'c:/repo'], + ['\\\\Server\\Share\\Repo', '//server/share/repo'] + ])('keeps the physical Windows root in event payloads for %s', async (watchPath, closePath) => { + vi.useFakeTimers() + try { + vi.mocked(stat).mockResolvedValue({ isDirectory: () => true } as never) + let watcherCallback: ( + error: Error | null, + events: { type: 'update'; path: string }[] + ) => void = () => undefined + const unsubscribeMock = vi.fn() + vi.mocked(subscribeParcelWatcher).mockImplementation(async (_path, callback) => { + watcherCallback = callback as typeof watcherCallback + return { unsubscribe: unsubscribeMock } as never + }) + const sender = { + isDestroyed: () => false, + send: vi.fn(), + once: vi.fn(), + id: 1 + } + + await handlers['fs:watchWorktree']({ sender }, { worktreePath: watchPath }) + watcherCallback(null, [{ type: 'update', path: `${watchPath}\\file.txt` }]) + await vi.advanceTimersByTimeAsync(150) + + expect(sender.send).toHaveBeenCalledWith( + 'fs:changed', + expect.objectContaining({ worktreePath: watchPath }) + ) + await closeLocalWatcherForWorktreePath(closePath) + } finally { + vi.useRealTimers() + } + }) + + it('propagates watcher termination failure to worktree deletion', async () => { + vi.mocked(stat).mockResolvedValue({ isDirectory: () => true } as never) + const terminationError = new Error('watcher child did not exit') + const unsubscribeMock = vi.fn().mockRejectedValue(terminationError) + vi.mocked(subscribeParcelWatcher).mockResolvedValue({ unsubscribe: unsubscribeMock } as never) + const sender = { isDestroyed: () => false, send: vi.fn(), once: vi.fn(), id: 1 } + + await handlers['fs:watchWorktree']({ sender }, { worktreePath: '/tmp/repo' }) + + await expect(closeLocalWatcherForWorktreePath('/tmp/repo')).rejects.toBe(terminationError) + expect(unsubscribeMock).toHaveBeenCalledTimes(1) + }) + + it('awaits and propagates sender cleanup already tearing down the same root', async () => { + vi.mocked(stat).mockResolvedValue({ isDirectory: () => true } as never) + const terminationError = new Error('watcher child did not exit') + let rejectUnsubscribe: (error: Error) => void = () => {} + const unsubscribeMock = vi.fn( + () => + new Promise((_resolve, reject) => { + rejectUnsubscribe = reject + }) + ) + vi.mocked(subscribeParcelWatcher).mockResolvedValue({ unsubscribe: unsubscribeMock } as never) + const destroyedCallbacks: (() => void)[] = [] + const sender = { + isDestroyed: () => false, + send: vi.fn(), + once: vi.fn((event: string, callback: () => void) => { + if (event === 'destroyed') { + destroyedCallbacks.push(callback) + } + }), + id: 1 + } + await handlers['fs:watchWorktree']({ sender }, { worktreePath: '/tmp/repo' }) + destroyedCallbacks[0]() + await vi.waitFor(() => expect(unsubscribeMock).toHaveBeenCalledTimes(1)) + + let settled = false + const closeFailure = expect( + closeLocalWatcherForWorktreePath('/tmp/repo').finally(() => { + settled = true + }) + ).rejects.toBe(terminationError) + await Promise.resolve() + expect(settled).toBe(false) + rejectUnsubscribe(terminationError) + await closeFailure + }) + + it('retains teardown failure only until the unkillable child physically exits', async () => { + vi.mocked(stat).mockResolvedValue({ isDirectory: () => true } as never) + let signalPhysicalExit: () => void = () => {} + const physicalExit = new Promise((resolve) => { + signalPhysicalExit = resolve + }) + const terminationError = new WatcherProcessFailure( + 'file watcher process did not exit after termination deadline', + 'supervisor', + 'process_unavailable', + physicalExit + ) + const unsubscribeMock = vi.fn().mockRejectedValue(terminationError) + vi.mocked(subscribeParcelWatcher).mockResolvedValue({ unsubscribe: unsubscribeMock } as never) + const destroyedCallbacks: (() => void)[] = [] + const sender = { + isDestroyed: () => false, + send: vi.fn(), + once: vi.fn((event: string, callback: () => void) => { + if (event === 'destroyed') { + destroyedCallbacks.push(callback) + } + }), + id: 1 + } + await handlers['fs:watchWorktree']({ sender }, { worktreePath: '/tmp/repo' }) + destroyedCallbacks[0]() + await vi.waitFor(() => expect(unsubscribeMock).toHaveBeenCalledTimes(1)) + + await expect(closeLocalWatcherForWorktreePath('/tmp/repo')).rejects.toBe(terminationError) + signalPhysicalExit() + await Promise.resolve() + await expect(closeLocalWatcherForWorktreePath('/tmp/repo')).resolves.toBeUndefined() + }) + + it('retains callback terminal failure when the cleared subscription later unsubscribes cleanly', async () => { + vi.mocked(stat).mockResolvedValue({ isDirectory: () => true } as never) + let watcherCallback: (error: Error | null, events: []) => void = () => {} + let signalPhysicalExit: () => void = () => {} + const physicalExit = new Promise((resolve) => { + signalPhysicalExit = resolve + }) + const terminationError = new WatcherProcessFailure( + 'file watcher process did not exit after termination deadline', + 'supervisor', + 'process_unavailable', + physicalExit + ) + const unsubscribe = vi.fn().mockResolvedValue(undefined) + vi.mocked(subscribeViaWatcherProcess).mockImplementationOnce(async (_dir, callback) => { + watcherCallback = callback as typeof watcherCallback + return { unsubscribe } + }) + const sender = { isDestroyed: () => false, send: vi.fn(), once: vi.fn(), id: 1 } + await handlers['fs:watchWorktree']({ sender }, { worktreePath: '/tmp/repo' }) + + watcherCallback(terminationError, []) + await vi.waitFor(() => expect(unsubscribe).toHaveBeenCalledTimes(1)) + + await expect(closeLocalWatcherForWorktreePath('/tmp/repo')).rejects.toBe(terminationError) + signalPhysicalExit() + await physicalExit + await expect(closeLocalWatcherForWorktreePath('/tmp/repo')).resolves.toBeUndefined() + }) + + it('propagates terminal child failure while deletion cancels an active crawl', async () => { + vi.mocked(stat).mockResolvedValue({ isDirectory: () => true } as never) + let signalPhysicalExit: () => void = () => undefined + const physicalExit = new Promise((resolve) => { + signalPhysicalExit = resolve + }) + const terminationError = new WatcherProcessFailure( + 'file watcher process did not exit after termination deadline', + 'supervisor', + 'process_unavailable', + physicalExit + ) + vi.mocked(subscribeViaWatcherProcess).mockImplementationOnce( + (_dir, _callback, _opts, hooks) => + new Promise((_resolve, reject) => { + hooks?.signal?.addEventListener('abort', () => reject(terminationError), { once: true }) + }) + ) + const sender = { isDestroyed: () => false, send: vi.fn(), once: vi.fn(), id: 1 } + const watchPromise = handlers['fs:watchWorktree']( + { sender }, + { worktreePath: '/tmp/repo' } + ) as Promise + const watchFailure = expect(watchPromise).rejects.toBe(terminationError) + await vi.waitFor(() => expect(subscribeViaWatcherProcess).toHaveBeenCalledTimes(1)) + + await expect(closeLocalWatcherForWorktreePath('/tmp/repo')).rejects.toBe(terminationError) + await watchFailure + await expect(closeLocalWatcherForWorktreePath('/tmp/repo')).rejects.toBe(terminationError) + + signalPhysicalExit() + await physicalExit + await expect(closeLocalWatcherForWorktreePath('/tmp/repo')).resolves.toBeUndefined() + }) + it('closes a pending grace-teardown watcher immediately for worktree deletion', async () => { vi.mocked(stat).mockResolvedValue({ isDirectory: () => true } as never) const unsubscribeMock = vi.fn() diff --git a/src/main/ipc/filesystem-watcher-native-capacity.test.ts b/src/main/ipc/filesystem-watcher-native-capacity.test.ts new file mode 100644 index 00000000000..658793e2f5e --- /dev/null +++ b/src/main/ipc/filesystem-watcher-native-capacity.test.ts @@ -0,0 +1,100 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { handleMock, statMock, subscribeViaWatcherProcessMock, disposeWatcherProcessMock } = + vi.hoisted(() => ({ + handleMock: vi.fn(), + statMock: vi.fn(), + subscribeViaWatcherProcessMock: vi.fn(), + disposeWatcherProcessMock: vi.fn() + })) + +vi.mock('electron', () => ({ ipcMain: { handle: handleMock } })) +vi.mock('node:fs/promises', () => ({ stat: statMock })) +vi.mock('./parcel-watcher-process', () => ({ + subscribeViaWatcherProcess: subscribeViaWatcherProcessMock, + disposeWatcherProcess: disposeWatcherProcessMock +})) +vi.mock('./filesystem-watcher-wsl', () => ({ createWslWatcher: vi.fn() })) +vi.mock('../providers/ssh-filesystem-dispatch', () => ({ + getSshFilesystemProvider: vi.fn() +})) + +import { closeAllWatchers, registerFilesystemWatcherHandlers } from './filesystem-watcher' +import { + MAX_PHYSICAL_WATCHER_CHILDREN, + reserveWatcherChild, + resetWatcherChildRegistryForTest, + WatcherChildCapacityError +} from './parcel-watcher-child-registry' + +type HandlerMap = Record Promise | unknown> + +function fillWatcherChildCapacity(): (() => void)[] { + return Array.from({ length: MAX_PHYSICAL_WATCHER_CHILDREN }, () => { + const release = reserveWatcherChild() + if (!release) { + throw new Error('expected watcher child reservation') + } + return release + }) +} + +describe('native filesystem watcher capacity recovery', () => { + const handlers: HandlerMap = {} + + beforeEach(async () => { + handleMock.mockReset() + statMock.mockReset() + subscribeViaWatcherProcessMock.mockReset() + disposeWatcherProcessMock.mockReset() + resetWatcherChildRegistryForTest() + for (const key of Object.keys(handlers)) { + delete handlers[key] + } + handleMock.mockImplementation((channel, handler) => { + handlers[channel] = handler + }) + statMock.mockResolvedValue({ isDirectory: () => true }) + registerFilesystemWatcherHandlers() + await closeAllWatchers() + }) + + afterEach(async () => { + await closeAllWatchers() + resetWatcherChildRegistryForTest() + }) + + it('automatically retries a native root when a child slot is released', async () => { + const releases = fillWatcherChildCapacity() + const unsubscribe = vi.fn().mockResolvedValue(undefined) + subscribeViaWatcherProcessMock + .mockRejectedValueOnce(new WatcherChildCapacityError()) + .mockResolvedValueOnce({ unsubscribe }) + const sender = { isDestroyed: () => false, send: vi.fn(), once: vi.fn(), id: 1 } + const args = { worktreePath: '/tmp/native-capacity-root' } + + await handlers['fs:watchWorktree']({ sender }, args) + expect(subscribeViaWatcherProcessMock).toHaveBeenCalledOnce() + + releases.pop()?.() + await vi.waitFor(() => expect(subscribeViaWatcherProcessMock).toHaveBeenCalledTimes(2)) + + releases.forEach((release) => release()) + }) + + it('cancels the native capacity wait when its renderer unwatches', async () => { + const releases = fillWatcherChildCapacity() + subscribeViaWatcherProcessMock.mockRejectedValue(new WatcherChildCapacityError()) + const sender = { isDestroyed: () => false, send: vi.fn(), once: vi.fn(), id: 1 } + const args = { worktreePath: '/tmp/native-capacity-root' } + + await handlers['fs:watchWorktree']({ sender }, args) + handlers['fs:unwatchWorktree']({ sender: { id: sender.id } }, args) + releases.pop()?.() + await Promise.resolve() + await Promise.resolve() + + expect(subscribeViaWatcherProcessMock).toHaveBeenCalledOnce() + releases.forEach((release) => release()) + }) +}) diff --git a/src/main/ipc/filesystem-watcher-wsl.test.ts b/src/main/ipc/filesystem-watcher-wsl.test.ts index 4258c0e675e..4c8333d508e 100644 --- a/src/main/ipc/filesystem-watcher-wsl.test.ts +++ b/src/main/ipc/filesystem-watcher-wsl.test.ts @@ -11,6 +11,12 @@ vi.mock('child_process', () => ({ })) import { createWslWatcher } from './filesystem-watcher-wsl' +import { WSL_WATCHER_PHYSICAL_EXIT_TIMEOUT_MS } from './wsl-watcher-process-exit' +import { + MAX_PHYSICAL_WATCHER_CHILDREN, + reserveWatcherChild, + resetWatcherChildRegistryForTest +} from './parcel-watcher-child-registry' import type { WatchedRoot, WslWatcherDeps } from './filesystem-watcher-wsl' const SNAPSHOT_START = '\x1e' @@ -71,6 +77,7 @@ async function resolveInitialSnapshot( describe('createWslWatcher', () => { beforeEach(() => { spawnMock.mockReset() + resetWatcherChildRegistryForTest() }) afterEach(() => { @@ -93,6 +100,54 @@ describe('createWslWatcher', () => { ) }) + it('counts WSL watcher processes against the global physical child cap', async () => { + const releases = Array.from({ length: MAX_PHYSICAL_WATCHER_CHILDREN }, () => + reserveWatcherChild() + ) + + await expect(createWslWatcher(ROOT_KEY, ROOT_KEY, makeDeps())).rejects.toThrow( + 'Physical file watcher process limit reached' + ) + expect(spawnMock).not.toHaveBeenCalled() + releases.forEach((release) => release?.()) + }) + + it('releases the WSL physical child reservation only after close', async () => { + const siblingReleases = Array.from({ length: MAX_PHYSICAL_WATCHER_CHILDREN - 1 }, () => + reserveWatcherChild() + ) + const { child, promise } = startWatcher() + await resolveInitialSnapshot(child, promise) + + expect(reserveWatcherChild()).toBeNull() + child.emit('close', null, 'SIGTERM') + const afterClose = reserveWatcherChild() + expect(afterClose).not.toBeNull() + + afterClose?.() + siblingReleases.forEach((release) => release?.()) + }) + + it('releases the WSL child reservation after a synchronous script write failure', async () => { + const siblingReleases = Array.from({ length: MAX_PHYSICAL_WATCHER_CHILDREN - 1 }, () => + reserveWatcherChild() + ) + const child = new FakeChildProcess() + vi.spyOn(child.stdin, 'end').mockImplementation(() => { + throw new Error('script write failed') + }) + spawnMock.mockReturnValueOnce(child) + + await expect(createWslWatcher(ROOT_KEY, ROOT_KEY, makeDeps())).rejects.toThrow( + 'script write failed' + ) + const afterFailure = reserveWatcherChild() + expect(afterFailure).not.toBeNull() + + afterFailure?.() + siblingReleases.forEach((release) => release?.()) + }) + it('diffs WSL snapshots into create, update, and delete events', async () => { const scheduleBatchFlush = vi.fn() const { child, promise } = startWatcher(makeDeps(scheduleBatchFlush)) @@ -145,6 +200,69 @@ describe('createWslWatcher', () => { expect(scheduleBatchFlush).not.toHaveBeenCalled() }) + it('keeps unsubscribe pending until the WSL child physically closes', async () => { + const child = new FakeChildProcess() + child.kill = vi.fn(() => true) + spawnMock.mockReturnValueOnce(child) + const promise = createWslWatcher(ROOT_KEY, ROOT_KEY, makeDeps()) + const root = await resolveInitialSnapshot(child, promise) + + let settled = false + const unsubscribe = root.subscription.unsubscribe().finally(() => { + settled = true + }) + await Promise.resolve() + expect(settled).toBe(false) + + child.emit('close', null, 'SIGTERM') + await unsubscribe + expect(settled).toBe(true) + }) + + it('retains a timed-out WSL watcher owner until a later physical close', async () => { + vi.useFakeTimers() + try { + const child = new FakeChildProcess() + child.kill = vi.fn(() => true) + spawnMock.mockReturnValueOnce(child) + const promise = createWslWatcher(ROOT_KEY, ROOT_KEY, makeDeps()) + const root = await resolveInitialSnapshot(child, promise) + + const unsubscribe = root.subscription.unsubscribe() + const rejected = expect(unsubscribe).rejects.toMatchObject({ + code: 'process_unavailable', + physicalExit: expect.any(Promise) + }) + await vi.advanceTimersByTimeAsync(WSL_WATCHER_PHYSICAL_EXIT_TIMEOUT_MS) + await rejected + + const retry = root.subscription.unsubscribe() + child.emit('close', null, 'SIGTERM') + await expect(retry).resolves.toBeUndefined() + expect(child.kill).toHaveBeenCalledTimes(1) + } finally { + vi.useRealTimers() + } + }) + + it('retains the WSL owner when the termination signal is rejected', async () => { + const child = new FakeChildProcess() + child.kill = vi.fn().mockReturnValueOnce(false).mockReturnValue(true) + spawnMock.mockReturnValueOnce(child) + const promise = createWslWatcher(ROOT_KEY, ROOT_KEY, makeDeps()) + const root = await resolveInitialSnapshot(child, promise) + + await expect(root.subscription.unsubscribe()).rejects.toMatchObject({ + code: 'process_unavailable', + physicalExit: expect.any(Promise) + }) + + const retry = root.subscription.unsubscribe() + child.emit('close', null, 'SIGTERM') + await expect(retry).resolves.toBeUndefined() + expect(child.kill).toHaveBeenCalledTimes(2) + }) + it('rejects when the WSL process exits before the first snapshot', async () => { const { child, promise } = startWatcher() @@ -185,6 +303,30 @@ describe('createWslWatcher', () => { expect(child.kill).toHaveBeenCalledOnce() }) + it('does not acknowledge startup abort before the WSL child closes', async () => { + const child = new FakeChildProcess() + child.kill = vi.fn(() => true) + spawnMock.mockReturnValueOnce(child) + const controller = new AbortController() + const promise = createWslWatcher(ROOT_KEY, ROOT_KEY, makeDeps(), controller.signal) + + controller.abort() + let settled = false + void promise.then( + () => { + settled = true + }, + () => { + settled = true + } + ) + await Promise.resolve() + expect(settled).toBe(false) + + child.emit('close', null, 'SIGTERM') + await expect(promise).rejects.toMatchObject({ name: 'AbortError' }) + }) + it('rejects immediately when the install abort signal is already aborted', async () => { const controller = new AbortController() controller.abort() diff --git a/src/main/ipc/filesystem-watcher-wsl.ts b/src/main/ipc/filesystem-watcher-wsl.ts index 54e58415894..b73f58d5122 100644 --- a/src/main/ipc/filesystem-watcher-wsl.ts +++ b/src/main/ipc/filesystem-watcher-wsl.ts @@ -11,6 +11,8 @@ import type { WebContents } from 'electron' import type { Event as WatcherEvent } from '@parcel/watcher' import { queueWatcherEvents } from './filesystem-watcher-event-batch' import { parseWslUncPath } from '../../shared/wsl-paths' +import { createWslWatcherProcessExit, createWslWatcherStartup } from './wsl-watcher-process-exit' +import { reserveWatcherChild, WatcherChildCapacityError } from './parcel-watcher-child-registry' export type WatcherSubscription = { unsubscribe(): Promise @@ -27,6 +29,7 @@ export type WatchedRoot = { subscription: WatcherSubscription listeners: Map batch: DebouncedBatch + rootPath?: string } export type WslWatcherDeps = { @@ -35,11 +38,9 @@ export type WslWatcherDeps = { watchedRoots: Map } -const POLL_INTERVAL_MS = 2000 -const POLL_INTERVAL_SECONDS = Math.max(1, Math.ceil(POLL_INTERVAL_MS / 1000)) +const POLL_INTERVAL_SECONDS = 2 const STARTUP_TIMEOUT_MS = 10_000 -const SNAPSHOT_START = '\x1e' -const SNAPSHOT_END = '\x1f' +const [SNAPSHOT_START, SNAPSHOT_END] = ['\x1e', '\x1f'] const MAX_STREAM_BUFFER_CHARS = 10 * 1024 * 1024 type WslSnapshotEntry = { @@ -153,8 +154,7 @@ export async function createWslWatcher( deps: WslWatcherDeps, signal?: AbortSignal ): Promise { - // Why: cancelled local installs pass the same abort controller used for - // native Parcel subscribe; honor it before spawning a WSL snapshot process. + // Why: honor local-install cancellation before spawning a WSL snapshot process. if (signal?.aborted) { throw new DOMException('WSL watcher subscription aborted', 'AbortError') } @@ -169,7 +169,8 @@ export async function createWslWatcher( const root: WatchedRoot = { subscription: null!, listeners: new Map(), - batch: { events: [], overflowed: false, timer: null, firstEventAt: 0 } + batch: { events: [], overflowed: false, timer: null, firstEventAt: 0 }, + rootPath: worktreePath } let disposed = false @@ -180,25 +181,7 @@ export async function createWslWatcher( const stderrDecoder = new StringDecoder('utf8') let stderrTail = '' - let resolveInitial!: () => void - let rejectInitial!: (error: Error) => void - let initialSettled = false - const initialSnapshotReady = new Promise((resolve, reject) => { - resolveInitial = resolve - rejectInitial = reject - }) - - function settleInitial(error?: Error): void { - if (initialSettled) { - return - } - initialSettled = true - if (error) { - rejectInitial(error) - } else { - resolveInitial() - } - } + const startup = createWslWatcherStartup() function signalWatcherStopped(): void { if (stopped) { @@ -217,7 +200,7 @@ export async function createWslWatcher( const nextSnapshot = parseSnapshotFrame(frame, distro) if (!prevSnapshot) { prevSnapshot = nextSnapshot - settleInitial() + startup.settle() return } const events = diffSnapshots(prevSnapshot, nextSnapshot) @@ -255,31 +238,40 @@ export async function createWslWatcher( } let child: ChildProcessWithoutNullStreams + const releaseChildReservation = reserveWatcherChild() + if (!releaseChildReservation) { + throw new WatcherChildCapacityError() + } try { child = spawn('wsl.exe', ['-d', distro, '--', 'sh', '-s', '--', linuxPath], { stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true }) } catch (error) { + releaseChildReservation() throw error instanceof Error ? error : new Error(String(error)) } + // Why: the physical-exit owner releases the shared process reservation on + // exactly the same close/error proof that settles destructive cleanup. + const processExit = createWslWatcherProcessExit(child, worktreePath, releaseChildReservation) + const onAbort = (): void => { - settleInitial(new DOMException('WSL watcher subscription aborted', 'AbortError')) - child.kill() + startup.settle(new DOMException('WSL watcher subscription aborted', 'AbortError')) + processExit.requestStopBestEffort() } signal?.addEventListener('abort', onAbort, { once: true }) const startupTimer = setTimeout(() => { - settleInitial(new Error(`Timed out starting WSL watcher for ${worktreePath}`)) - child.kill() + startup.settle(new Error(`Timed out starting WSL watcher for ${worktreePath}`)) + processExit.requestStopBestEffort() }, STARTUP_TIMEOUT_MS) child.stdin.on('error', (error) => { // Why: WSL can exit before reading the script; handle EPIPE here so the // startup failure rejects the watcher instead of crashing on a stream error. - if (!initialSettled) { - settleInitial(error) + if (!startup.settled) { + startup.settle(error) } }) @@ -296,8 +288,8 @@ export async function createWslWatcher( }) child.stdout.on('error', (error) => { - if (!initialSettled) { - settleInitial(error) + if (!startup.settled) { + startup.settle(error) return } if (!disposed) { @@ -310,8 +302,13 @@ export async function createWslWatcher( }) child.once('error', (error) => { - if (!initialSettled) { - settleInitial(error) + // Why: a child that never acquired a pid has no OS owner to await. Kill + // errors for an already-spawned child still require the close event. + if (child.pid === undefined) { + processExit.markPhysicalExit() + } + if (!startup.settled) { + startup.settle(error) return } if (!disposed) { @@ -320,9 +317,10 @@ export async function createWslWatcher( }) child.once('close', (code, signal) => { - if (!initialSettled) { + processExit.markPhysicalExit() + if (!startup.settled) { const suffix = stderrTail.trim() ? `: ${stderrTail.trim()}` : '' - settleInitial( + startup.settle( new Error(`WSL watcher exited before first snapshot (${code ?? signal})${suffix}`) ) return @@ -332,10 +330,16 @@ export async function createWslWatcher( } }) - child.stdin.end(buildSnapshotScript(deps.ignoreDirs)) - try { - await initialSnapshotReady + child.stdin.end(buildSnapshotScript(deps.ignoreDirs)) + await startup.ready + } catch (error) { + // Why: cancellation is part of destructive removal. Do not let the failed + // install disappear from ownership until WSL confirms the child exited. + startup.settle(error instanceof Error ? error : new Error(String(error))) + await startup.ready.catch(() => undefined) + await processExit.stopAndWait() + throw error } finally { clearTimeout(startupTimer) signal?.removeEventListener('abort', onAbort) @@ -344,7 +348,7 @@ export async function createWslWatcher( root.subscription = { unsubscribe: async () => { disposed = true - child.kill() + await processExit.stopAndWait() } } diff --git a/src/main/ipc/filesystem-watcher.test.ts b/src/main/ipc/filesystem-watcher.test.ts index f776e0d494d..ed66c7963da 100644 --- a/src/main/ipc/filesystem-watcher.test.ts +++ b/src/main/ipc/filesystem-watcher.test.ts @@ -27,9 +27,21 @@ vi.mock('../providers/ssh-filesystem-dispatch', () => ({ getSshFilesystemProvider: getSshFilesystemProviderMock })) -import { closeAllWatchers, registerFilesystemWatcherHandlers } from './filesystem-watcher' +import { + closeAllWatchers, + closeRemoteWatcherForWorktreePath, + registerFilesystemWatcherHandlers +} from './filesystem-watcher' import { stat } from 'node:fs/promises' import { subscribe as subscribeParcelWatcher } from '@parcel/watcher' +import { createWslWatcher } from './filesystem-watcher-wsl' +import { + MAX_PHYSICAL_WATCHER_CHILDREN, + reserveWatcherChild, + resetWatcherChildRegistryForTest, + WatcherChildCapacityError +} from './parcel-watcher-child-registry' +import { acquireWatcherRemovalGate } from './watcher-removal-gate' type HandlerMap = Record Promise | unknown> @@ -43,6 +55,8 @@ describe('registerFilesystemWatcherHandlers', () => { getSshFilesystemProviderMock.mockReset() vi.mocked(stat).mockReset() vi.mocked(subscribeParcelWatcher).mockReset() + vi.mocked(createWslWatcher).mockReset() + resetWatcherChildRegistryForTest() Object.defineProperty(process, 'platform', { configurable: true, value: originalPlatform @@ -79,6 +93,80 @@ describe('registerFilesystemWatcherHandlers', () => { await closeAllWatchers() }) + it('automatically retries a WSL watcher when child capacity becomes available', async () => { + Object.defineProperty(process, 'platform', { + configurable: true, + value: 'win32' + }) + vi.mocked(stat).mockResolvedValue({ isDirectory: () => true } as never) + const heldReservations = Array.from({ length: MAX_PHYSICAL_WATCHER_CHILDREN }, () => + reserveWatcherChild() + ) + vi.mocked(createWslWatcher).mockImplementation(async () => { + const release = reserveWatcherChild() + if (!release) { + throw new WatcherChildCapacityError() + } + return { + subscription: { unsubscribe: vi.fn(async () => release()) }, + listeners: new Map(), + batch: { events: [], overflowed: false, timer: null, firstEventAt: 0 } + } + }) + const sender = { isDestroyed: () => false, send: vi.fn(), once: vi.fn(), id: 1 } + const args = { worktreePath: '\\\\wsl.localhost\\Ubuntu\\home\\me\\repo' } + + await expect(handlers['fs:watchWorktree']({ sender }, args)).resolves.toBeUndefined() + expect(createWslWatcher).toHaveBeenCalledOnce() + + heldReservations.pop()?.() + await vi.waitFor(() => expect(createWslWatcher).toHaveBeenCalledTimes(2)) + await closeAllWatchers() + heldReservations.forEach((release) => release?.()) + }) + + it('cancels a pending WSL capacity retry when the renderer unwatches', async () => { + Object.defineProperty(process, 'platform', { + configurable: true, + value: 'win32' + }) + vi.mocked(stat).mockResolvedValue({ isDirectory: () => true } as never) + const heldReservations = Array.from({ length: MAX_PHYSICAL_WATCHER_CHILDREN }, () => + reserveWatcherChild() + ) + vi.mocked(createWslWatcher).mockRejectedValue(new WatcherChildCapacityError()) + const sender = { isDestroyed: () => false, send: vi.fn(), once: vi.fn(), id: 1 } + const args = { worktreePath: '\\\\wsl.localhost\\Ubuntu\\home\\me\\repo' } + + await handlers['fs:watchWorktree']({ sender }, args) + handlers['fs:unwatchWorktree']({ sender: { id: sender.id } }, args) + heldReservations.pop()?.() + await Promise.resolve() + await Promise.resolve() + + expect(createWslWatcher).toHaveBeenCalledOnce() + heldReservations.forEach((release) => release?.()) + }) + + it('rejects installs during destructive removal and allows a retry afterward', async () => { + vi.mocked(stat).mockResolvedValue({ isDirectory: () => true } as never) + vi.mocked(subscribeParcelWatcher).mockResolvedValue({ unsubscribe: vi.fn() } as never) + const sender = { isDestroyed: () => false, send: vi.fn(), once: vi.fn(), id: 1 } + const removal = acquireWatcherRemovalGate('/repo') + await removal.ready + + await expect( + handlers['fs:watchWorktree']({ sender }, { worktreePath: '/repo' }) + ).rejects.toMatchObject({ code: 'watcher_removal_in_progress' }) + expect(subscribeParcelWatcher).not.toHaveBeenCalled() + + removal.release() + await expect( + handlers['fs:watchWorktree']({ sender }, { worktreePath: '/repo' }) + ).resolves.toBeUndefined() + expect(subscribeParcelWatcher).toHaveBeenCalledTimes(1) + }) + it('quietly skips SSH worktree watches while the filesystem provider is unavailable', async () => { vi.useFakeTimers() const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) @@ -125,7 +213,8 @@ describe('registerFilesystemWatcherHandlers', () => { await vi.advanceTimersByTimeAsync(1_000) expect(watchMock).toHaveBeenCalledWith('/home/me/repo', expect.any(Function), { - signal: expect.any(AbortSignal) + signal: expect.any(AbortSignal), + onTerminalError: expect.any(Function) }) const onEvents = watchMock.mock.calls[0][1] onEvents([{ path: '/home/me/repo/file.txt', type: 'update' }]) @@ -164,7 +253,8 @@ describe('registerFilesystemWatcherHandlers', () => { await vi.advanceTimersByTimeAsync(1_000) expect(retryWatchMock).toHaveBeenCalledWith('/home/me/repo', expect.any(Function), { - signal: expect.any(AbortSignal) + signal: expect.any(AbortSignal), + onTerminalError: expect.any(Function) }) handlers['fs:unwatchWorktree']( { sender: { id: 1 } }, @@ -195,6 +285,47 @@ describe('registerFilesystemWatcherHandlers', () => { vi.useRealTimers() }) + it('retries all live renderer owners after a terminal relay watch failure', async () => { + vi.useFakeTimers() + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const senderOne = { isDestroyed: () => false, send: vi.fn(), once: vi.fn(), id: 1 } + const senderTwo = { isDestroyed: () => false, send: vi.fn(), once: vi.fn(), id: 2 } + const watchMock = vi.fn().mockResolvedValue(vi.fn()) + getSshFilesystemProviderMock.mockReturnValue({ watch: watchMock }) + + await handlers['fs:watchWorktree']( + { sender: senderOne }, + { worktreePath: '/home/me/repo', connectionId: 'conn-1' } + ) + await handlers['fs:watchWorktree']( + { sender: senderTwo }, + { worktreePath: '/home/me/repo', connectionId: 'conn-1' } + ) + const staleTerminal = watchMock.mock.calls[0][2].onTerminalError as (error: Error) => void + + staleTerminal(new Error('relay watcher recovery failed')) + handlers['fs:unwatchWorktree']( + { sender: senderOne }, + { worktreePath: '/home/me/repo', connectionId: 'conn-1' } + ) + await vi.advanceTimersByTimeAsync(1_000) + + expect(watchMock).toHaveBeenCalledTimes(2) + const recoveredEvents = watchMock.mock.calls[1][1] as (events: unknown[]) => void + recoveredEvents([{ kind: 'update', absolutePath: '/home/me/repo/file.ts' }]) + expect(senderOne.send).not.toHaveBeenCalled() + expect(senderTwo.send).toHaveBeenCalledWith('fs:changed', { + worktreePath: '/home/me/repo', + events: [{ kind: 'update', absolutePath: '/home/me/repo/file.ts' }] + }) + + staleTerminal(new Error('late stale failure')) + await vi.advanceTimersByTimeAsync(1_000) + expect(watchMock).toHaveBeenCalledTimes(2) + warnSpy.mockRestore() + vi.useRealTimers() + }) + it('shares SSH worktree watchers across renderer senders until the last unwatch', async () => { const sendOne = vi.fn() const sendTwo = vi.fn() @@ -232,6 +363,71 @@ describe('registerFilesystemWatcherHandlers', () => { expect(unwatchMock).toHaveBeenCalledTimes(1) }) + it('awaits acknowledged SSH watcher teardown before remote deletion', async () => { + let resolveClose: () => void = () => {} + const closeWatch = vi.fn( + () => + new Promise((resolve) => { + resolveClose = resolve + }) + ) + const provider = { watch: vi.fn().mockResolvedValue(vi.fn()), closeWatch } + getSshFilesystemProviderMock.mockReturnValue(provider) + await handlers['fs:watchWorktree']( + { sender: { isDestroyed: () => false, send: vi.fn(), once: vi.fn(), id: 1 } }, + { worktreePath: '/home/me/repo', connectionId: 'conn-1' } + ) + + let closed = false + const close = closeRemoteWatcherForWorktreePath('conn-1', '/home/me/repo').then(() => { + closed = true + }) + await Promise.resolve() + expect(closeWatch).toHaveBeenCalledWith('/home/me/repo') + expect(closed).toBe(false) + + resolveClose() + await close + }) + + it('preserves remote event routing when acknowledged teardown rejects', async () => { + const sendOne = vi.fn() + const sendTwo = vi.fn() + const senderOne = { isDestroyed: () => false, send: sendOne, once: vi.fn(), id: 1 } + const senderTwo = { isDestroyed: () => false, send: sendTwo, once: vi.fn(), id: 2 } + const watchMock = vi.fn().mockResolvedValue(vi.fn()) + const closeWatch = vi + .fn() + .mockRejectedValueOnce(new Error('Remote path is still watched by another client')) + .mockResolvedValueOnce(undefined) + getSshFilesystemProviderMock.mockReturnValue({ watch: watchMock, closeWatch }) + await handlers['fs:watchWorktree']( + { sender: senderOne }, + { worktreePath: '/home/me/repo', connectionId: 'conn-1' } + ) + + await expect(closeRemoteWatcherForWorktreePath('conn-1', '/home/me/repo')).rejects.toThrow( + 'still watched by another client' + ) + const onEvents = watchMock.mock.calls[0][1] + onEvents([{ path: '/home/me/repo/file.txt', type: 'update' }]) + expect(sendOne).toHaveBeenCalledTimes(1) + + await handlers['fs:watchWorktree']( + { sender: senderTwo }, + { worktreePath: '/home/me/repo', connectionId: 'conn-1' } + ) + expect(watchMock).toHaveBeenCalledTimes(1) + onEvents([{ path: '/home/me/repo/file-2.txt', type: 'update' }]) + expect(sendOne).toHaveBeenCalledTimes(2) + expect(sendTwo).toHaveBeenCalledTimes(1) + + await expect( + closeRemoteWatcherForWorktreePath('conn-1', '/home/me/repo') + ).resolves.toBeUndefined() + expect(closeWatch).toHaveBeenCalledTimes(2) + }) + it('dedupes concurrent pending SSH worktree watcher installs', async () => { const sendOne = vi.fn() const sendTwo = vi.fn() diff --git a/src/main/ipc/filesystem-watcher.ts b/src/main/ipc/filesystem-watcher.ts index 665b543779f..89b83561800 100644 --- a/src/main/ipc/filesystem-watcher.ts +++ b/src/main/ipc/filesystem-watcher.ts @@ -8,12 +8,22 @@ import * as path from 'node:path' import { stat } from 'node:fs/promises' import type { Event as WatcherEvent } from '@parcel/watcher' import type { FsChangeEvent, FsChangedPayload } from '../../shared/types' +import { + isWindowsAbsolutePathLike, + normalizeRuntimePathForComparison +} from '../../shared/cross-platform-path' import { isWslPath } from '../wsl' import { createWslWatcher } from './filesystem-watcher-wsl' import type { WatchedRoot } from './filesystem-watcher-wsl' import { getSshFilesystemProvider } from '../providers/ssh-filesystem-dispatch' import { MAX_BATCHED_WATCHER_EVENTS, queueWatcherEvents } from './filesystem-watcher-event-batch' import { disposeWatcherProcess, subscribeViaWatcherProcess } from './parcel-watcher-process' +import { isWatcherProcessFailure } from './parcel-watcher-process-failure' +import { + onWatcherChildCapacityAvailable, + WatcherChildCapacityError +} from './parcel-watcher-child-registry' +import { beginWatcherInstall, isWatcherRemovalInProgressError } from './watcher-removal-gate' // Why: high-churn directories are suppressed at the native watcher level so // events never leave the OS/daemon. This list is separate from the File // Explorer display filter (which only hides rows). Directories like `dist` @@ -69,16 +79,23 @@ const pendingTeardowns = new Map>() // Why: @parcel/watcher unsubscribe completes native async work. Sender-destroy // cleanup can start it before app shutdown, so will-quit must still await it. const pendingLocalUnsubscribes = new Set>() +const pendingLocalUnsubscribesByRoot = new Map>>() +const failedLocalUnsubscribes = new Map() type LocalWatcherInstallToken = { cancelled: boolean listeners: Map abortController: AbortController } -type LocalWatcherInstallResult = 'installed' | 'unavailable' | 'cancelled' +type LocalWatcherInstallResult = 'installed' | 'unavailable' | 'capacity' | 'cancelled' +type LocalWatcherCapacityRetry = { + listeners: Map + cancelWait: () => void +} // Why: native watcher creation is async. Concurrent local watch requests for // the same root must share one install or later resolves can orphan listeners. const inFlightLocalInstalls = new Map() const pendingLocalInstallPromises = new Map>() +const pendingLocalCapacityRetries = new Map() function addInFlightLocalInstallListener( token: LocalWatcherInstallToken, @@ -102,12 +119,84 @@ function cleanupInFlightLocalInstallsForSender(senderId: number): void { token.abortController.abort() } } + for (const [rootKey, retry] of pendingLocalCapacityRetries) { + retry.listeners.delete(senderId) + if (retry.listeners.size === 0) { + retry.cancelWait() + pendingLocalCapacityRetries.delete(rootKey) + } + } +} + +function takeLocalCapacityRetryListeners(rootKey: string): WebContents[] { + const retry = pendingLocalCapacityRetries.get(rootKey) + if (!retry) { + return [] + } + retry.cancelWait() + pendingLocalCapacityRetries.delete(rootKey) + return [...retry.listeners.values()].filter((listener) => !listener.isDestroyed()) +} + +function clearLocalCapacityRetry(rootKey: string): void { + const retry = pendingLocalCapacityRetries.get(rootKey) + retry?.cancelWait() + pendingLocalCapacityRetries.delete(rootKey) +} + +function scheduleLocalCapacityRetry( + rootKey: string, + worktreePath: string, + listeners: Map +): void { + const existing = pendingLocalCapacityRetries.get(rootKey) + if (existing) { + for (const listener of listeners.values()) { + if (!listener.isDestroyed()) { + existing.listeners.set(listener.id, listener) + } + } + return + } + + let retry!: LocalWatcherCapacityRetry + const cancelWait = onWatcherChildCapacityAvailable(async () => { + if (pendingLocalCapacityRetries.get(rootKey) !== retry) { + return + } + pendingLocalCapacityRetries.delete(rootKey) + await Promise.all( + [...retry.listeners.values()].map(async (listener) => { + if (listener.isDestroyed()) { + return + } + await subscribe(worktreePath, listener).catch((error: unknown) => { + if (!isWatcherRemovalInProgressError(error)) { + console.error(`[filesystem-watcher] capacity retry failed for ${rootKey}:`, error) + } + }) + }) + ) + }) + retry = { listeners: new Map(), cancelWait } + pendingLocalCapacityRetries.set(rootKey, retry) + for (const listener of listeners.values()) { + if (!listener.isDestroyed()) { + retry.listeners.set(listener.id, listener) + registerSenderCleanup(listener) + } + } + if (retry.listeners.size === 0) { + clearLocalCapacityRetry(rootKey) + } } // ── Path normalization ─────────────────────────────────────────────── function normalizeRootPath(rootPath: string): string { - let resolved = path.resolve(rootPath) + let resolved = isWindowsAbsolutePathLike(rootPath) + ? path.win32.resolve(rootPath) + : path.resolve(rootPath) // Why: on Windows, watcher events may report lowercase drive letters while // stored worktree paths use uppercase. Normalizing here ensures the renderer's // POSIX normalization produces casing-consistent results (see design §4.4). @@ -117,6 +206,16 @@ function normalizeRootPath(rootPath: string): string { return resolved } +function localWatcherRoot(rootPath: string): { key: string; path: string } { + const normalizedPath = normalizeRootPath(rootPath) + return { + // Why: Windows drive and UNC paths are case-insensitive; destructive + // cleanup must find the owner even when Git returns a different spelling. + key: normalizeRuntimePathForComparison(normalizedPath), + path: normalizedPath + } +} + function normalizeEventPath(eventPath: string): string { let resolved = path.resolve(eventPath) if (/^[a-zA-Z]:/.test(resolved)) { @@ -198,9 +297,10 @@ async function tryStatIsDirectory(filePath: string): Promise { ) const payload: FsChangedPayload = { - worktreePath: rootKey, + worktreePath: root.rootPath ?? rootKey, events } @@ -290,7 +390,8 @@ async function createWatcher( const root: WatchedRoot = { subscription: null!, listeners: new Map(), - batch: { events: [], overflowed: false, timer: null, firstEventAt: 0 } + batch: { events: [], overflowed: false, timer: null, firstEventAt: 0 }, + rootPath } try { @@ -338,6 +439,7 @@ async function createWatcher( // is deleted or inaccessible at startup). Guard against null so // the cleanup path doesn't crash the main process. if (root.subscription) { + retainLocalWatcherPhysicalFailure(rootKey, err) void trackLocalUnsubscribe(rootKey, root) } errorCleanedUp = true @@ -402,18 +504,40 @@ function cleanupLocalWatchersForSender(senderId: number): void { } function trackLocalUnsubscribe(rootKey: string, root: WatchedRoot): Promise { + const rootUnsubscribes = pendingLocalUnsubscribesByRoot.get(rootKey) ?? new Set>() + pendingLocalUnsubscribesByRoot.set(rootKey, rootUnsubscribes) const unsubscribePromise = Promise.resolve() .then(() => root.subscription.unsubscribe()) - .catch((err: unknown) => { - console.error(`[filesystem-watcher] unsubscribe error for ${rootKey}:`, err) - }) .finally(() => { pendingLocalUnsubscribes.delete(unsubscribePromise) + rootUnsubscribes.delete(unsubscribePromise) + if (rootUnsubscribes.size === 0) { + pendingLocalUnsubscribesByRoot.delete(rootKey) + } }) pendingLocalUnsubscribes.add(unsubscribePromise) + rootUnsubscribes.add(unsubscribePromise) + // Why: background cleanup must not create unhandled rejections, but the + // original promise stays rejected so later destructive cleanup can fail closed. + void unsubscribePromise.catch((error: unknown) => { + retainLocalWatcherPhysicalFailure(rootKey, error) + console.error(`[filesystem-watcher] unsubscribe error for ${rootKey}:`, error) + }) return unsubscribePromise } +function retainLocalWatcherPhysicalFailure(rootKey: string, error: unknown): void { + if (!isWatcherProcessFailure(error) || !error.physicalExit) { + return + } + failedLocalUnsubscribes.set(rootKey, error) + void error.physicalExit.then(() => { + if (failedLocalUnsubscribes.get(rootKey) === error) { + failedLocalUnsubscribes.delete(rootKey) + } + }) +} + function registerSenderCleanup(sender: WebContents): void { if (senderCleanupRegistered.has(sender.id)) { return @@ -436,7 +560,19 @@ function addLocalWatchListener(rootKey: string, sender: WebContents): void { } async function subscribe(worktreePath: string, sender: WebContents): Promise { - const rootKey = normalizeRootPath(worktreePath) + const finishInstall = beginWatcherInstall(worktreePath) + try { + await subscribeWhileRemovalAllowed(worktreePath, sender) + } finally { + finishInstall() + } +} + +async function subscribeWhileRemovalAllowed( + worktreePath: string, + sender: WebContents +): Promise { + const { key: rootKey, path: rootPath } = localWatcherRoot(worktreePath) if (sender.isDestroyed()) { return } @@ -455,8 +591,12 @@ async function subscribe(worktreePath: string, sender: WebContents): Promise [listener.id, listener]) + ) + retryListeners.set(sender.id, sender) + scheduleLocalCapacityRetry(rootKey, worktreePath, retryListeners) + } + } if ( result === 'installed' && watchedRoots.has(rootKey) && @@ -487,8 +643,11 @@ async function subscribe(worktreePath: string, sender: WebContents): Promise { let root: WatchedRoot try { - const s = await stat(rootKey) + const s = await stat(rootPath) if (!s.isDirectory()) { console.warn(`[filesystem-watcher] not a directory: ${rootKey}`) rememberUnwatchableRoot(rootKey) @@ -532,13 +692,23 @@ async function doInstallLocalWatcher( }, cancelToken.abortController.signal ) - : await createWatcher(rootKey, rootKey, cancelToken.abortController.signal) - } catch { - // Why: createWatcher / createWslWatcher already logged the error. Swallow - // it here so the renderer's watchWorktree call resolves without crashing. + : await createWatcher(rootKey, rootPath, cancelToken.abortController.signal) + } catch (error) { + // Why: setup may fail after its child misses the exit deadline; retain that + // owner even when the ordinary renderer-facing setup error is swallowed. + retainLocalWatcherPhysicalFailure(rootKey, error) if (cancelToken.cancelled) { + if (isWatcherProcessFailure(error) && error.code === 'process_unavailable') { + throw error + } return 'cancelled' } + // Why: capacity is transient; once another physical child exits this root + // must be allowed to retry instead of entering the permanent-failure cache. + if (error instanceof WatcherChildCapacityError) { + scheduleLocalCapacityRetry(rootKey, worktreePath, cancelToken.listeners) + return 'capacity' + } rememberUnwatchableRoot(rootKey) return 'unavailable' } finally { @@ -567,7 +737,14 @@ async function doInstallLocalWatcher( } function unsubscribe(worktreePath: string, senderId: number): void { - const rootKey = normalizeRootPath(worktreePath) + const { key: rootKey } = localWatcherRoot(worktreePath) + const capacityRetry = pendingLocalCapacityRetries.get(rootKey) + if (capacityRetry) { + capacityRetry.listeners.delete(senderId) + if (capacityRetry.listeners.size === 0) { + clearLocalCapacityRetry(rootKey) + } + } const inFlight = inFlightLocalInstalls.get(rootKey) if (inFlight) { inFlight.listeners.delete(senderId) @@ -615,7 +792,8 @@ function unsubscribe(worktreePath: string, senderId: number): void { } export async function closeLocalWatcherForWorktreePath(worktreePath: string): Promise { - const rootKey = normalizeRootPath(worktreePath) + const { key: rootKey } = localWatcherRoot(worktreePath) + clearLocalCapacityRetry(rootKey) const pendingTeardown = pendingTeardowns.get(rootKey) if (pendingTeardown) { clearTimeout(pendingTeardown) @@ -630,7 +808,14 @@ export async function closeLocalWatcherForWorktreePath(worktreePath: string): Pr inFlight.cancelled = true inFlight.abortController.abort() } - await pendingLocalInstallPromises.get(rootKey)?.catch(() => undefined) + await pendingLocalInstallPromises.get(rootKey) + const pendingUnsubscribes = pendingLocalUnsubscribesByRoot.get(rootKey) + if (pendingUnsubscribes) { + await Promise.all(Array.from(pendingUnsubscribes)) + } + if (failedLocalUnsubscribes.has(rootKey)) { + throw failedLocalUnsubscribes.get(rootKey) + } const root = watchedRoots.get(rootKey) if (!root) { @@ -647,6 +832,7 @@ export async function closeLocalWatcherForWorktreePath(worktreePath: string): Pr type RemoteWatcherState = { unwatch: () => void listeners: Map + installToken: RemoteWatcherInstallToken } type RemoteWatcherInstallToken = { @@ -654,14 +840,19 @@ type RemoteWatcherInstallToken = { listeners: Map abortController: AbortController abortScheduled: boolean + terminalError?: Error } // Key: `${connectionId}:${worktreePath}`, Value: shared remote watch state. const remoteWatchers = new Map() const loggedUnavailableRemoteWatchers = new Set() const pendingRemoteWatcherRetries = new Map>() -// Why: track in-flight `provider.watch()` calls so last-listener cleanup can -// abort relay setup, while late success is still unwatched instead of leaked. +const pendingRemoteWatcherRetryListeners = new Map< + string, + { listeners: Map; startedAt: number } +>() +// Why: last-listener cleanup aborts relay setup, while late success is still +// unwatched instead of being installed after the renderer stopped watching. const inFlightRemoteInstalls = new Map() // Why: dedupe concurrent installRemoteWatcher calls for the same key so // overlapping fs:watchWorktree IPCs share one native watcher and one listener @@ -680,6 +871,31 @@ let remoteWatcherLifecycleGeneration = 0 const REMOTE_WATCH_RETRY_MS = 1_000 const REMOTE_WATCH_RETRY_TIMEOUT_MS = 60_000 +export async function closeRemoteWatcherForWorktreePath( + connectionId: string, + worktreePath: string +): Promise { + const key = remoteWatcherKey(connectionId, worktreePath) + const retryTimer = pendingRemoteWatcherRetries.get(key) + if (retryTimer) { + clearTimeout(retryTimer) + pendingRemoteWatcherRetries.delete(key) + pendingRemoteWatcherRetryListeners.delete(key) + } + const inFlight = inFlightRemoteInstalls.get(key) + if (inFlight) { + inFlight.listeners.clear() + inFlight.cancelled = true + } + const state = remoteWatchers.get(key) + const provider = getSshFilesystemProvider(connectionId) + await (provider?.closeWatch + ? provider.closeWatch(worktreePath) + : Promise.resolve(state?.unwatch())) + remoteWatchers.delete(key) + loggedUnavailableRemoteWatchers.delete(key) +} + function addInFlightRemoteInstallListener( token: RemoteWatcherInstallToken, sender: WebContents @@ -713,6 +929,17 @@ function cleanupInFlightRemoteInstallsForSender(senderId: number): void { token.listeners.delete(senderId) cancelInFlightRemoteInstallIfUnowned(token) } + for (const [key, retry] of pendingRemoteWatcherRetryListeners) { + retry.listeners.delete(senderId) + if (retry.listeners.size === 0) { + const timer = pendingRemoteWatcherRetries.get(key) + if (timer) { + clearTimeout(timer) + pendingRemoteWatcherRetries.delete(key) + } + pendingRemoteWatcherRetryListeners.delete(key) + } + } } function addRemoteWatchListener(key: string, sender: WebContents): void { @@ -759,12 +986,31 @@ async function installRemoteWatcher( if (remoteWatchersClosed || generation !== remoteWatcherLifecycleGeneration) { return 'cancelled' } + const finishInstall = beginWatcherInstall(worktreePath, connectionId) + try { + return await installRemoteWatcherWhileRemovalAllowed( + sender, + connectionId, + worktreePath, + generation + ) + } finally { + finishInstall() + } +} + +async function installRemoteWatcherWhileRemovalAllowed( + sender: WebContents, + connectionId: string, + worktreePath: string, + generation: number +): Promise { const provider = getSshFilesystemProvider(connectionId) if (!provider || sender.isDestroyed()) { return 'unavailable' } - const key = `${connectionId}:${worktreePath}` + const key = remoteWatcherKey(connectionId, worktreePath) const existing = remoteWatchers.get(key) if (existing) { addRemoteWatchListener(key, sender) @@ -817,7 +1063,13 @@ async function installRemoteWatcher( } inFlightRemoteInstalls.set(key, cancelToken) addInFlightRemoteInstallListener(cancelToken, sender) - const installPromise = doInstallRemoteWatcher(provider, key, worktreePath, cancelToken) + const installPromise = doInstallRemoteWatcher( + provider, + key, + connectionId, + worktreePath, + cancelToken + ) pendingRemoteInstallPromises.set(key, installPromise) try { return await installPromise @@ -831,6 +1083,7 @@ async function installRemoteWatcher( async function doInstallRemoteWatcher( provider: NonNullable>, key: string, + connectionId: string, worktreePath: string, cancelToken: RemoteWatcherInstallToken ): Promise { @@ -853,7 +1106,11 @@ async function doInstallRemoteWatcher( } satisfies FsChangedPayload) } }, - { signal: cancelToken.abortController.signal } + { + signal: cancelToken.abortController.signal, + onTerminalError: (error) => + handleRemoteWatcherTerminalError(key, connectionId, worktreePath, cancelToken, error) + } ) } catch (err) { if (cancelToken.cancelled || cancelToken.abortController.signal.aborted) { @@ -877,7 +1134,10 @@ async function doInstallRemoteWatcher( } return 'cancelled' } - remoteWatchers.set(key, { unwatch, listeners: liveListeners }) + if (cancelToken.terminalError) { + return 'unavailable' + } + remoteWatchers.set(key, { unwatch, listeners: liveListeners, installToken: cancelToken }) for (const listener of liveListeners.values()) { registerSenderCleanup(listener) } @@ -885,29 +1145,62 @@ async function doInstallRemoteWatcher( return 'installed' } +function handleRemoteWatcherTerminalError( + key: string, + connectionId: string, + worktreePath: string, + installToken: RemoteWatcherInstallToken, + error: Error +): void { + installToken.terminalError = error + const state = remoteWatchers.get(key) + if (!state || state.installToken !== installToken) { + return + } + remoteWatchers.delete(key) + console.warn(`[filesystem-watcher] SSH watcher terminated for ${key}:`, error) + for (const listener of state.listeners.values()) { + scheduleRemoteWatcherRetry(listener, connectionId, worktreePath) + } +} + function scheduleRemoteWatcherRetry( sender: WebContents, connectionId: string, worktreePath: string, startedAt = Date.now() ): void { - const key = `${connectionId}:${worktreePath}` - if (pendingRemoteWatcherRetries.has(key)) { + const key = remoteWatcherKey(connectionId, worktreePath) + const existingRetry = pendingRemoteWatcherRetryListeners.get(key) + if (existingRetry) { + if (!sender.isDestroyed()) { + existingRetry.listeners.set(sender.id, sender) + } return } + const retry = { + listeners: new Map(sender.isDestroyed() ? [] : [[sender.id, sender]]), + startedAt + } + pendingRemoteWatcherRetryListeners.set(key, retry) + if (Date.now() - startedAt >= REMOTE_WATCH_RETRY_TIMEOUT_MS || sender.isDestroyed()) { pendingRemoteWatcherRetries.delete(key) + pendingRemoteWatcherRetryListeners.delete(key) loggedUnavailableRemoteWatchers.delete(key) // Why: the original `fs:watchWorktree` handler resolved successfully // when the retry was first scheduled, so the renderer believes the // watch is live. After giving up, emit a one-shot overflow so the // renderer falls back to a manual refresh instead of waiting forever. - if (!sender.isDestroyed()) { + for (const listener of retry.listeners.values()) { + if (listener.isDestroyed()) { + continue + } console.warn( `[filesystem-watcher] giving up SSH watch retry for ${worktreePath} on connection ${connectionId} after ${REMOTE_WATCH_RETRY_TIMEOUT_MS}ms` ) - sender.send('fs:changed', { + listener.send('fs:changed', { worktreePath, events: [{ kind: 'overflow', absolutePath: worktreePath }] } satisfies FsChangedPayload) @@ -917,18 +1210,31 @@ function scheduleRemoteWatcherRetry( const retryTimer = setTimeout(() => { pendingRemoteWatcherRetries.delete(key) - void installRemoteWatcher(sender, connectionId, worktreePath) - .then((result) => { + pendingRemoteWatcherRetryListeners.delete(key) + const listeners = Array.from(retry.listeners.values()).filter( + (listener) => !listener.isDestroyed() + ) + void Promise.all( + listeners.map((listener) => installRemoteWatcher(listener, connectionId, worktreePath)) + ) + .then((results) => { // Why: 'cancelled' means an unwatch/shutdown raced with this install // attempt. Re-arming the retry would reschedule for a worktree the // renderer explicitly stopped watching, eventually firing a stale // overflow when the 60s window expires. - if (result === 'unavailable') { - scheduleRemoteWatcherRetry(sender, connectionId, worktreePath, startedAt) + if (results.some((result) => result === 'unavailable')) { + for (const listener of listeners) { + scheduleRemoteWatcherRetry(listener, connectionId, worktreePath, retry.startedAt) + } } }) - .catch(() => { - scheduleRemoteWatcherRetry(sender, connectionId, worktreePath, startedAt) + .catch((error: unknown) => { + if (isWatcherRemovalInProgressError(error)) { + return + } + for (const listener of listeners) { + scheduleRemoteWatcherRetry(listener, connectionId, worktreePath, retry.startedAt) + } }) }, REMOTE_WATCH_RETRY_MS) pendingRemoteWatcherRetries.set(key, retryTimer) @@ -944,7 +1250,7 @@ export function registerFilesystemWatcherHandlers(): void { // Why: a real new watch reopens the subsystem after closeAllWatchers // latched it shut (also how tests reset between cases). remoteWatchersClosed = false - const key = `${args.connectionId}:${args.worktreePath}` + const key = remoteWatcherKey(args.connectionId, args.worktreePath) const result = await installRemoteWatcher( event.sender, args.connectionId, @@ -970,11 +1276,14 @@ export function registerFilesystemWatcherHandlers(): void { 'fs:unwatchWorktree', (_event, args: { worktreePath: string; connectionId?: string }): void => { if (args.connectionId) { - const key = `${args.connectionId}:${args.worktreePath}` + const key = remoteWatcherKey(args.connectionId, args.worktreePath) + const retry = pendingRemoteWatcherRetryListeners.get(key) + retry?.listeners.delete(_event.sender.id) const retryTimer = pendingRemoteWatcherRetries.get(key) - if (retryTimer) { + if (retryTimer && retry?.listeners.size === 0) { clearTimeout(retryTimer) pendingRemoteWatcherRetries.delete(key) + pendingRemoteWatcherRetryListeners.delete(key) } // Why: a `provider.watch()` call may still be in flight from a // retry tick. Mark it cancelled so installRemoteWatcher discards @@ -995,10 +1304,18 @@ export function registerFilesystemWatcherHandlers(): void { ) } +function remoteWatcherKey(connectionId: string, worktreePath: string): string { + return JSON.stringify([connectionId, normalizeRuntimePathForComparison(worktreePath)]) +} + /** Tear down all watchers on app shutdown. */ export async function closeAllWatchers(): Promise { senderCleanupRegistered.clear() unwatchableRoots.clear() + for (const retry of pendingLocalCapacityRetries.values()) { + retry.cancelWait() + } + pendingLocalCapacityRetries.clear() // Cancel any pending grace-period teardowns — we're tearing down everything. for (const timer of pendingTeardowns.values()) { @@ -1010,6 +1327,7 @@ export async function closeAllWatchers(): Promise { clearTimeout(timer) } pendingRemoteWatcherRetries.clear() + pendingRemoteWatcherRetryListeners.clear() loggedUnavailableRemoteWatchers.clear() // Why: latch the subsystem shut and drop the dedup map so a late install that // begins after teardown is refused instead of registering post-shutdown. Bump @@ -1035,10 +1353,11 @@ export async function closeAllWatchers(): Promise { if (root.batch.timer) { clearTimeout(root.batch.timer) } - await trackLocalUnsubscribe(rootKey, root) + await trackLocalUnsubscribe(rootKey, root).catch(() => undefined) } watchedRoots.clear() await Promise.allSettled(Array.from(pendingLocalUnsubscribes)) + failedLocalUnsubscribes.clear() // Why: with every local subscription released, drop the forked watcher // process outright — process death frees any remaining native handles // without running watcher.node's crash-prone async teardown in this process. diff --git a/src/main/ipc/parcel-watcher-cancellation-restart.ts b/src/main/ipc/parcel-watcher-cancellation-restart.ts new file mode 100644 index 00000000000..21f59d07f1d --- /dev/null +++ b/src/main/ipc/parcel-watcher-cancellation-restart.ts @@ -0,0 +1,86 @@ +import type { ChildProcess } from 'node:child_process' +import { + failAllWatcherSubscriptions, + type PendingWatcherUnsubscribe, + resolvePendingWatcherUnsubscribes +} from './parcel-watcher-host-subscriptions' +import type { WatcherCancellationTracker } from './parcel-watcher-cancellation-tracker' +import { + createWatcherChildTerminationFailure, + terminateWatcherChild +} from './parcel-watcher-child-termination' +import { resetPendingSubscribeAttempt } from './parcel-watcher-pending-subscribe' +import { WatcherProcessFailure } from './parcel-watcher-process-failure' +import type { WatcherProcessSubscriptionRecord } from './parcel-watcher-process-subscription' + +export function restoreWatcherRecordsAfterCancellation( + records: Map, + ensureWatcherProcess: () => ChildProcess | null, + sendSubscribe: (child: ChildProcess, record: WatcherProcessSubscriptionRecord) => void, + onUnavailable: () => void +): void { + if (records.size === 0) { + return + } + const replacement = ensureWatcherProcess() + if (!replacement) { + failAllWatcherSubscriptions( + records, + new WatcherProcessFailure( + 'file watcher process unavailable after subscription cancellation', + 'supervisor', + 'process_unavailable' + ) + ) + onUnavailable() + return + } + for (const record of records.values()) { + record.interrupted = true + resetPendingSubscribeAttempt(record) + sendSubscribe(replacement, record) + } +} + +export function failWatcherRecordsAfterTerminationDeadline( + records: Map, + error = new WatcherProcessFailure( + 'file watcher process did not exit after termination deadline', + 'supervisor', + 'process_unavailable' + ) +): void { + failAllWatcherSubscriptions(records, error) +} + +export async function restartCancelledWatcherChild( + child: ChildProcess, + records: Map, + pendingUnsubscribes: Map, + cancelledSubscribes: WatcherCancellationTracker, + onTerminationFinished: (exited: boolean) => boolean, + ensureWatcherProcess: () => ChildProcess | null, + sendSubscribe: (child: ChildProcess, record: WatcherProcessSubscriptionRecord) => void, + onUnavailable: () => void +): Promise { + const exited = await terminateWatcherChild(child) + const shouldRestore = onTerminationFinished(exited) + if (!exited) { + const error = createWatcherChildTerminationFailure(child) + cancelledSubscribes.finishRestart(child, error) + resolvePendingWatcherUnsubscribes(pendingUnsubscribes, error) + failWatcherRecordsAfterTerminationDeadline(records, error) + throw error + } + cancelledSubscribes.finishRestart(child) + resolvePendingWatcherUnsubscribes(pendingUnsubscribes) + if (!shouldRestore) { + return + } + restoreWatcherRecordsAfterCancellation( + records, + ensureWatcherProcess, + sendSubscribe, + onUnavailable + ) +} diff --git a/src/main/ipc/parcel-watcher-cancellation-tracker.ts b/src/main/ipc/parcel-watcher-cancellation-tracker.ts new file mode 100644 index 00000000000..78f39e3bbee --- /dev/null +++ b/src/main/ipc/parcel-watcher-cancellation-tracker.ts @@ -0,0 +1,77 @@ +import type { ChildProcess } from 'node:child_process' +import { RUNTIME_FILE_WATCH_CANCEL_TIMEOUT_MS } from '../../shared/runtime-file-watch-limits' + +type CancelledSubscribe = { + child: ChildProcess + error: Error + reject: (error: Error) => void + timer: ReturnType +} + +// Why: cancellation may queue behind a legitimate crawl, but must not let a +// wedged native unsubscribe block destructive worktree cleanup forever. +export const WATCHER_PROCESS_CANCEL_TIMEOUT_MS = RUNTIME_FILE_WATCH_CANCEL_TIMEOUT_MS + +export class WatcherCancellationTracker { + private readonly entries = new Map() + private readonly restartingChildren = new Set() + + begin( + id: number, + child: ChildProcess, + error: Error, + reject: (error: Error) => void, + onTimeout: () => void + ): void { + const timer = setTimeout(onTimeout, WATCHER_PROCESS_CANCEL_TIMEOUT_MS) + timer.unref?.() + this.entries.set(id, { child, error, reject, timer }) + } + + has(id: number): boolean { + return this.entries.has(id) + } + + beginRestart(child: ChildProcess): boolean { + if (this.restartingChildren.has(child)) { + return false + } + this.restartingChildren.add(child) + for (const cancelled of this.entries.values()) { + if (cancelled.child === child) { + clearTimeout(cancelled.timer) + } + } + return true + } + + finishRestart(child: ChildProcess, error?: Error): void { + this.completeForChild(child, error) + this.restartingChildren.delete(child) + } + + complete(id: number, error?: Error): boolean { + const cancelled = this.entries.get(id) + if (!cancelled) { + return false + } + this.entries.delete(id) + clearTimeout(cancelled.timer) + cancelled.reject(error ?? cancelled.error) + return true + } + + completeAll(): void { + for (const id of this.entries.keys()) { + this.complete(id) + } + } + + completeForChild(child: ChildProcess, error?: Error): void { + for (const [id, cancelled] of this.entries) { + if (cancelled.child === child) { + this.complete(id, error) + } + } + } +} diff --git a/src/main/ipc/parcel-watcher-child-launch.test.ts b/src/main/ipc/parcel-watcher-child-launch.test.ts new file mode 100644 index 00000000000..bc2e8635d72 --- /dev/null +++ b/src/main/ipc/parcel-watcher-child-launch.test.ts @@ -0,0 +1,76 @@ +import { EventEmitter } from 'node:events' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type * as WatcherChildRegistry from './parcel-watcher-child-registry' + +const { forkMock, releaseReservationMock, signalPhysicalExitMock, writeFileSyncMock } = vi.hoisted( + () => ({ + forkMock: vi.fn(), + releaseReservationMock: vi.fn(), + signalPhysicalExitMock: vi.fn(), + writeFileSyncMock: vi.fn() + }) +) + +vi.mock('node:child_process', () => ({ fork: forkMock })) +vi.mock('node:fs', () => ({ writeFileSync: writeFileSyncMock })) +vi.mock('./parcel-watcher-canary-directory', () => ({ + createWatcherCanaryDirectory: vi.fn(() => null), + removeWatcherCanaryDirectory: vi.fn() +})) +vi.mock('./parcel-watcher-child-registry', async (importOriginal) => ({ + ...(await importOriginal()), + reserveWatcherChild: vi.fn(() => releaseReservationMock) +})) +vi.mock('./parcel-watcher-child-termination', () => ({ + registerWatcherChildPhysicalExit: vi.fn(() => signalPhysicalExitMock) +})) + +import { launchWatcherChild } from './parcel-watcher-child-launch' +import { reserveWatcherChild, WatcherChildCapacityError } from './parcel-watcher-child-registry' + +class LaunchChild extends EventEmitter { + pid = 1234 + stderr = new EventEmitter() +} + +describe('launchWatcherChild', () => { + beforeEach(() => { + vi.stubEnv('ORCA_WATCHER_CHILD_PID_FILE', '/tmp/orca-watcher.pid') + forkMock.mockReturnValue(new LaunchChild()) + releaseReservationMock.mockReset() + signalPhysicalExitMock.mockReset() + writeFileSyncMock.mockReset() + vi.mocked(reserveWatcherChild).mockReturnValue(releaseReservationMock) + }) + + it('preserves typed transient capacity exhaustion without forking', () => { + vi.mocked(reserveWatcherChild).mockReturnValueOnce(null) + + expect(() => launchWatcherChild('/watcher.js', null, vi.fn(), vi.fn())).toThrow( + WatcherChildCapacityError + ) + expect(forkMock).not.toHaveBeenCalled() + }) + + it('creates the fault-harness pid file without clobbering an existing path', () => { + expect(launchWatcherChild('/watcher.js', null, vi.fn(), vi.fn())).not.toBeNull() + expect(writeFileSyncMock).toHaveBeenCalledWith('/tmp/orca-watcher.pid', '1234', { + flag: 'wx' + }) + }) + + it('releases physical capacity when an async spawn failure closes without exit', () => { + const child = new LaunchChild() + forkMock.mockReturnValue(child) + const onGone = vi.fn() + + expect(launchWatcherChild('/watcher.js', null, vi.fn(), onGone)).not.toBeNull() + child.emit('error', Object.assign(new Error('spawn failed'), { code: 'ENOENT' })) + child.emit('close', -2, null) + child.emit('exit', -2, null) + + expect(onGone).toHaveBeenCalledTimes(2) + expect(signalPhysicalExitMock).toHaveBeenCalledTimes(1) + expect(releaseReservationMock).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/ipc/parcel-watcher-child-launch.ts b/src/main/ipc/parcel-watcher-child-launch.ts index 6a72bb98145..325bbed2d10 100644 --- a/src/main/ipc/parcel-watcher-child-launch.ts +++ b/src/main/ipc/parcel-watcher-child-launch.ts @@ -4,6 +4,8 @@ import { createWatcherCanaryDirectory, removeWatcherCanaryDirectory } from './parcel-watcher-canary-directory' +import { reserveWatcherChild, WatcherChildCapacityError } from './parcel-watcher-child-registry' +import { registerWatcherChildPhysicalExit } from './parcel-watcher-child-termination' import type { WatcherToHostMessage } from './parcel-watcher-process-protocol' export type LaunchedWatcherChild = { @@ -17,6 +19,13 @@ export function launchWatcherChild( onMessage: (child: ChildProcess, message: WatcherToHostMessage) => void, onGone: (child: ChildProcess, code?: number | null, signal?: NodeJS.Signals | null) => void ): LaunchedWatcherChild | null { + const releaseReservation = reserveWatcherChild() + if (!releaseReservation) { + console.error('[parcel-watcher-process] physical watcher child limit reached') + // Why: capacity is transient and has an event-driven retry owner. Preserve + // its type so the desktop root is not cached permanently unavailable. + throw new WatcherChildCapacityError() + } const canaryDir = currentCanaryDir ?? createWatcherCanaryDirectory() let child: ChildProcess try { @@ -30,6 +39,7 @@ export function launchWatcherChild( ...(process.platform === 'win32' ? { windowsHide: true } : {}) }) } catch (error) { + releaseReservation() removeWatcherCanaryDirectory(canaryDir) console.error('[parcel-watcher-process] failed to fork watcher process:', error) return null @@ -48,10 +58,26 @@ export function launchWatcherChild( console.error('[parcel-watcher-process]', String(chunk).trimEnd()) }) child.on('message', (message) => onMessage(child, message as WatcherToHostMessage)) + const signalPhysicalExit = registerWatcherChildPhysicalExit(child) + let finalized = false + const finalizePhysicalExit = (): void => { + if (finalized) { + return + } + finalized = true + signalPhysicalExit() + releaseReservation() + } // Why: Node can close IPC before emitting exit. Recover at disconnect so a // concurrent subscribe cannot replace the child without restoring old roots. child.on('disconnect', () => onGone(child)) child.on('error', () => onGone(child)) - child.on('exit', (code, signal) => onGone(child, code, signal)) + // Why: asynchronous spawn failures emit error then close without exit; close + // must release the physical-process budget even though no child was created. + child.once('close', finalizePhysicalExit) + child.on('exit', (code, signal) => { + finalizePhysicalExit() + onGone(child, code, signal) + }) return { child, canaryDir } } diff --git a/src/main/ipc/parcel-watcher-child-messaging.ts b/src/main/ipc/parcel-watcher-child-messaging.ts new file mode 100644 index 00000000000..73e429abed5 --- /dev/null +++ b/src/main/ipc/parcel-watcher-child-messaging.ts @@ -0,0 +1,10 @@ +import type { ChildProcess } from 'node:child_process' +import type { HostToWatcherMessage } from './parcel-watcher-process-protocol' + +export function sendToWatcherChild(proc: ChildProcess, message: HostToWatcherMessage): void { + try { + proc.send(message) + } catch { + // The child's exit handler owns recovery after its IPC channel closes. + } +} diff --git a/src/main/ipc/parcel-watcher-child-recovery.ts b/src/main/ipc/parcel-watcher-child-recovery.ts new file mode 100644 index 00000000000..b0329474735 --- /dev/null +++ b/src/main/ipc/parcel-watcher-child-recovery.ts @@ -0,0 +1,94 @@ +import type { ChildProcess } from 'node:child_process' +import { + failAllWatcherSubscriptions, + type PendingWatcherUnsubscribe, + resolvePendingWatcherUnsubscribes +} from './parcel-watcher-host-subscriptions' +import type { WatcherCancellationTracker } from './parcel-watcher-cancellation-tracker' +import { + createWatcherChildTerminationFailure, + terminateWatcherChild +} from './parcel-watcher-child-termination' +import { resetPendingSubscribeAttempt } from './parcel-watcher-pending-subscribe' +import type { WatcherProcessCrashFuse } from './parcel-watcher-crash-fuse' +import { WatcherProcessFailure } from './parcel-watcher-process-failure' +import type { WatcherProcessSubscriptionRecord } from './parcel-watcher-process-subscription' + +export function recoverWatcherRecordsAfterChildGone( + records: Map, + crashFuse: WatcherProcessCrashFuse, + shutdownRequested: boolean, + ensureWatcherProcess: () => ChildProcess | null, + sendSubscribe: (child: ChildProcess, record: WatcherProcessSubscriptionRecord) => void, + removeCanary: () => void, + code?: number | null, + signal?: NodeJS.Signals | null +): void { + if (code !== undefined && (code !== 0 || signal)) { + console.error( + `[parcel-watcher-process] watcher process exited (code=${code}, signal=${signal})` + ) + } + if (shutdownRequested || records.size === 0) { + return + } + crashFuse.recordCrash() + for (const record of records.values()) { + record.interrupted = true + resetPendingSubscribeAttempt(record) + } + const replacement = ensureWatcherProcess() + if (!replacement) { + console.error( + '[parcel-watcher-process] watcher process crashed repeatedly; disabling file watching' + ) + failAllWatcherSubscriptions( + records, + new WatcherProcessFailure( + 'file watcher process crashed repeatedly', + 'supervisor', + 'supervisor_crash_fuse' + ) + ) + removeCanary() + return + } + console.error( + `[parcel-watcher-process] watcher process crashed; resubscribing ${records.size} root(s)` + ) + for (const record of records.values()) { + sendSubscribe(replacement, record) + } +} + +export async function terminateDisconnectedWatcherChild( + child: ChildProcess, + records: Map, + pendingUnsubscribes: Map, + cancelledSubscribes: WatcherCancellationTracker, + crashFuse: WatcherProcessCrashFuse, + onTerminationFinished: (exited: boolean) => boolean, + ensureWatcherProcess: () => ChildProcess | null, + sendSubscribe: (child: ChildProcess, record: WatcherProcessSubscriptionRecord) => void, + removeCanary: () => void +): Promise { + const exited = await terminateWatcherChild(child) + const error = exited ? undefined : createWatcherChildTerminationFailure(child) + cancelledSubscribes.completeForChild(child, error) + const shouldRestore = onTerminationFinished(exited) + resolvePendingWatcherUnsubscribes(pendingUnsubscribes, error) + if (error) { + failAllWatcherSubscriptions(records, error) + throw error + } + if (shouldRestore) { + recoverWatcherRecordsAfterChildGone( + records, + crashFuse, + false, + ensureWatcherProcess, + sendSubscribe, + removeCanary + ) + } +} diff --git a/src/main/ipc/parcel-watcher-child-registry.test.ts b/src/main/ipc/parcel-watcher-child-registry.test.ts new file mode 100644 index 00000000000..b85ef4a6820 --- /dev/null +++ b/src/main/ipc/parcel-watcher-child-registry.test.ts @@ -0,0 +1,84 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + MAX_PHYSICAL_WATCHER_CHILDREN, + onWatcherChildCapacityAvailable, + reserveWatcherChild, + resetWatcherChildRegistryForTest +} from './parcel-watcher-child-registry' + +function fillWatcherChildCapacity(): (() => void)[] { + return Array.from({ length: MAX_PHYSICAL_WATCHER_CHILDREN }, () => { + const release = reserveWatcherChild() + if (!release) { + throw new Error('expected watcher child reservation') + } + return release + }) +} + +describe('parcel watcher child registry capacity notifications', () => { + afterEach(() => resetWatcherChildRegistryForTest()) + + it('notifies after capacity was released before listener registration', async () => { + const releases = fillWatcherChildCapacity() + expect(reserveWatcherChild()).toBeNull() + releases.pop()?.() + const listener = vi.fn() + + onWatcherChildCapacityAvailable(listener) + await Promise.resolve() + + expect(listener).toHaveBeenCalledOnce() + releases.forEach((release) => release()) + }) + + it('cancels a pending capacity notification without leaking the listener', () => { + const releases = fillWatcherChildCapacity() + const listener = vi.fn() + const cancel = onWatcherChildCapacityAvailable(listener) + + cancel() + releases.pop()?.() + + expect(listener).not.toHaveBeenCalled() + releases.forEach((release) => release()) + }) + + it('delivers each capacity notification at most once', () => { + const releases = fillWatcherChildCapacity() + const listener = vi.fn() + onWatcherChildCapacityAvailable(listener) + + releases.pop()?.() + releases.pop()?.() + + expect(listener).toHaveBeenCalledOnce() + releases.forEach((release) => release()) + }) + + it('hands one released slot to one reserving waiter at a time', async () => { + const releases = fillWatcherChildCapacity() + let firstReservation: (() => void) | null = null + let secondReservation: (() => void) | null = null + const first = vi.fn(() => { + firstReservation = reserveWatcherChild() + }) + const second = vi.fn(() => { + secondReservation = reserveWatcherChild() + }) + onWatcherChildCapacityAvailable(first) + onWatcherChildCapacityAvailable(second) + + releases.pop()?.() + await vi.waitFor(() => expect(first).toHaveBeenCalledOnce()) + expect(firstReservation).toBeTypeOf('function') + expect(second).not.toHaveBeenCalled() + + ;(firstReservation as (() => void) | null)?.() + await vi.waitFor(() => expect(second).toHaveBeenCalledOnce()) + expect(secondReservation).toBeTypeOf('function') + + ;(secondReservation as (() => void) | null)?.() + releases.forEach((release) => release()) + }) +}) diff --git a/src/main/ipc/parcel-watcher-child-registry.ts b/src/main/ipc/parcel-watcher-child-registry.ts new file mode 100644 index 00000000000..82d8011ba5a --- /dev/null +++ b/src/main/ipc/parcel-watcher-child-registry.ts @@ -0,0 +1,85 @@ +// Why: stuck retiring children still consume RSS even after their logical pool +// slot is retired, so they must count against one process-wide hard cap. +export const MAX_PHYSICAL_WATCHER_CHILDREN = 8 + +export class WatcherChildCapacityError extends Error { + readonly code = 'watcher_child_capacity' + + constructor() { + super('Physical file watcher process limit reached') + this.name = 'WatcherChildCapacityError' + } +} + +let reservedChildren = 0 +const capacityListeners = new Set<() => Promise>() +let capacityNotificationsInProgress = 0 +let capacityNotificationGeneration = 0 + +export function reserveWatcherChild(): (() => void) | null { + if (reservedChildren >= MAX_PHYSICAL_WATCHER_CHILDREN) { + return null + } + reservedChildren++ + let released = false + return () => { + if (!released) { + released = true + reservedChildren-- + notifyCapacityListeners() + } + } +} + +export function onWatcherChildCapacityAvailable(listener: () => void | Promise): () => void { + let active = true + const notify = async (): Promise => { + if (!active) { + return + } + active = false + capacityListeners.delete(notify) + await listener() + } + capacityListeners.add(notify) + // Why: a child may exit between a failed reservation and listener setup. + // Recheck asynchronously so that release cannot become a lost wake-up. + if (reservedChildren < MAX_PHYSICAL_WATCHER_CHILDREN) { + queueMicrotask(notifyCapacityListeners) + } + return () => { + active = false + capacityListeners.delete(notify) + } +} + +function notifyCapacityListeners(): void { + // Why: an installing callback claims one free slot until it either reserves + // the child or finishes, so one release cannot wake every waiting root. + while (capacityNotificationsInProgress < MAX_PHYSICAL_WATCHER_CHILDREN - reservedChildren) { + const listener = capacityListeners.values().next().value + if (!listener) { + return + } + const generation = capacityNotificationGeneration + capacityNotificationsInProgress++ + void listener() + .catch((error: unknown) => { + console.error('[parcel-watcher-child-registry] capacity listener failed:', error) + }) + .finally(() => { + if (generation !== capacityNotificationGeneration) { + return + } + capacityNotificationsInProgress-- + notifyCapacityListeners() + }) + } +} + +export function resetWatcherChildRegistryForTest(): void { + reservedChildren = 0 + capacityListeners.clear() + capacityNotificationsInProgress = 0 + capacityNotificationGeneration++ +} diff --git a/src/main/ipc/parcel-watcher-child-termination.ts b/src/main/ipc/parcel-watcher-child-termination.ts new file mode 100644 index 00000000000..d4376468574 --- /dev/null +++ b/src/main/ipc/parcel-watcher-child-termination.ts @@ -0,0 +1,140 @@ +import type { ChildProcess } from 'node:child_process' +import { + type PendingWatcherUnsubscribe, + resolvePendingWatcherUnsubscribes +} from './parcel-watcher-host-subscriptions' +import { WatcherProcessFailure } from './parcel-watcher-process-failure' +import { RUNTIME_FILE_WATCH_EXIT_DEADLINE_MS } from '../../shared/runtime-file-watch-limits' + +export const WATCHER_PROCESS_HARD_KILL_DELAY_MS = 5_000 +export const WATCHER_PROCESS_EXIT_DEADLINE_MS = RUNTIME_FILE_WATCH_EXIT_DEADLINE_MS + +const physicalExitPromises = new WeakMap>() + +export function registerWatcherChildPhysicalExit(child: ChildProcess): () => void { + let resolveExit: () => void = () => undefined + const promise = new Promise((resolve) => { + resolveExit = resolve + }) + physicalExitPromises.set(child, promise) + return resolveExit +} + +export function ignoreWatcherTermination(promise: Promise): void { + void promise.catch(() => undefined) +} + +export class WatcherTerminationQueue { + private current: Promise | null = null + + waitFor(operation: () => Promise): Promise | null { + return this.current?.then(operation) ?? null + } + + getCurrent(): Promise | null { + return this.current + } + + track(promise: Promise): Promise { + const tracked = promise.finally(() => { + if (this.current === tracked) { + this.current = null + } + }) + this.current = tracked + ignoreWatcherTermination(tracked) + return tracked + } + + resetForTest(): void { + this.current = null + } +} + +export function terminateWatcherChild(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) { + return Promise.resolve(true) + } + return new Promise((resolve) => { + let settled = false + const finish = (exited: boolean): void => { + if (settled) { + return + } + settled = true + clearTimeout(hardKillTimer) + clearTimeout(exitDeadlineTimer) + child.removeListener('exit', onExit) + child.removeListener('close', onClose) + resolve(exited) + } + const onExit = (): void => finish(true) + // Why: an asynchronous spawn failure can close without emitting exit; the + // close event is still definitive proof that the child owns no OS handles. + const onClose = (): void => finish(true) + child.once('exit', onExit) + child.once('close', onClose) + const hardKillTimer = setTimeout(() => { + try { + child.kill('SIGKILL') + } catch { + // The finite deadline below reports termination failure. + } + }, WATCHER_PROCESS_HARD_KILL_DELAY_MS) + hardKillTimer.unref?.() + const exitDeadlineTimer = setTimeout(() => finish(false), WATCHER_PROCESS_EXIT_DEADLINE_MS) + exitDeadlineTimer.unref?.() + try { + child.kill() + } catch { + finish(false) + } + }) +} + +export function createWatcherChildTerminationFailure(child: ChildProcess): WatcherProcessFailure { + const physicalExit = + child.exitCode !== null || child.signalCode !== null + ? Promise.resolve() + : (physicalExitPromises.get(child) ?? + new Promise((resolve) => { + const finish = (): void => { + child.removeListener('exit', finish) + child.removeListener('close', finish) + resolve() + } + child.once('exit', finish) + child.once('close', finish) + })) + return new WatcherProcessFailure( + 'file watcher process did not exit after termination deadline', + 'supervisor', + 'process_unavailable', + physicalExit + ) +} + +export async function requireWatcherChildTermination(child: ChildProcess): Promise { + if (!(await terminateWatcherChild(child))) { + throw createWatcherChildTerminationFailure(child) + } +} + +export async function terminateIdleWatcherChild( + child: ChildProcess, + pendingUnsubscribes: Map, + onFinished: (exited: boolean) => void +): Promise { + try { + await requireWatcherChildTermination(child) + onFinished(true) + } catch (error) { + onFinished(false) + resolvePendingWatcherUnsubscribes( + pendingUnsubscribes, + error instanceof Error ? error : new Error(String(error)) + ) + throw error + } + resolvePendingWatcherUnsubscribes(pendingUnsubscribes) +} diff --git a/src/main/ipc/parcel-watcher-disconnect-termination.test.ts b/src/main/ipc/parcel-watcher-disconnect-termination.test.ts new file mode 100644 index 00000000000..b478775d1b7 --- /dev/null +++ b/src/main/ipc/parcel-watcher-disconnect-termination.test.ts @@ -0,0 +1,413 @@ +import { EventEmitter } from 'node:events' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { WATCHER_PROCESS_CANCEL_TIMEOUT_MS } from './parcel-watcher-cancellation-tracker' +import { WATCHER_PROCESS_EXIT_DEADLINE_MS } from './parcel-watcher-child-termination' + +const { existsSyncMock, forkMock, mkdtempSyncMock, rmSyncMock } = vi.hoisted(() => ({ + existsSyncMock: vi.fn(), + forkMock: vi.fn(), + mkdtempSyncMock: vi.fn(() => '/tmp/orca-watcher-disconnect-test'), + rmSyncMock: vi.fn() +})) + +vi.mock('node:child_process', () => ({ fork: forkMock })) +vi.mock('node:fs', () => ({ + existsSync: existsSyncMock, + mkdtempSync: mkdtempSyncMock, + rmSync: rmSyncMock +})) + +import { + createWatcherProcessSupervisor, + disposeWatcherProcess, + resetWatcherProcessForTest, + subscribeViaWatcherProcess +} from './parcel-watcher-process' +import type { WatcherProcessSubscription } from './parcel-watcher-process-subscription' + +type SentMessage = { op: string; id: number } + +class FakeChild extends EventEmitter { + connected = true + exitCode: number | null = null + signalCode: NodeJS.Signals | null = null + sent: SentMessage[] = [] + stderr = new EventEmitter() + kill = vi.fn(() => { + this.connected = false + }) + send = vi.fn((message: SentMessage) => { + this.sent.push(message) + return true + }) +} + +function currentChild(): FakeChild { + return forkMock.mock.results.at(-1)?.value as FakeChild +} + +function ackSubscribe(child: FakeChild): void { + const message = child.sent.findLast((candidate) => candidate.op === 'subscribe') + child.emit('message', { op: 'subscribed', id: message?.id }) +} + +async function subscribePair(): Promise<{ child: FakeChild; unsubscribe: () => Promise }> { + const firstPromise = subscribeViaWatcherProcess('/first', vi.fn(), {}) + const child = currentChild() + ackSubscribe(child) + const first = await firstPromise + const siblingPromise = subscribeViaWatcherProcess('/sibling', vi.fn(), {}) + ackSubscribe(child) + await siblingPromise + return { child, unsubscribe: first.unsubscribe } +} + +describe('watcher disconnect termination', () => { + beforeEach(() => { + resetWatcherProcessForTest() + vi.stubEnv('VITEST', '') + existsSyncMock.mockReturnValue(true) + forkMock.mockImplementation(() => new FakeChild()) + }) + + afterEach(() => { + disposeWatcherProcess() + vi.unstubAllEnvs() + vi.clearAllMocks() + }) + + it('keeps unsubscribe pending until physical exit', async () => { + const { child, unsubscribe } = await subscribePair() + let settled = false + const pending = unsubscribe().then(() => { + settled = true + }) + child.connected = false + child.emit('disconnect') + await Promise.resolve() + + expect(settled).toBe(false) + expect(forkMock).toHaveBeenCalledTimes(1) + child.emit('exit', 0, null) + await pending + expect(settled).toBe(true) + }) + + it('forces disconnected ready-subscription teardown even with a live sibling', async () => { + const { child, unsubscribe } = await subscribePair() + child.connected = false + let settled = false + const pending = unsubscribe().then(() => { + settled = true + }) + + await Promise.resolve() + expect(child.kill).toHaveBeenCalledTimes(1) + expect(settled).toBe(false) + + child.emit('exit', 0, null) + await pending + expect(forkMock).toHaveBeenCalledTimes(2) + }) + + it('forces disconnected pending-crawl teardown even with a live sibling', async () => { + const siblingPromise = subscribeViaWatcherProcess('/sibling', vi.fn(), {}) + const child = currentChild() + ackSubscribe(child) + await siblingPromise + const controller = new AbortController() + const subscribing = subscribeViaWatcherProcess( + '/pending', + vi.fn(), + {}, + { + signal: controller.signal + } + ) + child.connected = false + controller.abort() + let settled = false + void subscribing.then( + () => { + settled = true + }, + () => { + settled = true + } + ) + + await Promise.resolve() + expect(child.kill).toHaveBeenCalledTimes(1) + expect(settled).toBe(false) + + child.emit('exit', 0, null) + await expect(subscribing).rejects.toMatchObject({ code: 'subscribe_aborted' }) + expect(forkMock).toHaveBeenCalledTimes(2) + }) + + it('joins a cancellation deadline to active disconnect termination', async () => { + vi.useFakeTimers() + try { + const siblingPromise = subscribeViaWatcherProcess('/sibling', vi.fn(), {}) + const child = currentChild() + ackSubscribe(child) + await siblingPromise + const controller = new AbortController() + const pending = subscribeViaWatcherProcess( + '/pending', + vi.fn(), + {}, + { + signal: controller.signal + } + ) + const pendingId = child.sent.at(-1)?.id + child.emit('message', { op: 'subscribe-started', id: pendingId }) + controller.abort() + const rejection = expect(pending).rejects.toMatchObject({ code: 'subscribe_aborted' }) + + await vi.advanceTimersByTimeAsync(WATCHER_PROCESS_CANCEL_TIMEOUT_MS - 1) + child.connected = false + child.emit('disconnect') + expect(child.kill).toHaveBeenCalledTimes(1) + + await vi.advanceTimersByTimeAsync(1) + expect(child.kill).toHaveBeenCalledTimes(1) + child.emit('exit', 0, null) + await rejection + + const replacement = currentChild() + expect(forkMock).toHaveBeenCalledTimes(2) + expect(replacement.sent.filter((message) => message.op === 'subscribe')).toEqual([ + expect.objectContaining({ dir: '/sibling' }) + ]) + } finally { + vi.useRealTimers() + } + }) + + it('reports physical-exit failure when a timed-out recovery child cannot terminate', async () => { + vi.useFakeTimers() + try { + const supervisor = createWatcherProcessSupervisor() + let subscription: WatcherProcessSubscription + let terminalUnsubscribeResult: Promise | undefined + const onTerminalError = vi.fn(() => { + terminalUnsubscribeResult = subscription.unsubscribe().catch((error: unknown) => error) + }) + const initial = supervisor.subscribe( + '/stuck-recovery', + vi.fn(), + {}, + { + onTerminalError, + subscribeTimeoutMs: 100 + } + ) + const first = currentChild() + ackSubscribe(first) + subscription = await initial + + first.connected = false + first.emit('exit', null, 'SIGSEGV') + const replacement = currentChild() + replacement.emit('message', { + op: 'subscribe-started', + id: replacement.sent[0].id + }) + + await vi.advanceTimersByTimeAsync(100 + WATCHER_PROCESS_EXIT_DEADLINE_MS) + + await vi.waitFor(() => { + expect(onTerminalError).toHaveBeenCalledTimes(1) + expect(onTerminalError).toHaveBeenCalledWith( + expect.objectContaining({ + code: 'process_unavailable', + message: 'file watcher process did not exit after termination deadline', + physicalExit: expect.any(Promise) + }) + ) + }) + await expect(terminalUnsubscribeResult).resolves.toMatchObject({ + code: 'process_unavailable', + physicalExit: expect.any(Promise) + }) + expect(forkMock).toHaveBeenCalledTimes(2) + replacement.emit('exit', 0, null) + await Promise.resolve() + await expect(subscription.unsubscribe()).resolves.toBeUndefined() + supervisor.dispose() + } finally { + vi.useRealTimers() + } + }) + + it('reports a recovery timeout only after its replacement child physically exits', async () => { + vi.useFakeTimers() + try { + const supervisor = createWatcherProcessSupervisor() + const onTerminalError = vi.fn() + const initial = supervisor.subscribe( + '/slow-recovery', + vi.fn(), + {}, + { + onTerminalError, + subscribeTimeoutMs: 100 + } + ) + const first = currentChild() + ackSubscribe(first) + const subscription = await initial + + first.connected = false + first.emit('exit', null, 'SIGSEGV') + const replacement = currentChild() + replacement.emit('message', { + op: 'subscribe-started', + id: replacement.sent[0].id + }) + + await vi.advanceTimersByTimeAsync(100) + + expect(onTerminalError).not.toHaveBeenCalled() + expect(replacement.kill).toHaveBeenCalledTimes(1) + expect(forkMock).toHaveBeenCalledTimes(2) + let unsubscribeSettled = false + const unsubscribe = subscription.unsubscribe().then(() => { + unsubscribeSettled = true + }) + await Promise.resolve() + expect(unsubscribeSettled).toBe(false) + + replacement.emit('exit', 0, null) + await unsubscribe + await vi.waitFor(() => + expect(onTerminalError).toHaveBeenCalledWith( + expect.objectContaining({ + code: 'subscribe_timeout', + message: 'file watcher resubscription timed out after 100ms' + }) + ) + ) + supervisor.dispose() + } finally { + vi.useRealTimers() + } + }) + + it('keeps pending-crawl cancellation joined to disconnected child exit', async () => { + const controller = new AbortController() + const subscribing = subscribeViaWatcherProcess( + '/pending', + vi.fn(), + {}, + { + signal: controller.signal + } + ) + const child = currentChild() + child.connected = false + child.emit('disconnect') + controller.abort() + let settled = false + void subscribing.then( + () => { + settled = true + }, + () => { + settled = true + } + ) + + await Promise.resolve() + expect(settled).toBe(false) + + child.emit('exit', 0, null) + await expect(subscribing).rejects.toMatchObject({ code: 'subscribe_aborted' }) + }) + + it('reports physical-exit failure instead of early pending-crawl cancellation', async () => { + vi.useFakeTimers() + try { + const controller = new AbortController() + const subscribing = subscribeViaWatcherProcess( + '/pending', + vi.fn(), + {}, + { + signal: controller.signal + } + ) + const child = currentChild() + child.connected = false + child.emit('disconnect') + controller.abort() + const failure = expect(subscribing).rejects.toMatchObject({ + code: 'process_unavailable', + physicalExit: expect.any(Promise) + }) + + await vi.advanceTimersByTimeAsync(WATCHER_PROCESS_EXIT_DEADLINE_MS) + + await failure + } finally { + vi.useRealTimers() + } + }) + + it('rejects unsubscribe at the physical-exit deadline', async () => { + vi.useFakeTimers() + try { + const { child, unsubscribe } = await subscribePair() + const failure = expect(unsubscribe()).rejects.toThrow( + 'file watcher process did not exit after termination deadline' + ) + child.connected = false + child.emit('disconnect') + await vi.advanceTimersByTimeAsync(WATCHER_PROCESS_EXIT_DEADLINE_MS) + + await failure + expect(forkMock).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it('retires the child before settling a rejected native unsubscribe', async () => { + const firstPromise = subscribeViaWatcherProcess('/first', vi.fn(), {}) + const child = currentChild() + ackSubscribe(child) + const first = await firstPromise + const siblingInterruption = vi.fn() + const siblingPromise = subscribeViaWatcherProcess( + '/sibling', + vi.fn(), + {}, + { + onInterruption: siblingInterruption + } + ) + ackSubscribe(child) + await siblingPromise + + let settled = false + const unsubscribe = first.unsubscribe().then(() => { + settled = true + }) + child.emit('message', { op: 'unsubscribe-failed', id: 1, message: 'native handle active' }) + await Promise.resolve() + + expect(settled).toBe(false) + expect(child.kill).toHaveBeenCalledTimes(1) + expect(forkMock).toHaveBeenCalledTimes(1) + child.emit('exit', 0, null) + await unsubscribe + await vi.waitFor(() => expect(forkMock).toHaveBeenCalledTimes(2)) + const replacement = currentChild() + expect(replacement.sent).toContainEqual(expect.objectContaining({ dir: '/sibling' })) + ackSubscribe(replacement) + expect(siblingInterruption).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/ipc/parcel-watcher-entry-path.test.ts b/src/main/ipc/parcel-watcher-entry-path.test.ts index f8b59becc78..2b29c6c9016 100644 --- a/src/main/ipc/parcel-watcher-entry-path.test.ts +++ b/src/main/ipc/parcel-watcher-entry-path.test.ts @@ -1,6 +1,9 @@ import path from 'node:path' import { describe, expect, it } from 'vitest' -import { resolveWatcherProcessEntryPath } from './parcel-watcher-entry-path' +import { + resolveWatcherProcessEntryPath, + resolveWatcherProcessEntryPathWithoutApp +} from './parcel-watcher-entry-path' describe('resolveWatcherProcessEntryPath', () => { it('uses an adjacent entry when electron-vite appPath is already out/main', () => { @@ -37,4 +40,35 @@ describe('resolveWatcherProcessEntryPath', () => { ) ) }) + + it('uses resourcesPath for packaged Electron-as-Node serve processes', () => { + const resourcesPath = path.join('Applications', 'Orca.app', 'Contents', 'Resources') + const packagedEntry = path.join( + resourcesPath, + 'app.asar.unpacked', + 'out', + 'main', + 'parcel-watcher-process-entry.js' + ) + + expect( + resolveWatcherProcessEntryPathWithoutApp( + path.join(resourcesPath, 'unrelated-cwd'), + resourcesPath, + (candidate) => candidate === packagedEntry + ) + ).toBe(packagedEntry) + }) + + it('keeps the cwd build fallback when resourcesPath has no packaged entry', () => { + const appRoot = path.join('workspace', 'orca') + + expect( + resolveWatcherProcessEntryPathWithoutApp( + appRoot, + path.join('node_modules', 'electron', 'Resources'), + () => false + ) + ).toBe(path.join(appRoot, 'out', 'main', 'parcel-watcher-process-entry.js')) + }) }) diff --git a/src/main/ipc/parcel-watcher-entry-path.ts b/src/main/ipc/parcel-watcher-entry-path.ts index 90133146715..7eba4c205da 100644 --- a/src/main/ipc/parcel-watcher-entry-path.ts +++ b/src/main/ipc/parcel-watcher-entry-path.ts @@ -28,10 +28,32 @@ export function resolveWatcherProcessEntryPath( return join(basePath, 'out', 'main', 'parcel-watcher-process-entry.js') } +export function resolveWatcherProcessEntryPathWithoutApp( + cwd: string, + resourcesPath: string | undefined, + pathExists: (candidate: string) => boolean = existsSync +): string { + if (resourcesPath) { + const packagedEntry = join( + resourcesPath, + 'app.asar.unpacked', + 'out', + 'main', + 'parcel-watcher-process-entry.js' + ) + // Why: ELECTRON_RUN_AS_NODE exposes resourcesPath but not electron.app. + // Prefer the unpacked packaged entry without breaking dev Node fallbacks. + if (pathExists(packagedEntry)) { + return packagedEntry + } + } + return resolveWatcherProcessEntryPath(cwd, false, pathExists) +} + export function getWatcherProcessEntryPath(): string { const app = loadElectronApp() - return resolveWatcherProcessEntryPath( - app?.getAppPath() ?? process.cwd(), - app?.isPackaged === true - ) + if (app) { + return resolveWatcherProcessEntryPath(app.getAppPath(), app.isPackaged) + } + return resolveWatcherProcessEntryPathWithoutApp(process.cwd(), process.resourcesPath) } diff --git a/src/main/ipc/parcel-watcher-host-subscriptions.ts b/src/main/ipc/parcel-watcher-host-subscriptions.ts index 71a3d82868f..24f1803ee68 100644 --- a/src/main/ipc/parcel-watcher-host-subscriptions.ts +++ b/src/main/ipc/parcel-watcher-host-subscriptions.ts @@ -3,12 +3,19 @@ import { resetPendingSubscribeAttempt, takePendingSubscribe } from './parcel-watcher-pending-subscribe' -import { WatcherProcessFailure } from './parcel-watcher-process-failure' +import { isWatcherProcessFailure, WatcherProcessFailure } from './parcel-watcher-process-failure' import type { HostToWatcherMessage, WatcherToHostMessage } from './parcel-watcher-process-protocol' import type { WatcherProcessSubscription, WatcherProcessSubscriptionRecord } from './parcel-watcher-process-subscription' +import { RUNTIME_FILE_WATCH_CANCEL_TIMEOUT_MS } from '../../shared/runtime-file-watch-limits' + +export type PendingWatcherUnsubscribe = (error?: Error) => void + +export const WATCHER_PROCESS_UNSUBSCRIBE_TIMEOUT_MS = RUNTIME_FILE_WATCH_CANCEL_TIMEOUT_MS + +const terminalTeardownFailures = new WeakMap() export function handleWatcherHostMessage( message: Exclude< @@ -16,14 +23,14 @@ export function handleWatcherHostMessage( { op: 'subscribe-started' } | { op: 'cancel-requires-restart' } >, records: Map, - pendingUnsubscribes: Map void>, + pendingUnsubscribes: Map, reportTerminalError: (record: WatcherProcessSubscriptionRecord, error: Error) => void, killWatcherChildIfIdle: () => void ): void { if (message.op === 'unsubscribed') { - const resolve = pendingUnsubscribes.get(message.id) + const settle = pendingUnsubscribes.get(message.id) pendingUnsubscribes.delete(message.id) - resolve?.() + settle?.() return } const record = records.get(message.id) @@ -67,6 +74,15 @@ export function reportWatcherTerminalError( record: WatcherProcessSubscriptionRecord, error: Error ): void { + if (isWatcherProcessFailure(error) && error.physicalExit) { + terminalTeardownFailures.set(record, error) + const clearFailure = (): void => { + if (terminalTeardownFailures.get(record) === error) { + terminalTeardownFailures.delete(record) + } + } + void error.physicalExit.then(clearFailure, clearFailure) + } if (record.hooks.onTerminalError) { record.hooks.onTerminalError(error) return @@ -92,10 +108,11 @@ export function failAllWatcherSubscriptions( } export function resolvePendingWatcherUnsubscribes( - pendingUnsubscribes: Map void> + pendingUnsubscribes: Map, + error?: Error ): void { - for (const resolve of pendingUnsubscribes.values()) { - resolve() + for (const settle of pendingUnsubscribes.values()) { + settle(error) } pendingUnsubscribes.clear() } @@ -118,9 +135,11 @@ export function disposeWatcherSupervisorSubscriptions( export type CreateHostWatcherSubscriptionOptions = { record: WatcherProcessSubscriptionRecord records: Map - pendingUnsubscribes: Map void> + pendingUnsubscribes: Map getChild: () => ChildProcess | null - killWatcherChildIfIdle: () => void + getTerminationPromise: () => Promise | null + killWatcherChildIfIdle: () => Promise + terminateUnavailableChild: (child: ChildProcess | null) => Promise sendToChild: (child: ChildProcess, message: HostToWatcherMessage) => void } @@ -129,25 +148,52 @@ export function createHostWatcherSubscription({ records, pendingUnsubscribes, getChild, + getTerminationPromise, killWatcherChildIfIdle, + terminateUnavailableChild, sendToChild }: CreateHostWatcherSubscriptionOptions): WatcherProcessSubscription { return { unsubscribe: (): Promise => { + const terminalFailure = terminalTeardownFailures.get(record) + if (terminalFailure) { + return Promise.reject(terminalFailure) + } + const termination = getTerminationPromise() if (!records.delete(record.id)) { - return Promise.resolve() + return termination ?? Promise.resolve() } resetPendingSubscribeAttempt(record) - const child = getChild() - if (!child?.connected) { - return Promise.resolve() + if (termination) { + return termination } if (records.size === 0) { - killWatcherChildIfIdle() - return Promise.resolve() + return killWatcherChildIfIdle() } - return new Promise((resolve) => { - pendingUnsubscribes.set(record.id, resolve) + const child = getChild() + if (!child?.connected) { + return terminateUnavailableChild(child) + } + return new Promise((resolve, reject) => { + const onTimeout = (): void => { + if (pendingUnsubscribes.get(record.id) !== settle) { + return + } + // Why: canary setup itself can fail, so native teardown needs an + // independent host deadline that exits the child and releases handles. + void terminateUnavailableChild(child).catch(() => undefined) + } + const timer = setTimeout(onTimeout, WATCHER_PROCESS_UNSUBSCRIBE_TIMEOUT_MS) + timer.unref?.() + const settle: PendingWatcherUnsubscribe = (error) => { + clearTimeout(timer) + if (error) { + reject(error) + } else { + resolve() + } + } + pendingUnsubscribes.set(record.id, settle) sendToChild(child, { op: 'unsubscribe', id: record.id }) }) } diff --git a/src/main/ipc/parcel-watcher-interrupted-cancellation.ts b/src/main/ipc/parcel-watcher-interrupted-cancellation.ts new file mode 100644 index 00000000000..af23240cae1 --- /dev/null +++ b/src/main/ipc/parcel-watcher-interrupted-cancellation.ts @@ -0,0 +1,27 @@ +import { resetPendingSubscribeAttempt } from './parcel-watcher-pending-subscribe' +import type { WatcherProcessFailure } from './parcel-watcher-process-failure' +import type { WatcherProcessSubscriptionRecord } from './parcel-watcher-process-subscription' + +export function cancelInterruptedWatcherSubscribe(args: { + record: WatcherProcessSubscriptionRecord + error: WatcherProcessFailure + records: Map + reportTerminalError: (record: WatcherProcessSubscriptionRecord, error: Error) => void + restartChild: () => Promise +}): void { + const { record } = args + if (!record.interrupted || record.pendingSubscribe || !args.records.delete(record.id)) { + return + } + resetPendingSubscribeAttempt(record) + // Why: recovery may replace this root synchronously from the terminal hook; + // publish failure only after its predecessor releases native handles. + void args.restartChild().then( + () => args.reportTerminalError(record, args.error), + (terminationError: unknown) => + args.reportTerminalError( + record, + terminationError instanceof Error ? terminationError : new Error(String(terminationError)) + ) + ) +} diff --git a/src/main/ipc/parcel-watcher-pending-cancellation.ts b/src/main/ipc/parcel-watcher-pending-cancellation.ts new file mode 100644 index 00000000000..9fd37d19a8d --- /dev/null +++ b/src/main/ipc/parcel-watcher-pending-cancellation.ts @@ -0,0 +1,51 @@ +import type { ChildProcess } from 'node:child_process' +import type { WatcherCancellationTracker } from './parcel-watcher-cancellation-tracker' +import { takePendingSubscribe } from './parcel-watcher-pending-subscribe' +import type { WatcherProcessFailure } from './parcel-watcher-process-failure' +import type { WatcherProcessSubscriptionRecord } from './parcel-watcher-process-subscription' + +type CancelPendingWatcherSubscribeOptions = { + record: WatcherProcessSubscriptionRecord + error: WatcherProcessFailure + records: Map + child: ChildProcess | null + cancelledSubscribes: WatcherCancellationTracker + onChildUnavailable: (child: ChildProcess | null) => Promise + restartChild: (child: ChildProcess) => Promise + sendCancel: (child: ChildProcess, id: number) => void +} + +export function cancelPendingWatcherSubscribe({ + record, + error, + records, + child, + cancelledSubscribes, + onChildUnavailable, + restartChild, + sendCancel +}: CancelPendingWatcherSubscribeOptions): void { + if (!record.pendingSubscribe || !records.delete(record.id)) { + return + } + const pending = takePendingSubscribe(record) + if (!pending) { + return + } + if (!child?.connected) { + // Why: disconnect starts physical termination before cancellation sees the + // child; destructive callers must join that exact termination generation. + void onChildUnavailable(child).then( + () => pending.reject(error), + (terminationError: unknown) => + pending.reject( + terminationError instanceof Error ? terminationError : new Error(String(terminationError)) + ) + ) + return + } + // Why: only the child knows whether native setup is queued, active, or + // resolved, so its physical teardown must precede caller settlement. + cancelledSubscribes.begin(record.id, child, error, pending.reject, () => restartChild(child)) + sendCancel(child, record.id) +} diff --git a/src/main/ipc/parcel-watcher-pending-subscribe.ts b/src/main/ipc/parcel-watcher-pending-subscribe.ts index 67a020b9d5e..f37c7b3dbf6 100644 --- a/src/main/ipc/parcel-watcher-pending-subscribe.ts +++ b/src/main/ipc/parcel-watcher-pending-subscribe.ts @@ -53,15 +53,13 @@ export function startInterruptedSubscribeTimeout( record: WatcherProcessSubscriptionRecord, cancel: (error: WatcherProcessFailure) => void ): void { - if ( - !record.interrupted || - record.pendingSubscribe || - record.resubscribeTimer || - record.hooks.subscribeTimeoutMs === undefined - ) { + if (!record.interrupted || record.pendingSubscribe) { return } record.crawlStarted = true + if (record.resubscribeTimer || record.hooks.subscribeTimeoutMs === undefined) { + return + } record.resubscribeTimer = setTimeout(() => { cancel( new WatcherProcessFailure( diff --git a/src/main/ipc/parcel-watcher-process-entry.test.ts b/src/main/ipc/parcel-watcher-process-entry.test.ts index 19740f17fff..62b264f47b3 100644 --- a/src/main/ipc/parcel-watcher-process-entry.test.ts +++ b/src/main/ipc/parcel-watcher-process-entry.test.ts @@ -259,6 +259,29 @@ describe('parcel watcher process canary', () => { expect(sendMock).not.toHaveBeenCalledWith({ op: 'cancel-requires-restart', id: 1 }) }) + it('reports native unsubscribe rejection without acknowledging handle release', async () => { + const unsubscribeError = new Error('native handle still active') + subscribeMock + .mockResolvedValueOnce({ unsubscribe: vi.fn() }) + .mockResolvedValueOnce({ unsubscribe: vi.fn().mockRejectedValue(unsubscribeError) }) + const sendMock = vi.fn() + process.send = sendMock + + await import('./parcel-watcher-process-entry') + await vi.advanceTimersByTimeAsync(0) + process.emit('message', { op: 'subscribe', id: 1, dir: '/repo', opts: {} }) + await vi.advanceTimersByTimeAsync(0) + process.emit('message', { op: 'unsubscribe', id: 1 }) + await vi.advanceTimersByTimeAsync(0) + + expect(sendMock).toHaveBeenCalledWith({ + op: 'unsubscribe-failed', + id: 1, + message: 'native handle still active' + }) + expect(sendMock).not.toHaveBeenCalledWith({ op: 'unsubscribed', id: 1 }) + }) + it('asks the host to restart when an active crawl is cancelled', async () => { let finishCrawl: ((subscription: { unsubscribe: () => Promise }) => void) | undefined subscribeMock.mockResolvedValueOnce({ unsubscribe: vi.fn() }).mockReturnValueOnce( diff --git a/src/main/ipc/parcel-watcher-process-entry.ts b/src/main/ipc/parcel-watcher-process-entry.ts index 1e043739a4a..4bfaefdc930 100644 --- a/src/main/ipc/parcel-watcher-process-entry.ts +++ b/src/main/ipc/parcel-watcher-process-entry.ts @@ -249,9 +249,10 @@ function main(): void { await subscription?.unsubscribe() }) } catch (err) { - process.stderr.write( - `[parcel-watcher-process] unsubscribe ${id} failed: ${errorMessage(err)}\n` - ) + const message = errorMessage(err) + process.stderr.write(`[parcel-watcher-process] unsubscribe ${id} failed: ${message}\n`) + send({ op: 'unsubscribe-failed', id, message }) + return } finally { subscriptionActivityRevision++ } diff --git a/src/main/ipc/parcel-watcher-process-failure.ts b/src/main/ipc/parcel-watcher-process-failure.ts index 80eb79ddd33..336cc69caa4 100644 --- a/src/main/ipc/parcel-watcher-process-failure.ts +++ b/src/main/ipc/parcel-watcher-process-failure.ts @@ -13,7 +13,8 @@ export class WatcherProcessFailure extends Error { constructor( message: string, readonly scope: WatcherProcessFailureScope, - readonly code: WatcherProcessFailureCode + readonly code: WatcherProcessFailureCode, + readonly physicalExit?: Promise ) { super(message) this.name = 'WatcherProcessFailure' diff --git a/src/main/ipc/parcel-watcher-process-protocol.ts b/src/main/ipc/parcel-watcher-process-protocol.ts index 17dbcc36a88..05fb6247b0f 100644 --- a/src/main/ipc/parcel-watcher-process-protocol.ts +++ b/src/main/ipc/parcel-watcher-process-protocol.ts @@ -34,4 +34,5 @@ export type WatcherToHostMessage = | { op: 'overflow'; id: number } | { op: 'watch-error'; id: number; message: string } | { op: 'cancel-requires-restart'; id: number } + | { op: 'unsubscribe-failed'; id: number; message: string } | { op: 'unsubscribed'; id: number } diff --git a/src/main/ipc/parcel-watcher-process-supervisor.ts b/src/main/ipc/parcel-watcher-process-supervisor.ts index 9be9b232b08..324056b4418 100644 --- a/src/main/ipc/parcel-watcher-process-supervisor.ts +++ b/src/main/ipc/parcel-watcher-process-supervisor.ts @@ -1,26 +1,28 @@ import type { ChildProcess } from 'node:child_process' import { existsSync } from 'node:fs' +import { restartCancelledWatcherChild } from './parcel-watcher-cancellation-restart' +import { WatcherCancellationTracker } from './parcel-watcher-cancellation-tracker' import { getWatcherProcessEntryPath } from './parcel-watcher-entry-path' import { removeWatcherCanaryDirectory } from './parcel-watcher-canary-directory' +import * as termination from './parcel-watcher-child-termination' import { launchWatcherChild } from './parcel-watcher-child-launch' -import { WatcherProcessCrashFuse } from './parcel-watcher-crash-fuse' +import { sendToWatcherChild } from './parcel-watcher-child-messaging' import { - disposeWatcherSupervisorSubscriptions, - failAllWatcherSubscriptions, - handleWatcherHostMessage, + recoverWatcherRecordsAfterChildGone, + terminateDisconnectedWatcherChild +} from './parcel-watcher-child-recovery' +import { resetWatcherChildRegistryForTest } from './parcel-watcher-child-registry' +import { WatcherSupervisorCapacityWait } from './parcel-watcher-supervisor-capacity-wait' +import { WatcherProcessCrashFuse } from './parcel-watcher-crash-fuse' +import { cancelInterruptedWatcherSubscribe } from './parcel-watcher-interrupted-cancellation' +import { + type PendingWatcherUnsubscribe, reportWatcherTerminalError, resolvePendingWatcherUnsubscribes } from './parcel-watcher-host-subscriptions' -import { - resetPendingSubscribeAttempt, - startInterruptedSubscribeTimeout, - startPendingSubscribeTimeout, - takePendingSubscribe -} from './parcel-watcher-pending-subscribe' -import { watcherHostFailure } from './parcel-watcher-process-failure' +import { cancelPendingWatcherSubscribe } from './parcel-watcher-pending-cancellation' import type { WatcherProcessFailure } from './parcel-watcher-process-failure' import type { - HostToWatcherMessage, WatcherProcessSubscribeOptions, WatcherToHostMessage } from './parcel-watcher-process-protocol' @@ -31,14 +33,12 @@ import type { WatcherProcessSubscriptionRecord } from './parcel-watcher-process-subscription' import type { WatcherProcessSupervisorOptions } from './parcel-watcher-process-supervisor-options' -import { subscribeThroughWatcherSupervisor } from './parcel-watcher-supervisor-subscribe' - -export type { - WatcherProcessCallback, - WatcherProcessHooks, - WatcherProcessSubscription -} from './parcel-watcher-process-subscription' -export type { WatcherProcessSupervisorOptions } from './parcel-watcher-process-supervisor-options' +import { + sendWatcherSubscribe, + subscribeThroughWatcherSupervisor +} from './parcel-watcher-supervisor-subscribe' +import { disposeWatcherSupervisor } from './parcel-watcher-supervisor-disposal' +import { handleWatcherSupervisorMessage } from './parcel-watcher-supervisor-message' export class WatcherProcessSupervisor { private child: ChildProcess | null = null @@ -46,9 +46,12 @@ export class WatcherProcessSupervisor { private readonly crashFuse = new WatcherProcessCrashFuse() private shutdownRequested = false private canaryDir: string | null = null + private terminatingChild: ChildProcess | null = null + private readonly terminationQueue = new termination.WatcherTerminationQueue() private readonly records = new Map() - private readonly pendingUnsubscribes = new Map void>() - private readonly cancelledSubscribesAwaitingChild = new Set() + private readonly pendingUnsubscribes = new Map() + private readonly cancelledSubscribes = new WatcherCancellationTracker() + private readonly capacityWait = new WatcherSupervisorCapacityWait() constructor(private readonly options: WatcherProcessSupervisorOptions = {}) {} @@ -58,51 +61,63 @@ export class WatcherProcessSupervisor { opts: WatcherProcessSubscribeOptions, hooks: WatcherProcessHooks = {} ): Promise { - return subscribeThroughWatcherSupervisor({ - dir, - callback, - opts, - hooks, - shutdownRequested: this.shutdownRequested, - entryPath: this.options.entryPath ?? getWatcherProcessEntryPath(), - useInProcessVitestFallback: this.options.useInProcessVitestFallback ?? true, - allocateId: () => this.nextSubscriptionId++, - records: this.records, - pendingUnsubscribes: this.pendingUnsubscribes, - ensureWatcherProcess: (entryPath) => this.ensureWatcherProcess(entryPath), - getChild: () => this.child, - killWatcherChildIfIdle: () => this.killWatcherChildIfIdle(), - sendSubscribe: (child, record) => this.sendSubscribe(child, record), - sendToChild: (child, message) => this.sendToChild(child, message), - cancelPendingSubscribe: (record, error) => this.cancelPendingSubscribe(record, error) - }) + const queued = this.terminationQueue.waitFor(() => this.subscribe(dir, callback, opts, hooks)) + if (queued) { + return queued + } + return this.capacityWait.run( + subscribeThroughWatcherSupervisor({ + dir, + callback, + opts, + hooks, + shutdownRequested: this.shutdownRequested, + entryPath: this.options.entryPath ?? getWatcherProcessEntryPath(), + useInProcessVitestFallback: this.options.useInProcessVitestFallback ?? true, + allocateId: () => this.nextSubscriptionId++, + records: this.records, + pendingUnsubscribes: this.pendingUnsubscribes, + ensureWatcherProcess: (entryPath) => this.ensureWatcherProcess(entryPath), + getChild: () => this.child, + getTerminationPromise: () => this.terminationQueue.getCurrent(), + killWatcherChildIfIdle: () => this.killWatcherChildIfIdle(), + terminateUnavailableChild: (child) => this.terminateUnavailableChild(child), + sendSubscribe: sendWatcherSubscribe, + sendToChild: sendToWatcherChild, + cancelPendingSubscribe: (record, error) => this.cancelPendingSubscribe(record, error) + }), + () => this.subscribe(dir, callback, opts, hooks), + hooks.signal + ) } dispose(): void { this.shutdownRequested = true + this.capacityWait.dispose() const proc = this.child this.child = null - const error = watcherHostFailure('file watcher supervisor disposed', 'supervisor_disposed') - disposeWatcherSupervisorSubscriptions( + this.canaryDir = disposeWatcherSupervisor( + proc, this.records, this.pendingUnsubscribes, - this.cancelledSubscribesAwaitingChild, - error + this.cancelledSubscribes, + this.canaryDir ) - proc?.kill() - this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) } resetForTest(): void { this.dispose() this.shutdownRequested = false + this.terminatingChild = null + this.terminationQueue.resetForTest() this.crashFuse.reset() + resetWatcherChildRegistryForTest() } private ensureWatcherProcess( entryPath = this.options.entryPath ?? getWatcherProcessEntryPath() ): ChildProcess | null { - if (this.shutdownRequested) { + if (this.shutdownRequested || this.terminatingChild) { return null } if (this.child?.connected) { @@ -135,43 +150,30 @@ export class WatcherProcessSupervisor { } private handleChildMessage(message: WatcherToHostMessage): void { - if (message.op === 'subscribe-started') { - const record = this.records.get(message.id) - if (record) { - startPendingSubscribeTimeout(record, (error) => this.cancelPendingSubscribe(record, error)) - startInterruptedSubscribeTimeout(record, (error) => - this.cancelInterruptedSubscribe(record, error) - ) - } - return - } - if (message.op === 'cancel-requires-restart') { - if (this.cancelledSubscribesAwaitingChild.delete(message.id)) { - this.restartAfterCancelledSubscribe() - } - return - } - if (message.op === 'unsubscribed') { - const completedCancellation = this.cancelledSubscribesAwaitingChild.delete(message.id) - handleWatcherHostMessage( - message, - this.records, - this.pendingUnsubscribes, - reportWatcherTerminalError, - () => this.killWatcherChildIfIdle() - ) - if (completedCancellation) { - this.killWatcherChildIfIdle() - } - return - } - handleWatcherHostMessage( - message, - this.records, - this.pendingUnsubscribes, - reportWatcherTerminalError, - () => this.killWatcherChildIfIdle() - ) + const child = this.child + handleWatcherSupervisorMessage(message, { + records: this.records, + pendingUnsubscribes: this.pendingUnsubscribes, + cancelledSubscribes: this.cancelledSubscribes, + child, + cancelPendingSubscribe: (record, error) => this.cancelPendingSubscribe(record, error), + cancelInterruptedSubscribe: (record, error) => + cancelInterruptedWatcherSubscribe({ + record, + error, + records: this.records, + reportTerminalError: reportWatcherTerminalError, + restartChild: () => this.restartAfterCancelledSubscribe(child) + }), + restartAfterCancelledSubscribe: (child) => this.restartAfterCancelledSubscribe(child), + terminateUnavailableChild: (child) => { + if (child) { + this.terminateUnavailableChild(child) + } + }, + killWatcherChildIfIdle: () => + termination.ignoreWatcherTermination(this.killWatcherChildIfIdle()) + }) } private handleChildGone( @@ -182,135 +184,130 @@ export class WatcherProcessSupervisor { if (this.child !== proc) { return } - if (code !== undefined && (code !== 0 || signal)) { - console.error( - `[parcel-watcher-process] watcher process exited (code=${code}, signal=${signal})` - ) + if (code === undefined) { + this.terminateUnavailableChild(proc) + return } this.child = null - this.cancelledSubscribesAwaitingChild.clear() + this.cancelledSubscribes.completeForChild(proc) resolvePendingWatcherUnsubscribes(this.pendingUnsubscribes) - if (this.shutdownRequested || this.records.size === 0) { - return - } - this.crashFuse.recordCrash() - for (const record of this.records.values()) { - record.interrupted = true - resetPendingSubscribeAttempt(record) - } - const replacement = this.ensureWatcherProcess() - if (!replacement) { - console.error( - '[parcel-watcher-process] watcher process crashed repeatedly; disabling file watching' - ) - failAllWatcherSubscriptions( - this.records, - watcherHostFailure('file watcher process crashed repeatedly', 'supervisor_crash_fuse') - ) - this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) - return - } - console.error( - `[parcel-watcher-process] watcher process crashed; resubscribing ${this.records.size} root(s)` + recoverWatcherRecordsAfterChildGone( + this.records, + this.crashFuse, + this.shutdownRequested, + () => this.ensureWatcherProcess(), + sendWatcherSubscribe, + () => { + this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) + }, + code, + signal ) - for (const record of this.records.values()) { - this.sendSubscribe(replacement, record) - } } - private killWatcherChildIfIdle(): void { - const proc = this.child - if (!proc || this.records.size > 0) { - return + private terminateUnavailableChild(requestedChild: ChildProcess | null): Promise { + const currentTermination = this.terminationQueue.getCurrent() + if (currentTermination) { + return currentTermination + } + const proc = requestedChild ?? this.terminatingChild + if (!proc) { + return Promise.resolve() } this.child = null - resolvePendingWatcherUnsubscribes(this.pendingUnsubscribes) - proc.kill() + this.terminatingChild = proc this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) + return this.terminationQueue.track( + terminateDisconnectedWatcherChild( + proc, + this.records, + this.pendingUnsubscribes, + this.cancelledSubscribes, + this.crashFuse, + (exited) => { + this.terminatingChild = null + if (!exited) { + this.shutdownRequested = true + } + return !this.shutdownRequested + }, + () => this.ensureWatcherProcess(), + sendWatcherSubscribe, + () => { + this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) + } + ) + ) + } + + private killWatcherChildIfIdle(): Promise { + const terminationPromise = this.terminationQueue.getCurrent() + if (terminationPromise) { + return terminationPromise + } + const proc = this.child + if (!proc || this.records.size > 0) { + return Promise.resolve() + } + this.child = null + this.terminatingChild = proc + // Why: destructive Windows cleanup must await exit to release directory handles. + this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) + return this.terminationQueue.track( + termination.terminateIdleWatcherChild(proc, this.pendingUnsubscribes, (exited) => { + this.terminatingChild = null + if (!exited) { + this.shutdownRequested = true + } + }) + ) } private cancelPendingSubscribe( record: WatcherProcessSubscriptionRecord, error: WatcherProcessFailure ): void { - if (!record.pendingSubscribe || !this.records.delete(record.id)) { - return - } - const crawlStarted = record.crawlStarted - const pending = takePendingSubscribe(record) - pending?.reject(error) - if (crawlStarted) { - this.restartAfterCancelledSubscribe() - return - } - const proc = this.child - if (!proc?.connected) { - this.killWatcherChildIfIdle() - return - } - this.cancelledSubscribesAwaitingChild.add(record.id) - this.sendToChild(proc, { op: 'cancel-subscribe', id: record.id }) - } - - private cancelInterruptedSubscribe( - record: WatcherProcessSubscriptionRecord, - error: WatcherProcessFailure - ): void { - if (!record.interrupted || record.pendingSubscribe || !this.records.delete(record.id)) { - return - } - resetPendingSubscribeAttempt(record) - // Why: one slow recovery root must not pin every healthy root in its shard. - // End that subscription so the runtime pool can retry it in quarantine. - reportWatcherTerminalError(record, error) - this.restartAfterCancelledSubscribe() - } - - private restartAfterCancelledSubscribe(): void { - const proc = this.child - this.child = null - this.cancelledSubscribesAwaitingChild.clear() - resolvePendingWatcherUnsubscribes(this.pendingUnsubscribes) - proc?.kill() - if (this.records.size === 0) { - this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) - return - } - const replacement = this.ensureWatcherProcess() - if (!replacement) { - failAllWatcherSubscriptions( - this.records, - watcherHostFailure( - 'file watcher process unavailable after subscription cancellation', - 'process_unavailable' - ) - ) - this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) - return - } - for (const liveRecord of this.records.values()) { - liveRecord.interrupted = true - resetPendingSubscribeAttempt(liveRecord) - this.sendSubscribe(replacement, liveRecord) - } - } - - private sendSubscribe(proc: ChildProcess, record: WatcherProcessSubscriptionRecord): void { - resetPendingSubscribeAttempt(record) - this.sendToChild(proc, { - op: 'subscribe', - id: record.id, - dir: record.dir, - opts: record.opts, - delivery: record.hooks.delivery + cancelPendingWatcherSubscribe({ + record, + error, + records: this.records, + child: this.child, + cancelledSubscribes: this.cancelledSubscribes, + onChildUnavailable: (child) => this.terminateUnavailableChild(child), + restartChild: (child) => this.restartAfterCancelledSubscribe(child), + sendCancel: (child, id) => sendToWatcherChild(child, { op: 'cancel-subscribe', id }) }) } - private sendToChild(proc: ChildProcess, message: HostToWatcherMessage): void { - try { - proc.send(message) - } catch { - // Channel already closed — the child's exit handler recovers. + private restartAfterCancelledSubscribe(proc: ChildProcess | null): Promise { + const activeTermination = this.terminationQueue.getCurrent() + if (activeTermination || !proc || !this.cancelledSubscribes.beginRestart(proc)) { + return activeTermination ?? Promise.resolve() } + if (this.child === proc) { + this.child = null + } + this.terminatingChild = proc + this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) + return this.terminationQueue.track( + restartCancelledWatcherChild( + proc, + this.records, + this.pendingUnsubscribes, + this.cancelledSubscribes, + (exited) => { + this.terminatingChild = null + if (!exited) { + this.shutdownRequested = true + } + return !this.shutdownRequested + }, + () => this.ensureWatcherProcess(), + sendWatcherSubscribe, + () => { + this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir) + } + ) + ) } } diff --git a/src/main/ipc/parcel-watcher-process-test-child.ts b/src/main/ipc/parcel-watcher-process-test-child.ts new file mode 100644 index 00000000000..d4b8af4333f --- /dev/null +++ b/src/main/ipc/parcel-watcher-process-test-child.ts @@ -0,0 +1,46 @@ +import { EventEmitter } from 'node:events' +import { vi } from 'vitest' + +type SentMessage = { op: string; id: number; dir?: string } + +export class FakeWatcherChild extends EventEmitter { + connected = true + pid = 1234 + exitCode: number | null = null + signalCode: NodeJS.Signals | null = null + sent: SentMessage[] = [] + stderr = new EventEmitter() + kill = vi.fn(() => { + this.connected = false + }) + send = vi.fn((message: SentMessage) => { + this.sent.push(message) + return true + }) +} + +export function currentWatcherChild(forkMock: ReturnType): FakeWatcherChild { + const result = forkMock.mock.results.at(-1) + if (!result) { + throw new Error('fork was not called') + } + return result.value as FakeWatcherChild +} + +export function acknowledgeWatcherSubscribe(child: FakeWatcherChild, index = -1): number { + const message = child.sent.filter((candidate) => candidate.op === 'subscribe').at(index) + if (!message) { + throw new Error('no subscribe message sent') + } + child.emit('message', { op: 'subscribed', id: message.id }) + return message.id +} + +export function trackPromiseSettlement(promise: Promise): () => boolean { + let settled = false + void promise.then( + () => (settled = true), + () => (settled = true) + ) + return () => settled +} diff --git a/src/main/ipc/parcel-watcher-process.test.ts b/src/main/ipc/parcel-watcher-process.test.ts index 1809ad484a7..805281bbea6 100644 --- a/src/main/ipc/parcel-watcher-process.test.ts +++ b/src/main/ipc/parcel-watcher-process.test.ts @@ -1,31 +1,35 @@ // Regression tests for issue #7547: local @parcel/watcher subscriptions run // in a forked watcher process, and a native watcher crash must be contained — // respawn, resubscribe, notify interruption — instead of killing the app. -import { EventEmitter } from 'node:events' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { WATCHER_PROCESS_CANCEL_TIMEOUT_MS } from './parcel-watcher-cancellation-tracker' +import { + terminateWatcherChild, + WATCHER_PROCESS_EXIT_DEADLINE_MS, + WATCHER_PROCESS_HARD_KILL_DELAY_MS +} from './parcel-watcher-child-termination' +import { + acknowledgeWatcherSubscribe as ackSubscribe, + currentWatcherChild, + FakeWatcherChild as FakeChild, + trackPromiseSettlement +} from './parcel-watcher-process-test-child' -const { - forkMock, - existsSyncMock, - mkdtempSyncMock, - parcelSubscribeMock, - rmSyncMock, - writeFileSyncMock -} = vi.hoisted(() => ({ - forkMock: vi.fn(), - existsSyncMock: vi.fn(), - mkdtempSyncMock: vi.fn(() => '/tmp/orca-watcher-canary-supervisor-test'), - parcelSubscribeMock: vi.fn(), - rmSyncMock: vi.fn(), - writeFileSyncMock: vi.fn() -})) +const { forkMock, existsSyncMock, mkdtempSyncMock, parcelSubscribeMock, rmSyncMock } = vi.hoisted( + () => ({ + forkMock: vi.fn(), + existsSyncMock: vi.fn(), + mkdtempSyncMock: vi.fn(() => '/tmp/orca-watcher-canary-supervisor-test'), + parcelSubscribeMock: vi.fn(), + rmSyncMock: vi.fn() + }) +) vi.mock('node:child_process', () => ({ fork: forkMock })) vi.mock('node:fs', () => ({ existsSync: existsSyncMock, mkdtempSync: mkdtempSyncMock, - rmSync: rmSyncMock, - writeFileSync: writeFileSyncMock + rmSync: rmSyncMock })) vi.mock('@parcel/watcher', () => ({ subscribe: parcelSubscribeMock })) @@ -38,38 +42,7 @@ import { subscribeViaWatcherProcess } from './parcel-watcher-process' -type SentMessage = { op: string; id: number; dir?: string } - -class FakeChild extends EventEmitter { - connected = true - pid = 1234 - sent: SentMessage[] = [] - stderr = new EventEmitter() - kill = vi.fn(() => { - this.connected = false - }) - send = vi.fn((message: SentMessage) => { - this.sent.push(message) - return true - }) -} - -function currentChild(): FakeChild { - const result = forkMock.mock.results.at(-1) - if (!result) { - throw new Error('fork was not called') - } - return result.value as FakeChild -} - -function ackSubscribe(child: FakeChild, index = -1): number { - const message = child.sent.filter((m) => m.op === 'subscribe').at(index) - if (!message) { - throw new Error('no subscribe message sent') - } - child.emit('message', { op: 'subscribed', id: message.id }) - return message.id -} +const currentChild = (): FakeChild => currentWatcherChild(forkMock) describe('subscribeViaWatcherProcess', () => { beforeEach(() => { @@ -89,6 +62,21 @@ describe('subscribeViaWatcherProcess', () => { vi.clearAllMocks() }) + it('accepts close without exit as physical child termination', async () => { + vi.useFakeTimers() + try { + const child = new FakeChild() + + const termination = terminateWatcherChild(child as never) + child.emit('close', -2, null) + + await expect(termination).resolves.toBe(true) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) + it('subscribes through the watcher process and forwards events', async () => { const callback = vi.fn() const promise = subscribeViaWatcherProcess('/repo', callback, { ignore: ['**/.git'] }) @@ -102,19 +90,6 @@ describe('subscribeViaWatcherProcess', () => { expect(callback).toHaveBeenCalledWith(null, events) }) - it('creates the fault-harness pid file without clobbering an existing path', async () => { - vi.stubEnv('ORCA_WATCHER_CHILD_PID_FILE', '/tmp/orca-watcher.pid') - - const promise = subscribeViaWatcherProcess('/repo', vi.fn(), {}) - const child = currentChild() - - expect(writeFileSyncMock).toHaveBeenCalledWith('/tmp/orca-watcher.pid', '1234', { - flag: 'wx' - }) - ackSubscribe(child) - await promise - }) - it('forwards watcher errors to the callback', async () => { const callback = vi.fn() const promise = subscribeViaWatcherProcess('/repo', callback, {}) @@ -126,6 +101,31 @@ describe('subscribeViaWatcherProcess', () => { expect(callback).toHaveBeenCalledWith(expect.objectContaining({ message: 'boom' }), []) }) + it('rejects and physically cancels a watcher error before subscription readiness', async () => { + const callback = vi.fn() + const promise = subscribeViaWatcherProcess('/repo', callback, {}) + const child = currentChild() + const id = child.sent[0].id + child.emit('message', { op: 'subscribe-started', id }) + + child.emit('message', { op: 'watch-error', id, message: 'root disappeared during crawl' }) + + let settled = false + void promise.catch(() => { + settled = true + }) + expect(child.sent.at(-1)).toEqual({ op: 'cancel-subscribe', id }) + expect(settled).toBe(false) + child.emit('message', { op: 'cancel-requires-restart', id }) + expect(settled).toBe(false) + expect(callback).not.toHaveBeenCalled() + expect(child.kill).toHaveBeenCalledTimes(1) + child.emit('exit', 0, null) + await expect(promise).rejects.toThrow('root disappeared during crawl') + child.emit('message', { op: 'subscribed', id }) + expect(forkMock).toHaveBeenCalledTimes(1) + }) + it('reports a terminal resubscribe failure separately from recoverable watch errors', async () => { const callback = vi.fn() const onTerminalError = vi.fn() @@ -227,6 +227,36 @@ describe('subscribeViaWatcherProcess', () => { expect(callback).toHaveBeenCalledWith(null, events) }) + it('rejects a replacement watch error before crash recovery is ready', async () => { + const onTerminalError = vi.fn() + const promise = subscribeViaWatcherProcess('/repo', vi.fn(), {}, { onTerminalError }) + const first = currentChild() + const id = ackSubscribe(first) + await promise + + first.connected = false + first.emit('exit', null, 'SIGSEGV') + const replacement = currentChild() + replacement.emit('message', { op: 'subscribe-started', id }) + replacement.emit('message', { + op: 'watch-error', + id, + message: 'root disappeared during replacement crawl' + }) + + expect(replacement.kill).toHaveBeenCalledTimes(1) + replacement.emit('message', { op: 'subscribed', id }) + expect(onTerminalError).not.toHaveBeenCalled() + replacement.emit('exit', 0, null) + + await vi.waitFor(() => + expect(onTerminalError).toHaveBeenCalledWith( + expect.objectContaining({ message: 'root disappeared during replacement crawl' }) + ) + ) + expect(forkMock).toHaveBeenCalledTimes(2) + }) + it('recovers existing records when IPC disconnects before child exit', async () => { const firstPromise = subscribeViaWatcherProcess('/existing', vi.fn(), {}) const first = currentChild() @@ -235,9 +265,12 @@ describe('subscribeViaWatcherProcess', () => { first.connected = false first.emit('disconnect') - const replacement = currentChild() const secondPromise = subscribeViaWatcherProcess('/new', vi.fn(), {}) + expect(forkMock).toHaveBeenCalledTimes(1) + first.emit('exit', 0, null) + await vi.waitFor(() => expect(forkMock).toHaveBeenCalledTimes(2)) + const replacement = currentChild() expect(replacement).not.toBe(first) expect(replacement.sent.filter((message) => message.op === 'subscribe')).toEqual([ expect.objectContaining({ dir: '/existing' }), @@ -248,7 +281,7 @@ describe('subscribeViaWatcherProcess', () => { await secondPromise }) - it('spreads runtime roots across healthy children and contains a shard crash', async () => { + it('shares healthy runtime roots and recovers the shard after a crash', async () => { const firstInterruption = vi.fn() const secondInterruption = vi.fn() const firstPromise = subscribeViaRuntimeWatcherProcess( @@ -272,21 +305,21 @@ describe('subscribeViaWatcherProcess', () => { } ) const secondChild = currentChild() - expect(secondChild).not.toBe(firstChild) + expect(secondChild).toBe(firstChild) ackSubscribe(secondChild) await secondPromise - expect(forkMock).toHaveBeenCalledTimes(2) + expect(forkMock).toHaveBeenCalledTimes(1) firstChild.connected = false firstChild.emit('exit', null, 'SIGSEGV') const replacement = currentChild() expect(replacement).not.toBe(firstChild) - expect(replacement.sent.filter((message) => message.op === 'subscribe')).toHaveLength(1) - ackSubscribe(replacement) + expect(replacement.sent.filter((message) => message.op === 'subscribe')).toHaveLength(2) + ackSubscribe(replacement, 0) + ackSubscribe(replacement, 1) expect(firstInterruption).toHaveBeenCalledTimes(1) - expect(secondInterruption).not.toHaveBeenCalled() - expect(secondChild.connected).toBe(true) + expect(secondInterruption).toHaveBeenCalledTimes(1) }) it('completes an in-flight subscribe across a crash-respawn', async () => { @@ -330,12 +363,63 @@ describe('subscribeViaWatcherProcess', () => { const queuedId = child.sent.at(-1)?.id controller.abort() - await expect(pending).rejects.toThrow('aborted') + const isSettled = trackPromiseSettlement(pending) expect(child.kill).not.toHaveBeenCalled() expect(child.sent.at(-1)).toEqual({ op: 'cancel-subscribe', id: queuedId }) + expect(isSettled()).toBe(false) + child.emit('message', { op: 'unsubscribed', id: queuedId }) + await expect(pending).rejects.toThrow('aborted') expect(healthyInterruption).not.toHaveBeenCalled() }) + it('kills a shard when resolved-child cancellation teardown never acknowledges', async () => { + vi.useFakeTimers() + try { + const healthyInterruption = vi.fn() + const healthyPromise = subscribeViaWatcherProcess( + '/healthy', + vi.fn(), + {}, + { onInterruption: healthyInterruption } + ) + const first = currentChild() + ackSubscribe(first) + await healthyPromise + + const controller = new AbortController() + const pending = subscribeViaWatcherProcess( + '/resolved-but-unacked', + vi.fn(), + {}, + { + signal: controller.signal + } + ) + const pendingId = first.sent.at(-1)?.id + controller.abort() + + const isSettled = trackPromiseSettlement(pending) + expect(first.sent.at(-1)).toEqual({ op: 'cancel-subscribe', id: pendingId }) + await vi.advanceTimersByTimeAsync(WATCHER_PROCESS_CANCEL_TIMEOUT_MS) + + expect(first.kill).toHaveBeenCalledTimes(1) + expect(isSettled()).toBe(false) + expect(forkMock).toHaveBeenCalledTimes(1) + + first.emit('exit', 0, null) + await expect(pending).rejects.toThrow('aborted') + const replacement = currentChild() + expect(replacement).not.toBe(first) + expect(replacement.sent.filter((message) => message.op === 'subscribe')).toEqual([ + expect.objectContaining({ dir: '/healthy' }) + ]) + ackSubscribe(replacement) + expect(healthyInterruption).toHaveBeenCalledTimes(1) + } finally { + vi.useRealTimers() + } + }) + it('cancels an active crawl and restores healthy roots when its owner aborts', async () => { const healthyInterruption = vi.fn() const healthyPromise = subscribeViaWatcherProcess( @@ -352,11 +436,18 @@ describe('subscribeViaWatcherProcess', () => { const controller = new AbortController() const pending = subscribeViaWatcherProcess('/slow', vi.fn(), {}, { signal: controller.signal }) - first.emit('message', { op: 'subscribe-started', id: first.sent.at(-1)?.id }) + const pendingId = first.sent.at(-1)?.id + first.emit('message', { op: 'subscribe-started', id: pendingId }) controller.abort() - await expect(pending).rejects.toThrow('aborted') + const isSettled = trackPromiseSettlement(pending) + expect(first.sent.at(-1)).toEqual({ op: 'cancel-subscribe', id: pendingId }) + first.emit('message', { op: 'cancel-requires-restart', id: pendingId }) expect(first.kill).toHaveBeenCalledTimes(1) + expect(isSettled()).toBe(false) + expect(forkMock).toHaveBeenCalledTimes(1) + first.emit('exit', 0, null) + await expect(pending).rejects.toThrow('aborted') const replacement = currentChild() expect(replacement).not.toBe(first) expect(replacement.sent.filter((message) => message.op === 'subscribe')).toEqual([ @@ -366,6 +457,94 @@ describe('subscribeViaWatcherProcess', () => { expect(healthyInterruption).toHaveBeenCalledTimes(1) }) + it('keeps process count bounded when a killed child never exits', async () => { + vi.useFakeTimers() + try { + const healthyPromise = subscribeViaWatcherProcess('/healthy', vi.fn(), {}) + const child = currentChild() + ackSubscribe(child) + await healthyPromise + + const controller = new AbortController() + const pending = subscribeViaWatcherProcess( + '/slow', + vi.fn(), + {}, + { + signal: controller.signal + } + ) + const pendingId = child.sent.at(-1)?.id + child.emit('message', { op: 'subscribe-started', id: pendingId }) + controller.abort() + const rejected = expect(pending).rejects.toThrow( + 'file watcher process did not exit after termination deadline' + ) + child.emit('message', { op: 'cancel-requires-restart', id: pendingId }) + + await vi.advanceTimersByTimeAsync(WATCHER_PROCESS_HARD_KILL_DELAY_MS) + expect(child.kill).toHaveBeenNthCalledWith(2, 'SIGKILL') + expect(forkMock).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync( + WATCHER_PROCESS_EXIT_DEADLINE_MS - WATCHER_PROCESS_HARD_KILL_DELAY_MS + ) + + await rejected + expect(forkMock).toHaveBeenCalledTimes(1) + expect(child.listenerCount('exit')).toBe(1) + expect(vi.getTimerCount()).toBe(0) + await expect(subscribeViaWatcherProcess('/later', vi.fn(), {})).rejects.toThrow('disposed') + } finally { + vi.useRealTimers() + } + }) + + it('queues subscribe and unsubscribe behind the retiring child', async () => { + const healthyPromise = subscribeViaWatcherProcess('/healthy', vi.fn(), {}) + const first = currentChild() + ackSubscribe(first) + const healthy = await healthyPromise + + const controller = new AbortController() + const cancelled = subscribeViaWatcherProcess( + '/slow', + vi.fn(), + {}, + { + signal: controller.signal + } + ) + const cancelledId = first.sent.at(-1)?.id + first.emit('message', { op: 'subscribe-started', id: cancelledId }) + controller.abort() + void cancelled.catch(() => undefined) + first.emit('message', { op: 'cancel-requires-restart', id: cancelledId }) + + let unsubscribeSettled = false + const unsubscribe = healthy.unsubscribe().then(() => { + unsubscribeSettled = true + }) + let queuedSettled = false + const queued = subscribeViaWatcherProcess('/queued-during-termination', vi.fn(), {}).then( + (subscription) => { + queuedSettled = true + return subscription + } + ) + await Promise.resolve() + expect(unsubscribeSettled).toBe(false) + expect(queuedSettled).toBe(false) + expect(forkMock).toHaveBeenCalledTimes(1) + + first.emit('exit', 0, null) + await expect(cancelled).rejects.toThrow('aborted') + await unsubscribe + const replacement = currentChild() + expect(replacement).not.toBe(first) + ackSubscribe(replacement) + await expect(queued).resolves.toMatchObject({ unsubscribe: expect.any(Function) }) + }) + it('bounds a pending crawl and restores healthy roots after setup timeout', async () => { vi.useFakeTimers() try { @@ -384,9 +563,13 @@ describe('subscribeViaWatcherProcess', () => { const pending = subscribeViaWatcherProcess('/slow', vi.fn(), {}, { subscribeTimeoutMs: 100 }) first.emit('message', { op: 'subscribe-started', id: first.sent.at(-1)?.id }) + const pendingId = first.sent.at(-1)?.id const timedOut = expect(pending).rejects.toThrow('timed out') await vi.advanceTimersByTimeAsync(100) + expect(first.sent.at(-1)).toEqual({ op: 'cancel-subscribe', id: pendingId }) + first.emit('message', { op: 'cancel-requires-restart', id: pendingId }) + first.emit('exit', 0, null) await timedOut const replacement = currentChild() expect(replacement).not.toBe(first) @@ -397,48 +580,6 @@ describe('subscribeViaWatcherProcess', () => { } }) - it('ends a root whose crash resubscription crawl exceeds its deadline', async () => { - vi.useFakeTimers() - try { - const supervisor = createWatcherProcessSupervisor() - const onTerminalError = vi.fn() - const initial = supervisor.subscribe( - '/slow-recovery', - vi.fn(), - {}, - { - onTerminalError, - subscribeTimeoutMs: 100 - } - ) - const first = currentChild() - ackSubscribe(first) - await initial - - first.connected = false - first.emit('exit', null, 'SIGSEGV') - const replacement = currentChild() - replacement.emit('message', { - op: 'subscribe-started', - id: replacement.sent[0].id - }) - - await vi.advanceTimersByTimeAsync(100) - - expect(onTerminalError).toHaveBeenCalledWith( - expect.objectContaining({ - code: 'subscribe_timeout', - message: 'file watcher resubscription timed out after 100ms' - }) - ) - expect(replacement.kill).toHaveBeenCalledTimes(1) - expect(forkMock).toHaveBeenCalledTimes(2) - supervisor.dispose() - } finally { - vi.useRealTimers() - } - }) - it('starts the setup timeout only when the child begins that crawl', async () => { vi.useFakeTimers() try { @@ -465,9 +606,12 @@ describe('subscribeViaWatcherProcess', () => { expect(settled).toBe(false) expect(child.kill).not.toHaveBeenCalled() - child.emit('message', { op: 'subscribe-started', id: child.sent.at(-1)?.id }) + const pendingId = child.sent.at(-1)?.id + child.emit('message', { op: 'subscribe-started', id: pendingId }) const timedOut = expect(pending).rejects.toThrow('timed out') await vi.advanceTimersByTimeAsync(100) + child.emit('message', { op: 'cancel-requires-restart', id: pendingId }) + child.emit('exit', 0, null) await timedOut } finally { vi.useRealTimers() @@ -518,8 +662,10 @@ describe('subscribeViaWatcherProcess', () => { ackSubscribe(first) const subscription = await promise - await expect(subscription.unsubscribe()).resolves.toBeUndefined() + const unsubscribe = subscription.unsubscribe() + const isSettled = trackPromiseSettlement(unsubscribe) expect(first.kill).toHaveBeenCalledTimes(1) + expect(isSettled()).toBe(false) expect(rmSyncMock).toHaveBeenCalledWith('/tmp/orca-watcher-canary-supervisor-test', { recursive: true, force: true @@ -529,6 +675,7 @@ describe('subscribeViaWatcherProcess', () => { // and no crash-fuse advance when the killed child's exit event lands. first.connected = false first.emit('exit', null, 'SIGTERM') + await expect(unsubscribe).resolves.toBeUndefined() expect(forkMock).toHaveBeenCalledTimes(1) const respawnPromise = subscribeViaWatcherProcess('/repo2', vi.fn(), {}) @@ -550,6 +697,8 @@ describe('subscribeViaWatcherProcess', () => { }) await expect(promise).rejects.toThrow('Error opening directory') expect(first.kill).toHaveBeenCalledTimes(1) + first.emit('exit', 0, null) + await new Promise((resolve) => setImmediate(resolve)) const respawnPromise = subscribeViaWatcherProcess('/repo', vi.fn(), {}) expect(forkMock).toHaveBeenCalledTimes(2) @@ -582,21 +731,38 @@ describe('subscribeViaWatcherProcess', () => { expect(child.kill).not.toHaveBeenCalled() expect(child.sent.at(-1)).toMatchObject({ op: 'unsubscribe' }) - // The last unsubscribe kills the idle child, which also completes the - // still-pending unsubscribe ack from the first record. + // The last unsubscribe kills the idle child; both callers wait for its + // physical exit because either may gate destructive worktree cleanup. const secondUnsub = second.unsubscribe() expect(child.kill).toHaveBeenCalledTimes(1) + child.emit('exit', 0, null) await expect(firstUnsub).resolves.toBeUndefined() await expect(secondUnsub).resolves.toBeUndefined() }) - it('uses the in-process watcher under vitest', async () => { + it('disposes production runtime watcher children during app shutdown', async () => { + const pending = subscribeViaRuntimeWatcherProcess('/runtime-root', vi.fn(), {}) + const child = currentChild() + ackSubscribe(child) + await pending + + disposeWatcherProcess() + + expect(child.kill).toHaveBeenCalledTimes(1) + await expect( + subscribeViaRuntimeWatcherProcess('/runtime-root', vi.fn(), {}) + ).rejects.toMatchObject({ code: 'supervisor_disposed' }) + expect(forkMock).toHaveBeenCalledTimes(1) + }) + + it('keeps the runtime pool reusable with the in-process watcher under vitest', async () => { vi.stubEnv('VITEST', 'true') + disposeWatcherProcess() const unsubscribe = vi.fn().mockResolvedValue(undefined) parcelSubscribeMock.mockResolvedValue({ unsubscribe }) const callback = vi.fn() - const subscription = await subscribeViaWatcherProcess('/repo', callback, { + const subscription = await subscribeViaRuntimeWatcherProcess('/repo', callback, { ignore: ['**/.git'] }) expect(forkMock).not.toHaveBeenCalled() @@ -638,6 +804,26 @@ describe('subscribeViaWatcherProcess', () => { await vi.waitFor(() => expect(unsubscribe).toHaveBeenCalledTimes(1)) }) + it('contains rejection while releasing a late in-process subscription', async () => { + vi.stubEnv('VITEST', 'true') + const unsubscribe = vi.fn().mockRejectedValue(new Error('late unsubscribe failed')) + let resolveSubscription!: (subscription: { unsubscribe: typeof unsubscribe }) => void + parcelSubscribeMock.mockReturnValue( + new Promise((resolve) => { + resolveSubscription = resolve + }) + ) + const controller = new AbortController() + const pending = subscribeViaWatcherProcess('/repo', vi.fn(), {}, { signal: controller.signal }) + await vi.waitFor(() => expect(parcelSubscribeMock).toHaveBeenCalledTimes(1)) + controller.abort() + await expect(pending).rejects.toThrow('aborted') + + resolveSubscription({ unsubscribe }) + + await vi.waitFor(() => expect(unsubscribe).toHaveBeenCalledTimes(1)) + }) + it('fails closed when the isolated watcher entry is absent outside tests', async () => { existsSyncMock.mockReturnValue(false) diff --git a/src/main/ipc/parcel-watcher-process.ts b/src/main/ipc/parcel-watcher-process.ts index 2d6b45ee97f..978edb21c6b 100644 --- a/src/main/ipc/parcel-watcher-process.ts +++ b/src/main/ipc/parcel-watcher-process.ts @@ -1,19 +1,19 @@ // Host API for crash-isolated @parcel/watcher children. Desktop uses one // supervisor; runtime roots use a bounded pool with independent crash fuses. -import { - WatcherProcessSupervisor, - type WatcherProcessCallback, - type WatcherProcessHooks, - type WatcherProcessSubscription -} from './parcel-watcher-process-supervisor' +import { WatcherProcessSupervisor } from './parcel-watcher-process-supervisor' import type { WatcherProcessSubscribeOptions } from './parcel-watcher-process-protocol' +import type { + WatcherProcessCallback, + WatcherProcessHooks, + WatcherProcessSubscription +} from './parcel-watcher-process-subscription' import { RuntimeWatcherProcessPool } from './runtime-watcher-process-pool' export type { WatcherProcessCallback, WatcherProcessHooks, WatcherProcessSubscription -} from './parcel-watcher-process-supervisor' +} from './parcel-watcher-process-subscription' export type { WatcherProcessDeliveryOptions, WatcherProcessEvent, @@ -21,8 +21,11 @@ export type { } from './parcel-watcher-process-protocol' const sharedWatcherProcessSupervisor = new WatcherProcessSupervisor() -// Why: healthy and quarantine pools each cap at four children, containing a -// failed shard without scaling process count with its number of roots. +// Why: the canary pauses during legitimate crawls, so production callers need +// a separate liveness fuse once a child reports that it owns native setup. +export const WATCHER_PROCESS_SUBSCRIBE_TIMEOUT_MS = 60_000 +// Why: healthy roots share one child; only fault quarantine scales to four, +// containing a failed shard without paying that RSS cost during normal use. const runtimeWatcherProcessPool = new RuntimeWatcherProcessPool() export function createWatcherProcessSupervisor(): WatcherProcessSupervisor { diff --git a/src/main/ipc/parcel-watcher-supervisor-capacity-wait.test.ts b/src/main/ipc/parcel-watcher-supervisor-capacity-wait.test.ts new file mode 100644 index 00000000000..4f0845ebd26 --- /dev/null +++ b/src/main/ipc/parcel-watcher-supervisor-capacity-wait.test.ts @@ -0,0 +1,35 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + MAX_PHYSICAL_WATCHER_CHILDREN, + reserveWatcherChild, + resetWatcherChildRegistryForTest +} from './parcel-watcher-child-registry' +import { WatcherSupervisorCapacityWait } from './parcel-watcher-supervisor-capacity-wait' + +describe('WatcherSupervisorCapacityWait', () => { + beforeEach(() => resetWatcherChildRegistryForTest()) + afterEach(() => resetWatcherChildRegistryForTest()) + + it('removes ten thousand cancelled capacity callers while one anchor remains', async () => { + const releases = Array.from({ length: MAX_PHYSICAL_WATCHER_CHILDREN }, () => + reserveWatcherChild() + ) + expect(releases.every(Boolean)).toBe(true) + const capacity = new WatcherSupervisorCapacityWait() + const anchor = capacity.wait() + const controllers = Array.from({ length: 10_000 }, () => new AbortController()) + const cancelled = controllers.map((controller) => + capacity.wait(controller.signal).catch((error) => error) + ) + + for (const controller of controllers) { + controller.abort() + } + await Promise.all(cancelled) + + expect(capacity.waiterCount).toBe(1) + releases[0]?.() + await expect(anchor).resolves.toBeUndefined() + expect(capacity.waiterCount).toBe(0) + }) +}) diff --git a/src/main/ipc/parcel-watcher-supervisor-capacity-wait.ts b/src/main/ipc/parcel-watcher-supervisor-capacity-wait.ts new file mode 100644 index 00000000000..bda018da6b0 --- /dev/null +++ b/src/main/ipc/parcel-watcher-supervisor-capacity-wait.ts @@ -0,0 +1,111 @@ +import { + onWatcherChildCapacityAvailable, + WatcherChildCapacityError +} from './parcel-watcher-child-registry' +import { watcherHostFailure, WatcherProcessFailure } from './parcel-watcher-process-failure' +import { PromiseSettlementWaiters } from '../../shared/promise-settlement-waiters' + +type CapacityWaitState = { + waiters: PromiseSettlementWaiters + resolve: () => void + reject: (error: Error) => void + cancel: () => void +} + +export class WatcherSupervisorCapacityWait { + private state: CapacityWaitState | null = null + private lifecycleGeneration = 0 + + get waiterCount(): number { + return this.state?.waiters.waiterCount ?? 0 + } + + run(operation: Promise, retry: () => Promise, signal?: AbortSignal): Promise { + const lifecycleGeneration = this.lifecycleGeneration + return operation.catch(async (error: unknown) => { + if (!(error instanceof WatcherChildCapacityError)) { + throw error + } + if (lifecycleGeneration !== this.lifecycleGeneration) { + throw watcherHostFailure('file watcher supervisor disposed', 'supervisor_disposed') + } + await this.wait(signal) + if (lifecycleGeneration !== this.lifecycleGeneration) { + throw watcherHostFailure('file watcher supervisor disposed', 'supervisor_disposed') + } + // Why: subscribe() wraps each retry in a fresh capacity wait, so a slot + // reclaimed by crash recovery remains pending without polling or failure. + return retry() + }) + } + + wait(signal?: AbortSignal): Promise { + if (signal?.aborted) { + return Promise.reject(createCapacityAbortError()) + } + const state = this.state ?? this.createState() + return state.waiters.wait({ + signal, + createAbortError: createCapacityAbortError, + onAbandon: () => { + if (state.waiters.waiterCount === 0 && this.state === state) { + // Why: one cancelled root must not strand a shared listener after all + // same-supervisor capacity waiters have left. + this.state = null + state.cancel() + state.reject(createCapacityAbortError()) + } + } + }) + } + + dispose(): void { + this.lifecycleGeneration++ + const state = this.state + if (!state) { + return + } + this.state = null + state.cancel() + state.reject( + new WatcherProcessFailure( + 'file watcher supervisor disposed', + 'supervisor', + 'supervisor_disposed' + ) + ) + } + + private createState(): CapacityWaitState { + let resolve = (): void => undefined + let reject = (_error: Error): void => undefined + const promise = new Promise((nextResolve, nextReject) => { + resolve = nextResolve + reject = nextReject + }) + const state: CapacityWaitState = { + waiters: new PromiseSettlementWaiters(promise), + resolve, + reject, + cancel: () => undefined + } + this.state = state + // Why: one physical slot launches the shared supervisor child; every root + // awaiting this barrier can then subscribe through that same child. + state.cancel = onWatcherChildCapacityAvailable(() => { + if (this.state === state) { + this.state = null + } + state.resolve() + }) + return state + } +} + +function createCapacityAbortError(): WatcherProcessFailure { + return new WatcherProcessFailure( + 'file watcher subscription aborted', + 'subscription', + 'subscribe_aborted' + ) +} diff --git a/src/main/ipc/parcel-watcher-supervisor-capacity.test.ts b/src/main/ipc/parcel-watcher-supervisor-capacity.test.ts new file mode 100644 index 00000000000..e1a26c03b2a --- /dev/null +++ b/src/main/ipc/parcel-watcher-supervisor-capacity.test.ts @@ -0,0 +1,192 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { WATCHER_PROCESS_EXIT_DEADLINE_MS } from './parcel-watcher-child-termination' +import { + MAX_PHYSICAL_WATCHER_CHILDREN, + resetWatcherChildRegistryForTest +} from './parcel-watcher-child-registry' +import { + acknowledgeWatcherSubscribe, + currentWatcherChild, + FakeWatcherChild, + trackPromiseSettlement +} from './parcel-watcher-process-test-child' +import type { WatcherProcessSupervisor } from './parcel-watcher-process-supervisor' + +const { existsSyncMock, forkMock, mkdtempSyncMock, rmSyncMock } = vi.hoisted(() => ({ + existsSyncMock: vi.fn(), + forkMock: vi.fn(), + mkdtempSyncMock: vi.fn(() => '/tmp/orca-watcher-capacity-supervisor-test'), + rmSyncMock: vi.fn() +})) + +vi.mock('node:child_process', () => ({ fork: forkMock })) +vi.mock('node:fs', () => ({ + existsSync: existsSyncMock, + mkdtempSync: mkdtempSyncMock, + rmSync: rmSyncMock +})) + +import { createWatcherProcessSupervisor } from './parcel-watcher-process' + +const supervisors: WatcherProcessSupervisor[] = [] + +function createSupervisor(): WatcherProcessSupervisor { + const supervisor = createWatcherProcessSupervisor() + supervisors.push(supervisor) + return supervisor +} + +function currentChild(): FakeWatcherChild { + return currentWatcherChild(forkMock) +} + +describe('WatcherProcessSupervisor capacity wait', () => { + beforeEach(() => { + resetWatcherChildRegistryForTest() + vi.stubEnv('VITEST', '') + existsSyncMock.mockReturnValue(true) + forkMock.mockImplementation(() => new FakeWatcherChild()) + }) + + afterEach(() => { + for (const supervisor of supervisors.splice(0)) { + supervisor.dispose() + } + resetWatcherChildRegistryForTest() + vi.unstubAllEnvs() + vi.clearAllMocks() + vi.useRealTimers() + }) + + it('shares one released slot across same-supervisor capacity waiters', async () => { + vi.useFakeTimers() + const pending: Promise[] = [] + const children: FakeWatcherChild[] = [] + for (let index = 0; index < MAX_PHYSICAL_WATCHER_CHILDREN; index++) { + const supervisor = createSupervisor() + const controller = new AbortController() + const subscription = supervisor.subscribe( + `/stuck-${index}`, + vi.fn(), + {}, + { + signal: controller.signal + } + ) + pending.push(subscription.catch((error) => error)) + const child = currentChild() + children.push(child) + const id = child.sent.at(-1)?.id + child.emit('message', { op: 'subscribe-started', id }) + controller.abort() + child.emit('message', { op: 'cancel-requires-restart', id }) + } + expect(forkMock).toHaveBeenCalledTimes(MAX_PHYSICAL_WATCHER_CHILDREN) + + const extraSupervisor = createSupervisor() + const firstExtra = extraSupervisor.subscribe('/over-cap-a', vi.fn(), {}) + const secondExtra = extraSupervisor.subscribe('/over-cap-b', vi.fn(), {}) + const firstSettled = trackPromiseSettlement(firstExtra) + const secondSettled = trackPromiseSettlement(secondExtra) + await Promise.resolve() + expect(firstSettled()).toBe(false) + expect(secondSettled()).toBe(false) + await vi.advanceTimersByTimeAsync(WATCHER_PROCESS_EXIT_DEADLINE_MS) + await Promise.all(pending) + + expect(forkMock).toHaveBeenCalledTimes(MAX_PHYSICAL_WATCHER_CHILDREN) + expect(children.every((child) => child.listenerCount('exit') === 1)).toBe(true) + expect(vi.getTimerCount()).toBe(0) + + children[0].emit('exit', 0, null) + await vi.waitFor(() => + expect(forkMock).toHaveBeenCalledTimes(MAX_PHYSICAL_WATCHER_CHILDREN + 1) + ) + const replacement = currentChild() + const subscribeMessages = replacement.sent.filter(({ op }) => op === 'subscribe') + expect(subscribeMessages.map(({ dir }) => dir)).toEqual(['/over-cap-a', '/over-cap-b']) + acknowledgeWatcherSubscribe(replacement, 0) + acknowledgeWatcherSubscribe(replacement, 1) + await expect(Promise.all([firstExtra, secondExtra])).resolves.toHaveLength(2) + + const blockedSupervisor = createSupervisor() + const firstController = new AbortController() + const secondController = new AbortController() + const firstBlocked = blockedSupervisor.subscribe( + '/cancelled-over-cap-a', + vi.fn(), + {}, + { + signal: firstController.signal + } + ) + const secondBlocked = blockedSupervisor.subscribe( + '/cancelled-over-cap-b', + vi.fn(), + {}, + { + signal: secondController.signal + } + ) + await Promise.resolve() + expect(forkMock).toHaveBeenCalledTimes(MAX_PHYSICAL_WATCHER_CHILDREN + 1) + firstController.abort() + await expect(firstBlocked).rejects.toMatchObject({ code: 'subscribe_aborted' }) + expect(trackPromiseSettlement(secondBlocked)()).toBe(false) + secondController.abort() + await expect(secondBlocked).rejects.toMatchObject({ code: 'subscribe_aborted' }) + + const racedDisposeSupervisor = createSupervisor() + const racedDisposeWait = racedDisposeSupervisor.subscribe('/raced-dispose', vi.fn(), {}) + racedDisposeSupervisor.dispose() + await expect(racedDisposeWait).rejects.toMatchObject({ code: 'supervisor_disposed' }) + + const activeDisposeSupervisor = createSupervisor() + const activeDisposeWait = activeDisposeSupervisor.subscribe('/active-dispose', vi.fn(), {}) + await Promise.resolve() + activeDisposeSupervisor.dispose() + await expect(activeDisposeWait).rejects.toMatchObject({ code: 'supervisor_disposed' }) + + children[1].emit('exit', 0, null) + await Promise.resolve() + await Promise.resolve() + expect(forkMock).toHaveBeenCalledTimes(MAX_PHYSICAL_WATCHER_CHILDREN + 1) + }) + + it('keeps waiting when crash recovery reclaims the announced slot', async () => { + const children: FakeWatcherChild[] = [] + for (let index = 0; index < MAX_PHYSICAL_WATCHER_CHILDREN; index++) { + const supervisor = createSupervisor() + const subscription = supervisor.subscribe(`/active-${index}`, vi.fn(), {}) + const child = currentChild() + children.push(child) + acknowledgeWatcherSubscribe(child) + await subscription + } + + const waitingSupervisor = createSupervisor() + const waiting = waitingSupervisor.subscribe('/waiting-root', vi.fn(), {}) + const waitingSettled = trackPromiseSettlement(waiting) + await Promise.resolve() + expect(waitingSettled()).toBe(false) + + children[0].connected = false + children[0].emit('exit', null, 'SIGSEGV') + await vi.waitFor(() => + expect(forkMock).toHaveBeenCalledTimes(MAX_PHYSICAL_WATCHER_CHILDREN + 1) + ) + await Promise.resolve() + await Promise.resolve() + expect(waitingSettled()).toBe(false) + + supervisors[1].dispose() + children[1].emit('exit', 0, null) + await vi.waitFor(() => + expect(forkMock).toHaveBeenCalledTimes(MAX_PHYSICAL_WATCHER_CHILDREN + 2) + ) + const waitingChild = currentChild() + expect(waitingChild.sent).toContainEqual(expect.objectContaining({ dir: '/waiting-root' })) + acknowledgeWatcherSubscribe(waitingChild) + await expect(waiting).resolves.toMatchObject({ unsubscribe: expect.any(Function) }) + }) +}) diff --git a/src/main/ipc/parcel-watcher-supervisor-disposal.ts b/src/main/ipc/parcel-watcher-supervisor-disposal.ts new file mode 100644 index 00000000000..42dc918e05b --- /dev/null +++ b/src/main/ipc/parcel-watcher-supervisor-disposal.ts @@ -0,0 +1,32 @@ +import type { ChildProcess } from 'node:child_process' +import { removeWatcherCanaryDirectory } from './parcel-watcher-canary-directory' +import type { WatcherCancellationTracker } from './parcel-watcher-cancellation-tracker' +import { + type PendingWatcherUnsubscribe, + resolvePendingWatcherUnsubscribes +} from './parcel-watcher-host-subscriptions' +import { + resetPendingSubscribeAttempt, + takePendingSubscribe +} from './parcel-watcher-pending-subscribe' +import { watcherHostFailure } from './parcel-watcher-process-failure' +import type { WatcherProcessSubscriptionRecord } from './parcel-watcher-process-subscription' + +export function disposeWatcherSupervisor( + child: ChildProcess | null, + records: Map, + pendingUnsubscribes: Map, + cancelledSubscribes: WatcherCancellationTracker, + canaryDir: string | null +): null { + const error = watcherHostFailure('file watcher supervisor disposed', 'supervisor_disposed') + for (const record of records.values()) { + resetPendingSubscribeAttempt(record) + takePendingSubscribe(record)?.reject(error) + } + resolvePendingWatcherUnsubscribes(pendingUnsubscribes) + cancelledSubscribes.completeAll() + records.clear() + child?.kill() + return removeWatcherCanaryDirectory(canaryDir) +} diff --git a/src/main/ipc/parcel-watcher-supervisor-message.ts b/src/main/ipc/parcel-watcher-supervisor-message.ts new file mode 100644 index 00000000000..e5fbce4342f --- /dev/null +++ b/src/main/ipc/parcel-watcher-supervisor-message.ts @@ -0,0 +1,104 @@ +import type { ChildProcess } from 'node:child_process' +import type { WatcherCancellationTracker } from './parcel-watcher-cancellation-tracker' +import { + handleWatcherHostMessage, + type PendingWatcherUnsubscribe, + reportWatcherTerminalError +} from './parcel-watcher-host-subscriptions' +import { + startInterruptedSubscribeTimeout, + startPendingSubscribeTimeout +} from './parcel-watcher-pending-subscribe' +import { WatcherProcessFailure } from './parcel-watcher-process-failure' +import type { WatcherToHostMessage } from './parcel-watcher-process-protocol' +import type { WatcherProcessSubscriptionRecord } from './parcel-watcher-process-subscription' + +type WatcherSupervisorMessageContext = { + records: Map + pendingUnsubscribes: Map + cancelledSubscribes: WatcherCancellationTracker + child: ChildProcess | null + cancelPendingSubscribe: ( + record: WatcherProcessSubscriptionRecord, + error: WatcherProcessFailure + ) => void + cancelInterruptedSubscribe: ( + record: WatcherProcessSubscriptionRecord, + error: WatcherProcessFailure + ) => void + restartAfterCancelledSubscribe: (child: ChildProcess | null) => void + terminateUnavailableChild: (child: ChildProcess | null) => void + killWatcherChildIfIdle: () => void +} + +export function handleWatcherSupervisorMessage( + message: WatcherToHostMessage, + context: WatcherSupervisorMessageContext +): void { + const record = context.records.get(message.id) + if (message.op === 'subscribe-started') { + if (record) { + startPendingSubscribeTimeout(record, (error) => context.cancelPendingSubscribe(record, error)) + startInterruptedSubscribeTimeout(record, (error) => + context.cancelInterruptedSubscribe(record, error) + ) + } + return + } + if (message.op === 'watch-error' && record) { + const error = new WatcherProcessFailure(message.message, 'subscription', 'subscribe_failed') + if (record.pendingSubscribe) { + // Why: an error before readiness means native setup failed. Cancel the + // physical crawl so a later subscribed ack cannot expose a dead root. + context.cancelPendingSubscribe(record, error) + return + } + if (record.interrupted && record.crawlStarted) { + // Why: crash recovery has no pending caller promise, but its replacement + // crawl is equally unready and must not accept a later subscribed ack. + context.cancelInterruptedSubscribe(record, error) + return + } + } + if (message.op === 'watch-error') { + handleWatcherHostMessage( + message, + context.records, + context.pendingUnsubscribes, + reportWatcherTerminalError, + context.killWatcherChildIfIdle + ) + return + } + if (message.op === 'cancel-requires-restart') { + if (context.cancelledSubscribes.has(message.id)) { + context.restartAfterCancelledSubscribe(context.child) + } + return + } + if (message.op === 'unsubscribe-failed') { + context.terminateUnavailableChild(context.child) + return + } + if (message.op === 'unsubscribed') { + const completedCancellation = context.cancelledSubscribes.complete(message.id) + handleWatcherHostMessage( + message, + context.records, + context.pendingUnsubscribes, + reportWatcherTerminalError, + context.killWatcherChildIfIdle + ) + if (completedCancellation) { + context.killWatcherChildIfIdle() + } + return + } + handleWatcherHostMessage( + message, + context.records, + context.pendingUnsubscribes, + reportWatcherTerminalError, + context.killWatcherChildIfIdle + ) +} diff --git a/src/main/ipc/parcel-watcher-supervisor-subscribe.ts b/src/main/ipc/parcel-watcher-supervisor-subscribe.ts index abfa9ffe540..f508f9b117b 100644 --- a/src/main/ipc/parcel-watcher-supervisor-subscribe.ts +++ b/src/main/ipc/parcel-watcher-supervisor-subscribe.ts @@ -1,8 +1,13 @@ import type { ChildProcess } from 'node:child_process' import { existsSync } from 'node:fs' +import { sendToWatcherChild } from './parcel-watcher-child-messaging' import { createHostWatcherSubscription } from './parcel-watcher-host-subscriptions' +import type { PendingWatcherUnsubscribe } from './parcel-watcher-host-subscriptions' import { subscribeWithInProcessWatcher } from './parcel-watcher-in-process-fallback' -import { installPendingSubscribeControls } from './parcel-watcher-pending-subscribe' +import { + installPendingSubscribeControls, + resetPendingSubscribeAttempt +} from './parcel-watcher-pending-subscribe' import { WatcherProcessFailure } from './parcel-watcher-process-failure' import type { HostToWatcherMessage, @@ -25,10 +30,12 @@ type WatcherSupervisorSubscribeOptions = { useInProcessVitestFallback: boolean allocateId: () => number records: Map - pendingUnsubscribes: Map void> + pendingUnsubscribes: Map ensureWatcherProcess: (entryPath: string) => ChildProcess | null getChild: () => ChildProcess | null - killWatcherChildIfIdle: () => void + getTerminationPromise: () => Promise | null + killWatcherChildIfIdle: () => Promise + terminateUnavailableChild: (child: ChildProcess | null) => Promise sendSubscribe: (child: ChildProcess, record: WatcherProcessSubscriptionRecord) => void sendToChild: (child: ChildProcess, message: HostToWatcherMessage) => void cancelPendingSubscribe: ( @@ -37,6 +44,20 @@ type WatcherSupervisorSubscribeOptions = { ) => void } +export function sendWatcherSubscribe( + child: ChildProcess, + record: WatcherProcessSubscriptionRecord +): void { + resetPendingSubscribeAttempt(record) + sendToWatcherChild(child, { + op: 'subscribe', + id: record.id, + dir: record.dir, + opts: record.opts, + delivery: record.hooks.delivery + }) +} + export function subscribeThroughWatcherSupervisor({ dir, callback, @@ -50,7 +71,9 @@ export function subscribeThroughWatcherSupervisor({ pendingUnsubscribes, ensureWatcherProcess, getChild, + getTerminationPromise, killWatcherChildIfIdle, + terminateUnavailableChild, sendSubscribe, sendToChild, cancelPendingSubscribe @@ -117,7 +140,9 @@ export function subscribeThroughWatcherSupervisor({ records, pendingUnsubscribes, getChild, + getTerminationPromise, killWatcherChildIfIdle, + terminateUnavailableChild, sendToChild }) ), diff --git a/src/main/ipc/parcel-watcher-unsubscribe-timeout.test.ts b/src/main/ipc/parcel-watcher-unsubscribe-timeout.test.ts new file mode 100644 index 00000000000..026578938e7 --- /dev/null +++ b/src/main/ipc/parcel-watcher-unsubscribe-timeout.test.ts @@ -0,0 +1,116 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { WATCHER_PROCESS_EXIT_DEADLINE_MS } from './parcel-watcher-child-termination' +import { WATCHER_PROCESS_UNSUBSCRIBE_TIMEOUT_MS } from './parcel-watcher-host-subscriptions' +import { + acknowledgeWatcherSubscribe as ackSubscribe, + currentWatcherChild, + FakeWatcherChild, + trackPromiseSettlement +} from './parcel-watcher-process-test-child' + +const { existsSyncMock, forkMock, mkdtempSyncMock, rmSyncMock } = vi.hoisted(() => ({ + existsSyncMock: vi.fn(), + forkMock: vi.fn(), + mkdtempSyncMock: vi.fn(() => '/tmp/orca-watcher-unsubscribe-timeout-test'), + rmSyncMock: vi.fn() +})) + +vi.mock('node:child_process', () => ({ fork: forkMock })) +vi.mock('node:fs', () => ({ + existsSync: existsSyncMock, + mkdtempSync: mkdtempSyncMock, + rmSync: rmSyncMock +})) + +import { + disposeWatcherProcess, + resetWatcherProcessForTest, + subscribeViaWatcherProcess +} from './parcel-watcher-process' + +const currentChild = (): FakeWatcherChild => currentWatcherChild(forkMock) + +describe('watcher native unsubscribe timeout', () => { + beforeEach(() => { + resetWatcherProcessForTest() + vi.stubEnv('VITEST', '') + existsSyncMock.mockReturnValue(true) + forkMock.mockImplementation(() => new FakeWatcherChild()) + }) + + afterEach(() => { + disposeWatcherProcess() + vi.unstubAllEnvs() + vi.clearAllMocks() + }) + + it('restarts the shard and restores a healthy sibling', async () => { + vi.useFakeTimers() + try { + const firstPromise = subscribeViaWatcherProcess('/repo', vi.fn(), {}) + const first = currentChild() + ackSubscribe(first, 0) + const firstSubscription = await firstPromise + const siblingInterruption = vi.fn() + const siblingPromise = subscribeViaWatcherProcess( + '/sibling', + vi.fn(), + {}, + { onInterruption: siblingInterruption } + ) + ackSubscribe(first) + await siblingPromise + + const unsubscribe = firstSubscription.unsubscribe() + const isSettled = trackPromiseSettlement(unsubscribe) + await vi.advanceTimersByTimeAsync(WATCHER_PROCESS_UNSUBSCRIBE_TIMEOUT_MS) + + expect(first.kill).toHaveBeenCalledTimes(1) + expect(isSettled()).toBe(false) + first.emit('exit', 0, null) + await expect(unsubscribe).resolves.toBeUndefined() + + const replacement = currentChild() + expect(replacement).not.toBe(first) + expect(replacement.sent.filter((message) => message.op === 'subscribe')).toEqual([ + expect.objectContaining({ dir: '/sibling' }) + ]) + ackSubscribe(replacement) + expect(siblingInterruption).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it('rejects only after an unkillable child reaches its exit deadline', async () => { + vi.useFakeTimers() + try { + const firstPromise = subscribeViaWatcherProcess('/repo', vi.fn(), {}) + const child = currentChild() + ackSubscribe(child, 0) + const firstSubscription = await firstPromise + const siblingPromise = subscribeViaWatcherProcess('/sibling', vi.fn(), {}) + ackSubscribe(child) + await siblingPromise + + const result = firstSubscription.unsubscribe().catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(WATCHER_PROCESS_UNSUBSCRIBE_TIMEOUT_MS) + const isSettled = trackPromiseSettlement(result) + expect(isSettled()).toBe(false) + + await vi.advanceTimersByTimeAsync(WATCHER_PROCESS_EXIT_DEADLINE_MS) + const error = (await result) as Error & { physicalExit?: Promise } + expect(error).toMatchObject({ + message: 'file watcher process did not exit after termination deadline', + physicalExit: expect.any(Promise) + }) + expect(vi.getTimerCount()).toBe(0) + + child.emit('exit', 0, null) + await error.physicalExit + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index 4838711803d..3a798d35144 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -204,6 +204,7 @@ import { unregisterSshPtyProvider, getLocalPtyProvider } from './pty' +import { _resetLocalPtyProviderStateForTest } from '../providers/local-pty-provider' import { _resetHiddenRendererPtyDeliveryGateForTest, isHiddenRendererPty @@ -220,6 +221,7 @@ import { SSH_SESSION_EXPIRED_ERROR } from '../providers/ssh-pty-provider' import { _resetWslCachesForTests, _setWslCachesForTests } from '../wsl' +import { acquireWatcherRemovalGate } from './watcher-removal-gate' const POWERSHELL_OSC133_ARGS = [ '-NoLogo', @@ -422,6 +424,7 @@ describe('registerPtyHandlers', () => { }) afterEach(() => { + _resetLocalPtyProviderStateForTest() _resetWslCachesForTests() vi.useRealTimers() // Why: sshProviders is module-level state; any id left registered leaks @@ -823,6 +826,19 @@ describe('registerPtyHandlers', () => { describe('spawn environment', () => { it('marks local Claude launches live until the PTY is killed', async () => { + let exitCb: ((info: { exitCode: number }) => void) | undefined + spawnMock.mockReturnValue({ + onData: vi.fn(() => makeDisposable()), + onExit: vi.fn((cb: (info: { exitCode: number }) => void) => { + exitCb = cb + return makeDisposable() + }), + write: vi.fn(), + resize: vi.fn(), + kill: vi.fn(() => exitCb?.({ exitCode: -1 })), + process: 'zsh', + pid: 12345 + }) const prepareClaudeAuth = vi.fn(async () => ({ configDir: '/tmp/claude', envPatch: {}, @@ -6584,6 +6600,85 @@ describe('registerPtyHandlers', () => { expect(options.cwd).toBe('/tmp/floating-notes') }) + it('rejects a renderer spawn while destructive worktree removal holds the gate', async () => { + const removal = acquireWatcherRemovalGate('/repo/app') + await removal.ready + registerPtyHandlers(mainWindow as never) + + try { + await expect( + handlers.get('pty:spawn')!(null, { + cols: 80, + rows: 24, + worktreeId: 'repo-1::/repo/app' + }) + ).rejects.toMatchObject({ code: 'terminal_removal_in_progress' }) + expect(spawnMock).not.toHaveBeenCalled() + } finally { + removal.release() + } + }) + + it('rejects a sibling-worktree terminal cwd inside a worktree being removed', async () => { + const removal = acquireWatcherRemovalGate('/repo/app') + await removal.ready + registerPtyHandlers(mainWindow as never) + + try { + await expect( + handlers.get('pty:spawn')!(null, { + cols: 80, + rows: 24, + cwd: '/repo/app/nested', + worktreeId: 'repo-1::/repo/sibling' + }) + ).rejects.toMatchObject({ code: 'terminal_removal_in_progress' }) + expect(spawnMock).not.toHaveBeenCalled() + } finally { + removal.release() + } + const siblingRemoval = acquireWatcherRemovalGate('/repo/sibling') + await siblingRemoval.ready + siblingRemoval.release() + }) + + it('rejects a runtime sibling-worktree cwd inside a removing worktree', async () => { + const runtime = { + setPtyController: vi.fn(), + registerPty: vi.fn(), + onPtySpawned: vi.fn(), + onPtyExit: vi.fn(), + onPtyData: vi.fn() + } + registerPtyHandlers(mainWindow as never, runtime as never) + const controller = runtime.setPtyController.mock.calls[0]?.[0] as { + spawn(args: { + cols: number + rows: number + cwd?: string + worktreeId?: string + env?: Record + }): Promise<{ id: string }> + } + const removal = acquireWatcherRemovalGate('/repo/app') + await removal.ready + + try { + await expect( + controller.spawn({ + cols: 80, + rows: 24, + cwd: '/repo/app/nested', + worktreeId: 'repo-1::/repo/sibling', + env: {} + }) + ).rejects.toMatchObject({ code: 'terminal_removal_in_progress' }) + expect(spawnMock).not.toHaveBeenCalled() + } finally { + removal.release() + } + }) + it('falls back to the worktree root when a saved local cwd no longer exists', async () => { registerPtyHandlers(mainWindow as never) // Why: issue #7239 reproduced in a Japanese-named worktree; the fallback @@ -10643,9 +10738,10 @@ describe('registerPtyHandlers', () => { expect(piClearPtyMock).toHaveBeenCalledWith(spawnResult.id) }) - it('disposes PTY listeners before manual kill IPC', async () => { + it('retains PTY listeners until physical exit after manual kill IPC', async () => { const onDataDisposable = makeDisposable() const onExitDisposable = makeDisposable() + let exitCb: ((info: { exitCode: number }) => void) | undefined // Why: hold a stable reference to the kill spy. On POSIX, destroyPtyProcess // in local-pty-provider reassigns proc.kill to a no-op to defuse the // SIGHUP-to-recycled-pid hazard (see docs/fix-pty-fd-leak.md). Reading @@ -10653,7 +10749,10 @@ describe('registerPtyHandlers', () => { const killSpy = vi.fn() const proc = { onData: vi.fn(() => onDataDisposable), - onExit: vi.fn(() => onExitDisposable), + onExit: vi.fn((cb: (info: { exitCode: number }) => void) => { + exitCb = cb + return onExitDisposable + }), write: vi.fn(), resize: vi.fn(), kill: killSpy, @@ -10668,23 +10767,30 @@ describe('registerPtyHandlers', () => { rows: 24 })) as { id: string } - await handlers.get('pty:kill')!(null, { id: spawnResult.id }) + const killPromise = handlers.get('pty:kill')!(null, { id: spawnResult.id }) as Promise - expect(onDataDisposable.dispose.mock.invocationCallOrder[0]).toBeLessThan( - killSpy.mock.invocationCallOrder[0] - ) - expect(onExitDisposable.dispose.mock.invocationCallOrder[0]).toBeLessThan( - killSpy.mock.invocationCallOrder[0] - ) + expect(killSpy).toHaveBeenCalledTimes(1) + expect(onDataDisposable.dispose).not.toHaveBeenCalled() + expect(onExitDisposable.dispose).not.toHaveBeenCalled() + + exitCb?.({ exitCode: -1 }) + await killPromise + + expect(onDataDisposable.dispose).toHaveBeenCalledTimes(1) + expect(onExitDisposable.dispose).toHaveBeenCalledTimes(1) }) - it('disposes PTY listeners before runtime controller kill', async () => { + it('retains PTY listeners until physical exit after runtime controller kill', async () => { const onDataDisposable = makeDisposable() const onExitDisposable = makeDisposable() + let exitCb: ((info: { exitCode: number }) => void) | undefined const killSpy = vi.fn() const proc = { onData: vi.fn(() => onDataDisposable), - onExit: vi.fn(() => onExitDisposable), + onExit: vi.fn((cb: (info: { exitCode: number }) => void) => { + exitCb = cb + return onExitDisposable + }), write: vi.fn(), resize: vi.fn(), kill: killSpy, @@ -10711,21 +10817,26 @@ describe('registerPtyHandlers', () => { } expect(runtimeController.kill(spawnResult.id)).toBe(true) - expect(onDataDisposable.dispose.mock.invocationCallOrder[0]).toBeLessThan( - killSpy.mock.invocationCallOrder[0] - ) - expect(onExitDisposable.dispose.mock.invocationCallOrder[0]).toBeLessThan( - killSpy.mock.invocationCallOrder[0] - ) + await vi.waitFor(() => expect(killSpy).toHaveBeenCalledTimes(1)) + expect(onDataDisposable.dispose).not.toHaveBeenCalled() + expect(onExitDisposable.dispose).not.toHaveBeenCalled() + + exitCb?.({ exitCode: -1 }) + await vi.waitFor(() => expect(onExitDisposable.dispose).toHaveBeenCalledTimes(1)) + expect(onDataDisposable.dispose).toHaveBeenCalledTimes(1) }) - it('disposes PTY listeners before did-finish-load orphan cleanup', async () => { + it('retains the PTY exit listener through did-finish-load orphan cleanup', async () => { const onDataDisposable = makeDisposable() const onExitDisposable = makeDisposable() + let exitCb: ((info: { exitCode: number }) => void) | undefined const killSpy = vi.fn() const proc = { onData: vi.fn(() => onDataDisposable), - onExit: vi.fn(() => onExitDisposable), + onExit: vi.fn((cb: (info: { exitCode: number }) => void) => { + exitCb = cb + return onExitDisposable + }), write: vi.fn(), resize: vi.fn(), kill: killSpy, @@ -10764,9 +10875,10 @@ describe('registerPtyHandlers', () => { expect(onDataDisposable.dispose.mock.invocationCallOrder[0]).toBeLessThan( killSpy.mock.invocationCallOrder[0] ) - expect(onExitDisposable.dispose.mock.invocationCallOrder[0]).toBeLessThan( - killSpy.mock.invocationCallOrder[0] - ) + expect(onExitDisposable.dispose).not.toHaveBeenCalled() + + exitCb?.({ exitCode: -1 }) + expect(onExitDisposable.dispose).toHaveBeenCalledTimes(1) }) it('removes the previous orphan-cleanup listener from its original webContents', () => { @@ -10894,10 +11006,16 @@ describe('registerPtyHandlers', () => { // Why: guard against over-suppression — when no recovery reload is in flight the // sweep MUST still reclaim genuinely orphaned local PTYs. it('still sweeps orphaned local PTYs when no recovery reload is in flight', async () => { - const killSpy = vi.fn() + let exitCb: ((info: { exitCode: number }) => void) | undefined + const killSpy = vi.fn(() => { + queueMicrotask(() => exitCb?.({ exitCode: -1 })) + }) const proc = { onData: vi.fn(() => makeDisposable()), - onExit: vi.fn(() => makeDisposable()), + onExit: vi.fn((cb: (info: { exitCode: number }) => void) => { + exitCb = cb + return makeDisposable() + }), write: vi.fn(), resize: vi.fn(), kill: killSpy, @@ -10938,6 +11056,7 @@ describe('registerPtyHandlers', () => { // prior-load orphan. With the flag false the guard must NOT suppress the sweep. didFinishLoad() didFinishLoad() + await Promise.resolve() expect(killSpy).toHaveBeenCalled() expect(runtime.onPtyExit).toHaveBeenCalledWith(spawnResult.id, -1) @@ -11010,10 +11129,14 @@ describe('registerPtyHandlers', () => { expect(ids).toContain(ptyB.id) }) - it('clears PTY state even when kill reports the process is already gone', async () => { + it('retains PTY state when kill fails until physical exit arrives', async () => { + let exitCb: ((info: { exitCode: number }) => void) | undefined const proc = { onData: vi.fn(() => makeDisposable()), - onExit: vi.fn(() => makeDisposable()), + onExit: vi.fn((cb: (info: { exitCode: number }) => void) => { + exitCb = cb + return makeDisposable() + }), write: vi.fn(), resize: vi.fn(), kill: vi.fn(() => { @@ -11030,9 +11153,21 @@ describe('registerPtyHandlers', () => { rows: 24 })) as { id: string } - await handlers.get('pty:kill')!(null, { id: spawnResult.id }) + await expect(handlers.get('pty:kill')!(null, { id: spawnResult.id })).rejects.toThrow( + 'already dead' + ) - expect(await handlers.get('pty:hasChildProcesses')!(null, { id: spawnResult.id })).toBe(false) + expect((await getLocalPtyProvider().listProcesses()).map(({ id }) => id)).toContain( + spawnResult.id + ) + expect(openCodeClearPtyMock).not.toHaveBeenCalled() + expect(piClearPtyMock).not.toHaveBeenCalled() + + exitCb?.({ exitCode: -1 }) + + expect((await getLocalPtyProvider().listProcesses()).map(({ id }) => id)).not.toContain( + spawnResult.id + ) expect(openCodeClearPtyMock).toHaveBeenCalledWith(spawnResult.id) expect(piClearPtyMock).toHaveBeenCalledWith(spawnResult.id) }) diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 0dd30b24b1e..63e5fd99b3c 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -19,6 +19,7 @@ export { getBashShellReadyRcfileContent } from '../providers/local-pty-shell-rea import type { OrcaRuntimeService } from '../runtime/orca-runtime' import type { Store } from '../persistence' import type { GlobalSettings, TuiAgent } from '../../shared/types' +import { normalizeRuntimePathForComparison } from '../../shared/cross-platform-path' import { terminalOutputBacklogCapChars } from '../../shared/terminal-scrollback-policy' import type { PtyDeliveryWriteOff, @@ -108,6 +109,7 @@ import { type TerminalStartupCwdMissingDirFallback } from '../../shared/terminal-startup-cwd' import { isWslUncPath } from '../../shared/wsl-paths' +import { splitWorktreeIdForFilesystem } from '../../shared/worktree-id' import { clearMigrationUnsupportedPty, clearMigrationUnsupportedPtysForPaneKey @@ -116,6 +118,7 @@ import { parseWslPath } from '../wsl' import { mergePersistedWindowsPath } from '../pty/windows-environment-path' import { addOrcaWslInteropEnv } from '../pty/wsl-orca-env' import { PtyProducerFlowController } from './pty-producer-flow-control' +import { beginTerminalInstall } from './watcher-removal-gate' import { clearHiddenRendererPtyDeliveryState, getHiddenRendererPtyDeliveryDebug, @@ -1050,6 +1053,34 @@ function routesFreshSpawnsToLocalProvider( return (provider as FreshLocalFallbackProvider).routesFreshSpawnsToLocalProvider === true } +function beginPtySpawnForWorktree( + worktreeId: string | undefined, + cwd: string | undefined, + connectionId: string | null | undefined +): () => void { + const worktreePath = worktreeId + ? splitWorktreeIdForFilesystem(worktreeId)?.worktreePath + : undefined + const installPaths = new Map() + for (const candidate of [worktreePath, cwd]) { + if (candidate) { + installPaths.set(normalizeRuntimePathForComparison(candidate), candidate) + } + } + const finishes: (() => void)[] = [] + try { + for (const candidate of installPaths.values()) { + finishes.push(beginTerminalInstall(candidate, connectionId ?? undefined)) + } + } catch (error) { + // Why: the worktree ID and actual cwd can belong to different roots. If + // either is deleting, release any earlier admission before rejecting. + finishes.toReversed().forEach((finish) => finish()) + throw error + } + return () => finishes.toReversed().forEach((finish) => finish()) +} + /** Register an SSH PTY provider for a connection. */ export function registerSshPtyProvider(connectionId: string, provider: IPtyProvider): void { sshProviders.set(connectionId, provider) @@ -2865,13 +2896,9 @@ export function registerPtyHandlers( if (options?.isRecoveryReloadInFlight?.(mainWindow.webContents.id)) { return } - const killed = lp.killOrphanedPtys(generation - 1) - for (const { id } of killed) { - clearProviderPtyState(id) - ptyOwnership.delete(id) - markClaudePtyExited(id) - runtime?.onPtyExit(id, -1) - } + // Why: the retained provider onExit callback is the only physical-exit + // proof; it clears ownership and notifies runtime after the OS reaps it. + lp.killOrphanedPtys(generation - 1) } didFinishLoadWebContents = mainWindow.webContents mainWindow.webContents.on('did-finish-load', didFinishLoadHandler) @@ -3116,6 +3143,11 @@ export function registerPtyHandlers( if (existingPaneSpawn) { return await existingPaneSpawn.promise } + const finishTerminalInstall = beginPtySpawnForWorktree( + args.worktreeId, + cwd, + args.connectionId + ) const paneSpawnReservation = materializedPaneKey ? reservePaneSpawn(materializedPaneKey) : null @@ -3293,6 +3325,8 @@ export function registerPtyHandlers( // no-op once the reservation has already resolved. rejectPaneSpawnReservation(materializedPaneKey, paneSpawnReservation, err) throw err + } finally { + finishTerminalInstall() } }, write: (ptyId, data) => { @@ -4004,6 +4038,11 @@ export function registerPtyHandlers( if (existingPaneSpawn) { return await existingPaneSpawn.promise } + const finishTerminalInstall = beginPtySpawnForWorktree( + args.worktreeId, + cwd, + args.connectionId + ) const paneSpawnReservation = reservationPaneKey ? reservePaneSpawn(reservationPaneKey) : null const initiallyHidden = args.initiallyHidden === true // Why pre-spawn for daemon-host sessions (id minted up front): daemon @@ -4409,6 +4448,8 @@ export function registerPtyHandlers( // no-op once the reservation has already resolved. rejectPaneSpawnReservation(reservationPaneKey, paneSpawnReservation, err) throw err + } finally { + finishTerminalInstall() } } ) diff --git a/src/main/ipc/runtime-watcher-pending-assignment.test.ts b/src/main/ipc/runtime-watcher-pending-assignment.test.ts new file mode 100644 index 00000000000..e73468aa799 --- /dev/null +++ b/src/main/ipc/runtime-watcher-pending-assignment.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it, vi } from 'vitest' +import { RuntimeWatcherPendingAssignment } from './runtime-watcher-pending-assignment' + +describe('RuntimeWatcherPendingAssignment', () => { + it('removes ten thousand cancelled assignment callers while one anchor remains', async () => { + let resolveAssignment: (value: number) => void = () => undefined + const assignment = new RuntimeWatcherPendingAssignment( + new Promise((resolve) => { + resolveAssignment = resolve + }), + vi.fn(), + vi.fn() + ) + const anchor = assignment.wait({}, vi.fn()) + const controllers = Array.from({ length: 10_000 }, () => new AbortController()) + const cancelled = controllers.map((controller) => + assignment.wait({ signal: controller.signal }, vi.fn()).catch((error) => error) + ) + + for (const controller of controllers) { + controller.abort() + } + await Promise.all(cancelled) + + expect(assignment.waiterCount).toBe(1) + resolveAssignment(42) + await expect(anchor).resolves.toBe(42) + expect(assignment.waiterCount).toBe(0) + }) + + it('releases ownership for an already-aborted first caller', async () => { + const controller = new AbortController() + controller.abort() + const onNoWaiters = vi.fn() + const onSettled = vi.fn() + const assignment = new RuntimeWatcherPendingAssignment( + new Promise(() => {}), + onNoWaiters, + onSettled + ) + + await expect(assignment.wait({ signal: controller.signal }, vi.fn())).rejects.toMatchObject({ + code: 'subscribe_aborted' + }) + + expect(onNoWaiters).toHaveBeenCalledOnce() + expect(onSettled).toHaveBeenCalledOnce() + expect(assignment.waiterCount).toBe(0) + }) +}) diff --git a/src/main/ipc/runtime-watcher-pending-assignment.ts b/src/main/ipc/runtime-watcher-pending-assignment.ts new file mode 100644 index 00000000000..9a638fe1b7d --- /dev/null +++ b/src/main/ipc/runtime-watcher-pending-assignment.ts @@ -0,0 +1,64 @@ +import { WatcherProcessFailure } from './parcel-watcher-process-failure' +import type { WatcherProcessHooks } from './parcel-watcher-process-subscription' +import { PromiseSettlementWaiters } from '../../shared/promise-settlement-waiters' + +const DEFAULT_QUARANTINE_WAIT_TIMEOUT_MS = 60_000 + +export class RuntimeWatcherPendingAssignment { + private readonly settlementWaiters: PromiseSettlementWaiters + private settledNotified = false + + constructor( + basePromise: Promise, + private readonly onNoWaiters: () => void, + private readonly onSettled: () => void + ) { + this.settlementWaiters = new PromiseSettlementWaiters(basePromise, () => this.notifySettled()) + } + + get waiterCount(): number { + return this.settlementWaiters.waiterCount + } + + wait(hooks: WatcherProcessHooks, onGranted: (assignment: T) => void): Promise { + return this.settlementWaiters.wait({ + signal: hooks.signal, + timeoutMs: hooks.subscribeTimeoutMs ?? DEFAULT_QUARANTINE_WAIT_TIMEOUT_MS, + createAbortError: createAssignmentAbortError, + createTimeoutError: createAssignmentTimeoutError, + onFulfilled: onGranted, + onAbandon: () => { + if (this.settlementWaiters.waiterCount === 0) { + // Why: delete the logically abandoned assignment synchronously so a + // same-turn replacement cannot join its soon-to-reject promise. + this.notifySettled() + this.onNoWaiters() + } + } + }) + } + + private notifySettled(): void { + if (this.settledNotified) { + return + } + this.settledNotified = true + this.onSettled() + } +} + +function createAssignmentAbortError(): WatcherProcessFailure { + return new WatcherProcessFailure( + 'file watcher subscription aborted', + 'subscription', + 'subscribe_aborted' + ) +} + +function createAssignmentTimeoutError(): WatcherProcessFailure { + return new WatcherProcessFailure( + 'file watcher subscription timed out waiting for quarantine capacity', + 'subscription', + 'subscribe_timeout' + ) +} diff --git a/src/main/ipc/runtime-watcher-pool-lifecycle.ts b/src/main/ipc/runtime-watcher-pool-lifecycle.ts new file mode 100644 index 00000000000..041d9e7675f --- /dev/null +++ b/src/main/ipc/runtime-watcher-pool-lifecycle.ts @@ -0,0 +1,41 @@ +import { WatcherProcessFailure } from './parcel-watcher-process-failure' + +export class RuntimeWatcherPoolLifecycle { + readonly isolatedRoots = new Set() + readonly failedQuarantineRoots = new Set() + private disposed = false + + assertActive(): void { + if (this.disposed) { + throw new WatcherProcessFailure( + 'file watcher supervisor disposed', + 'supervisor', + 'supervisor_disposed' + ) + } + } + + isIsolated(root: string): boolean { + return this.isolatedRoots.has(root) + } + + quarantineOrFuse(root: string, isolated: boolean): void { + // Why: one quarantine attempt is the recovery budget; a second failure is fused. + if (isolated) { + this.isolatedRoots.delete(root) + this.failedQuarantineRoots.add(root) + return + } + this.isolatedRoots.add(root) + } + + dispose(): void { + this.disposed = true + this.isolatedRoots.clear() + this.failedQuarantineRoots.clear() + } + + reset(): void { + this.disposed = false + } +} diff --git a/src/main/ipc/runtime-watcher-pool-state.ts b/src/main/ipc/runtime-watcher-pool-state.ts new file mode 100644 index 00000000000..e6c016b9844 --- /dev/null +++ b/src/main/ipc/runtime-watcher-pool-state.ts @@ -0,0 +1,22 @@ +import type { WatcherProcessSupervisor } from './parcel-watcher-process-supervisor' + +export type RuntimeWatcherPoolSupervisor = Pick + +export type RuntimeWatcherPoolSlot = { + supervisor: RuntimeWatcherPoolSupervisor + roots: Set + isolated: boolean + retired: boolean + disposed: boolean +} + +export type RuntimeWatcherPoolAssignment = { + slot: RuntimeWatcherPoolSlot + leases: number +} + +export type RuntimeWatcherProcessPoolOptions = { + maxSharedSupervisors?: number + maxQuarantineSupervisors?: number + createSupervisor?: () => RuntimeWatcherPoolSupervisor +} diff --git a/src/main/ipc/runtime-watcher-predecessor-barriers.ts b/src/main/ipc/runtime-watcher-predecessor-barriers.ts new file mode 100644 index 00000000000..f5301eb5432 --- /dev/null +++ b/src/main/ipc/runtime-watcher-predecessor-barriers.ts @@ -0,0 +1,35 @@ +import type { WatcherProcessFailure } from './parcel-watcher-process-failure' + +/** Retain exact failed-child ownership until its native handles are gone. */ +export class RuntimeWatcherPredecessorBarriers { + private readonly failures = new Map() + + throwIfRetained(rootPath: string): void { + const failure = this.failures.get(rootPath) + if (failure) { + throw failure + } + } + + retain(rootPaths: Iterable, failure: WatcherProcessFailure): void { + if (!failure.physicalExit) { + return + } + const retainedRoots = Array.from(rootPaths) + for (const rootPath of retainedRoots) { + this.failures.set(rootPath, failure) + } + const clearExactFailure = (): void => { + for (const rootPath of retainedRoots) { + if (this.failures.get(rootPath) === failure) { + this.failures.delete(rootPath) + } + } + } + void failure.physicalExit.then(clearExactFailure, clearExactFailure) + } + + clear(): void { + this.failures.clear() + } +} diff --git a/src/main/ipc/runtime-watcher-process-pool.test.ts b/src/main/ipc/runtime-watcher-process-pool.test.ts index 5cd624589b2..c722b5ccdd3 100644 --- a/src/main/ipc/runtime-watcher-process-pool.test.ts +++ b/src/main/ipc/runtime-watcher-process-pool.test.ts @@ -13,6 +13,14 @@ type InstalledSubscription = { unsubscribe: ReturnType Promise>> } +function deferred(): { promise: Promise; resolve: () => void } { + let resolve = (): void => undefined + const promise = new Promise((nextResolve) => { + resolve = nextResolve + }) + return { promise, resolve } +} + class FakeSupervisor { readonly subscriptions: InstalledSubscription[] = [] readonly dispose = vi.fn() @@ -40,7 +48,6 @@ describe('RuntimeWatcherProcessPool', () => { beforeEach(() => { supervisors = [] pool = new RuntimeWatcherProcessPool({ - maxSharedSupervisors: 4, createSupervisor: () => { const supervisor = new FakeSupervisor() supervisors.push(supervisor) @@ -49,14 +56,14 @@ describe('RuntimeWatcherProcessPool', () => { }) }) - it('spreads roots across a bounded healthy pool before sharing children', async () => { + it('shares one healthy child until a fault requires quarantine isolation', async () => { for (const dir of ['/a', '/b', '/c', '/d', '/e']) { await pool.subscribe(dir, vi.fn(), {}, {}) } - expect(supervisors).toHaveLength(4) + expect(supervisors).toHaveLength(1) expect(supervisors.map((supervisor) => supervisor.subscriptions.map(({ dir }) => dir))).toEqual( - [['/a', '/e'], ['/b'], ['/c'], ['/d']] + [['/a', '/b', '/c', '/d', '/e']] ) }) @@ -104,7 +111,30 @@ describe('RuntimeWatcherProcessPool', () => { expect(supervisors[2].subscriptions.map(({ dir }) => dir)).toEqual(['/b']) }) - it('bounds quarantine supervisors when a fused shard contains many roots', async () => { + it('does not assign a replacement until the failed child physically exits', async () => { + const physicalExit = deferred() + await pool.subscribe('/held', vi.fn(), {}, {}) + const failure = new WatcherProcessFailure( + 'file watcher process did not exit after termination deadline', + 'supervisor', + 'process_unavailable', + physicalExit.promise + ) + + supervisors[0].subscriptions[0].hooks.onTerminalError?.(failure) + + await expect(pool.subscribe('/held', vi.fn(), {}, {})).rejects.toBe(failure) + expect(supervisors).toHaveLength(1) + + physicalExit.resolve() + await physicalExit.promise + await expect(pool.subscribe('/held', vi.fn(), {}, {})).resolves.toBeDefined() + + expect(supervisors).toHaveLength(2) + expect(supervisors[1].subscriptions.map(({ dir }) => dir)).toEqual(['/held']) + }) + + it('bounds single-root quarantine supervisors and their wait queue', async () => { pool = new RuntimeWatcherProcessPool({ maxSharedSupervisors: 1, createSupervisor: () => { @@ -113,7 +143,7 @@ describe('RuntimeWatcherProcessPool', () => { return supervisor } }) - const roots = Array.from({ length: 12 }, (_, index) => `/root-${index}`) + const roots = Array.from({ length: 9 }, (_, index) => `/root-${index}`) for (const root of roots) { await pool.subscribe(root, vi.fn(), {}, {}) } @@ -127,14 +157,278 @@ describe('RuntimeWatcherProcessPool', () => { for (const installed of failedShard.subscriptions) { installed.hooks.onTerminalError?.(failure) } - for (const root of roots) { - await pool.subscribe(root, vi.fn(), {}, {}) - } + const recoveries = roots.map((root) => pool.subscribe(root, vi.fn(), {}, {})) + const queueOverflow = expect(recoveries[8]).rejects.toThrow('quarantine capacity exhausted') + const active = await Promise.all(recoveries.slice(0, 4)) expect(supervisors).toHaveLength(5) expect(supervisors.slice(1).map(({ subscriptions }) => subscriptions.length)).toEqual([ - 3, 3, 3, 3 + 1, 1, 1, 1 ]) + + for (let index = 0; index < active.length; index++) { + await active[index].unsubscribe() + await expect(recoveries[index + 4]).resolves.toBeDefined() + } + await queueOverflow + + expect(supervisors).toHaveLength(5) + expect(supervisors.slice(1).map(({ subscriptions }) => subscriptions.length)).toEqual([ + 2, 2, 2, 2 + ]) + }) + + it('does not fail a healthy queued root when a quarantine root fuses again', async () => { + pool = new RuntimeWatcherProcessPool({ + maxSharedSupervisors: 1, + maxQuarantineSupervisors: 1, + createSupervisor: () => { + const supervisor = new FakeSupervisor() + supervisors.push(supervisor) + return supervisor + } + }) + const failure = new WatcherProcessFailure( + 'file watcher process crashed repeatedly', + 'supervisor', + 'supervisor_crash_fuse' + ) + await pool.subscribe('/poison', vi.fn(), {}, {}) + await pool.subscribe('/healthy', vi.fn(), {}, {}) + for (const installed of supervisors[0].subscriptions) { + installed.hooks.onTerminalError?.(failure) + } + + await pool.subscribe('/poison', vi.fn(), {}, {}) + const healthyRecovery = pool.subscribe('/healthy', vi.fn(), {}, {}) + supervisors[1].subscriptions[0].hooks.onTerminalError?.(failure) + + await expect(healthyRecovery).resolves.toBeDefined() + await expect(pool.subscribe('/poison', vi.fn(), {}, {})).rejects.toThrow( + 'failed again in quarantine' + ) + expect(supervisors[1].subscriptions.map(({ dir }) => dir)).toEqual(['/poison']) + expect(supervisors[2].subscriptions.map(({ dir }) => dir)).toEqual(['/healthy']) + }) + + it('bounds how long a root can wait for quarantine capacity', async () => { + vi.useFakeTimers() + try { + pool = new RuntimeWatcherProcessPool({ + maxSharedSupervisors: 1, + maxQuarantineSupervisors: 1, + createSupervisor: () => { + const supervisor = new FakeSupervisor() + supervisors.push(supervisor) + return supervisor + } + }) + const failure = new WatcherProcessFailure( + 'file watcher process crashed repeatedly', + 'supervisor', + 'supervisor_crash_fuse' + ) + await pool.subscribe('/occupied', vi.fn(), {}, {}) + await pool.subscribe('/waiting', vi.fn(), {}, {}) + for (const installed of supervisors[0].subscriptions) { + installed.hooks.onTerminalError?.(failure) + } + await pool.subscribe('/occupied', vi.fn(), {}, {}) + + const waiting = pool.subscribe('/waiting', vi.fn(), {}, { subscribeTimeoutMs: 25 }) + const timedOut = expect(waiting).rejects.toThrow('timed out waiting for quarantine capacity') + await vi.advanceTimersByTimeAsync(25) + + await timedOut + expect(supervisors).toHaveLength(2) + } finally { + pool.dispose() + expect(vi.getTimerCount()).toBe(0) + vi.useRealTimers() + } + }) + + it('dedupes concurrent same-root assignments waiting for quarantine capacity', async () => { + pool = new RuntimeWatcherProcessPool({ + maxSharedSupervisors: 1, + maxQuarantineSupervisors: 1, + createSupervisor: () => { + const supervisor = new FakeSupervisor() + supervisors.push(supervisor) + return supervisor + } + }) + const failure = new WatcherProcessFailure( + 'file watcher process crashed repeatedly', + 'supervisor', + 'supervisor_crash_fuse' + ) + await pool.subscribe('/occupied', vi.fn(), {}, {}) + await pool.subscribe('/same', vi.fn(), {}, {}) + for (const installed of supervisors[0].subscriptions) { + installed.hooks.onTerminalError?.(failure) + } + const occupied = await pool.subscribe('/occupied', vi.fn(), {}, {}) + const first = pool.subscribe('/same', vi.fn(), {}, {}) + const second = pool.subscribe('/same', vi.fn(), {}, {}) + + await occupied.unsubscribe() + const [firstSubscription, secondSubscription] = await Promise.all([first, second]) + + expect(supervisors).toHaveLength(2) + expect(supervisors[1].subscriptions.map(({ dir }) => dir)).toEqual([ + '/occupied', + '/same', + '/same' + ]) + await firstSubscription.unsubscribe() + await secondSubscription.unsubscribe() + }) + + it('keeps a healthy same-root quarantine waiter when the first waiter aborts', async () => { + pool = new RuntimeWatcherProcessPool({ + maxSharedSupervisors: 1, + maxQuarantineSupervisors: 1, + createSupervisor: () => { + const supervisor = new FakeSupervisor() + supervisors.push(supervisor) + return supervisor + } + }) + const failure = new WatcherProcessFailure( + 'file watcher process crashed repeatedly', + 'supervisor', + 'supervisor_crash_fuse' + ) + await pool.subscribe('/occupied', vi.fn(), {}, {}) + await pool.subscribe('/same', vi.fn(), {}, {}) + for (const installed of supervisors[0].subscriptions) { + installed.hooks.onTerminalError?.(failure) + } + const occupied = await pool.subscribe('/occupied', vi.fn(), {}, {}) + const firstController = new AbortController() + const first = pool.subscribe('/same', vi.fn(), {}, { signal: firstController.signal }) + const second = pool.subscribe('/same', vi.fn(), {}, {}) + + firstController.abort() + await expect(first).rejects.toMatchObject({ code: 'subscribe_aborted' }) + await occupied.unsubscribe() + const secondSubscription = await second + + expect(supervisors[1].subscriptions.map(({ dir }) => dir)).toEqual(['/occupied', '/same']) + await secondSubscription.unsubscribe() + }) + + it('does not let an aborted zero-waiter assignment poison a same-turn replacement', async () => { + pool = new RuntimeWatcherProcessPool({ + maxSharedSupervisors: 1, + maxQuarantineSupervisors: 1, + createSupervisor: () => { + const supervisor = new FakeSupervisor() + supervisors.push(supervisor) + return supervisor + } + }) + const failure = new WatcherProcessFailure( + 'file watcher process crashed repeatedly', + 'supervisor', + 'supervisor_crash_fuse' + ) + await pool.subscribe('/occupied', vi.fn(), {}, {}) + await pool.subscribe('/same', vi.fn(), {}, {}) + for (const installed of supervisors[0].subscriptions) { + installed.hooks.onTerminalError?.(failure) + } + const occupied = await pool.subscribe('/occupied', vi.fn(), {}, {}) + const controller = new AbortController() + const abandoned = pool.subscribe('/same', vi.fn(), {}, { signal: controller.signal }) + + controller.abort() + const replacement = pool.subscribe('/same', vi.fn(), {}, {}) + await expect(abandoned).rejects.toMatchObject({ code: 'subscribe_aborted' }) + await occupied.unsubscribe() + const replacementSubscription = await replacement + + expect(supervisors[1].subscriptions.map(({ dir }) => dir)).toEqual(['/occupied', '/same']) + await replacementSubscription.unsubscribe() + }) + + it('rejects an aborted later same-root waiter without cancelling the first', async () => { + pool = new RuntimeWatcherProcessPool({ + maxSharedSupervisors: 1, + maxQuarantineSupervisors: 1, + createSupervisor: () => { + const supervisor = new FakeSupervisor() + supervisors.push(supervisor) + return supervisor + } + }) + const failure = new WatcherProcessFailure( + 'file watcher process crashed repeatedly', + 'supervisor', + 'supervisor_crash_fuse' + ) + await pool.subscribe('/occupied', vi.fn(), {}, {}) + await pool.subscribe('/same', vi.fn(), {}, {}) + for (const installed of supervisors[0].subscriptions) { + installed.hooks.onTerminalError?.(failure) + } + const occupied = await pool.subscribe('/occupied', vi.fn(), {}, {}) + const first = pool.subscribe('/same', vi.fn(), {}, {}) + const secondController = new AbortController() + const second = pool.subscribe('/same', vi.fn(), {}, { signal: secondController.signal }) + + secondController.abort() + await expect(second).rejects.toMatchObject({ code: 'subscribe_aborted' }) + let firstSettled = false + void first.finally(() => { + firstSettled = true + }) + await Promise.resolve() + expect(firstSettled).toBe(false) + + await occupied.unsubscribe() + const firstSubscription = await first + expect(supervisors[1].subscriptions.map(({ dir }) => dir)).toEqual(['/occupied', '/same']) + await firstSubscription.unsubscribe() + }) + + it('releases a granted assignment when its last waiter aborts before lease publication', async () => { + pool = new RuntimeWatcherProcessPool({ + maxSharedSupervisors: 1, + maxQuarantineSupervisors: 1, + createSupervisor: () => { + const supervisor = new FakeSupervisor() + supervisors.push(supervisor) + return supervisor + } + }) + const failure = new WatcherProcessFailure( + 'file watcher process crashed repeatedly', + 'supervisor', + 'supervisor_crash_fuse' + ) + for (const root of ['/occupied', '/aborted', '/following']) { + await pool.subscribe(root, vi.fn(), {}, {}) + } + for (const installed of supervisors[0].subscriptions) { + installed.hooks.onTerminalError?.(failure) + } + const occupied = await pool.subscribe('/occupied', vi.fn(), {}, {}) + const controller = new AbortController() + const aborted = pool.subscribe('/aborted', vi.fn(), {}, { signal: controller.signal }) + + const release = occupied.unsubscribe() + queueMicrotask(() => controller.abort()) + await release + + await expect(aborted).rejects.toMatchObject({ code: 'subscribe_aborted' }) + const following = pool.subscribe('/following', vi.fn(), {}, { subscribeTimeoutMs: 25 }) + const followingSubscription = await following + expect(supervisors).toHaveLength(3) + expect(supervisors[2].subscriptions.map(({ dir }) => dir)).toEqual(['/following']) + expect(supervisors[1].dispose).toHaveBeenCalledTimes(1) + await followingSubscription.unsubscribe() }) it('moves an explicitly timed-out root into quarantine for its retry', async () => { @@ -324,5 +618,7 @@ describe('RuntimeWatcherProcessPool', () => { pool.resetForTest() expect(supervisors.every((supervisor) => supervisor.dispose.mock.calls.length === 1)).toBe(true) + await expect(pool.subscribe('/after-reset', vi.fn(), {}, {})).resolves.toBeDefined() + expect(supervisors).toHaveLength(2) }) }) diff --git a/src/main/ipc/runtime-watcher-process-pool.ts b/src/main/ipc/runtime-watcher-process-pool.ts index b4e46cb3d2c..4bafc38208d 100644 --- a/src/main/ipc/runtime-watcher-process-pool.ts +++ b/src/main/ipc/runtime-watcher-process-pool.ts @@ -6,44 +6,41 @@ import type { WatcherProcessHooks, WatcherProcessSubscription } from './parcel-watcher-process-subscription' +import { RuntimeWatcherPendingAssignment } from './runtime-watcher-pending-assignment' +import { RuntimeWatcherPoolLifecycle } from './runtime-watcher-pool-lifecycle' +import { RuntimeWatcherPredecessorBarriers } from './runtime-watcher-predecessor-barriers' +import { RuntimeWatcherQuarantineQueue } from './runtime-watcher-quarantine-queue' +import { handleRuntimeWatcherSubscriptionFailure } from './runtime-watcher-subscription-failure' +import type { + RuntimeWatcherPoolAssignment, + RuntimeWatcherPoolSlot, + RuntimeWatcherPoolSupervisor, + RuntimeWatcherProcessPoolOptions +} from './runtime-watcher-pool-state' -const DEFAULT_MAX_SHARED_SUPERVISORS = 4 +export type { RuntimeWatcherProcessPoolOptions } from './runtime-watcher-pool-state' + +// Why: extra ~50 MiB processes are justified only when quarantine isolates a fault. +const DEFAULT_MAX_SHARED_SUPERVISORS = 1 const DEFAULT_MAX_QUARANTINE_SUPERVISORS = 4 -type WatcherSupervisor = Pick - -type RuntimeWatcherPoolSlot = { - supervisor: WatcherSupervisor - roots: Set - isolated: boolean - retired: boolean - disposed: boolean -} - -type RuntimeWatcherPoolAssignment = { - slot: RuntimeWatcherPoolSlot - leases: number -} - -export type RuntimeWatcherProcessPoolOptions = { - maxSharedSupervisors?: number - maxQuarantineSupervisors?: number - createSupervisor?: () => WatcherSupervisor -} - /** - * Share a bounded set of watcher children while healthy, then move roots from - * a failed shard into a separate bounded quarantine pool. + * Share children while healthy, then move failed roots into bounded quarantine slots. */ export class RuntimeWatcherProcessPool { private readonly maxSharedSupervisors: number private readonly maxQuarantineSupervisors: number - private readonly createSupervisor: () => WatcherSupervisor + private readonly createSupervisor: () => RuntimeWatcherPoolSupervisor private readonly activeSlots = new Set() private readonly allSlots = new Set() private readonly assignments = new Map() - private readonly isolatedRoots = new Set() - private readonly failedQuarantineRoots = new Set() + private readonly pendingAssignments = new Map< + string, + RuntimeWatcherPendingAssignment + >() + private readonly lifecycle = new RuntimeWatcherPoolLifecycle() + private readonly predecessorBarriers = new RuntimeWatcherPredecessorBarriers() + private readonly quarantineQueue: RuntimeWatcherQuarantineQueue constructor(options: RuntimeWatcherProcessPoolOptions = {}) { this.maxSharedSupervisors = Math.max( @@ -54,6 +51,7 @@ export class RuntimeWatcherProcessPool { 1, options.maxQuarantineSupervisors ?? DEFAULT_MAX_QUARANTINE_SUPERVISORS ) + this.quarantineQueue = new RuntimeWatcherQuarantineQueue(this.maxQuarantineSupervisors) this.createSupervisor = options.createSupervisor ?? (() => new WatcherProcessSupervisor()) } @@ -63,7 +61,11 @@ export class RuntimeWatcherProcessPool { opts: WatcherProcessSubscribeOptions, hooks: WatcherProcessHooks = {} ): Promise { - const assignment = this.assignmentForRoot(dir) + this.lifecycle.assertActive() + const assignmentOrPromise = this.assignmentForRoot(dir, hooks) + const assignment = + assignmentOrPromise instanceof Promise ? await assignmentOrPromise : assignmentOrPromise + this.lifecycle.assertActive() const { slot } = assignment let assignmentReleased = false const releaseAssignment = (): void => { @@ -74,11 +76,12 @@ export class RuntimeWatcherProcessPool { this.releaseRoot(assignment, dir) } const onTerminalError = (error: Error): void => { - if (isWatcherProcessFailure(error) && error.scope === 'supervisor') { - this.retireSlot(slot) - } else { - this.releaseFailedRoot(assignment, dir, error, releaseAssignment) - } + handleRuntimeWatcherSubscriptionFailure( + error, + releaseAssignment, + (failure) => this.retireSlot(slot, failure), + () => this.lifecycle.quarantineOrFuse(dir, slot.isolated) + ) hooks.onTerminalError?.(error) } let subscription: WatcherProcessSubscription @@ -88,20 +91,29 @@ export class RuntimeWatcherProcessPool { onTerminalError }) } catch (error) { - if (isWatcherProcessFailure(error) && error.scope === 'supervisor') { - this.retireSlot(slot) - } else { - this.releaseFailedRoot(assignment, dir, error, releaseAssignment) - } + handleRuntimeWatcherSubscriptionFailure( + error, + releaseAssignment, + (failure) => this.retireSlot(slot, failure), + () => this.lifecycle.quarantineOrFuse(dir, slot.isolated) + ) throw error } let unsubscribePromise: Promise | undefined return { unsubscribe: (): Promise => { - unsubscribePromise ??= subscription.unsubscribe().finally(() => { - releaseAssignment() - }) + unsubscribePromise ??= subscription.unsubscribe().then( + () => releaseAssignment(), + (error: unknown) => { + if (isWatcherProcessFailure(error) && error.scope === 'supervisor') { + this.retireSlot(slot, error) + } else { + releaseAssignment() + } + throw error + } + ) return unsubscribePromise } } @@ -109,33 +121,46 @@ export class RuntimeWatcherProcessPool { /** Kill every pooled watcher child (production shutdown or test reset). */ dispose(): void { + this.lifecycle.dispose() // disposeSlot deletes the current slot from allSlots; deleting the // in-progress element during Set iteration is safe, so no snapshot needed. + const disposedError = new WatcherProcessFailure( + 'file watcher supervisor disposed', + 'supervisor', + 'supervisor_disposed' + ) + this.quarantineQueue.failAll(disposedError) for (const slot of this.allSlots) { this.disposeSlot(slot) } this.activeSlots.clear() this.allSlots.clear() this.assignments.clear() - this.isolatedRoots.clear() - this.failedQuarantineRoots.clear() + this.pendingAssignments.clear() + this.predecessorBarriers.clear() } resetForTest(): void { this.dispose() + this.lifecycle.reset() } forgetRoot(dir: string): void { // Physical subscriptions release their assignment through unsubscribe or // terminal callbacks. This clears fault history after setup gives up. if (!this.assignments.has(dir)) { - this.isolatedRoots.delete(dir) - this.failedQuarantineRoots.delete(dir) + this.quarantineQueue.failRoot(dir) + this.lifecycle.isolatedRoots.delete(dir) + this.lifecycle.failedQuarantineRoots.delete(dir) } } - private assignmentForRoot(dir: string): RuntimeWatcherPoolAssignment { - if (this.failedQuarantineRoots.has(dir)) { + private assignmentForRoot( + dir: string, + hooks: WatcherProcessHooks + ): RuntimeWatcherPoolAssignment | Promise { + this.predecessorBarriers.throwIfRetained(dir) + if (this.lifecycle.failedQuarantineRoots.has(dir)) { throw new WatcherProcessFailure( 'file watcher process failed again in quarantine', 'supervisor', @@ -147,8 +172,49 @@ export class RuntimeWatcherProcessPool { assigned.leases++ return assigned } - const slot = this.isolatedRoots.has(dir) ? this.quarantineSlot() : this.sharedSlot() - const assignment = { slot, leases: 1 } + const pending = this.pendingAssignments.get(dir) + if (pending) { + return pending.wait(hooks, (assignment) => assignment.leases++) + } + const slot = this.lifecycle.isIsolated(dir) ? this.quarantineSlot(dir) : this.sharedSlot() + if (slot instanceof Promise) { + let abandoned = false + let grantedAssignment: RuntimeWatcherPoolAssignment | undefined + const basePromise = slot.then((resolvedSlot) => { + grantedAssignment = this.assignRootToSlot(dir, resolvedSlot, 0) + if (abandoned) { + this.releaseRoot(grantedAssignment, dir, false) + } + return grantedAssignment + }) + const pendingAssignment = new RuntimeWatcherPendingAssignment( + basePromise, + () => { + abandoned = true + if (grantedAssignment) { + this.releaseRoot(grantedAssignment, dir, false) + } else { + this.quarantineQueue.failRoot(dir) + } + }, + () => { + if (this.pendingAssignments.get(dir) === pendingAssignment) { + this.pendingAssignments.delete(dir) + } + } + ) + this.pendingAssignments.set(dir, pendingAssignment) + return pendingAssignment.wait(hooks, (assignment) => assignment.leases++) + } + return this.assignRootToSlot(dir, slot) + } + + private assignRootToSlot( + dir: string, + slot: RuntimeWatcherPoolSlot, + leases = 1 + ): RuntimeWatcherPoolAssignment { + const assignment = { slot, leases } slot.roots.add(dir) this.assignments.set(dir, assignment) return assignment @@ -166,16 +232,14 @@ export class RuntimeWatcherProcessPool { ) } - private quarantineSlot(): RuntimeWatcherPoolSlot { + private quarantineSlot(dir: string): RuntimeWatcherPoolSlot | Promise { const quarantineSlots = Array.from(this.activeSlots).filter( (slot) => slot.isolated && !slot.retired ) if (quarantineSlots.length < this.maxQuarantineSupervisors) { return this.createSlot(true) } - return quarantineSlots.reduce((leastLoaded, candidate) => - candidate.roots.size < leastLoaded.roots.size ? candidate : leastLoaded - ) + return this.quarantineQueue.wait(dir) } private createSlot(isolated: boolean): RuntimeWatcherPoolSlot { @@ -191,65 +255,64 @@ export class RuntimeWatcherProcessPool { return slot } - private retireSlot(slot: RuntimeWatcherPoolSlot): void { + private retireSlot(slot: RuntimeWatcherPoolSlot, error: WatcherProcessFailure): void { if (slot.retired) { return } slot.retired = true this.activeSlots.delete(slot) + // Why: a replacement cannot safely reacquire a root while the failed child + // may still own its native watcher handle (notably on Windows). + this.predecessorBarriers.retain(slot.roots, error) for (const root of slot.roots) { if (this.assignments.get(root)?.slot === slot) { this.assignments.delete(root) } - this.quarantineOrFuse(root, slot.isolated) + this.lifecycle.quarantineOrFuse(root, slot.isolated) } slot.roots.clear() // Why: failAllSubscriptions is still iterating callbacks; defer disposal // so every logical root receives the supervisor failure first. - queueMicrotask(() => this.disposeSlot(slot)) + queueMicrotask(() => { + this.disposeSlot(slot) + this.drainQuarantineWaiters() + }) } - private releaseRoot(assignment: RuntimeWatcherPoolAssignment, dir: string): void { + private releaseRoot( + assignment: RuntimeWatcherPoolAssignment, + dir: string, + releaseLease = true + ): void { if (this.assignments.get(dir) !== assignment) { return } - assignment.leases-- + if (releaseLease) { + assignment.leases-- + } if (assignment.leases > 0) { return } const { slot } = assignment this.assignments.delete(dir) slot.roots.delete(dir) - this.isolatedRoots.delete(dir) + this.lifecycle.isolatedRoots.delete(dir) if (slot.isolated && slot.roots.size === 0 && !slot.retired) { - this.activeSlots.delete(slot) - this.disposeSlot(slot) + if (!this.quarantineQueue.grantNext(slot)) { + this.activeSlots.delete(slot) + this.disposeSlot(slot) + } } } - private releaseFailedRoot( - assignment: RuntimeWatcherPoolAssignment, - dir: string, - error: unknown, - releaseAssignment: () => void - ): void { - releaseAssignment() - if (!isWatcherProcessFailure(error) || error.code !== 'subscribe_timeout') { - return + private drainQuarantineWaiters(): void { + while ( + this.quarantineQueue.length > 0 && + Array.from(this.activeSlots).filter((slot) => slot.isolated && !slot.retired).length < + this.maxQuarantineSupervisors + ) { + this.quarantineQueue.grantNext(this.createSlot(true)) } - this.quarantineOrFuse(dir, assignment.slot.isolated) - } - - // Why: one bounded quarantine attempt is the recovery budget per watch - // lifetime; an already-isolated root that fails again is fused, not re-isolated, - // so it cannot spawn another child generation. - private quarantineOrFuse(dir: string, isolated: boolean): void { - if (isolated) { - this.isolatedRoots.delete(dir) - this.failedQuarantineRoots.add(dir) - return - } - this.isolatedRoots.add(dir) } private disposeSlot(slot: RuntimeWatcherPoolSlot): void { diff --git a/src/main/ipc/runtime-watcher-quarantine-queue.ts b/src/main/ipc/runtime-watcher-quarantine-queue.ts new file mode 100644 index 00000000000..e326b299943 --- /dev/null +++ b/src/main/ipc/runtime-watcher-quarantine-queue.ts @@ -0,0 +1,77 @@ +import { WatcherProcessFailure } from './parcel-watcher-process-failure' + +type QuarantineWaiter = { + dir: string + grant: (value: T) => void + fail: (error: WatcherProcessFailure) => void +} + +export class RuntimeWatcherQuarantineQueue { + private readonly waiters: QuarantineWaiter[] = [] + + constructor(private readonly capacity: number) {} + + get length(): number { + return this.waiters.length + } + + wait(dir: string): Promise { + if (this.waiters.length >= this.capacity) { + return Promise.reject( + new WatcherProcessFailure( + 'file watcher quarantine capacity exhausted', + 'supervisor', + 'process_unavailable' + ) + ) + } + return new Promise((resolve, reject) => { + let settled = false + const finish = (callback: () => void): void => { + if (settled) { + return + } + settled = true + const index = this.waiters.indexOf(waiter) + if (index >= 0) { + this.waiters.splice(index, 1) + } + callback() + } + const waiter: QuarantineWaiter = { + dir, + grant: (value) => finish(() => resolve(value)), + fail: (error) => finish(() => reject(error)) + } + this.waiters.push(waiter) + }) + } + + grantNext(value: T): boolean { + const waiter = this.waiters.shift() + waiter?.grant(value) + return Boolean(waiter) + } + + failRoot(dir: string): void { + for (let index = this.waiters.length - 1; index >= 0; index--) { + if (this.waiters[index].dir === dir) { + this.waiters[index].fail(createQuarantineAbortError()) + } + } + } + + failAll(error: WatcherProcessFailure): void { + for (const waiter of this.waiters.splice(0)) { + waiter.fail(error) + } + } +} + +function createQuarantineAbortError(): WatcherProcessFailure { + return new WatcherProcessFailure( + 'file watcher subscription aborted', + 'subscription', + 'subscribe_aborted' + ) +} diff --git a/src/main/ipc/runtime-watcher-subscription-failure.ts b/src/main/ipc/runtime-watcher-subscription-failure.ts new file mode 100644 index 00000000000..5e238cab8ed --- /dev/null +++ b/src/main/ipc/runtime-watcher-subscription-failure.ts @@ -0,0 +1,20 @@ +import { + isWatcherProcessFailure, + type WatcherProcessFailure +} from './parcel-watcher-process-failure' + +export function handleRuntimeWatcherSubscriptionFailure( + error: unknown, + releaseAssignment: () => void, + retireSupervisor: (failure: WatcherProcessFailure) => void, + quarantineOrFuseRoot: () => void +): void { + if (isWatcherProcessFailure(error) && error.scope === 'supervisor') { + retireSupervisor(error) + return + } + releaseAssignment() + if (isWatcherProcessFailure(error) && error.code === 'subscribe_timeout') { + quarantineOrFuseRoot() + } +} diff --git a/src/main/ipc/watcher-removal-gate.test.ts b/src/main/ipc/watcher-removal-gate.test.ts new file mode 100644 index 00000000000..5bb7bc50f7f --- /dev/null +++ b/src/main/ipc/watcher-removal-gate.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from 'vitest' +import { + acquireWatcherRemovalGate, + beginTerminalInstall, + beginWatcherInstall, + TerminalRemovalInProgressError, + WatcherRemovalInProgressError +} from './watcher-removal-gate' + +describe('watcher removal gate', () => { + it('waits for an existing install and rejects later equivalent-path installs', async () => { + const finishInstall = beginWatcherInstall('C:\\Repo') + const removal = acquireWatcherRemovalGate('c:/repo/') + let ready = false + void removal.ready.then(() => { + ready = true + }) + + await Promise.resolve() + expect(ready).toBe(false) + expect(() => beginWatcherInstall('C:/REPO')).toThrow(WatcherRemovalInProgressError) + + finishInstall() + await removal.ready + removal.release() + + const finishLaterInstall = beginWatcherInstall('C:/repo') + finishLaterInstall() + }) + + it('scopes identical roots to their execution host', async () => { + const removal = acquireWatcherRemovalGate('/repo', 'ssh-a') + await removal.ready + + const finishOtherHostInstall = beginWatcherInstall('/repo', 'ssh-b') + expect(() => beginWatcherInstall('/repo', 'ssh-a')).toThrow(WatcherRemovalInProgressError) + + finishOtherHostInstall() + removal.release() + }) + + it('rejects a second destructive owner across removal entry points', () => { + const removal = acquireWatcherRemovalGate('/repo') + + expect(() => acquireWatcherRemovalGate('/repo')).toThrow( + 'Worktree deletion already in progress' + ) + + removal.release() + const retry = acquireWatcherRemovalGate('/repo') + retry.release() + }) + + it('waits for an existing terminal spawn and rejects later same-host spawns', async () => { + const finishSpawn = beginTerminalInstall('/repo', 'ssh-a') + const removal = acquireWatcherRemovalGate('/repo', 'ssh-a') + let ready = false + void removal.ready.then(() => { + ready = true + }) + + await Promise.resolve() + expect(ready).toBe(false) + expect(() => beginTerminalInstall('/repo', 'ssh-a')).toThrow(TerminalRemovalInProgressError) + + finishSpawn() + await removal.ready + removal.release() + }) + + it('waits descendant installs and rejects overlapping removal roots', async () => { + const finishSpawn = beginTerminalInstall('/repo/nested') + const removal = acquireWatcherRemovalGate('/repo') + let ready = false + void removal.ready.then(() => { + ready = true + }) + + await Promise.resolve() + expect(ready).toBe(false) + expect(() => beginTerminalInstall('/repo/late')).toThrow(TerminalRemovalInProgressError) + expect(() => acquireWatcherRemovalGate('/repo/nested')).toThrow( + 'Worktree deletion already in progress' + ) + + finishSpawn() + await removal.ready + removal.release() + }) + + it('does not fence a distinct POSIX root containing a literal backslash', async () => { + const removal = acquireWatcherRemovalGate('/srv/team\\repo') + await removal.ready + + const finishInstall = beginWatcherInstall('/srv/team/repo') + finishInstall() + removal.release() + }) +}) diff --git a/src/main/ipc/watcher-removal-gate.ts b/src/main/ipc/watcher-removal-gate.ts new file mode 100644 index 00000000000..accf279961b --- /dev/null +++ b/src/main/ipc/watcher-removal-gate.ts @@ -0,0 +1,169 @@ +import { + isPathInsideOrEqual, + normalizeRuntimePathForComparison +} from '../../shared/cross-platform-path' + +type WatcherRemovalGateState = { + connectionId: string | null + rootPath: string + installCount: number + removalCount: number + installDrainWaiters: Set<() => void> +} + +export type WatcherRemovalGate = { + ready: Promise + release(): void +} + +export class WatcherRemovalInProgressError extends Error { + readonly code = 'watcher_removal_in_progress' + + constructor() { + super('File watcher cannot start while the worktree is being removed') + this.name = 'WatcherRemovalInProgressError' + } +} + +export class TerminalRemovalInProgressError extends Error { + readonly code = 'terminal_removal_in_progress' + + constructor() { + super('Terminal cannot start while the worktree is being removed') + this.name = 'TerminalRemovalInProgressError' + } +} + +const states = new Map() + +export function beginWatcherInstall(rootPath: string, connectionId?: string): () => void { + return beginRemovalSensitiveInstall( + rootPath, + connectionId, + () => new WatcherRemovalInProgressError() + ) +} + +export function beginTerminalInstall(rootPath: string, connectionId?: string): () => void { + return beginRemovalSensitiveInstall( + rootPath, + connectionId, + () => new TerminalRemovalInProgressError() + ) +} + +function beginRemovalSensitiveInstall( + rootPath: string, + connectionId: string | undefined, + createRemovalError: () => Error +): () => void { + const normalizedRoot = normalizeRuntimePathForComparison(rootPath) + if ( + matchingHostStates(connectionId).some( + (state) => state.removalCount > 0 && isPathInsideOrEqual(state.rootPath, normalizedRoot) + ) + ) { + throw createRemovalError() + } + const key = watcherRemovalGateKey(normalizedRoot, connectionId) + const state = states.get(key) ?? createState(key, normalizedRoot, connectionId) + state.installCount++ + let released = false + return () => { + if (released) { + return + } + released = true + state.installCount-- + if (state.installCount === 0) { + for (const resolve of state.installDrainWaiters) { + resolve() + } + state.installDrainWaiters.clear() + deleteIdleState(key, state) + } + } +} + +export function acquireWatcherRemovalGate( + rootPath: string, + connectionId?: string +): WatcherRemovalGate { + const normalizedRoot = normalizeRuntimePathForComparison(rootPath) + const hostStates = matchingHostStates(connectionId) + if ( + hostStates.some( + (state) => + state.removalCount > 0 && + (isPathInsideOrEqual(state.rootPath, normalizedRoot) || + isPathInsideOrEqual(normalizedRoot, state.rootPath)) + ) + ) { + // Why: desktop and runtime removal entry points have separate request + // dedupe; the shared root fence must still prevent two destructive runs. + throw new Error('Worktree deletion already in progress') + } + const key = watcherRemovalGateKey(normalizedRoot, connectionId) + const state = states.get(key) ?? createState(key, normalizedRoot, connectionId) + state.removalCount++ + // Why: deleting a parent root must wait for native installs already admitted + // under that root, not only installs keyed to the exact same spelling. + const drains = matchingHostStates(connectionId) + .filter( + (candidate) => + candidate.installCount > 0 && isPathInsideOrEqual(normalizedRoot, candidate.rootPath) + ) + .map((candidate) => new Promise((resolve) => candidate.installDrainWaiters.add(resolve))) + const ready = drains.length === 0 ? Promise.resolve() : Promise.all(drains).then(() => undefined) + let released = false + return { + ready, + release: () => { + if (released) { + return + } + released = true + state.removalCount-- + deleteIdleState(key, state) + } + } +} + +export function isWatcherRemovalInProgressError( + error: unknown +): error is WatcherRemovalInProgressError { + return error instanceof WatcherRemovalInProgressError +} + +function watcherRemovalGateKey(normalizedRoot: string, connectionId?: string): string { + // Why: the same path can exist on local and multiple SSH hosts, while + // Windows-equivalent spellings must still share one physical-removal fence. + return JSON.stringify([connectionId ?? null, normalizedRoot]) +} + +function matchingHostStates(connectionId?: string): WatcherRemovalGateState[] { + const host = connectionId ?? null + return [...states.values()].filter((state) => state.connectionId === host) +} + +function createState( + key: string, + rootPath: string, + connectionId?: string +): WatcherRemovalGateState { + const state = { + connectionId: connectionId ?? null, + rootPath, + installCount: 0, + removalCount: 0, + installDrainWaiters: new Set<() => void>() + } + states.set(key, state) + return state +} + +function deleteIdleState(key: string, state: WatcherRemovalGateState): void { + if (state.installCount === 0 && state.removalCount === 0 && states.get(key) === state) { + states.delete(key) + } +} diff --git a/src/main/ipc/worktree-symlinks.test.ts b/src/main/ipc/worktree-symlinks.test.ts index d3c8de1ff8c..cfb53effcfe 100644 --- a/src/main/ipc/worktree-symlinks.test.ts +++ b/src/main/ipc/worktree-symlinks.test.ts @@ -17,6 +17,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { createWorktreeLinkedPaths, createWorktreeSymlinks, + findExistingWorktreeSymlinkPaths, removeWorktreeLinkedPaths, removeWorktreeSymlinks } from './worktree-symlinks' @@ -434,6 +435,16 @@ describe('removeWorktreeSymlinks', () => { expect(statSync(join(worktree, '.env')).isFile()).toBe(true) }) + it('identifies only actual symlinks for removal preflight', async () => { + writeFileSync(join(primary, '.env'), 'PRIMARY=1\n') + symlinkSync(join(primary, '.env'), join(worktree, '.env')) + writeFileSync(join(worktree, 'config.json'), '{}\n') + + await expect( + findExistingWorktreeSymlinkPaths(worktree, ['.env', 'config.json', 'missing']) + ).resolves.toEqual(['.env']) + }) + it('leaves APFS clone-copied regular files for git removal to judge', async () => { writeFileSync(join(worktree, '.env'), 'CLONED=1\n') diff --git a/src/main/ipc/worktree-symlinks.ts b/src/main/ipc/worktree-symlinks.ts index 27e45ac3450..98e74e753fb 100644 --- a/src/main/ipc/worktree-symlinks.ts +++ b/src/main/ipc/worktree-symlinks.ts @@ -336,6 +336,27 @@ export async function removeWorktreeLinkedPaths( } } +export async function findExistingWorktreeSymlinkPaths( + worktreePath: string, + paths: readonly string[] +): Promise { + const symlinkPaths: string[] = [] + for (const rawPath of paths) { + const safePath = getSafeRelativePath(rawPath) + if (!safePath.safe) { + continue + } + try { + if ((await lstat(resolve(worktreePath, safePath.rel))).isSymbolicLink()) { + symlinkPaths.push(safePath.rel) + } + } catch { + // Why: only a positively identified symlink may bypass dirty preflight. + } + } + return symlinkPaths +} + /** Remove previously-created symlinks from a worktree before deletion. * * Why: `git worktree remove` refuses to delete a worktree that has modified diff --git a/src/main/ipc/worktrees-windows.test.ts b/src/main/ipc/worktrees-windows.test.ts index 73d10495b1f..a2f43b7dac6 100644 --- a/src/main/ipc/worktrees-windows.test.ts +++ b/src/main/ipc/worktrees-windows.test.ts @@ -264,7 +264,12 @@ describe('registerWorktreeHandlers – Windows path handling', () => { emitWorktreeBaseStatus: vi.fn(), recordOptimisticReconcileToken: vi.fn().mockReturnValue('token-1'), reconcileWorktreeBaseStatus: vi.fn(), - clearOptimisticReconcileToken: vi.fn() + clearOptimisticReconcileToken: vi.fn(), + closeFileWatchersForRemoval: vi.fn().mockResolvedValue(undefined), + acquireFileWatcherRemoval: vi.fn().mockResolvedValue({ + ready: Promise.resolve(), + release: vi.fn() + }) } registerWorktreeHandlers(mainWindow as never, store as never, runtimeStub as never) }) diff --git a/src/main/ipc/worktrees.test.ts b/src/main/ipc/worktrees.test.ts index 9f4dea40f92..ef6127ec2f9 100644 --- a/src/main/ipc/worktrees.test.ts +++ b/src/main/ipc/worktrees.test.ts @@ -9,6 +9,7 @@ import * as localWorktreeFilesystem from '../local-worktree-filesystem' const ORIGINAL_PLATFORM = process.platform const removeWorktreeLinkedPathsMock = vi.hoisted(() => vi.fn()) +const findExistingWorktreeSymlinkPathsMock = vi.hoisted(() => vi.fn()) function setPlatform(platform: NodeJS.Platform): void { Object.defineProperty(process, 'platform', { @@ -175,6 +176,7 @@ vi.mock('../providers/ssh-filesystem-dispatch', () => ({ vi.mock('./worktree-symlinks', () => ({ createWorktreeLinkedPaths: vi.fn(), + findExistingWorktreeSymlinkPaths: findExistingWorktreeSymlinkPathsMock, removeWorktreeLinkedPaths: removeWorktreeLinkedPathsMock })) @@ -225,12 +227,17 @@ vi.mock('../ports/advertised-url-watcher', () => ({ } })) -const { killAllProcessesForWorktreeMock, clearProviderPtyStateMock, getLocalPtyProviderMock } = - vi.hoisted(() => ({ - killAllProcessesForWorktreeMock: vi.fn(), - clearProviderPtyStateMock: vi.fn(), - getLocalPtyProviderMock: vi.fn() - })) +const { + killAllProcessesForWorktreeMock, + clearProviderPtyStateMock, + getLocalPtyProviderMock, + getSshPtyProviderMock +} = vi.hoisted(() => ({ + killAllProcessesForWorktreeMock: vi.fn(), + clearProviderPtyStateMock: vi.fn(), + getLocalPtyProviderMock: vi.fn(), + getSshPtyProviderMock: vi.fn() +})) vi.mock('../runtime/worktree-teardown', () => ({ killAllProcessesForWorktree: killAllProcessesForWorktreeMock @@ -238,7 +245,8 @@ vi.mock('../runtime/worktree-teardown', () => ({ vi.mock('./pty', () => ({ clearProviderPtyState: clearProviderPtyStateMock, - getLocalPtyProvider: getLocalPtyProviderMock + getLocalPtyProvider: getLocalPtyProviderMock, + getSshPtyProvider: getSshPtyProviderMock })) import { @@ -290,6 +298,8 @@ describe('registerWorktreeHandlers', () => { createTerminal: ReturnType splitTerminal: ReturnType notifyWorktreesChangedForRemoteClients: ReturnType + closeFileWatchersForRemoval: ReturnType + acquireFileWatcherRemoval: ReturnType } beforeEach(() => { @@ -351,8 +361,10 @@ describe('registerWorktreeHandlers', () => { killAllProcessesForWorktreeMock, clearProviderPtyStateMock, getLocalPtyProviderMock, + getSshPtyProviderMock, deleteWorktreeHistoryDirMock, advertisedUrlWatcherForgetWorktreeMock, + findExistingWorktreeSymlinkPathsMock, removeWorktreeLinkedPathsMock ]) { m.mockReset() @@ -363,7 +375,9 @@ describe('registerWorktreeHandlers', () => { registryStopped: 0 }) assertWorktreeCleanForRemovalMock.mockResolvedValue(undefined) + findExistingWorktreeSymlinkPathsMock.mockResolvedValue([]) getLocalPtyProviderMock.mockReturnValue({} as never) + getSshPtyProviderMock.mockReturnValue({} as never) for (const key of Object.keys(handlers)) { delete handlers[key] @@ -501,8 +515,21 @@ describe('registerWorktreeHandlers', () => { tabId: 'tab-startup', paneRuntimeId: -1 }), - notifyWorktreesChangedForRemoteClients: vi.fn() + notifyWorktreesChangedForRemoteClients: vi.fn(), + closeFileWatchersForRemoval: vi.fn().mockResolvedValue(undefined), + acquireFileWatcherRemoval: vi.fn() } + runtimeStub.acquireFileWatcherRemoval.mockImplementation( + async (worktreePath: string, connectionId?: string) => { + await ( + runtimeStub.closeFileWatchersForRemoval as ( + worktreePath: string, + connectionId?: string + ) => Promise + )(worktreePath, connectionId) + return { ready: Promise.resolve(), release: vi.fn() } + } + ) registerWorktreeHandlers(mainWindow as never, store as never, runtimeStub as never) }) @@ -6778,6 +6805,50 @@ describe('registerWorktreeHandlers', () => { ) }) + it('does not remove a worktree when watcher teardown cannot release it', async () => { + mockKnownFeatureWorktree() + store.getRepo.mockReturnValue({ + id: 'repo-1', + path: '/workspace/repo', + displayName: 'repo', + badgeColor: '#000', + addedAt: 0, + symlinkPaths: ['node_modules'] + }) + runtimeStub.closeFileWatchersForRemoval.mockRejectedValue( + new Error('file watcher process did not exit after termination deadline') + ) + + await expect( + handlers['worktrees:remove'](null, { + worktreeId: 'repo-1::/workspace/feature-wt' + }) + ).rejects.toThrow('file watcher process did not exit after termination deadline') + + expect(removeWorktreeMock).not.toHaveBeenCalled() + expect(removeWorktreeLinkedPathsMock).not.toHaveBeenCalled() + expect(store.removeWorktreeMeta).not.toHaveBeenCalled() + }) + + it('releases the watcher-install fence when worktree deletion fails', async () => { + mockKnownFeatureWorktree() + const release = vi.fn() + runtimeStub.acquireFileWatcherRemoval.mockResolvedValueOnce({ + ready: Promise.resolve(), + release + }) + removeWorktreeMock.mockRejectedValueOnce(new Error('delete failed')) + + await expect( + handlers['worktrees:remove'](null, { + worktreeId: 'repo-1::/workspace/feature-wt' + }) + ).rejects.toThrow('delete failed') + + expect(release).toHaveBeenCalledTimes(1) + expect(store.removeWorktreeMeta).not.toHaveBeenCalled() + }) + it('skips the archive hook on remove when skipArchive is true', async () => { mockKnownFeatureWorktree() removeWorktreeMock.mockResolvedValue(undefined) @@ -6819,6 +6890,9 @@ describe('registerWorktreeHandlers', () => { worktreeBaseRef: null } const callOrder: string[] = [] + runtimeStub.closeFileWatchersForRemoval.mockImplementationOnce(async () => { + callOrder.push('watchers') + }) const provider = { listWorktrees: vi.fn().mockResolvedValue([ { @@ -6877,7 +6951,11 @@ describe('registerWorktreeHandlers', () => { }) ) expect(provider.removeWorktree).toHaveBeenCalledWith('/remote/feature-wt', undefined) - expect(callOrder).toEqual(['archive', 'preflight', 'remove']) + expect(runtimeStub.closeFileWatchersForRemoval).toHaveBeenCalledWith( + '/remote/feature-wt', + 'conn-1' + ) + expect(callOrder).toEqual(['archive', 'preflight', 'watchers', 'remove']) expect(runHookMock).not.toHaveBeenCalled() }) @@ -7745,7 +7823,10 @@ describe('registerWorktreeHandlers', () => { }) await expect(lstat(orphanPath)).rejects.toMatchObject({ code: 'ENOENT' }) - expect(killAllProcessesForWorktreeMock).not.toHaveBeenCalled() + expect(killAllProcessesForWorktreeMock).toHaveBeenCalledWith( + worktreeId, + expect.objectContaining({ requirePhysicalStop: true }) + ) expect(runHookMock).not.toHaveBeenCalled() expect(removeWorktreeMock).not.toHaveBeenCalled() expect(runtimeStub.clearOptimisticReconcileToken).toHaveBeenCalledWith(worktreeId) @@ -7837,7 +7918,10 @@ describe('registerWorktreeHandlers', () => { ).resolves.toEqual({}) await expect(lstat(leftoverPath)).rejects.toMatchObject({ code: 'ENOENT' }) - expect(killAllProcessesForWorktreeMock).not.toHaveBeenCalled() + expect(killAllProcessesForWorktreeMock).toHaveBeenCalledWith( + worktreeId, + expect.objectContaining({ requirePhysicalStop: true }) + ) expect(runHookMock).not.toHaveBeenCalled() expect(removeWorktreeMock).not.toHaveBeenCalled() expect(runtimeStub.clearOptimisticReconcileToken).toHaveBeenCalledWith(worktreeId) @@ -8146,13 +8230,31 @@ describe('registerWorktreeHandlers', () => { 'repo-1::/workspace/feature-wt', expect.objectContaining({ localProvider: expect.anything(), - onPtyStopped: clearProviderPtyStateMock + onPtyStopped: clearProviderPtyStateMock, + requirePhysicalStop: true }) ) expect(removeWorktreeMock).toHaveBeenCalled() expect(callOrder).toEqual(['preflight', 'kill', 'git']) }) + it('does not start Git removal when physical PTY teardown cannot be proven', async () => { + mockKnownFeatureWorktree() + getEffectiveHooksMock.mockReturnValue(null) + killAllProcessesForWorktreeMock.mockRejectedValueOnce( + new Error('Timed out waiting for physical PTY teardown') + ) + + await expect( + handlers['worktrees:remove'](null, { + worktreeId: 'repo-1::/workspace/feature-wt' + }) + ).rejects.toThrow('Timed out waiting for physical PTY teardown') + + expect(removeWorktreeMock).not.toHaveBeenCalled() + expect(store.removeWorktreeMeta).not.toHaveBeenCalled() + }) + it('routes local worktree removal through the selected WSL project runtime', async () => { mockSelectedWslProjectRuntime() mockKnownFeatureWorktree() @@ -8193,6 +8295,16 @@ describe('registerWorktreeHandlers', () => { it('fails dirty non-force deletes before PTY teardown', async () => { mockKnownFeatureWorktree() + const repoWithConfiguredRegularFile = { + id: 'repo-1', + path: '/workspace/repo', + displayName: 'repo', + badgeColor: '#000', + addedAt: 0, + symlinkPaths: ['scratch.txt'] + } + store.getRepo.mockReturnValue(repoWithConfiguredRegularFile) + store.getRepos.mockReturnValue([repoWithConfiguredRegularFile]) getEffectiveHooksMock.mockReturnValue(null) assertWorktreeCleanForRemovalMock.mockRejectedValue( Object.assign(new Error('Worktree has uncommitted or untracked changes.'), { @@ -8206,6 +8318,28 @@ describe('registerWorktreeHandlers', () => { }) ).rejects.toThrow('Failed to delete worktree at /workspace/feature-wt. ?? scratch.txt') + expect(findExistingWorktreeSymlinkPathsMock).toHaveBeenCalledWith('/workspace/feature-wt', [ + 'scratch.txt' + ]) + expect(assertWorktreeCleanForRemovalMock).toHaveBeenCalledWith('/workspace/feature-wt', false) + expect(runtimeStub.closeFileWatchersForRemoval).not.toHaveBeenCalled() + expect(removeWorktreeLinkedPathsMock).not.toHaveBeenCalled() + expect(killAllProcessesForWorktreeMock).not.toHaveBeenCalled() + expect(removeWorktreeMock).not.toHaveBeenCalled() + }) + + it('propagates a timed-out removal preflight before watcher teardown', async () => { + mockKnownFeatureWorktree() + getEffectiveHooksMock.mockReturnValue(null) + assertWorktreeCleanForRemovalMock.mockRejectedValue(new Error('git timed out.')) + + await expect( + handlers['worktrees:remove'](null, { + worktreeId: 'repo-1::/workspace/feature-wt' + }) + ).rejects.toThrow('Failed to delete worktree at /workspace/feature-wt. git timed out.') + + expect(runtimeStub.closeFileWatchersForRemoval).not.toHaveBeenCalled() expect(killAllProcessesForWorktreeMock).not.toHaveBeenCalled() expect(removeWorktreeMock).not.toHaveBeenCalled() }) @@ -8330,17 +8464,17 @@ describe('registerWorktreeHandlers', () => { worktreeId: 'repo-1::/workspace/feature-wt' }) - expect(killAllProcessesForWorktreeMock).not.toHaveBeenCalled() + expect(killAllProcessesForWorktreeMock).toHaveBeenCalledWith( + 'repo-1::/workspace/feature-wt', + expect.objectContaining({ requirePhysicalStop: true }) + ) expect(removeWorktreeMock).toHaveBeenCalled() expect(gitExecFileAsyncMock).toHaveBeenCalledWith(['worktree', 'prune'], { cwd: '/workspace/repo' }) }) - it('skips the PTY teardown for SSH-backed repos (design §6 out-of-scope)', async () => { - // Why: SSH-backed PTYs live on the remote host and are handled by the - // remote provider's own teardown. The local-host helper must not run for - // SSH repos, because it would sweep registry entries for other worktrees. + it('fails closed when the SSH PTY provider is unavailable', async () => { const repo = { id: 'repo-ssh', path: '/remote/repo', @@ -8352,16 +8486,32 @@ describe('registerWorktreeHandlers', () => { } store.getRepos.mockReturnValue([repo]) store.getRepo.mockReturnValue(repo) + const removeWorktree = vi.fn() + getSshGitProviderMock.mockReturnValue({ + listWorktrees: vi.fn().mockResolvedValue([ + { path: '/remote/repo', isMainWorktree: true }, + { + path: '/remote/feature-wt', + head: 'feature', + branch: 'feature', + isMainWorktree: false + } + ]), + worktreeIsClean: vi.fn().mockResolvedValue({ clean: true }), + removeWorktree + }) + getSshFilesystemProviderMock.mockReturnValue({ + readFile: vi.fn().mockRejectedValue(new Error('missing')) + }) + getSshPtyProviderMock.mockReturnValue(undefined) - // The test can't easily mock the SSH provider without more plumbing — the - // call will throw about 'no git provider for connection'. What matters - // here is that the kill helper was NOT called for the SSH branch. - await ( + await expect( handlers['worktrees:remove'](null, { worktreeId: 'repo-ssh::/remote/feature-wt' - }) as Promise - ).catch(() => {}) + }) + ).rejects.toThrow('PTY provider unavailable for worktree deletion') + expect(removeWorktree).not.toHaveBeenCalled() expect(killAllProcessesForWorktreeMock).not.toHaveBeenCalled() }) diff --git a/src/main/ipc/worktrees.ts b/src/main/ipc/worktrees.ts index 06397b341db..429c5634e5d 100644 --- a/src/main/ipc/worktrees.ts +++ b/src/main/ipc/worktrees.ts @@ -88,11 +88,10 @@ import { isENOENT, registerWorktreeRootsForRepo } from './filesystem-auth' -import { closeLocalWatcherForWorktreePath } from './filesystem-watcher' import type { OrcaRuntimeService } from '../runtime/orca-runtime' import { killAllProcessesForWorktree } from '../runtime/worktree-teardown' -import { clearProviderPtyState, getLocalPtyProvider } from './pty' -import { removeWorktreeLinkedPaths } from './worktree-symlinks' +import { clearProviderPtyState, getLocalPtyProvider, getSshPtyProvider } from './pty' +import { findExistingWorktreeSymlinkPaths, removeWorktreeLinkedPaths } from './worktree-symlinks' import { track } from '../telemetry/client' import { getCohortAtEmit } from '../telemetry/cohort-classifier' import { workspaceSourceSchema, type WorkspaceSource } from '../../shared/telemetry-events' @@ -114,6 +113,31 @@ type RemoveWorktreeArgs = { skipArchive?: boolean } +async function stopPtysForDestructiveWorktreeRemoval( + runtime: OrcaRuntimeService, + worktreeId: string, + connectionId?: string +): Promise { + const provider = connectionId ? getSshPtyProvider(connectionId) : getLocalPtyProvider() + if (!provider) { + throw new Error(`PTY provider unavailable for worktree deletion: ${worktreeId}`) + } + const teardownResult = await killAllProcessesForWorktree(worktreeId, { + runtime, + localProvider: provider, + onPtyStopped: clearProviderPtyState, + requirePhysicalStop: true, + ...(connectionId ? { includeLocalRegistry: false } : {}) + }) + const total = + teardownResult.runtimeStopped + teardownResult.providerStopped + teardownResult.registryStopped + if (total > 0) { + console.info( + `[worktree-teardown] ${worktreeId} killed runtime=${teardownResult.runtimeStopped} provider=${teardownResult.providerStopped} registry=${teardownResult.registryStopped}` + ) + } +} + function getRepoForWorktreeRemoval( store: Store, repoId: string, @@ -208,12 +232,6 @@ function removeWorktreeMetadataAndTransientState(store: Store, worktreeId: strin pruneWorktreePRRefreshAliases(worktreeId) } -async function closeLocalWatcherForRemoval(worktreePath: string): Promise { - await closeLocalWatcherForWorktreePath(worktreePath).catch((err) => { - console.warn(`[filesystem-watcher] failed to close ${worktreePath}:`, err) - }) -} - function getProjectHostSetupMetaUpdates( store: Store, repo: Repo, @@ -1498,7 +1516,20 @@ export function registerWorktreeHandlers( throw new Error(ORPHANED_WORKTREE_DIRECTORY_MESSAGE) } if (repo.connectionId) { - await fsProvider!.deletePath(worktreePath, true) + const removalGate = await runtime.acquireFileWatcherRemoval( + worktreePath, + repo.connectionId + ) + try { + await stopPtysForDestructiveWorktreeRemoval( + runtime, + args.worktreeId, + repo.connectionId + ) + await fsProvider!.deletePath(worktreePath, true) + } finally { + removalGate.release() + } await cleanupUnusedWorktreePushTargetRemoteSsh( provider!, repo.path, @@ -1507,8 +1538,13 @@ export function registerWorktreeHandlers( store ) } else { - await closeLocalWatcherForRemoval(worktreePath) - await removeLocalWorktreePath(worktreePath, localWorktreeGitOptions) + const removalGate = await runtime.acquireFileWatcherRemoval(worktreePath) + try { + await stopPtysForDestructiveWorktreeRemoval(runtime, args.worktreeId) + await removeLocalWorktreePath(worktreePath, localWorktreeGitOptions) + } finally { + removalGate.release() + } await cleanupUnusedWorktreePushTargetRemote( repo.path, args.worktreeId, @@ -1545,8 +1581,13 @@ export function registerWorktreeHandlers( if (!args.force) { throw new Error(ORPHANED_WORKTREE_DIRECTORY_MESSAGE) } - await closeLocalWatcherForRemoval(worktreePath) - await removeLocalWorktreePath(worktreePath, localWorktreeGitOptions) + const removalGate = await runtime.acquireFileWatcherRemoval(worktreePath) + try { + await stopPtysForDestructiveWorktreeRemoval(runtime, args.worktreeId) + await removeLocalWorktreePath(worktreePath, localWorktreeGitOptions) + } finally { + removalGate.release() + } await cleanupUnusedWorktreePushTargetRemote( repo.path, args.worktreeId, @@ -1648,8 +1689,6 @@ export function registerWorktreeHandlers( return removalResult ?? {} } - let shouldTearDownPtys = true - // Run archive hook before removal so teardown scripts still see the worktree directory. const hooks = await getArchiveHooksForRemoval(repo) if (hooks?.scripts.archive && !args.skipArchive) { @@ -1683,9 +1722,19 @@ export function registerWorktreeHandlers( } const remoteRemoveOptions = !deleteBranch ? { deleteBranch } : {} - const rawRemovalResult = await (Object.keys(remoteRemoveOptions).length > 0 - ? provider!.removeWorktree(canonicalWorktreePath, args.force, remoteRemoveOptions) - : provider!.removeWorktree(canonicalWorktreePath, args.force)) + const removalGate = await runtime.acquireFileWatcherRemoval( + canonicalWorktreePath, + repo.connectionId + ) + let rawRemovalResult: RemoveWorktreeResult | undefined + try { + await stopPtysForDestructiveWorktreeRemoval(runtime, args.worktreeId, repo.connectionId) + rawRemovalResult = await (Object.keys(remoteRemoveOptions).length > 0 + ? provider!.removeWorktree(canonicalWorktreePath, args.force, remoteRemoveOptions) + : provider!.removeWorktree(canonicalWorktreePath, args.force)) + } finally { + removalGate.release() + } const removalResult = preserveBranchHeadFallback( rawRemovalResult, registeredWorktree.head @@ -1732,139 +1781,131 @@ export function registerWorktreeHandlers( ) } - // Why: `git worktree remove` (non-force) refuses to delete a worktree - // that has untracked files. User-configured symlinks look untracked, - // so unlink them before the clean check; regular APFS clone-copied - // files are left for git to judge so we never delete user edits. - if (repo.symlinkPaths && repo.symlinkPaths.length > 0) { - await removeWorktreeLinkedPaths(canonicalWorktreePath, repo.symlinkPaths) - } - + const linkedPaths = repo.symlinkPaths ?? [] + const ignoredLinkedPaths = args.force + ? [] + : await findExistingWorktreeSymlinkPaths(canonicalWorktreePath, linkedPaths) try { await (hasLocalWorktreeGitOptions - ? assertWorktreeCleanForRemoval( - canonicalWorktreePath, - args.force ?? false, - localWorktreeGitOptions - ) - : assertWorktreeCleanForRemoval(canonicalWorktreePath, args.force ?? false)) + ? assertWorktreeCleanForRemoval(canonicalWorktreePath, args.force ?? false, { + ...localWorktreeGitOptions, + ...(ignoredLinkedPaths.length > 0 + ? { ignoredUntrackedPaths: ignoredLinkedPaths } + : {}) + }) + : ignoredLinkedPaths.length > 0 + ? assertWorktreeCleanForRemoval(canonicalWorktreePath, args.force ?? false, { + ignoredUntrackedPaths: ignoredLinkedPaths + }) + : assertWorktreeCleanForRemoval(canonicalWorktreePath, args.force ?? false)) } catch (error) { if (!isOrphanCompatiblePreflightError(error)) { throw new Error( formatWorktreeRemovalError(error, canonicalWorktreePath, args.force ?? false) ) } - // Why: orphan cleanup does not need live shells to be killed first, - // and preflight did not prove the worktree is cleanly removable. - shouldTearDownPtys = false - } - - await closeLocalWatcherForRemoval(canonicalWorktreePath) - - if (shouldTearDownPtys) { - // Why: once preflight proves normal deletion is clean, kill PTYs before - // git-level removal so Windows handles cannot keep the directory busy. - await killAllProcessesForWorktree(args.worktreeId, { - runtime, - localProvider: getLocalPtyProvider(), - onPtyStopped: clearProviderPtyState - }) - .then((r) => { - const total = r.runtimeStopped + r.providerStopped + r.registryStopped - if (total > 0) { - console.info( - `[worktree-teardown] ${args.worktreeId} killed runtime=${r.runtimeStopped} provider=${r.providerStopped} registry=${r.registryStopped}` - ) - } - }) - .catch((err) => { - console.warn(`[worktree-teardown] failed for ${args.worktreeId}:`, err) - }) + // Why: Git can still classify this as an orphan after preflight; + // retain strict PTY teardown before any recursive fallback deletion. } let removalResult: RemoveWorktreeResult | undefined + const removalGate = await runtime.acquireFileWatcherRemoval(canonicalWorktreePath) try { - const removeOptions = { - ...(!deleteBranch ? { deleteBranch } : {}), - // Why: this handler already paid for an authoritative worktree - // list to validate the target; reuse it instead of rescanning - // every sibling worktree during the hot delete path. - knownRemovedWorktree: refreshedRegisteredWorktree, - ...(hasLocalWorktreeGitOptions ? localWorktreeGitOptions : {}) + // Why: preflight ignores only these configured paths without mutating + // the worktree; keep new watcher installs fenced through Git removal. + if (linkedPaths.length > 0) { + await removeWorktreeLinkedPaths(canonicalWorktreePath, linkedPaths) } - removalResult = preserveBranchHeadFallback( - await removeWorktree( - repo.path, - canonicalWorktreePath, - args.force ?? false, - removeOptions - ), - refreshedRegisteredWorktree.head - ) - } catch (error) { - // Why: Git for Windows can deregister a clean worktree before its - // recursive filesystem deletion fails transiently. - const recoveredRemovalResult = await recoverLocalWindowsWorktreeRemoval({ - error, - force: args.force ?? false, - canonicalWorktreePath, - repoPath: repo.path, - localWorktreeGitOptions, - registeredWorktree: refreshedRegisteredWorktree, - deleteBranch, - closeWatcher: closeLocalWatcherForRemoval - }) - if (recoveredRemovalResult) { - removalResult = recoveredRemovalResult - } else if (isOrphanedWorktreeError(error)) { - // If git no longer tracks this worktree, clean up the directory and metadata - console.warn( - `[worktrees] Orphaned worktree detected at ${canonicalWorktreePath}, cleaning up` + + // Why: hold the watcher/terminal gate until Git and any recursive + // fallback complete, so no late spawn can recreate a native handle. + await stopPtysForDestructiveWorktreeRemoval(runtime, args.worktreeId) + + try { + const removeOptions = { + ...(!deleteBranch ? { deleteBranch } : {}), + // Why: this handler already paid for an authoritative worktree + // list to validate the target; reuse it instead of rescanning + // every sibling worktree during the hot delete path. + knownRemovedWorktree: refreshedRegisteredWorktree, + ...(hasLocalWorktreeGitOptions ? localWorktreeGitOptions : {}) + } + removalResult = preserveBranchHeadFallback( + await removeWorktree( + repo.path, + canonicalWorktreePath, + args.force ?? false, + removeOptions + ), + refreshedRegisteredWorktree.head ) - const access = getLocalWorktreePathAccess(localWorktreeGitOptions) - if ( - await canSafelyRemoveOrphanedWorktreeDirectory( - toLocalWorktreeRuntimePath(canonicalWorktreePath, localWorktreeGitOptions), - toLocalWorktreeRuntimePath(repo.path, localWorktreeGitOptions), - access.statPath, - access.readPath - ) - ) { - await closeLocalWatcherForRemoval(canonicalWorktreePath) - await removeLocalWorktreePath(canonicalWorktreePath, localWorktreeGitOptions).catch( - () => {} - ) - } else { + } catch (error) { + // Why: Git for Windows can deregister a clean worktree before its + // recursive filesystem deletion fails transiently. + const recoveredRemovalResult = await recoverLocalWindowsWorktreeRemoval({ + error, + force: args.force ?? false, + canonicalWorktreePath, + repoPath: repo.path, + localWorktreeGitOptions, + registeredWorktree: refreshedRegisteredWorktree, + deleteBranch, + closeWatcher: (worktreePath) => runtime.closeFileWatchersForRemoval(worktreePath) + }) + if (recoveredRemovalResult) { + removalResult = recoveredRemovalResult + } else if (isOrphanedWorktreeError(error)) { + // If git no longer tracks this worktree, clean up the directory and metadata console.warn( - `[worktrees] Refusing recursive cleanup for unproven worktree directory: ${canonicalWorktreePath}` + `[worktrees] Orphaned worktree detected at ${canonicalWorktreePath}, cleaning up` + ) + const access = getLocalWorktreePathAccess(localWorktreeGitOptions) + if ( + await canSafelyRemoveOrphanedWorktreeDirectory( + toLocalWorktreeRuntimePath(canonicalWorktreePath, localWorktreeGitOptions), + toLocalWorktreeRuntimePath(repo.path, localWorktreeGitOptions), + access.statPath, + access.readPath + ) + ) { + await runtime.closeFileWatchersForRemoval(canonicalWorktreePath) + await removeLocalWorktreePath(canonicalWorktreePath, localWorktreeGitOptions).catch( + () => {} + ) + } else { + console.warn( + `[worktrees] Refusing recursive cleanup for unproven worktree directory: ${canonicalWorktreePath}` + ) + } + // Why: `git worktree remove` failed, so git's internal worktree tracking + // (`.git/worktrees/`) is still intact. Without pruning, `git worktree + // list` continues to show the stale entry and the branch it had checked out + // remains locked — other worktrees cannot check it out. + await gitExecFileAsync(['worktree', 'prune'], { + cwd: repo.path, + ...localWorktreeGitOptions + }).catch(() => {}) + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + args.worktreeId, + removedPushTarget, + store, + localWorktreeGitOptions + ) + runtime.clearOptimisticReconcileToken(args.worktreeId) + removeWorktreeMetadataAndTransientState(store, args.worktreeId) + preservedBranchCleanupByWorktreeId.delete(args.worktreeId) + invalidateAuthorizedRootsCache() + notifyWorktreesChanged(mainWindow, repoId) + return {} + } else { + throw new Error( + formatWorktreeRemovalError(error, canonicalWorktreePath, args.force ?? false) ) } - // Why: `git worktree remove` failed, so git's internal worktree tracking - // (`.git/worktrees/`) is still intact. Without pruning, `git worktree - // list` continues to show the stale entry and the branch it had checked out - // remains locked — other worktrees cannot check it out. - await gitExecFileAsync(['worktree', 'prune'], { - cwd: repo.path, - ...localWorktreeGitOptions - }).catch(() => {}) - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - args.worktreeId, - removedPushTarget, - store, - localWorktreeGitOptions - ) - runtime.clearOptimisticReconcileToken(args.worktreeId) - removeWorktreeMetadataAndTransientState(store, args.worktreeId) - preservedBranchCleanupByWorktreeId.delete(args.worktreeId) - invalidateAuthorizedRootsCache() - notifyWorktreesChanged(mainWindow, repoId) - return {} - } else { - throw new Error( - formatWorktreeRemovalError(error, canonicalWorktreePath, args.force ?? false) - ) } + } finally { + removalGate.release() } await cleanupUnusedWorktreePushTargetRemote( repo.path, diff --git a/src/main/ipc/wsl-watcher-process-exit.ts b/src/main/ipc/wsl-watcher-process-exit.ts new file mode 100644 index 00000000000..ebb897dfe96 --- /dev/null +++ b/src/main/ipc/wsl-watcher-process-exit.ts @@ -0,0 +1,106 @@ +import type { ChildProcessWithoutNullStreams } from 'node:child_process' +import { PhysicalExitTracker } from '../../shared/physical-exit-tracker' +import { WatcherProcessFailure } from './parcel-watcher-process-failure' + +export const WSL_WATCHER_PHYSICAL_EXIT_TIMEOUT_MS = 8_000 + +export function createWslWatcherStartup(): { + ready: Promise + readonly settled: boolean + settle: (error?: Error) => void +} { + let settled = false + let resolve!: () => void + let reject!: (error: Error) => void + const ready = new Promise((done, fail) => { + resolve = done + reject = fail + }) + return { + ready, + get settled() { + return settled + }, + settle(error) { + if (settled) { + return + } + settled = true + if (error) { + reject(error) + } else { + resolve() + } + } + } +} + +export function createWslWatcherProcessExit( + child: ChildProcessWithoutNullStreams, + worktreePath: string, + onPhysicalExit?: () => void +): { + markPhysicalExit: () => void + requestStopBestEffort: () => void + stopAndWait: () => Promise +} { + let physicalExited = false + let stopRequested = false + const physicalExit = new PhysicalExitTracker() + const markPhysicalExit = (): void => { + if (!physicalExited) { + physicalExited = true + physicalExit.markExited() + onPhysicalExit?.() + } + } + const requestStop = (): void => { + if (physicalExited || stopRequested) { + return + } + stopRequested = true + try { + if (!child.kill()) { + throw new Error('WSL watcher process rejected the termination signal') + } + } catch (error) { + stopRequested = false + throw error + } + } + const requestStopBestEffort = (): void => { + try { + requestStop() + } catch { + // The awaited owner retries and retains the child on failure. + } + } + const stopAndWait = async (): Promise => { + try { + requestStop() + } catch (error) { + throw processExitFailure( + `Failed to terminate WSL watcher: ${String(error)}`, + physicalExit.exitedPromise + ) + } + await waitForPhysicalExit(worktreePath, physicalExit) + } + return { markPhysicalExit, requestStopBestEffort, stopAndWait } +} + +function waitForPhysicalExit( + worktreePath: string, + physicalExit: PhysicalExitTracker +): Promise { + return physicalExit.waitForExit(WSL_WATCHER_PHYSICAL_EXIT_TIMEOUT_MS, () => + processExitFailure( + `WSL watcher process did not exit after deadline: ${worktreePath}`, + physicalExit.exitedPromise + ) + ) +} + +function processExitFailure(message: string, physicalExit: Promise): WatcherProcessFailure { + return new WatcherProcessFailure(message, 'supervisor', 'process_unavailable', physicalExit) +} diff --git a/src/main/local-worktree-removal-recovery.test.ts b/src/main/local-worktree-removal-recovery.test.ts index cb6dbcbb51e..1be0fd02cce 100644 --- a/src/main/local-worktree-removal-recovery.test.ts +++ b/src/main/local-worktree-removal-recovery.test.ts @@ -116,6 +116,28 @@ describe('recoverLocalWindowsWorktreeRemoval', () => { }) }) + it('does not recursively delete while watcher teardown remains unconfirmed', async () => { + await withPlatform('win32', async () => { + const watcherError = new Error('file watcher process did not exit after termination deadline') + + await expect( + recoverLocalWindowsWorktreeRemoval({ + error: new Error('git worktree remove failed'), + force: true, + canonicalWorktreePath: 'C:/repo/worktree/feature', + repoPath: 'C:/repo', + localWorktreeGitOptions: {}, + registeredWorktree: { branch: 'refs/heads/feature', head: 'abc123' }, + deleteBranch: true, + closeWatcher: vi.fn().mockRejectedValue(watcherError) + }) + ).rejects.toBe(watcherError) + + expect(removeLocalWorktreePathMock).not.toHaveBeenCalled() + expect(gitExecFileAsyncMock).not.toHaveBeenCalled() + }) + }) + it('recovers localized Windows failures after Git already removed the registration', async () => { await withPlatform('win32', async () => { const result = await recoverLocalWindowsWorktreeRemoval({ diff --git a/src/main/local-worktree-removal-recovery.ts b/src/main/local-worktree-removal-recovery.ts index 25f93e2b2de..5a920a4ee6f 100644 --- a/src/main/local-worktree-removal-recovery.ts +++ b/src/main/local-worktree-removal-recovery.ts @@ -120,7 +120,9 @@ export async function recoverLocalWindowsWorktreeRemoval( // Why: this error means Git accepted removal and started deleting the path; // finish that partial success with Windows retries, then verify Git metadata. - await args.closeWatcher(args.canonicalWorktreePath).catch(() => {}) + // Why: Windows recovery recursively deletes the remaining directory, so it + // must fail closed while a watcher process may still own a native handle. + await args.closeWatcher(args.canonicalWorktreePath) try { await removeLocalWorktreePath(args.canonicalWorktreePath, args.localWorktreeGitOptions) } catch (error) { diff --git a/src/main/providers/local-pty-provider.test.ts b/src/main/providers/local-pty-provider.test.ts index 66f68d6e868..c31e8e5f59c 100644 --- a/src/main/providers/local-pty-provider.test.ts +++ b/src/main/providers/local-pty-provider.test.ts @@ -84,7 +84,13 @@ vi.mock('../wsl', () => ({ wslUncDirectoryExists: () => true })) -import { LocalPtyProvider } from './local-pty-provider' +import { + _resetLocalPtyProviderStateForTest, + LOCAL_PTY_FORCE_KILL_RETRY_MS, + LOCAL_PTY_GRACEFUL_FORCE_TIMEOUT_MS, + LOCAL_PTY_PHYSICAL_EXIT_TIMEOUT_MS, + LocalPtyProvider +} from './local-pty-provider' import { isRootLikePath } from './pty-path-safety' import { POWERLEVEL10K_WIZARD_DISABLE_ENV } from '../pty/powerlevel10k-wizard-env' @@ -157,6 +163,7 @@ describe('LocalPtyProvider', () => { }) afterEach(() => { + _resetLocalPtyProviderStateForTest() if (origPlatform) { Object.defineProperty(process, 'platform', origPlatform) } @@ -974,12 +981,191 @@ describe('LocalPtyProvider', () => { }) const { id } = await provider.spawn({ cols: 80, rows: 24 }) - await provider.shutdown(id, { immediate: true }) + const shutdown = provider.shutdown(id, { immediate: true }) + exitCb?.({ exitCode: -1 }) + await shutdown expect(killSpy).toHaveBeenCalledTimes(1) expect(destroySpy).not.toHaveBeenCalled() }) + it('keeps shutdown and ownership pending until node-pty reports physical exit', async () => { + const killSpy = vi.fn() + mockProc.kill = killSpy + const { id } = await provider.spawn({ cols: 80, rows: 24 }) + + let settled = false + const shutdown = provider.shutdown(id, { immediate: true }).finally(() => { + settled = true + }) + await Promise.resolve() + + expect(killSpy).toHaveBeenCalledWith('SIGKILL') + expect(settled).toBe(false) + expect(provider.hasPty(id)).toBe(true) + exitCb?.({ exitCode: 137 }) + await shutdown + expect(provider.hasPty(id)).toBe(false) + }) + + it('keeps physical-exit tracking when orphan cleanup races immediate shutdown', async () => { + const killSpy = vi.fn(() => { + queueMicrotask(() => exitCb?.({ exitCode: 137 })) + }) + mockProc.kill = killSpy + const { id } = await provider.spawn({ cols: 80, rows: 24 }) + + const shutdown = provider.shutdown(id, { immediate: true }) + expect(provider.killOrphanedPtys(1)).toEqual([{ id }]) + expect(provider.hasPty(id)).toBe(true) + + await expect(shutdown).resolves.toBeUndefined() + expect(killSpy).toHaveBeenCalledTimes(1) + expect(provider.hasPty(id)).toBe(false) + }) + + it('escalates graceful shutdown before orphan cleanup disables the kill handle', async () => { + vi.useFakeTimers() + try { + const killSpy = vi.fn() + mockProc.kill = killSpy + const { id } = await provider.spawn({ cols: 80, rows: 24 }) + + const graceful = provider.shutdown(id, { immediate: false }) + expect(killSpy.mock.calls).toEqual([['SIGTERM']]) + + expect(provider.killOrphanedPtys(1)).toEqual([{ id }]) + expect(killSpy.mock.calls).toEqual([['SIGTERM'], ['SIGKILL']]) + + // The original graceful deadline was replaced by the immediate + // escalation, so it cannot call the now-neutralized proc.kill later. + await vi.advanceTimersByTimeAsync(LOCAL_PTY_GRACEFUL_FORCE_TIMEOUT_MS) + expect(killSpy).toHaveBeenCalledTimes(2) + exitCb?.({ exitCode: 137 }) + await graceful + expect(provider.hasPty(id)).toBe(false) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it('escalates a graceful shutdown when destructive cleanup joins it', async () => { + const killSpy = vi.fn() + mockProc.kill = killSpy + const { id } = await provider.spawn({ cols: 80, rows: 24 }) + + const graceful = provider.shutdown(id, { immediate: false }) + const immediate = provider.shutdown(id, { immediate: true }) + expect(killSpy.mock.calls).toEqual([['SIGTERM'], ['SIGKILL']]) + + exitCb?.({ exitCode: 137 }) + await Promise.all([graceful, immediate]) + expect(provider.hasPty(id)).toBe(false) + }) + + it('force-kills a POSIX PTY that ignores graceful shutdown', async () => { + vi.useFakeTimers() + try { + const killSpy = vi.fn() + mockProc.kill = killSpy + const { id } = await provider.spawn({ cols: 80, rows: 24 }) + + const graceful = provider.shutdown(id, { immediate: false }) + expect(killSpy.mock.calls).toEqual([['SIGTERM']]) + + await vi.advanceTimersByTimeAsync(LOCAL_PTY_GRACEFUL_FORCE_TIMEOUT_MS) + expect(killSpy.mock.calls).toEqual([['SIGTERM'], ['SIGKILL']]) + exitCb?.({ exitCode: 137 }) + await graceful + expect(provider.hasPty(id)).toBe(false) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it('retries a rejected graceful-deadline SIGKILL before the physical timeout', async () => { + vi.useFakeTimers() + try { + let forceAttempts = 0 + const killSpy = vi.fn((signal: string) => { + if (signal === 'SIGKILL' && forceAttempts++ === 0) { + throw new Error('transient force-kill failure') + } + }) + mockProc.kill = killSpy + const { id } = await provider.spawn({ cols: 80, rows: 24 }) + + const graceful = provider.shutdown(id, { immediate: false }) + await vi.advanceTimersByTimeAsync(LOCAL_PTY_GRACEFUL_FORCE_TIMEOUT_MS) + expect(killSpy.mock.calls).toEqual([['SIGTERM'], ['SIGKILL']]) + expect(provider.hasPty(id)).toBe(true) + + await vi.advanceTimersByTimeAsync(LOCAL_PTY_FORCE_KILL_RETRY_MS) + expect(killSpy.mock.calls).toEqual([['SIGTERM'], ['SIGKILL'], ['SIGKILL']]) + exitCb?.({ exitCode: 137 }) + await graceful + expect(provider.hasPty(id)).toBe(false) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it('does not double-kill ConPTY when destructive cleanup joins shutdown', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + const killSpy = vi.fn() + mockProc.kill = killSpy + const { id } = await provider.spawn({ cols: 80, rows: 24 }) + + const graceful = provider.shutdown(id, { immediate: false }) + const immediate = provider.shutdown(id, { immediate: true }) + expect(killSpy.mock.calls).toEqual([[]]) + + exitCb?.({ exitCode: 137 }) + await Promise.all([graceful, immediate]) + expect(killSpy).toHaveBeenCalledTimes(1) + expect(provider.hasPty(id)).toBe(false) + }) + + it('rejects a physical-exit timeout but retains the owner for a successful retry', async () => { + vi.useFakeTimers() + try { + const killSpy = vi.fn() + mockProc.kill = killSpy + const { id } = await provider.spawn({ cols: 80, rows: 24 }) + + const shutdown = provider.shutdown(id, { immediate: true }) + const rejected = expect(shutdown).rejects.toThrow('Timed out waiting for PTY process exit') + await vi.advanceTimersByTimeAsync(LOCAL_PTY_PHYSICAL_EXIT_TIMEOUT_MS) + await rejected + expect(provider.hasPty(id)).toBe(true) + + const retry = provider.shutdown(id, { immediate: true }) + expect(killSpy).toHaveBeenCalledTimes(1) + exitCb?.({ exitCode: 137 }) + await expect(retry).resolves.toBeUndefined() + expect(provider.hasPty(id)).toBe(false) + } finally { + vi.useRealTimers() + } + }) + + it('propagates kill failure without dropping the physical owner', async () => { + mockProc.kill = vi.fn(() => { + throw new Error('kill denied') + }) + const { id } = await provider.spawn({ cols: 80, rows: 24 }) + + await expect(provider.shutdown(id, { immediate: true })).rejects.toThrow('kill denied') + expect(provider.hasPty(id)).toBe(true) + + mockProc.kill = vi.fn(() => exitCb?.({ exitCode: 137 })) + await expect(provider.shutdown(id, { immediate: true })).resolves.toBeUndefined() + expect(provider.hasPty(id)).toBe(false) + }) + it('cancels pending shell-ready startup delivery on forced shutdown', async () => { vi.useFakeTimers() try { @@ -1105,13 +1291,14 @@ describe('LocalPtyProvider', () => { describe('listProcesses', () => { it('returns spawned PTYs', async () => { const before = await provider.listProcesses() - await provider.spawn({ cols: 80, rows: 24 }) + await provider.spawn({ cols: 80, rows: 24, cwd: '/tmp/owned-cwd' }) await provider.spawn({ cols: 80, rows: 24 }) const after = await provider.listProcesses() expect(after.length - before.length).toBe(2) const newEntries = after.slice(before.length) expect(newEntries[0]).toHaveProperty('id') expect(newEntries[0]).toHaveProperty('title', 'zsh') + expect(newEntries[0]).toHaveProperty('cwd', '/tmp/owned-cwd') }) }) diff --git a/src/main/providers/local-pty-provider.ts b/src/main/providers/local-pty-provider.ts index 3d9962fcb48..5af6742a3c1 100644 --- a/src/main/providers/local-pty-provider.ts +++ b/src/main/providers/local-pty-provider.ts @@ -58,6 +58,7 @@ import { readWindowsConptyProcessIds } from './windows-conpty-process-membership import { shouldUseShellReadyStartupDelivery } from '../../shared/codex-startup-delivery' import { assertSafeAgentStartupCwd, resolveSafePtyDefaultCwd } from './pty-default-cwd' import { ORCA_HERMES_STARTUP_QUERY_ENV } from '../../shared/hermes-startup-query' +import { PhysicalExitTracker } from '../../shared/physical-exit-tracker' const PANE_IDENTITY_ENV_KEYS = [ 'ORCA_PANE_KEY', @@ -71,12 +72,19 @@ const ptyProcesses = new Map() const ptyShellName = new Map() const ptyAgentForegroundContextPaths = new Map() const ptyTerminalHandle = new Map() -// Why: node-pty's onData/onExit register native NAPI ThreadSafeFunction -// callbacks. If the PTY is killed without disposing these listeners, the -// stale callbacks survive into node::FreeEnvironment() where NAPI attempts -// to invoke/clean them up on a destroyed environment, triggering a SIGABRT. +const ptyInitialCwd = new Map() +// Why: node-pty callbacks must be disposed before environment teardown, but +// onExit separately owns physical process-exit proof during termination. const ptyDisposables = new Map void }[]>() +const ptyExitDisposables = new Map void }>() const ptyCleanupCallbacks = new Map void>() +const ptyTerminationMode = new Map() +const ptyPhysicalExits = new Map() +const ptyForceKillTimers = new Map>() + +export const LOCAL_PTY_PHYSICAL_EXIT_TIMEOUT_MS = 8_000 +export const LOCAL_PTY_GRACEFUL_FORCE_TIMEOUT_MS = 5_000 +export const LOCAL_PTY_FORCE_KILL_RETRY_MS = 250 let loadGeneration = 0 const ptyLoadGeneration = new Map() @@ -139,6 +147,19 @@ function disposePtyListeners(id: string): void { } } +function disposePtyExitListener(id: string): void { + ptyExitDisposables.get(id)?.dispose() + ptyExitDisposables.delete(id) +} + +function clearLocalPtyForceKillTimer(id: string): void { + const timer = ptyForceKillTimers.get(id) + if (timer) { + clearTimeout(timer) + ptyForceKillTimers.delete(id) + } +} + function runPtyCleanup(id: string): void { const cleanup = ptyCleanupCallbacks.get(id) if (!cleanup) { @@ -177,13 +198,74 @@ function getWslContextFromPreferredDistro( * Removes all local tracking state for a PTY id after teardown. */ function clearPtyState(id: string): void { + clearLocalPtyForceKillTimer(id) runPtyCleanup(id) disposePtyListeners(id) + disposePtyExitListener(id) ptyProcesses.delete(id) ptyShellName.delete(id) ptyAgentForegroundContextPaths.delete(id) ptyTerminalHandle.delete(id) + ptyInitialCwd.delete(id) ptyLoadGeneration.delete(id) + ptyTerminationMode.delete(id) + ptyPhysicalExits.delete(id) +} + +function createPtyPhysicalExit(id: string): void { + ptyPhysicalExits.set(id, new PhysicalExitTracker()) +} + +function waitForPtyPhysicalExit(id: string, physicalExit?: PhysicalExitTracker): Promise { + if (!physicalExit) { + return Promise.reject(new Error(`PTY "${id}" exit tracking unavailable`)) + } + return physicalExit.waitForExit( + LOCAL_PTY_PHYSICAL_EXIT_TIMEOUT_MS, + () => new Error(`Timed out waiting for PTY process exit: ${id}`) + ) +} + +function killLocalPtyProcess(proc: pty.IPty, immediate: boolean): void { + if (process.platform === 'win32') { + proc.kill() + return + } + proc.kill(immediate ? 'SIGKILL' : 'SIGTERM') +} + +function armLocalPtyForceKill( + id: string, + proc: pty.IPty, + options: { delayMs?: number; attemptsRemaining?: number } = {} +): void { + if (ptyProcesses.get(id) !== proc || ptyTerminationMode.get(id) !== 'graceful') { + return + } + const attemptsRemaining = options.attemptsRemaining ?? 2 + const timer = setTimeout(() => { + ptyForceKillTimers.delete(id) + if (ptyProcesses.get(id) !== proc || ptyTerminationMode.get(id) !== 'graceful') { + return + } + ptyTerminationMode.set(id, 'force') + try { + killLocalPtyProcess(proc, true) + } catch (error) { + ptyTerminationMode.set(id, 'graceful') + console.error('[pty] failed to force-kill PTY after graceful deadline', { id, error }) + // Why: a transient native rejection must not consume the only SIGKILL + // owner while the logical shutdown continues waiting for physical exit. + if (attemptsRemaining > 1) { + armLocalPtyForceKill(id, proc, { + delayMs: LOCAL_PTY_FORCE_KILL_RETRY_MS, + attemptsRemaining: attemptsRemaining - 1 + }) + } + } + }, options.delayMs ?? LOCAL_PTY_GRACEFUL_FORCE_TIMEOUT_MS) + timer.unref?.() + ptyForceKillTimers.set(id, timer) } /** @@ -274,18 +356,36 @@ function destroyPtyProcess(proc: pty.IPty, options: { alreadyKilled?: boolean } } /** - * Kills a local PTY and clears all associated local provider state. + * Requests local PTY termination while retaining physical-exit ownership. */ -function safeKillAndClean(id: string, proc: pty.IPty): void { +function requestPtyTermination(id: string, proc: pty.IPty): void { runPtyCleanup(id) disposePtyListeners(id) - try { - proc.kill() - } catch { - /* Process may already be dead */ + const previousMode = ptyTerminationMode.get(id) + // Why: destructive cleanup neutralizes proc.kill below, so an outstanding + // graceful request must be escalated before its deadline can be disabled. + if (previousMode !== 'force') { + clearLocalPtyForceKillTimer(id) + ptyTerminationMode.set(id, 'force') + try { + killLocalPtyProcess(proc, true) + } catch { + if (previousMode === 'graceful') { + ptyTerminationMode.set(id, previousMode) + armLocalPtyForceKill(id, proc, { + delayMs: LOCAL_PTY_FORCE_KILL_RETRY_MS, + attemptsRemaining: 1 + }) + } else { + ptyTerminationMode.delete(id) + } + /* Process may already be dead. */ + return + } } + // Why: shutdown and orphan cleanup can race; node-pty's onExit listener and + // tracker must remain installed until the OS proves the child was reaped. destroyPtyProcess(proc, { alreadyKilled: true }) - clearPtyState(id) } export type LocalPtyProviderOptions = { @@ -688,7 +788,9 @@ export class LocalPtyProvider implements IPtyProvider { } const proc = spawnResult.process + createPtyPhysicalExit(id) ptyProcesses.set(id, proc) + ptyInitialCwd.set(id, cwd) ptyShellName.set(id, getSpawnedShellName(shellPath)) if (finalEnv.ORCA_TERMINAL_HANDLE) { ptyTerminalHandle.set(id, finalEnv.ORCA_TERMINAL_HANDLE) @@ -783,6 +885,8 @@ export class LocalPtyProvider implements IPtyProvider { } const onExitDisposable = proc.onExit(({ exitCode }) => { + const wasTerminationRequested = ptyTerminationMode.has(id) + ptyPhysicalExits.get(id)?.markExited() // Why: neutralize proc.kill the instant the child is reaped, before any // other work in this callback. node-pty's UnixTerminal installs a // `_socket.once('close', () => this.kill('SIGHUP'))` handler at destroy @@ -804,14 +908,14 @@ export class LocalPtyProvider implements IPtyProvider { // Why: release the master ptmx fd on the natural-exit path — without // this, a shell that exits cleanly (the common case) never releases its // fd until the next GC. See docs/fix-pty-fd-leak.md. - destroyPtyProcess(proc) + destroyPtyProcess(proc, { alreadyKilled: wasTerminationRequested }) this.opts.onExit?.(id, exitCode) for (const cb of exitListeners) { cb({ id, code: exitCode }) } }) if (onExitDisposable) { - disposables.push(onExitDisposable) + ptyExitDisposables.set(id, onExitDisposable) } ptyDisposables.set(id, disposables) @@ -886,29 +990,39 @@ export class LocalPtyProvider implements IPtyProvider { return { cols: proc.cols, rows: proc.rows } } - async shutdown(id: string, _opts: { immediate?: boolean; keepHistory?: boolean }): Promise { + async shutdown(id: string, opts: { immediate?: boolean; keepHistory?: boolean }): Promise { const proc = ptyProcesses.get(id) if (!proc) { return } - // Why: disposePtyListeners removes the onExit callback, so the natural - // exit cleanup path from node-pty won't fire. Cleanup and notification - // must happen unconditionally after the try/catch. - // Timer/writer cleanup must happen here too: disposing listeners prevents - // the natural onExit callback from running the usual clearPtyState path. + const physicalExit = ptyPhysicalExits.get(id) + // Why: cancel startup delivery immediately, but keep node-pty's onExit + // listener and all ownership maps until the OS confirms the child reaped. runPtyCleanup(id) - disposePtyListeners(id) - try { - proc.kill() - } catch { - /* Process may already be dead */ - } - destroyPtyProcess(proc, { alreadyKilled: true }) - clearPtyState(id) - this.opts.onExit?.(id, -1) - for (const cb of exitListeners) { - cb({ id, code: -1 }) + const previousMode = ptyTerminationMode.get(id) + // Why: ConPTY has no graceful signal; its first bare node-pty kill closes + // the pseudoconsole and must be treated as the final force request. + const requestedMode = + opts.immediate === true || process.platform === 'win32' ? 'force' : 'graceful' + if (!previousMode || (requestedMode === 'force' && previousMode !== 'force')) { + ptyTerminationMode.set(id, requestedMode) + try { + killLocalPtyProcess(proc, opts.immediate === true) + if (requestedMode === 'graceful') { + armLocalPtyForceKill(id, proc) + } else { + clearLocalPtyForceKillTimer(id) + } + } catch (error) { + if (previousMode) { + ptyTerminationMode.set(id, previousMode) + } else { + ptyTerminationMode.delete(id) + } + throw error + } } + await waitForPtyPhysicalExit(id, physicalExit) } async sendSignal(id: string, signal: string): Promise { @@ -1036,7 +1150,7 @@ export class LocalPtyProvider implements IPtyProvider { async listProcesses(): Promise { return Array.from(ptyProcesses.entries()).map(([id, proc]) => ({ id, - cwd: '', + cwd: ptyInitialCwd.get(id) ?? '', title: proc.process || ptyShellName.get(id) || 'shell', ...(ptyTerminalHandle.get(id) ? { terminalHandle: ptyTerminalHandle.get(id) } : {}) })) @@ -1090,7 +1204,7 @@ export class LocalPtyProvider implements IPtyProvider { const killed: { id: string }[] = [] for (const [id, proc] of ptyProcesses) { if ((ptyLoadGeneration.get(id) ?? -1) < currentGeneration) { - safeKillAndClean(id, proc) + requestPtyTermination(id, proc) killed.push({ id }) } } @@ -1110,7 +1224,27 @@ export class LocalPtyProvider implements IPtyProvider { /** Kill all in-process local PTYs. Call on app quit. */ killAll(): void { for (const [id, proc] of ptyProcesses) { - safeKillAndClean(id, proc) + runPtyCleanup(id) + disposePtyListeners(id) + disposePtyExitListener(id) + if (!(process.platform === 'win32' && ptyTerminationMode.has(id))) { + try { + proc.kill() + } catch { + /* Process may already be dead. */ + } + } + // Why: app quit cannot retain NAPI callbacks into FreeEnvironment; the + // process exit itself is the final physical handle boundary here. + destroyPtyProcess(proc, { alreadyKilled: true }) + clearPtyState(id) } } } + +export function _resetLocalPtyProviderStateForTest(): void { + for (const id of ptyProcesses.keys()) { + clearPtyState(id) + } + loadGeneration = 0 +} diff --git a/src/main/providers/ssh-filesystem-provider-watch-waiters.test.ts b/src/main/providers/ssh-filesystem-provider-watch-waiters.test.ts new file mode 100644 index 00000000000..375ed34848d --- /dev/null +++ b/src/main/providers/ssh-filesystem-provider-watch-waiters.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it, vi } from 'vitest' +import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' +import { registerSshFilesystemWatch, type WatchRegistration } from './ssh-filesystem-provider-watch' + +describe('registerSshFilesystemWatch waiters', () => { + it('removes ten thousand cancelled callers while one registration anchor remains', async () => { + let resolveWatch: () => void = () => undefined + const mux = { + request: vi.fn( + () => + new Promise((resolve) => { + resolveWatch = resolve + }) + ), + notify: vi.fn() + } as unknown as SshChannelMultiplexer + const registrations = new Map() + const watch = (signal?: AbortSignal) => + registerSshFilesystemWatch({ + mux, + registrations, + rootPath: '/home/user/project', + callback: vi.fn(), + signal, + disposed: () => false + }) + const anchor = watch() + const controllers = Array.from({ length: 10_000 }, () => new AbortController()) + const cancelled = controllers.map((controller) => + watch(controller.signal).catch((error) => error) + ) + + for (const controller of controllers) { + controller.abort() + } + await Promise.all(cancelled) + + expect(registrations.values().next().value?.setupWaiters.waiterCount).toBe(1) + resolveWatch() + await expect(anchor).resolves.toEqual(expect.any(Function)) + expect(registrations.values().next().value?.setupWaiters.waiterCount).toBe(0) + }) +}) diff --git a/src/main/providers/ssh-filesystem-provider-watch.ts b/src/main/providers/ssh-filesystem-provider-watch.ts index 02b87f3d005..8ef523860bd 100644 --- a/src/main/providers/ssh-filesystem-provider-watch.ts +++ b/src/main/providers/ssh-filesystem-provider-watch.ts @@ -1,10 +1,18 @@ import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer' import type { FsChangeEvent } from '../../shared/types' +import { isMethodNotFoundError } from '../ssh/ssh-filesystem-stream-reader' +import { normalizeRuntimePathForComparison } from '../../shared/cross-platform-path' +import { PromiseSettlementWaiters } from '../../shared/promise-settlement-waiters' + +let nextRemoteWatchId = 1 export type WatchRegistration = { + rootPath: string callbacks: Set<(events: FsChangeEvent[]) => void> - setupPromise: Promise + terminalCallbacks: Map<(events: FsChangeEvent[]) => void, (error: Error) => void> + setupWaiters: PromiseSettlementWaiters setupAbortController: AbortController + remoteWatchId: number | null ready: boolean stopping: boolean unwatchSent: boolean @@ -17,37 +25,12 @@ function createWatchAbortError(): Error { } async function awaitSetupWithOptionalAbort( - setupPromise: Promise, + setupWaiters: PromiseSettlementWaiters, signal?: AbortSignal ): Promise { - if (!signal) { - await setupPromise - return - } - if (signal.aborted) { - throw createWatchAbortError() - } - await new Promise((resolve, reject) => { - const onAbort = (): void => { - cleanup() - reject(createWatchAbortError()) - } - const onSettled = (error?: unknown): void => { - cleanup() - if (error === undefined) { - resolve() - } else { - reject(error) - } - } - const cleanup = (): void => { - signal.removeEventListener('abort', onAbort) - } - signal.addEventListener('abort', onAbort, { once: true }) - setupPromise.then( - () => onSettled(), - (error) => onSettled(error) - ) + await setupWaiters.wait({ + signal, + createAbortError: createWatchAbortError }) } @@ -57,6 +40,7 @@ export async function registerSshFilesystemWatch(args: { registrations: Map rootPath: string callback: (events: FsChangeEvent[]) => void + onTerminalError?: (error: Error) => void signal?: AbortSignal }): Promise<() => void> { if (args.disposed()) { @@ -66,16 +50,18 @@ export async function registerSshFilesystemWatch(args: { throw createWatchAbortError() } - let registration = args.registrations.get(args.rootPath) + const rootKey = sshFilesystemWatchKey(args.rootPath) + let registration = args.registrations.get(rootKey) if (registration) { if (registration.stopping) { throw createWatchAbortError() } registration.callbacks.add(args.callback) + registration.terminalCallbacks.set(args.callback, args.onTerminalError ?? (() => undefined)) try { // Why: each caller may leave a shared setup independently; registration // ownership decides whether the physical relay request should stop. - await awaitSetupWithOptionalAbort(registration.setupPromise, args.signal) + await awaitSetupWithOptionalAbort(registration.setupWaiters, args.signal) assertActiveWatch(args, registration) return createSshFilesystemWatchUnsubscribe(args, registration) } catch (error) { @@ -86,19 +72,30 @@ export async function registerSshFilesystemWatch(args: { const callbacks = new Set<(events: FsChangeEvent[]) => void>([args.callback]) const setupAbortController = new AbortController() + const remoteWatchId = nextRemoteWatchId++ + if (!Number.isSafeInteger(nextRemoteWatchId)) { + nextRemoteWatchId = 1 + } registration = { + rootPath: args.rootPath, callbacks, + terminalCallbacks: new Map([[args.callback, args.onTerminalError ?? (() => undefined)]]), setupAbortController, + remoteWatchId, ready: false, stopping: false, unwatchSent: false, - setupPromise: Promise.resolve() + setupWaiters: new PromiseSettlementWaiters(Promise.resolve()) } const createdRegistration = registration // Why: the shared registration, not its first caller, owns relay setup. // This keeps a same-root joiner alive when the original caller disconnects. - registration.setupPromise = args.mux - .request('fs.watch', { rootPath: args.rootPath }, { signal: setupAbortController.signal }) + const setupPromise = args.mux + .request( + 'fs.watch', + { rootPath: args.rootPath, watchId: remoteWatchId }, + { signal: setupAbortController.signal } + ) .then( () => { createdRegistration.ready = true @@ -106,24 +103,25 @@ export async function registerSshFilesystemWatch(args: { createdRegistration.stopping || createdRegistration.callbacks.size === 0 || args.disposed() || - args.registrations.get(args.rootPath) !== createdRegistration + args.registrations.get(rootKey) !== createdRegistration ) { - if (args.registrations.get(args.rootPath) === createdRegistration) { - args.registrations.delete(args.rootPath) + if (args.registrations.get(rootKey) === createdRegistration) { + args.registrations.delete(rootKey) } - sendSshFilesystemUnwatchOnce(args.mux, args.rootPath, createdRegistration) + sendSshFilesystemUnwatchOnce(args.mux, createdRegistration) } }, (error) => { - if (args.registrations.get(args.rootPath) === createdRegistration) { - args.registrations.delete(args.rootPath) + if (args.registrations.get(rootKey) === createdRegistration) { + args.registrations.delete(rootKey) } throw error } ) - args.registrations.set(args.rootPath, registration) + registration.setupWaiters = new PromiseSettlementWaiters(setupPromise) + args.registrations.set(rootKey, registration) try { - await awaitSetupWithOptionalAbort(registration.setupPromise, args.signal) + await awaitSetupWithOptionalAbort(registration.setupWaiters, args.signal) assertActiveWatch(args, registration) return createSshFilesystemWatchUnsubscribe(args, registration) } catch (error) { @@ -138,6 +136,51 @@ export function notifySshFilesystemUnwatch(mux: SshChannelMultiplexer, rootPath: } catch {} } +export async function closeSshFilesystemWatch( + mux: SshChannelMultiplexer, + registrations: Map, + rootPath: string +): Promise { + const rootKey = sshFilesystemWatchKey(rootPath) + try { + await mux.request('fs.unwatchAndWait', { rootPath }) + } catch (error) { + if (!isMethodNotFoundError(error)) { + throw error + } + throw new Error('Remote watcher teardown is unavailable. Reconnect the SSH target and retry.') + } + registrations.get(rootKey)?.callbacks.clear() + registrations.get(rootKey)?.terminalCallbacks.clear() + registrations.delete(rootKey) +} + +export function failSshFilesystemWatchRegistration( + registrations: Map, + rootPath: string, + remoteWatchId: number, + error: Error +): void { + const rootKey = sshFilesystemWatchKey(rootPath) + const registration = registrations.get(rootKey) + if (!registration || registration.remoteWatchId !== remoteWatchId) { + return + } + registrations.delete(rootKey) + registration.stopping = true + registration.unwatchSent = true + const terminalCallbacks = Array.from(registration.terminalCallbacks.values()) + registration.callbacks.clear() + registration.terminalCallbacks.clear() + for (const onTerminalError of terminalCallbacks) { + try { + onTerminalError(error) + } catch (callbackError) { + console.error('[ssh-fs] terminal watch callback failed', callbackError) + } + } +} + function assertActiveWatch( args: { disposed: () => boolean @@ -146,7 +189,10 @@ function assertActiveWatch( }, registration: WatchRegistration ): void { - if (args.disposed() || args.registrations.get(args.rootPath) !== registration) { + if ( + args.disposed() || + args.registrations.get(sshFilesystemWatchKey(args.rootPath)) !== registration + ) { throw new Error('SSH filesystem provider disposed') } } @@ -175,13 +221,15 @@ function releaseSshFilesystemWatchCallback( registration: WatchRegistration ): void { registration.callbacks.delete(args.callback) - if (registration.callbacks.size > 0 || args.registrations.get(args.rootPath) !== registration) { + registration.terminalCallbacks.delete(args.callback) + const rootKey = sshFilesystemWatchKey(args.rootPath) + if (registration.callbacks.size > 0 || args.registrations.get(rootKey) !== registration) { return } registration.stopping = true if (registration.ready) { - args.registrations.delete(args.rootPath) - sendSshFilesystemUnwatchOnce(args.mux, args.rootPath, registration) + args.registrations.delete(rootKey) + sendSshFilesystemUnwatchOnce(args.mux, registration) } else { // Keep the cancelling registration indexed until its request settles so a // new same-root setup cannot race an old late success and be unwatched. @@ -191,22 +239,24 @@ function releaseSshFilesystemWatchCallback( export function stopSshFilesystemWatchRegistration( mux: SshChannelMultiplexer, - rootPath: string, registration: WatchRegistration ): void { registration.stopping = true registration.setupAbortController.abort() - sendSshFilesystemUnwatchOnce(mux, rootPath, registration) + sendSshFilesystemUnwatchOnce(mux, registration) } function sendSshFilesystemUnwatchOnce( mux: SshChannelMultiplexer, - rootPath: string, registration: WatchRegistration ): void { if (registration.unwatchSent) { return } registration.unwatchSent = true - notifySshFilesystemUnwatch(mux, rootPath) + notifySshFilesystemUnwatch(mux, registration.rootPath) +} + +function sshFilesystemWatchKey(rootPath: string): string { + return normalizeRuntimePathForComparison(rootPath) } diff --git a/src/main/providers/ssh-filesystem-provider.test.ts b/src/main/providers/ssh-filesystem-provider.test.ts index d583a93d7cc..13ab2846baa 100644 --- a/src/main/providers/ssh-filesystem-provider.test.ts +++ b/src/main/providers/ssh-filesystem-provider.test.ts @@ -540,7 +540,7 @@ describe('SshFilesystemProvider', () => { expect(mux.request).toHaveBeenCalledWith( 'fs.watch', - { rootPath: '/home/user/project' }, + { rootPath: '/home/user/project', watchId: expect.any(Number) }, { signal: expect.any(AbortSignal) } ) expect(typeof unsub).toBe('function') @@ -555,7 +555,7 @@ describe('SshFilesystemProvider', () => { expect(mux.request).toHaveBeenCalledWith( 'fs.watch', - { rootPath: '/home/user/project' }, + { rootPath: '/home/user/project', watchId: expect.any(Number) }, { signal: expect.any(AbortSignal) } ) const physicalSignal = mux.request.mock.calls[0][2]?.signal @@ -642,6 +642,50 @@ describe('SshFilesystemProvider', () => { expect(callback).toHaveBeenCalledWith(events) }) + it('invalidates only the matching watch generation after relay recovery terminates', async () => { + const firstTerminal = vi.fn() + const secondTerminal = vi.fn() + await provider.watch('/home/user/project', vi.fn(), { + onTerminalError: firstTerminal + }) + await provider.watch('/home/user/project', vi.fn(), { + onTerminalError: secondTerminal + }) + const notifHandler = mux.onNotification.mock.calls[0][0] + const firstWatchId = mux.request.mock.calls[0][1].watchId as number + + notifHandler('fs.watchFailed', { + rootPath: '/home/user/project', + watchId: firstWatchId + 1, + message: 'stale failure' + }) + expect(firstTerminal).not.toHaveBeenCalled() + + notifHandler('fs.watchFailed', { + rootPath: '/home/user/project', + watchId: firstWatchId, + message: 'bounded recovery failed' + }) + expect(firstTerminal).toHaveBeenCalledWith( + expect.objectContaining({ message: 'bounded recovery failed' }) + ) + expect(secondTerminal).toHaveBeenCalledTimes(1) + + const replacementTerminal = vi.fn() + await provider.watch('/home/user/project', vi.fn(), { + onTerminalError: replacementTerminal + }) + const replacementWatchId = mux.request.mock.calls[1][1].watchId as number + expect(replacementWatchId).not.toBe(firstWatchId) + + notifHandler('fs.watchFailed', { + rootPath: '/home/user/project', + watchId: firstWatchId, + message: 'late stale failure' + }) + expect(replacementTerminal).not.toHaveBeenCalled() + }) + it('fans out same-root watch events and unwatches only after the last subscriber', async () => { const first = vi.fn() const second = vi.fn() @@ -651,7 +695,7 @@ describe('SshFilesystemProvider', () => { expect(mux.request).toHaveBeenCalledTimes(1) expect(mux.request).toHaveBeenCalledWith( 'fs.watch', - { rootPath: '/home/user/project' }, + { rootPath: '/home/user/project', watchId: expect.any(Number) }, { signal: expect.any(AbortSignal) } ) @@ -670,6 +714,40 @@ describe('SshFilesystemProvider', () => { expect(mux.notify).toHaveBeenCalledWith('fs.unwatch', { rootPath: '/home/user/project' }) }) + it('shares equivalent Windows spellings under one provider watch generation', async () => { + const first = vi.fn() + const second = vi.fn() + const firstTerminal = vi.fn() + const secondTerminal = vi.fn() + const unsubFirst = await provider.watch('C:\\Repo', first, { + onTerminalError: firstTerminal + }) + const unsubSecond = await provider.watch('c:/repo/', second, { + onTerminalError: secondTerminal + }) + + expect(mux.request).toHaveBeenCalledTimes(1) + const watchId = mux.request.mock.calls[0][1].watchId as number + const notification = mux.onNotification.mock.calls[0][0] + notification('fs.changed', { + events: [{ kind: 'update', absolutePath: 'c:/repo/file.ts' }] + }) + expect(first).toHaveBeenCalledTimes(1) + expect(second).toHaveBeenCalledTimes(1) + + unsubFirst() + expect(mux.notify).not.toHaveBeenCalledWith('fs.unwatch', { rootPath: 'C:\\Repo' }) + notification('fs.watchFailed', { + rootPath: 'c:/repo', + watchId, + message: 'bounded recovery failed' + }) + expect(firstTerminal).not.toHaveBeenCalled() + expect(secondTerminal).toHaveBeenCalledTimes(1) + + unsubSecond() + }) + it('shares an in-flight same-root watch setup across concurrent subscribers', async () => { let resolveWatch: () => void = () => {} mux.request.mockImplementationOnce( @@ -760,6 +838,42 @@ describe('SshFilesystemProvider', () => { expect(mux.notify).toHaveBeenCalledWith('fs.unwatch', { rootPath: '/home/user/project' }) }) + it('awaits acknowledged watcher teardown for destructive cleanup', async () => { + const callback = vi.fn() + await provider.watch('/home/user/project', callback) + mux.request.mockClear() + + await provider.closeWatch('/home/user/project') + + expect(mux.request).toHaveBeenCalledWith('fs.unwatchAndWait', { + rootPath: '/home/user/project' + }) + const notifHandler = mux.onNotification.mock.calls[0][0] + notifHandler('fs.changed', { + events: [{ kind: 'update', absolutePath: '/home/user/project/file.ts' }] + }) + expect(callback).not.toHaveBeenCalled() + }) + + it('fails destructive teardown closed on an older relay', async () => { + const callback = vi.fn() + await provider.watch('/home/user/project', callback) + mux.request.mockRejectedValueOnce( + Object.assign(new Error('Method not found'), { code: JsonRpcErrorCode.MethodNotFound }) + ) + + await expect(provider.closeWatch('/home/user/project')).rejects.toThrow( + 'Reconnect the SSH target' + ) + expect(mux.notify).not.toHaveBeenCalledWith('fs.unwatch', { + rootPath: '/home/user/project' + }) + const notifHandler = mux.onNotification.mock.calls[0][0] + const events = [{ kind: 'update' as const, absolutePath: '/home/user/project/file.ts' }] + notifHandler('fs.changed', { events }) + expect(callback).toHaveBeenCalledWith(events) + }) + it('sends fs.unwatch for active roots when disposed', async () => { const callback = vi.fn() await provider.watch('/home/user/project', callback) diff --git a/src/main/providers/ssh-filesystem-provider.ts b/src/main/providers/ssh-filesystem-provider.ts index ed4520e8f5a..507bfd98fdb 100644 --- a/src/main/providers/ssh-filesystem-provider.ts +++ b/src/main/providers/ssh-filesystem-provider.ts @@ -8,6 +8,7 @@ import { type SshRawTransferOptions } from './ssh-filesystem-file-upload' import { + closeSshFilesystemWatch, registerSshFilesystemWatch, stopSshFilesystemWatchRegistration, type WatchRegistration @@ -20,7 +21,7 @@ import type { TerminalArtifactAccessOptions } from './types' import type { DirEntry, FsChangeEvent, SearchOptions, SearchResult } from '../../shared/types' -import { isPathInsideOrEqual } from '../../shared/cross-platform-path' +import { routeSshFilesystemWatchNotification } from './ssh-filesystem-watch-notifications' import type { WorkspaceSpaceDirectoryScanResult } from '../../shared/workspace-space-types' const WORKSPACE_SPACE_SCAN_TIMEOUT_MS = 130_000 @@ -42,19 +43,9 @@ export class SshFilesystemProvider implements IFilesystemProvider { this.connectionId = connectionId this.mux = mux - this.unsubscribeNotifications = mux.onNotification((method, params) => { - if (method === 'fs.changed') { - const events = params.events as FsChangeEvent[] - for (const [rootPath, registration] of this.watchListeners) { - const matching = events.filter((e) => isPathInsideOrEqual(rootPath, e.absolutePath)) - if (matching.length > 0) { - for (const cb of registration.callbacks) { - cb(matching) - } - } - } - } - }) + this.unsubscribeNotifications = mux.onNotification((method, params) => + routeSshFilesystemWatchNotification(this.watchListeners, method, params) + ) } dispose(): void { @@ -66,8 +57,8 @@ export class SshFilesystemProvider implements IFilesystemProvider { this.unsubscribeNotifications() this.unsubscribeNotifications = null } - for (const [rootPath, registration] of this.watchListeners) { - stopSshFilesystemWatchRegistration(this.mux, rootPath, registration) + for (const registration of this.watchListeners.values()) { + stopSshFilesystemWatchRegistration(this.mux, registration) } this.watchListeners.clear() } @@ -319,7 +310,7 @@ export class SshFilesystemProvider implements IFilesystemProvider { async watch( rootPath: string, callback: (events: FsChangeEvent[]) => void, - options?: { signal?: AbortSignal } + options?: { signal?: AbortSignal; onTerminalError?: (error: Error) => void } ): Promise<() => void> { return registerSshFilesystemWatch({ mux: this.mux, @@ -327,7 +318,12 @@ export class SshFilesystemProvider implements IFilesystemProvider { registrations: this.watchListeners, rootPath, callback, + onTerminalError: options?.onTerminalError, signal: options?.signal }) } + + async closeWatch(rootPath: string): Promise { + await closeSshFilesystemWatch(this.mux, this.watchListeners, rootPath) + } } diff --git a/src/main/providers/ssh-filesystem-watch-notifications.ts b/src/main/providers/ssh-filesystem-watch-notifications.ts new file mode 100644 index 00000000000..97e3a3a4cdb --- /dev/null +++ b/src/main/providers/ssh-filesystem-watch-notifications.ts @@ -0,0 +1,37 @@ +import type { FsChangeEvent } from '../../shared/types' +import { isPathInsideOrEqual } from '../../shared/cross-platform-path' +import { + failSshFilesystemWatchRegistration, + type WatchRegistration +} from './ssh-filesystem-provider-watch' + +export function routeSshFilesystemWatchNotification( + registrations: Map, + method: string, + params: Record +): void { + if (method === 'fs.changed') { + const events = params.events as FsChangeEvent[] + for (const registration of registrations.values()) { + const matching = events.filter((event) => + isPathInsideOrEqual(registration.rootPath, event.absolutePath) + ) + if (matching.length > 0) { + for (const callback of registration.callbacks) { + callback(matching) + } + } + } + return + } + if (method !== 'fs.watchFailed') { + return + } + const rootPath = typeof params.rootPath === 'string' ? params.rootPath : null + const watchId = typeof params.watchId === 'number' ? params.watchId : null + if (rootPath && watchId !== null) { + const message = + typeof params.message === 'string' ? params.message : 'remote file watcher failed' + failSshFilesystemWatchRegistration(registrations, rootPath, watchId, new Error(message)) + } +} diff --git a/src/main/providers/ssh-pty-provider.test.ts b/src/main/providers/ssh-pty-provider.test.ts index ca0a9fd8261..04c3d4c6e24 100644 --- a/src/main/providers/ssh-pty-provider.test.ts +++ b/src/main/providers/ssh-pty-provider.test.ts @@ -486,10 +486,12 @@ describe('SshPtyProvider', () => { }) it('listProcesses returns process list', async () => { - const processes = [{ id: 'pty-1', cwd: '/home', title: 'zsh' }] + const processes = [{ id: 'pty-1', cwd: '/home', title: 'zsh', worktreeId: 'repo::/home' }] mux.request.mockResolvedValue(processes) const result = await provider.listProcesses() - expect(result).toEqual([{ id: scopedPty1, cwd: '/home', title: 'zsh' }]) + expect(result).toEqual([ + { id: scopedPty1, cwd: '/home', title: 'zsh', worktreeId: 'repo::/home' } + ]) }) it('getDefaultShell returns shell path', async () => { diff --git a/src/main/providers/types.ts b/src/main/providers/types.ts index 66642e38ad1..788d2aad596 100644 --- a/src/main/providers/types.ts +++ b/src/main/providers/types.ts @@ -148,6 +148,8 @@ export type PtyProcessInfo = { id: string cwd: string title: string + /** Owning worktree when the provider can report it authoritatively. */ + worktreeId?: string /** Trusted ORCA_TERMINAL_HANDLE exported into this PTY, when known. */ terminalHandle?: string } @@ -282,8 +284,9 @@ export type IFilesystemProvider = { watch( rootPath: string, callback: (events: FsChangeEvent[]) => void, - options?: { signal?: AbortSignal } + options?: { signal?: AbortSignal; onTerminalError?: (error: Error) => void } ): Promise<() => void> + closeWatch?(rootPath: string): Promise } export type FileUploadSession = { diff --git a/src/main/runtime/file-watcher-host.test.ts b/src/main/runtime/file-watcher-host.test.ts index e64a7e9d145..10f88f228f5 100644 --- a/src/main/runtime/file-watcher-host.test.ts +++ b/src/main/runtime/file-watcher-host.test.ts @@ -28,6 +28,8 @@ vi.mock('../ipc/parcel-watcher-process', () => ({ })) import { + closeFileExplorerWatcherInWatcherProcess, + getRuntimeRootWatchWaiterCountForTest, resetRuntimeRootWatchersForTest, watchFileExplorerInWatcherProcess } from './file-watcher-host' @@ -108,6 +110,31 @@ describe('watchFileExplorerInWatcherProcess', () => { expect(watch.unsubscribe).toHaveBeenCalledTimes(1) }) + it('joins root-scoped cleanup to an in-progress subscriber teardown', async () => { + let resolveUnsubscribe: () => void = () => undefined + const unsubscribe = vi.fn( + () => + new Promise((resolve) => { + resolveUnsubscribe = resolve + }) + ) + subscribeViaRuntimeWatcherProcessMock.mockResolvedValue({ unsubscribe }) + const dispose = await watchFileExplorerInWatcherProcess('/repo', vi.fn()) + + let rootCloseSettled = false + const subscriberClose = dispose() + const rootClose = closeFileExplorerWatcherInWatcherProcess('/repo').then(() => { + rootCloseSettled = true + }) + await Promise.resolve() + expect(rootCloseSettled).toBe(false) + expect(unsubscribe).toHaveBeenCalledTimes(1) + + resolveUnsubscribe() + await Promise.all([subscriberClose, rootClose]) + expect(rootCloseSettled).toBe(true) + }) + it('maps child-enriched watcher events without host-process stat work', async () => { const watch = installSuccessfulWatch() const onEvents = vi.fn<(events: FsChangeEvent[]) => void>() @@ -175,6 +202,116 @@ describe('watchFileExplorerInWatcherProcess', () => { expect(secondEvents).toHaveBeenCalledTimes(2) }) + it('waits for recovery-generation cancellation before releasing the root', async () => { + const watch = installSuccessfulWatch() + let rejectRecovery: ((error: Error) => void) | undefined + let abortObserved = false + subscribeViaRuntimeWatcherProcessMock.mockImplementationOnce( + (_rootPath, _callback, _options, hooks: WatcherProcessHooks) => + new Promise((_resolve, reject) => { + rejectRecovery = reject + hooks.signal?.addEventListener( + 'abort', + () => { + abortObserved = true + }, + { once: true } + ) + }) + ) + const dispose = await watchFileExplorerInWatcherProcess('/repo', vi.fn()) + watch.hooks.onTerminalError?.( + new WatcherProcessFailure('crashed repeatedly', 'supervisor', 'supervisor_crash_fuse') + ) + await vi.waitFor(() => expect(rejectRecovery).toBeDefined()) + + let released = false + const release = dispose().then(() => { + released = true + }) + await Promise.resolve() + expect(released).toBe(false) + expect(abortObserved).toBe(true) + + rejectRecovery?.( + new WatcherProcessFailure( + 'file watcher subscription aborted', + 'subscription', + 'subscribe_aborted' + ) + ) + await release + expect(released).toBe(true) + }) + + it('propagates recovery-generation physical termination failure', async () => { + const watch = installSuccessfulWatch() + let rejectRecovery: ((error: Error) => void) | undefined + subscribeViaRuntimeWatcherProcessMock.mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + rejectRecovery = reject + }) + ) + const dispose = await watchFileExplorerInWatcherProcess('/repo', vi.fn()) + watch.hooks.onTerminalError?.( + new WatcherProcessFailure('crashed repeatedly', 'supervisor', 'supervisor_crash_fuse') + ) + await vi.waitFor(() => expect(rejectRecovery).toBeDefined()) + const release = dispose() + const teardownError = new WatcherProcessFailure( + 'file watcher process did not exit after termination deadline', + 'supervisor', + 'process_unavailable' + ) + + rejectRecovery?.(teardownError) + await expect(release).rejects.toBe(teardownError) + }) + + it('retains an early recovery termination failure until physical exit', async () => { + const watch = installSuccessfulWatch() + let rejectRecovery: ((error: Error) => void) | undefined + subscribeViaRuntimeWatcherProcessMock.mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + rejectRecovery = reject + }) + ) + let resolvePhysicalExit: () => void = () => {} + const physicalExit = new Promise((resolve) => { + resolvePhysicalExit = resolve + }) + const teardownError = new WatcherProcessFailure( + 'file watcher process did not exit after termination deadline', + 'supervisor', + 'process_unavailable', + physicalExit + ) + const dispose = await watchFileExplorerInWatcherProcess('/repo', vi.fn()) + watch.hooks.onTerminalError?.( + new WatcherProcessFailure('crashed repeatedly', 'supervisor', 'supervisor_crash_fuse') + ) + await vi.waitFor(() => expect(rejectRecovery).toBeDefined()) + rejectRecovery?.(teardownError) + await Promise.resolve() + + await expect(dispose()).rejects.toBe(teardownError) + await expect(watchFileExplorerInWatcherProcess('/repo', vi.fn())).rejects.toBe(teardownError) + expect(subscribeViaRuntimeWatcherProcessMock).toHaveBeenCalledTimes(2) + resolvePhysicalExit() + await physicalExit + await vi.waitFor(() => + expect(forgetRuntimeWatcherProcessRootMock).toHaveBeenCalledWith('/repo') + ) + + const replacement = installSuccessfulWatch() + await expect(watchFileExplorerInWatcherProcess('/repo', vi.fn())).resolves.toEqual( + expect.any(Function) + ) + expect(replacement.unsubscribe).not.toHaveBeenCalled() + }) + it('ends subscribers only after isolated recovery also fails', async () => { const watch = installSuccessfulWatch() subscribeViaRuntimeWatcherProcessMock.mockRejectedValueOnce(new Error('isolated failure')) @@ -216,6 +353,61 @@ describe('watchFileExplorerInWatcherProcess', () => { expect(physicalSignal?.aborted).toBe(true) }) + it('keeps pre-ready cancellation pending until the root can be replaced', async () => { + let physicalSignal: AbortSignal | undefined + let rejectPhysical: ((error: unknown) => void) | undefined + subscribeViaRuntimeWatcherProcessMock.mockImplementationOnce( + (_rootPath, _callback, _options, hooks: WatcherProcessHooks) => { + physicalSignal = hooks.signal + return new Promise((_resolve, reject) => { + rejectPhysical = reject + }) + } + ) + const controllers = [new AbortController(), new AbortController()] + const pending = controllers.map((controller) => + watchFileExplorerInWatcherProcess('/repo', vi.fn(), vi.fn(), controller.signal) + ) + const settled = [false, false] + pending.forEach((promise, index) => { + void promise.then( + () => { + settled[index] = true + }, + () => { + settled[index] = true + } + ) + }) + + controllers.forEach((controller) => controller.abort()) + await Promise.resolve() + + expect(physicalSignal?.aborted).toBe(true) + expect(settled).toEqual([false, false]) + + rejectPhysical?.( + new WatcherProcessFailure('physical watch aborted', 'subscription', 'subscribe_aborted') + ) + const results = await Promise.allSettled(pending) + expect(results).toEqual([ + expect.objectContaining({ + status: 'rejected', + reason: expect.objectContaining({ message: 'file watcher subscription aborted' }) + }), + expect.objectContaining({ + status: 'rejected', + reason: expect.objectContaining({ message: 'file watcher subscription aborted' }) + }) + ]) + + installSuccessfulWatch() + await expect(watchFileExplorerInWatcherProcess('/repo', vi.fn())).resolves.toEqual( + expect.any(Function) + ) + expect(subscribeViaRuntimeWatcherProcessMock).toHaveBeenCalledTimes(2) + }) + it('keeps pending setup alive when another same-root client still owns it', async () => { let physicalSignal: AbortSignal | undefined let resolvePhysical: ((subscription: WatcherProcessSubscription) => void) | undefined @@ -241,6 +433,40 @@ describe('watchFileExplorerInWatcherProcess', () => { expect(subscribeViaRuntimeWatcherProcessMock).toHaveBeenCalledTimes(1) }) + it('removes ten thousand cancelled root callers while one anchor remains', async () => { + let resolvePhysical: ((subscription: WatcherProcessSubscription) => void) | undefined + const unsubscribe = vi.fn(async () => undefined) + subscribeViaRuntimeWatcherProcessMock.mockImplementation( + () => + new Promise((resolve) => { + resolvePhysical = resolve + }) + ) + const anchorController = new AbortController() + const anchor = watchFileExplorerInWatcherProcess( + '/repo', + vi.fn(), + vi.fn(), + anchorController.signal + ) + const controllers = Array.from({ length: 10_000 }, () => new AbortController()) + const cancelled = controllers.map((controller) => + watchFileExplorerInWatcherProcess('/repo', vi.fn(), vi.fn(), controller.signal).catch( + (error) => error + ) + ) + + for (const controller of controllers) { + controller.abort() + } + await Promise.all(cancelled) + + expect(getRuntimeRootWatchWaiterCountForTest('/repo')).toBe(1) + resolvePhysical?.({ unsubscribe }) + await expect(anchor).resolves.toEqual(expect.any(Function)) + expect(getRuntimeRootWatchWaiterCountForTest('/repo')).toBe(0) + }) + it('shares one native subscription per root across same-root callers', async () => { const firstRoot = installSuccessfulWatch() const secondRoot = installSuccessfulWatch() @@ -298,4 +524,33 @@ describe('watchFileExplorerInWatcherProcess', () => { ) expect(subscribeViaRuntimeWatcherProcessMock).toHaveBeenCalledTimes(1) }) + + it('retains failed initial root ownership until the exact child physically exits', async () => { + let resolvePhysicalExit: () => void = () => {} + const physicalExit = new Promise((resolve) => { + resolvePhysicalExit = resolve + }) + const teardownError = new WatcherProcessFailure( + 'initial watcher child did not exit', + 'subscription', + 'subscribe_failed', + physicalExit + ) + subscribeViaRuntimeWatcherProcessMock.mockRejectedValueOnce(teardownError) + + await expect(watchFileExplorerInWatcherProcess('/repo', vi.fn())).rejects.toBe(teardownError) + await expect(closeFileExplorerWatcherInWatcherProcess('/repo')).rejects.toBe(teardownError) + await expect(watchFileExplorerInWatcherProcess('/repo', vi.fn())).rejects.toBe(teardownError) + expect(subscribeViaRuntimeWatcherProcessMock).toHaveBeenCalledTimes(1) + expect(forgetRuntimeWatcherProcessRootMock).not.toHaveBeenCalled() + + resolvePhysicalExit() + await physicalExit + const replacement = installSuccessfulWatch() + await expect(watchFileExplorerInWatcherProcess('/repo', vi.fn())).resolves.toEqual( + expect.any(Function) + ) + expect(replacement.unsubscribe).not.toHaveBeenCalled() + await expect(closeFileExplorerWatcherInWatcherProcess('/repo')).resolves.toBeUndefined() + }) }) diff --git a/src/main/runtime/file-watcher-host.ts b/src/main/runtime/file-watcher-host.ts index 68e5711150d..59a8071bd58 100644 --- a/src/main/runtime/file-watcher-host.ts +++ b/src/main/runtime/file-watcher-host.ts @@ -6,15 +6,24 @@ import { type WatcherProcessEvent, type WatcherProcessSubscription } from '../ipc/parcel-watcher-process' -import { isWatcherProcessFailure } from '../ipc/parcel-watcher-process-failure' +import { + isWatcherProcessFailure, + type WatcherProcessFailure +} from '../ipc/parcel-watcher-process-failure' import { WATCHER_IGNORE_DIRS, buildParcelWatcherIgnoreOptions } from '../ipc/filesystem-watcher-ignore' +import { createRuntimeRootOwnershipReleaser } from './runtime-root-watch-ownership' +import { closeRuntimeRootWatch } from './runtime-root-watch-teardown' +import { + RUNTIME_FILE_WATCH_CRAWL_TIMEOUT_MS, + RUNTIME_FILE_WATCH_MAX_SETUP_ATTEMPTS +} from '../../shared/runtime-file-watch-limits' +import { PromiseSettlementWaiters } from '../../shared/promise-settlement-waiters' const RUNTIME_FILE_WATCH_IGNORE_OPTIONS = buildParcelWatcherIgnoreOptions(WATCHER_IGNORE_DIRS) const RUNTIME_FILE_WATCH_EVENT_LIMIT = 200 -const RUNTIME_FILE_WATCH_CRAWL_TIMEOUT_MS = 60_000 type RuntimeFileWatchSubscriber = { onEvents: (events: FsChangeEvent[]) => void @@ -24,16 +33,22 @@ type RuntimeFileWatchSubscriber = { type RuntimeRootWatch = { rootPath: string subscribers: Set - start: Promise + startWaiters: PromiseSettlementWaiters subscription: WatcherProcessSubscription | null abortController: AbortController generation: number closed: boolean + closePromise: Promise | null + terminalReleaseError: WatcherProcessFailure | null } // Why: paired clients watching the same worktree share one native subscription. // Different roots are assigned across the bounded crash-isolated child pool. const runtimeRootWatches = new Map() +const { + release: releaseRuntimeRootOwnership, + releaseAfterFailure: releaseRuntimeRootAfterFailure +} = createRuntimeRootOwnershipReleaser(runtimeRootWatches, forgetRuntimeWatcherProcessRoot) function overflowEvent(rootPath: string): FsChangeEvent[] { return [{ kind: 'overflow', absolutePath: rootPath }] @@ -63,10 +78,7 @@ function terminateRootWatch(root: RuntimeRootWatch, error: Error): void { root.closed = true root.generation++ root.abortController.abort() - if (runtimeRootWatches.get(root.rootPath) === root) { - runtimeRootWatches.delete(root.rootPath) - } - forgetRuntimeWatcherProcessRoot(root.rootPath) + releaseRuntimeRootAfterFailure(root, error) const subscribers = Array.from(root.subscribers) root.subscribers.clear() for (const subscriber of subscribers) { @@ -74,14 +86,14 @@ function terminateRootWatch(root: RuntimeRootWatch, error: Error): void { } } -function closeInitialRootWatch(root: RuntimeRootWatch): void { +function closeInitialRootWatch(root: RuntimeRootWatch, error: unknown): void { root.closed = true root.generation++ root.abortController.abort() - if (runtimeRootWatches.get(root.rootPath) === root) { - runtimeRootWatches.delete(root.rootPath) - } - forgetRuntimeWatcherProcessRoot(root.rootPath) + releaseRuntimeRootAfterFailure( + root, + error instanceof Error ? error : new Error('file watcher setup failed') + ) root.subscribers.clear() } @@ -142,19 +154,22 @@ function subscribeRuntimeRootWatch(root: RuntimeRootWatch): Promise { } async function startInitialRuntimeRootWatch(root: RuntimeRootWatch): Promise { - try { - await subscribeRuntimeRootWatch(root) - } catch (firstError) { - if (root.closed || !shouldRetryInitialWatch(firstError)) { - closeInitialRootWatch(root) - throw firstError - } + for (let attempt = 0; attempt < RUNTIME_FILE_WATCH_MAX_SETUP_ATTEMPTS; attempt++) { try { await subscribeRuntimeRootWatch(root) - emitToSubscribers(root, overflowEvent(root.rootPath)) - } catch (isolatedError) { - closeInitialRootWatch(root) - throw isolatedError + if (attempt > 0) { + emitToSubscribers(root, overflowEvent(root.rootPath)) + } + return + } catch (error) { + if ( + root.closed || + attempt + 1 >= RUNTIME_FILE_WATCH_MAX_SETUP_ATTEMPTS || + !shouldRetryInitialWatch(error) + ) { + closeInitialRootWatch(root, error) + throw error + } } } } @@ -169,25 +184,30 @@ function recoverRuntimeRootWatch( } root.subscription = null emitToSubscribers(root, overflowEvent(root.rootPath)) - root.start = subscribeRuntimeRootWatch(root).catch((recoveryError: unknown) => { + const recovery = subscribeRuntimeRootWatch(root).catch((recoveryError: unknown) => { if (!root.closed) { terminateRootWatch(root, recoveryError instanceof Error ? recoveryError : terminalError) + return } + throw recoveryError }) + root.startWaiters = new PromiseSettlementWaiters(recovery) } function createRuntimeRootWatch(rootPath: string): RuntimeRootWatch { const root: RuntimeRootWatch = { rootPath, subscribers: new Set(), - start: Promise.resolve(), + startWaiters: new PromiseSettlementWaiters(Promise.resolve()), subscription: null, abortController: new AbortController(), generation: 0, - closed: false + closed: false, + closePromise: null, + terminalReleaseError: null } runtimeRootWatches.set(rootPath, root) - root.start = startInitialRuntimeRootWatch(root) + root.startWaiters = new PromiseSettlementWaiters(startInitialRuntimeRootWatch(root)) return root } @@ -196,16 +216,40 @@ async function releaseRuntimeRootWatch( subscriber: RuntimeFileWatchSubscriber ): Promise { root.subscribers.delete(subscriber) - if (root.closed || root.subscribers.size > 0) { + if (root.closed) { + if (root.closePromise) { + await root.closePromise + } + if (root.terminalReleaseError) { + throw root.terminalReleaseError + } return } - root.closed = true - root.generation++ - root.abortController.abort() - if (runtimeRootWatches.get(root.rootPath) === root) { - runtimeRootWatches.delete(root.rootPath) + if (root.subscribers.size > 0) { + return } - await root.subscription?.unsubscribe() + await closeRuntimeRootWatchOnce(root) +} + +function closeRuntimeRootWatchOnce(root: RuntimeRootWatch): Promise { + root.closePromise ??= closeRuntimeRootWatch( + root, + root.startWaiters.promise, + () => releaseRuntimeRootOwnership(root), + (error) => releaseRuntimeRootAfterFailure(root, error) + ) + return root.closePromise +} + +export async function closeFileExplorerWatcherInWatcherProcess(rootPath: string): Promise { + const root = runtimeRootWatches.get(rootPath) + if (!root) { + return + } + // Why: destructive cleanup owns every same-root subscriber, including a + // setup that failed before it could publish an unsubscribe callback. + root.subscribers.clear() + await closeRuntimeRootWatchOnce(root) } /** @@ -219,13 +263,28 @@ export async function watchFileExplorerInWatcherProcess( onTerminalError: (error: Error) => void = () => undefined, signal?: AbortSignal ): Promise<() => Promise> { - const root = runtimeRootWatches.get(rootPath) ?? createRuntimeRootWatch(rootPath) + const existingRoot = runtimeRootWatches.get(rootPath) + if (existingRoot?.closed) { + if (!existingRoot.closePromise) { + throw existingRoot.terminalReleaseError ?? new Error('file watcher closed during setup') + } + // Why: a replacement cannot share a native root whose prior generation + // is still closing; join its exact release before creating the next owner. + await existingRoot.closePromise + if (signal?.aborted) { + throw createRuntimeRootAbortError() + } + return watchFileExplorerInWatcherProcess(rootPath, callback, onTerminalError, signal) + } + const root = existingRoot ?? createRuntimeRootWatch(rootPath) const subscriber: RuntimeFileWatchSubscriber = { onEvents: callback, onTerminalError } root.subscribers.add(subscriber) try { await waitForRuntimeRootStart(root, subscriber, signal) } catch (error) { - root.subscribers.delete(subscriber) + // Why: paired unwatch cannot acknowledge until native cancellation or + // physical exit releases this root for an immediate replacement watch. + await releaseRuntimeRootWatch(root, subscriber) throw error } if (root.closed) { @@ -245,34 +304,23 @@ function waitForRuntimeRootStart( signal: AbortSignal | undefined ): Promise { if (!signal) { - return root.start + return root.startWaiters.promise } if (signal.aborted) { - void releaseRuntimeRootWatch(root, subscriber) + root.subscribers.delete(subscriber) return Promise.reject(new Error('file watcher subscription aborted')) } - return new Promise((resolve, reject) => { - let settled = false - const finish = (callback: () => void): void => { - if (settled) { - return - } - settled = true - signal.removeEventListener('abort', handleAbort) - callback() - } - const handleAbort = (): void => { - void releaseRuntimeRootWatch(root, subscriber) - finish(() => reject(new Error('file watcher subscription aborted'))) - } - signal.addEventListener('abort', handleAbort, { once: true }) - root.start.then( - () => finish(resolve), - (error: unknown) => finish(() => reject(error)) - ) + return root.startWaiters.wait({ + signal, + createAbortError: createRuntimeRootAbortError, + onAbandon: () => root.subscribers.delete(subscriber) }) } +function createRuntimeRootAbortError(): Error { + return new Error('file watcher subscription aborted') +} + export function resetRuntimeRootWatchersForTest(): void { for (const root of runtimeRootWatches.values()) { root.closed = true @@ -284,3 +332,7 @@ export function resetRuntimeRootWatchersForTest(): void { runtimeRootWatches.clear() resetRuntimeWatcherProcessForTest() } + +export function getRuntimeRootWatchWaiterCountForTest(rootPath: string): number { + return runtimeRootWatches.get(rootPath)?.startWaiters.waiterCount ?? 0 +} diff --git a/src/main/runtime/orca-runtime-files-watch.test.ts b/src/main/runtime/orca-runtime-files-watch.test.ts index 7986e0fc9b0..248330472e6 100644 --- a/src/main/runtime/orca-runtime-files-watch.test.ts +++ b/src/main/runtime/orca-runtime-files-watch.test.ts @@ -1,20 +1,25 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { EventEmitter } from 'node:events' import type * as Fs from 'node:fs' import type * as FsPromises from 'node:fs/promises' import type * as FilesystemAuth from '../ipc/filesystem-auth' import type { FsChangeEvent } from '../../shared/types' +import { WatcherProcessFailure } from '../ipc/parcel-watcher-process-failure' +import { acquireWatcherRemovalGate } from '../ipc/watcher-removal-gate' const { resolveAuthorizedPathMock, statMock, watchMock, watchInWatcherProcessMock, + closeWatcherInWatcherProcessMock, getSshFilesystemProviderMock } = vi.hoisted(() => ({ resolveAuthorizedPathMock: vi.fn(), statMock: vi.fn(), watchMock: vi.fn(), watchInWatcherProcessMock: vi.fn(), + closeWatcherInWatcherProcessMock: vi.fn(), getSshFilesystemProviderMock: vi.fn() })) @@ -36,6 +41,7 @@ vi.mock('fs/promises', async () => { // The local (non-Windows, non-SSH) watch path delegates to the isolated watcher process. vi.mock('./file-watcher-host', () => ({ + closeFileExplorerWatcherInWatcherProcess: closeWatcherInWatcherProcessMock, watchFileExplorerInWatcherProcess: watchInWatcherProcessMock })) @@ -51,7 +57,18 @@ vi.mock('../providers/ssh-filesystem-dispatch', () => ({ getSshFilesystemProvider: getSshFilesystemProviderMock })) -import { awaitRuntimeFileWatcherUnsubscribes, RuntimeFileCommands } from './orca-runtime-files' +import { + _getRuntimeFileWatcherReleaseCountForTests, + awaitRuntimeFileWatcherUnsubscribes, + RuntimeFileCommands, + WINDOWS_RUNTIME_FILE_WATCH_CLOSE_DEADLINE_MS +} from './orca-runtime-files' + +function createWindowsWatcher(close: () => void) { + const watcher = new EventEmitter() as EventEmitter & { close: ReturnType } + watcher.close = vi.fn(close) + return watcher +} function createRuntimeFileCommands(rootPath: string) { const store = { getRepo: vi.fn(() => undefined) } @@ -87,6 +104,7 @@ describe('RuntimeFileCommands file watching', () => { statMock.mockReset() watchMock.mockReset() watchInWatcherProcessMock.mockReset() + closeWatcherInWatcherProcessMock.mockReset() getSshFilesystemProviderMock.mockReset() Object.defineProperty(process, 'platform', { configurable: true, @@ -109,12 +127,13 @@ describe('RuntimeFileCommands file watching', () => { value: 'win32' }) - const close = vi.fn() - const on = vi.fn() + const watcher = createWindowsWatcher(() => { + queueMicrotask(() => watcher.emit('close')) + }) let listener: (() => void) | null = null watchMock.mockImplementation((_rootPath, _options, callback) => { listener = callback - return { close, on } + return watcher }) resolveAuthorizedPathMock.mockResolvedValue('C:\\repo') statMock.mockResolvedValue({ isDirectory: () => true }) @@ -138,8 +157,110 @@ describe('RuntimeFileCommands file watching', () => { expect(onEvents).toHaveBeenCalledTimes(1) expect(onEvents).toHaveBeenCalledWith([{ kind: 'overflow', absolutePath: 'C:\\repo' }]) - unsubscribe() - expect(close).toHaveBeenCalledTimes(1) + await unsubscribe() + expect(watcher.close).toHaveBeenCalledTimes(1) + }) + + it('waits for the Windows watcher close event before allowing deletion', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + const watcher = createWindowsWatcher(() => undefined) + watchMock.mockReturnValue(watcher) + resolveAuthorizedPathMock.mockResolvedValue('C:\\repo') + statMock.mockResolvedValue({ isDirectory: () => true }) + const { commands } = createRuntimeFileCommands('C:\\repo') + await commands.watchFileExplorer('id:wt-1', vi.fn()) + + let settled = false + const closePromise = commands.closeFileExplorerWatchersForPath('C:\\repo').then(() => { + settled = true + }) + await Promise.resolve() + expect(settled).toBe(false) + + watcher.emit('close') + await closePromise + expect(settled).toBe(true) + }) + + it('treats a Windows watcher error before cleanup as physical close', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + const watcher = createWindowsWatcher(() => undefined) + watchMock.mockReturnValue(watcher) + resolveAuthorizedPathMock.mockResolvedValue('C:\\repo') + statMock.mockResolvedValue({ isDirectory: () => true }) + const { commands } = createRuntimeFileCommands('C:\\repo') + const onEvents = vi.fn() + const onTerminalError = vi.fn() + await commands.watchFileExplorer('id:wt-1', onEvents, onTerminalError) + const watchError = new Error('native directory handle closed') + + watcher.emit('error', watchError) + + expect(onEvents).toHaveBeenCalledWith([{ kind: 'overflow', absolutePath: 'C:\\repo' }]) + expect(onTerminalError).toHaveBeenCalledWith(watchError) + await expect(commands.closeFileExplorerWatchersForPath('C:\\repo')).resolves.toBeUndefined() + expect(watcher.close).toHaveBeenCalledTimes(1) + expect(watcher.listenerCount('close')).toBe(0) + expect(watcher.listenerCount('error')).toBe(0) + expect(_getRuntimeFileWatcherReleaseCountForTests()).toBe(0) + expect(vi.getTimerCount()).toBe(0) + }) + + it('retains Windows close ownership until late physical exit without retry leaks', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + const watcher = createWindowsWatcher(() => undefined) + watchMock.mockReturnValue(watcher) + resolveAuthorizedPathMock.mockResolvedValue('C:\\repo') + statMock.mockResolvedValue({ isDirectory: () => true }) + const { commands } = createRuntimeFileCommands('C:\\repo') + await commands.watchFileExplorer('id:wt-1', vi.fn()) + + const closePromise = commands.closeFileExplorerWatchersForPath('C:\\repo') + const closeResult = closePromise.catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(WINDOWS_RUNTIME_FILE_WATCH_CLOSE_DEADLINE_MS) + const closeError = (await closeResult) as WatcherProcessFailure + expect(closeError).toBeInstanceOf(WatcherProcessFailure) + expect(closeError).toMatchObject({ + message: 'Windows watcher did not close before deletion deadline', + physicalExit: expect.any(Promise) + }) + expect(_getRuntimeFileWatcherReleaseCountForTests()).toBe(1) + expect(vi.getTimerCount()).toBe(0) + + const retries = await Promise.allSettled( + Array.from({ length: 100 }, () => commands.closeFileExplorerWatchersForPath('C:\\repo')) + ) + expect(retries.every((result) => result.status === 'rejected')).toBe(true) + expect(watcher.close).toHaveBeenCalledTimes(1) + expect(watcher.listenerCount('close')).toBe(1) + expect(vi.getTimerCount()).toBe(0) + + watcher.emit('close') + await closeError.physicalExit + await Promise.resolve() + expect(_getRuntimeFileWatcherReleaseCountForTests()).toBe(0) + await expect(commands.closeFileExplorerWatchersForPath('C:\\repo')).resolves.toBeUndefined() + expect(watcher.close).toHaveBeenCalledTimes(1) + }) + + it('retries a failed Windows watcher close before allowing deletion', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + const closeError = new Error('Windows watcher handle still active') + const watcher = createWindowsWatcher(() => { + if (watcher.close.mock.calls.length === 1) { + throw closeError + } + queueMicrotask(() => watcher.emit('close')) + }) + watchMock.mockReturnValue(watcher) + resolveAuthorizedPathMock.mockResolvedValue('C:\\repo') + statMock.mockResolvedValue({ isDirectory: () => true }) + const { commands } = createRuntimeFileCommands('C:\\repo') + await commands.watchFileExplorer('id:wt-1', vi.fn()) + + await expect(commands.closeFileExplorerWatchersForPath('C:\\repo')).rejects.toBe(closeError) + await expect(commands.closeFileExplorerWatchersForPath('C:\\repo')).resolves.toBeUndefined() + expect(watcher.close).toHaveBeenCalledTimes(2) }) // Issues #5308/#8212: the local recursive watch runs out of process so the @@ -196,6 +317,101 @@ describe('RuntimeFileCommands file watching', () => { ) }) + posixWatcherProcessIt( + 'retains pre-publication watcher setup ownership through destructive cleanup', + async () => { + resolveAuthorizedPathMock.mockResolvedValue('/repo') + statMock.mockResolvedValue({ isDirectory: () => true }) + let resolvePhysicalExit: () => void = () => undefined + const physicalExit = new Promise((resolve) => { + resolvePhysicalExit = resolve + }) + const teardownError = new WatcherProcessFailure( + 'file watcher process did not exit after termination deadline', + 'supervisor', + 'process_unavailable', + physicalExit + ) + watchInWatcherProcessMock.mockRejectedValue(teardownError) + closeWatcherInWatcherProcessMock.mockRejectedValueOnce(teardownError) + const { commands } = createRuntimeFileCommands('/repo') + + await expect(commands.watchFileExplorer('id:wt-1', vi.fn())).rejects.toBe(teardownError) + await expect(commands.closeFileExplorerWatchersForPath('/repo')).rejects.toBe(teardownError) + expect(closeWatcherInWatcherProcessMock).toHaveBeenCalledWith('/repo') + + resolvePhysicalExit() + await physicalExit + await expect(commands.closeFileExplorerWatchersForPath('/repo')).resolves.toBeUndefined() + } + ) + + posixWatcherProcessIt('rejects a runtime watch throughout destructive removal', async () => { + resolveAuthorizedPathMock.mockResolvedValue('/repo') + statMock.mockResolvedValue({ isDirectory: () => true }) + const watcherDispose = vi.fn() + watchInWatcherProcessMock.mockResolvedValue(watcherDispose) + const { commands } = createRuntimeFileCommands('/repo') + const removal = acquireWatcherRemovalGate('/repo') + await removal.ready + + await expect(commands.watchFileExplorer('id:wt-1', vi.fn())).rejects.toMatchObject({ + code: 'watcher_removal_in_progress' + }) + expect(watchInWatcherProcessMock).not.toHaveBeenCalled() + + removal.release() + const unsubscribe = await commands.watchFileExplorer('id:wt-1', vi.fn()) + await unsubscribe() + expect(watchInWatcherProcessMock).toHaveBeenCalledTimes(1) + }) + + posixWatcherProcessIt('awaits root-scoped runtime watcher teardown for deletion', async () => { + resolveAuthorizedPathMock.mockResolvedValue('/repo') + statMock.mockResolvedValue({ isDirectory: () => true }) + const teardownError = new Error('runtime watcher teardown failed') + const watcherDispose = vi.fn().mockRejectedValueOnce(teardownError) + watchInWatcherProcessMock.mockResolvedValue(watcherDispose) + const { commands } = createRuntimeFileCommands('/repo') + + await commands.watchFileExplorer('id:wt-1', vi.fn()) + + await expect(commands.closeFileExplorerWatchersForPath('/repo')).rejects.toBe(teardownError) + await expect(commands.closeFileExplorerWatchersForPath('/repo')).resolves.toBeUndefined() + expect(watcherDispose).toHaveBeenCalledTimes(2) + }) + + posixWatcherProcessIt('clears a failed runtime release after physical child exit', async () => { + resolveAuthorizedPathMock.mockResolvedValue('/repo') + statMock.mockResolvedValue({ isDirectory: () => true }) + let resolvePhysicalExit: () => void = () => {} + const physicalExit = new Promise((resolve) => { + resolvePhysicalExit = resolve + }) + const teardownError = new WatcherProcessFailure( + 'file watcher process did not exit after termination deadline', + 'supervisor', + 'process_unavailable', + physicalExit + ) + const watcherDispose = vi.fn().mockRejectedValue(teardownError) + watchInWatcherProcessMock.mockResolvedValue(watcherDispose) + const { commands } = createRuntimeFileCommands('/repo') + + await commands.watchFileExplorer('id:wt-1', vi.fn()) + + await expect(commands.closeFileExplorerWatchersForPath('/repo')).rejects.toBe(teardownError) + await expect(commands.closeFileExplorerWatchersForPath('/repo')).rejects.toBe(teardownError) + expect(watcherDispose).toHaveBeenCalledTimes(1) + + resolvePhysicalExit() + await physicalExit + await Promise.resolve() + + await expect(commands.closeFileExplorerWatchersForPath('/repo')).resolves.toBeUndefined() + expect(watcherDispose).toHaveBeenCalledTimes(1) + }) + posixWatcherProcessIt('tracks watcher unsubscribe work so shutdown can await it', async () => { resolveAuthorizedPathMock.mockResolvedValue('/repo') statMock.mockResolvedValue({ isDirectory: () => true }) @@ -251,11 +467,13 @@ describe('RuntimeFileCommands file watching', () => { openFile: vi.fn() } as never) const controller = new AbortController() + const onTerminalError = vi.fn() - await commands.watchFileExplorer('id:wt-1', vi.fn(), vi.fn(), controller.signal) + await commands.watchFileExplorer('id:wt-1', vi.fn(), onTerminalError, controller.signal) expect(watch).toHaveBeenCalledWith('/remote/repo', expect.any(Function), { - signal: controller.signal + signal: controller.signal, + onTerminalError }) expect(watchInWatcherProcessMock).not.toHaveBeenCalled() }) diff --git a/src/main/runtime/orca-runtime-files.test.ts b/src/main/runtime/orca-runtime-files.test.ts index d43e48c4695..e38d0adf0d0 100644 --- a/src/main/runtime/orca-runtime-files.test.ts +++ b/src/main/runtime/orca-runtime-files.test.ts @@ -19,6 +19,7 @@ const { resolveAuthorizedPathMock, statMock, watchInWatcherProcessMock, + closeWatcherInWatcherProcessMock, checkRgAvailableMock, getLocalGitOptionsForRegisteredWorktreeMock, wslAwareSpawnMock, @@ -33,6 +34,7 @@ const { resolveAuthorizedPathMock: vi.fn(), statMock: vi.fn(), watchInWatcherProcessMock: vi.fn(), + closeWatcherInWatcherProcessMock: vi.fn(), wslAwareSpawnMock: vi.fn(), watchMock: vi.fn() })) @@ -61,6 +63,7 @@ vi.mock('fs/promises', async () => { }) vi.mock('./file-watcher-host', () => ({ + closeFileExplorerWatcherInWatcherProcess: closeWatcherInWatcherProcessMock, watchFileExplorerInWatcherProcess: watchInWatcherProcessMock })) @@ -200,6 +203,7 @@ describe('RuntimeFileCommands', () => { resolveAuthorizedPathMock.mockReset() statMock.mockReset() watchInWatcherProcessMock.mockReset() + closeWatcherInWatcherProcessMock.mockReset() watchMock.mockReset() checkRgAvailableMock.mockReset() vi.mocked(getSshFilesystemProvider).mockReset() @@ -410,12 +414,12 @@ describe('RuntimeFileCommands', () => { value: 'win32' }) - const close = vi.fn() - const on = vi.fn() + const watcher = new EventEmitter() as EventEmitter & { close: ReturnType } + watcher.close = vi.fn(() => queueMicrotask(() => watcher.emit('close'))) let listener: (() => void) | null = null watchMock.mockImplementation((_rootPath, _options, callback) => { listener = callback - return { close, on } + return watcher }) resolveAuthorizedPathMock.mockResolvedValue('C:\\repo') statMock.mockResolvedValue({ isDirectory: () => true }) @@ -437,8 +441,8 @@ describe('RuntimeFileCommands', () => { expect(onEvents).toHaveBeenCalledTimes(1) expect(onEvents).toHaveBeenCalledWith([{ kind: 'overflow', absolutePath: 'C:\\repo' }]) - unsubscribe() - expect(close).toHaveBeenCalledTimes(1) + await unsubscribe() + expect(watcher.close).toHaveBeenCalledTimes(1) }) it('delegates local recursive watching to the watcher process', async () => { @@ -465,6 +469,77 @@ describe('RuntimeFileCommands', () => { expect(dispose).toHaveBeenCalledTimes(1) }) + it('keeps SSH runtime watches on the remote filesystem provider', async () => { + const remoteDispose = vi.fn() + const providerWatch = vi.fn(() => remoteDispose) + vi.mocked(getSshFilesystemProvider).mockReturnValue({ watch: providerWatch } as never) + const { commands, store } = createRuntimeFileCommands({ path: '/remote/repo' }) + store.getRepo.mockReturnValue({ connectionId: 'ssh-1' }) + const onEvents = vi.fn() + + const unsubscribe = await commands.watchFileExplorer('id:wt-1', onEvents) + + expect(providerWatch).toHaveBeenCalledWith('/remote/repo', onEvents, { + signal: undefined, + onTerminalError: expect.any(Function) + }) + expect(watchInWatcherProcessMock).not.toHaveBeenCalled() + await unsubscribe() + expect(remoteDispose).toHaveBeenCalledTimes(1) + }) + + it('indexes SSH runtime watches so remote deletion can await them', async () => { + let resolveDispose: () => void = () => {} + const remoteDispose = vi.fn( + () => + new Promise((resolve) => { + resolveDispose = resolve + }) + ) + vi.mocked(getSshFilesystemProvider).mockReturnValue({ + watch: vi.fn(() => remoteDispose) + } as never) + const { commands, store } = createRuntimeFileCommands({ path: '/remote/repo' }) + store.getRepo.mockReturnValue({ connectionId: 'ssh-1' }) + await commands.watchFileExplorer('id:wt-1', vi.fn()) + + let closed = false + const close = commands.closeFileExplorerWatchersForPath('/remote/repo', 'ssh-1').then(() => { + closed = true + }) + await Promise.resolve() + expect(remoteDispose).toHaveBeenCalledTimes(1) + expect(closed).toBe(false) + + resolveDispose() + await close + }) + + it('scopes same-path runtime watcher teardown to its SSH execution host', async () => { + const firstDispose = vi.fn() + const secondDispose = vi.fn() + vi.mocked(getSshFilesystemProvider).mockImplementation( + (connectionId) => + ({ + watch: vi.fn(() => (connectionId === 'ssh-1' ? firstDispose : secondDispose)) + }) as never + ) + const first = createRuntimeFileCommands({ path: '/same/repo' }) + const second = createRuntimeFileCommands({ path: '/same/repo' }) + first.store.getRepo.mockReturnValue({ connectionId: 'ssh-1' }) + second.store.getRepo.mockReturnValue({ connectionId: 'ssh-2' }) + + await first.commands.watchFileExplorer('id:wt-1', vi.fn()) + await second.commands.watchFileExplorer('id:wt-1', vi.fn()) + await first.commands.closeFileExplorerWatchersForPath('/same/repo', 'ssh-1') + + expect(firstDispose).toHaveBeenCalledTimes(1) + expect(secondDispose).not.toHaveBeenCalled() + + await second.commands.closeFileExplorerWatchersForPath('/same/repo', 'ssh-2') + expect(secondDispose).toHaveBeenCalledTimes(1) + }) + it('settles and detaches runtime rg searches when timeout kill is ignored', async () => { const resolveRuntimeFileTarget = vi.fn(async () => ({ worktree: { diff --git a/src/main/runtime/orca-runtime-files.ts b/src/main/runtime/orca-runtime-files.ts index acdf3616f47..fdf818fe3ba 100644 --- a/src/main/runtime/orca-runtime-files.ts +++ b/src/main/runtime/orca-runtime-files.ts @@ -33,9 +33,11 @@ import { isPathInsideOrEqual, isRuntimePathAbsolute, isWindowsAbsolutePathLike, + normalizeRuntimePathForComparison, relativePathInsideRoot, resolveRuntimePath } from '../../shared/cross-platform-path' +import { PhysicalExitTracker } from '../../shared/physical-exit-tracker' import type { RuntimeFileListResult, RuntimeFileOpenResult, @@ -44,7 +46,10 @@ import type { RuntimeFileReadResult, RuntimeTerminalPathResolution } from '../../shared/runtime-types' -import { watchFileExplorerInWatcherProcess } from './file-watcher-host' +import { + closeFileExplorerWatcherInWatcherProcess, + watchFileExplorerInWatcherProcess +} from './file-watcher-host' import { wslAwareSpawn } from '../git/runner' import { parseWslPath, toWindowsWslPath } from '../wsl' import { isENOENT, resolveAuthorizedPath } from '../ipc/filesystem-auth' @@ -70,18 +75,25 @@ import { SSH_FILESYSTEM_PROVIDER_UNAVAILABLE_MESSAGE } from '../providers/ssh-filesystem-dispatch' import type { FileStat, IFilesystemProvider } from '../providers/types' +import { + isWatcherProcessFailure, + WatcherProcessFailure +} from '../ipc/parcel-watcher-process-failure' import { assertNoClobberRenameDestinationAvailable } from '../../shared/filesystem-rename-collision' import { joinWorktreeRelativePath, normalizeRuntimeRelativePath } from './runtime-relative-paths' +import { beginWatcherInstall } from '../ipc/watcher-removal-gate' const MOBILE_FILE_LIST_LIMIT = 5000 const MOBILE_FILE_READ_MAX_BYTES = 512 * 1024 const RUNTIME_PREVIEWABLE_BINARY_MAX_BYTES = 10 * 1024 * 1024 const WINDOWS_RUNTIME_FILE_WATCH_DEBOUNCE_MS = 150 +export const WINDOWS_RUNTIME_FILE_WATCH_CLOSE_DEADLINE_MS = 10_000 const TERMINAL_FILE_GRANT_TTL_MS = 10 * 60 * 1000 const OPEN_NOFOLLOW = typeof constants.O_NOFOLLOW === 'number' ? constants.O_NOFOLLOW : 0 // Why: runtime files.watch subscriptions are cleaned up through synchronous RPC // callbacks. Track native Parcel unsubscribe work so app shutdown can drain it. const pendingRuntimeFileWatcherUnsubscribes = new Set>() +const runtimeFileWatcherReleasesByOwnerAndRoot = new Map Promise>>() const MOBILE_BINARY_EXTENSIONS = new Set([ '.avif', '.bmp', @@ -157,22 +169,95 @@ const RUNTIME_PREVIEWABLE_BINARY_MIME_TYPES: Record = { function trackRuntimeFileWatcherUnsubscribe( rootPath: string, unsubscribe: () => Promise -): void { +): Promise { const promise = Promise.resolve() .then(unsubscribe) - .catch((err: unknown) => { - console.error('[runtime-files.watch] unsubscribe error', { rootPath, err }) - }) .finally(() => { pendingRuntimeFileWatcherUnsubscribes.delete(promise) }) pendingRuntimeFileWatcherUnsubscribes.add(promise) + void promise.catch((err: unknown) => { + console.error('[runtime-files.watch] unsubscribe error', { rootPath, err }) + }) + return promise +} + +function normalizeRuntimeWatcherRoot(rootPath: string): string { + return normalizeRuntimePathForComparison(rootPath) +} + +function runtimeWatcherReleaseKey( + runtimeId: string, + connectionId: string | undefined, + rootPath: string +): string { + // Why: identical absolute paths are valid on local and multiple SSH hosts; + // destructive teardown must stay scoped to the execution host that owns it. + return JSON.stringify([runtimeId, connectionId ?? null, normalizeRuntimeWatcherRoot(rootPath)]) +} + +function registerRuntimeFileWatcherRelease( + runtimeId: string, + connectionId: string | undefined, + rootPaths: string[], + unsubscribe: () => Promise +): () => Promise { + const keys = Array.from( + new Set( + rootPaths.map((rootPath) => runtimeWatcherReleaseKey(runtimeId, connectionId, rootPath)) + ) + ) + let releasePromise: Promise | undefined + const removeRelease = (): void => { + for (const key of keys) { + const releases = runtimeFileWatcherReleasesByOwnerAndRoot.get(key) + releases?.delete(release) + if (releases?.size === 0) { + runtimeFileWatcherReleasesByOwnerAndRoot.delete(key) + } + } + } + const release = (): Promise => { + if (!releasePromise) { + releasePromise = trackRuntimeFileWatcherUnsubscribe(rootPaths[0], unsubscribe) + releasePromise.then(removeRelease, (error: unknown) => { + if (isWatcherProcessFailure(error) && error.physicalExit) { + void error.physicalExit.then(() => { + releasePromise = Promise.resolve() + removeRelease() + }) + } else { + // Why: a synchronous Windows close failure has no process-exit + // signal; retain the root and allow a later destructive retry. + releasePromise = undefined + } + }) + } + return releasePromise + } + for (const key of keys) { + const releases = + runtimeFileWatcherReleasesByOwnerAndRoot.get(key) ?? new Set<() => Promise>() + releases.add(release) + runtimeFileWatcherReleasesByOwnerAndRoot.set(key, releases) + } + return release } export async function awaitRuntimeFileWatcherUnsubscribes(): Promise { await Promise.allSettled(Array.from(pendingRuntimeFileWatcherUnsubscribes)) } +export function _getRuntimeFileWatcherReleaseCountForTests(): number { + const releases = new Set<() => Promise>() + for (const rootReleases of runtimeFileWatcherReleasesByOwnerAndRoot.values()) { + for (const release of rootReleases) { + releases.add(release) + } + } + return releases.size +} + export type ResolvedRuntimeFileWorktree = Worktree & { git: GitWorktreeInfo } export type ResolvedRuntimeFileTarget = { worktree: ResolvedRuntimeFileWorktree @@ -926,34 +1011,68 @@ export class RuntimeFileCommands { signal?: AbortSignal ): Promise<() => void> { const target = await this.resolveFileExplorerPath(worktreeSelector, '') - const provider = target.connectionId ? getSshFilesystemProvider(target.connectionId) : null - if (target.connectionId) { - if (!provider) { - throw new Error(SSH_FILESYSTEM_PROVIDER_UNAVAILABLE_MESSAGE) + const finishInstall = beginWatcherInstall(target.path, target.connectionId) + try { + const provider = target.connectionId ? getSshFilesystemProvider(target.connectionId) : null + if (target.connectionId) { + if (!provider) { + throw new Error(SSH_FILESYSTEM_PROVIDER_UNAVAILABLE_MESSAGE) + } + // Why: the RPC layer already threads AbortSignal for local watches; SSH + // must cancel the remote fs.watch request instead of waiting it out. + const close = await provider.watch(target.path, callback, { signal, onTerminalError }) + return registerRuntimeFileWatcherRelease( + this.host.getRuntimeId(), + target.connectionId, + [target.path], + async () => close() + ) } - // Why: the RPC layer already threads AbortSignal for local watches; SSH - // must cancel the remote fs.watch request instead of waiting it out. - return provider.watch(target.path, callback, { signal }) - } - const rootPath = await resolveAuthorizedPath(target.path, this.host.requireStore()) - const rootStats = await stat(rootPath) - if (!rootStats.isDirectory()) { - throw new Error('not_a_directory') + const rootPath = await resolveAuthorizedPath(target.path, this.host.requireStore()) + const rootStats = await stat(rootPath) + if (!rootStats.isDirectory()) { + throw new Error('not_a_directory') + } + if (process.platform === 'win32') { + const close = watchWindowsRuntimeFileExplorer(rootPath, callback, onTerminalError) + return registerRuntimeFileWatcherRelease( + this.host.getRuntimeId(), + undefined, + [target.path, rootPath], + async () => close() + ) + } + // Why: the forked watcher keeps the blocking crawl and native faults out + // of the main/`serve` process (issues #5308 and #8212). + const dispose = await watchFileExplorerInWatcherProcess( + rootPath, + callback, + onTerminalError, + signal + ) + return registerRuntimeFileWatcherRelease( + this.host.getRuntimeId(), + undefined, + [target.path, rootPath], + dispose + ) + } finally { + finishInstall() } - if (process.platform === 'win32') { - return watchWindowsRuntimeFileExplorer(rootPath, callback) + } + + async closeFileExplorerWatchersForPath(rootPath: string, connectionId?: string): Promise { + const key = runtimeWatcherReleaseKey(this.host.getRuntimeId(), connectionId, rootPath) + const releases = runtimeFileWatcherReleasesByOwnerAndRoot.get(key) + if (releases) { + await Promise.all(Array.from(releases, (release) => release())) } - // Why: the forked watcher keeps the blocking crawl and native faults out - // of the main/`serve` process (issues #5308 and #8212). - const dispose = await watchFileExplorerInWatcherProcess( - rootPath, - callback, - onTerminalError, - signal - ) - return () => { - trackRuntimeFileWatcherUnsubscribe(rootPath, dispose) + if (!connectionId) { + // Why: setup can fail before registerRuntimeFileWatcherRelease publishes + // its callback, while the host still retains an unkillable child owner. + const resolvedRootPath = await resolveAuthorizedPath(rootPath, this.host.requireStore()) + await closeFileExplorerWatcherInWatcherProcess(resolvedRootPath) } } @@ -1512,10 +1631,13 @@ export class RuntimeFileCommands { function watchWindowsRuntimeFileExplorer( rootPath: string, - callback: (events: FsChangeEvent[]) => void -): () => void { + callback: (events: FsChangeEvent[]) => void, + onTerminalError: (error: Error) => void +): () => Promise { let disposed = false let timer: ReturnType | null = null + let closeStarted = false + const physicalClose = new PhysicalExitTracker() const emitOverflow = (): void => { timer = null @@ -1539,21 +1661,61 @@ function watchWindowsRuntimeFileExplorer( // watcher can abort the headless server process. For remote Windows runtimes, // a conservative overflow refresh is safer than a process-wide native crash. const watcher = watchFs(rootPath, { recursive: true }, scheduleOverflow) - watcher.on('error', (err) => { + const onClose = (): void => { + watcher.removeListener('error', onError) + physicalClose.markExited() + } + const onError = (err: Error): void => { console.error('[runtime-files.watch] Windows watcher error', { rootPath, err }) - scheduleOverflow() - }) + if (timer) { + clearTimeout(timer) + timer = null + } + watcher.removeListener('close', onClose) + watcher.removeListener('error', onError) + // Why: Node closes and nulls FSWatcher's native handle on error without a + // close event; that error is positive physical-exit proof for deletion. + physicalClose.markExited() + if (!disposed) { + try { + callback([{ kind: 'overflow', absolutePath: rootPath }]) + } finally { + onTerminalError(err) + } + } + } + watcher.once('close', onClose) + watcher.on('error', onError) - return () => { + return async () => { disposed = true if (timer) { clearTimeout(timer) timer = null } + if (!closeStarted) { + try { + watcher.close() + } catch (err) { + console.error('[runtime-files.watch] Windows watcher close error', { rootPath, err }) + throw err + } + closeStarted = true + } try { - watcher.close() - } catch (err) { - console.error('[runtime-files.watch] Windows watcher close error', { rootPath, err }) + await physicalClose.waitForExit( + WINDOWS_RUNTIME_FILE_WATCH_CLOSE_DEADLINE_MS, + () => new Error('Windows watcher did not close before deletion deadline') + ) + } catch (error) { + // Why: late Windows close still owns native directory handles; expose its + // exact completion so destructive cleanup retains and then clears the root. + throw new WatcherProcessFailure( + error instanceof Error ? error.message : String(error), + 'supervisor', + 'process_unavailable', + physicalClose.exitedPromise + ) } } } diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index e7b11ef8bb2..a9756c7049b 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -92,14 +92,17 @@ import { FOLDER_WORKSPACE_INSTANCE_SEPARATOR } from '../../shared/worktree-id' import { RpcDispatcher } from './rpc/dispatcher' import type { RpcRequest } from './rpc/core' import { TERMINAL_METHODS } from './rpc/methods/terminal' +import { beginWatcherInstall } from '../ipc/watcher-removal-gate' const ORIGINAL_PLATFORM = process.platform const ORIGINAL_PLATFORM_DESCRIPTOR = Object.getOwnPropertyDescriptor(process, 'platform') const removeWorktreeLinkedPathsMock = vi.hoisted(() => vi.fn()) +const findExistingWorktreeSymlinkPathsMock = vi.hoisted(() => vi.fn()) const resolveLocalGitUsernameMock = vi.hoisted(() => vi.fn(async () => '')) vi.mock('../ipc/worktree-symlinks', () => ({ createWorktreeLinkedPaths: vi.fn(), + findExistingWorktreeSymlinkPaths: findExistingWorktreeSymlinkPathsMock, removeWorktreeLinkedPaths: removeWorktreeLinkedPathsMock })) @@ -152,7 +155,14 @@ const electronMocks = vi.hoisted(() => { } }) +const closeLocalWatcherForWorktreePathMock = vi.hoisted(() => vi.fn()) +const closeRemoteWatcherForWorktreePathMock = vi.hoisted(() => vi.fn()) + vi.mock('electron', () => electronMocks) +vi.mock('../ipc/filesystem-watcher', () => ({ + closeLocalWatcherForWorktreePath: closeLocalWatcherForWorktreePathMock, + closeRemoteWatcherForWorktreePath: closeRemoteWatcherForWorktreePathMock +})) const { MOCK_GIT_WORKTREES, @@ -558,12 +568,15 @@ function resetRuntimeTestMocks(): void { electronMocks.ipcMain.on.mockClear() electronMocks.ipcMain.removeListener.mockClear() electronMocks.ipcMain.emit.mockClear() + closeLocalWatcherForWorktreePathMock.mockReset().mockResolvedValue(undefined) + closeRemoteWatcherForWorktreePathMock.mockReset().mockResolvedValue(undefined) vi.mocked(listWorktrees).mockResolvedValue(MOCK_GIT_WORKTREES) vi.mocked(listWorktreesStrict).mockResolvedValue(MOCK_GIT_WORKTREES) vi.mocked(addWorktree).mockReset() vi.mocked(assertWorktreeCleanForRemoval).mockReset() vi.mocked(assertWorktreeCleanForRemoval).mockResolvedValue(undefined) vi.mocked(removeWorktree).mockReset() + findExistingWorktreeSymlinkPathsMock.mockReset().mockResolvedValue([]) removeWorktreeLinkedPathsMock.mockReset() resolveLocalGitUsernameMock.mockReset().mockResolvedValue('') vi.mocked(forceDeleteLocalBranchMock).mockReset() @@ -4766,7 +4779,20 @@ describe('OrcaRuntimeService', () => { removeWorktree: vi.fn().mockResolvedValue(undefined) } registerSshGitProvider('ssh-1', gitProvider as never) - const runtime = new OrcaRuntimeService(remoteStore as never) + const ptyProvider = { + listProcesses: vi.fn().mockResolvedValue([ + { + id: 'pty-remote', + cwd: '/remote/feature', + title: 'shell', + worktreeId: `${TEST_REPO_ID}::/remote/feature` + } + ]), + shutdown: vi.fn().mockResolvedValue(undefined) + } + const runtime = new OrcaRuntimeService(remoteStore as never, undefined, { + getSshProvider: () => ptyProvider as never + }) try { await runtime.removeManagedWorktree('path:/remote/feature', true, false) @@ -4775,6 +4801,14 @@ describe('OrcaRuntimeService', () => { } expect(gitProvider.removeWorktree).toHaveBeenCalledWith('/remote/feature', true) + expect(ptyProvider.shutdown).toHaveBeenCalledWith('pty-remote', { immediate: true }) + expect(ptyProvider.shutdown.mock.invocationCallOrder[0]).toBeLessThan( + gitProvider.removeWorktree.mock.invocationCallOrder[0] + ) + expect(closeRemoteWatcherForWorktreePathMock).toHaveBeenCalledWith('ssh-1', '/remote/feature') + expect(closeRemoteWatcherForWorktreePathMock.mock.invocationCallOrder[0]).toBeLessThan( + gitProvider.removeWorktree.mock.invocationCallOrder[0] + ) expect(removeWorktree).not.toHaveBeenCalled() expect(listWorktrees).not.toHaveBeenCalled() expect(deleteWorktreeHistoryDirMock).toHaveBeenCalledWith(`${TEST_REPO_ID}::/remote/feature`) @@ -21463,6 +21497,70 @@ describe('OrcaRuntimeService', () => { expect(secondStop).toHaveBeenCalledTimes(1) }) + it('dedupes async subscription cleanup and retains a failed cleanup for retry', async () => { + const runtime = createRuntime() + const cleanupError = new Error('physical teardown incomplete') + const firstCleanup = deferred() + const cleanup = vi + .fn() + .mockReturnValueOnce(firstCleanup.promise) + .mockRejectedValue(cleanupError) + runtime.registerSubscriptionCleanup('files-watch-1', cleanup, 'conn-1') + + const first = runtime.cleanupSubscriptionAndWait('files-watch-1') + const duplicate = runtime.cleanupSubscriptionAndWait('files-watch-1') + expect(cleanup).toHaveBeenCalledTimes(1) + firstCleanup.reject(cleanupError) + await expect(first).rejects.toBe(cleanupError) + await expect(duplicate).rejects.toBe(cleanupError) + + await expect(runtime.cleanupSubscriptionAndWait('files-watch-1')).rejects.toBe(cleanupError) + expect(cleanup).toHaveBeenCalledTimes(2) + }) + + it('does not let an old connection cleanup tear down its replacement subscription', async () => { + const runtime = createRuntime() + const oldDone = deferred() + const oldCleanup = vi.fn(() => oldDone.promise) + const replacementCleanup = vi.fn() + + runtime.registerSubscriptionCleanup('terminal:stable', oldCleanup, 'conn-old') + runtime.registerSubscriptionCleanup('terminal:stable', replacementCleanup, 'conn-new') + expect(oldCleanup).toHaveBeenCalledTimes(1) + + runtime.cleanupSubscriptionsForConnection('conn-old') + expect(replacementCleanup).not.toHaveBeenCalled() + + runtime.cleanupSubscriptionsForConnection('conn-new') + expect(replacementCleanup).toHaveBeenCalledTimes(1) + oldDone.resolve() + await Promise.resolve() + }) + + it('does not let a delayed cleanup retry tear down its replacement subscription', async () => { + const runtime = createRuntime() + const physicalExit = deferred() + const oldDone = deferred() + const cleanupError = new Error('physical teardown incomplete') + const oldCleanup = vi.fn(() => oldDone.promise) + const replacementCleanup = vi.fn() + + runtime.registerSubscriptionCleanup('files-watch:stable', oldCleanup, 'conn-old') + const oldAttempt = runtime.cleanupSubscriptionAndWait('files-watch:stable') + runtime.retrySubscriptionCleanupAfter('files-watch:stable', oldCleanup, physicalExit.promise) + runtime.registerSubscriptionCleanup('files-watch:stable', replacementCleanup, 'conn-new') + + oldDone.reject(cleanupError) + await expect(oldAttempt).rejects.toBe(cleanupError) + physicalExit.resolve() + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect(oldCleanup).toHaveBeenCalledTimes(1) + expect(replacementCleanup).not.toHaveBeenCalled() + await runtime.cleanupSubscriptionAndWait('files-watch:stable') + expect(replacementCleanup).toHaveBeenCalledTimes(1) + }) + it('does not deliver or accept browser screencast frames before ready', async () => { const runtime = createRuntime() const done = deferred() @@ -23015,6 +23113,40 @@ describe('OrcaRuntimeService', () => { expect(killed).toBe(false) }) + it('awaits physical PTY stop when destructive teardown supplies shared dedupe', async () => { + const runtime = new OrcaRuntimeService(store) + const physicalStop = makeDeferred() + const kill = vi.fn(() => true) + const stopAndWait = vi.fn(async () => { + await physicalStop.promise + return true + }) + runtime.setPtyController({ + write: () => true, + kill, + stopAndWait, + getForegroundProcess: async () => null + }) + syncSinglePty(runtime) + const stopPty = vi.fn(async (_ptyId: string, stop: () => boolean | Promise) => ({ + stopped: await stop(), + owner: true + })) + + const stopping = runtime.stopTerminalsForWorktree(`id:${TEST_WORKTREE_ID}`, { stopPty }) + await vi.waitFor(() => expect(stopAndWait).toHaveBeenCalledWith('pty-1')) + expect(kill).not.toHaveBeenCalled() + let settled = false + void stopping.then(() => { + settled = true + }) + await Promise.resolve() + expect(settled).toBe(false) + + physicalStop.resolve() + await expect(stopping).resolves.toEqual({ stopped: 1 }) + }) + it('fails terminal listing closed if the graph reloads during selector resolution', async () => { const runtime = new OrcaRuntimeService(store) @@ -23105,6 +23237,59 @@ describe('OrcaRuntimeService', () => { expect(killed).toBe(false) }) + it('does not stop a reused PTY after the teardown deadline passes', async () => { + vi.useFakeTimers() + try { + const runtime = new OrcaRuntimeService(store) + const killed = vi.fn(() => true) + runtime.setPtyController({ + write: () => true, + kill: killed, + getForegroundProcess: async () => null + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { + tabs: [ + { + tabId: 'tab-1', + worktreeId: 'repo-1::/tmp/worktree-a', + title: 'Claude', + activeLeafId: 'pane:1', + layout: null + } + ], + leaves: [ + { + tabId: 'tab-1', + worktreeId: 'repo-1::/tmp/worktree-a', + leafId: 'pane:1', + paneRuntimeId: 1, + ptyId: 'reused-pty' + } + ] + }) + + let releaseListWorktrees = () => {} + vi.mocked(listWorktrees).mockImplementationOnce( + () => + new Promise((resolve) => { + releaseListWorktrees = () => resolve(MOCK_GIT_WORKTREES) + }) + ) + const stopPromise = runtime.stopTerminalsForWorktree('branch:feature/foo', { + deadline: Date.now() + 25 + }) + + await vi.advanceTimersByTimeAsync(25) + releaseListWorktrees() + + await expect(stopPromise).resolves.toEqual({ stopped: 0 }) + expect(killed).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) + it('stops exactly the expected live PTYs for a worktree', async () => { const runtime = new OrcaRuntimeService(store) const stopped: string[] = [] @@ -28470,8 +28655,19 @@ describe('OrcaRuntimeService', () => { } }) + function createWorktreeRemovalRuntime(runtimeStore: unknown = store): OrcaRuntimeService { + const emptyPtyProvider = { + listProcesses: vi.fn(async () => []), + shutdown: vi.fn(async () => {}) + } + return new OrcaRuntimeService(runtimeStore as never, undefined, { + getLocalProvider: () => emptyPtyProvider as never, + getSshProvider: () => emptyPtyProvider as never + }) + } + it('skips archive hooks for CLI worktree removal by default', async () => { - const runtime = new OrcaRuntimeService(store) + const runtime = createWorktreeRemovalRuntime() vi.mocked(getEffectiveHooks).mockReturnValue({ scripts: { archive: 'pnpm worktree:archive' @@ -28496,9 +28692,47 @@ describe('OrcaRuntimeService', () => { ) }) + it('does not remove a runtime worktree when watcher teardown cannot release it', async () => { + const repo = { ...store.getRepos()[0], symlinkPaths: ['node_modules'] } + const runtimeStore = { ...store, getRepos: () => [repo], getRepo: () => repo } + const runtime = createWorktreeRemovalRuntime(runtimeStore) + vi.mocked(getEffectiveHooks).mockReturnValue(null) + deleteWorktreeHistoryDirMock.mockClear() + closeLocalWatcherForWorktreePathMock.mockRejectedValue( + new Error('file watcher process did not exit after termination deadline') + ) + + await expect(runtime.removeManagedWorktree(TEST_WORKTREE_ID)).rejects.toThrow( + 'file watcher process did not exit after termination deadline' + ) + + expect(removeWorktree).not.toHaveBeenCalled() + expect(removeWorktreeLinkedPathsMock).not.toHaveBeenCalled() + expect(deleteWorktreeHistoryDirMock).not.toHaveBeenCalled() + }) + + it('closes before and after draining a pre-removal watcher install', async () => { + const runtime = createRuntime() + const finishInstall = beginWatcherInstall(TEST_WORKTREE_PATH) + + const acquiring = runtime.acquireFileWatcherRemoval(TEST_WORKTREE_PATH) + await vi.waitFor(() => expect(closeLocalWatcherForWorktreePathMock).toHaveBeenCalledTimes(1)) + expect(() => beginWatcherInstall(TEST_WORKTREE_PATH)).toThrow( + 'cannot start while the worktree is being removed' + ) + + finishInstall() + const gate = await acquiring + expect(closeLocalWatcherForWorktreePathMock).toHaveBeenCalledTimes(2) + gate.release() + + const finishRetry = beginWatcherInstall(TEST_WORKTREE_PATH) + finishRetry() + }) + it('recovers forced Windows runtime long-path removal and keeps skipped-hook warnings', async () => { setPlatform('win32') - const runtime = new OrcaRuntimeService(store) + const runtime = createWorktreeRemovalRuntime() await mkdir(TEST_WORKTREE_PATH, { recursive: true }) await writeFile(join(TEST_WORKTREE_PATH, 'scratch.txt'), 'delete me') const gitSpy = vi.spyOn(gitRunner, 'gitExecFileAsync').mockResolvedValue({ @@ -28547,7 +28781,7 @@ describe('OrcaRuntimeService', () => { ...store, removeWorktreeMeta } - const runtime = new OrcaRuntimeService(runtimeStore as never) + const runtime = createWorktreeRemovalRuntime(runtimeStore) const gitSpy = vi.spyOn(gitRunner, 'gitExecFileAsync').mockResolvedValue({ stdout: '', stderr: '' @@ -28580,7 +28814,7 @@ describe('OrcaRuntimeService', () => { const missingWorktreePath = 'C:\\workspace\\already-removed' const worktreeId = `${TEST_REPO_ID}::${missingWorktreePath}` const { runtimeStore, removeWorktreeMeta } = createStaleRuntimeWorktreeStore(worktreeId) - const runtime = new OrcaRuntimeService(runtimeStore as never) + const runtime = createWorktreeRemovalRuntime(runtimeStore) const registeredWorktrees = [ { path: TEST_REPO_PATH, @@ -28631,7 +28865,7 @@ describe('OrcaRuntimeService', () => { const missingWorktreePath = 'C:\\workspace\\locked-already-removed' const worktreeId = `${TEST_REPO_ID}::${missingWorktreePath}` const { runtimeStore, removeWorktreeMeta } = createStaleRuntimeWorktreeStore(worktreeId) - const runtime = new OrcaRuntimeService(runtimeStore as never) + const runtime = createWorktreeRemovalRuntime(runtimeStore) const registeredWorktrees = [ { path: missingWorktreePath, @@ -28674,7 +28908,7 @@ describe('OrcaRuntimeService', () => { localWindowsRuntimeDefault: { kind: 'windows-host' } }) } - const runtime = new OrcaRuntimeService(runtimeStore as never) + const runtime = createWorktreeRemovalRuntime(runtimeStore) vi.mocked(getEffectiveHooks).mockReturnValue(null) vi.mocked(removeWorktree).mockResolvedValue({}) @@ -28731,7 +28965,7 @@ describe('OrcaRuntimeService', () => { localWindowsRuntimeDefault: { kind: 'windows-host' } }) } - const runtime = new OrcaRuntimeService(runtimeStore as never) + const runtime = createWorktreeRemovalRuntime(runtimeStore) vi.mocked(getEffectiveHooks).mockReturnValue(null) vi.mocked(listWorktrees).mockResolvedValue([ { @@ -28788,7 +29022,7 @@ describe('OrcaRuntimeService', () => { localWindowsRuntimeDefault: { kind: 'windows-host' } }) } - const runtime = new OrcaRuntimeService(runtimeStore as never) + const runtime = createWorktreeRemovalRuntime(runtimeStore) vi.mocked(listWorktrees).mockResolvedValue(MOCK_GIT_WORKTREES) vi.mocked(listWorktreesStrict).mockRejectedValue(new Error('wsl git list failed')) @@ -28803,7 +29037,7 @@ describe('OrcaRuntimeService', () => { }) it('force-deletes a branch that was preserved by runtime worktree removal', async () => { - const runtime = new OrcaRuntimeService(store) + const runtime = createWorktreeRemovalRuntime() vi.mocked(removeWorktree).mockResolvedValue({ preservedBranch: { branchName: 'feature/test', head: 'def456' } }) @@ -28872,7 +29106,7 @@ describe('OrcaRuntimeService', () => { }) } registerSshGitProvider('ssh-1', provider as never) - const runtime = new OrcaRuntimeService(runtimeStore as never) + const runtime = createWorktreeRemovalRuntime(runtimeStore) try { await runtime.removeManagedWorktree(remoteWorktreeId) @@ -28914,7 +29148,7 @@ describe('OrcaRuntimeService', () => { localWindowsRuntimeDefault: { kind: 'windows-host' } }) } - const runtime = new OrcaRuntimeService(runtimeStore as never) + const runtime = createWorktreeRemovalRuntime(runtimeStore) vi.mocked(removeWorktree).mockResolvedValue({ preservedBranch: { branchName: 'feature/test', head: 'def456' } }) @@ -28936,7 +29170,7 @@ describe('OrcaRuntimeService', () => { }) it('rejects stale preserved-branch runtime cleanup actions with an old head', async () => { - const runtime = new OrcaRuntimeService(store) + const runtime = createWorktreeRemovalRuntime() vi.mocked(removeWorktree).mockResolvedValue({ preservedBranch: { branchName: 'feature/test', head: 'new456' } }) @@ -28950,7 +29184,7 @@ describe('OrcaRuntimeService', () => { }) it('coalesces concurrent runtime worktree removals for the same worktree id', async () => { - const runtime = new OrcaRuntimeService(store) + const runtime = createWorktreeRemovalRuntime() const removeStarted = deferred() const finishRemoval = deferred() vi.mocked(removeWorktree).mockImplementation(async () => { @@ -28971,7 +29205,7 @@ describe('OrcaRuntimeService', () => { }) it('rejects concurrent runtime worktree removals for the same id with different options', async () => { - const runtime = new OrcaRuntimeService(store) + const runtime = createWorktreeRemovalRuntime() const removeStarted = deferred() const finishRemoval = deferred() vi.mocked(removeWorktree).mockImplementation(async () => { @@ -28997,7 +29231,7 @@ describe('OrcaRuntimeService', () => { const missingWorktreePath = join(parentDir, 'already-deleted') const worktreeId = `${TEST_REPO_ID}::${missingWorktreePath}` const { runtimeStore, removeWorktreeMeta } = createStaleRuntimeWorktreeStore(worktreeId) - const runtime = new OrcaRuntimeService(runtimeStore as never) + const runtime = createWorktreeRemovalRuntime(runtimeStore) const notifier = { worktreesChanged: vi.fn() } runtime.setNotifier(notifier as never) @@ -29021,7 +29255,7 @@ describe('OrcaRuntimeService', () => { const missingWorktreePath = join(parentDir, 'already-deleted') const worktreeId = `${TEST_REPO_ID}::${missingWorktreePath}` const { runtimeStore, removeWorktreeMeta } = createStaleRuntimeWorktreeStore(worktreeId) - const runtime = new OrcaRuntimeService(runtimeStore as never) + const runtime = createWorktreeRemovalRuntime(runtimeStore) const notifier = { worktreesChanged: vi.fn() } runtime.setNotifier(notifier as never) @@ -29075,7 +29309,7 @@ describe('OrcaRuntimeService', () => { } : undefined } - const runtime = new OrcaRuntimeService(runtimeStoreWithRepoPath as never) + const runtime = createWorktreeRemovalRuntime(runtimeStoreWithRepoPath) const notifier = { worktreesChanged: vi.fn() } runtime.setNotifier(notifier as never) @@ -29085,6 +29319,7 @@ describe('OrcaRuntimeService', () => { await expect(runtime.removeManagedWorktree(worktreeId, true)).resolves.toEqual({}) await expect(lstat(orphanPath)).rejects.toMatchObject({ code: 'ENOENT' }) + expect(closeLocalWatcherForWorktreePathMock).toHaveBeenCalledWith(orphanPath) expect(removeWorktree).not.toHaveBeenCalled() expect(removeWorktreeMeta).toHaveBeenCalledWith(worktreeId) expect(deleteWorktreeHistoryDirMock).toHaveBeenCalledWith(worktreeId) @@ -29129,7 +29364,7 @@ describe('OrcaRuntimeService', () => { } : undefined } - const runtime = new OrcaRuntimeService(runtimeStoreWithRepoPath as never) + const runtime = createWorktreeRemovalRuntime(runtimeStoreWithRepoPath) const notifier = { worktreesChanged: vi.fn() } runtime.setNotifier(notifier as never) const gitSpy = vi.spyOn(gitRunner, 'gitExecFileAsync').mockImplementation(async (args) => { @@ -29198,7 +29433,7 @@ describe('OrcaRuntimeService', () => { } : undefined } - const runtime = new OrcaRuntimeService(runtimeStoreWithRepoPath as never) + const runtime = createWorktreeRemovalRuntime(runtimeStoreWithRepoPath) try { vi.mocked(listWorktrees).mockResolvedValue([]) @@ -29259,7 +29494,7 @@ describe('OrcaRuntimeService', () => { ]) } registerSshGitProvider(repo.connectionId, gitProvider as never) - const runtime = new OrcaRuntimeService(runtimeStore as never) + const runtime = createWorktreeRemovalRuntime(runtimeStore) try { await expect(runtime.removeManagedWorktree(`id:${worktreeId}`, true)).rejects.toThrow( @@ -29279,7 +29514,7 @@ describe('OrcaRuntimeService', () => { const existingWorktreePath = await mkdtemp(join(tmpdir(), 'orca-runtime-remove-existing-')) const worktreeId = `${TEST_REPO_ID}::${existingWorktreePath}` const { runtimeStore, removeWorktreeMeta } = createStaleRuntimeWorktreeStore(worktreeId) - const runtime = new OrcaRuntimeService(runtimeStore as never) + const runtime = createWorktreeRemovalRuntime(runtimeStore) try { vi.mocked(listWorktrees).mockResolvedValue([]) @@ -29296,7 +29531,7 @@ describe('OrcaRuntimeService', () => { }) it('rejects CLI worktree removal when the target contains another registered worktree', async () => { - const runtime = new OrcaRuntimeService(store) + const runtime = createWorktreeRemovalRuntime() vi.mocked(listWorktrees).mockResolvedValue([ { path: TEST_REPO_PATH, @@ -29359,7 +29594,7 @@ describe('OrcaRuntimeService', () => { }) it('fails dirty non-force deletes before PTY teardown', async () => { - const runtime = new OrcaRuntimeService(store) + const runtime = createWorktreeRemovalRuntime() const killSpy = vi.fn().mockReturnValue(true) runtime.setPtyController({ write: () => true, @@ -29378,6 +29613,8 @@ describe('OrcaRuntimeService', () => { `Failed to delete worktree at ${TEST_WORKTREE_PATH}. ?? scratch.txt` ) + expect(closeLocalWatcherForWorktreePathMock).not.toHaveBeenCalled() + expect(removeWorktreeLinkedPathsMock).not.toHaveBeenCalled() expect(killSpy).not.toHaveBeenCalled() expect(removeWorktree).not.toHaveBeenCalled() }) @@ -29389,7 +29626,7 @@ describe('OrcaRuntimeService', () => { getRepos: () => [repo], getRepo: () => repo } - const runtime = new OrcaRuntimeService(runtimeStore as never) + const runtime = createWorktreeRemovalRuntime(runtimeStore) const killSpy = vi.fn().mockReturnValue(true) runtime.setPtyController({ write: () => true, @@ -29426,7 +29663,7 @@ describe('OrcaRuntimeService', () => { getRepos: () => [repo], getRepo: () => repo } - const runtime = new OrcaRuntimeService(runtimeStore as never) + const runtime = createWorktreeRemovalRuntime(runtimeStore) const killSpy = vi.fn().mockReturnValue(true) runtime.setPtyController({ write: () => true, @@ -29460,7 +29697,7 @@ describe('OrcaRuntimeService', () => { }) it('formats preflight subprocess failures and skips PTY teardown', async () => { - const runtime = new OrcaRuntimeService(store) + const runtime = createWorktreeRemovalRuntime() const killSpy = vi.fn().mockReturnValue(true) runtime.setPtyController({ write: () => true, @@ -29484,7 +29721,7 @@ describe('OrcaRuntimeService', () => { }) it('falls through to orphan cleanup when preflight reports missing/non-repo worktree', async () => { - const runtime = new OrcaRuntimeService(store) + const runtime = createWorktreeRemovalRuntime() vi.mocked(getEffectiveHooks).mockReturnValue(null) vi.mocked(assertWorktreeCleanForRemoval).mockRejectedValue( Object.assign(new Error('status failed'), { @@ -29511,7 +29748,7 @@ describe('OrcaRuntimeService', () => { }) it('runs archive hooks for CLI worktree removal when hooks are explicitly enabled', async () => { - const runtime = new OrcaRuntimeService(store) + const runtime = createWorktreeRemovalRuntime() vi.mocked(getEffectiveHooks).mockReturnValue({ scripts: { archive: 'pnpm worktree:archive' @@ -29540,7 +29777,7 @@ describe('OrcaRuntimeService', () => { }) it('clears optimistic reconcile tokens when a CLI worktree removal succeeds', async () => { - const runtime = new OrcaRuntimeService(store) + const runtime = createWorktreeRemovalRuntime() const worktreeBaseStatus = vi.fn() runtime.setNotifier({ worktreesChanged: vi.fn(), @@ -30127,11 +30364,14 @@ describe('OrcaRuntimeService', () => { } } - it('RPC-initiated delete kills matching PTYs before git', async () => { + it('RPC-initiated delete awaits matching PTYs before git', async () => { // Seed the runtime with a live leaf whose worktreeId matches the target. - const killSpy = vi.fn().mockReturnValue(true) - const localProvider = createProviderStub(async () => []) const callOrder: string[] = [] + const stopAndWait = vi.fn(async (id: string) => { + callOrder.push(`stop-and-wait:${id}`) + return true + }) + const localProvider = createProviderStub(async () => []) vi.mocked(assertWorktreeCleanForRemoval).mockImplementation(async () => { callOrder.push('preflight') }) @@ -30148,26 +30388,44 @@ describe('OrcaRuntimeService', () => { }) runtime.setPtyController({ write: () => true, - kill: (id) => { - callOrder.push(`kill:${id}`) - return killSpy(id) as boolean - }, + kill: vi.fn(() => true), + stopAndWait, getForegroundProcess: async () => null }) syncSinglePty(runtime, 'pty-1') await runtime.removeManagedWorktree(TEST_WORKTREE_ID) - expect(killSpy).toHaveBeenCalledWith('pty-1') + expect(stopAndWait).toHaveBeenCalledWith('pty-1') // The provider-prefix sweep and the git removal must happen AFTER the - // runtime-graph kill. Git removal must NOT happen before any kill. + // runtime-graph physical stop. Git removal must NOT start before it. const preflightIdx = callOrder.indexOf('preflight') - const killIdx = callOrder.indexOf('kill:pty-1') + const stopIdx = callOrder.indexOf('stop-and-wait:pty-1') const gitIdx = callOrder.indexOf('git-removeWorktree') expect(preflightIdx).toBeGreaterThanOrEqual(0) - expect(killIdx).toBeGreaterThan(preflightIdx) - expect(killIdx).toBeGreaterThanOrEqual(0) - expect(gitIdx).toBeGreaterThan(killIdx) + expect(stopIdx).toBeGreaterThan(preflightIdx) + expect(stopIdx).toBeGreaterThanOrEqual(0) + expect(gitIdx).toBeGreaterThan(stopIdx) + }) + + it('does not start Git removal when physical PTY stop cannot be proven', async () => { + const localProvider = createProviderStub(async () => []) + const runtime = new OrcaRuntimeService(store, undefined, { + getLocalProvider: () => localProvider as never + }) + runtime.setPtyController({ + write: () => true, + kill: vi.fn(() => true), + stopAndWait: vi.fn(async () => false), + getForegroundProcess: async () => null + }) + syncSinglePty(runtime, 'pty-1') + + await expect(runtime.removeManagedWorktree(TEST_WORKTREE_ID)).rejects.toThrow( + 'Failed to physically stop every PTY' + ) + + expect(removeWorktree).not.toHaveBeenCalled() }) it('thunk resolves the installed provider lazily, not at construction time', async () => { diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 96e73e3f965..c3ab1ada07a 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -664,7 +664,11 @@ import { getDefaultVoiceSettings } from '../../shared/constants' import { listRepoWorktrees } from '../repo-worktrees' -import { createWorktreeLinkedPaths, removeWorktreeLinkedPaths } from '../ipc/worktree-symlinks' +import { + createWorktreeLinkedPaths, + findExistingWorktreeSymlinkPaths, + removeWorktreeLinkedPaths +} from '../ipc/worktree-symlinks' import { deleteWorktreeHistoryDir } from '../terminal-history' import { cleanupUnusedWorktreePushTargetRemote, @@ -713,7 +717,11 @@ import { import { prefetchWorktreeCreateBase } from '../worktree-create-base-prefetch' import { invalidateAuthorizedRootsCache } from '../ipc/filesystem-auth' import { prepareLocalWorktreeRootForRepo } from '../worktree-root-preparation' -import { closeLocalWatcherForWorktreePath } from '../ipc/filesystem-watcher' +import { + closeLocalWatcherForWorktreePath, + closeRemoteWatcherForWorktreePath +} from '../ipc/filesystem-watcher' +import { acquireWatcherRemovalGate, type WatcherRemovalGate } from '../ipc/watcher-removal-gate' import { HeadlessEmulator } from '../daemon/headless-emulator' import { isNativeWindowsConptyPty, @@ -2209,7 +2217,11 @@ export class OrcaRuntimeService { > >() private driverListeners = new Map void>>() - private subscriptionCleanups = new Map void>() + private subscriptionCleanups = new Map void | Promise>() + private subscriptionCleanupPromises = new Map< + string, + { cleanup: () => void | Promise; promise: Promise } + >() // Why: index of subscriptionIds by per-WebSocket connectionId so the // server can sweep all subscriptions for a closing socket without // touching subscriptions on other live sockets that share the same @@ -2503,6 +2515,7 @@ export class OrcaRuntimeService { private removeManagedWorktreeInFlight = new Map() private preservedBranchCleanupByWorktreeId = new Map() private readonly getLocalProviderFn: (() => IPtyProvider) | null + private readonly getSshProviderFn: ((connectionId: string) => IPtyProvider | undefined) | null private readonly onPtyStopped: ((ptyId: string) => void) | null private readonly onTerminalAgentStatus: ((event: RuntimeTerminalAgentStatusEvent) => void) | null private readonly onTerminalSideEffects: ((batch: TerminalSideEffectBatch) => void) | null @@ -2530,6 +2543,7 @@ export class OrcaRuntimeService { stats?: StatsCollector, deps?: { getLocalProvider?: () => IPtyProvider + getSshProvider?: (connectionId: string) => IPtyProvider | undefined onPtyStopped?: (ptyId: string) => void onTerminalAgentStatus?: (event: RuntimeTerminalAgentStatusEvent) => void onTerminalSideEffects?: (batch: TerminalSideEffectBatch) => void @@ -2567,6 +2581,7 @@ export class OrcaRuntimeService { // lazily via thunk so teardown always sees the currently-installed // provider (design §4.3 wire-up). this.getLocalProviderFn = deps?.getLocalProvider ?? null + this.getSshProviderFn = deps?.getSshProvider ?? null this.onPtyStopped = deps?.onPtyStopped ?? null this.onTerminalAgentStatus = deps?.onTerminalAgentStatus ?? null this.buildAgentHookPtyEnv = deps?.buildAgentHookPtyEnv ?? null @@ -2591,6 +2606,32 @@ export class OrcaRuntimeService { return this.getLocalProviderFn ? this.getLocalProviderFn() : null } + private async stopPtysForDestructiveWorktreeRemoval( + worktreeId: string, + connectionId?: string + ): Promise { + const provider = connectionId ? this.getSshProviderFn?.(connectionId) : this.getLocalProvider() + if (!provider) { + throw new Error(`PTY provider unavailable for worktree deletion: ${worktreeId}`) + } + const teardownResult = await killAllProcessesForWorktree(worktreeId, { + runtime: this, + localProvider: provider, + onPtyStopped: this.onPtyStopped ?? undefined, + requirePhysicalStop: true, + ...(connectionId ? { includeLocalRegistry: false } : {}) + }) + const total = + teardownResult.runtimeStopped + + teardownResult.providerStopped + + teardownResult.registryStopped + if (total > 0) { + console.info( + `[worktree-teardown] ${worktreeId} killed runtime=${teardownResult.runtimeStopped} provider=${teardownResult.providerStopped} registry=${teardownResult.registryStopped}` + ) + } + } + getStatsSummary(): StatsSummary | null { return this.stats?.getSummary() ?? null } @@ -5388,6 +5429,36 @@ export class OrcaRuntimeService { this.fileCommands.readFileExplorerDir.bind(this.fileCommands) watchFileExplorer: RuntimeFileCommands['watchFileExplorer'] = this.fileCommands.watchFileExplorer.bind(this.fileCommands) + closeFileWatchersForRemoval = async ( + worktreePath: string, + connectionId?: string + ): Promise => { + // Why: desktop and paired runtime clients own independent native watches; + // destructive cleanup is safe only after both release the worktree path. + await Promise.all([ + connectionId + ? closeRemoteWatcherForWorktreePath(connectionId, worktreePath) + : closeLocalWatcherForWorktreePath(worktreePath), + this.fileCommands.closeFileExplorerWatchersForPath(worktreePath, connectionId) + ]) + } + acquireFileWatcherRemoval = async ( + worktreePath: string, + connectionId?: string + ): Promise => { + const gate = acquireWatcherRemovalGate(worktreePath, connectionId) + try { + // Why: the first pass aborts desktop setup immediately; the second catches + // any pre-gate runtime install that published after the first snapshot. + await this.closeFileWatchersForRemoval(worktreePath, connectionId) + await gate.ready + await this.closeFileWatchersForRemoval(worktreePath, connectionId) + return gate + } catch (error) { + gate.release() + throw error + } + } readFileExplorerPreview: RuntimeFileCommands['readFileExplorerPreview'] = this.fileCommands.readFileExplorerPreview.bind(this.fileCommands) readFileExplorerChunk: RuntimeFileCommands['readFileExplorerChunk'] = @@ -7389,7 +7460,7 @@ export class OrcaRuntimeService { registerSubscriptionCleanup( subscriptionId: string, - cleanup: () => void, + cleanup: () => void | Promise, connectionId?: string ): void { // Why: mobile clients reconnect frequently (phone lock, network switch). @@ -7398,6 +7469,9 @@ export class OrcaRuntimeService { // listener leaks in dataListeners and duplicates every PTY data event. const existing = this.subscriptionCleanups.get(subscriptionId) if (existing) { + // Why: the stable id is about to belong to a newer connection; detach + // the old owner before its asynchronous cleanup can overlap the rebind. + this.removeSubscriptionConnectionIndex(subscriptionId) this.cleanupSubscription(subscriptionId) } this.subscriptionCleanups.set(subscriptionId, cleanup) @@ -7413,21 +7487,83 @@ export class OrcaRuntimeService { } cleanupSubscription(subscriptionId: string): void { - const cleanup = this.subscriptionCleanups.get(subscriptionId) - if (cleanup) { - this.subscriptionCleanups.delete(subscriptionId) - const connectionId = this.subscriptionConnectionByEntry.get(subscriptionId) - if (connectionId) { - this.subscriptionConnectionByEntry.delete(subscriptionId) - const set = this.subscriptionsByConnection.get(connectionId) - if (set) { - set.delete(subscriptionId) - if (set.size === 0) { - this.subscriptionsByConnection.delete(connectionId) + void this.cleanupSubscriptionAndWait(subscriptionId).catch((error) => { + console.error(`[runtime] subscription cleanup failed for ${subscriptionId}:`, error) + }) + } + + retrySubscriptionCleanupAfter( + subscriptionId: string, + cleanupOwner: () => void | Promise, + gate: Promise + ): void { + const failedGeneration = this.subscriptionCleanupPromises.get(subscriptionId) + void gate.then( + async () => { + await (failedGeneration?.cleanup === cleanupOwner + ? failedGeneration.promise.catch(() => undefined) + : undefined) + while (this.subscriptionCleanups.get(subscriptionId) === cleanupOwner) { + const newerGeneration = this.subscriptionCleanupPromises.get(subscriptionId) + if (newerGeneration?.cleanup === cleanupOwner) { + // Why: a caller may already be retrying this owner; wait for that + // exact generation so a rejected join cannot consume our retry. + await newerGeneration.promise.catch(() => undefined) + continue } + this.cleanupSubscription(subscriptionId) + return + } + }, + () => undefined + ) + } + + async cleanupSubscriptionAndWait(subscriptionId: string): Promise { + const cleanup = this.subscriptionCleanups.get(subscriptionId) + if (!cleanup) { + return + } + const inFlight = this.subscriptionCleanupPromises.get(subscriptionId) + if (inFlight?.cleanup === cleanup) { + return inFlight.promise + } + let cleanupResult: void | Promise + try { + cleanupResult = cleanup() + } catch (error) { + cleanupResult = Promise.reject(error) + } + const promise = Promise.resolve(cleanupResult) + .then(() => { + // Why: a reconnect can replace this id while old async cleanup runs; + // only the generation that registered this callback may remove it. + if (this.subscriptionCleanups.get(subscriptionId) !== cleanup) { + return + } + this.subscriptionCleanups.delete(subscriptionId) + this.removeSubscriptionConnectionIndex(subscriptionId) + }) + .finally(() => { + if (this.subscriptionCleanupPromises.get(subscriptionId)?.promise === promise) { + this.subscriptionCleanupPromises.delete(subscriptionId) + } + }) + this.subscriptionCleanupPromises.set(subscriptionId, { cleanup, promise }) + return promise + } + + private removeSubscriptionConnectionIndex(subscriptionId: string): void { + const connectionId = this.subscriptionConnectionByEntry.get(subscriptionId) + if (connectionId) { + this.subscriptionConnectionByEntry.delete(subscriptionId) + const set = this.subscriptionsByConnection.get(connectionId) + if (set) { + set.delete(subscriptionId) + if (set.size === 0) { + this.subscriptionsByConnection.delete(connectionId) } } - cleanup() } } @@ -7451,8 +7587,15 @@ export class OrcaRuntimeService { // mutates both the set and the index map. const ids = Array.from(set) for (const id of ids) { + if (this.subscriptionConnectionByEntry.get(id) !== connectionId) { + set.delete(id) + continue + } this.cleanupSubscription(id) } + if (set.size === 0) { + this.subscriptionsByConnection.delete(connectionId) + } } // Why: mobile clients subscribe via notifications.subscribe streaming RPC. @@ -17190,7 +17333,16 @@ export class OrcaRuntimeService { throw new Error(ORPHANED_WORKTREE_DIRECTORY_MESSAGE) } if (repo.connectionId) { - await fsProvider!.deletePath(removalTarget.path, true) + const removalGate = await this.acquireFileWatcherRemoval( + removalTarget.path, + repo.connectionId + ) + try { + await this.stopPtysForDestructiveWorktreeRemoval(removalTarget.id, repo.connectionId) + await fsProvider!.deletePath(removalTarget.path, true) + } finally { + removalGate.release() + } await cleanupUnusedWorktreePushTargetRemoteSsh( provider!, repo.path, @@ -17199,7 +17351,13 @@ export class OrcaRuntimeService { store ) } else { - await removeLocalWorktreePath(removalTarget.path, localWorktreeGitOptions) + const removalGate = await this.acquireFileWatcherRemoval(removalTarget.path) + try { + await this.stopPtysForDestructiveWorktreeRemoval(removalTarget.id) + await removeLocalWorktreePath(removalTarget.path, localWorktreeGitOptions) + } finally { + removalGate.release() + } await cleanupUnusedWorktreePushTargetRemote( repo.path, removalTarget.id, @@ -17237,10 +17395,13 @@ export class OrcaRuntimeService { if (!force) { throw new Error(ORPHANED_WORKTREE_DIRECTORY_MESSAGE) } - await closeLocalWatcherForWorktreePath(removalTarget.path).catch((err) => { - console.warn(`[filesystem-watcher] failed to close ${removalTarget.path}:`, err) - }) - await removeLocalWorktreePath(removalTarget.path, localWorktreeGitOptions) + const removalGate = await this.acquireFileWatcherRemoval(removalTarget.path) + try { + await this.stopPtysForDestructiveWorktreeRemoval(removalTarget.id) + await removeLocalWorktreePath(removalTarget.path, localWorktreeGitOptions) + } finally { + removalGate.release() + } await cleanupUnusedWorktreePushTargetRemote( repo.path, removalTarget.id, @@ -17341,9 +17502,19 @@ export class OrcaRuntimeService { } if (repo.connectionId) { const remoteRemoveOptions = !deleteBranch ? { deleteBranch } : {} - const rawRemovalResult = await (Object.keys(remoteRemoveOptions).length > 0 - ? provider!.removeWorktree(canonicalWorktreePath, force, remoteRemoveOptions) - : provider!.removeWorktree(canonicalWorktreePath, force)) + const removalGate = await this.acquireFileWatcherRemoval( + canonicalWorktreePath, + repo.connectionId + ) + let rawRemovalResult: RemoveWorktreeResult | undefined + try { + await this.stopPtysForDestructiveWorktreeRemoval(removalTarget.id, repo.connectionId) + rawRemovalResult = await (Object.keys(remoteRemoveOptions).length > 0 + ? provider!.removeWorktree(canonicalWorktreePath, force, remoteRemoveOptions) + : provider!.removeWorktree(canonicalWorktreePath, force)) + } finally { + removalGate.release() + } const removalResult = this.preserveBranchHeadFallback( rawRemovalResult, registeredWorktree.head @@ -17409,133 +17580,118 @@ export class OrcaRuntimeService { throw new Error(formatWorktreeRemovalError(error, canonicalWorktreePath, force)) } - let shouldTearDownPtys = true - if (repo.symlinkPaths && repo.symlinkPaths.length > 0) { - await removeWorktreeLinkedPaths(canonicalWorktreePath, repo.symlinkPaths) - } + const linkedPaths = repo.symlinkPaths ?? [] + const ignoredLinkedPaths = force + ? [] + : await findExistingWorktreeSymlinkPaths(canonicalWorktreePath, linkedPaths) try { await (hasLocalWorktreeGitOptions - ? assertWorktreeCleanForRemoval(canonicalWorktreePath, force, localWorktreeGitOptions) - : assertWorktreeCleanForRemoval(canonicalWorktreePath, force)) + ? assertWorktreeCleanForRemoval(canonicalWorktreePath, force, { + ...localWorktreeGitOptions, + ...(ignoredLinkedPaths.length > 0 + ? { ignoredUntrackedPaths: ignoredLinkedPaths } + : {}) + }) + : ignoredLinkedPaths.length > 0 + ? assertWorktreeCleanForRemoval(canonicalWorktreePath, force, { + ignoredUntrackedPaths: ignoredLinkedPaths + }) + : assertWorktreeCleanForRemoval(canonicalWorktreePath, force)) } catch (error) { if (!isOrphanCompatiblePreflightError(error)) { throw new Error(formatWorktreeRemovalError(error, canonicalWorktreePath, force)) } - // Why: orphan cleanup does not need live shells to be killed first, - // and preflight did not prove the worktree is cleanly removable. - shouldTearDownPtys = false - } - - const localProvider = this.getLocalProvider() - await closeLocalWatcherForWorktreePath(canonicalWorktreePath).catch((err) => { - console.warn(`[filesystem-watcher] failed to close ${canonicalWorktreePath}:`, err) - }) - if (localProvider && shouldTearDownPtys) { - // Why: once preflight proves normal deletion is clean, kill PTYs before - // git-level removal so Windows handles cannot keep the directory busy. This also - // closes the headless-CLI leak for confirmed-removable worktrees. - await killAllProcessesForWorktree(removalTarget.id, { - runtime: this, - localProvider, - onPtyStopped: this.onPtyStopped ?? undefined - }) - .then((r) => { - const total = r.runtimeStopped + r.providerStopped + r.registryStopped - if (total > 0) { - // Why (design §4.4 observability): breadcrumb lets ops - // distinguish a renderer-state-induced leak (diff-path purge - // non-empty) from a backend-induced one (nothing to kill but - // memory still pinned). Emit only when the sweep actually did - // work so steady-state logs stay quiet. - console.info( - `[worktree-teardown] ${removalTarget.id} killed runtime=${r.runtimeStopped} provider=${r.providerStopped} registry=${r.registryStopped}` - ) - } - }) - .catch((err) => { - console.warn(`[worktree-teardown] failed for ${removalTarget.id}:`, err) - }) + // Why: Git can still classify this as an orphan after preflight; + // retain strict PTY teardown before any recursive fallback deletion. } let removalResult: RemoveWorktreeResult | undefined + const removalGate = await this.acquireFileWatcherRemoval(canonicalWorktreePath) try { - const removeOptions = { - ...(!deleteBranch ? { deleteBranch } : {}), - // Why: removal already validated the Git row under the selected - // project runtime; keep branch cleanup on that same canonical row. - knownRemovedWorktree: refreshedRegisteredWorktree, - ...localWorktreeGitOptions + if (linkedPaths.length > 0) { + await removeWorktreeLinkedPaths(canonicalWorktreePath, linkedPaths) } - removalResult = this.preserveBranchHeadFallback( - await removeWorktree(repo.path, canonicalWorktreePath, force, removeOptions), - refreshedRegisteredWorktree.head - ) - } catch (error) { - // Why: Git for Windows can deregister a clean worktree before its - // recursive filesystem deletion fails transiently. - const recoveredRemovalResult = await recoverLocalWindowsWorktreeRemoval({ - error, - force, - canonicalWorktreePath, - repoPath: repo.path, - localWorktreeGitOptions, - registeredWorktree: refreshedRegisteredWorktree, - deleteBranch, - closeWatcher: (worktreePath) => - closeLocalWatcherForWorktreePath(worktreePath).catch((err) => { - console.warn(`[filesystem-watcher] failed to close ${worktreePath}:`, err) - }) - }) - if (recoveredRemovalResult) { - removalResult = recoveredRemovalResult - } else if (isOrphanedWorktreeError(error)) { - const access = getLocalWorktreePathAccess(localWorktreeGitOptions) - if ( - await canSafelyRemoveOrphanedWorktreeDirectory( - toLocalWorktreeRuntimePath(canonicalWorktreePath, localWorktreeGitOptions), - toLocalWorktreeRuntimePath(repo.path, localWorktreeGitOptions), - access.statPath, - access.readPath - ) - ) { - await closeLocalWatcherForWorktreePath(canonicalWorktreePath).catch((err) => { - console.warn(`[filesystem-watcher] failed to close ${canonicalWorktreePath}:`, err) - }) - await removeLocalWorktreePath(canonicalWorktreePath, localWorktreeGitOptions).catch( - () => {} - ) - } else { - console.warn( - `[worktrees] Refusing recursive cleanup for unproven worktree directory: ${canonicalWorktreePath}` - ) - } - // Why: `git worktree remove` failed, so git's internal worktree tracking - // (`.git/worktrees/`) is still intact. Without pruning, `git worktree - // list` continues to show the stale entry and the branch it had checked out - // remains locked — other worktrees cannot check it out. - await gitExecFileAsync(['worktree', 'prune'], { - cwd: repo.path, + + // Why: the gate spans terminal shutdown through Git and any recursive + // fallback so a late headless spawn cannot recreate a native handle. + await this.stopPtysForDestructiveWorktreeRemoval(removalTarget.id) + + try { + const removeOptions = { + ...(!deleteBranch ? { deleteBranch } : {}), + // Why: removal already validated the Git row under the selected + // project runtime; keep branch cleanup on that same canonical row. + knownRemovedWorktree: refreshedRegisteredWorktree, ...localWorktreeGitOptions - }).catch(() => {}) - await cleanupUnusedWorktreePushTargetRemote( - repo.path, - removalTarget.id, - removedPushTarget, - store, - localWorktreeGitOptions - ) - this.clearOptimisticReconcileToken(removalTarget.id) - this.removeWorktreeMetadataAndHistory(store, removalTarget.id) - this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) - this.invalidateResolvedWorktreeCache() - invalidateAuthorizedRootsCache() - this.notifyWorktreesChanged(repo.id) - return { - ...(warning ? { warning } : {}) } - } else { - throw new Error(formatWorktreeRemovalError(error, canonicalWorktreePath, force)) + removalResult = this.preserveBranchHeadFallback( + await removeWorktree(repo.path, canonicalWorktreePath, force, removeOptions), + refreshedRegisteredWorktree.head + ) + } catch (error) { + // Why: Git for Windows can deregister a clean worktree before its + // recursive filesystem deletion fails transiently. + const recoveredRemovalResult = await recoverLocalWindowsWorktreeRemoval({ + error, + force, + canonicalWorktreePath, + repoPath: repo.path, + localWorktreeGitOptions, + registeredWorktree: refreshedRegisteredWorktree, + deleteBranch, + closeWatcher: (worktreePath) => this.closeFileWatchersForRemoval(worktreePath) + }) + if (recoveredRemovalResult) { + removalResult = recoveredRemovalResult + } else if (isOrphanedWorktreeError(error)) { + const access = getLocalWorktreePathAccess(localWorktreeGitOptions) + if ( + await canSafelyRemoveOrphanedWorktreeDirectory( + toLocalWorktreeRuntimePath(canonicalWorktreePath, localWorktreeGitOptions), + toLocalWorktreeRuntimePath(repo.path, localWorktreeGitOptions), + access.statPath, + access.readPath + ) + ) { + await this.closeFileWatchersForRemoval(canonicalWorktreePath) + await removeLocalWorktreePath(canonicalWorktreePath, localWorktreeGitOptions).catch( + () => {} + ) + } else { + console.warn( + `[worktrees] Refusing recursive cleanup for unproven worktree directory: ${canonicalWorktreePath}` + ) + } + // Why: `git worktree remove` failed, so git's internal worktree tracking + // (`.git/worktrees/`) is still intact. Without pruning, `git worktree + // list` continues to show the stale entry and the branch it had checked out + // remains locked — other worktrees cannot check it out. + await gitExecFileAsync(['worktree', 'prune'], { + cwd: repo.path, + ...localWorktreeGitOptions + }).catch(() => {}) + await cleanupUnusedWorktreePushTargetRemote( + repo.path, + removalTarget.id, + removedPushTarget, + store, + localWorktreeGitOptions + ) + this.clearOptimisticReconcileToken(removalTarget.id) + this.removeWorktreeMetadataAndHistory(store, removalTarget.id) + this.preservedBranchCleanupByWorktreeId.delete(removalTarget.id) + this.invalidateResolvedWorktreeCache() + invalidateAuthorizedRootsCache() + this.notifyWorktreesChanged(repo.id) + return { + ...(warning ? { warning } : {}) + } + } else { + throw new Error(formatWorktreeRemovalError(error, canonicalWorktreePath, force)) + } } + } finally { + removalGate.release() } await cleanupUnusedWorktreePushTargetRemote( @@ -18985,12 +19141,24 @@ export class OrcaRuntimeService { }) } - async stopTerminalsForWorktree(worktreeSelector: string): Promise<{ stopped: number }> { + async stopTerminalsForWorktree( + worktreeSelector: string, + options: { + deadline?: number + stopPty?: ( + ptyId: string, + stop: () => boolean | Promise + ) => Promise<{ stopped: boolean; owner: boolean }> + } = {} + ): Promise<{ stopped: number }> { // Why: this mutates live PTYs, so the runtime must reject it while the // renderer graph is reloading instead of acting on cached leaf ownership. const graphEpoch = this.captureReadyGraphEpoch() const worktree = await this.resolveWorktreeSelector(worktreeSelector) this.assertStableReadyGraph(graphEpoch) + if (options.deadline !== undefined && Date.now() >= options.deadline) { + return { stopped: 0 } + } const ptyIds = new Set() for (const leaf of this.leaves.values()) { if (leaf.worktreeId === worktree.id && leaf.ptyId) { @@ -19005,7 +19173,24 @@ export class OrcaRuntimeService { let stopped = 0 for (const ptyId of ptyIds) { - if (this.ptyController?.kill(ptyId)) { + if (options.deadline !== undefined && Date.now() >= options.deadline) { + break + } + const stop = (): boolean | Promise => { + if (options.deadline !== undefined && Date.now() >= options.deadline) { + return false + } + if (options.stopPty) { + // Why: destructive worktree cleanup must not let its cross-surface + // dedupe treat fire-and-forget controller.kill as physical exit. + return this.ptyController?.stopAndWait?.(ptyId) ?? false + } + return Boolean(this.ptyController?.kill(ptyId)) + } + const stopResult = options.stopPty + ? await options.stopPty(ptyId, stop) + : { stopped: stop(), owner: true } + if (stopResult.owner && stopResult.stopped) { stopped += 1 } } diff --git a/src/main/runtime/rpc/methods/file-watch-stream-lifecycle.ts b/src/main/runtime/rpc/methods/file-watch-stream-lifecycle.ts new file mode 100644 index 00000000000..f612f27dc0d --- /dev/null +++ b/src/main/runtime/rpc/methods/file-watch-stream-lifecycle.ts @@ -0,0 +1,146 @@ +import { isWatcherProcessFailure } from '../../../ipc/parcel-watcher-process-failure' +import type { OrcaRuntimeService } from '../../orca-runtime' +import { createFileWatchEventBatcher } from './file-watch-event-batcher' + +export async function runFileWatchStream(args: { + runtime: OrcaRuntimeService + worktree: string + connectionId?: string + signal?: AbortSignal + subscriptionId: string + emit: (event: unknown) => void +}): Promise { + if (args.signal?.aborted) { + return + } + await new Promise((resolve, reject) => { + let settled = false + let setupFailed = false + let watchReady = false + let unwatch: (() => void) | null = null + let terminalError: Error | null = null + let setupPromise: Promise<() => void> | null = null + let cleanupPromise: Promise | null = null + let logicalCleanupStarted = false + let endEmitted = false + const setupAbortController = new AbortController() + const eventBatcher = createFileWatchEventBatcher(args.worktree, args.emit) + const settle = (callback: () => void): void => { + if (settled) { + return + } + settled = true + args.signal?.removeEventListener('abort', handleAbort) + callback() + } + const cleanup = (): Promise => { + if (cleanupPromise) { + return cleanupPromise + } + const attempt = (async () => { + if (!logicalCleanupStarted) { + logicalCleanupStarted = true + // Mark cleanup complete before emitting so a synchronous transport + // abort cannot re-enter this function and duplicate error/end. + if (!setupFailed) { + settle(resolve) + } + setupAbortController.abort() + if (terminalError) { + // Why: recovery emits overflow before giving up. Flush that final + // refresh so clients never end on a knowingly stale snapshot. + eventBatcher.flush() + eventBatcher.dispose() + args.emit({ type: 'error', message: terminalError.message }) + } else if (watchReady) { + eventBatcher.flush() + eventBatcher.dispose() + } else { + // Why: setup can queue events before it returns an unsubscribe; + // cancellation before ready must not publish that partial crawl. + eventBatcher.dispose() + } + } + try { + if (!unwatch && setupPromise) { + try { + unwatch = await setupPromise + } catch (error) { + if (!setupAbortController.signal.aborted) { + throw error + } + } + } + await unwatch?.() + } catch (error) { + if (isWatcherProcessFailure(error) && error.physicalExit) { + args.runtime.retrySubscriptionCleanupAfter( + args.subscriptionId, + cleanup, + error.physicalExit + ) + } + throw error + } finally { + if (!setupFailed && !endEmitted) { + endEmitted = true + args.emit({ type: 'end' }) + } + } + })() + cleanupPromise = attempt + void attempt.catch(() => { + if (cleanupPromise === attempt) { + // Why: callers deliberately retry transient unwatch failures; only + // concurrent callers should share the rejected cleanup generation. + cleanupPromise = null + } + }) + return attempt + } + const handleTerminalError = (error: Error): void => { + if (settled || terminalError) { + return + } + terminalError = error + args.runtime.cleanupSubscription(args.subscriptionId) + } + function handleAbort(): void { + args.runtime.cleanupSubscription(args.subscriptionId) + } + + args.signal?.addEventListener('abort', handleAbort, { once: true }) + // Why: paired clients must be able to cancel while native setup is + // waiting indefinitely for global child capacity, before ready exists. + args.runtime.registerSubscriptionCleanup(args.subscriptionId, cleanup, args.connectionId) + args.emit({ type: 'starting', subscriptionId: args.subscriptionId }) + setupPromise = args.runtime.watchFileExplorer( + args.worktree, + (events) => eventBatcher.push(events), + handleTerminalError, + setupAbortController.signal + ) + void setupPromise + .then((nextUnwatch) => { + if (cleanupPromise || settled) { + return + } + unwatch = nextUnwatch + watchReady = true + args.emit({ type: 'ready', subscriptionId: args.subscriptionId }) + }) + .catch(async (error) => { + if (cleanupPromise || settled || setupAbortController.signal.aborted) { + return + } + setupFailed = true + await args.runtime.cleanupSubscriptionAndWait(args.subscriptionId).catch((cleanupError) => { + console.error('[runtime-files.watch] failed-setup cleanup failed', cleanupError) + }) + settle(() => reject(error)) + }) + if (args.signal?.aborted) { + handleAbort() + } + }) +} diff --git a/src/main/runtime/rpc/methods/files-watch-cancellation.test.ts b/src/main/runtime/rpc/methods/files-watch-cancellation.test.ts new file mode 100644 index 00000000000..067c23aa83a --- /dev/null +++ b/src/main/runtime/rpc/methods/files-watch-cancellation.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it, vi } from 'vitest' +import type { RpcRequest } from '../core' +import { RpcDispatcher } from '../dispatcher' +import type { OrcaRuntimeService } from '../../orca-runtime' +import { FILE_METHODS } from './files' + +function makeRequest(method: string, params?: unknown): RpcRequest { + return { id: 'req-1', authToken: 'tok', method, params } +} + +describe('file watch RPC cancellation', () => { + it('cancels a file watch waiting for setup before ready', async () => { + let setupSignal: AbortSignal | undefined + const watchFileExplorer = vi.fn( + ( + _worktree: string, + _callback: (events: unknown[]) => void, + _onTerminalError: (error: Error) => void, + signal?: AbortSignal + ) => { + setupSignal = signal + return new Promise<() => void>((_resolve, reject) => { + signal?.addEventListener('abort', () => reject(new Error('setup aborted')), { + once: true + }) + }) + } + ) + const cleanups = new Map void | Promise>() + const cleanupSubscriptionAndWait = vi.fn(async (id: string) => { + const cleanup = cleanups.get(id) + await cleanup?.() + cleanups.delete(id) + }) + const runtime = { + getRuntimeId: () => 'test-runtime', + watchFileExplorer, + registerSubscriptionCleanup: vi.fn((id, cleanup) => cleanups.set(id, cleanup)), + cleanupSubscriptionAndWait, + cleanupSubscription: vi.fn((id) => { + void cleanupSubscriptionAndWait(id) + }) + } as unknown as OrcaRuntimeService + const dispatcher = new RpcDispatcher({ runtime, methods: FILE_METHODS }) + const replies: { result?: { type?: string; subscriptionId?: string } }[] = [] + + const stream = dispatcher.dispatchStreaming( + makeRequest('files.watch', { worktree: 'id:wt-1' }), + (response) => replies.push(JSON.parse(response)), + { connectionId: 'conn-1' } + ) + await vi.waitFor(() => expect(replies[0]?.result?.type).toBe('starting')) + const subscriptionId = replies[0]?.result?.subscriptionId + expect(subscriptionId).toBeTruthy() + + const response = await dispatcher.dispatch(makeRequest('files.unwatch', { subscriptionId })) + await stream + + expect(response).toMatchObject({ ok: true, result: { unsubscribed: true } }) + expect(setupSignal?.aborted).toBe(true) + expect(replies.map((reply) => reply.result?.type)).toEqual(['starting', 'end']) + expect(cleanups.size).toBe(0) + }) + + it('drops queued file watch events when aborted before setup resolves', async () => { + vi.useFakeTimers() + try { + type WatchCallback = ( + events: { kind: 'update'; absolutePath: string; isDirectory?: boolean }[] + ) => void + const unwatch = vi.fn() + let resolveWatch: (value: () => void) => void = () => {} + const watchFileExplorer = vi.fn((_worktree: string, callback: WatchCallback) => { + callback([{ kind: 'update', absolutePath: '/repo/queued.ts', isDirectory: false }]) + return new Promise<() => void>((resolve) => { + resolveWatch = resolve + }) + }) + const cleanups = new Map void | Promise>() + const runtime = { + getRuntimeId: () => 'test-runtime', + watchFileExplorer, + registerSubscriptionCleanup: vi.fn((id, cleanup) => cleanups.set(id, cleanup)), + cleanupSubscription: vi.fn((id) => { + void Promise.resolve(cleanups.get(id)?.()) + }) + } as unknown as OrcaRuntimeService + const dispatcher = new RpcDispatcher({ runtime, methods: FILE_METHODS }) + const abortController = new AbortController() + const replies: unknown[] = [] + + const dispatch = dispatcher.dispatchStreaming( + makeRequest('files.watch', { worktree: 'id:wt-1' }), + (response) => replies.push(JSON.parse(response)), + { connectionId: 'conn-1', signal: abortController.signal } + ) + await vi.waitFor(() => expect(watchFileExplorer).toHaveBeenCalled()) + + abortController.abort() + await dispatch + await vi.runOnlyPendingTimersAsync() + resolveWatch(unwatch) + await vi.waitFor(() => expect(unwatch).toHaveBeenCalled()) + + expect(runtime.registerSubscriptionCleanup).toHaveBeenCalledTimes(1) + expect(replies).toEqual([ + expect.objectContaining({ result: expect.objectContaining({ type: 'starting' }) }), + expect.objectContaining({ result: { type: 'end' } }) + ]) + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/main/runtime/rpc/methods/files-watch-cleanup.test.ts b/src/main/runtime/rpc/methods/files-watch-cleanup.test.ts new file mode 100644 index 00000000000..991dc23b732 --- /dev/null +++ b/src/main/runtime/rpc/methods/files-watch-cleanup.test.ts @@ -0,0 +1,104 @@ +import { expect, it, vi } from 'vitest' +import { WatcherProcessFailure } from '../../../ipc/parcel-watcher-process-failure' +import type { OrcaRuntimeService } from '../../orca-runtime' +import type { RpcRequest } from '../core' +import { RpcDispatcher } from '../dispatcher' +import { FILE_METHODS } from './files' + +function makeRequest(method: string, params?: unknown): RpcRequest { + return { id: 'req-1', authToken: 'tok', method, params } +} + +it.each(['late', 'already-resolved', 'transient'] as const)( + 'retries retained file-watch cleanup after %s failure', + async (failureTiming) => { + let resolvePhysicalExit: () => void = () => {} + const physicalExit = + failureTiming === 'already-resolved' + ? Promise.resolve() + : failureTiming === 'late' + ? new Promise((resolve) => { + resolvePhysicalExit = resolve + }) + : undefined + const cleanupError = physicalExit + ? new WatcherProcessFailure( + 'file watcher process did not exit after termination deadline', + 'supervisor', + 'process_unavailable', + physicalExit + ) + : new Error('transient unwatch failure') + const unwatch = vi.fn().mockRejectedValueOnce(cleanupError).mockResolvedValue(undefined) + let cleanup: (() => Promise) | undefined + let subscriptionId = '' + let inFlight: Promise | null = null + const cleanupSubscriptionAndWait = vi.fn(() => { + if (inFlight) { + return inFlight + } + let tracked: Promise + tracked = Promise.resolve(cleanup?.()).finally(() => { + if (inFlight === tracked) { + inFlight = null + } + }) + inFlight = tracked + return tracked + }) + const cleanupSubscription = vi.fn(() => { + void cleanupSubscriptionAndWait().catch(() => {}) + }) + const retrySubscriptionCleanupAfter = vi.fn( + (_id: string, _owner: () => void | Promise, gate: Promise) => { + void gate.then(async () => { + await inFlight?.catch(() => undefined) + cleanupSubscription() + }) + } + ) + const runtime = { + getRuntimeId: () => 'test-runtime', + watchFileExplorer: vi.fn().mockResolvedValue(unwatch), + registerSubscriptionCleanup: vi.fn((id, nextCleanup) => { + subscriptionId = id + cleanup = nextCleanup + }), + cleanupSubscription, + cleanupSubscriptionAndWait, + retrySubscriptionCleanupAfter + } as unknown as OrcaRuntimeService + const dispatcher = new RpcDispatcher({ runtime, methods: FILE_METHODS }) + const responses: { result?: { type?: string } }[] = [] + + const dispatch = dispatcher.dispatchStreaming( + makeRequest('files.watch', { worktree: 'id:wt-1' }), + (response) => responses.push(JSON.parse(response)) + ) + await vi.waitFor(() => expect(cleanup).toBeDefined()) + + await expect(cleanupSubscriptionAndWait()).rejects.toBe(cleanupError) + await dispatch + + if (failureTiming === 'late') { + expect(unwatch).toHaveBeenCalledTimes(1) + resolvePhysicalExit() + } else if (failureTiming === 'transient') { + await expect(cleanupSubscriptionAndWait()).resolves.toBeUndefined() + } + await vi.waitFor(() => expect(unwatch).toHaveBeenCalledTimes(2)) + expect(cleanupSubscriptionAndWait).toHaveBeenCalledTimes(2) + if (physicalExit) { + await vi.waitFor(() => expect(cleanupSubscription).toHaveBeenCalledTimes(1)) + expect(retrySubscriptionCleanupAfter).toHaveBeenCalledWith( + subscriptionId, + cleanup, + physicalExit + ) + } else { + expect(cleanupSubscription).not.toHaveBeenCalled() + expect(retrySubscriptionCleanupAfter).not.toHaveBeenCalled() + } + expect(responses.filter((response) => response.result?.type === 'end')).toHaveLength(1) + } +) diff --git a/src/main/runtime/rpc/methods/files.test.ts b/src/main/runtime/rpc/methods/files.test.ts index 9f535d9a637..24b415e751b 100644 --- a/src/main/runtime/rpc/methods/files.test.ts +++ b/src/main/runtime/rpc/methods/files.test.ts @@ -126,15 +126,20 @@ describe('file RPC methods', () => { ) await vi.waitFor(() => { - expect(replies).toHaveLength(1) + expect(replies).toHaveLength(2) }) expect(runtime.watchFileExplorer).toHaveBeenCalledWith( 'id:wt-1', expect.any(Function), expect.any(Function), - undefined + expect.any(AbortSignal) ) expect(replies[0]).toMatchObject({ + ok: true, + streaming: true, + result: { type: 'starting', subscriptionId: expect.stringContaining('files-watch-') } + }) + expect(replies[1]).toMatchObject({ ok: true, streaming: true, result: { type: 'ready', subscriptionId: expect.stringContaining('files-watch-') } @@ -146,11 +151,11 @@ describe('file RPC methods', () => { emitWatchChange?.([ { kind: 'update', absolutePath: '/repo/package.json', isDirectory: false } ]) - expect(replies).toHaveLength(1) + expect(replies).toHaveLength(2) await vi.runOnlyPendingTimersAsync() - expect(replies[1]).toMatchObject({ + expect(replies[2]).toMatchObject({ ok: true, streaming: true, result: { @@ -163,11 +168,11 @@ describe('file RPC methods', () => { } }) - const ready = replies[0] as { result: { subscriptionId: string } } + const ready = replies[1] as { result: { subscriptionId: string } } cleanups.get(ready.result.subscriptionId)?.() await dispatch - expect(replies[2]).toMatchObject({ + expect(replies[3]).toMatchObject({ ok: true, streaming: true, result: { type: 'end' } @@ -205,14 +210,20 @@ describe('file RPC methods', () => { makeRequest('files.watch', { worktree: 'id:wt-1' }), (response) => replies.push(JSON.parse(response)) ) - await vi.waitFor(() => expect(replies[0]?.result?.type).toBe('ready')) + await vi.waitFor(() => expect(replies[1]?.result?.type).toBe('ready')) emitWatchChange?.([{ kind: 'overflow', absolutePath: '/repo' }]) onTerminalError?.(new Error('file watcher process crashed repeatedly')) await dispatch - expect(replies.map((reply) => reply.result?.type)).toEqual(['ready', 'changed', 'error', 'end']) - expect(replies[2]?.result?.message).toContain('crashed repeatedly') + expect(replies.map((reply) => reply.result?.type)).toEqual([ + 'starting', + 'ready', + 'changed', + 'error', + 'end' + ]) + expect(replies[3]?.result?.message).toContain('crashed repeatedly') expect(unwatch).toHaveBeenCalledTimes(1) expect(cleanups.size).toBe(0) }) @@ -234,10 +245,14 @@ describe('file RPC methods', () => { resolveWatch = resolve }) ) + const cleanups = new Map void | Promise>() const runtime = { getRuntimeId: () => 'test-runtime', watchFileExplorer, - registerSubscriptionCleanup: vi.fn() + registerSubscriptionCleanup: vi.fn((id, cleanup) => cleanups.set(id, cleanup)), + cleanupSubscription: vi.fn((id) => { + void Promise.resolve(cleanups.get(id)?.()) + }) } as unknown as OrcaRuntimeService const dispatcher = new RpcDispatcher({ runtime, methods: FILE_METHODS }) const abortController = new AbortController() @@ -251,63 +266,39 @@ describe('file RPC methods', () => { await vi.waitFor(() => { expect(watchFileExplorer).toHaveBeenCalled() }) - expect(watchFileExplorer.mock.calls[0]?.[3]).toBe(abortController.signal) + const setupSignal = watchFileExplorer.mock.calls[0]?.[3] + expect(setupSignal).not.toBe(abortController.signal) abortController.abort() + expect(setupSignal?.aborted).toBe(true) await dispatch resolveWatch(unwatch) await vi.waitFor(() => { expect(unwatch).toHaveBeenCalled() }) - expect(runtime.registerSubscriptionCleanup).not.toHaveBeenCalled() - expect(replies).toEqual([]) + expect(runtime.registerSubscriptionCleanup).toHaveBeenCalledTimes(1) + expect(replies).toEqual([ + expect.objectContaining({ result: expect.objectContaining({ type: 'starting' }) }), + expect.objectContaining({ result: { type: 'end' } }) + ]) }) - it('drops queued file watch events when aborted before setup resolves', async () => { - vi.useFakeTimers() - try { - type WatchCallback = ( - events: { kind: 'update'; absolutePath: string; isDirectory?: boolean }[] - ) => void - const unwatch = vi.fn() - let resolveWatch: (value: () => void) => void = () => {} - const watchFileExplorer = vi.fn((_worktree: string, callback: WatchCallback) => { - callback([{ kind: 'update', absolutePath: '/repo/queued.ts', isDirectory: false }]) - return new Promise<() => void>((resolve) => { - resolveWatch = resolve - }) - }) - const runtime = { - getRuntimeId: () => 'test-runtime', - watchFileExplorer, - registerSubscriptionCleanup: vi.fn() - } as unknown as OrcaRuntimeService - const dispatcher = new RpcDispatcher({ runtime, methods: FILE_METHODS }) - const abortController = new AbortController() - const replies: unknown[] = [] + it('reports files.unwatch failure instead of acknowledging incomplete teardown', async () => { + const cleanupError = new Error('file watcher process did not exit after termination deadline') + const runtime = { + getRuntimeId: () => 'test-runtime', + cleanupSubscriptionAndWait: vi.fn().mockRejectedValue(cleanupError) + } as unknown as OrcaRuntimeService + const dispatcher = new RpcDispatcher({ runtime, methods: FILE_METHODS }) - const dispatch = dispatcher.dispatchStreaming( - makeRequest('files.watch', { worktree: 'id:wt-1' }), - (response) => replies.push(JSON.parse(response)), - { connectionId: 'conn-1', signal: abortController.signal } - ) - await vi.waitFor(() => { - expect(watchFileExplorer).toHaveBeenCalled() - }) + const response = await dispatcher.dispatch( + makeRequest('files.unwatch', { subscriptionId: 'files-watch-inproc-1' }) + ) - abortController.abort() - await dispatch - await vi.runOnlyPendingTimersAsync() - resolveWatch(unwatch) - await vi.waitFor(() => { - expect(unwatch).toHaveBeenCalled() - }) - - expect(runtime.registerSubscriptionCleanup).not.toHaveBeenCalled() - expect(replies).toEqual([]) - } finally { - vi.useRealTimers() - } + expect(response).toMatchObject({ + ok: false, + error: { message: expect.stringContaining(cleanupError.message) } + }) }) it('reads a relative file path for a selected worktree', async () => { diff --git a/src/main/runtime/rpc/methods/files.ts b/src/main/runtime/rpc/methods/files.ts index b1fb70234a6..a46d24eb1b1 100644 --- a/src/main/runtime/rpc/methods/files.ts +++ b/src/main/runtime/rpc/methods/files.ts @@ -1,7 +1,7 @@ /* oxlint-disable max-lines -- Why: file RPC routing coverage stays together so the dispatcher contract for read, write, mutation, and watch methods is easy to audit. */ import { z } from 'zod' import { defineMethod, defineStreamingMethod, type RpcAnyMethod } from '../core' -import { createFileWatchEventBatcher } from './file-watch-event-batcher' +import { runFileWatchStream } from './file-watch-stream-lifecycle' let filesWatchSubscriptionSeq = 0 const RUNTIME_FILE_BASE64_PATTERN = /^[A-Za-z0-9+/]*={0,2}$/ @@ -386,91 +386,13 @@ export const FILE_METHODS: RpcAnyMethod[] = [ handler: async (params, { runtime, connectionId, signal }, emit) => { const seq = ++filesWatchSubscriptionSeq const subscriptionId = `files-watch-${connectionId ?? 'inproc'}-${seq}` - if (signal?.aborted) { - return - } - await new Promise((resolve, reject) => { - let settled = false - let unwatch: (() => void) | null = null - let terminalError: Error | null = null - const eventBatcher = createFileWatchEventBatcher(params.worktree, emit) - const finish = (): void => { - if (settled) { - return - } - settled = true - signal?.removeEventListener('abort', handleAbort) - resolve() - } - const cleanup = (): void => { - // Mark cleanup complete before emitting so a synchronous transport - // abort cannot re-enter this function and duplicate error/end. - finish() - if (terminalError) { - // Why: recovery emits overflow before giving up. Flush that final - // refresh so clients never end on a knowingly stale snapshot. - eventBatcher.flush() - eventBatcher.dispose() - emit({ type: 'error', message: terminalError.message }) - } else { - eventBatcher.flush() - eventBatcher.dispose() - } - unwatch?.() - emit({ type: 'end' }) - } - const handleTerminalError = (error: Error): void => { - if (settled || terminalError) { - return - } - terminalError = error - if (unwatch) { - runtime.cleanupSubscription(subscriptionId) - } else { - cleanup() - } - } - const handleAbort = (): void => { - if (unwatch) { - cleanup() - } else { - // Why: watch setup may have queued events before resolving its - // unwatch callback. Dispose that transient batcher on early abort. - eventBatcher.dispose() - finish() - } - } - signal?.addEventListener('abort', handleAbort, { once: true }) - void runtime - .watchFileExplorer( - params.worktree, - (events) => { - eventBatcher.push(events) - }, - handleTerminalError, - signal - ) - .then((nextUnwatch) => { - if (signal?.aborted || settled) { - // Why: the connection can close while watch setup is still - // resolving. Tear down the late watcher immediately instead of - // registering cleanup on a connection that was already reaped. - eventBatcher.dispose() - nextUnwatch() - return - } - unwatch = nextUnwatch - runtime.registerSubscriptionCleanup(subscriptionId, cleanup, connectionId) - emit({ type: 'ready', subscriptionId }) - }) - .catch((error) => { - if (!settled) { - settled = true - signal?.removeEventListener('abort', handleAbort) - eventBatcher.dispose() - reject(error) - } - }) + await runFileWatchStream({ + runtime, + worktree: params.worktree, + connectionId, + signal, + subscriptionId, + emit }) } }), @@ -478,7 +400,7 @@ export const FILE_METHODS: RpcAnyMethod[] = [ name: 'files.unwatch', params: FileUnwatch, handler: async (params, { runtime }) => { - runtime.cleanupSubscription(params.subscriptionId) + await runtime.cleanupSubscriptionAndWait(params.subscriptionId) return { unsubscribed: true } } }) diff --git a/src/main/runtime/runtime-root-watch-ownership.ts b/src/main/runtime/runtime-root-watch-ownership.ts new file mode 100644 index 00000000000..890fa802033 --- /dev/null +++ b/src/main/runtime/runtime-root-watch-ownership.ts @@ -0,0 +1,41 @@ +import { + isWatcherProcessFailure, + type WatcherProcessFailure +} from '../ipc/parcel-watcher-process-failure' + +type RuntimeRootOwnership = { + rootPath: string + terminalReleaseError: WatcherProcessFailure | null +} + +export function createRuntimeRootOwnershipReleaser( + roots: Map, + forgetRoot: (rootPath: string) => void +): { + release: (root: T) => void + releaseAfterFailure: (root: T, error: Error) => void +} { + const release = (root: T): void => { + if (roots.get(root.rootPath) !== root) { + return + } + roots.delete(root.rootPath) + forgetRoot(root.rootPath) + } + const releaseAfterFailure = (root: T, error: Error): void => { + if (!(isWatcherProcessFailure(error) && error.physicalExit)) { + release(root) + return + } + root.terminalReleaseError = error + // Why: an unkillable child can still own the native root after logical + // teardown; reserve the exact root until that child's physical exit. + void error.physicalExit.then(() => { + if (root.terminalReleaseError === error) { + root.terminalReleaseError = null + release(root) + } + }) + } + return { release, releaseAfterFailure } +} diff --git a/src/main/runtime/runtime-root-watch-teardown.ts b/src/main/runtime/runtime-root-watch-teardown.ts new file mode 100644 index 00000000000..e9353d1da49 --- /dev/null +++ b/src/main/runtime/runtime-root-watch-teardown.ts @@ -0,0 +1,49 @@ +import { isWatcherProcessFailure } from '../ipc/parcel-watcher-process-failure' + +type RuntimeRootWatchTeardown = { + closed: boolean + generation: number + abortController: AbortController + subscription: { unsubscribe(): Promise } | null + terminalReleaseError: Error | null +} + +export async function closeRuntimeRootWatch( + root: RuntimeRootWatchTeardown, + pendingGeneration: Promise, + release: () => void, + releaseAfterFailure: (error: Error) => void +): Promise { + if (root.closed) { + if (root.terminalReleaseError) { + throw root.terminalReleaseError + } + return + } + root.closed = true + root.generation++ + root.abortController.abort() + try { + await pendingGeneration + } catch (error) { + if (!(isWatcherProcessFailure(error) && error.code === 'subscribe_aborted')) { + if (error instanceof Error) { + releaseAfterFailure(error) + } else { + release() + } + throw error + } + } + try { + await root.subscription?.unsubscribe() + release() + } catch (error) { + if (error instanceof Error) { + releaseAfterFailure(error) + } else { + release() + } + throw error + } +} diff --git a/src/main/runtime/worktree-teardown.test.ts b/src/main/runtime/worktree-teardown.test.ts index 63e207cd8da..c2f18cec03a 100644 --- a/src/main/runtime/worktree-teardown.test.ts +++ b/src/main/runtime/worktree-teardown.test.ts @@ -8,12 +8,10 @@ vi.mock('../memory/pty-registry', () => ({ listRegisteredPtys: listRegisteredPtysMock })) -import { killAllProcessesForWorktree } from './worktree-teardown' -import type { IPtyProvider } from '../providers/types' +import { killAllProcessesForWorktree, WORKTREE_PROCESS_SWEEP_TIMEOUT_MS } from './worktree-teardown' +import type { IPtyProvider, PtyProcessInfo } from '../providers/types' -function createProviderStub( - listProcesses: () => Promise<{ id: string; cwd: string; title: string }[]> -): IPtyProvider { +function createProviderStub(listProcesses: () => Promise): IPtyProvider { return { spawn: vi.fn(), attach: vi.fn(), @@ -92,6 +90,49 @@ describe('killAllProcessesForWorktree', () => { expect(onPtyStopped).toHaveBeenCalledWith('1') }) + it('stops cwd-owned PTYs even when their ids have no worktree prefix', async () => { + const localProvider = createProviderStub(async () => [ + { id: 'floating-1', cwd: '/repo/app/nested', title: 'shell' }, + { id: 'outside-2', cwd: '/repo/application', title: 'shell' }, + { id: 'repo-1::/repo/app@@legacy-3', title: 'shell' } as PtyProcessInfo + ]) + listRegisteredPtysMock.mockReturnValue([]) + + const result = await killAllProcessesForWorktree('repo-1::/repo/app', { + localProvider, + requirePhysicalStop: true + }) + + expect(localProvider.shutdown).toHaveBeenCalledWith('floating-1', { immediate: true }) + expect(localProvider.shutdown).toHaveBeenCalledWith('repo-1::/repo/app@@legacy-3', { + immediate: true + }) + expect(localProvider.shutdown).not.toHaveBeenCalledWith('outside-2', { immediate: true }) + expect(result.providerStopped).toBe(2) + }) + + it('uses authoritative remote worktree ownership without sweeping the local registry', async () => { + const remoteProvider = createProviderStub(async () => [ + { id: 'pty-remote', cwd: '/remote/w1', title: 'shell', worktreeId: 'w1' }, + { id: 'pty-sibling', cwd: '/remote/w2', title: 'shell', worktreeId: 'w2' } + ]) + listRegisteredPtysMock.mockReturnValue([ + { ptyId: 'local-1', worktreeId: 'w1', sessionId: null, paneKey: null, pid: 200 } + ]) + + const result = await killAllProcessesForWorktree('w1', { + localProvider: remoteProvider, + includeLocalRegistry: false, + requirePhysicalStop: true + }) + + expect(remoteProvider.shutdown).toHaveBeenCalledWith('pty-remote', { immediate: true }) + expect(remoteProvider.shutdown).not.toHaveBeenCalledWith('pty-sibling', { immediate: true }) + expect(remoteProvider.shutdown).not.toHaveBeenCalledWith('local-1', { immediate: true }) + expect(listRegisteredPtysMock).not.toHaveBeenCalled() + expect(result).toEqual({ runtimeStopped: 0, providerStopped: 1, registryStopped: 0 }) + }) + it('best-effort: swallows errors from listProcesses and shutdown', async () => { const localProvider = createProviderStub(() => Promise.reject(new Error('boom'))) listRegisteredPtysMock.mockReturnValue([ @@ -157,10 +198,145 @@ describe('killAllProcessesForWorktree', () => { const result = await killAllProcessesForWorktree('w1', { runtime, localProvider }) - expect(stopTerminalsForWorktree).toHaveBeenCalledWith('w1') + expect(stopTerminalsForWorktree).toHaveBeenCalledWith('w1', { + deadline: expect.any(Number), + stopPty: expect.any(Function) + }) expect(result.runtimeStopped).toBe(3) }) + it('claims duplicate provider and registry PTY ids only once', async () => { + const localProvider = createProviderStub(async () => [ + { id: 'w1@@same', cwd: '/tmp/w1', title: 'shell' } + ]) + listRegisteredPtysMock.mockReturnValue([ + { ptyId: 'w1@@same', worktreeId: 'w1', sessionId: null, paneKey: null, pid: 100 } + ]) + + const result = await killAllProcessesForWorktree('w1', { localProvider }) + + expect(localProvider.shutdown).toHaveBeenCalledTimes(1) + expect(localProvider.shutdown).toHaveBeenCalledWith('w1@@same', { immediate: true }) + expect(result.providerStopped + result.registryStopped).toBe(1) + }) + + it('falls back to provider shutdown when the runtime cannot stop an overlapping PTY', async () => { + const stopTerminalsForWorktree = vi.fn( + async ( + _worktreeId: string, + options: { + stopPty: ( + ptyId: string, + stop: () => boolean + ) => Promise<{ stopped: boolean; owner: boolean }> + } + ) => ({ + stopped: (await options.stopPty('w1@@same', () => false)).owner ? 1 : 0 + }) + ) + const runtime = { + stopTerminalsForWorktree + } as unknown as Parameters[1]['runtime'] + const localProvider = createProviderStub(async () => []) + listRegisteredPtysMock.mockReturnValue([ + { ptyId: 'w1@@same', worktreeId: 'w1', sessionId: null, paneKey: null, pid: 100 } + ]) + + const result = await killAllProcessesForWorktree('w1', { runtime, localProvider }) + + expect(result.runtimeStopped).toBe(0) + expect(result.registryStopped).toBe(1) + expect(localProvider.shutdown).toHaveBeenCalledTimes(1) + }) + + it('keeps duplicate sweeps behind the runtime physical-stop promise', async () => { + let releasePhysicalStop: () => void = () => undefined + const physicalStop = new Promise((resolve) => { + releasePhysicalStop = () => resolve(true) + }) + const stopTerminalsForWorktree = vi.fn( + async ( + _worktreeId: string, + options: { + stopPty: ( + ptyId: string, + stop: () => Promise + ) => Promise<{ stopped: boolean; owner: boolean }> + } + ) => ({ + stopped: (await options.stopPty('w1@@same', () => physicalStop)).owner ? 1 : 0 + }) + ) + const runtime = { + stopTerminalsForWorktree + } as unknown as Parameters[1]['runtime'] + const localProvider = createProviderStub(async () => [ + { id: 'w1@@same', cwd: '/tmp/w1', title: 'shell' } + ]) + listRegisteredPtysMock.mockReturnValue([ + { ptyId: 'w1@@same', worktreeId: 'w1', sessionId: null, paneKey: null, pid: 100 } + ]) + + const teardown = killAllProcessesForWorktree('w1', { runtime, localProvider }) + await vi.waitFor(() => expect(stopTerminalsForWorktree).toHaveBeenCalledTimes(1)) + expect(localProvider.shutdown).not.toHaveBeenCalled() + + releasePhysicalStop() + await expect(teardown).resolves.toEqual({ + runtimeStopped: 1, + providerStopped: 0, + registryStopped: 0 + }) + expect(localProvider.shutdown).not.toHaveBeenCalled() + }) + + it('fails destructive teardown closed when physical stop misses the deadline', async () => { + vi.useFakeTimers() + try { + const physicalStop = new Promise(() => {}) + const stopTerminalsForWorktree = vi.fn( + async ( + _worktreeId: string, + options: { + stopPty: ( + ptyId: string, + stop: () => Promise + ) => Promise<{ stopped: boolean; owner: boolean }> + } + ) => ({ + stopped: (await options.stopPty('w1@@same', () => physicalStop)).owner ? 1 : 0 + }) + ) + const runtime = { + stopTerminalsForWorktree + } as unknown as Parameters[1]['runtime'] + const localProvider = createProviderStub(async () => [ + { id: 'w1@@same', cwd: '/tmp/w1', title: 'shell' } + ]) + listRegisteredPtysMock.mockReturnValue([ + { ptyId: 'w1@@same', worktreeId: 'w1', sessionId: null, paneKey: null, pid: 100 } + ]) + + const teardown = killAllProcessesForWorktree('w1', { + runtime, + localProvider, + timeoutMs: 25, + requirePhysicalStop: true + }) + const failure = expect(teardown).rejects.toThrow( + 'Timed out waiting for physical PTY teardown' + ) + await vi.advanceTimersByTimeAsync(24) + expect(localProvider.shutdown).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + + await failure + expect(localProvider.shutdown).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) + it('tolerates runtime.stopTerminalsForWorktree throwing (headless assertGraphReady reject)', async () => { const stopTerminalsForWorktree = vi.fn().mockRejectedValue(new Error('graph not ready')) const runtime = { @@ -174,4 +350,92 @@ describe('killAllProcessesForWorktree', () => { expect(result.runtimeStopped).toBe(0) }) + + it('bounds the entire process sweep when a provider never settles', async () => { + vi.useFakeTimers() + try { + const localProvider = createProviderStub(() => new Promise(() => {})) + listRegisteredPtysMock.mockReturnValue([ + { ptyId: 'registry-1', worktreeId: 'w1', sessionId: null, paneKey: null, pid: 12 } + ]) + let completed = false + const teardown = killAllProcessesForWorktree('w1', { + localProvider, + timeoutMs: WORKTREE_PROCESS_SWEEP_TIMEOUT_MS + }).then((result) => { + completed = true + return result + }) + + await vi.advanceTimersByTimeAsync(WORKTREE_PROCESS_SWEEP_TIMEOUT_MS - 1) + expect(completed).toBe(false) + await vi.advanceTimersByTimeAsync(1) + + await expect(teardown).resolves.toEqual({ + runtimeStopped: 0, + providerStopped: 0, + registryStopped: 1 + }) + expect(localProvider.shutdown).toHaveBeenCalledWith('registry-1', { immediate: true }) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it('does not issue shutdown after a timed-out provider list settles late', async () => { + vi.useFakeTimers() + try { + let resolveList: (sessions: { id: string; cwd: string; title: string }[]) => void = () => {} + const localProvider = createProviderStub( + () => + new Promise((resolve) => { + resolveList = resolve + }) + ) + listRegisteredPtysMock.mockReturnValue([]) + const teardown = killAllProcessesForWorktree('w1', { localProvider, timeoutMs: 25 }) + + await vi.advanceTimersByTimeAsync(25) + await expect(teardown).resolves.toMatchObject({ providerStopped: 0 }) + resolveList([{ id: 'w1@@replacement', cwd: '/new', title: 'new' }]) + await Promise.resolve() + + expect(localProvider.shutdown).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) + + it('does not mutate PTY state after a shutdown settles beyond the deadline', async () => { + vi.useFakeTimers() + try { + let resolveShutdown: () => void = () => {} + const localProvider = createProviderStub(async () => []) + ;(localProvider.shutdown as unknown as ReturnType).mockImplementation( + () => + new Promise((resolve) => { + resolveShutdown = resolve + }) + ) + listRegisteredPtysMock.mockReturnValue([ + { ptyId: 'registry-1', worktreeId: 'w1', sessionId: null, paneKey: null, pid: 12 } + ]) + const onPtyStopped = vi.fn() + const teardown = killAllProcessesForWorktree('w1', { + localProvider, + onPtyStopped, + timeoutMs: 25 + }) + + await vi.advanceTimersByTimeAsync(25) + await expect(teardown).resolves.toMatchObject({ registryStopped: 0 }) + resolveShutdown() + await Promise.resolve() + + expect(onPtyStopped).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) }) diff --git a/src/main/runtime/worktree-teardown.ts b/src/main/runtime/worktree-teardown.ts index 7572b9bbd77..f89c340739d 100644 --- a/src/main/runtime/worktree-teardown.ts +++ b/src/main/runtime/worktree-teardown.ts @@ -1,11 +1,16 @@ import type { IPtyProvider } from '../providers/types' import type { OrcaRuntimeService } from './orca-runtime' import { listRegisteredPtys } from '../memory/pty-registry' +import { isPathInsideOrEqual } from '../../shared/cross-platform-path' +import { splitWorktreeIdForFilesystem } from '../../shared/worktree-id' export type WorktreeTeardownDeps = { runtime?: OrcaRuntimeService localProvider: IPtyProvider onPtyStopped?: (ptyId: string) => void + timeoutMs?: number + requirePhysicalStop?: boolean + includeLocalRegistry?: boolean } export type WorktreeTeardownResult = { @@ -14,6 +19,8 @@ export type WorktreeTeardownResult = { registryStopped: number } +export const WORKTREE_PROCESS_SWEEP_TIMEOUT_MS = 10_000 + /** * Kills every PTY we can prove belongs to `worktreeId`, across all three * registration surfaces (renderer graph, installed PTY provider session list, @@ -32,9 +39,8 @@ export type WorktreeTeardownResult = { * canonical source for memory attribution; it also redundantly backstops * daemon spawns. * - * Best-effort throughout: each sweep catches its own errors. The caller - * (removeManagedWorktree, worktrees:remove IPC) must run the git-level - * removal regardless of what this returns. + * Sweeps are best-effort by default. Destructive removal callers set + * `requirePhysicalStop` so a timeout or unproven stop blocks filesystem work. */ export async function killAllProcessesForWorktree( worktreeId: string, @@ -45,45 +51,177 @@ export async function killAllProcessesForWorktree( providerStopped: 0, registryStopped: 0 } - - if (deps.runtime) { - const r = await deps.runtime.stopTerminalsForWorktree(worktreeId).catch(() => ({ stopped: 0 })) - result.runtimeStopped = r.stopped + const deadline = Date.now() + Math.max(1, deps.timeoutMs ?? WORKTREE_PROCESS_SWEEP_TIMEOUT_MS) + const deadlineError = new Error(`Timed out waiting for physical PTY teardown: ${worktreeId}`) + const worktreePath = splitWorktreeIdForFilesystem(worktreeId)?.worktreePath + const stopAttempts = new Map>() + const stopPty = ( + ptyId: string, + stop: () => boolean | Promise + ): Promise<{ stopped: boolean; owner: boolean }> => { + const previous = stopAttempts.get(ptyId) ?? Promise.resolve(false) + const current = previous + .then(async (stopped) => { + if (stopped) { + return { stopped: true, owner: false } + } + const didStop = await stop() + return { stopped: didStop, owner: didStop } + }) + .catch(() => ({ stopped: false, owner: false })) + stopAttempts.set( + ptyId, + current.then(({ stopped }) => stopped) + ) + return current } - result.providerStopped = await sweepProviderByPrefix( - worktreeId, - deps.localProvider, - deps.onPtyStopped - ) - result.registryStopped = await sweepRegistryForWorktree( - worktreeId, - deps.localProvider, - deps.onPtyStopped + const runtimeSweep = deps.runtime + ? settleBeforeDeadline( + () => deps.runtime!.stopTerminalsForWorktree(worktreeId, { deadline, stopPty }), + { stopped: 0 }, + deadline, + deps.requirePhysicalStop ? deadlineError : undefined, + false + ) + : Promise.resolve({ stopped: 0 }) + const providerSweep = settleBeforeDeadline( + () => + sweepProviderByPrefix( + worktreeId, + worktreePath, + deps.localProvider, + deadline, + stopPty, + deps.onPtyStopped, + deps.requirePhysicalStop + ), + 0, + deadline, + deps.requirePhysicalStop ? deadlineError : undefined ) + const registrySweep = + deps.includeLocalRegistry === false + ? Promise.resolve(0) + : settleBeforeDeadline( + () => + sweepRegistryForWorktree( + worktreeId, + deps.localProvider, + deadline, + stopPty, + deps.onPtyStopped + ), + 0, + deadline, + deps.requirePhysicalStop ? deadlineError : undefined + ) + const [runtimeResult, providerStopped, registryStopped] = await Promise.all([ + runtimeSweep, + providerSweep, + registrySweep + ]) + result.runtimeStopped = runtimeResult.stopped + result.providerStopped = providerStopped + result.registryStopped = registryStopped + if (deps.requirePhysicalStop) { + const stops = await Promise.all(stopAttempts.values()) + if (stops.some((stopped) => !stopped)) { + throw new Error(`Failed to physically stop every PTY for worktree: ${worktreeId}`) + } + } return result } +async function settleBeforeDeadline( + run: () => Promise, + fallback: T, + deadline: number, + failClosedError?: Error, + failClosedOnRunError = true +): Promise { + const remaining = deadline - Date.now() + if (remaining <= 0) { + if (failClosedError) { + throw failClosedError + } + return fallback + } + return new Promise((resolve, reject) => { + let settled = false + const finish = (value: T): void => { + if (settled) { + return + } + settled = true + clearTimeout(timer) + resolve(value) + } + const fail = (error: unknown): void => { + if (settled) { + return + } + settled = true + clearTimeout(timer) + reject(error) + } + const timer = setTimeout( + () => (failClosedError ? fail(failClosedError) : finish(fallback)), + remaining + ) + timer.unref?.() + void run().then(finish, (error: unknown) => + failClosedError && failClosedOnRunError ? fail(error) : finish(fallback) + ) + }) +} + async function sweepProviderByPrefix( worktreeId: string, + worktreePath: string | undefined, provider: IPtyProvider, - onPtyStopped?: (ptyId: string) => void + deadline: number, + stopPty: ( + ptyId: string, + stop: () => Promise + ) => Promise<{ stopped: boolean; owner: boolean }>, + onPtyStopped?: (ptyId: string) => void, + failClosed = false ): Promise { const prefix = `${worktreeId}@@` - const sessions = await provider.listProcesses().catch(() => []) + const sessions = failClosed + ? await provider.listProcesses() + : await provider.listProcesses().catch(() => []) let killed = 0 for (const s of sessions) { - if (!s.id.startsWith(prefix)) { + if (Date.now() >= deadline) { + break + } + // Why: older daemon/relay process rows may omit cwd; their established ID + // and authoritative worktree ownership must remain usable during teardown. + const cwdOwned = + worktreePath !== undefined && + typeof s.cwd === 'string' && + s.cwd.length > 0 && + isPathInsideOrEqual(worktreePath, s.cwd) + if (!s.id.startsWith(prefix) && s.worktreeId !== worktreeId && !cwdOwned) { continue } - try { - await provider.shutdown(s.id, { immediate: true }) + const stopResult = await stopPty(s.id, async () => { + if (Date.now() >= deadline) { + return false + } + try { + await provider.shutdown(s.id, { immediate: true }) + return Date.now() < deadline + } catch { + return false + } + }) + if (stopResult.owner && Date.now() < deadline) { clearStoppedPtyState(s.id, onPtyStopped) killed += 1 - } catch { - // Already dead, or the backend dropped the session — treat as success. - killed += 1 } } return killed @@ -92,17 +230,33 @@ async function sweepProviderByPrefix( async function sweepRegistryForWorktree( worktreeId: string, localProvider: IPtyProvider, + deadline: number, + stopPty: ( + ptyId: string, + stop: () => Promise + ) => Promise<{ stopped: boolean; owner: boolean }>, onPtyStopped?: (ptyId: string) => void ): Promise { const entries = listRegisteredPtys().filter((r) => r.worktreeId === worktreeId) let killed = 0 for (const entry of entries) { - try { - await localProvider.shutdown(entry.ptyId, { immediate: true }) + if (Date.now() >= deadline) { + break + } + const stopResult = await stopPty(entry.ptyId, async () => { + if (Date.now() >= deadline) { + return false + } + try { + await localProvider.shutdown(entry.ptyId, { immediate: true }) + return Date.now() < deadline + } catch { + return false + } + }) + if (stopResult.owner && Date.now() < deadline) { clearStoppedPtyState(entry.ptyId, onPtyStopped) killed += 1 - } catch { - /* ignore — best-effort */ } } return killed diff --git a/src/relay/dispatcher.test.ts b/src/relay/dispatcher.test.ts index 0a9a3908172..87f5cb69053 100644 --- a/src/relay/dispatcher.test.ts +++ b/src/relay/dispatcher.test.ts @@ -201,6 +201,23 @@ describe('RelayDispatcher', () => { expect(socketWritten).toHaveLength(1) }) + it('targets terminal ownership notifications to one attached client', () => { + const firstWritten: Buffer[] = [] + const secondWritten: Buffer[] = [] + const firstId = dispatcher.attachClient((data) => { + firstWritten.push(Buffer.from(data)) + }) + dispatcher.attachClient((data) => { + secondWritten.push(Buffer.from(data)) + }) + + dispatcher.notifyClient(firstId, 'fs.watchFailed', { watchId: 7 }) + + expect(written).toHaveLength(0) + expect(firstWritten).toHaveLength(1) + expect(secondWritten).toHaveLength(0) + }) + it('forwards relay-originated requests to an owning socket client instead of the caller', async () => { dispatcher.invalidateClient() const ownerWritten: Buffer[] = [] diff --git a/src/relay/dispatcher.ts b/src/relay/dispatcher.ts index 7dfdf5dc5f4..a036d5449bb 100644 --- a/src/relay/dispatcher.ts +++ b/src/relay/dispatcher.ts @@ -189,6 +189,21 @@ export class RelayDispatcher { } } + notifyClient(clientId: number, method: string, params?: Record): void { + if (this.disposed) { + return + } + const client = this.clients.get(clientId) + if (!client || client.closed) { + return + } + this.sendFrame(client, { + jsonrpc: '2.0', + method, + ...(params !== undefined ? { params } : {}) + }) + } + /** * Bulk-lane notification. Sends are serialized per client and the returned * promise resolves only after the sink accepted the frame without reporting diff --git a/src/relay/fs-handler.test.ts b/src/relay/fs-handler.test.ts index 63e399072e9..23dd28c35cb 100644 --- a/src/relay/fs-handler.test.ts +++ b/src/relay/fs-handler.test.ts @@ -61,6 +61,7 @@ function createMockDispatcher() { notify: vi.fn((method: string, params?: Record) => { notifications.push({ method, params }) }), + notifyClient: vi.fn(), onClientDetached: vi.fn((listener: (clientId: number) => void) => { detachListeners.add(listener) return () => detachListeners.delete(listener) @@ -153,6 +154,7 @@ describe('FsHandler', () => { expect(methods).toContain('fs.listFiles') expect(methods).toContain('fs.workspaceSpaceScan') expect(methods).toContain('fs.watch') + expect(methods).toContain('fs.unwatchAndWait') const notifMethods = Array.from(dispatcher._notificationHandlers.keys()) expect(notifMethods).toContain('fs.unwatch') @@ -493,6 +495,28 @@ describe('FsHandler', () => { await expect(fs.access(filePath)).rejects.toThrow() }) + it('holds the relay watcher fence through recursive directory deletion', async () => { + const directoryPath = path.join(tmpDir, 'watched-orphan') + mkdirSync(directoryPath) + const unsubscribe = vi.fn() + mockSubscribe.mockResolvedValue({ unsubscribe }) + await dispatcher.callRequest( + 'fs.watch', + { rootPath: directoryPath, watchId: 77 }, + { clientId: 3, isStale: () => false } + ) + + await dispatcher.callRequest('fs.deletePath', { targetPath: directoryPath, recursive: true }) + + expect(unsubscribe).toHaveBeenCalledTimes(1) + expect(dispatcher.notifyClient).toHaveBeenCalledWith(3, 'fs.watchFailed', { + rootPath: directoryPath, + watchId: 77, + message: 'Remote worktree is being removed' + }) + await expect(fs.access(directoryPath)).rejects.toThrow() + }) + it('createFile creates an empty file with parent dirs', async () => { const filePath = path.join(tmpDir, 'deep', 'nested', 'file.txt') await dispatcher.callRequest('fs.createFile', { filePath }) @@ -734,6 +758,129 @@ describe('FsHandler', () => { expect(unsubscribe).toHaveBeenCalledTimes(1) }) + it('settles acknowledged unwatch only after native unsubscribe completes', async () => { + let resolveUnsubscribe: () => void = () => {} + const unsubscribe = vi.fn( + () => + new Promise((resolve) => { + resolveUnsubscribe = resolve + }) + ) + mockSubscribe.mockResolvedValue({ unsubscribe }) + await dispatcher.callRequest('fs.watch', { rootPath: tmpDir }) + + let settled = false + const unwatch = dispatcher.callRequest('fs.unwatchAndWait', { rootPath: tmpDir }).then(() => { + settled = true + }) + await vi.waitFor(() => expect(unsubscribe).toHaveBeenCalledTimes(1)) + expect(settled).toBe(false) + + resolveUnsubscribe() + await unwatch + expect(settled).toBe(true) + }) + + it('waits for in-flight native setup before acknowledging teardown', async () => { + handler.dispose() + let resolveSubscribe: (value: { unsubscribe: () => Promise }) => void = () => {} + const unsubscribe = vi.fn(async () => undefined) + const subscribe = vi.fn( + () => + new Promise<{ unsubscribe: () => Promise }>((resolve) => { + resolveSubscribe = resolve + }) + ) + handler = new FsHandler(dispatcher as unknown as RelayDispatcher, new RelayContext(), { + dispose: vi.fn(), + forgetRoot: vi.fn(), + subscribe + }) + + const watch = dispatcher.callRequest('fs.watch', { rootPath: tmpDir }) + let unwatchSettled = false + const unwatch = dispatcher.callRequest('fs.unwatchAndWait', { rootPath: tmpDir }).then(() => { + unwatchSettled = true + }) + await vi.waitFor(() => expect(subscribe).toHaveBeenCalledTimes(1)) + expect(unwatchSettled).toBe(false) + + resolveSubscribe({ unsubscribe }) + await Promise.all([watch, unwatch]) + expect(unsubscribe).toHaveBeenCalledTimes(1) + }) + + it('joins a physical unsubscribe already started by the notification path', async () => { + let resolveUnsubscribe: () => void = () => {} + const unsubscribe = vi.fn( + () => + new Promise((resolve) => { + resolveUnsubscribe = resolve + }) + ) + mockSubscribe.mockResolvedValue({ unsubscribe }) + await dispatcher.callRequest('fs.watch', { rootPath: tmpDir }) + dispatcher.callNotification('fs.unwatch', { rootPath: tmpDir }) + + let settled = false + const joined = dispatcher.callRequest('fs.unwatchAndWait', { rootPath: tmpDir }).then(() => { + settled = true + }) + await Promise.resolve() + expect(settled).toBe(false) + + resolveUnsubscribe() + await joined + }) + + it('blocks replacement watches behind physical unsubscribe and counts the pending slot', async () => { + let resolveUnsubscribe: () => void = () => {} + const unsubscribe = vi.fn( + () => + new Promise((resolve) => { + resolveUnsubscribe = resolve + }) + ) + mockSubscribe.mockResolvedValue({ unsubscribe }) + await dispatcher.callRequest('fs.watch', { rootPath: tmpDir }) + dispatcher.callNotification('fs.unwatch', { rootPath: tmpDir }) + + const replacement = dispatcher.callRequest('fs.watch', { rootPath: tmpDir }) + for (let index = 0; index < 19; index += 1) { + await dispatcher.callRequest('fs.watch', { + rootPath: path.join(tmpDir, `pending-cap-${index}`) + }) + } + await expect( + dispatcher.callRequest('fs.watch', { rootPath: path.join(tmpDir, 'over-pending-cap') }) + ).rejects.toThrow('Maximum number of file watchers reached') + expect(mockSubscribe).toHaveBeenCalledTimes(20) + + resolveUnsubscribe() + await replacement + expect(mockSubscribe).toHaveBeenCalledTimes(21) + }) + + it('retains a failed native unsubscribe slot until acknowledged retry succeeds', async () => { + const unsubscribe = vi + .fn() + .mockRejectedValueOnce(new Error('native handle still active')) + .mockResolvedValueOnce(undefined) + mockSubscribe.mockResolvedValue({ unsubscribe }) + await dispatcher.callRequest('fs.watch', { rootPath: tmpDir }) + + await expect(dispatcher.callRequest('fs.unwatchAndWait', { rootPath: tmpDir })).rejects.toThrow( + 'native handle still active' + ) + await expect( + dispatcher.callRequest('fs.unwatchAndWait', { rootPath: tmpDir }) + ).resolves.toBeUndefined() + expect(unsubscribe).toHaveBeenCalledTimes(2) + + await expect(dispatcher.callRequest('fs.watch', { rootPath: tmpDir })).resolves.toBeUndefined() + expect(mockSubscribe).toHaveBeenCalledTimes(2) + }) + it('allows a shared watch attach even when the root watch cap is full', async () => { mockSubscribe.mockResolvedValue({ unsubscribe: vi.fn() }) diff --git a/src/relay/fs-handler.ts b/src/relay/fs-handler.ts index 38f0dec9b5d..baea79a88c8 100644 --- a/src/relay/fs-handler.ts +++ b/src/relay/fs-handler.ts @@ -96,6 +96,10 @@ export class FsHandler { }) } + getWatchRegistry(): RelayFilesystemWatchRegistry { + return this.watchRegistry + } + private registerHandlers(): void { this.dispatcher.onRequest('fs.readDir', (p) => this.readDir(p)) this.dispatcher.onRequest('fs.readFile', (p) => this.readFile(p)) @@ -118,7 +122,14 @@ export class FsHandler { this.dispatcher.onRequest('fs.listFiles', (p, c) => this.listFiles(p, c)) this.dispatcher.onRequest('fs.workspaceSpaceScan', (p, c) => this.workspaceSpaceScan(p, c)) this.dispatcher.onRequest('fs.watch', (p, context) => - this.watchRegistry.watch(expandTilde(p.rootPath as string), context) + this.watchRegistry.watch( + expandTilde(p.rootPath as string), + context, + typeof p.watchId === 'number' && Number.isSafeInteger(p.watchId) ? p.watchId : undefined + ) + ) + this.dispatcher.onRequest('fs.unwatchAndWait', (p, context) => + this.watchRegistry.unwatchAndWait(expandTilde(p.rootPath as string), context) ) this.dispatcher.onNotification('fs.unwatch', (p, context) => this.watchRegistry.unwatch(expandTilde(p.rootPath as string), context) @@ -247,7 +258,14 @@ export class FsHandler { if (stats.isDirectory() && !recursive) { throw new Error('Cannot delete directory without recursive flag') } - await rm(targetPath, { recursive: !!recursive, force: true }) + const remove = () => rm(targetPath, { recursive: !!recursive, force: true }) + if (stats.isDirectory()) { + // Why: forced orphan cleanup bypasses git.removeWorktree but must hold + // the same relay-wide watcher fence through recursive deletion. + await this.watchRegistry.runWithRemovalFence(targetPath, remove) + return + } + await remove() } private async createFile(params: Record) { diff --git a/src/relay/git-handler.test.ts b/src/relay/git-handler.test.ts index d674af9949e..ca787ac1e1b 100644 --- a/src/relay/git-handler.test.ts +++ b/src/relay/git-handler.test.ts @@ -139,6 +139,22 @@ describe('GitHandler', () => { expect(methods).toContain('git.isGitRepo') }) + it('runs remote worktree deletion inside the relay watcher fence', async () => { + const removalError = new Error('fenced before Git') + const runWithRemovalFence = vi.fn(async () => { + throw removalError + }) + handler.dispose() + handler = new GitHandler(dispatcher as unknown as RelayDispatcher, new RelayContext(), { + runWithRemovalFence + }) + + await expect( + dispatcher.callRequest('git.removeWorktree', { worktreePath: '/repo-feature' }) + ).rejects.toBe(removalError) + expect(runWithRemovalFence).toHaveBeenCalledWith('/repo-feature', expect.any(Function)) + }) + describe('abortMerge', () => { it('aborts an in-progress merge', async () => { gitInit(tmpDir) diff --git a/src/relay/git-handler.ts b/src/relay/git-handler.ts index 510dd60f84b..d46a900c7a2 100644 --- a/src/relay/git-handler.ts +++ b/src/relay/git-handler.ts @@ -69,6 +69,7 @@ import { syncForkDefaultBranch, validateGitForkSyncExpectedUpstream } from '../s import { InFlightPromiseDedupe, stableInFlightKey } from '../shared/in-flight-promise-dedupe' import { GIT_FETCH_SKIP_AUTO_MAINTENANCE_CONFIG_ARGS } from '../shared/git-fetch-auto-maintenance' import { GitCapabilityCache } from '../shared/git-capability-cache' +import type { RelayFilesystemWatchRegistry } from './relay-filesystem-watch-registry' import { hasUnsupportedRevParsePathFormatEcho, isUnsupportedRevParsePathFormatError @@ -178,7 +179,11 @@ export class GitHandler { // Why: RelayContext is accepted for protocol back-compat (see // docs/relay-fs-allowlist-removal.md) but no longer consulted on git ops. - constructor(dispatcher: RelayDispatcher, _context: RelayContext) { + constructor( + dispatcher: RelayDispatcher, + _context: RelayContext, + private readonly watcherRegistry?: Pick + ) { this.dispatcher = dispatcher this.registerHandlers() // Why: a detached client's git.responseAck frames will never arrive; wake @@ -1390,9 +1395,14 @@ export class GitHandler { } private async removeWorktree(params: Record) { - return this.runWithGitReadCacheClear(() => - removeWorktreeOp(this.git.bind(this), params, this.gitCapabilities) - ) + const remove = () => + this.runWithGitReadCacheClear(() => + removeWorktreeOp(this.git.bind(this), params, this.gitCapabilities) + ) + const worktreePath = params.worktreePath + return this.watcherRegistry && typeof worktreePath === 'string' + ? this.watcherRegistry.runWithRemovalFence(expandTilde(worktreePath), remove) + : remove() } private async worktreeIsClean(params: Record) { diff --git a/src/relay/pty-handler.test.ts b/src/relay/pty-handler.test.ts index c90e874800e..03c2d731012 100644 --- a/src/relay/pty-handler.test.ts +++ b/src/relay/pty-handler.test.ts @@ -31,7 +31,12 @@ vi.mock('node-pty', () => ({ spawn: mockPtySpawn })) -import { PtyHandler, attachIdentityMismatches } from './pty-handler' +import { + IMMEDIATE_PTY_EXIT_TIMEOUT_MS, + MAX_RELAY_PTY_SESSIONS, + PtyHandler, + attachIdentityMismatches +} from './pty-handler' import type { RelayDispatcher } from './dispatcher' function createMockDispatcher() { @@ -107,8 +112,10 @@ describe('PtyHandler', () => { handler = new PtyHandler(dispatcher as unknown as RelayDispatcher) }) - afterEach(() => { - handler.dispose() + afterEach(async () => { + const cleanup = handler.dispose({ waitForPhysicalExit: false }) + await vi.runAllTimersAsync() + await cleanup.catch(() => {}) vi.useRealTimers() }) @@ -174,6 +181,37 @@ describe('PtyHandler', () => { expect(handler.activePtyCount).toBe(1) }) + it('atomically caps concurrent PTY spawn admission', async () => { + const results = await Promise.allSettled( + Array.from({ length: MAX_RELAY_PTY_SESSIONS + 1 }, () => + dispatcher.callRequest('pty.spawn', { cols: 80, rows: 24 }) + ) + ) + + expect(results.filter(({ status }) => status === 'fulfilled')).toHaveLength( + MAX_RELAY_PTY_SESSIONS + ) + expect(results.filter(({ status }) => status === 'rejected')).toHaveLength(1) + expect(results.find(({ status }) => status === 'rejected')).toMatchObject({ + reason: expect.objectContaining({ message: 'Maximum number of PTY sessions reached (50)' }) + }) + expect(mockPtySpawn).toHaveBeenCalledTimes(MAX_RELAY_PTY_SESSIONS) + expect(handler.activePtyCount).toBe(MAX_RELAY_PTY_SESSIONS) + }) + + it('spawns a PTY without post-Node-18 array copy methods', async () => { + const descriptor = Object.getOwnPropertyDescriptor(Array.prototype, 'toReversed') + Reflect.deleteProperty(Array.prototype, 'toReversed') + try { + await expect(dispatcher.callRequest('pty.spawn', {})).resolves.toMatchObject({ id: 'pty-1' }) + expect(handler.activePtyCount).toBe(1) + } finally { + if (descriptor) { + Object.defineProperty(Array.prototype, 'toReversed', descriptor) + } + } + }) + it('uses an explicit shell override and falls back to the default shell otherwise', async () => { const originalPlatform = process.platform Object.defineProperty(process, 'platform', { @@ -650,6 +688,35 @@ describe('PtyHandler', () => { ).rejects.toThrow('PTY "pty-1" not found') }) + it('settles concurrent immediate shutdown when attach proves the shell exited', async () => { + const mockKill = vi.fn() + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + onData: vi.fn(), + onExit: vi.fn() + }) + await dispatcher.callRequest('pty.spawn', {}) + + let shutdown: Promise | undefined + const aliveSpy = vi.spyOn(ptyShellUtils, 'isProcessAlive').mockImplementation(() => { + shutdown = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + return false + }) + try { + await expect(dispatcher.callRequest('pty.attach', { id: 'pty-1' })).rejects.toThrow( + 'PTY "pty-1" not found' + ) + await expect(shutdown).resolves.toBeUndefined() + } finally { + aliveSpy.mockRestore() + } + + expect(mockKill).toHaveBeenCalledWith('SIGKILL') + expect(handler.activePtyCount).toBe(0) + expect(vi.getTimerCount()).toBe(0) + }) + it('replays for a reattach whose backing shell is still alive', async () => { // Guard the other direction: a live pid must never be reaped, so a quiet // but live session still returns its buffered replay on reattach. @@ -678,8 +745,16 @@ describe('PtyHandler', () => { }) it('terminates spawned PTY when request becomes stale before response', async () => { + let onExitCb: ((evt: { exitCode: number }) => void) | undefined const killSpy = vi.fn() - const term = { ...mockPtyInstance, kill: killSpy, onData: vi.fn(), onExit: vi.fn() } + const term = { + ...mockPtyInstance, + kill: killSpy, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) + } mockPtySpawn.mockReturnValue(term) await dispatcher.callRequest('pty.spawn', {}, { isStale: () => true }) @@ -692,6 +767,10 @@ describe('PtyHandler', () => { expect(killSpy).toHaveBeenCalledWith('SIGTERM') vi.advanceTimersByTime(5000) expect(killSpy).toHaveBeenCalledWith('SIGKILL') + expect(handler.activePtyCount).toBe(1) + + onExitCb?.({ exitCode: 137 }) + expect(handler.activePtyCount).toBe(0) }) it('does not submit provider-delivered commands for stale spawn responses', async () => { @@ -712,8 +791,16 @@ describe('PtyHandler', () => { }) it('releases pending provider-delivered commands on shutdown before delivery', async () => { + let onExitCb: ((evt: { exitCode: number }) => void) | undefined const killSpy = vi.fn() - const term = { ...mockPtyInstance, kill: killSpy, onData: vi.fn(), onExit: vi.fn() } + const term = { + ...mockPtyInstance, + kill: killSpy, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) + } mockPtySpawn.mockReturnValue(term) await dispatcher.callRequest('pty.spawn', { @@ -722,7 +809,9 @@ describe('PtyHandler', () => { }) expect(handler.retainedStartupCommandCount).toBe(1) - await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + const shutdown = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + onExitCb!({ exitCode: 137 }) + await shutdown vi.advanceTimersByTime(50) expect(handler.retainedStartupCommandCount).toBe(0) @@ -1062,46 +1151,96 @@ describe('PtyHandler', () => { it('on immediate shutdown', async () => { await withWindowsPlatform(async () => { - const mockKill = mockKillablePty() + let onExitCb: ((evt: { exitCode: number }) => void) | undefined + const mockKill = vi.fn() + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) + }) await dispatcher.callRequest('pty.spawn', {}) - await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + const shutdown = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + onExitCb!({ exitCode: 137 }) + await shutdown expectBareKills(mockKill, 1) }) }) - it('on stale-spawn cleanup including the SIGKILL fallback', async () => { + it('does not double-kill when immediate cleanup joins graceful shutdown', async () => { + await withWindowsPlatform(async () => { + let onExitCb: ((evt: { exitCode: number }) => void) | undefined + const mockKill = vi.fn() + const destroy = vi.fn() + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + destroy, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) + }) + await dispatcher.callRequest('pty.spawn', {}) + await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: false }) + const immediate = dispatcher.callRequest('pty.shutdown', { + id: 'pty-1', + immediate: true + }) + + expectBareKills(mockKill, 1) + onExitCb!({ exitCode: 137 }) + await immediate + expectBareKills(mockKill, 1) + expect(destroy).not.toHaveBeenCalled() + }) + }) + + it('does not retry stale-spawn cleanup after the Windows kill deadline', async () => { await withWindowsPlatform(async () => { const mockKill = mockKillablePty() await dispatcher.callRequest('pty.spawn', {}, { isStale: () => true }) expectBareKills(mockKill, 1) vi.advanceTimersByTime(5000) - expectBareKills(mockKill, 2) + expectBareKills(mockKill, 1) }) }) - it('on graceful-shutdown SIGKILL fallback', async () => { + it('does not retry graceful shutdown after the Windows kill deadline', async () => { await withWindowsPlatform(async () => { const mockKill = mockKillablePty() await dispatcher.callRequest('pty.spawn', {}) await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: false }) expectBareKills(mockKill, 1) vi.advanceTimersByTime(5000) - expectBareKills(mockKill, 2) + expectBareKills(mockKill, 1) }) }) it('on dispose', async () => { await withWindowsPlatform(async () => { - const mockKill = mockKillablePty() + const mockKill = vi.fn() + const destroy = vi.fn() + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + destroy, + onData: vi.fn(), + onExit: vi.fn() + }) await dispatcher.callRequest('pty.spawn', {}) - handler.dispose() + await handler.dispose({ waitForPhysicalExit: false }) expectBareKills(mockKill, 1) + expect(destroy).not.toHaveBeenCalled() }) }) }) it('flushes pending PTY output before immediate shutdown cleanup', async () => { let dataCallback: ((data: string) => void) | undefined + let onExitCb: ((evt: { exitCode: number }) => void) | undefined const mockKill = vi.fn() mockPtySpawn.mockReturnValue({ ...mockPtyInstance, @@ -1109,12 +1248,16 @@ describe('PtyHandler', () => { onData: vi.fn((cb: (data: string) => void) => { dataCallback = cb }), - onExit: vi.fn() + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) }) await dispatcher.callRequest('pty.spawn', {}) dataCallback!('last words') - await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + const shutdown = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + onExitCb!({ exitCode: 137 }) + await shutdown expect(dispatcher.notify).toHaveBeenNthCalledWith(1, 'pty.data', { id: 'pty-1', @@ -1140,26 +1283,69 @@ describe('PtyHandler', () => { await dispatcher.callRequest('pty.spawn', { env: { ORCA_PANE_KEY: 'tab-fallback:0' } }) await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: false }) vi.advanceTimersByTime(5000) + + expect(handler.activePtyCount).toBe(1) + expect(exits).toEqual([]) + expect(dispatcher.notify).not.toHaveBeenCalledWith('pty.exit', expect.anything()) onExitCb!({ exitCode: 137 }) expect(mockKill).toHaveBeenCalledWith('SIGTERM') expect(mockKill).toHaveBeenCalledWith('SIGKILL') - expect(dispatcher.notify).toHaveBeenCalledWith('pty.exit', { id: 'pty-1', code: -1 }) + expect(dispatcher.notify).toHaveBeenCalledWith('pty.exit', { id: 'pty-1', code: 137 }) expect(exits).toEqual([{ id: 'pty-1', paneKey: 'tab-fallback:0' }]) expect(handler.activePtyCount).toBe(0) }) + it('retries a rejected graceful SIGKILL fallback while retaining ownership', async () => { + let onExitCb: ((evt: { exitCode: number }) => void) | undefined + let forceAttempts = 0 + const mockKill = vi.fn((signal: string) => { + if (signal === 'SIGKILL' && forceAttempts++ === 0) { + throw new Error('transient kill failure') + } + }) + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) + }) + + await dispatcher.callRequest('pty.spawn', {}) + await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: false }) + vi.advanceTimersByTime(5000) + + expect(mockKill.mock.calls).toEqual([['SIGTERM'], ['SIGKILL']]) + expect(handler.activePtyCount).toBe(1) + expect(vi.getTimerCount()).toBe(1) + + vi.runOnlyPendingTimers() + expect(mockKill.mock.calls).toEqual([['SIGTERM'], ['SIGKILL'], ['SIGKILL']]) + expect(handler.activePtyCount).toBe(1) + + onExitCb!({ exitCode: 137 }) + expect(handler.activePtyCount).toBe(0) + expect(vi.getTimerCount()).toBe(0) + }) + it('kills PTY on shutdown with SIGKILL when immediate', async () => { + let onExitCb: ((evt: { exitCode: number }) => void) | undefined const mockKill = vi.fn() mockPtySpawn.mockReturnValue({ ...mockPtyInstance, kill: mockKill, onData: vi.fn(), - onExit: vi.fn() + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) }) await dispatcher.callRequest('pty.spawn', {}) - await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + const shutdown = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + onExitCb!({ exitCode: 137 }) + await shutdown expect(mockKill).toHaveBeenCalledWith('SIGKILL') }) @@ -1444,6 +1630,43 @@ describe('PtyHandler', () => { expect(spawnEnv.env.TERM).toBe('xterm-256color') }) + it('admits default-cwd spawns through the worktree removal coordinator', async () => { + const finishCreation = vi.fn() + const beginWorktreePtySpawn = vi.fn((_operationPath: string) => finishCreation) + handler.setWorktreeRemovalCoordinator({ beginWorktreePtySpawn }) + + await dispatcher.callRequest('pty.spawn', {}) + + expect(beginWorktreePtySpawn).toHaveBeenCalledWith(expect.any(String)) + expect(beginWorktreePtySpawn.mock.calls[0][0]).not.toBe('') + expect(finishCreation).toHaveBeenCalledTimes(1) + }) + + it('fences both sibling worktree identity and removing cwd with rollback', async () => { + const finishSiblingAdmission = vi.fn() + const beginWorktreePtySpawn = vi.fn((operationPath: string) => { + if (operationPath === '/repo/removing/nested') { + throw new Error('Remote worktree deletion already in progress') + } + return finishSiblingAdmission + }) + handler.setWorktreeRemovalCoordinator({ beginWorktreePtySpawn }) + + await expect( + dispatcher.callRequest('pty.spawn', { + cwd: '/repo/removing/nested', + env: { ORCA_WORKTREE_ID: 'repo-id::/repo/sibling' } + }) + ).rejects.toThrow('Remote worktree deletion already in progress') + + expect(beginWorktreePtySpawn.mock.calls.map(([operationPath]) => operationPath)).toEqual([ + '/repo/sibling', + '/repo/removing/nested' + ]) + expect(finishSiblingAdmission).toHaveBeenCalledOnce() + expect(mockPtySpawn).not.toHaveBeenCalled() + }) + it('lets relay env augmenters resolve the original sequenced startup command hint', async () => { handler.addEnvAugmenter((ctx) => ({ SEEN_LAUNCH_COMMAND_HINT: resolveSetupAgentSequenceLaunchCommand(ctx.env, ctx.command) ?? '' @@ -1529,7 +1752,7 @@ describe('PtyHandler', () => { }) const state = (await dispatcher.callRequest('pty.serialize', { ids: ['pty-1'] })) as string - handler.dispose() + await handler.dispose({ waitForPhysicalExit: false }) mockPtySpawn.mockClear() dispatcher = createMockDispatcher() handler = new PtyHandler(dispatcher as unknown as RelayDispatcher) @@ -1555,6 +1778,91 @@ describe('PtyHandler', () => { expect(callArgs.env.TERM_PROGRAM).toBe('Orca') }) + it('fences both revived worktree identity and cwd with rollback', async () => { + const finishSiblingAdmission = vi.fn() + const beginWorktreePtySpawn = vi.fn((operationPath: string) => { + if (operationPath === '/repo/removing/nested') { + throw new Error('Remote worktree deletion already in progress') + } + return finishSiblingAdmission + }) + handler.setWorktreeRemovalCoordinator({ beginWorktreePtySpawn }) + const state = JSON.stringify([ + { + id: 'pty-7', + pid: process.pid, + cols: 80, + rows: 24, + cwd: '/repo/removing/nested', + worktreeId: 'repo-id::/repo/sibling' + } + ]) + const killSpy = vi.spyOn(process, 'kill').mockImplementation(() => true) + try { + await expect(dispatcher.callRequest('pty.revive', { state })).rejects.toThrow( + 'Remote worktree deletion already in progress' + ) + } finally { + killSpy.mockRestore() + } + + expect(beginWorktreePtySpawn.mock.calls.map(([operationPath]) => operationPath)).toEqual([ + '/repo/sibling', + '/repo/removing/nested' + ]) + expect(finishSiblingAdmission).toHaveBeenCalledOnce() + expect(mockPtySpawn).not.toHaveBeenCalled() + }) + + it('applies the physical PTY cap to untrusted revive state', async () => { + const state = JSON.stringify( + Array.from({ length: MAX_RELAY_PTY_SESSIONS + 1 }, (_, index) => ({ + id: `pty-${index + 1}`, + pid: process.pid, + cols: 80, + rows: 24, + cwd: '/repo', + worktreeId: 'repo-id::/repo' + })) + ) + const killSpy = vi.spyOn(process, 'kill').mockImplementation(() => true) + try { + await expect(dispatcher.callRequest('pty.revive', { state })).rejects.toThrow( + 'Maximum number of PTY sessions reached (50)' + ) + } finally { + killSpy.mockRestore() + } + + expect(mockPtySpawn).toHaveBeenCalledTimes(MAX_RELAY_PTY_SESSIONS) + expect(handler.activePtyCount).toBe(MAX_RELAY_PTY_SESSIONS) + }) + + it('deduplicates concurrent revive requests for the same physical PTY id', async () => { + const state = JSON.stringify([ + { + id: 'pty-7', + pid: process.pid, + cols: 80, + rows: 24, + cwd: '/repo', + worktreeId: 'repo-id::/repo' + } + ]) + const killSpy = vi.spyOn(process, 'kill').mockImplementation(() => true) + try { + await Promise.all([ + dispatcher.callRequest('pty.revive', { state }), + dispatcher.callRequest('pty.revive', { state }) + ]) + } finally { + killSpy.mockRestore() + } + + expect(mockPtySpawn).toHaveBeenCalledTimes(1) + expect(handler.activePtyCount).toBe(1) + }) + it('normalizes an explicit empty TERM and preserves sanitized env deletions on revive', async () => { await dispatcher.callRequest('pty.spawn', { env: { TERM: '' }, @@ -1576,7 +1884,7 @@ describe('PtyHandler', () => { expect(serialized.explicitTerm).toBeUndefined() expect(serialized.envToDelete).toEqual(['ORCA_ATTRIBUTION_SHIM_DIR']) - handler.dispose() + await handler.dispose({ waitForPhysicalExit: false }) mockPtySpawn.mockClear() dispatcher = createMockDispatcher() handler = new PtyHandler(dispatcher as unknown as RelayDispatcher) @@ -1649,7 +1957,7 @@ describe('PtyHandler', () => { const killSpy = vi.spyOn(process, 'kill').mockImplementation(() => true) try { - handler.dispose() + await handler.dispose({ waitForPhysicalExit: false }) mockPtySpawn.mockClear() dispatcher = createMockDispatcher() handler = new PtyHandler(dispatcher as unknown as RelayDispatcher) @@ -1673,7 +1981,7 @@ describe('PtyHandler', () => { } ]) - handler.dispose() + await handler.dispose({ waitForPhysicalExit: false }) mockPtySpawn.mockClear() dispatcher = createMockDispatcher() handler = new PtyHandler(dispatcher as unknown as RelayDispatcher) @@ -1747,7 +2055,7 @@ describe('PtyHandler', () => { } const state = (await dispatcher.callRequest('pty.serialize', { ids: ['pty-1'] })) as string - handler.dispose() + await handler.dispose({ waitForPhysicalExit: false }) mockPtySpawn.mockClear() dispatcher = createMockDispatcher() handler = new PtyHandler(dispatcher as unknown as RelayDispatcher) @@ -1805,7 +2113,7 @@ describe('PtyHandler', () => { expect(exits).toEqual([{ id: 'pty-1', paneKey: 'tab-2:1' }]) }) - it('immediate shutdown invokes the exit listener once even if onExit arrives later', async () => { + it('keeps immediate shutdown pending until onExit and invokes the exit listener once', async () => { let onExitCb: ((evt: { exitCode: number }) => void) | undefined const mockKill = vi.fn() mockPtySpawn.mockReturnValue({ @@ -1822,21 +2130,219 @@ describe('PtyHandler', () => { await dispatcher.callRequest('pty.spawn', { env: { ORCA_PANE_KEY: 'tab-shutdown:0' } }) - await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + let settled = false + const shutdown = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + void shutdown.then(() => { + settled = true + }) + await Promise.resolve() + + expect(settled).toBe(false) + expect(exits).toEqual([]) + expect(handler.activePtyCount).toBe(1) onExitCb!({ exitCode: 0 }) + await shutdown expect(mockKill).toHaveBeenCalledWith('SIGKILL') expect(exits).toEqual([{ id: 'pty-1', paneKey: 'tab-shutdown:0' }]) expect(handler.activePtyCount).toBe(0) }) - it('dispose kills all PTYs with SIGKILL and invokes exit listeners', async () => { + it('physically stops only matching worktree PTYs before relay deletion', async () => { + let firstExit: ((evt: { exitCode: number }) => void) | undefined + let secondExit: ((evt: { exitCode: number }) => void) | undefined + const firstKill = vi.fn() + const secondKill = vi.fn() + mockPtySpawn + .mockReturnValueOnce({ + ...mockPtyInstance, + kill: firstKill, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + firstExit = cb + }) + }) + .mockReturnValueOnce({ + ...mockPtyInstance, + kill: secondKill, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + secondExit = cb + }) + }) + + await dispatcher.callRequest('pty.spawn', { + cwd: '/repo', + env: { ORCA_WORKTREE_ID: 'repo-id::/repo' } + }) + await dispatcher.callRequest('pty.spawn', { + cwd: '/sibling', + env: { ORCA_WORKTREE_ID: 'repo-id::/sibling' } + }) + + let settled = false + const shutdown = handler.shutdownForWorktreePath('/repo').finally(() => { + settled = true + }) + await Promise.resolve() + expect(firstKill).toHaveBeenCalledWith('SIGKILL') + expect(secondKill).not.toHaveBeenCalled() + expect(settled).toBe(false) + expect(handler.activePtyCount).toBe(2) + + firstExit?.({ exitCode: 137 }) + await shutdown + expect(secondExit).toBeDefined() + expect(handler.activePtyCount).toBe(1) + }) + + it('rejects timed-out immediate shutdown while retaining the physical owner', async () => { + let onExitCb: ((evt: { exitCode: number }) => void) | undefined const mockKill = vi.fn() mockPtySpawn.mockReturnValue({ ...mockPtyInstance, kill: mockKill, onData: vi.fn(), - onExit: vi.fn() + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) + }) + + await dispatcher.callRequest('pty.spawn', {}) + const shutdown = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + const rejected = expect(shutdown).rejects.toThrow('Timed out waiting for PTY process exit') + await vi.advanceTimersByTimeAsync(IMMEDIATE_PTY_EXIT_TIMEOUT_MS) + await rejected + + expect(mockKill).toHaveBeenCalledTimes(1) + expect(handler.activePtyCount).toBe(1) + const retry = dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: true }) + expect(mockKill).toHaveBeenCalledTimes(1) + onExitCb!({ exitCode: 137 }) + await retry + expect(handler.activePtyCount).toBe(0) + }) + + it('fences late creation and drains an admitted spawn before the disposal snapshot', async () => { + let onExitCb: ((evt: { exitCode: number }) => void) | undefined + const mockKill = vi.fn() + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) + }) + + const admittedSpawn = dispatcher.callRequest('pty.spawn', {}) + const dispose = handler.dispose() + await admittedSpawn + + expect(mockKill).toHaveBeenCalledWith('SIGKILL') + expect(handler.activePtyCount).toBe(1) + await expect(dispatcher.callRequest('pty.spawn', {})).rejects.toThrow( + 'PTY handler is shutting down' + ) + const aliveSpy = vi.spyOn(process, 'kill').mockImplementation(() => true) + try { + await expect( + dispatcher.callRequest('pty.revive', { + state: JSON.stringify([ + { id: 'pty-late', pid: process.pid, cols: 80, rows: 24, cwd: '/tmp' } + ]) + }) + ).rejects.toThrow('PTY handler is shutting down') + } finally { + aliveSpy.mockRestore() + } + + onExitCb!({ exitCode: 137 }) + await dispose + expect(handler.activePtyCount).toBe(0) + expect(handler.dispose()).toBe(dispose) + }) + + it('retries a rejected force kill during dispose and waits for physical exit', async () => { + let onExitCb: ((evt: { exitCode: number }) => void) | undefined + let forceAttempts = 0 + const mockKill = vi.fn((signal: string) => { + if (signal === 'SIGKILL' && forceAttempts++ === 0) { + throw new Error('transient dispose kill failure') + } + }) + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) + }) + + await dispatcher.callRequest('pty.spawn', {}) + const dispose = handler.dispose() + await Promise.resolve() + + expect(mockKill.mock.calls).toEqual([['SIGKILL']]) + expect(handler.activePtyCount).toBe(1) + await vi.advanceTimersByTimeAsync(250) + expect(mockKill.mock.calls).toEqual([['SIGKILL'], ['SIGKILL']]) + expect(handler.activePtyCount).toBe(1) + + onExitCb!({ exitCode: 137 }) + await dispose + expect(handler.activePtyCount).toBe(0) + expect(vi.getTimerCount()).toBe(0) + }) + + it('takes ownership when dispose overlaps a queued graceful force-kill retry', async () => { + let onExitCb: ((evt: { exitCode: number }) => void) | undefined + let forceAttempts = 0 + const mockKill = vi.fn((signal: string) => { + if (signal === 'SIGKILL' && forceAttempts++ < 2) { + throw new Error('transient overlapping kill failure') + } + }) + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCb = cb + }) + }) + + await dispatcher.callRequest('pty.spawn', {}) + await dispatcher.callRequest('pty.shutdown', { id: 'pty-1', immediate: false }) + vi.advanceTimersByTime(5000) + expect(mockKill.mock.calls).toEqual([['SIGTERM'], ['SIGKILL']]) + expect(vi.getTimerCount()).toBe(1) + + const dispose = handler.dispose() + await Promise.resolve() + expect(mockKill.mock.calls).toEqual([['SIGTERM'], ['SIGKILL'], ['SIGKILL']]) + expect(vi.getTimerCount()).toBe(1) + await vi.advanceTimersByTimeAsync(250) + expect(mockKill.mock.calls).toEqual([['SIGTERM'], ['SIGKILL'], ['SIGKILL'], ['SIGKILL']]) + expect(handler.activePtyCount).toBe(1) + + onExitCb!({ exitCode: 137 }) + await dispose + expect(handler.activePtyCount).toBe(0) + expect(vi.getTimerCount()).toBe(0) + }) + + it('dispose kills all PTYs with SIGKILL and invokes exit listeners', async () => { + const mockKill = vi.fn() + const onExitCallbacks: ((evt: { exitCode: number }) => void)[] = [] + mockPtySpawn.mockReturnValue({ + ...mockPtyInstance, + kill: mockKill, + onData: vi.fn(), + onExit: vi.fn((cb: (evt: { exitCode: number }) => void) => { + onExitCallbacks.push(cb) + }) }) const exits: { id: string; paneKey?: string }[] = [] handler.setExitListener((evt) => exits.push(evt)) @@ -1845,12 +2351,18 @@ describe('PtyHandler', () => { await dispatcher.callRequest('pty.spawn', { env: { ORCA_PANE_KEY: 'tab-dispose:1' } }) expect(handler.activePtyCount).toBe(2) - handler.dispose() + const dispose = handler.dispose() + await Promise.resolve() // Why: dispose uses SIGKILL (not SIGTERM) because the relay process is // exiting. A SIGTERM-ignoring remote shell (editor with unsaved buffers, // wedged process, uninterruptible sleep) would survive SIGTERM + immediate // destroy() as an orphan on the remote host. SIGKILL is not ignorable. expect(mockKill).toHaveBeenCalledWith('SIGKILL') + expect(handler.activePtyCount).toBe(2) + for (const onExit of onExitCallbacks) { + onExit({ exitCode: 137 }) + } + await dispose expect(exits).toEqual([ { id: 'pty-1', paneKey: 'tab-dispose:0' }, { id: 'pty-2', paneKey: 'tab-dispose:1' } diff --git a/src/relay/pty-handler.ts b/src/relay/pty-handler.ts index d7dd995005a..1aabb27c4d6 100644 --- a/src/relay/pty-handler.ts +++ b/src/relay/pty-handler.ts @@ -18,6 +18,12 @@ import { DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS } from '../shared/ssh-types' import { shouldUseShellReadyStartupDelivery } from '../shared/codex-startup-delivery' import { buildStartupCommandSubmission } from '../shared/startup-command-submission' import { resolveSetupAgentSequenceLaunchCommand } from '../shared/setup-agent-sequencing' +import { + isPathInsideOrEqual, + normalizeRuntimePathForComparison +} from '../shared/cross-platform-path' +import { splitWorktreeId } from '../shared/worktree-id' +import { PhysicalExitTracker } from '../shared/physical-exit-tracker' import { createShellReadyScanState, drainShellReadyHeldBytes, @@ -54,9 +60,7 @@ type ManagedPty = { * entry-point calls into a clean "not found" error instead of a silent no-op * (POSIX proc.kill is neutralized inside disposeManagedPty). */ disposed?: boolean - /** True once external cleanup observers have been notified. Forced cleanup - * paths can run before node-pty emits onExit; this prevents duplicate - * overlay/cache cleanup if onExit arrives later. */ + /** True once external cleanup observers have been notified. */ exitListenerNotified?: boolean /** Renderer-supplied paneKey from spawn env (ORCA_PANE_KEY). Captured so * external observers (the relay-hook-server cache) can evict per-pane @@ -71,6 +75,9 @@ type ManagedPty = { explicitTerm?: string envToDelete: string[] startupCommand?: ManagedStartupCommand + physicalExit?: PhysicalExitTracker + forceKillSent?: boolean + gracefulKillSent?: boolean } type PendingPtyOutput = { @@ -97,6 +104,13 @@ function killPtyProcess(pty: IPty, signal: string): void { pty.kill(signal) } +function finishPtyCreationOperations(operations: readonly (() => void)[]): void { + // Why: the relay still targets Node 18, which lacks Array.prototype.toReversed. + for (let index = operations.length - 1; index >= 0; index--) { + operations[index]() + } +} + function disposeManagedPty(managed: ManagedPty): void { if (managed.disposed) { return @@ -119,6 +133,10 @@ function disposeManagedPty(managed: ManagedPty): void { // IS a kill() call via _deferNoArgs — neutralizing it leaks the ConPTY agent. if (process.platform !== 'win32') { ;(managed.pty as unknown as { kill: (sig?: string) => void }).kill = () => {} + } else if (managed.gracefulKillSent || managed.forceKillSent) { + // Why: WindowsTerminal.destroy() calls kill() internally; any prior bare + // kill already closed ConPTY, so destroy would double-close the handle. + return } try { ;(managed.pty as unknown as { destroy?: () => void }).destroy?.() @@ -127,6 +145,8 @@ function disposeManagedPty(managed: ManagedPty): void { } } const DEFAULT_GRACE_TIME_MS = DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS * 1000 +export const IMMEDIATE_PTY_EXIT_TIMEOUT_MS = 8_000 +export const MAX_RELAY_PTY_SESSIONS = 50 export const REPLAY_BUFFER_MAX = 100 * 1024 const PTY_OUTPUT_BATCH_INTERVAL_MS = 8 const PTY_OUTPUT_DRAIN_CONTINUE_MS = 1 @@ -138,6 +158,8 @@ const INTERACTIVE_REDRAW_MAX_CHARS = PTY_OUTPUT_FLUSH_CHUNK_CHARS const INTERACTIVE_OUTPUT_BUDGET_CHARS = 32 * 1024 const STARTUP_COMMAND_WRITE_DELAY_MS = 50 const STARTUP_COMMAND_SHELL_READY_FALLBACK_MS = 1500 +const PTY_FORCE_KILL_RETRY_DELAY_MS = 250 +const PTY_FORCE_KILL_MAX_ATTEMPTS = 2 const ALLOWED_SIGNALS = new Set([ 'SIGINT', 'SIGTERM', @@ -180,6 +202,7 @@ type PtyProcessSummary = { id: string cwd: string title: string + worktreeId?: string terminalHandle?: string } @@ -239,6 +262,10 @@ export type PtyEnvAugmenter = (ctx: { command?: string }) => Record +export type RelayPtyWorktreeRemovalCoordinator = { + beginWorktreePtySpawn(operationPath: string): () => void +} + export class PtyHandler { private ptys = new Map() private nextId = 1 @@ -249,6 +276,12 @@ export class PtyHandler { private pendingOutputByPty = new Map() private lastInputAtByPty = new Map() private interactiveOutputCharsByPty = new Map() + private pendingSpawnCount = 0 + private pendingReviveIds = new Set() + private creationFenced = false + private pendingCreationDrainResolvers = new Set<() => void>() + private worktreeRemovalCoordinator: RelayPtyWorktreeRemovalCoordinator | null = null + private disposePromise: Promise | null = null // Why: external observers need to drop per-pane state when a PTY exits. // Today the relay composes multiple consumers (hook-server cache eviction // and plugin-overlay dir cleanup) into a single callback at the call site @@ -273,6 +306,25 @@ export class PtyHandler { this.graceTimeMs = Math.max(0, Math.floor(graceTimeMs)) } + setWorktreeRemovalCoordinator(coordinator: RelayPtyWorktreeRemovalCoordinator | null): void { + this.worktreeRemovalCoordinator = coordinator + } + + async shutdownForWorktreePath(rootPath: string): Promise { + const matchingIds = [...this.ptys.values()] + .filter((managed) => { + const ownedPath = managed.worktreeId + ? splitWorktreeId(managed.worktreeId)?.worktreePath + : undefined + return ( + (ownedPath !== undefined && isPathInsideOrEqual(rootPath, ownedPath)) || + isPathInsideOrEqual(rootPath, managed.initialCwd) + ) + }) + .map((managed) => managed.id) + await Promise.all(matchingIds.map((id) => this.shutdown({ id, immediate: true }))) + } + get configuredGraceTimeMs(): number { return this.graceTimeMs } @@ -411,6 +463,7 @@ export class PtyHandler { /** Wire onData/onExit listeners for a managed PTY and store it. */ private wireAndStore(managed: ManagedPty): void { + managed.physicalExit = new PhysicalExitTracker() this.ptys.set(managed.id, managed) managed.pty.onData((data: string) => { const startup = managed.startupCommand @@ -425,6 +478,7 @@ export class PtyHandler { this.enqueuePtyOutput(managed.id, data) }) managed.pty.onExit(({ exitCode }: { exitCode: number }) => { + managed.physicalExit?.markExited() if (managed.disposed) { return } @@ -464,9 +518,7 @@ export class PtyHandler { } managed.exitListenerNotified = true // Why: external observers own relay-hook cache eviction and plugin-overlay - // cleanup. Natural exits, immediate shutdown, SIGKILL fallback, and relay - // process disposal all need the same cleanup even when node-pty never - // delivers onExit. + // cleanup. Physical exits and whole-relay disposal both need it exactly once. if (this.exitListener) { try { this.exitListener({ id: managed.id, paneKey: managed.paneKey }) @@ -598,13 +650,81 @@ export class PtyHandler { this.clearOutputFlushTimerIfIdle() } + private beginPtyCreation(operationPaths: readonly (string | undefined)[]): () => void { + if (this.creationFenced) { + throw new Error('PTY handler is shutting down') + } + const distinctPaths = new Map() + for (const operationPath of operationPaths) { + if (operationPath) { + distinctPaths.set(normalizeRuntimePathForComparison(operationPath), operationPath) + } + } + const finishRemovalOperations: (() => void)[] = [] + try { + if (this.worktreeRemovalCoordinator) { + for (const operationPath of distinctPaths.values()) { + finishRemovalOperations.push( + this.worktreeRemovalCoordinator.beginWorktreePtySpawn(operationPath) + ) + } + } + if (this.ptys.size + this.pendingSpawnCount >= MAX_RELAY_PTY_SESSIONS) { + throw new Error('Maximum number of PTY sessions reached (50)') + } + } catch (error) { + // Why: worktree identity and cwd can belong to different roots. A later + // rejection must release every earlier admission before propagating. + finishPtyCreationOperations(finishRemovalOperations) + throw error + } + this.pendingSpawnCount++ + let finished = false + return () => { + if (finished) { + return + } + finished = true + this.pendingSpawnCount-- + if (this.pendingSpawnCount === 0) { + for (const resolve of this.pendingCreationDrainResolvers) { + resolve() + } + this.pendingCreationDrainResolvers.clear() + } + finishPtyCreationOperations(finishRemovalOperations) + } + } + + private waitForPendingPtyCreations(): Promise { + if (this.pendingSpawnCount === 0) { + return Promise.resolve() + } + return new Promise((resolve) => { + this.pendingCreationDrainResolvers.add(resolve) + }) + } + private async spawn( params: Record, context?: RequestContext ): Promise<{ id: string }> { - if (this.ptys.size >= 50) { - throw new Error('Maximum number of PTY sessions reached (50)') + const env = params.env as Record | undefined + const worktreeId = env?.ORCA_WORKTREE_ID + const worktreePath = worktreeId ? splitWorktreeId(worktreeId)?.worktreePath : undefined + const cwd = typeof params.cwd === 'string' ? params.cwd : resolveDefaultCwd() + const finishCreation = this.beginPtyCreation([worktreePath, cwd]) + try { + return await this.spawnAfterAdmission(params, context) + } finally { + finishCreation() } + } + + private async spawnAfterAdmission( + params: Record, + context?: RequestContext + ): Promise<{ id: string }> { const pty = await loadPty() if (!pty) { throw new Error('node-pty is not available on this remote host') @@ -627,7 +747,10 @@ export class PtyHandler { typeof params.shellOverride === 'string' ? params.shellOverride.trim() : '' const resolvedShellOverride = resolvePtyShellOverride(shellOverride) const shell = resolvedShellOverride || resolveDefaultShell() - const id = `pty-${this.nextId++}` + let id: string + do { + id = `pty-${this.nextId++}` + } while (this.ptys.has(id) || this.pendingReviveIds.has(id)) // Why: server-side augmenter values (ORCA_AGENT_HOOK_* and plugin overlay // dirs) override renderer-supplied env so live remote paths and hook coords @@ -722,20 +845,7 @@ export class PtyHandler { // response is discarded and no renderer can own this PTY. Shut it down // immediately so it does not linger as an unreachable remote shell. this.releaseStartupCommand(managed) - killPtyProcess(term, 'SIGTERM') - managed.killTimer = setTimeout(() => { - const still = this.ptys.get(id) - if (still && !still.disposed) { - killPtyProcess(still.pty, 'SIGKILL') - // Why: stale-spawn cleanup has no client who will ever attach. If - // SIGKILL's onExit is missed (kernel edge case, uninterruptible - // sleep), the managed entry + ptmx fd would leak forever. Dispose - // synchronously so the entry is gone regardless of onExit timing. - this.notifyExitListener(still) - disposeManagedPty(still) - this.ptys.delete(id) - } - }, 5000) + this.requestGracefulKill(managed, 'terminate stale') } else if (managed.startupCommand) { this.scheduleStartupCommandDelivery( managed, @@ -767,6 +877,7 @@ export class PtyHandler { // drops the dead lease and spawns fresh — the same recovery path an expired // grace window already takes. if (managed.pty.pid && !isProcessAlive(managed.pty.pid)) { + managed.physicalExit?.markExited() this.notifyExitListener(managed) disposeManagedPty(managed) this.ptys.delete(id) @@ -851,58 +962,14 @@ export class PtyHandler { if (immediate) { this.releaseStartupCommand(managed) this.flushPtyOutput(id) - killPtyProcess(managed.pty, 'SIGKILL') - // Why: SIGKILL has already reaped the child; release the ptmx fd on the - // same tick. Deferring to onExit leaves a window where the fd is live - // with a dead child. Idempotent via the disposed guard — if onExit fires - // later and also calls disposeManagedPty, the second call is a no-op. - disposeManagedPty(managed) - // Why: mirror the graceful-shutdown killTimer cleanup. If SIGKILL's - // onExit never fires (kernel edge case: uninterruptible sleep, - // D-state child on a bad NFS mount), the disposed managed entry would - // linger in this.ptys forever. Each stranded entry consumes a slot in - // the 50-PTY cap and is returned by listProcesses/serialize. Deleting - // here makes the map hygiene a hard guarantee, not "hopefully onExit - // runs". If onExit DOES fire later, its own `this.ptys.delete(id)` is - // a no-op. - this.notifyExitListener(managed) - this.ptys.delete(id) - this.clearPtyFlowState(id) + this.requestForceKill(managed) + // Why: remote Git deletion must not race native handles still owned by + // an uninterruptible child. Timeout rejects but deliberately keeps the + // map entry so a later onExit or retry retains the physical owner. + await this.waitForPhysicalExit(managed, IMMEDIATE_PTY_EXIT_TIMEOUT_MS) } else { this.releaseStartupCommand(managed) - killPtyProcess(managed.pty, 'SIGTERM') - - // Why: Some processes ignore SIGTERM (e.g. a hung child, a custom signal - // handler). Without a SIGKILL fallback the PTY process would leak and the - // managed entry would never be cleaned up. The 5-second window gives - // well-behaved processes time to flush and exit gracefully. The timer is - // cleared in the onExit handler if the process terminates on its own. - // Do NOT call disposeManagedPty here: destroy()-right-after-SIGTERM - // collapses the graceful-shutdown window and risks interrupting shell - // EXIT traps. Fd release happens via onExit (natural exit) or via the - // killTimer → SIGKILL → disposeManagedPty chain below. - managed.killTimer = setTimeout(() => { - const still = this.ptys.get(id) - if (still && !still.disposed) { - killPtyProcess(still.pty, 'SIGKILL') - this.flushPtyOutput(id) - // Why: emit pty.exit BEFORE disposeManagedPty sets disposed=true. - // The natural onExit short-circuits on `managed.disposed`, so - // without this notify the renderer never learns the pane is dead - // when the SIGKILL fallback fires for a SIGTERM-ignoring child. - this.dispatcher.notify('pty.exit', { id, code: -1 }) - // Why: if SIGKILL's onExit never fires (kernel edge case, - // uninterruptible sleep, child wedged on a bad NFS mount), the - // fd and map entry would leak forever. Dispose synchronously so - // graceful-shutdown's SIGKILL fallback is a hard guarantee, not - // "hopefully onExit will run". The disposed guard inside - // disposeManagedPty makes a later onExit's dispose a no-op. - this.notifyExitListener(still) - disposeManagedPty(still) - this.ptys.delete(id) - this.clearPtyFlowState(id) - } - }, 5000) + this.requestGracefulKill(managed, 'force-kill') } } @@ -922,6 +989,90 @@ export class PtyHandler { managed.pty.kill(signal) } + private waitForPhysicalExit(managed: ManagedPty, timeoutMs: number): Promise { + const physicalExit = managed.physicalExit + if (!physicalExit) { + return Promise.reject(new Error(`PTY "${managed.id}" exit tracking unavailable`)) + } + return physicalExit.waitForExit( + timeoutMs, + () => new Error(`Timed out waiting for PTY process exit: ${managed.id}`) + ) + } + + private requestGracefulKill( + managed: ManagedPty, + fallbackAction: 'terminate stale' | 'force-kill' + ): void { + if (managed.gracefulKillSent) { + return + } + managed.gracefulKillSent = true + if (process.platform === 'win32') { + // Why: bare node-pty kill is already force-final on ConPTY, so no later + // fallback, immediate cleanup, or destroy may close the handle again. + managed.forceKillSent = true + } + try { + killPtyProcess(managed.pty, 'SIGTERM') + } catch (error) { + managed.gracefulKillSent = false + managed.forceKillSent = false + throw error + } + if (process.platform === 'win32') { + return + } + // Why: POSIX children may ignore SIGTERM; only onExit releases ownership + // after the bounded SIGKILL fallback. + this.armForceKillFallback(managed, fallbackAction, 5000, PTY_FORCE_KILL_MAX_ATTEMPTS) + } + + private armForceKillFallback( + managed: ManagedPty, + fallbackAction: 'terminate stale' | 'force-kill', + delayMs: number, + attemptsRemaining: number + ): void { + managed.killTimer = setTimeout(() => { + managed.killTimer = undefined + const still = this.ptys.get(managed.id) + if (!still || still.disposed) { + return + } + try { + this.requestForceKill(still) + } catch (error) { + process.stderr.write( + `[pty-handler] failed to ${fallbackAction} PTY ${managed.id}: ${error instanceof Error ? error.message : String(error)}\n` + ) + // Why: a transient native SIGKILL failure must not strand an + // unreachable remote shell after the only cleanup owner returned. + if (attemptsRemaining > 1 && this.ptys.get(still.id) === still && !still.disposed) { + this.armForceKillFallback( + still, + fallbackAction, + PTY_FORCE_KILL_RETRY_DELAY_MS, + attemptsRemaining - 1 + ) + } + } + }, delayMs) + } + + private requestForceKill(managed: ManagedPty): void { + if (managed.forceKillSent || (process.platform === 'win32' && managed.gracefulKillSent)) { + return + } + managed.forceKillSent = true + try { + killPtyProcess(managed.pty, 'SIGKILL') + } catch (error) { + managed.forceKillSent = false + throw error + } + } + private async getCwd(params: Record): Promise { const id = params.id as string const managed = this.ptys.get(id) @@ -975,6 +1126,7 @@ export class PtyHandler { id, cwd: managed.initialCwd, title, + ...(managed.worktreeId ? { worktreeId: managed.worktreeId } : {}), ...(managed.terminalHandle ? { terminalHandle: managed.terminalHandle } : {}) }) } @@ -1013,7 +1165,7 @@ export class PtyHandler { const entries = JSON.parse(state) as SerializedPtyEntry[] for (const entry of entries) { - if (this.ptys.has(entry.id)) { + if (this.ptys.has(entry.id) || this.pendingReviveIds.has(entry.id)) { continue } // Only re-attach if the original process is still alive @@ -1022,90 +1174,82 @@ export class PtyHandler { } catch { continue } - const ptyMod = await loadPty() - if (!ptyMod) { - continue + const ownedPath = entry.worktreeId + ? splitWorktreeId(entry.worktreeId)?.worktreePath + : undefined + const finishCreation = this.beginPtyCreation([ownedPath, entry.cwd]) + this.pendingReviveIds.add(entry.id) + try { + await this.reviveEntry(entry) + } finally { + this.pendingReviveIds.delete(entry.id) + finishCreation() } - // Why: revive must apply the same hook env as spawn(). The hook-server - // coords come from augmenters, while pane identity comes from the - // serialized PTY entry because managed hook scripts exit without - // ORCA_PANE_KEY. - const revivedEnv: Record = {} - if (entry.paneKey) { - revivedEnv.ORCA_PANE_KEY = entry.paneKey - } - if (entry.tabId) { - revivedEnv.ORCA_TAB_ID = entry.tabId - } - if (entry.worktreeId) { - revivedEnv.ORCA_WORKTREE_ID = entry.worktreeId - } - if (entry.terminalHandle) { - revivedEnv.ORCA_TERMINAL_HANDLE = entry.terminalHandle - } - const explicitTerm = - typeof entry.explicitTerm === 'string' && entry.explicitTerm.length > 0 - ? entry.explicitTerm - : undefined - if (explicitTerm !== undefined) { - revivedEnv.TERM = explicitTerm - } - // Why: serialized state can come from an older or untrusted client, so - // revive reapplies the same bounds as a fresh spawn before retaining it. - const envToDelete = sanitizeEnvToDelete(entry.envToDelete) - const shell = resolveDefaultShell() - // Why: `command` is intentionally absent from this revive path because - // SerializedPtyEntry (see line 99) does not persist it — ManagedPty - // never stored the renderer-chosen launch command. The Pi/OMP extension - // installer in src/relay/relay.ts therefore sees `ctx.command === - // undefined` for revived PTYs and prepares the Pi default plus OMP's - // typed-command wrapper. Plumbing `command` through serialization is a - // separate, larger change. - const spawnEnv = this.buildSpawnEnv( - revivedEnv, - { - id: entry.id, - paneKey: entry.paneKey, - shell - }, - envToDelete - ) - const shellLaunch = getRelayShellLaunchConfig(shell, spawnEnv) - const term = ptyMod.spawn(shell, shellLaunch.args, { - // Why: revive must preserve the same terminal identity as fresh spawn. - name: spawnEnv.TERM ?? 'xterm-256color', - cols: entry.cols, - rows: entry.rows, - cwd: entry.cwd, - // Why: revived shells should not inherit an ambient shell-ready marker - // because no provider-delivered startup command is waiting on it. - env: { ...spawnEnv, ORCA_SHELL_READY_MARKER: '0', ...shellLaunch.env } - }) - this.wireAndStore({ - id: entry.id, - pty: term, - initialCwd: entry.cwd, - buffered: '', - paneKey: entry.paneKey, - tabId: entry.tabId, - attachIdentity: entry.attachIdentity, - worktreeId: entry.worktreeId, - ...(explicitTerm !== undefined ? { explicitTerm } : {}), - envToDelete, - ...(entry.terminalHandle ? { terminalHandle: entry.terminalHandle } : {}) - }) + } + } - // Why: nextId starts at 1 and is only incremented by spawn(). Revived - // PTYs carry their original IDs (e.g. "pty-3"), so without this bump the - // next spawn() would generate an ID that collides with an already-active - // revived PTY. - const match = entry.id.match(/^pty-(\d+)$/) - if (match) { - const revivedNum = Number.parseInt(match[1], 10) - if (revivedNum >= this.nextId) { - this.nextId = revivedNum + 1 - } - } + private async reviveEntry(entry: SerializedPtyEntry): Promise { + const ptyMod = await loadPty() + if (!ptyMod) { + return + } + // Why: revive must apply the same hook env as spawn(). The hook-server + // coords come from augmenters, while pane identity comes from the + // serialized PTY entry because managed hook scripts exit without + // ORCA_PANE_KEY. + const revivedEnv: Record = {} + if (entry.paneKey) { + revivedEnv.ORCA_PANE_KEY = entry.paneKey + } + if (entry.tabId) { + revivedEnv.ORCA_TAB_ID = entry.tabId + } + if (entry.worktreeId) { + revivedEnv.ORCA_WORKTREE_ID = entry.worktreeId + } + if (entry.terminalHandle) { + revivedEnv.ORCA_TERMINAL_HANDLE = entry.terminalHandle + } + const explicitTerm = + typeof entry.explicitTerm === 'string' && entry.explicitTerm.length > 0 + ? entry.explicitTerm + : undefined + if (explicitTerm !== undefined) { + revivedEnv.TERM = explicitTerm + } + const envToDelete = sanitizeEnvToDelete(entry.envToDelete) + const shell = resolveDefaultShell() + const spawnEnv = this.buildSpawnEnv( + revivedEnv, + { id: entry.id, paneKey: entry.paneKey, shell }, + envToDelete + ) + const shellLaunch = getRelayShellLaunchConfig(shell, spawnEnv) + const term = ptyMod.spawn(shell, shellLaunch.args, { + name: spawnEnv.TERM ?? 'xterm-256color', + cols: entry.cols, + rows: entry.rows, + cwd: entry.cwd, + // Why: no provider-delivered command is waiting for a ready marker. + env: { ...spawnEnv, ORCA_SHELL_READY_MARKER: '0', ...shellLaunch.env } + }) + this.wireAndStore({ + id: entry.id, + pty: term, + initialCwd: entry.cwd, + buffered: '', + paneKey: entry.paneKey, + tabId: entry.tabId, + attachIdentity: entry.attachIdentity, + worktreeId: entry.worktreeId, + ...(explicitTerm !== undefined ? { explicitTerm } : {}), + envToDelete, + ...(entry.terminalHandle ? { terminalHandle: entry.terminalHandle } : {}) + }) + + const match = entry.id.match(/^pty-(\d+)$/) + if (match) { + this.nextId = Math.max(this.nextId, Number.parseInt(match[1], 10) + 1) } } @@ -1129,8 +1273,28 @@ export class PtyHandler { } } - dispose(): void { + dispose(options: { waitForPhysicalExit?: boolean } = {}): Promise { + // Why: fence creation synchronously before the first await so a spawn or + // revive cannot appear after the disposal snapshot and escape process exit. + this.creationFenced = true + if (this.disposePromise) { + return this.disposePromise + } + const disposePromise = this.disposePtys(options.waitForPhysicalExit !== false) + this.disposePromise = disposePromise + void disposePromise.catch(() => { + // Why: a rejected native kill retains ownership so a later shutdown + // signal can retry instead of joining a permanently rejected promise. + if (this.disposePromise === disposePromise) { + this.disposePromise = null + } + }) + return disposePromise + } + + private async disposePtys(waitForPhysicalExit: boolean): Promise { this.cancelGraceTimer() + await this.waitForPendingPtyCreations() if (this.outputFlushTimer !== null) { clearTimeout(this.outputFlushTimer) this.outputFlushTimer = null @@ -1138,28 +1302,76 @@ export class PtyHandler { this.pendingOutputByPty.clear() this.lastInputAtByPty.clear() this.interactiveOutputCharsByPty.clear() - for (const [, managed] of this.ptys) { - if (managed.killTimer) { - clearTimeout(managed.killTimer) - managed.killTimer = undefined - } - this.clearStartupCommandTimer(managed) - // Why: SIGKILL (not SIGTERM) before destroy. The relay process is - // exiting; any SIGTERM-ignoring remote shell (editor with unsaved - // buffers, a hung child with a bad handler, a process in - // uninterruptible sleep) would survive SIGTERM + immediate destroy() - // as an orphan on the remote host. SIGKILL is not ignorable and the - // ptmx fd release via disposeManagedPty is synchronous, so there is - // no graceful-shutdown window to preserve at this point. + const results = await Promise.allSettled( + [...this.ptys.values()].map((managed) => + this.disposePtyForRelayShutdown(managed, waitForPhysicalExit) + ) + ) + const rejected = results.find( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ) + if (rejected) { + throw rejected.reason + } + } + + private async disposePtyForRelayShutdown( + managed: ManagedPty, + waitForPhysicalExit: boolean + ): Promise { + if (managed.killTimer) { + clearTimeout(managed.killTimer) + managed.killTimer = undefined + } + this.clearStartupCommandTimer(managed) + // Why: relay exit must retain the native owner until SIGKILL is accepted + // (with one bounded retry) or onExit proves the process is already gone. + await this.requestForceKillForRelayShutdown(managed) + if (waitForPhysicalExit && this.ptys.get(managed.id) === managed && !managed.disposed) { try { - killPtyProcess(managed.pty, 'SIGKILL') + await this.waitForPhysicalExit(managed, IMMEDIATE_PTY_EXIT_TIMEOUT_MS) } catch { - /* child may already be dead */ + // An accepted SIGKILL is the bounded final boundary when a child is + // uninterruptible and cannot report exit before relay shutdown. } + } + if (this.ptys.get(managed.id) === managed && !managed.disposed) { this.notifyExitListener(managed) disposeManagedPty(managed) + this.ptys.delete(managed.id) + this.clearPtyFlowState(managed.id) } - this.ptys.clear() + } + + private async requestForceKillForRelayShutdown(managed: ManagedPty): Promise { + let lastError: unknown + for (let attempt = 0; attempt < PTY_FORCE_KILL_MAX_ATTEMPTS; attempt++) { + if (this.ptys.get(managed.id) !== managed || managed.disposed) { + return + } + try { + this.requestForceKill(managed) + return + } catch (error) { + lastError = error + } + if (attempt + 1 < PTY_FORCE_KILL_MAX_ATTEMPTS) { + const tracker = managed.physicalExit + if (!tracker) { + throw lastError + } + try { + await tracker.waitForExit( + PTY_FORCE_KILL_RETRY_DELAY_MS, + () => new Error(`Retrying force-kill for PTY ${managed.id}`) + ) + return + } catch { + // The bounded waiter detached; retry the still-owned native handle. + } + } + } + throw lastError } get activePtyCount(): number { diff --git a/src/relay/relay-filesystem-watch-registry.test.ts b/src/relay/relay-filesystem-watch-registry.test.ts index 7c505ae652d..fd6d2ca37d1 100644 --- a/src/relay/relay-filesystem-watch-registry.test.ts +++ b/src/relay/relay-filesystem-watch-registry.test.ts @@ -13,12 +13,13 @@ import { RelayFilesystemWatchRegistry } from './relay-filesystem-watch-registry' import { createRelayWatcherProcessPool } from './relay-watcher-process-pool' type InstalledWatch = { + rootPath: string callback: WatcherProcessCallback hooks: WatcherProcessHooks unsubscribe: ReturnType Promise>> } -type InstalledSupervisorWatch = InstalledWatch & { dir: string } +type InstalledSupervisorWatch = Omit & { dir: string } function stubRelayWatcherSupervisors() { const installed = new Map() @@ -47,13 +48,13 @@ class FakeWatcherPool { readonly forgetRoot = vi.fn() async subscribe( - _rootPath: string, + rootPath: string, callback: WatcherProcessCallback, _options: object, hooks: WatcherProcessHooks ): Promise { const unsubscribe = vi.fn(async () => undefined) - this.installed.push({ callback, hooks, unsubscribe }) + this.installed.push({ rootPath, callback, hooks, unsubscribe }) return { unsubscribe } } } @@ -62,6 +63,7 @@ function createDispatcher() { const detached = new Set<(clientId: number) => void>() return { notify: vi.fn(), + notifyClient: vi.fn(), onClientDetached: vi.fn((listener: (clientId: number) => void) => { detached.add(listener) return () => detached.delete(listener) @@ -127,6 +129,33 @@ describe('RelayFilesystemWatchRegistry', () => { expect(pool.installed[1].unsubscribe).toHaveBeenCalledTimes(1) }) + it('notifies each owning client when bounded recovery terminates a live watch', async () => { + await registry.watch('/repo', context(1), 101) + await registry.watch('/repo', context(2), 202) + const first = pool.installed[0] + vi.spyOn(pool, 'subscribe').mockRejectedValueOnce(new Error('quarantine recovery failed')) + + first.hooks.onTerminalError?.( + new WatcherProcessFailure( + 'file watcher process crashed repeatedly', + 'supervisor', + 'supervisor_crash_fuse' + ) + ) + + await vi.waitFor(() => expect(dispatcher.notifyClient).toHaveBeenCalledTimes(2)) + expect(dispatcher.notifyClient).toHaveBeenNthCalledWith(1, 1, 'fs.watchFailed', { + rootPath: '/repo', + watchId: 101, + message: 'quarantine recovery failed' + }) + expect(dispatcher.notifyClient).toHaveBeenNthCalledWith(2, 2, 'fs.watchFailed', { + rootPath: '/repo', + watchId: 202, + message: 'quarantine recovery failed' + }) + }) + it('aborts a pending crawl only after the last same-root client leaves', async () => { let sharedSignal: AbortSignal | undefined let rejectSubscribe: ((error: Error) => void) | undefined @@ -169,6 +198,139 @@ describe('RelayFilesystemWatchRegistry', () => { expect(sharedSignal?.aborted).toBe(true) expect(rejectSubscribe).toBeDefined() }) + + it('rejects acknowledged teardown while another client still owns the watch', async () => { + await registry.watch('/repo', context(1)) + await registry.watch('/repo', context(2)) + const unsubscribe = pool.installed[0].unsubscribe + + await expect(registry.unwatchAndWait('/repo', context(1))).rejects.toThrow( + 'still watched by another client' + ) + expect(unsubscribe).not.toHaveBeenCalled() + + registry.unwatch('/repo', context(1)) + expect(unsubscribe).not.toHaveBeenCalled() + registry.unwatch('/repo', context(2)) + expect(unsubscribe).toHaveBeenCalledTimes(1) + }) + + it('notifies old clients and fences new ones while removal owns the root', async () => { + await registry.watch('/repo', context(1), 101) + let failRemoval: (error: Error) => void = () => undefined + const removal = registry.runWithRemovalFence( + '/repo', + () => + new Promise((_resolve, reject) => { + failRemoval = reject + }) + ) + await vi.waitFor(() => expect(pool.installed[0].unsubscribe).toHaveBeenCalledTimes(1)) + expect(dispatcher.notifyClient).toHaveBeenCalledWith(1, 'fs.watchFailed', { + rootPath: '/repo', + watchId: 101, + message: 'Remote worktree is being removed' + }) + + await expect(registry.watch('/repo', context(2))).rejects.toThrow( + 'deletion already in progress' + ) + failRemoval(new Error('Git removal failed')) + await expect(removal).rejects.toThrow('Git removal failed') + + await expect(registry.watch('/repo', context(2))).resolves.toBeUndefined() + }) + + it('waits admitted PTY creation, rejects late creation, then tears down before removal', async () => { + const finishPtyCreation = registry.beginWorktreePtySpawn('/repo/nested') + const order: string[] = [] + registry.setWorktreePtyTeardown(async (rootPath) => { + order.push(`pty:${rootPath}`) + }) + + const removal = registry.runWithRemovalFence('/repo', async () => { + order.push('remove') + }) + await Promise.resolve() + expect(order).toEqual([]) + expect(() => registry.beginWorktreePtySpawn('/repo/late')).toThrow( + 'deletion already in progress' + ) + + finishPtyCreation() + await removal + expect(order).toEqual(['pty:/repo', 'remove']) + }) + + it.each([ + ['drive spelling', 'C:\\Repo', 'c:/repo/'], + ['UNC spelling', '\\\\Server\\Share\\Repo', '//server/share/repo/'] + ])('shares one Windows root across equivalent %s', async (_label, firstPath, secondPath) => { + await registry.watch(firstPath, context(1)) + await registry.watch(secondPath, context(2)) + + expect(pool.installed).toHaveLength(1) + expect(pool.installed[0].rootPath).toBe(firstPath) + await expect(registry.unwatchAndWait(secondPath, context(1))).rejects.toThrow( + 'still watched by another client' + ) + + registry.unwatch(secondPath, context(1)) + await registry.unwatchAndWait(firstPath, context(2)) + + expect(pool.installed[0].unsubscribe).toHaveBeenCalledTimes(1) + expect(pool.forgetRoot).toHaveBeenCalledWith(firstPath) + }) + + it('keeps literal POSIX backslash roots physically distinct', async () => { + await registry.watch('/srv/team\\repo', context(1)) + await registry.watch('/srv/team/repo', context(2)) + + expect(pool.installed.map(({ rootPath }) => rootPath)).toEqual([ + '/srv/team\\repo', + '/srv/team/repo' + ]) + + await registry.unwatchAndWait('/srv/team\\repo', context(1)) + await registry.unwatchAndWait('/srv/team/repo', context(2)) + }) + + it('retains pending-setup teardown failure until the child physically exits', async () => { + let resolveSubscribe: (subscription: WatcherProcessSubscription) => void = () => undefined + let resolvePhysicalExit: () => void = () => undefined + const physicalExit = new Promise((resolve) => { + resolvePhysicalExit = resolve + }) + const terminationError = new WatcherProcessFailure( + 'file watcher process did not exit after termination deadline', + 'supervisor', + 'process_unavailable', + physicalExit + ) + const unsubscribe = vi.fn().mockRejectedValue(terminationError) + vi.spyOn(pool, 'subscribe').mockImplementation( + () => + new Promise((resolve) => { + resolveSubscribe = resolve + }) + ) + + const watch = registry.watch('/repo', context(1)) + const close = registry.unwatchAndWait('/repo', context(1)) + resolveSubscribe({ unsubscribe }) + + await expect(watch).rejects.toBe(terminationError) + await expect(close).rejects.toBe(terminationError) + await expect(registry.unwatchAndWait('/repo', context(1))).rejects.toBe(terminationError) + expect(pool.forgetRoot).not.toHaveBeenCalled() + + resolvePhysicalExit() + await physicalExit + await Promise.resolve() + + await expect(registry.unwatchAndWait('/repo', context(1))).resolves.toBeUndefined() + expect(pool.forgetRoot).toHaveBeenCalledWith('/repo') + }) }) describe('createRelayWatcherProcessPool', () => { diff --git a/src/relay/relay-filesystem-watch-registry.ts b/src/relay/relay-filesystem-watch-registry.ts index a1ccda05826..2e9c3809916 100644 --- a/src/relay/relay-filesystem-watch-registry.ts +++ b/src/relay/relay-filesystem-watch-registry.ts @@ -1,10 +1,6 @@ import type { RelayDispatcher, RequestContext } from './dispatcher' import { MAX_BATCHED_WATCHER_EVENTS } from '../main/ipc/filesystem-watcher-event-batch' import { isWatcherProcessFailure } from '../main/ipc/parcel-watcher-process-failure' -import type { - WatcherProcessEvent, - WatcherProcessSubscription -} from '../main/ipc/parcel-watcher-process' import { WATCHER_IGNORE_DIRS, buildParcelWatcherIgnoreOptions @@ -13,118 +9,204 @@ import { createRelayWatcherProcessPool, type RelayWatcherProcessPool } from './relay-watcher-process-pool' +import { emitRelayWatcherEvents, emitRelayWatcherOverflow } from './relay-watcher-event-emitter' +import { awaitRelayWatcherSetup, shouldRetryInitialRelayWatch } from './relay-watcher-setup-wait' +import { + RelayWatcherTeardownTracker, + type RelayWatcherTeardownState +} from './relay-watcher-teardown-tracker' +import { emitRelayWatcherTerminalFailure } from './relay-watcher-terminal-notifier' +import { assertRelayWatcherRootCapacity } from './relay-watcher-root-capacity' +import { normalizeRuntimePathForComparison } from '../shared/cross-platform-path' +import { + trackRelayWatcherSetup, + type RelayWatcherPendingSetup +} from './relay-watcher-setup-tracking' +import { RelayWatcherRemovalFence } from './relay-watcher-removal-fence' +import { releaseStaleRelayWatches } from './relay-watcher-stale-client-release' +import { PromiseSettlementWaiters } from '../shared/promise-settlement-waiters' +import { joinRelayWatcherPendingSetup } from './relay-watcher-pending-setup-join' +import { createRelayWatcherState } from './relay-watcher-state' -const MAX_RELAY_WATCH_ROOTS = 20 -const RELAY_WATCH_CRAWL_TIMEOUT_MS = 60_000 const RELAY_WATCH_OPTIONS = buildParcelWatcherIgnoreOptions(WATCHER_IGNORE_DIRS) -type RelayWatchState = { - rootPath: string - clients: Map boolean> - setupPromise: Promise - subscription: WatcherProcessSubscription | null - abortController: AbortController - generation: number - closed: boolean -} - -function overflowEvent(rootPath: string): Record { - return { events: [{ kind: 'overflow', absolutePath: rootPath }] } -} - -function createWatchAbortError(): Error { - const error = new Error('Request "fs.watch" was cancelled') - error.name = 'AbortError' - return error -} - -function shouldRetryInitialWatch(error: unknown): boolean { - return ( - isWatcherProcessFailure(error) && - error.code !== 'entry_missing' && - error.code !== 'subscribe_aborted' && - error.code !== 'supervisor_disposed' && - (error.scope === 'supervisor' || error.code === 'subscribe_timeout') - ) -} - export class RelayFilesystemWatchRegistry { - private readonly watches = new Map() + private readonly watches = new Map() + private readonly pendingSetups = new Map() + private readonly teardownTracker: RelayWatcherTeardownTracker + private readonly removalFence: RelayWatcherRemovalFence constructor( private readonly dispatcher: RelayDispatcher, private readonly watcherPool: RelayWatcherProcessPool = createRelayWatcherProcessPool() ) { + this.teardownTracker = new RelayWatcherTeardownTracker((rootPath) => + this.watcherPool.forgetRoot(rootPath) + ) + this.removalFence = new RelayWatcherRemovalFence( + this.watches, + this.pendingSetups, + this.teardownTracker, + this.dispatcher + ) this.dispatcher.onClientDetached?.((clientId) => this.releaseClientWatches(clientId)) } - async watch(rootPath: string, context?: RequestContext): Promise { - this.releaseStaleWatches() + watch(rootPath: string, context?: RequestContext, watchId?: number): Promise { + const rootKey = normalizeRuntimePathForComparison(rootPath) + if (this.removalFence.isActive(rootKey)) { + return Promise.reject(new Error('Remote worktree deletion already in progress')) + } + const existing = this.watches.get(rootKey) + if (existing) { + const hasLiveClient = [...existing.clients.values()].some((isStale) => !isStale()) + if (hasLiveClient) { + const clientId = context?.clientId ?? 0 + existing.clients.set(clientId, context?.isStale ?? (() => false)) + if (watchId !== undefined) { + existing.clientWatchIds.set(clientId, watchId) + } + return this.awaitSetupForClient(existing, clientId, context) + } + void this.closeWatch(existing).catch(() => {}) + } + const pendingSetup = this.pendingSetups.get(rootKey) + if (pendingSetup) { + const retry = (): Promise => this.watch(rootPath, context, watchId) + return joinRelayWatcherPendingSetup(pendingSetup, context?.signal, retry) + } + const setup = this.installWatch(rootKey, rootPath, context, watchId) + return trackRelayWatcherSetup(this.pendingSetups, rootKey, setup) + } + + private async installWatch( + rootKey: string, + rootPath: string, + context?: RequestContext, + watchId?: number + ): Promise { + const staleTeardown = releaseStaleRelayWatches(this.watches.values(), (state) => + this.closeWatch(state) + ) + if (staleTeardown) { + await staleTeardown + } + const rootTeardown = this.teardownTracker.join(rootKey) + if (rootTeardown) { + await rootTeardown + } const clientId = context?.clientId ?? 0 const isStale = context?.isStale ?? (() => false) - const existing = this.watches.get(rootPath) + const existing = this.watches.get(rootKey) if (existing) { existing.clients.set(clientId, isStale) + if (watchId !== undefined) { + existing.clientWatchIds.set(clientId, watchId) + } await this.awaitSetupForClient(existing, clientId, context) return } - if (this.watches.size >= MAX_RELAY_WATCH_ROOTS) { - throw new Error('Maximum number of file watchers reached') - } + assertRelayWatcherRootCapacity( + this.watches.keys(), + this.pendingSetups.keys(), + this.teardownTracker.rootPaths(), + rootKey + ) - const state: RelayWatchState = { - rootPath, - clients: new Map([[clientId, isStale]]), - setupPromise: Promise.resolve(), - subscription: null, - abortController: new AbortController(), - generation: 0, - closed: false - } - this.watches.set(rootPath, state) - state.setupPromise = this.startInitialWatch(state) + const state = createRelayWatcherState(rootKey, rootPath, clientId, isStale, watchId) + this.watches.set(rootKey, state) + state.setupWaiters = new PromiseSettlementWaiters(this.startInitialWatch(state)) await this.awaitSetupForClient(state, clientId, context) } unwatch(rootPath: string, context?: RequestContext): void { - const state = this.watches.get(rootPath) + const rootKey = normalizeRuntimePathForComparison(rootPath) + const state = this.watches.get(rootKey) if (state) { this.releaseWatchClient(state, context?.clientId ?? 0) + return } + const setup = this.pendingSetups.get(rootKey) + if (setup) { + // Why: notification teardown must not disappear before async setup publishes ownership. + void setup.promise.then(() => this.unwatch(rootPath, context)).catch(() => {}) + } + } + + async unwatchAndWait(rootPath: string, context?: RequestContext): Promise { + const rootKey = normalizeRuntimePathForComparison(rootPath) + let state = this.watches.get(rootKey) + if (!state) { + await this.pendingSetups.get(rootKey)?.promise.catch(() => undefined) + state = this.watches.get(rootKey) + } + if (!state) { + const failed = this.teardownTracker.failedState(rootKey) + if (failed) { + await this.closeWatch(failed) + return + } + await this.teardownTracker.join(rootKey) + return + } + const clientId = context?.clientId ?? 0 + for (const [registeredClientId, isStale] of state.clients) { + if (registeredClientId !== clientId && isStale()) { + state.clients.delete(registeredClientId) + state.clientWatchIds.delete(registeredClientId) + } + } + if ([...state.clients.keys()].some((registeredClientId) => registeredClientId !== clientId)) { + // Why: destructive cleanup cannot acknowledge while another client owns the handle. + throw new Error('Remote path is still watched by another client') + } + state.clients.delete(clientId) + state.clientWatchIds.delete(clientId) + await this.closeWatch(state) + } + + async runWithRemovalFence(rootPath: string, operation: () => Promise): Promise { + return this.removalFence.run(normalizeRuntimePathForComparison(rootPath), operation) + } + + beginWorktreePtySpawn(path: string): () => void { + return this.removalFence.beginOperation(normalizeRuntimePathForComparison(path)) + } + + setWorktreePtyTeardown(teardown: (rootPath: string) => Promise): void { + this.removalFence.setBeforeRemove(teardown) } dispose(): void { - for (const state of Array.from(this.watches.values())) { - this.closeWatch(state) - } + this.watches.forEach((state) => void this.closeWatch(state).catch(() => {})) this.watcherPool.dispose() } - private async startInitialWatch(state: RelayWatchState): Promise { + private async startInitialWatch(state: RelayWatcherTeardownState): Promise { try { await this.subscribeState(state) } catch (firstError) { - if (!state.closed && shouldRetryInitialWatch(firstError)) { + if (!state.closed && shouldRetryInitialRelayWatch(firstError)) { try { await this.subscribeState(state) - this.emitOverflow(state) + emitRelayWatcherOverflow(this.dispatcher, state.rootPath, state.closed) return } catch (quarantineError) { - this.closeWatch(state) + void this.closeWatch(state).catch(() => {}) throw quarantineError } } - this.closeWatch(state) + void this.closeWatch(state).catch(() => {}) throw firstError } } - private subscribeState(state: RelayWatchState): Promise { + private subscribeState(state: RelayWatcherTeardownState): Promise { const generation = ++state.generation const emitOverflow = (): void => { if (state.generation === generation) { - this.emitOverflow(state) + emitRelayWatcherOverflow(this.dispatcher, state.rootPath, state.closed) } } return this.watcherPool @@ -141,7 +223,7 @@ export class RelayFilesystemWatchRegistry { emitOverflow() return } - this.emitEvents(state, events) + emitRelayWatcherEvents(this.dispatcher, state.closed, events) }, RELAY_WATCH_OPTIONS, { @@ -150,14 +232,14 @@ export class RelayFilesystemWatchRegistry { onOverflow: emitOverflow, onTerminalError: (error) => this.recoverWatch(state, generation, error), signal: state.abortController.signal, - subscribeTimeoutMs: RELAY_WATCH_CRAWL_TIMEOUT_MS + subscribeTimeoutMs: 60_000 } ) .then(async (subscription) => { if ( state.closed || state.generation !== generation || - this.watches.get(state.rootPath) !== state + this.watches.get(state.rootKey) !== state ) { await subscription.unsubscribe() return @@ -166,51 +248,37 @@ export class RelayFilesystemWatchRegistry { }) } - private recoverWatch(state: RelayWatchState, failedGeneration: number, error: Error): void { + private recoverWatch( + state: RelayWatcherTeardownState, + failedGeneration: number, + error: Error + ): void { if (state.closed || state.generation !== failedGeneration) { return } state.subscription = null - this.emitOverflow(state) + emitRelayWatcherOverflow(this.dispatcher, state.rootPath, state.closed) const recovery = this.subscribeState(state) - state.setupPromise = recovery + state.setupWaiters = new PromiseSettlementWaiters(recovery) void recovery.catch((recoveryError: unknown) => { if (!state.closed) { const message = recoveryError instanceof Error ? recoveryError.message : error.message process.stderr.write( `[relay] File watcher disabled after bounded recovery for ${state.rootPath}: ${message}\n` ) - this.closeWatch(state) + emitRelayWatcherTerminalFailure(this.dispatcher, state, message) + void this.closeWatch(state).catch(() => {}) } }) } - private emitEvents(state: RelayWatchState, events: readonly WatcherProcessEvent[]): void { - if (state.closed || events.length === 0) { - return - } - this.dispatcher.notify('fs.changed', { - events: events.map((event) => ({ - kind: event.type, - absolutePath: event.path, - ...(event.isDirectory === undefined ? {} : { isDirectory: event.isDirectory }) - })) - }) - } - - private emitOverflow(state: RelayWatchState): void { - if (!state.closed) { - this.dispatcher.notify('fs.changed', overflowEvent(state.rootPath)) - } - } - private async awaitSetupForClient( - state: RelayWatchState, + state: RelayWatcherTeardownState, clientId: number, context?: RequestContext ): Promise { try { - await this.awaitSetupWithAbort(state.setupPromise, context?.signal) + await awaitRelayWatcherSetup(state.setupWaiters, context?.signal) } catch (error) { this.releaseWatchClient(state, clientId) const expectedAbort = @@ -229,84 +297,23 @@ export class RelayFilesystemWatchRegistry { } } - private awaitSetupWithAbort(setupPromise: Promise, signal?: AbortSignal): Promise { - if (!signal) { - return setupPromise + private releaseClientWatches(clientId: number): void { + this.watches.forEach((state) => this.releaseWatchClient(state, clientId)) + } + + private releaseWatchClient(state: RelayWatcherTeardownState, clientId: number): void { + state.clients.delete(clientId) + state.clientWatchIds.delete(clientId) + if (!state.closed && state.clients.size === 0) { + void this.closeWatch(state).catch(() => {}) } - if (signal.aborted) { - return Promise.reject(createWatchAbortError()) - } - return new Promise((resolve, reject) => { - const onAbort = (): void => { - cleanup() - reject(createWatchAbortError()) + } + + private closeWatch(state: RelayWatcherTeardownState): Promise { + return this.teardownTracker.close(state, () => { + if (this.watches.get(state.rootKey) === state) { + this.watches.delete(state.rootKey) } - const cleanup = (): void => signal.removeEventListener('abort', onAbort) - signal.addEventListener('abort', onAbort, { once: true }) - setupPromise.then( - () => { - cleanup() - resolve() - }, - (error) => { - cleanup() - reject(error) - } - ) }) } - - private releaseClientWatches(clientId: number): void { - for (const state of Array.from(this.watches.values())) { - this.releaseWatchClient(state, clientId) - } - } - - private releaseStaleWatches(): void { - for (const state of Array.from(this.watches.values())) { - for (const [clientId, isStale] of state.clients) { - if (isStale()) { - state.clients.delete(clientId) - } - } - if (state.clients.size === 0) { - this.closeWatch(state) - } - } - } - - private releaseWatchClient(state: RelayWatchState, clientId: number): void { - state.clients.delete(clientId) - if (!state.closed && state.clients.size === 0) { - this.closeWatch(state) - } - } - - private closeWatch(state: RelayWatchState): void { - if (state.closed) { - return - } - state.closed = true - state.generation++ - state.abortController.abort() - state.clients.clear() - if (this.watches.get(state.rootPath) === state) { - this.watches.delete(state.rootPath) - } - const subscription = state.subscription - state.subscription = null - if (subscription) { - // Why: a child can die during unwatch; release quarantine history even - // when physical teardown reports that already-contained failure. - void subscription.unsubscribe().then( - () => this.watcherPool.forgetRoot(state.rootPath), - () => this.watcherPool.forgetRoot(state.rootPath) - ) - return - } - void state.setupPromise.then( - () => this.watcherPool.forgetRoot(state.rootPath), - () => this.watcherPool.forgetRoot(state.rootPath) - ) - } } diff --git a/src/relay/relay-watcher-event-emitter.ts b/src/relay/relay-watcher-event-emitter.ts new file mode 100644 index 00000000000..46515e67959 --- /dev/null +++ b/src/relay/relay-watcher-event-emitter.ts @@ -0,0 +1,31 @@ +import type { WatcherProcessEvent } from '../main/ipc/parcel-watcher-process' +import type { RelayDispatcher } from './dispatcher' + +export function emitRelayWatcherEvents( + dispatcher: RelayDispatcher, + closed: boolean, + events: readonly WatcherProcessEvent[] +): void { + if (closed || events.length === 0) { + return + } + dispatcher.notify('fs.changed', { + events: events.map((event) => ({ + kind: event.type, + absolutePath: event.path, + ...(event.isDirectory === undefined ? {} : { isDirectory: event.isDirectory }) + })) + }) +} + +export function emitRelayWatcherOverflow( + dispatcher: RelayDispatcher, + rootPath: string, + closed: boolean +): void { + if (!closed) { + dispatcher.notify('fs.changed', { + events: [{ kind: 'overflow', absolutePath: rootPath }] + }) + } +} diff --git a/src/relay/relay-watcher-parent-removal.test.ts b/src/relay/relay-watcher-parent-removal.test.ts new file mode 100644 index 00000000000..892759204e0 --- /dev/null +++ b/src/relay/relay-watcher-parent-removal.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it, vi } from 'vitest' +import type { + WatcherProcessHooks, + WatcherProcessSubscription +} from '../main/ipc/parcel-watcher-process-subscription' +import type { RelayDispatcher, RequestContext } from './dispatcher' +import { RelayFilesystemWatchRegistry } from './relay-filesystem-watch-registry' + +class ParentRemovalPool { + readonly subscriptions: { rootPath: string; unsubscribe: ReturnType }[] = [] + readonly dispose = vi.fn() + readonly forgetRoot = vi.fn() + + async subscribe( + rootPath: string, + _callback: unknown, + _options: unknown, + _hooks: WatcherProcessHooks + ): Promise { + const unsubscribe = vi.fn(async () => undefined) + this.subscriptions.push({ rootPath, unsubscribe }) + return { unsubscribe } + } +} + +function context(clientId: number): RequestContext { + return { clientId, isStale: () => false } +} + +describe('RelayFilesystemWatchRegistry parent removal', () => { + it.each([ + ['/repo', '/repo/nested', '/repo-sibling'], + ['C:\\Repo', 'c:\\repo\\nested', 'C:\\Repo-Sibling'] + ])( + 'closes descendant watches before deleting parent %s and preserves siblings', + async (parent, descendant, sibling) => { + const pool = new ParentRemovalPool() + const dispatcher = { notify: vi.fn(), notifyClient: vi.fn(), onClientDetached: vi.fn() } + const registry = new RelayFilesystemWatchRegistry( + dispatcher as unknown as RelayDispatcher, + pool + ) + await registry.watch(descendant, context(1), 101) + await registry.watch(sibling, context(2), 202) + const remove = vi.fn(async () => { + expect(pool.subscriptions[0].unsubscribe).toHaveBeenCalledOnce() + expect(pool.subscriptions[1].unsubscribe).not.toHaveBeenCalled() + }) + + await registry.runWithRemovalFence(parent, remove) + + expect(remove).toHaveBeenCalledOnce() + expect(dispatcher.notifyClient).toHaveBeenCalledWith(1, 'fs.watchFailed', { + rootPath: descendant, + watchId: 101, + message: 'Remote worktree is being removed' + }) + } + ) + + it('waits for a descendant setup and closes its published subscription before removal', async () => { + const pool = new ParentRemovalPool() + const dispatcher = { notify: vi.fn(), notifyClient: vi.fn(), onClientDetached: vi.fn() } + const registry = new RelayFilesystemWatchRegistry( + dispatcher as unknown as RelayDispatcher, + pool + ) + let releaseSetup: () => void = () => undefined + const setupGate = new Promise((resolve) => { + releaseSetup = resolve + }) + const subscribe = pool.subscribe.bind(pool) + vi.spyOn(pool, 'subscribe').mockImplementation(async (...args) => { + await setupGate + return subscribe(...args) + }) + + const watch = registry.watch('/repo/nested', context(1), 101) + const remove = vi.fn(async () => { + expect(pool.subscriptions[0].unsubscribe).toHaveBeenCalledOnce() + }) + const removal = registry.runWithRemovalFence('/repo', remove) + await Promise.resolve() + expect(remove).not.toHaveBeenCalled() + + releaseSetup() + await Promise.all([watch, removal]) + expect(remove).toHaveBeenCalledOnce() + }) + + it('waits for an already-retiring descendant before removal', async () => { + const pool = new ParentRemovalPool() + const dispatcher = { notify: vi.fn(), notifyClient: vi.fn(), onClientDetached: vi.fn() } + const registry = new RelayFilesystemWatchRegistry( + dispatcher as unknown as RelayDispatcher, + pool + ) + await registry.watch('/repo/nested', context(1), 101) + let releaseTeardown: () => void = () => undefined + pool.subscriptions[0].unsubscribe.mockImplementation( + () => + new Promise((resolve) => { + releaseTeardown = resolve + }) + ) + const teardown = registry.unwatchAndWait('/repo/nested', context(1)) + const remove = vi.fn(async () => undefined) + const removal = registry.runWithRemovalFence('/repo', remove) + await Promise.resolve() + expect(remove).not.toHaveBeenCalled() + + releaseTeardown() + await Promise.all([teardown, removal]) + expect(remove).toHaveBeenCalledOnce() + }) +}) diff --git a/src/relay/relay-watcher-pending-setup-join.ts b/src/relay/relay-watcher-pending-setup-join.ts new file mode 100644 index 00000000000..17c7cb5427a --- /dev/null +++ b/src/relay/relay-watcher-pending-setup-join.ts @@ -0,0 +1,21 @@ +import type { RelayWatcherPendingSetup } from './relay-watcher-setup-tracking' +import { awaitRelayWatcherSetup } from './relay-watcher-setup-wait' + +export async function joinRelayWatcherPendingSetup( + pendingSetup: RelayWatcherPendingSetup, + signal: AbortSignal | undefined, + retry: () => Promise +): Promise { + try { + await awaitRelayWatcherSetup(pendingSetup, signal) + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + return + } + // Why: the previous caller's failure must not poison a later same-root + // attempt once its single serialized setup generation has released. + } + if (!signal?.aborted) { + await retry() + } +} diff --git a/src/relay/relay-watcher-pending-setup-waiters.test.ts b/src/relay/relay-watcher-pending-setup-waiters.test.ts new file mode 100644 index 00000000000..f3e6f8f897d --- /dev/null +++ b/src/relay/relay-watcher-pending-setup-waiters.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it, vi } from 'vitest' +import type { WatcherProcessSubscription } from '../main/ipc/parcel-watcher-process-subscription' +import type { PromiseSettlementWaiters } from '../shared/promise-settlement-waiters' +import type { RelayDispatcher, RequestContext } from './dispatcher' +import { RelayFilesystemWatchRegistry } from './relay-filesystem-watch-registry' + +class PendingSetupPool { + readonly installed: { unsubscribe: () => Promise }[] = [] + readonly dispose = vi.fn() + readonly forgetRoot = vi.fn() + + async subscribe(): Promise { + const subscription = { unsubscribe: vi.fn(async () => undefined) } + this.installed.push(subscription) + return subscription + } +} + +function context(clientId: number, signal?: AbortSignal): RequestContext { + return { clientId, isStale: () => false, signal } +} + +describe('RelayFilesystemWatchRegistry pending setup waiters', () => { + it('removes ten thousand aborted callers queued behind one teardown anchor', async () => { + const pool = new PendingSetupPool() + const dispatcher = { notify: vi.fn(), onClientDetached: vi.fn() } + const registry = new RelayFilesystemWatchRegistry( + dispatcher as unknown as RelayDispatcher, + pool + ) + await registry.watch('/repo', context(1)) + let releaseTeardown: () => void = () => undefined + pool.installed[0].unsubscribe = vi.fn( + () => + new Promise((resolve) => { + releaseTeardown = resolve + }) + ) + const teardown = registry.unwatchAndWait('/repo', context(1)) + const leader = registry.watch('/repo', context(2)) + const anchorController = new AbortController() + const anchor = registry.watch('/repo', context(3, anchorController.signal)) + const controllers = Array.from({ length: 10_000 }, () => new AbortController()) + const cancelled = controllers.map((controller, index) => + registry.watch('/repo', context(index + 4, controller.signal)) + ) + + for (const controller of controllers) { + controller.abort() + } + await Promise.all(cancelled) + + const pendingSetups = ( + registry as unknown as { pendingSetups: Map> } + ).pendingSetups + expect(pendingSetups.values().next().value?.waiterCount).toBe(1) + expect(pool.installed).toHaveLength(1) + releaseTeardown() + await Promise.all([teardown, leader, anchor]) + expect(pool.installed).toHaveLength(2) + expect(pendingSetups.size).toBe(0) + }) +}) diff --git a/src/relay/relay-watcher-removal-fence.ts b/src/relay/relay-watcher-removal-fence.ts new file mode 100644 index 00000000000..9b0b62ab780 --- /dev/null +++ b/src/relay/relay-watcher-removal-fence.ts @@ -0,0 +1,131 @@ +import type { + RelayWatcherTeardownState, + RelayWatcherTeardownTracker +} from './relay-watcher-teardown-tracker' +import type { RelayDispatcher } from './dispatcher' +import { emitRelayWatcherTerminalFailure } from './relay-watcher-terminal-notifier' +import { isPathInsideOrEqual } from '../shared/cross-platform-path' +import type { RelayWatcherPendingSetup } from './relay-watcher-setup-tracking' + +export class RelayWatcherRemovalFence { + private readonly roots = new Set() + private readonly operations = new Map() + private readonly operationWaiters = new Map void>>() + private beforeRemove: ((rootKey: string) => Promise) | null = null + + constructor( + private readonly watches: Map, + private readonly pendingSetups: Map, + private readonly teardownTracker: RelayWatcherTeardownTracker, + private readonly dispatcher: RelayDispatcher + ) {} + + isActive(rootKey: string): boolean { + return [...this.roots].some((activeRoot) => isPathInsideOrEqual(activeRoot, rootKey)) + } + + setBeforeRemove(beforeRemove: (rootKey: string) => Promise): void { + this.beforeRemove = beforeRemove + } + + beginOperation(operationPath: string): () => void { + if ([...this.roots].some((rootKey) => isPathInsideOrEqual(rootKey, operationPath))) { + throw new Error('Remote worktree deletion already in progress') + } + this.operations.set(operationPath, (this.operations.get(operationPath) ?? 0) + 1) + let finished = false + return () => { + if (finished) { + return + } + finished = true + const remaining = (this.operations.get(operationPath) ?? 1) - 1 + if (remaining > 0) { + this.operations.set(operationPath, remaining) + } else { + this.operations.delete(operationPath) + } + this.resolveOperationWaiters() + } + } + + async run(rootKey: string, operation: () => Promise): Promise { + if ( + [...this.roots].some( + (activeRoot) => + isPathInsideOrEqual(activeRoot, rootKey) || isPathInsideOrEqual(rootKey, activeRoot) + ) + ) { + throw new Error('Remote worktree deletion already in progress') + } + this.roots.add(rootKey) + try { + await this.waitForOperations(rootKey) + await this.closeRoot(rootKey) + await this.beforeRemove?.(rootKey) + return await operation() + } finally { + this.roots.delete(rootKey) + } + } + + private hasOperationInside(rootKey: string): boolean { + return [...this.operations.keys()].some((operationPath) => + isPathInsideOrEqual(rootKey, operationPath) + ) + } + + private waitForOperations(rootKey: string): Promise { + if (!this.hasOperationInside(rootKey)) { + return Promise.resolve() + } + return new Promise((resolve) => { + const waiters = this.operationWaiters.get(rootKey) ?? new Set<() => void>() + waiters.add(resolve) + this.operationWaiters.set(rootKey, waiters) + }) + } + + private resolveOperationWaiters(): void { + for (const [rootKey, waiters] of this.operationWaiters) { + if (this.hasOperationInside(rootKey)) { + continue + } + this.operationWaiters.delete(rootKey) + for (const resolve of waiters) { + resolve() + } + } + } + + private async closeRoot(rootKey: string): Promise { + const pending = [...this.pendingSetups.entries()].filter(([setupRoot]) => + isPathInsideOrEqual(rootKey, setupRoot) + ) + await Promise.all(pending.map(([, setup]) => setup.promise.catch(() => undefined))) + const states = [...this.watches.entries()] + .filter(([watchRoot]) => isPathInsideOrEqual(rootKey, watchRoot)) + .map(([, state]) => state) + for (const state of states) { + emitRelayWatcherTerminalFailure(this.dispatcher, state, 'Remote worktree is being removed') + state.clients.clear() + state.clientWatchIds.clear() + } + await Promise.all(states.map((state) => this.closeWatch(state))) + const trackedRoots = this.teardownTracker + .rootPaths() + .filter((trackedRoot) => isPathInsideOrEqual(rootKey, trackedRoot)) + for (const trackedRoot of trackedRoots) { + const failed = this.teardownTracker.failedState(trackedRoot) + await (failed ? this.closeWatch(failed) : this.teardownTracker.join(trackedRoot)) + } + } + + private closeWatch(state: RelayWatcherTeardownState): Promise { + return this.teardownTracker.close(state, () => { + if (this.watches.get(state.rootKey) === state) { + this.watches.delete(state.rootKey) + } + }) + } +} diff --git a/src/relay/relay-watcher-root-capacity.ts b/src/relay/relay-watcher-root-capacity.ts new file mode 100644 index 00000000000..025462e7bd3 --- /dev/null +++ b/src/relay/relay-watcher-root-capacity.ts @@ -0,0 +1,14 @@ +const MAX_RELAY_WATCH_ROOTS = 20 + +export function assertRelayWatcherRootCapacity( + activeRoots: Iterable, + pendingRoots: Iterable, + teardownRoots: Iterable, + prospectiveRoot: string +): void { + const physicalRoots = new Set([...activeRoots, ...pendingRoots, ...teardownRoots]) + physicalRoots.add(prospectiveRoot) + if (physicalRoots.size > MAX_RELAY_WATCH_ROOTS) { + throw new Error('Maximum number of file watchers reached') + } +} diff --git a/src/relay/relay-watcher-setup-tracking.ts b/src/relay/relay-watcher-setup-tracking.ts new file mode 100644 index 00000000000..edf3d3cb976 --- /dev/null +++ b/src/relay/relay-watcher-setup-tracking.ts @@ -0,0 +1,19 @@ +import { PromiseSettlementWaiters } from '../shared/promise-settlement-waiters' + +export type RelayWatcherPendingSetup = PromiseSettlementWaiters + +export function trackRelayWatcherSetup( + pendingSetups: Map, + rootKey: string, + setup: Promise +): Promise { + let tracked!: RelayWatcherPendingSetup + const trackedPromise = setup.finally(() => { + if (pendingSetups.get(rootKey) === tracked) { + pendingSetups.delete(rootKey) + } + }) + tracked = new PromiseSettlementWaiters(trackedPromise) + pendingSetups.set(rootKey, tracked) + return tracked.promise +} diff --git a/src/relay/relay-watcher-setup-wait.test.ts b/src/relay/relay-watcher-setup-wait.test.ts new file mode 100644 index 00000000000..69f31a4e59c --- /dev/null +++ b/src/relay/relay-watcher-setup-wait.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it, vi } from 'vitest' +import { PromiseSettlementWaiters } from '../shared/promise-settlement-waiters' +import { awaitRelayWatcherSetup } from './relay-watcher-setup-wait' + +describe('awaitRelayWatcherSetup', () => { + it('removes ten thousand cancelled relay callers while one anchor remains', async () => { + let resolveSetup: () => void = () => undefined + const setupPromise = new Promise((resolve) => { + resolveSetup = resolve + }) + const thenSpy = vi.spyOn(setupPromise, 'then') + const setupWaiters = new PromiseSettlementWaiters(setupPromise) + const anchor = awaitRelayWatcherSetup(setupWaiters) + const controllers = Array.from({ length: 10_000 }, () => new AbortController()) + const cancelled = controllers.map((controller) => + awaitRelayWatcherSetup(setupWaiters, controller.signal).catch((error) => error) + ) + + for (const controller of controllers) { + controller.abort() + } + await Promise.all(cancelled) + + expect(setupWaiters.waiterCount).toBe(1) + expect(thenSpy).toHaveBeenCalledOnce() + resolveSetup() + await expect(anchor).resolves.toBeUndefined() + expect(setupWaiters.waiterCount).toBe(0) + }) +}) diff --git a/src/relay/relay-watcher-setup-wait.ts b/src/relay/relay-watcher-setup-wait.ts new file mode 100644 index 00000000000..a4c7b348b02 --- /dev/null +++ b/src/relay/relay-watcher-setup-wait.ts @@ -0,0 +1,28 @@ +import { isWatcherProcessFailure } from '../main/ipc/parcel-watcher-process-failure' +import type { PromiseSettlementWaiters } from '../shared/promise-settlement-waiters' + +export function shouldRetryInitialRelayWatch(error: unknown): boolean { + return ( + isWatcherProcessFailure(error) && + error.code !== 'entry_missing' && + error.code !== 'subscribe_aborted' && + error.code !== 'supervisor_disposed' && + (error.scope === 'supervisor' || error.code === 'subscribe_timeout') + ) +} + +export function awaitRelayWatcherSetup( + setupWaiters: PromiseSettlementWaiters, + signal?: AbortSignal +): Promise { + return setupWaiters.wait({ + signal, + createAbortError: createRelayWatchAbortError + }) +} + +function createRelayWatchAbortError(): Error { + const error = new Error('Request "fs.watch" was cancelled') + error.name = 'AbortError' + return error +} diff --git a/src/relay/relay-watcher-stale-client-release.ts b/src/relay/relay-watcher-stale-client-release.ts new file mode 100644 index 00000000000..959866d0447 --- /dev/null +++ b/src/relay/relay-watcher-stale-client-release.ts @@ -0,0 +1,20 @@ +import type { RelayWatcherTeardownState } from './relay-watcher-teardown-tracker' + +export function releaseStaleRelayWatches( + states: Iterable, + closeWatch: (state: RelayWatcherTeardownState) => Promise +): Promise | undefined { + const teardowns: Promise[] = [] + for (const state of Array.from(states)) { + for (const [clientId, isStale] of state.clients) { + if (isStale()) { + state.clients.delete(clientId) + state.clientWatchIds.delete(clientId) + } + } + if (state.clients.size === 0) { + teardowns.push(closeWatch(state)) + } + } + return teardowns.length > 0 ? Promise.all(teardowns).then(() => undefined) : undefined +} diff --git a/src/relay/relay-watcher-state.ts b/src/relay/relay-watcher-state.ts new file mode 100644 index 00000000000..90d6b978a8d --- /dev/null +++ b/src/relay/relay-watcher-state.ts @@ -0,0 +1,22 @@ +import { PromiseSettlementWaiters } from '../shared/promise-settlement-waiters' +import type { RelayWatcherTeardownState } from './relay-watcher-teardown-tracker' + +export function createRelayWatcherState( + rootKey: string, + rootPath: string, + clientId: number, + isStale: () => boolean, + watchId?: number +): RelayWatcherTeardownState { + return { + clientWatchIds: watchId === undefined ? new Map() : new Map([[clientId, watchId]]), + rootKey, + rootPath, + clients: new Map([[clientId, isStale]]), + setupWaiters: new PromiseSettlementWaiters(Promise.resolve()), + subscription: null, + abortController: new AbortController(), + generation: 0, + closed: false + } +} diff --git a/src/relay/relay-watcher-teardown-tracker.ts b/src/relay/relay-watcher-teardown-tracker.ts new file mode 100644 index 00000000000..91b1460054d --- /dev/null +++ b/src/relay/relay-watcher-teardown-tracker.ts @@ -0,0 +1,109 @@ +import type { WatcherProcessSubscription } from '../main/ipc/parcel-watcher-process' +import { isWatcherProcessFailure } from '../main/ipc/parcel-watcher-process-failure' +import type { PromiseSettlementWaiters } from '../shared/promise-settlement-waiters' + +export type RelayWatcherTeardownState = { + rootKey: string + rootPath: string + clients: Map boolean> + clientWatchIds: Map + setupWaiters: PromiseSettlementWaiters + subscription: WatcherProcessSubscription | null + abortController: AbortController + generation: number + closed: boolean +} + +export class RelayWatcherTeardownTracker { + private readonly pending = new Map>() + private readonly failed = new Map() + + constructor(private readonly forgetRoot: (rootPath: string) => void) {} + + close(state: RelayWatcherTeardownState, removeActiveState: () => void): Promise { + const pending = this.pending.get(state.rootKey) + if (pending) { + return pending + } + if (!state.closed) { + state.closed = true + state.generation++ + state.abortController.abort() + state.clients.clear() + state.clientWatchIds.clear() + removeActiveState() + } + const subscription = state.subscription + const teardown = subscription + ? callUnsubscribe(subscription) + : state.setupWaiters.promise.then(() => undefined) + let tracked: Promise + tracked = teardown + .then( + () => { + state.subscription = null + this.failed.delete(state.rootKey) + this.forgetRoot(state.rootPath) + }, + (error) => { + const physicalExit = isWatcherProcessFailure(error) ? error.physicalExit : undefined + if (!subscription && !physicalExit) { + this.failed.delete(state.rootKey) + this.forgetRoot(state.rootPath) + return + } + // Why: failed physical teardown must remain retryable and consume capacity. + this.failed.set(state.rootKey, { state, error }) + if (physicalExit) { + // Why: an unkillable child still owns native handles until its later + // physical exit, even when teardown began before setup published. + void physicalExit.then(() => { + const failed = this.failed.get(state.rootKey) + if (failed?.state === state && failed.error === error) { + this.failed.delete(state.rootKey) + this.forgetRoot(state.rootPath) + } + }) + } + throw error + } + ) + .finally(() => { + if (this.pending.get(state.rootKey) === tracked) { + this.pending.delete(state.rootKey) + } + }) + this.pending.set(state.rootKey, tracked) + return tracked + } + + join(rootKey: string): Promise | undefined { + const pending = this.pending.get(rootKey) + const failed = this.failed.get(rootKey) + if (!pending) { + return failed ? Promise.reject(failed.error) : undefined + } + return pending.then(() => { + const settledFailure = this.failed.get(rootKey) + if (settledFailure) { + throw settledFailure.error + } + }) + } + + failedState(rootKey: string): RelayWatcherTeardownState | undefined { + return this.failed.get(rootKey)?.state + } + + rootPaths(): string[] { + return [...this.pending.keys(), ...this.failed.keys()] + } +} + +function callUnsubscribe(subscription: WatcherProcessSubscription): Promise { + try { + return Promise.resolve(subscription.unsubscribe()) + } catch (error) { + return Promise.reject(error) + } +} diff --git a/src/relay/relay-watcher-terminal-notifier.ts b/src/relay/relay-watcher-terminal-notifier.ts new file mode 100644 index 00000000000..add044f4460 --- /dev/null +++ b/src/relay/relay-watcher-terminal-notifier.ts @@ -0,0 +1,32 @@ +import type { RelayDispatcher } from './dispatcher' + +type RelayTerminalWatchOwners = { + rootPath: string + clients: Map boolean> + clientWatchIds: Map +} + +export function emitRelayWatcherTerminalFailure( + dispatcher: RelayDispatcher, + state: RelayTerminalWatchOwners, + message: string +): void { + for (const [clientId, isStale] of state.clients) { + if (isStale()) { + state.clients.delete(clientId) + state.clientWatchIds.delete(clientId) + continue + } + const watchId = state.clientWatchIds.get(clientId) + if (watchId === undefined) { + continue + } + // Why: a shared relay may serve multiple Orca clients; only owners of + // this exact logical watch should invalidate their provider state. + dispatcher.notifyClient(clientId, 'fs.watchFailed', { + rootPath: state.rootPath, + watchId, + message + }) + } +} diff --git a/src/relay/relay.ts b/src/relay/relay.ts index d47c2019582..debef3b85e8 100644 --- a/src/relay/relay.ts +++ b/src/relay/relay.ts @@ -467,7 +467,10 @@ async function main(): Promise { const ptyHandler = new PtyHandler(dispatcher, graceTimeMs) const fsHandler = new FsHandler(dispatcher, context) - const gitHandler = new GitHandler(dispatcher, context) + const watchRegistry = fsHandler.getWatchRegistry() + ptyHandler.setWorktreeRemovalCoordinator(watchRegistry) + watchRegistry.setWorktreePtyTeardown((rootPath) => ptyHandler.shutdownForWorktreePath(rootPath)) + const gitHandler = new GitHandler(dispatcher, context, watchRegistry) const _preflightHandler = new PreflightHandler(dispatcher) const _externalAutomationsHandler = new ExternalAutomationsHandler(dispatcher) @@ -1014,25 +1017,41 @@ async function main(): Promise { }) } + let shutdownInFlight = false function shutdown(): void { + if (shutdownInFlight) { + return + } + shutdownInFlight = true relayLogLine( `[relay] Shutdown: ptys=${ptyHandler.activePtyCount}, clients=${socketClients.size}, ownsSocket=${ownsSocketPath}` ) graceDeadlineAt = null graceReason = null - dispatcher.dispose() - ptyHandler.dispose() - fsHandler.dispose() - gitHandler.dispose() - hookServer.stop() - // Why: Node's Unix server.close() can unlink the listen path. If the path - // was externally removed and rebound by a newer relay, closing this older - // server would strand the newer daemon behind a missing socket. - if (socketServer && ownsCurrentSocketPath()) { - socketServer.close() - } - cleanupOwnedSocket() - process.exit(0) + void ptyHandler + .dispose() + .then(() => { + dispatcher.dispose() + fsHandler.dispose() + gitHandler.dispose() + hookServer.stop() + // Why: Node's Unix server.close() can unlink the listen path. If the path + // was externally removed and rebound by a newer relay, closing this older + // server would strand the newer daemon behind a missing socket. + if (socketServer && ownsCurrentSocketPath()) { + socketServer.close() + } + cleanupOwnedSocket() + process.exit(0) + }) + .catch((error) => { + // Why: keep owning a PTY whose native kill was rejected; exiting here + // would turn a transient signal failure into an orphan remote shell. + shutdownInFlight = false + relayLogLine( + `[relay] Shutdown deferred: ${error instanceof Error ? error.message : String(error)}` + ) + }) } process.on('SIGTERM', shutdown) diff --git a/src/relay/subprocess.test.ts b/src/relay/subprocess.test.ts index 219addc02a9..ee20461dd68 100644 --- a/src/relay/subprocess.test.ts +++ b/src/relay/subprocess.test.ts @@ -99,6 +99,10 @@ describe('Subprocess: Relay entry point', () => { await relay.sentinelReceived }, 10_000) + it('keeps the Node-18 relay bundle free of unsupported array copy methods', () => { + expect(readFileSync(relayEntry, 'utf8')).not.toContain('.toReversed(') + }) + it('responds to fs.stat over stdin/stdout', async () => { tmpDir = mkdtempSync(path.join(tmpdir(), 'relay-sub-')) writeFileSync(path.join(tmpDir, 'test.txt'), 'hello') diff --git a/src/renderer/src/runtime/runtime-file-client.ts b/src/renderer/src/runtime/runtime-file-client.ts index 6c70cd9e63c..6033ce58efe 100644 --- a/src/renderer/src/runtime/runtime-file-client.ts +++ b/src/renderer/src/runtime/runtime-file-client.ts @@ -99,6 +99,7 @@ type RuntimeImportResult = } type RuntimeFileWatchEvent = + | { type: 'starting'; subscriptionId: string } | { type: 'ready'; subscriptionId: string } | { type: 'changed'; worktree: string; events: FsChangedPayload['events'] } | { type: 'error'; message: string } @@ -932,7 +933,7 @@ function handleSharedRuntimeFileWatchResponse( const event = unwrapRuntimeRpcResult( response as RuntimeRpcResponse ) - if (event.type === 'ready') { + if (event.type === 'starting' || event.type === 'ready') { shared.remoteSubscriptionId = event.subscriptionId if (shared.closed) { shared.unsubscribe?.() @@ -989,7 +990,7 @@ function closeSharedRuntimeFileWatch(key: string, shared: SharedRuntimeFileWatch sharedRuntimeFileWatches.delete(key) if (shared.keepStreamUntilReady) { // Why: WebRuntimeClient owns shared-socket file-watch cleanup, including - // pre-ready fallback timers and late-ready files.unwatch. + // pre-ready cancellation ownership and late-ready files.unwatch. shared.unsubscribe?.() shared.unsubscribe = null return diff --git a/src/renderer/src/web/web-runtime-client.test.ts b/src/renderer/src/web/web-runtime-client.test.ts index 8f7e500c3d3..12187812165 100644 --- a/src/renderer/src/web/web-runtime-client.test.ts +++ b/src/renderer/src/web/web-runtime-client.test.ts @@ -220,12 +220,9 @@ describe('WebRuntimeClient', () => { const subscribeOnCurrentConnection = vi .spyOn(internals, 'subscribeOnCurrentConnection') .mockResolvedValue(handle) + const onResponse = vi.fn() - const subscription = await client.subscribe( - 'files.watch', - { worktree: 'wt-1' }, - { onResponse: vi.fn() } - ) + const subscription = await client.subscribe('files.watch', { worktree: 'wt-1' }, { onResponse }) expect(subscribeOnCurrentConnection).toHaveBeenCalledWith( 'files.watch', @@ -234,6 +231,14 @@ describe('WebRuntimeClient', () => { undefined ) expect(internals.childClients.size).toBe(0) + subscribeOnCurrentConnection.mock.calls[0]?.[2].onResponse({ + id: 'watch', + ok: true, + streaming: true, + result: { type: 'starting', subscriptionId: 'files-watch-starting' }, + _meta: { runtimeId: 'runtime-web-test' } + } as RuntimeRpcResponse & { streaming: true }) + expect(onResponse).not.toHaveBeenCalled() const frame = new Uint8Array([1]) subscription.sendBinary(frame) expect(handle.sendBinary).toHaveBeenCalledWith(frame) @@ -287,7 +292,7 @@ describe('WebRuntimeClient', () => { client.close() }) - it('removes the shared local subscription when remote unwatch fails', async () => { + it('retains and retries a shared local subscription when remote unwatch returns failure', async () => { const client = new WebRuntimeClient({ v: 2, endpoint: 'ws://127.0.0.1:6768', @@ -301,8 +306,19 @@ describe('WebRuntimeClient', () => { const subscribeOnCurrentConnection = vi .spyOn(internals, 'subscribeOnCurrentConnection') .mockResolvedValue(localHandle) - const unwatchError = new Error('remote unwatch failed') - const unwatch = vi.spyOn(client, 'call').mockRejectedValue(unwatchError) + const unwatch = vi + .spyOn(client, 'call') + .mockResolvedValueOnce({ + id: 'unwatch-failed', + ok: false, + error: { code: 'teardown_failed', message: 'remote unwatch failed' } + }) + .mockResolvedValueOnce({ + id: 'unwatch-retry', + ok: true, + result: { unsubscribed: true }, + _meta: { runtimeId: 'runtime-web-test' } + }) const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) try { @@ -327,8 +343,83 @@ describe('WebRuntimeClient', () => { { subscriptionId: 'files-watch-failing-unwatch' }, { timeoutMs: 5_000 } ) - await vi.waitFor(() => expect(localHandle.unsubscribe).toHaveBeenCalledTimes(1)) - expect(warn).toHaveBeenCalledWith('Failed to unwatch remote file subscription:', unwatchError) + await vi.waitFor(() => expect(warn).toHaveBeenCalledTimes(1)) + expect(localHandle.unsubscribe).not.toHaveBeenCalled() + expect(warn).toHaveBeenCalledWith( + 'Failed to unwatch remote file subscription:', + expect.objectContaining({ message: 'teardown_failed: remote unwatch failed' }) + ) + + await client.subscribe('files.watch', { worktree: 'wt-1' }, { onResponse: vi.fn() }) + expect(unwatch).toHaveBeenCalledTimes(2) + expect(localHandle.unsubscribe).toHaveBeenCalledTimes(1) + expect(subscribeOnCurrentConnection).toHaveBeenCalledTimes(2) + } finally { + client.close() + warn.mockRestore() + } + }) + + it('retries every failed same-root teardown before opening a replacement watch', async () => { + const client = new WebRuntimeClient({ + v: 2, + endpoint: 'ws://127.0.0.1:6768', + deviceToken: 'token', + publicKeyB64: Buffer.alloc(32).toString('base64') + }) + const handles = Array.from({ length: 3 }, () => ({ + unsubscribe: vi.fn(), + sendBinary: vi.fn() + })) + const internals = client as unknown as { + subscribeOnCurrentConnection: WebRuntimeClient['subscribe'] + } + const subscribeOnCurrentConnection = vi + .spyOn(internals, 'subscribeOnCurrentConnection') + .mockResolvedValueOnce(handles[0]) + .mockResolvedValueOnce(handles[1]) + .mockResolvedValueOnce(handles[2]) + const unwatch = vi + .spyOn(client, 'call') + .mockRejectedValueOnce(new Error('first teardown failed')) + .mockRejectedValueOnce(new Error('second teardown failed')) + .mockResolvedValue({ + id: 'unwatch-retry', + ok: true, + result: { unsubscribed: true }, + _meta: { runtimeId: 'runtime-web-test' } + }) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + const first = await client.subscribe( + 'files.watch', + { worktree: 'wt-1' }, + { onResponse: vi.fn() } + ) + const second = await client.subscribe( + 'files.watch', + { worktree: 'wt-1' }, + { onResponse: vi.fn() } + ) + for (const [index, subscriptionId] of ['watch-a', 'watch-b'].entries()) { + subscribeOnCurrentConnection.mock.calls[index]?.[2].onResponse({ + id: subscriptionId, + ok: true, + streaming: true, + result: { type: 'ready', subscriptionId }, + _meta: { runtimeId: 'runtime-web-test' } + } as RuntimeRpcResponse & { streaming: true }) + } + first.unsubscribe() + second.unsubscribe() + await vi.waitFor(() => expect(warn).toHaveBeenCalledTimes(2)) + + await client.subscribe('files.watch', { worktree: 'wt-1' }, { onResponse: vi.fn() }) + + expect(unwatch).toHaveBeenCalledTimes(4) + expect(handles[0].unsubscribe).toHaveBeenCalledTimes(1) + expect(handles[1].unsubscribe).toHaveBeenCalledTimes(1) + expect(subscribeOnCurrentConnection).toHaveBeenCalledTimes(3) } finally { client.close() warn.mockRestore() @@ -382,7 +473,7 @@ describe('WebRuntimeClient', () => { client.close() }) - it('cleans up a stopped pre-ready shared file watch if ready never arrives', async () => { + it('retains a stopped pre-ready file watch until its cancellation id arrives', async () => { vi.useFakeTimers() const timerWindow = window as unknown as { setTimeout: typeof setTimeout @@ -400,8 +491,15 @@ describe('WebRuntimeClient', () => { const internals = client as unknown as { subscribeOnCurrentConnection: WebRuntimeClient['subscribe'] } - vi.spyOn(internals, 'subscribeOnCurrentConnection').mockResolvedValue(localHandle) - const unwatch = vi.spyOn(client, 'call') + const subscribeOnCurrentConnection = vi + .spyOn(internals, 'subscribeOnCurrentConnection') + .mockResolvedValue(localHandle) + const unwatch = vi.spyOn(client, 'call').mockResolvedValue({ + id: 'unwatch', + ok: true, + result: { unsubscribed: true }, + _meta: { runtimeId: 'runtime-web-test' } + }) try { const subscription = await client.subscribe( @@ -414,12 +512,24 @@ describe('WebRuntimeClient', () => { expect(unwatch).not.toHaveBeenCalled() expect(localHandle.unsubscribe).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(4_999) + await vi.advanceTimersByTimeAsync(300_000) expect(localHandle.unsubscribe).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(1) - expect(localHandle.unsubscribe).toHaveBeenCalledTimes(1) - expect(unwatch).not.toHaveBeenCalled() + const wrappedCallbacks = subscribeOnCurrentConnection.mock.calls[0]?.[2] + wrappedCallbacks?.onResponse({ + id: 'watch', + ok: true, + streaming: true, + result: { type: 'starting', subscriptionId: 'files-watch-pending' }, + _meta: { runtimeId: 'runtime-web-test' } + } as RuntimeRpcResponse & { streaming: true }) + + expect(unwatch).toHaveBeenCalledWith( + 'files.unwatch', + { subscriptionId: 'files-watch-pending' }, + { timeoutMs: 5_000 } + ) + await vi.waitFor(() => expect(localHandle.unsubscribe).toHaveBeenCalledTimes(1)) } finally { client.close() vi.useRealTimers() diff --git a/src/renderer/src/web/web-runtime-client.ts b/src/renderer/src/web/web-runtime-client.ts index f94cf04476e..b21c64795c2 100644 --- a/src/renderer/src/web/web-runtime-client.ts +++ b/src/renderer/src/web/web-runtime-client.ts @@ -61,7 +61,6 @@ export type SubscribeOptions = { const REQUEST_TIMEOUT_MS = 30_000 const CONNECT_TIMEOUT_MS = 12_000 const HANDSHAKE_TIMEOUT_MS = 10_000 -const FILE_WATCH_READY_CLEANUP_TIMEOUT_MS = 5_000 const RECONNECT_DELAYS_MS = [500, 1000, 2000, 4000, 8000, 15_000] const SHARED_CONNECTION_SUBSCRIPTION_METHODS = new Set(['files.watch']) // Why: the browser WebSocket API hides protocol pings/pongs, so a half-open @@ -99,6 +98,7 @@ export class WebRuntimeClient { private lastHeartbeatTickAt = 0 private readonly pending = new Map() private readonly subscriptions = new Map() + private readonly fileWatchTeardownRetries = new Map Promise>>() private readonly childClients = new Set() private readonly waiters: { resolve: () => void; reject: (error: Error) => void }[] = [] private readonly serverPublicKey: Uint8Array @@ -187,67 +187,81 @@ export class WebRuntimeClient { callbacks: SubscriptionCallbacks, options?: { timeoutMs?: number } ): Promise { + const teardownKey = JSON.stringify(params) ?? String(params) + await Promise.all( + Array.from(this.fileWatchTeardownRetries.get(teardownKey) ?? [], (retry) => retry()) + ) let stopped = false let remoteSubscriptionId: string | null = null let unwatchStarted = false let handle: WebRuntimeSubscriptionHandle | null = null - let readyCleanupTimer: number | null = null - const clearReadyCleanupTimer = (): void => { - if (readyCleanupTimer === null) { - return - } - window.clearTimeout(readyCleanupTimer) - readyCleanupTimer = null - } const dropLocalSubscription = (): void => { - clearReadyCleanupTimer() handle?.unsubscribe() } - const schedulePreReadyCleanup = (): void => { - if (readyCleanupTimer !== null) { - return + let unwatchAttempt: Promise | null = null + const retryRemoteUnwatch = (): Promise => { + if (unwatchAttempt) { + return unwatchAttempt } - // Why: if a stopped watch never reaches ready/error, no remote id exists - // to unwatch; bound the lifetime of the local callback on this socket. - readyCleanupTimer = window.setTimeout(() => { - readyCleanupTimer = null - handle?.unsubscribe() - }, FILE_WATCH_READY_CLEANUP_TIMEOUT_MS) + unwatchStarted = true + const attempt = this.call( + 'files.unwatch', + { subscriptionId: remoteSubscriptionId! }, + { timeoutMs: 5_000 } + ) + .then((response) => { + if (response.ok === false) { + throw new Error(`${response.error.code}: ${response.error.message}`) + } + const retries = this.fileWatchTeardownRetries.get(teardownKey) + retries?.delete(retryRemoteUnwatch) + if (retries?.size === 0) { + this.fileWatchTeardownRetries.delete(teardownKey) + } + dropLocalSubscription() + }) + .catch((error: unknown) => { + console.warn('Failed to unwatch remote file subscription:', error) + throw error + }) + .finally(() => { + unwatchAttempt = null + unwatchStarted = false + }) + unwatchAttempt = attempt + return attempt } const unwatchAndDropLocalSubscription = (): void => { if (unwatchStarted) { return } - unwatchStarted = true if (!remoteSubscriptionId) { dropLocalSubscription() return } - clearReadyCleanupTimer() - // Why: shared files.watch streams stay on this socket, so stop the - // server watcher before removing the local callback that receives ready. - void this.call( - 'files.unwatch', - { subscriptionId: remoteSubscriptionId }, - { timeoutMs: 5_000 } - ) - .catch((error) => { - console.warn('Failed to unwatch remote file subscription:', error) - }) - .finally(() => { - dropLocalSubscription() - }) + // Why: retain the shared-socket callback and retry ownership until the + // server acknowledges physical teardown; a new watch joins this barrier. + const retries = this.fileWatchTeardownRetries.get(teardownKey) ?? new Set() + retries.add(retryRemoteUnwatch) + this.fileWatchTeardownRetries.set(teardownKey, retries) + void retryRemoteUnwatch().catch(() => {}) } const wrappedCallbacks: SubscriptionCallbacks = { ...callbacks, onResponse: (response) => { - if (isFileWatchReadyResponse(response)) { - remoteSubscriptionId = response.result.subscriptionId + const nextSubscriptionId = getFileWatchSubscriptionId(response) + if (nextSubscriptionId) { + remoteSubscriptionId = nextSubscriptionId if (stopped) { unwatchAndDropLocalSubscription() return } } + // Why: the server publishes cancellation ownership before native setup; + // callers should still become ready only after the watcher is live. + if (isFileWatchStartingResponse(response)) { + return + } if (!stopped) { callbacks.onResponse(response) } else if (response.ok === false) { @@ -280,9 +294,9 @@ export class WebRuntimeClient { stopped = true if (remoteSubscriptionId) { unwatchAndDropLocalSubscription() - } else { - schedulePreReadyCleanup() } + // Why: an older server may not publish its id until ready. Retain the + // callback so a late response can still physically unwatch the root. }, sendBinary: (bytes) => handle?.sendBinary(bytes) } @@ -329,6 +343,7 @@ export class WebRuntimeClient { child.close({ notifySubscriptions: shouldNotifySubscriptions }) } this.childClients.clear() + this.fileWatchTeardownRetries.clear() this.clearTimers() this.rejectAllPending('Remote Orca runtime connection closed.') this.rejectAllWaiters(new Error('Remote Orca runtime connection closed.')) @@ -796,18 +811,26 @@ function isRuntimeFailureResponse( ) } -function isFileWatchReadyResponse( - response: RuntimeRpcResponse -): response is RuntimeRpcSuccess<{ type: 'ready'; subscriptionId: string }> { +function getFileWatchSubscriptionId(response: RuntimeRpcResponse): string | null { if (!response.ok) { - return false + return null } const result = response.result + if (!result || typeof result !== 'object') { + return null + } + const subscriptionId = (result as { subscriptionId?: unknown }).subscriptionId + return typeof subscriptionId === 'string' ? subscriptionId : null +} + +function isFileWatchStartingResponse( + response: RuntimeRpcResponse +): response is RuntimeRpcSuccess<{ type: 'starting'; subscriptionId: string }> { return ( - !!result && - typeof result === 'object' && - (result as { type?: unknown }).type === 'ready' && - typeof (result as { subscriptionId?: unknown }).subscriptionId === 'string' + response.ok && + !!response.result && + typeof response.result === 'object' && + (response.result as { type?: unknown }).type === 'starting' ) } diff --git a/src/shared/cross-platform-path.test.ts b/src/shared/cross-platform-path.test.ts index 15777065bc0..4098e8f9b86 100644 --- a/src/shared/cross-platform-path.test.ts +++ b/src/shared/cross-platform-path.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import { isPathInsideOrEqual, isRuntimePathAbsolute, + normalizeRuntimePathForComparison, relativePathInsideRoot, resolveRuntimePath } from './cross-platform-path' @@ -14,6 +15,12 @@ describe('cross-platform path containment', () => { expect(relativePathInsideRoot('/repo/app/', '/repo/app/src/index.ts')).toBe('src/index.ts') }) + it('keeps literal POSIX backslashes distinct from separators', () => { + expect(normalizeRuntimePathForComparison('/srv/team\\repo')).toBe('/srv/team\\repo') + expect(normalizeRuntimePathForComparison('/srv/team/repo')).toBe('/srv/team/repo') + expect(isPathInsideOrEqual('/srv/team\\repo', '/srv/team/repo/file.ts')).toBe(false) + }) + it('handles Windows drive roots and sibling drives case-insensitively', () => { expect(isPathInsideOrEqual('C:\\Repo', 'c:\\repo\\src\\index.ts')).toBe(true) expect(relativePathInsideRoot('C:\\Repo', 'c:\\repo\\src\\index.ts')).toBe('src/index.ts') diff --git a/src/shared/cross-platform-path.ts b/src/shared/cross-platform-path.ts index e82bd9d2461..fee96842af8 100644 --- a/src/shared/cross-platform-path.ts +++ b/src/shared/cross-platform-path.ts @@ -11,14 +11,19 @@ export function normalizeRuntimePathSeparators(value: string): string { } export function normalizeRuntimePathForComparison(value: string): string { - const normalized = trimRuntimePathTrailingSlash(normalizeRuntimePathSeparators(value)) + const isWindowsPath = isWindowsAbsolutePathLike(value) + // Why: backslash is a valid POSIX filename character; fold it only when the + // path itself proves Windows drive/UNC semantics. + const normalized = trimRuntimePathTrailingSlash( + isWindowsPath ? normalizeRuntimePathSeparators(value) : value.replace(/\/+/g, '/') + ) const wslUnc = normalized.match(/^\/\/(?:wsl\.localhost|wsl\$)\/([^/]+)(\/[\s\S]*)?$/i) if (wslUnc) { // Why: Windows exposes the same case-sensitive WSL filesystem through two // UNC aliases, while the distro/server portion remains case-insensitive. return `//wsl/${wslUnc[1].toLowerCase()}${wslUnc[2] ?? ''}` } - return isWindowsAbsolutePathLike(value) ? normalized.toLowerCase() : normalized + return isWindowsPath ? normalized.toLowerCase() : normalized } export function isRuntimePathAbsolute( diff --git a/src/shared/physical-exit-tracker.test.ts b/src/shared/physical-exit-tracker.test.ts new file mode 100644 index 00000000000..10de6fe4443 --- /dev/null +++ b/src/shared/physical-exit-tracker.test.ts @@ -0,0 +1,34 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { PhysicalExitTracker } from './physical-exit-tracker' + +describe('PhysicalExitTracker', () => { + afterEach(() => vi.useRealTimers()) + + it('removes timed-out waiters while later retries can still observe exit', async () => { + vi.useFakeTimers() + const tracker = new PhysicalExitTracker() + const first = tracker.waitForExit(10, () => new Error('first deadline')) + const firstRejection = expect(first).rejects.toThrow('first deadline') + + await vi.advanceTimersByTimeAsync(10) + await firstRejection + + const retry = tracker.waitForExit(10, () => new Error('retry deadline')) + tracker.markExited() + + await expect(retry).resolves.toBeUndefined() + await expect(tracker.exitedPromise).resolves.toBeUndefined() + }) + + it('settles every active waiter once and resolves later waits immediately', async () => { + const tracker = new PhysicalExitTracker() + const first = tracker.waitForExit(100, () => new Error('deadline')) + const second = tracker.waitForExit(100, () => new Error('deadline')) + + tracker.markExited() + tracker.markExited() + + await expect(Promise.all([first, second])).resolves.toEqual([undefined, undefined]) + await expect(tracker.waitForExit(1, () => new Error('deadline'))).resolves.toBeUndefined() + }) +}) diff --git a/src/shared/physical-exit-tracker.ts b/src/shared/physical-exit-tracker.ts new file mode 100644 index 00000000000..1b896085ff8 --- /dev/null +++ b/src/shared/physical-exit-tracker.ts @@ -0,0 +1,48 @@ +type ExitWaiter = { + resolve: () => void + reject: (error: Error) => void + timer: ReturnType +} + +// Why: Promise.then reactions cannot be detached after a deadline. Explicit +// waiters let repeated retries time out without accumulating on a live child. +export class PhysicalExitTracker { + private exited = false + private readonly waiters = new Set() + private resolveExit!: () => void + readonly exitedPromise = new Promise((resolve) => { + this.resolveExit = resolve + }) + + markExited(): void { + if (this.exited) { + return + } + this.exited = true + this.resolveExit() + for (const waiter of this.waiters) { + clearTimeout(waiter.timer) + waiter.resolve() + } + this.waiters.clear() + } + + waitForExit(timeoutMs: number, timeoutError: () => Error): Promise { + if (this.exited) { + return Promise.resolve() + } + return new Promise((resolve, reject) => { + let waiter!: ExitWaiter + const timer = setTimeout( + () => { + this.waiters.delete(waiter) + waiter.reject(timeoutError()) + }, + Math.max(1, timeoutMs) + ) + timer.unref?.() + waiter = { resolve, reject, timer } + this.waiters.add(waiter) + }) + } +} diff --git a/src/shared/promise-settlement-waiters.test.ts b/src/shared/promise-settlement-waiters.test.ts new file mode 100644 index 00000000000..dfb9b50b7bb --- /dev/null +++ b/src/shared/promise-settlement-waiters.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it, vi } from 'vitest' +import { PromiseSettlementWaiters } from './promise-settlement-waiters' + +describe('PromiseSettlementWaiters', () => { + it('removes ten thousand aborted callers while one anchor remains pending', async () => { + let resolveBase: (value: number) => void = () => {} + const basePromise = new Promise((resolve) => { + resolveBase = resolve + }) + const thenSpy = vi.spyOn(basePromise, 'then') + const waiters = new PromiseSettlementWaiters(basePromise) + const anchor = waiters.wait() + const controllers = Array.from({ length: 10_000 }, () => new AbortController()) + const cancelled = controllers.map((controller) => + waiters.wait({ signal: controller.signal }).catch((error) => error) + ) + + for (const controller of controllers) { + controller.abort() + } + await Promise.all(cancelled) + + expect(waiters.waiterCount).toBe(1) + expect(thenSpy).toHaveBeenCalledOnce() + resolveBase(42) + await expect(anchor).resolves.toBe(42) + expect(waiters.waiterCount).toBe(0) + }) + + it('removes timed-out callers and clears their timers on settlement', async () => { + vi.useFakeTimers() + try { + let resolveBase: () => void = () => {} + const waiters = new PromiseSettlementWaiters( + new Promise((resolve) => { + resolveBase = resolve + }) + ) + const timedOut = waiters + .wait({ timeoutMs: 10, createTimeoutError: () => new Error('late') }) + .catch((error) => error) + const active = waiters.wait({ timeoutMs: 100 }) + + await vi.advanceTimersByTimeAsync(10) + await expect(timedOut).resolves.toMatchObject({ message: 'late' }) + expect(waiters.waiterCount).toBe(1) + resolveBase() + await expect(active).resolves.toBeUndefined() + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it('notifies ownership cleanup for an already-aborted caller', async () => { + const controller = new AbortController() + controller.abort() + const onAbandon = vi.fn() + const waiters = new PromiseSettlementWaiters(new Promise(() => {})) + + await expect(waiters.wait({ signal: controller.signal, onAbandon })).rejects.toMatchObject({ + name: 'AbortError' + }) + + expect(onAbandon).toHaveBeenCalledWith('abort') + expect(waiters.waiterCount).toBe(0) + }) +}) diff --git a/src/shared/promise-settlement-waiters.ts b/src/shared/promise-settlement-waiters.ts new file mode 100644 index 00000000000..98ec24b25c1 --- /dev/null +++ b/src/shared/promise-settlement-waiters.ts @@ -0,0 +1,139 @@ +type PromiseSettlement = + | { status: 'fulfilled'; value: T } + | { status: 'rejected'; error: unknown } + +type PromiseSettlementWaiter = { + resolve: (value: T) => void + reject: (error: unknown) => void + onFulfilled?: (value: T) => void + signal?: AbortSignal + onAbort: () => void + timer?: ReturnType +} + +export type PromiseSettlementWaitOptions = { + signal?: AbortSignal + timeoutMs?: number + createAbortError?: () => Error + createTimeoutError?: () => Error + onFulfilled?: (value: T) => void + onAbandon?: (reason: 'abort' | 'timeout') => void +} + +// Why: Promise.then reactions cannot be detached. One reaction fans settlement +// into explicit waiters so aborted or timed-out callers release their closures. +export class PromiseSettlementWaiters { + private readonly waiters = new Set>() + private settlement: PromiseSettlement | null = null + + constructor( + readonly promise: Promise, + onSettled?: () => void + ) { + void this.promise.then( + (value) => this.settle({ status: 'fulfilled', value }, onSettled), + (error: unknown) => this.settle({ status: 'rejected', error }, onSettled) + ) + } + + get waiterCount(): number { + return this.waiters.size + } + + wait(options: PromiseSettlementWaitOptions = {}): Promise { + if (options.signal?.aborted) { + options.onAbandon?.('abort') + return Promise.reject(options.createAbortError?.() ?? createDefaultAbortError()) + } + if (this.settlement) { + return settleWaiterImmediately(this.settlement, options.onFulfilled) + } + return new Promise((resolve, reject) => { + let waiter!: PromiseSettlementWaiter + const abandon = (reason: 'abort' | 'timeout', error: Error): void => { + if (!this.waiters.delete(waiter)) { + return + } + cleanupWaiter(waiter) + options.onAbandon?.(reason) + reject(error) + } + const onAbort = (): void => + abandon('abort', options.createAbortError?.() ?? createDefaultAbortError()) + waiter = { + resolve, + reject, + onFulfilled: options.onFulfilled, + signal: options.signal, + onAbort + } + if (options.timeoutMs !== undefined) { + waiter.timer = setTimeout( + () => + abandon( + 'timeout', + options.createTimeoutError?.() ?? new Error('Promise settlement wait timed out') + ), + Math.max(1, options.timeoutMs) + ) + waiter.timer.unref?.() + } + this.waiters.add(waiter) + options.signal?.addEventListener('abort', onAbort, { once: true }) + if (options.signal?.aborted) { + onAbort() + } + }) + } + + private settle(settlement: PromiseSettlement, onSettled?: () => void): void { + if (this.settlement) { + return + } + this.settlement = settlement + for (const waiter of this.waiters) { + this.waiters.delete(waiter) + cleanupWaiter(waiter) + settleWaiter(settlement, waiter) + } + onSettled?.() + } +} + +function cleanupWaiter(waiter: PromiseSettlementWaiter): void { + if (waiter.timer) { + clearTimeout(waiter.timer) + } + waiter.signal?.removeEventListener('abort', waiter.onAbort) +} + +function settleWaiter( + settlement: PromiseSettlement, + waiter: Pick, 'resolve' | 'reject' | 'onFulfilled'> +): void { + if (settlement.status === 'rejected') { + waiter.reject(settlement.error) + return + } + try { + waiter.onFulfilled?.(settlement.value) + waiter.resolve(settlement.value) + } catch (error) { + waiter.reject(error) + } +} + +function settleWaiterImmediately( + settlement: PromiseSettlement, + onFulfilled?: (value: T) => void +): Promise { + return new Promise((resolve, reject) => + settleWaiter(settlement, { resolve, reject, onFulfilled }) + ) +} + +function createDefaultAbortError(): Error { + const error = new Error('Promise settlement wait aborted') + error.name = 'AbortError' + return error +} diff --git a/src/shared/runtime-file-watch-limits.ts b/src/shared/runtime-file-watch-limits.ts new file mode 100644 index 00000000000..2f70fa0c7f0 --- /dev/null +++ b/src/shared/runtime-file-watch-limits.ts @@ -0,0 +1,4 @@ +export const RUNTIME_FILE_WATCH_CRAWL_TIMEOUT_MS = 60_000 +export const RUNTIME_FILE_WATCH_CANCEL_TIMEOUT_MS = 60_000 +export const RUNTIME_FILE_WATCH_EXIT_DEADLINE_MS = 10_000 +export const RUNTIME_FILE_WATCH_MAX_SETUP_ATTEMPTS = 2