fix(ssh): bound healthy relay watcher processes

config/reliability-gates.jsonc

@@ -3924,14 +3924,14 @@
         "remote-runtime",
         "ssh"
       ],
-      "coverageNotes": "Deterministic tests cover a four-child healthy runtime pool plus at most four bounded quarantine children, one quarantine attempt per watch lifetime, queue-aware cancellation, bounded initial and crash-resubscribe crawls, physical desktop-install abort, stale-generation rejection, per-child stat bounds, final-overflow RPC delivery, and renderer eviction before terminal callbacks can retry. A built-entry macOS harness drives the real supervisor, kills its child with SIGSEGV, then requires automatic resubscription and a later event. SSH now bundles the same child boundary beside relay.js, requires both artifacts for install completeness, and preserves registration-owned same-root cancellation. A built-relay harness kills that child and requires the original relay PID and existing PTY to survive before overflow and precise events resume.",
+      "coverageNotes": "Deterministic tests cover a four-child healthy runtime pool, a one-child healthy SSH relay pool, and at most four bounded quarantine children, with one quarantine attempt per watch lifetime, queue-aware cancellation, bounded initial and crash-resubscribe crawls, physical desktop-install abort, stale-generation rejection, per-child stat bounds, final-overflow RPC delivery, and renderer eviction before terminal callbacks can retry. The relay policy test maps a standard repository's base, Git common directory, and worktree roots to one healthy supervisor, then proves a shared-child failure recovers those roots in separate quarantine supervisors with overflow and resumed events. A built-entry macOS harness drives the real supervisor, kills its child with SIGSEGV, then requires automatic resubscription and a later event. SSH bundles the same child boundary beside relay.js, requires both artifacts for install completeness, and preserves registration-owned same-root cancellation. A built-relay harness kills that child and requires the original relay PID and existing PTY to survive before overflow and precise events resume.",
       "motivatingLinks": [
         "https://github.com/stablyai/orca/issues/5308",
         "https://github.com/stablyai/orca/issues/7547",
         "https://github.com/stablyai/orca/issues/8212"
       ],
-      "invariant": "Local runtime and SSH relay roots use at most four shared watcher children while healthy and at most four additional quarantine children after shard failure. Cancelling queued setup removes only that request; cancelling an active crawl restarts one shard and restores its healthy records. Setup deadlines start only when the child owns the crawl; the runtime RPC transport owns the normal 15-second initial budget, while a 60-second recovery deadline quarantines a root that would otherwise pin its shard. A crash-fused shared shard moves affected roots into the bounded quarantine pool, and a second fused quarantine ends those watch lifetimes instead of spawning replacements indefinitely. Every interrupted setup emits overflow after recovery, event delivery stays bounded, and the SSH relay, PTYs, Git, and other filesystem operations never load the watcher native addon or share its fault boundary.",
-      "oracle": "Fill the four-child healthy pool with multiple roots, inject IPC disconnect-before-exit, queued and active setup cancellation, initial and crash-resubscribe timeouts, and shard-level terminal failure, then assert healthy roots are restored, affected roots spread across no more than four quarantine supervisors, a second quarantine fuse creates no replacement, stale generations stop delivering, and existing same-root clients receive overflow plus later events without error/end. Abort a pending desktop install and require worktree close and app shutdown to settle before the native crawl resolves. Verify relay build hashes and remote install probes require both relay.js and relay-watcher.js. Launch the built relay with a real PTY and Parcel watch, kill only the watcher child, and require the same relay PID and PTY to answer before a replacement child delivers overflow and a post-crash event.",
+      "invariant": "Local runtime roots use at most four shared watcher children while healthy, while each SSH relay process uses at most one; either pool uses at most four additional quarantine children after shard failure. Cancelling queued setup removes only that request; cancelling an active crawl restarts one shard and restores its healthy records. Setup deadlines start only when the child owns the crawl; the runtime RPC transport owns the normal 15-second initial budget, while a 60-second recovery deadline quarantines a root that would otherwise pin its shard. A crash-fused shared shard moves affected roots into the bounded quarantine pool, and a second fused quarantine ends those watch lifetimes instead of spawning replacements indefinitely. Every interrupted setup emits overflow after recovery, event delivery stays bounded, and each SSH target's relay, PTYs, Git, and other filesystem operations remain isolated from other targets and never load the watcher native addon or share its fault boundary.",
+      "oracle": "Fill the four-child healthy runtime pool with multiple roots, and map a standard SSH repository's base, Git common directory, and worktree roots to one relay supervisor. Inject IPC disconnect-before-exit, queued and active setup cancellation, initial and crash-resubscribe timeouts, and shard-level terminal failure, then assert healthy roots are restored, affected roots spread across no more than four quarantine supervisors, a second quarantine fuse creates no replacement, stale generations stop delivering, and existing same-root clients receive overflow plus later events without error/end. Abort a pending desktop install and require worktree close and app shutdown to settle before the native crawl resolves. Verify relay build hashes and remote install probes require both relay.js and relay-watcher.js. Launch the built relay with a real PTY and Parcel watch, kill only the watcher child, and require the same relay PID and PTY to answer before a replacement child delivers overflow and a post-crash event.",
       "commands": [
         "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/parcel-watcher-process.test.ts src/main/ipc/runtime-watcher-process-pool.test.ts src/main/ipc/parcel-watcher-process-entry.test.ts src/main/ipc/parcel-watcher-event-delivery.test.ts src/main/ipc/filesystem-watcher-local-unsubscribe.test.ts src/main/ipc/filesystem-watcher-wsl.test.ts src/main/runtime/file-watcher-host.test.ts src/main/runtime/rpc/methods/files.test.ts src/main/providers/ssh-filesystem-provider.test.ts src/renderer/src/runtime/runtime-file-client.test.ts",
         "pnpm run build:electron-vite",
@@ -4043,6 +4043,8 @@
         {
           "file": "src/relay/relay-filesystem-watch-registry.test.ts",
           "assertions": [
+            "each SSH host keeps its own healthy watcher supervisor while a standard repository's three distinct roots share that one child",
+            "shared-child failure moves those roots into separate quarantine supervisors with overflow and resumed events",
             "replacement emits overflow before later ordered events",
             "terminal shard failure preserves same-root clients through recovery",
             "pending setup aborts only after the last same-root client leaves",
@@ -4155,11 +4157,11 @@
       },
       "redGreenEvidence": {
         "status": "partial",
-        "evidence": "Red tests reproduced the reviewed failures: IPC disconnect-before-exit orphaned existing roots, queue-admission deadlines cascaded shard replacement, a fused shard created one child per root, destructive desktop cleanup waited indefinitely for native setup, and synchronous renderer retry joined a terminal stream. Earlier red coverage also reproduced missing-entry fail-open, terminal recovery loss, stale same-root lease deletion, and unbounded per-child stat lanes. The bounded quarantine and cancellation rewrite passes those contracts, and the real-supervisor built-entry harness survives SIGSEGV with automatic replacement. CI/soak history is still absent."
+        "evidence": "Red tests reproduced the reviewed failures: IPC disconnect-before-exit orphaned existing roots, queue-admission deadlines cascaded shard replacement, a fused shard created one child per root, a standard SSH repository created three healthy watcher supervisors, destructive desktop cleanup waited indefinitely for native setup, and synchronous renderer retry joined a terminal stream. Earlier red coverage also reproduced missing-entry fail-open, terminal recovery loss, stale same-root lease deletion, and unbounded per-child stat lanes. The bounded quarantine and cancellation rewrite passes those contracts, the relay-specific healthy width test reduces the three-root case to one supervisor, and the real-supervisor built-entry harness survives SIGSEGV with automatic replacement. CI/soak history is still absent."
       },
       "performanceBudget": {
         "required": true,
-        "evidence": "Healthy operation is capped at four shared child processes instead of one child per root; based on the prior 51.9 MiB one-child measurement this bounds the unremeasured eight-root healthy ceiling near 207.6 MiB rather than the rejected 388.7 MiB per-root design. Degraded operation adds at most four quarantine children, distributes affected roots by least load, disposes empty slots, and permits no replacement after a quarantine shard itself fuses. Each child caps directory stats at eight across its assigned roots (at most 32 healthy or 64 total logical stat operations), each subscription retains at most one active plus one bounded pending batch, and no polling or serve-process stat fanout is added."
+        "evidence": "Local runtime operation remains capped at four shared child processes, while each SSH relay is capped at one healthy child instead of spreading a standard repository's three roots across three children. Based on the measured 51.9 MiB per child, normal SSH watcher overhead is bounded near 51.9 MiB rather than 155.7 MiB before relay and PTY overhead. Degraded operation adds at most four quarantine children, distributes affected roots by least load, disposes empty slots, and permits no replacement after a quarantine shard itself fuses. Each child caps directory stats at eight across its assigned roots (at most 32 healthy or 64 total logical stat operations for local runtime; 8 healthy or 40 total per SSH relay), each subscription retains at most one active plus one bounded pending batch, and no polling or serve-process stat fanout is added."
       },
       "promotionCriteria": [
         "Run the built-entry SIGSEGV harness in soak for at least 100 consecutive passes or 14 days on macOS.",
@@ -4168,11 +4170,11 @@
       ],
       "knownGaps": [
         "The fault harness bundles the real source supervisor and drives the built child, but it does not launch a packaged orca serve pairing session or exercise app.asar path resolution.",
-        "The four-child healthy and four-child quarantine memory ceilings are inferred from the earlier one-child measurement; direct degraded-state RSS plus sustained six-writer event-loop-delay and CPU soak evidence is still absent.",
+        "The one-child SSH healthy and four-child quarantine memory ceilings are inferred from the earlier one-child measurement; direct degraded-state RSS plus sustained six-writer event-loop-delay and CPU soak evidence is still absent.",
         "Linux native watcher fault containment is source-equivalent but not live fault-injected in this evidence run.",
         "Windows runtime watches bypass this path; SSH registration ownership is deterministic-contract tested but not live-relay fault-injected."
       ],
-      "demotionRule": "Demote or quarantine if the fault harness flakes without a product or harness bug, if healthy operation exceeds four runtime watcher children, if total healthy plus quarantine operation exceeds eight, if quarantine children outlive their roots or repeat after fusing, if event delivery becomes unbounded, or if metadata/stat work returns to the serve process."
+      "demotionRule": "Demote or quarantine if the fault harness flakes without a product or harness bug, if healthy operation exceeds four local runtime watcher children or one child in any SSH relay, if total healthy plus quarantine operation exceeds eight locally or five in an SSH relay, if quarantine children outlive their roots or repeat after fusing, if event delivery becomes unbounded, or if metadata/stat work returns to the serve process."
     }
   ]
 }

src/relay/relay-filesystem-watch-registry.test.ts

@@ -1,7 +1,8 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 import { join } from 'node:path'
 import { tmpdir } from 'node:os'
 import { WatcherProcessFailure } from '../main/ipc/parcel-watcher-process-failure'
+import { WatcherProcessSupervisor } from '../main/ipc/parcel-watcher-process-supervisor'
 import type {
   WatcherProcessCallback,
   WatcherProcessHooks,
@@ -17,6 +18,29 @@ type InstalledWatch = {
   unsubscribe: ReturnType<typeof vi.fn<() => Promise<void>>>
 }
 
+type InstalledSupervisorWatch = InstalledWatch & { dir: string }
+
+function stubRelayWatcherSupervisors() {
+  const installed = new Map<WatcherProcessSupervisor, InstalledSupervisorWatch[]>()
+  vi.spyOn(WatcherProcessSupervisor.prototype, 'subscribe').mockImplementation(function (
+    this: WatcherProcessSupervisor,
+    dir,
+    callback,
+    _options,
+    hooks = {}
+  ) {
+    const watches = installed.get(this) ?? []
+    const unsubscribe = vi.fn(async () => undefined)
+    watches.push({ dir, callback, hooks, unsubscribe })
+    installed.set(this, watches)
+    return Promise.resolve({ unsubscribe })
+  })
+  const dispose = vi
+    .spyOn(WatcherProcessSupervisor.prototype, 'dispose')
+    .mockImplementation(() => undefined)
+  return { installed, dispose }
+}
+
 class FakeWatcherPool {
   readonly installed: InstalledWatch[] = []
   readonly dispose = vi.fn()
@@ -148,6 +172,81 @@ describe('RelayFilesystemWatchRegistry', () => {
 })
 
 describe('createRelayWatcherProcessPool', () => {
+  afterEach(() => vi.restoreAllMocks())
+
+  it('uses one healthy watcher process per SSH host for each standard repo', async () => {
+    const { installed } = stubRelayWatcherSupervisors()
+    const firstHostPool = createRelayWatcherProcessPool()
+    const secondHostPool = createRelayWatcherProcessPool()
+    const repoBase = join(tmpdir(), 'ssh-repo')
+    const roots = [repoBase, join(repoBase, '.git'), join(repoBase, 'worktree')]
+    try {
+      for (const root of roots) {
+        await firstHostPool.subscribe(root, vi.fn(), {}, {})
+        await secondHostPool.subscribe(root, vi.fn(), {}, {})
+      }
+
+      expect(
+        Array.from(installed.values()).map((watches) => watches.map(({ dir }) => dir))
+      ).toEqual([roots, roots])
+    } finally {
+      firstHostPool.dispose()
+      secondHostPool.dispose()
+    }
+  })
+
+  it('quarantines and recovers each standard-repo root after the shared child fails', async () => {
+    const { installed, dispose } = stubRelayWatcherSupervisors()
+    const pool = createRelayWatcherProcessPool()
+    const dispatcher = createDispatcher()
+    const registry = new RelayFilesystemWatchRegistry(
+      dispatcher as unknown as RelayDispatcher,
+      pool
+    )
+    const repoBase = join(tmpdir(), 'ssh-repo-recovery')
+    const roots = [repoBase, join(repoBase, '.git'), join(repoBase, 'worktree')]
+    try {
+      for (const root of roots) {
+        await registry.watch(root, context(1))
+      }
+      const healthyWatches = Array.from(installed.values())[0]
+      const failure = new WatcherProcessFailure(
+        'file watcher process crashed repeatedly',
+        'supervisor',
+        'supervisor_crash_fuse'
+      )
+
+      for (const watch of healthyWatches) {
+        watch.hooks.onTerminalError?.(failure)
+      }
+      await Promise.resolve()
+
+      const recoveredWatches = Array.from(installed.values()).slice(1)
+      expect(dispose).toHaveBeenCalledTimes(1)
+      expect(recoveredWatches.map((watches) => watches.map(({ dir }) => dir))).toEqual(
+        roots.map((root) => [root])
+      )
+      expect(dispatcher.notify.mock.calls.slice(0, roots.length)).toEqual(
+        roots.map((root) => ['fs.changed', { events: [{ kind: 'overflow', absolutePath: root }] }])
+      )
+
+      for (const watches of recoveredWatches) {
+        const [{ callback, dir }] = watches
+        callback(null, [{ type: 'update', path: join(dir, 'recovered.txt') }])
+      }
+      expect(dispatcher.notify.mock.calls.slice(roots.length)).toEqual(
+        roots.map((root) => [
+          'fs.changed',
+          {
+            events: [{ kind: 'update', absolutePath: join(root, 'recovered.txt') }]
+          }
+        ])
+      )
+    } finally {
+      registry.dispose()
+    }
+  })
+
   it('fails closed instead of loading the native watcher in the relay process', async () => {
     const previousVitest = process.env.VITEST
     process.env.VITEST = 'true'

src/relay/relay-watcher-process-pool.ts

@@ -15,6 +15,9 @@ export function createRelayWatcherProcessPool(
   entryPath = getRelayWatcherProcessEntryPath()
 ): RelayWatcherProcessPool {
   return new RuntimeWatcherProcessPool({
+    // Why: one SSH workspace normally installs three roots, and each remote
+    // watcher child carries substantial Node/native-addon process overhead.
+    maxSharedSupervisors: 1,
     createSupervisor: () =>
       new WatcherProcessSupervisor({
         entryPath,