Fix ssh watcher isolation (#8463)

* fix(ssh): isolate relay filesystem watchers

* Fix relay watcher fault-harness pid file and in-process fallback isolati

- Use exclusive ('wx') creation for the fault-harness pid file so a leaked
  ORCA_WATCHER_CHILD_PID_FILE env var can't clobber an existing file, and
  have the harness remove the file after reading a replacement pid.
- Force useInProcessVitestFallback to false in the relay watcher pool so a
  leaked VITEST env var can never load the native watcher addon in-process
  on the relay; fail closed instead when the isolated child is missing.
- Thread an injectable RelayWatcherProcessPool into FsHandler/
  RelayFilesystemWatchRegistry for tests, and add coverage for both fixes.

src/main/ipc/parcel-watcher-child-launch.ts

@@ -1,4 +1,5 @@
 import { fork, type ChildProcess } from 'node:child_process'
+import { writeFileSync } from 'node:fs'
 import {
   createWatcherCanaryDirectory,
   removeWatcherCanaryDirectory
@@ -33,6 +34,16 @@ export function launchWatcherChild(
     console.error('[parcel-watcher-process] failed to fork watcher process:', error)
     return null
   }
+  const faultHarnessPidFile = process.env.ORCA_WATCHER_CHILD_PID_FILE
+  if (faultHarnessPidFile && child.pid) {
+    try {
+      // Why: exclusive creation lets the harness identify the child without a
+      // leaked test-only environment variable clobbering an existing file.
+      writeFileSync(faultHarnessPidFile, String(child.pid), { flag: 'wx' })
+    } catch {
+      // Fault-injection observability must never affect watcher availability.
+    }
+  }
   child.stderr?.on('data', (chunk: Buffer) => {
     console.error('[parcel-watcher-process]', String(chunk).trimEnd())
   })

src/main/ipc/parcel-watcher-host-subscriptions.ts

@@ -100,6 +100,21 @@ export function resolvePendingWatcherUnsubscribes(
   pendingUnsubscribes.clear()
 }
 
+export function disposeWatcherSupervisorSubscriptions(
+  records: Map<number, WatcherProcessSubscriptionRecord>,
+  pendingUnsubscribes: Map<number, () => void>,
+  cancelledSubscribesAwaitingChild: Set<number>,
+  error: Error
+): void {
+  for (const record of records.values()) {
+    resetPendingSubscribeAttempt(record)
+    takePendingSubscribe(record)?.reject(error)
+  }
+  resolvePendingWatcherUnsubscribes(pendingUnsubscribes)
+  cancelledSubscribesAwaitingChild.clear()
+  records.clear()
+}
+
 export type CreateHostWatcherSubscriptionOptions = {
   record: WatcherProcessSubscriptionRecord
   records: Map<number, WatcherProcessSubscriptionRecord>

src/main/ipc/parcel-watcher-process-failure.ts

@@ -20,6 +20,13 @@ export class WatcherProcessFailure extends Error {
   }
 }
 
+export function watcherHostFailure(
+  message: string,
+  code: WatcherProcessFailureCode
+): WatcherProcessFailure {
+  return new WatcherProcessFailure(message, 'supervisor', code)
+}
+
 export function isWatcherProcessFailure(error: unknown): error is WatcherProcessFailure {
   return error instanceof WatcherProcessFailure
 }

src/main/ipc/parcel-watcher-process-supervisor-options.ts

@@ -0,0 +1,4 @@
+export type WatcherProcessSupervisorOptions = {
+  entryPath?: string
+  useInProcessVitestFallback?: boolean
+}

src/main/ipc/parcel-watcher-process-supervisor.ts

@@ -5,6 +5,7 @@ import { removeWatcherCanaryDirectory } from './parcel-watcher-canary-directory'
 import { launchWatcherChild } from './parcel-watcher-child-launch'
 import { WatcherProcessCrashFuse } from './parcel-watcher-crash-fuse'
 import {
+  disposeWatcherSupervisorSubscriptions,
   failAllWatcherSubscriptions,
   handleWatcherHostMessage,
   reportWatcherTerminalError,
@@ -16,7 +17,8 @@ import {
   startPendingSubscribeTimeout,
   takePendingSubscribe
 } from './parcel-watcher-pending-subscribe'
-import { WatcherProcessFailure } from './parcel-watcher-process-failure'
+import { watcherHostFailure } from './parcel-watcher-process-failure'
+import type { WatcherProcessFailure } from './parcel-watcher-process-failure'
 import type {
   HostToWatcherMessage,
   WatcherProcessSubscribeOptions,
@@ -28,6 +30,7 @@ import type {
   WatcherProcessSubscription,
   WatcherProcessSubscriptionRecord
 } from './parcel-watcher-process-subscription'
+import type { WatcherProcessSupervisorOptions } from './parcel-watcher-process-supervisor-options'
 import { subscribeThroughWatcherSupervisor } from './parcel-watcher-supervisor-subscribe'
 
 export type {
@@ -35,6 +38,7 @@ export type {
   WatcherProcessHooks,
   WatcherProcessSubscription
 } from './parcel-watcher-process-subscription'
+export type { WatcherProcessSupervisorOptions } from './parcel-watcher-process-supervisor-options'
 
 export class WatcherProcessSupervisor {
   private child: ChildProcess | null = null
@@ -45,6 +49,9 @@ export class WatcherProcessSupervisor {
   private readonly records = new Map<number, WatcherProcessSubscriptionRecord>()
   private readonly pendingUnsubscribes = new Map<number, () => void>()
   private readonly cancelledSubscribesAwaitingChild = new Set<number>()
+
+  constructor(private readonly options: WatcherProcessSupervisorOptions = {}) {}
+
   subscribe(
     dir: string,
     callback: WatcherProcessCallback,
@@ -57,6 +64,8 @@ export class WatcherProcessSupervisor {
       opts,
       hooks,
       shutdownRequested: this.shutdownRequested,
+      entryPath: this.options.entryPath ?? getWatcherProcessEntryPath(),
+      useInProcessVitestFallback: this.options.useInProcessVitestFallback ?? true,
       allocateId: () => this.nextSubscriptionId++,
       records: this.records,
       pendingUnsubscribes: this.pendingUnsubscribes,
@@ -73,19 +82,13 @@ export class WatcherProcessSupervisor {
     this.shutdownRequested = true
     const proc = this.child
     this.child = null
-    const error = new WatcherProcessFailure(
-      'file watcher supervisor disposed',
-      'supervisor',
-      'supervisor_disposed'
+    const error = watcherHostFailure('file watcher supervisor disposed', 'supervisor_disposed')
+    disposeWatcherSupervisorSubscriptions(
+      this.records,
+      this.pendingUnsubscribes,
+      this.cancelledSubscribesAwaitingChild,
+      error
     )
-    for (const record of this.records.values()) {
-      resetPendingSubscribeAttempt(record)
-      const pending = takePendingSubscribe(record)
-      pending?.reject(error)
-    }
-    resolvePendingWatcherUnsubscribes(this.pendingUnsubscribes)
-    this.cancelledSubscribesAwaitingChild.clear()
-    this.records.clear()
     proc?.kill()
     this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir)
   }
@@ -96,7 +99,9 @@ export class WatcherProcessSupervisor {
     this.crashFuse.reset()
   }
 
-  private ensureWatcherProcess(entryPath = getWatcherProcessEntryPath()): ChildProcess | null {
+  private ensureWatcherProcess(
+    entryPath = this.options.entryPath ?? getWatcherProcessEntryPath()
+  ): ChildProcess | null {
     if (this.shutdownRequested) {
       return null
     }
@@ -200,11 +205,7 @@ export class WatcherProcessSupervisor {
       )
       failAllWatcherSubscriptions(
         this.records,
-        new WatcherProcessFailure(
-          'file watcher process crashed repeatedly',
-          'supervisor',
-          'supervisor_crash_fuse'
-        )
+        watcherHostFailure('file watcher process crashed repeatedly', 'supervisor_crash_fuse')
       )
       this.canaryDir = removeWatcherCanaryDirectory(this.canaryDir)
       return
@@ -279,9 +280,8 @@ export class WatcherProcessSupervisor {
     if (!replacement) {
       failAllWatcherSubscriptions(
         this.records,
-        new WatcherProcessFailure(
+        watcherHostFailure(
           'file watcher process unavailable after subscription cancellation',
-          'supervisor',
           'process_unavailable'
         )
       )

src/main/ipc/parcel-watcher-process.test.ts

@@ -4,21 +4,28 @@
 import { EventEmitter } from 'node:events'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
-const { forkMock, existsSyncMock, mkdtempSyncMock, parcelSubscribeMock, rmSyncMock } = vi.hoisted(
-  () => ({
-    forkMock: vi.fn(),
-    existsSyncMock: vi.fn(),
-    mkdtempSyncMock: vi.fn(() => '/tmp/orca-watcher-canary-supervisor-test'),
-    parcelSubscribeMock: vi.fn(),
-    rmSyncMock: vi.fn()
-  })
-)
+const {
+  forkMock,
+  existsSyncMock,
+  mkdtempSyncMock,
+  parcelSubscribeMock,
+  rmSyncMock,
+  writeFileSyncMock
+} = vi.hoisted(() => ({
+  forkMock: vi.fn(),
+  existsSyncMock: vi.fn(),
+  mkdtempSyncMock: vi.fn(() => '/tmp/orca-watcher-canary-supervisor-test'),
+  parcelSubscribeMock: vi.fn(),
+  rmSyncMock: vi.fn(),
+  writeFileSyncMock: vi.fn()
+}))
 
 vi.mock('node:child_process', () => ({ fork: forkMock }))
 vi.mock('node:fs', () => ({
   existsSync: existsSyncMock,
   mkdtempSync: mkdtempSyncMock,
-  rmSync: rmSyncMock
+  rmSync: rmSyncMock,
+  writeFileSync: writeFileSyncMock
 }))
 vi.mock('@parcel/watcher', () => ({ subscribe: parcelSubscribeMock }))
 
@@ -35,6 +42,7 @@ type SentMessage = { op: string; id: number; dir?: string }
 
 class FakeChild extends EventEmitter {
   connected = true
+  pid = 1234
   sent: SentMessage[] = []
   stderr = new EventEmitter()
   kill = vi.fn(() => {
@@ -94,6 +102,19 @@ describe('subscribeViaWatcherProcess', () => {
     expect(callback).toHaveBeenCalledWith(null, events)
   })
 
+  it('creates the fault-harness pid file without clobbering an existing path', async () => {
+    vi.stubEnv('ORCA_WATCHER_CHILD_PID_FILE', '/tmp/orca-watcher.pid')
+
+    const promise = subscribeViaWatcherProcess('/repo', vi.fn(), {})
+    const child = currentChild()
+
+    expect(writeFileSyncMock).toHaveBeenCalledWith('/tmp/orca-watcher.pid', '1234', {
+      flag: 'wx'
+    })
+    ackSubscribe(child)
+    await promise
+  })
+
   it('forwards watcher errors to the callback', async () => {
     const callback = vi.fn()
     const promise = subscribeViaWatcherProcess('/repo', callback, {})

src/main/ipc/parcel-watcher-supervisor-subscribe.ts

@@ -1,6 +1,5 @@
 import type { ChildProcess } from 'node:child_process'
 import { existsSync } from 'node:fs'
-import { getWatcherProcessEntryPath } from './parcel-watcher-entry-path'
 import { createHostWatcherSubscription } from './parcel-watcher-host-subscriptions'
 import { subscribeWithInProcessWatcher } from './parcel-watcher-in-process-fallback'
 import { installPendingSubscribeControls } from './parcel-watcher-pending-subscribe'
@@ -22,6 +21,8 @@ type WatcherSupervisorSubscribeOptions = {
   opts: WatcherProcessSubscribeOptions
   hooks: WatcherProcessHooks
   shutdownRequested: boolean
+  entryPath: string
+  useInProcessVitestFallback: boolean
   allocateId: () => number
   records: Map<number, WatcherProcessSubscriptionRecord>
   pendingUnsubscribes: Map<number, () => void>
@@ -42,6 +43,8 @@ export function subscribeThroughWatcherSupervisor({
   opts,
   hooks,
   shutdownRequested,
+  entryPath,
+  useInProcessVitestFallback,
   allocateId,
   records,
   pendingUnsubscribes,
@@ -72,10 +75,9 @@ export function subscribeThroughWatcherSupervisor({
   }
   // Why: under Vitest we cannot fork a real watcher child, so exercise the
   // subscription path in-process (against mocked @parcel/watcher) instead.
-  if (process.env.VITEST) {
+  if (process.env.VITEST && useInProcessVitestFallback) {
     return subscribeWithInProcessWatcher(dir, callback, opts, hooks)
   }
-  const entryPath = getWatcherProcessEntryPath()
   if (!existsSync(entryPath)) {
     return Promise.reject(
       new WatcherProcessFailure(

src/main/ssh/ssh-relay-versioned-install.test.ts

@@ -108,11 +108,12 @@ describe('isRelayAlreadyInstalled', () => {
     ).rejects.toBe(sessionLimitError)
   })
 
-  it('checks for relay.js AND .install-complete in addition to the dir', async () => {
+  it('checks for both relay process artifacts and .install-complete', async () => {
     mockExec.mockResolvedValueOnce('OK')
     await isRelayAlreadyInstalled(conn, '/r')
     const cmd = mockExec.mock.calls.at(-1)?.[1] ?? ''
     expect(cmd).toContain('relay.js')
+    expect(cmd).toContain('relay-watcher.js')
     expect(cmd).toContain('.install-complete')
   })
 })

src/main/ssh/ssh-relay-versioned-install.ts

@@ -126,10 +126,9 @@ export function computeRemoteRelayDir(
 /**
  * Probe whether a fully-installed relay already exists at remoteRelayDir.
  *
- * "Fully installed" means: the directory exists, contains relay.js, AND
- * contains the .install-complete sentinel written at the end of a successful
- * install. A directory missing .install-complete is either mid-install (lock
- * held) or a crashed-install partial — either way we re-run the deploy.
+ * "Fully installed" means: the directory contains relay.js, its isolated
+ * relay-watcher.js child, and the .install-complete sentinel written at the
+ * end of a successful install. Missing artifacts force a complete re-deploy.
  */
 export async function isRelayAlreadyInstalled(
   conn: SshConnection,

src/main/ssh/ssh-remote-commands.ts

@@ -59,11 +59,13 @@ export function probeRelayInstalledCommand(
   remoteRelayDir: string
 ): string {
   const relayJs = joinRemotePath(host, remoteRelayDir, 'relay.js')
+  const relayWatcherJs = joinRemotePath(host, remoteRelayDir, 'relay-watcher.js')
   const installComplete = joinRemotePath(host, remoteRelayDir, '.install-complete')
   if (!isWindowsRemoteHost(host)) {
     return (
       `test -d ${shellEscape(remoteRelayDir)} ` +
       `&& test -f ${shellEscape(relayJs)} ` +
+      `&& test -f ${shellEscape(relayWatcherJs)} ` +
       `&& test -f ${shellEscape(installComplete)} ` +
       `&& echo OK || echo MISSING`
     )
@@ -72,8 +74,9 @@ export function probeRelayInstalledCommand(
     [
       `$dir = ${powerShellLiteral(remoteRelayDir)}`,
       `$relay = ${powerShellLiteral(relayJs)}`,
+      `$watcher = ${powerShellLiteral(relayWatcherJs)}`,
       `$complete = ${powerShellLiteral(installComplete)}`,
-      "if ((Test-Path -LiteralPath $dir -PathType Container) -and (Test-Path -LiteralPath $relay -PathType Leaf) -and (Test-Path -LiteralPath $complete -PathType Leaf)) { 'OK' } else { 'MISSING' }"
+      "if ((Test-Path -LiteralPath $dir -PathType Container) -and (Test-Path -LiteralPath $relay -PathType Leaf) -and (Test-Path -LiteralPath $watcher -PathType Leaf) -and (Test-Path -LiteralPath $complete -PathType Leaf)) { 'OK' } else { 'MISSING' }"
     ].join('; ')
   )
 }