Files
orca/config/scripts/runtime-file-watcher-fault-harness.mjs
Jinjing 433df4be3c fix(runtime): prevent file watcher SIGSEGV from crashing orca serve (#8370)
* Fix crash-isolated file watcher process pool for orca-serve SIGSEGV afte

Replace the worker-thread runtime file watcher with a forked, crash-isolated
@parcel/watcher child process pool so a native FSEvents fault can no longer
take down the main/serve process, and add bounded event batching, delivery
backpressure, and quarantine-based recovery for faulty watch roots.

* Fix crash-isolated file watcher teardown and shutdown leaks

- Fault harness could throw before mkdtemp/realpath completed, skipping
  cleanup; now tracks each temp path independently and races an async
  watcher-callback error so it can't escape the try/finally unhandled.
- In-process fallback swallowed unsubscribe failures via a bare rejection
  handler that could still throw; use .catch() instead.
- Watcher process entry's cancel-subscribe handler now reuses the async
  unsubscribe path when a crawl already finished, releasing the native
  handle instead of leaking it (blocks worktree unlock on Windows).
- Runtime watcher process pool exposed no real dispose(); shutdown now
  kills pooled children so they don't outlive the main process.

* Fix disposeSlot double-iteration bug in file watcher pool teardown

Remove the unnecessary array snapshot in dispose(): disposeSlot mutates
allSlots by deleting the slot being visited, and deleting the
in-progress element during Set iteration is well-defined, so the spread
copy was dead weight left over from prior debugging.

* Fix pending file watcher installs not aborting on unsubscribe

- Local/WSL watcher installs and SSH fs.watch setup now honor the
  in-flight AbortSignal, so the last unwatch cancels a slow native
  subscribe or remote setup instead of waiting for it to finish.
- Thread signal through IFilesystemProvider.watch and SSH-backed
  file explorer watches for the same early-cancel behavior.

* Fix crash-resubscribe hangs and SSH watch teardown races in file watcher

- Add a bounded deadline for post-crash resubscription crawls so one
  stuck root quarantines instead of pinning its whole shard forever.
- Report FSEvents overflow as recoverable so delivery continues after
  a dropped-events error instead of surfacing as terminal.
- Make WSL watcher abort errors real DOMException instances so
  AbortSignal-based cancellation checks recognize them.
- Rework SSH watch registration so ownership of the shared setup
  request (not just the first caller) decides teardown, preventing
  one caller's abort from cancelling another's shared watch and
  guaranteeing exactly one fs.unwatch per registration.
- Reformat reliability-gates.jsonc arrays and refresh WSL/SSH coverage
  entries and evidence runs to match the above.

* Add CI gate to run the file-watcher SIGSEGV fault harness under Electron

- The reliability gate and release workflows (mac, Linux) previously only
  exercised the crash-isolation harness under vanilla Node, which doesn't
  catch runtime differences in the actual Electron binary that ships to
  users.
- Adds an `ELECTRON_RUN_AS_NODE=1 pnpm exec electron ...` run of the same
  harness alongside the existing Node run, so #8212's SIGSEGV-survival
  contract is proven against both runtimes before packaging.

* Add CI gate blocking Linux/macOS release packaging on watcher fault reco

Adds a contract test asserting release-cut.yml and release-mac-build.yml
run the runtime-file-watcher-fault-harness after building and before
publishing artifacts, so a regression in watcher process fault recovery
fails release packaging instead of shipping silently.

* Fix use-after-clear crash in failAllWatcherSubscriptions

Snapshot the records map before iterating, since onTerminalError
hooks can dispose the supervisor and clear `records` mid-loop,
causing a crash. Also update the matching test to assert against
the shared buildParcelWatcherIgnoreOptions helper instead of a
loose arrayContaining match.

* Fix use-after-clear crash in failAllWatcherSubscriptions

Snapshot watcher records with Array.from instead of spread, since
spread syntax over an iterator that's mutated mid-loop by
onTerminalError hooks can produce inconsistent results.
2026-07-12 18:15:15 -07:00

181 lines
5.6 KiB
JavaScript

import { createRequire } from 'node:module'
import { existsSync } from 'node:fs'
import { mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import { build } from 'esbuild'
const ENTRY_PATH = resolve('out/main/parcel-watcher-process-entry.js')
const SUPERVISOR_SOURCE = resolve('src/main/ipc/parcel-watcher-process-supervisor.ts')
const WAIT_TIMEOUT_MS = 15_000
const require = createRequire(import.meta.url)
function withTimeout(promise, label) {
return new Promise((resolvePromise, rejectPromise) => {
const timer = setTimeout(
() => rejectPromise(new Error(`Timed out waiting for ${label}`)),
WAIT_TIMEOUT_MS
)
promise.then(
(value) => {
clearTimeout(timer)
resolvePromise(value)
},
(error) => {
clearTimeout(timer)
rejectPromise(error)
}
)
})
}
function nextMatchingEvent(register, predicate, label) {
return withTimeout(
new Promise((resolveEvent) => {
register((events) => {
if (events.some(predicate)) {
resolveEvent(events)
}
})
}),
label
)
}
async function loadSupervisor(bundleDir) {
const outfile = join(bundleDir, 'watcher-supervisor.cjs')
await build({
entryPoints: [SUPERVISOR_SOURCE],
bundle: true,
platform: 'node',
format: 'cjs',
outfile,
external: ['@parcel/watcher', 'electron'],
logLevel: 'silent'
})
return require(outfile).WatcherProcessSupervisor
}
async function main() {
if (process.platform === 'win32') {
console.log('[runtime-file-watcher-fault] SKIP: SIGSEGV oracle is macOS/Linux only')
return
}
if (!existsSync(ENTRY_PATH)) {
throw new Error(`Missing ${ENTRY_PATH}; run pnpm run build:electron-vite first`)
}
// Why: mkdtemp/realpath/bundle/construction can fail before the body runs.
// Keep cleanup in finally from the first successful mkdtemp onward, and clean
// the original temp path if realpath never succeeds.
let createdRootPath
let bundleDir
let rootPath
let supervisor
let subscription
let watcherCanaryDir
let eventListener = () => undefined
let rejectWatcherError
const watcherError = new Promise((_, reject) => {
rejectWatcherError = reject
})
// Attach early so a callback rejection before the race cannot become unhandled.
watcherError.catch(() => undefined)
try {
createdRootPath = await mkdtemp(join(tmpdir(), 'orca-runtime-watcher-fault-'))
bundleDir = await mkdtemp(join(tmpdir(), 'orca-runtime-watcher-harness-'))
// Parcel reports canonical event paths on macOS, where tmpdir() may use the
// /var symlink spelling. Keep the oracle in the same path domain.
rootPath = await realpath(createdRootPath)
const WatcherProcessSupervisor = await loadSupervisor(bundleDir)
supervisor = new WatcherProcessSupervisor()
let resolveInterruption
const interrupted = withTimeout(
new Promise((resolveWait) => {
resolveInterruption = resolveWait
}),
'automatic watcher resubscription'
)
subscription = await supervisor.subscribe(
rootPath,
(error, events) => {
if (error) {
// Why: throws from the async watcher callback escape main()'s try/finally
// and skip teardown. Surface failures through a harness promise instead.
rejectWatcherError(error)
return
}
eventListener(events)
},
{ ignore: ['.git', 'node_modules'] },
{
delivery: { includeDirectoryMetadata: true, maxEventsPerBatch: 200 },
onInterruption: () => resolveInterruption()
}
)
watcherCanaryDir = supervisor.canaryDir
const beforeEvent = nextMatchingEvent(
(listener) => {
eventListener = listener
},
(event) => event.path === join(rootPath, 'before.txt'),
'pre-crash watch event'
)
await writeFile(join(rootPath, 'before.txt'), 'before')
await Promise.race([beforeEvent, watcherError])
const firstChildPid = supervisor.child?.pid
if (!firstChildPid) {
throw new Error('Watcher supervisor did not expose a live child')
}
process.kill(firstChildPid, 'SIGSEGV')
await Promise.race([interrupted, watcherError])
const replacementChildPid = supervisor.child?.pid
if (!replacementChildPid || replacementChildPid === firstChildPid) {
throw new Error('Watcher supervisor did not replace the faulted child')
}
const afterEvent = nextMatchingEvent(
(listener) => {
eventListener = listener
},
(event) => event.path === join(rootPath, 'after.txt'),
'post-crash watch event'
)
await writeFile(join(rootPath, 'after.txt'), 'after')
await Promise.race([afterEvent, watcherError])
console.log(
JSON.stringify({
hostPid: process.pid,
killedWatcherPid: firstChildPid,
replacementWatcherPid: replacementChildPid,
hostSurvived: true,
automaticResubscribe: true,
postCrashEventDelivered: true
})
)
} finally {
try {
await subscription?.unsubscribe()
} finally {
supervisor?.dispose()
await Promise.all([
createdRootPath ? rm(createdRootPath, { recursive: true, force: true }) : Promise.resolve(),
rootPath && rootPath !== createdRootPath
? rm(rootPath, { recursive: true, force: true })
: Promise.resolve(),
bundleDir ? rm(bundleDir, { recursive: true, force: true }) : Promise.resolve()
])
}
}
if (watcherCanaryDir && existsSync(watcherCanaryDir)) {
throw new Error(`Watcher supervisor leaked its canary directory: ${watcherCanaryDir}`)
}
}
await main()