mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 16:02:32 +00:00
* Fix crash-isolated file watcher process pool for orca-serve SIGSEGV afte Replace the worker-thread runtime file watcher with a forked, crash-isolated @parcel/watcher child process pool so a native FSEvents fault can no longer take down the main/serve process, and add bounded event batching, delivery backpressure, and quarantine-based recovery for faulty watch roots. * Fix crash-isolated file watcher teardown and shutdown leaks - Fault harness could throw before mkdtemp/realpath completed, skipping cleanup; now tracks each temp path independently and races an async watcher-callback error so it can't escape the try/finally unhandled. - In-process fallback swallowed unsubscribe failures via a bare rejection handler that could still throw; use .catch() instead. - Watcher process entry's cancel-subscribe handler now reuses the async unsubscribe path when a crawl already finished, releasing the native handle instead of leaking it (blocks worktree unlock on Windows). - Runtime watcher process pool exposed no real dispose(); shutdown now kills pooled children so they don't outlive the main process. * Fix disposeSlot double-iteration bug in file watcher pool teardown Remove the unnecessary array snapshot in dispose(): disposeSlot mutates allSlots by deleting the slot being visited, and deleting the in-progress element during Set iteration is well-defined, so the spread copy was dead weight left over from prior debugging. * Fix pending file watcher installs not aborting on unsubscribe - Local/WSL watcher installs and SSH fs.watch setup now honor the in-flight AbortSignal, so the last unwatch cancels a slow native subscribe or remote setup instead of waiting for it to finish. - Thread signal through IFilesystemProvider.watch and SSH-backed file explorer watches for the same early-cancel behavior. * Fix crash-resubscribe hangs and SSH watch teardown races in file watcher - Add a bounded deadline for post-crash resubscription crawls so one stuck root quarantines instead of pinning its whole shard forever. - Report FSEvents overflow as recoverable so delivery continues after a dropped-events error instead of surfacing as terminal. - Make WSL watcher abort errors real DOMException instances so AbortSignal-based cancellation checks recognize them. - Rework SSH watch registration so ownership of the shared setup request (not just the first caller) decides teardown, preventing one caller's abort from cancelling another's shared watch and guaranteeing exactly one fs.unwatch per registration. - Reformat reliability-gates.jsonc arrays and refresh WSL/SSH coverage entries and evidence runs to match the above. * Add CI gate to run the file-watcher SIGSEGV fault harness under Electron - The reliability gate and release workflows (mac, Linux) previously only exercised the crash-isolation harness under vanilla Node, which doesn't catch runtime differences in the actual Electron binary that ships to users. - Adds an `ELECTRON_RUN_AS_NODE=1 pnpm exec electron ...` run of the same harness alongside the existing Node run, so #8212's SIGSEGV-survival contract is proven against both runtimes before packaging. * Add CI gate blocking Linux/macOS release packaging on watcher fault reco Adds a contract test asserting release-cut.yml and release-mac-build.yml run the runtime-file-watcher-fault-harness after building and before publishing artifacts, so a regression in watcher process fault recovery fails release packaging instead of shipping silently. * Fix use-after-clear crash in failAllWatcherSubscriptions Snapshot the records map before iterating, since onTerminalError hooks can dispose the supervisor and clear `records` mid-loop, causing a crash. Also update the matching test to assert against the shared buildParcelWatcherIgnoreOptions helper instead of a loose arrayContaining match. * Fix use-after-clear crash in failAllWatcherSubscriptions Snapshot watcher records with Array.from instead of spread, since spread syntax over an iterator that's mutated mid-loop by onTerminalError hooks can produce inconsistent results.
181 lines
5.6 KiB
JavaScript
181 lines
5.6 KiB
JavaScript
import { createRequire } from 'node:module'
|
|
import { existsSync } from 'node:fs'
|
|
import { mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'
|
|
import { tmpdir } from 'node:os'
|
|
import { join, resolve } from 'node:path'
|
|
import { build } from 'esbuild'
|
|
|
|
const ENTRY_PATH = resolve('out/main/parcel-watcher-process-entry.js')
|
|
const SUPERVISOR_SOURCE = resolve('src/main/ipc/parcel-watcher-process-supervisor.ts')
|
|
const WAIT_TIMEOUT_MS = 15_000
|
|
const require = createRequire(import.meta.url)
|
|
|
|
function withTimeout(promise, label) {
|
|
return new Promise((resolvePromise, rejectPromise) => {
|
|
const timer = setTimeout(
|
|
() => rejectPromise(new Error(`Timed out waiting for ${label}`)),
|
|
WAIT_TIMEOUT_MS
|
|
)
|
|
promise.then(
|
|
(value) => {
|
|
clearTimeout(timer)
|
|
resolvePromise(value)
|
|
},
|
|
(error) => {
|
|
clearTimeout(timer)
|
|
rejectPromise(error)
|
|
}
|
|
)
|
|
})
|
|
}
|
|
|
|
function nextMatchingEvent(register, predicate, label) {
|
|
return withTimeout(
|
|
new Promise((resolveEvent) => {
|
|
register((events) => {
|
|
if (events.some(predicate)) {
|
|
resolveEvent(events)
|
|
}
|
|
})
|
|
}),
|
|
label
|
|
)
|
|
}
|
|
|
|
async function loadSupervisor(bundleDir) {
|
|
const outfile = join(bundleDir, 'watcher-supervisor.cjs')
|
|
await build({
|
|
entryPoints: [SUPERVISOR_SOURCE],
|
|
bundle: true,
|
|
platform: 'node',
|
|
format: 'cjs',
|
|
outfile,
|
|
external: ['@parcel/watcher', 'electron'],
|
|
logLevel: 'silent'
|
|
})
|
|
return require(outfile).WatcherProcessSupervisor
|
|
}
|
|
|
|
async function main() {
|
|
if (process.platform === 'win32') {
|
|
console.log('[runtime-file-watcher-fault] SKIP: SIGSEGV oracle is macOS/Linux only')
|
|
return
|
|
}
|
|
if (!existsSync(ENTRY_PATH)) {
|
|
throw new Error(`Missing ${ENTRY_PATH}; run pnpm run build:electron-vite first`)
|
|
}
|
|
|
|
// Why: mkdtemp/realpath/bundle/construction can fail before the body runs.
|
|
// Keep cleanup in finally from the first successful mkdtemp onward, and clean
|
|
// the original temp path if realpath never succeeds.
|
|
let createdRootPath
|
|
let bundleDir
|
|
let rootPath
|
|
let supervisor
|
|
let subscription
|
|
let watcherCanaryDir
|
|
let eventListener = () => undefined
|
|
let rejectWatcherError
|
|
const watcherError = new Promise((_, reject) => {
|
|
rejectWatcherError = reject
|
|
})
|
|
// Attach early so a callback rejection before the race cannot become unhandled.
|
|
watcherError.catch(() => undefined)
|
|
|
|
try {
|
|
createdRootPath = await mkdtemp(join(tmpdir(), 'orca-runtime-watcher-fault-'))
|
|
bundleDir = await mkdtemp(join(tmpdir(), 'orca-runtime-watcher-harness-'))
|
|
// Parcel reports canonical event paths on macOS, where tmpdir() may use the
|
|
// /var symlink spelling. Keep the oracle in the same path domain.
|
|
rootPath = await realpath(createdRootPath)
|
|
const WatcherProcessSupervisor = await loadSupervisor(bundleDir)
|
|
supervisor = new WatcherProcessSupervisor()
|
|
|
|
let resolveInterruption
|
|
const interrupted = withTimeout(
|
|
new Promise((resolveWait) => {
|
|
resolveInterruption = resolveWait
|
|
}),
|
|
'automatic watcher resubscription'
|
|
)
|
|
subscription = await supervisor.subscribe(
|
|
rootPath,
|
|
(error, events) => {
|
|
if (error) {
|
|
// Why: throws from the async watcher callback escape main()'s try/finally
|
|
// and skip teardown. Surface failures through a harness promise instead.
|
|
rejectWatcherError(error)
|
|
return
|
|
}
|
|
eventListener(events)
|
|
},
|
|
{ ignore: ['.git', 'node_modules'] },
|
|
{
|
|
delivery: { includeDirectoryMetadata: true, maxEventsPerBatch: 200 },
|
|
onInterruption: () => resolveInterruption()
|
|
}
|
|
)
|
|
watcherCanaryDir = supervisor.canaryDir
|
|
|
|
const beforeEvent = nextMatchingEvent(
|
|
(listener) => {
|
|
eventListener = listener
|
|
},
|
|
(event) => event.path === join(rootPath, 'before.txt'),
|
|
'pre-crash watch event'
|
|
)
|
|
await writeFile(join(rootPath, 'before.txt'), 'before')
|
|
await Promise.race([beforeEvent, watcherError])
|
|
|
|
const firstChildPid = supervisor.child?.pid
|
|
if (!firstChildPid) {
|
|
throw new Error('Watcher supervisor did not expose a live child')
|
|
}
|
|
process.kill(firstChildPid, 'SIGSEGV')
|
|
await Promise.race([interrupted, watcherError])
|
|
|
|
const replacementChildPid = supervisor.child?.pid
|
|
if (!replacementChildPid || replacementChildPid === firstChildPid) {
|
|
throw new Error('Watcher supervisor did not replace the faulted child')
|
|
}
|
|
const afterEvent = nextMatchingEvent(
|
|
(listener) => {
|
|
eventListener = listener
|
|
},
|
|
(event) => event.path === join(rootPath, 'after.txt'),
|
|
'post-crash watch event'
|
|
)
|
|
await writeFile(join(rootPath, 'after.txt'), 'after')
|
|
await Promise.race([afterEvent, watcherError])
|
|
|
|
console.log(
|
|
JSON.stringify({
|
|
hostPid: process.pid,
|
|
killedWatcherPid: firstChildPid,
|
|
replacementWatcherPid: replacementChildPid,
|
|
hostSurvived: true,
|
|
automaticResubscribe: true,
|
|
postCrashEventDelivered: true
|
|
})
|
|
)
|
|
} finally {
|
|
try {
|
|
await subscription?.unsubscribe()
|
|
} finally {
|
|
supervisor?.dispose()
|
|
await Promise.all([
|
|
createdRootPath ? rm(createdRootPath, { recursive: true, force: true }) : Promise.resolve(),
|
|
rootPath && rootPath !== createdRootPath
|
|
? rm(rootPath, { recursive: true, force: true })
|
|
: Promise.resolve(),
|
|
bundleDir ? rm(bundleDir, { recursive: true, force: true }) : Promise.resolve()
|
|
])
|
|
}
|
|
}
|
|
if (watcherCanaryDir && existsSync(watcherCanaryDir)) {
|
|
throw new Error(`Watcher supervisor leaked its canary directory: ${watcherCanaryDir}`)
|
|
}
|
|
}
|
|
|
|
await main()
|