mirror of
https://github.com/stablyai/orca.git
synced 2026-10-21 08:03:33 +00:00
* fix(runtime): retain watcher and PTY teardown ownership * fix(runtime): restore watchers after interrupted cleanup * fix(runtime): prevent stale watcher revival * test(runtime): cover watcher shutdown ownership * test(daemon): model physical PTY exit * fix(daemon): keep shutdown terminating when disposal cannot prove exit A rejecting host.dispose() (unreapable child past its exit deadline) left the shutdown RPC without its process.nextTick(shutdown) and skipped socket cleanup in shutdown(), stranding the daemon as an unreachable orphan after the stale-daemon replacement flow unlinks its socket. Log and continue: daemon exit reparents the child to init instead of blocking on it. * fix(runtime): keep local watching alive after an idle-kill deadline miss An idle child that outlived the exit deadline set shutdownRequested on the shared desktop supervisor, which has no retire-and-replace path — every later subscribe rejected supervisor_disposed and the roots were cached unwatchable, silently ending local file watching for the session. The idle path owns zero records, so there is no double-watch hazard; the zombie keeps its capacity reservation until physical exit and the next subscribe gets a fresh child. * fix(renderer): resync replayed paired-web file watches Transparent replay removed the implicit resync the old close-and-rebuild path provided: a replayed files.watch only reports changes from its own native setup, so changes during the reconnect gap were silently lost. Deliver a conservative overflow to consumers once the replayed watch is ready, matching the overflow-after-interruption contract everywhere else. * fix(runtime): address teardown review findings * fix(runtime): retry watches after teardown deadlines * Fix PTY descendant leaks on forced teardown * Fix jitter-sensitive terminal lifecycle test
339 lines
10 KiB
TypeScript
339 lines
10 KiB
TypeScript
import type { FsChangeEvent } from '../../shared/types'
|
|
import {
|
|
forgetRuntimeWatcherProcessRoot,
|
|
resetRuntimeWatcherProcessForTest,
|
|
subscribeViaRuntimeWatcherProcess,
|
|
type WatcherProcessEvent,
|
|
type WatcherProcessSubscription
|
|
} from '../ipc/parcel-watcher-process'
|
|
import {
|
|
isWatcherProcessFailure,
|
|
type WatcherProcessFailure
|
|
} from '../ipc/parcel-watcher-process-failure'
|
|
import {
|
|
WATCHER_IGNORE_DIRS,
|
|
buildParcelWatcherIgnoreOptions
|
|
} from '../ipc/filesystem-watcher-ignore'
|
|
import { createRuntimeRootOwnershipReleaser } from './runtime-root-watch-ownership'
|
|
import { closeRuntimeRootWatch } from './runtime-root-watch-teardown'
|
|
import {
|
|
RUNTIME_FILE_WATCH_CRAWL_TIMEOUT_MS,
|
|
RUNTIME_FILE_WATCH_MAX_SETUP_ATTEMPTS
|
|
} from '../../shared/runtime-file-watch-limits'
|
|
import { PromiseSettlementWaiters } from '../../shared/promise-settlement-waiters'
|
|
|
|
const RUNTIME_FILE_WATCH_IGNORE_OPTIONS = buildParcelWatcherIgnoreOptions(WATCHER_IGNORE_DIRS)
|
|
const RUNTIME_FILE_WATCH_EVENT_LIMIT = 200
|
|
|
|
type RuntimeFileWatchSubscriber = {
|
|
onEvents: (events: FsChangeEvent[]) => void
|
|
onTerminalError: (error: Error) => void
|
|
}
|
|
|
|
type RuntimeRootWatch = {
|
|
rootPath: string
|
|
subscribers: Set<RuntimeFileWatchSubscriber>
|
|
startWaiters: PromiseSettlementWaiters<void>
|
|
subscription: WatcherProcessSubscription | null
|
|
abortController: AbortController
|
|
generation: number
|
|
closed: boolean
|
|
closePromise: Promise<void> | null
|
|
terminalReleaseError: WatcherProcessFailure | null
|
|
}
|
|
|
|
// Why: paired clients watching the same worktree share one native subscription.
|
|
// Different roots are assigned across the bounded crash-isolated child pool.
|
|
const runtimeRootWatches = new Map<string, RuntimeRootWatch>()
|
|
const {
|
|
release: releaseRuntimeRootOwnership,
|
|
releaseAfterFailure: releaseRuntimeRootAfterFailure
|
|
} = createRuntimeRootOwnershipReleaser(runtimeRootWatches, forgetRuntimeWatcherProcessRoot)
|
|
|
|
function overflowEvent(rootPath: string): FsChangeEvent[] {
|
|
return [{ kind: 'overflow', absolutePath: rootPath }]
|
|
}
|
|
|
|
function mapWatcherEvents(events: readonly WatcherProcessEvent[]): FsChangeEvent[] {
|
|
return events.map((event) => ({
|
|
kind: event.type,
|
|
absolutePath: event.path,
|
|
isDirectory: event.isDirectory
|
|
}))
|
|
}
|
|
|
|
function emitToSubscribers(root: RuntimeRootWatch, events: FsChangeEvent[]): void {
|
|
if (root.closed) {
|
|
return
|
|
}
|
|
for (const subscriber of root.subscribers) {
|
|
subscriber.onEvents(events)
|
|
}
|
|
}
|
|
|
|
function terminateRootWatch(root: RuntimeRootWatch, error: Error): void {
|
|
if (root.closed) {
|
|
return
|
|
}
|
|
root.closed = true
|
|
root.generation++
|
|
root.abortController.abort()
|
|
releaseRuntimeRootAfterFailure(root, error)
|
|
const subscribers = Array.from(root.subscribers)
|
|
root.subscribers.clear()
|
|
for (const subscriber of subscribers) {
|
|
subscriber.onTerminalError(error)
|
|
}
|
|
}
|
|
|
|
function closeInitialRootWatch(root: RuntimeRootWatch, error: unknown): void {
|
|
root.closed = true
|
|
root.generation++
|
|
root.abortController.abort()
|
|
releaseRuntimeRootAfterFailure(
|
|
root,
|
|
error instanceof Error ? error : new Error('file watcher setup failed')
|
|
)
|
|
root.subscribers.clear()
|
|
}
|
|
|
|
function shouldRetryInitialWatch(error: unknown): boolean {
|
|
return (
|
|
isWatcherProcessFailure(error) &&
|
|
error.code !== 'entry_missing' &&
|
|
error.code !== 'subscribe_aborted' &&
|
|
error.code !== 'supervisor_disposed' &&
|
|
(error.scope === 'supervisor' || error.code === 'subscribe_timeout')
|
|
)
|
|
}
|
|
|
|
function subscribeRuntimeRootWatch(root: RuntimeRootWatch): Promise<void> {
|
|
const generation = ++root.generation
|
|
const emitOverflow = (): void => {
|
|
if (root.generation === generation) {
|
|
emitToSubscribers(root, overflowEvent(root.rootPath))
|
|
}
|
|
}
|
|
return subscribeViaRuntimeWatcherProcess(
|
|
root.rootPath,
|
|
(err, events) => {
|
|
if (root.generation !== generation || root.closed) {
|
|
return
|
|
}
|
|
if (err) {
|
|
console.error('[runtime-files.watch] watcher error', {
|
|
rootPath: root.rootPath,
|
|
error: err.message
|
|
})
|
|
emitOverflow()
|
|
return
|
|
}
|
|
emitToSubscribers(root, mapWatcherEvents(events))
|
|
},
|
|
RUNTIME_FILE_WATCH_IGNORE_OPTIONS,
|
|
{
|
|
delivery: {
|
|
includeDirectoryMetadata: true,
|
|
maxEventsPerBatch: RUNTIME_FILE_WATCH_EVENT_LIMIT
|
|
},
|
|
onInterruption: emitOverflow,
|
|
onOverflow: emitOverflow,
|
|
onTerminalError: (error) => recoverRuntimeRootWatch(root, generation, error),
|
|
// Why: the transport bounds initial setup at 15 seconds, but a later
|
|
// crash-resubscribe has no request deadline and must not freeze its shard forever.
|
|
subscribeTimeoutMs: RUNTIME_FILE_WATCH_CRAWL_TIMEOUT_MS,
|
|
signal: root.abortController.signal
|
|
}
|
|
).then(async (subscription) => {
|
|
if (root.closed || root.generation !== generation) {
|
|
await subscription.unsubscribe()
|
|
return
|
|
}
|
|
root.subscription = subscription
|
|
})
|
|
}
|
|
|
|
async function startInitialRuntimeRootWatch(root: RuntimeRootWatch): Promise<void> {
|
|
for (let attempt = 0; attempt < RUNTIME_FILE_WATCH_MAX_SETUP_ATTEMPTS; attempt++) {
|
|
try {
|
|
await subscribeRuntimeRootWatch(root)
|
|
if (attempt > 0) {
|
|
emitToSubscribers(root, overflowEvent(root.rootPath))
|
|
}
|
|
return
|
|
} catch (error) {
|
|
if (
|
|
root.closed ||
|
|
attempt + 1 >= RUNTIME_FILE_WATCH_MAX_SETUP_ATTEMPTS ||
|
|
!shouldRetryInitialWatch(error)
|
|
) {
|
|
closeInitialRootWatch(root, error)
|
|
throw error
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function recoverRuntimeRootWatch(
|
|
root: RuntimeRootWatch,
|
|
failedGeneration: number,
|
|
terminalError: Error
|
|
): void {
|
|
if (root.closed || root.generation !== failedGeneration) {
|
|
return
|
|
}
|
|
root.subscription = null
|
|
emitToSubscribers(root, overflowEvent(root.rootPath))
|
|
const recovery = subscribeRuntimeRootWatch(root).catch((recoveryError: unknown) => {
|
|
if (!root.closed) {
|
|
terminateRootWatch(root, recoveryError instanceof Error ? recoveryError : terminalError)
|
|
return
|
|
}
|
|
throw recoveryError
|
|
})
|
|
root.startWaiters = new PromiseSettlementWaiters(recovery)
|
|
}
|
|
|
|
function createRuntimeRootWatch(rootPath: string): RuntimeRootWatch {
|
|
const root: RuntimeRootWatch = {
|
|
rootPath,
|
|
subscribers: new Set(),
|
|
startWaiters: new PromiseSettlementWaiters(Promise.resolve()),
|
|
subscription: null,
|
|
abortController: new AbortController(),
|
|
generation: 0,
|
|
closed: false,
|
|
closePromise: null,
|
|
terminalReleaseError: null
|
|
}
|
|
runtimeRootWatches.set(rootPath, root)
|
|
root.startWaiters = new PromiseSettlementWaiters(startInitialRuntimeRootWatch(root))
|
|
return root
|
|
}
|
|
|
|
async function releaseRuntimeRootWatch(
|
|
root: RuntimeRootWatch,
|
|
subscriber: RuntimeFileWatchSubscriber
|
|
): Promise<void> {
|
|
root.subscribers.delete(subscriber)
|
|
if (root.closed) {
|
|
if (root.closePromise) {
|
|
await root.closePromise
|
|
}
|
|
if (root.terminalReleaseError) {
|
|
throw root.terminalReleaseError
|
|
}
|
|
return
|
|
}
|
|
if (root.subscribers.size > 0) {
|
|
return
|
|
}
|
|
await closeRuntimeRootWatchOnce(root)
|
|
}
|
|
|
|
function closeRuntimeRootWatchOnce(root: RuntimeRootWatch): Promise<void> {
|
|
root.closePromise ??= closeRuntimeRootWatch(
|
|
root,
|
|
root.startWaiters.promise,
|
|
() => releaseRuntimeRootOwnership(root),
|
|
(error) => releaseRuntimeRootAfterFailure(root, error)
|
|
)
|
|
return root.closePromise
|
|
}
|
|
|
|
export async function closeFileExplorerWatcherInWatcherProcess(rootPath: string): Promise<void> {
|
|
const root = runtimeRootWatches.get(rootPath)
|
|
if (!root) {
|
|
return
|
|
}
|
|
// Why: destructive cleanup owns every same-root subscriber, including a
|
|
// setup that failed before it could publish an unsubscribe callback.
|
|
root.subscribers.clear()
|
|
await closeRuntimeRootWatchOnce(root)
|
|
}
|
|
|
|
/**
|
|
* Start a recursive runtime-file watch in the crash-isolated process pool.
|
|
* The child owns crawl, event collapse, and stat metadata so neither native
|
|
* faults nor event storms can consume the serve process's libuv pool.
|
|
*/
|
|
export async function watchFileExplorerInWatcherProcess(
|
|
rootPath: string,
|
|
callback: (events: FsChangeEvent[]) => void,
|
|
onTerminalError: (error: Error) => void = () => undefined,
|
|
signal?: AbortSignal
|
|
): Promise<() => Promise<void>> {
|
|
const existingRoot = runtimeRootWatches.get(rootPath)
|
|
if (existingRoot?.closed) {
|
|
if (!existingRoot.closePromise) {
|
|
throw existingRoot.terminalReleaseError ?? new Error('file watcher closed during setup')
|
|
}
|
|
// Why: a replacement cannot share a native root whose prior generation
|
|
// is still closing; join its exact release before creating the next owner.
|
|
await existingRoot.closePromise
|
|
if (signal?.aborted) {
|
|
throw createRuntimeRootAbortError()
|
|
}
|
|
return watchFileExplorerInWatcherProcess(rootPath, callback, onTerminalError, signal)
|
|
}
|
|
const root = existingRoot ?? createRuntimeRootWatch(rootPath)
|
|
const subscriber: RuntimeFileWatchSubscriber = { onEvents: callback, onTerminalError }
|
|
root.subscribers.add(subscriber)
|
|
try {
|
|
await waitForRuntimeRootStart(root, subscriber, signal)
|
|
} catch (error) {
|
|
// Why: paired unwatch cannot acknowledge until native cancellation or
|
|
// physical exit releases this root for an immediate replacement watch.
|
|
await releaseRuntimeRootWatch(root, subscriber)
|
|
throw error
|
|
}
|
|
if (root.closed) {
|
|
throw new Error('file watcher closed during setup')
|
|
}
|
|
|
|
let releasePromise: Promise<void> | undefined
|
|
return (): Promise<void> => {
|
|
releasePromise ??= releaseRuntimeRootWatch(root, subscriber)
|
|
return releasePromise
|
|
}
|
|
}
|
|
|
|
function waitForRuntimeRootStart(
|
|
root: RuntimeRootWatch,
|
|
subscriber: RuntimeFileWatchSubscriber,
|
|
signal: AbortSignal | undefined
|
|
): Promise<void> {
|
|
if (!signal) {
|
|
return root.startWaiters.promise
|
|
}
|
|
if (signal.aborted) {
|
|
root.subscribers.delete(subscriber)
|
|
return Promise.reject(new Error('file watcher subscription aborted'))
|
|
}
|
|
return root.startWaiters.wait({
|
|
signal,
|
|
createAbortError: createRuntimeRootAbortError,
|
|
onAbandon: () => root.subscribers.delete(subscriber)
|
|
})
|
|
}
|
|
|
|
function createRuntimeRootAbortError(): Error {
|
|
return new Error('file watcher subscription aborted')
|
|
}
|
|
|
|
export function resetRuntimeRootWatchersForTest(): void {
|
|
for (const root of runtimeRootWatches.values()) {
|
|
root.closed = true
|
|
root.generation++
|
|
root.abortController.abort()
|
|
root.subscribers.clear()
|
|
void root.subscription?.unsubscribe()
|
|
}
|
|
runtimeRootWatches.clear()
|
|
resetRuntimeWatcherProcessForTest()
|
|
}
|
|
|
|
export function getRuntimeRootWatchWaiterCountForTest(rootPath: string): number {
|
|
return runtimeRootWatches.get(rootPath)?.startWaiters.waiterCount ?? 0
|
|
}
|