fix(ai-vault): retire idle service processes

This commit is contained in:
Brennan Benson
2026-08-09 18:32:27 -07:00
parent 37b4c46090
commit 909b048879
6 changed files with 190 additions and 48 deletions
@@ -84,3 +84,25 @@ export type AiVaultServiceReadyWaiter = {
reject: (error: Error) => void
timer: NodeJS.Timeout
}
export class AiVaultServiceIdleRetirement {
private timer: NodeJS.Timeout | null = null
clear(): void {
if (this.timer) {
clearTimeout(this.timer)
this.timer = null
}
}
schedule(busy: boolean, timeoutMs: number, retire: () => void): void {
if (busy || this.timer) {
return
}
this.timer = setTimeout(() => {
this.timer = null
retire()
}, timeoutMs)
this.timer.unref?.()
}
}
@@ -113,6 +113,44 @@ describe('AiVaultScannerServiceClient', () => {
client.dispose()
})
it('keeps invalidation-only children alive through acknowledgement, then retires them', async () => {
vi.useFakeTimers()
const children: AiVaultServiceTestChild[] = []
const client = new AiVaultScannerServiceClient({
processFactory: () => {
const child = new AiVaultServiceTestChild(12_345 + children.length)
children.push(child)
return child.asChildProcess()
},
init: { sessionParseCache: null },
idleTimeoutMs: 100
})
const first = client.request({ type: 'request', operation: 'titles', requests: [] })
readyAiVaultServiceChild(children[0]!)
await Promise.resolve()
children[0]!.emit('message', {
type: 'result',
id: aiVaultServiceRequestId(children[0]!, 'titles'),
operation: 'titles',
value: { titles: [] }
})
await first
vi.advanceTimersByTime(100)
const invalidation = client.invalidate(['/tmp/deleted.jsonl'])
readyAiVaultServiceChild(children[1]!)
await Promise.resolve()
vi.advanceTimersByTime(100)
expect(children[1]!.sent).not.toContainEqual({ type: 'shutdown' })
children[1]!.emit('message', { type: 'invalidated', generation: 1 })
await invalidation
vi.advanceTimersByTime(100)
expect(children[1]!.sent).toContainEqual({ type: 'shutdown' })
client.dispose()
})
it('faults the child on malformed output without affecting later processes', async () => {
const { child, client } = setup()
const request = client.request({ type: 'request', operation: 'titles', requests: [] })
@@ -6,6 +6,7 @@ import {
AI_VAULT_SERVICE_MAX_CALLS,
AI_VAULT_SERVICE_READY_TIMEOUT_MS,
AI_VAULT_SERVICE_SCAN_TIMEOUT_MS,
AiVaultServiceIdleRetirement,
armAiVaultServiceCancellationTimeout,
clearAiVaultServiceCall,
rejectAiVaultServiceCall,
@@ -35,7 +36,7 @@ export class AiVaultScannerServiceClient {
private readonly invalidations = new Map<number, AiVaultServiceInvalidation>()
private nextId = 1
private invalidationGeneration = 0
private idleTimer: NodeJS.Timeout | null = null
private readonly idleRetirement = new AiVaultServiceIdleRetirement()
private readonly restartPolicy = new AiVaultServiceRestartPolicy()
private disposed = false
@@ -68,7 +69,7 @@ export class AiVaultScannerServiceClient {
signal.addEventListener('abort', call.onAbort, { once: true })
}
this.queue.push(call)
this.clearIdleTimer()
this.idleRetirement.clear()
this.pump()
})
}
@@ -77,12 +78,14 @@ export class AiVaultScannerServiceClient {
if (paths.length === 0 || this.disposed) {
return
}
this.idleRetirement.clear()
const child = await this.ensureChild()
const generation = ++this.invalidationGeneration
return new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => {
this.invalidations.delete(generation)
reject(new Error('AI Vault service cache invalidation timed out.'))
this.scheduleIdleIfNeeded()
}, AI_VAULT_SERVICE_READY_TIMEOUT_MS)
timer.unref?.()
this.invalidations.set(generation, { resolve, reject, timer })
@@ -96,7 +99,7 @@ export class AiVaultScannerServiceClient {
}
this.disposed = true
this.restartPolicy.dispose()
this.clearIdleTimer()
this.idleRetirement.clear()
const error = new Error('AI Vault service client was disposed.')
for (const call of [...this.active.values(), ...this.queue]) {
rejectAiVaultServiceCall(call, error)
@@ -218,6 +221,7 @@ export class AiVaultScannerServiceClient {
clearTimeout(pending.timer)
this.invalidations.delete(message.generation)
pending.resolve()
this.scheduleIdleIfNeeded()
}
return
}
@@ -284,25 +288,15 @@ export class AiVaultScannerServiceClient {
}
private scheduleIdleIfNeeded(): void {
if (this.active.size > 0 || this.queue.length > 0 || !this.child || this.idleTimer) {
return
}
this.idleTimer = setTimeout(
() => this.retireChild(),
this.options.idleTimeoutMs ?? AI_VAULT_SERVICE_IDLE_TIMEOUT_MS
this.idleRetirement.schedule(
this.active.size > 0 || this.queue.length > 0 || this.invalidations.size > 0 || !this.child,
this.options.idleTimeoutMs ?? AI_VAULT_SERVICE_IDLE_TIMEOUT_MS,
() => this.retireChild()
)
this.idleTimer.unref?.()
}
private clearIdleTimer(): void {
if (this.idleTimer) {
clearTimeout(this.idleTimer)
this.idleTimer = null
}
}
private retireChild(): void {
this.clearIdleTimer()
this.idleRetirement.clear()
const child = this.child
this.child = null
if (!child) {
@@ -12,6 +12,7 @@ export const RELAY_AI_VAULT_READY_TIMEOUT_MS = 5_000
export const RELAY_AI_VAULT_SCAN_TIMEOUT_MS = 130_000
export const RELAY_AI_VAULT_TITLE_TIMEOUT_MS = 15_000
export const RELAY_AI_VAULT_MAX_CALLS = 16
export const RELAY_AI_VAULT_IDLE_TIMEOUT_MS = 10 * 60_000
export function relayAiVaultAbortError(): Error {
const error = new Error('The operation was aborted.')
@@ -31,6 +32,27 @@ export function armRelayAiVaultCancellationTimeout(
call.timer.unref?.()
}
export function settleRelayAiVaultServiceCall(
call: RelayAiVaultServiceCall,
value: Error | AiVaultListResult | AiVaultSessionTitlesResult
): void {
if (call.settled) {
return
}
call.settled = true
if (call.timer) {
clearTimeout(call.timer)
}
if (call.signal && call.onAbort) {
call.signal.removeEventListener('abort', call.onAbort)
}
if (value instanceof Error) {
call.reject(value)
} else {
call.resolve(value)
}
}
export type RelayAiVaultServiceCall = {
request: RelayAiVaultServiceRequest
signal?: AbortSignal
@@ -57,4 +79,34 @@ export type RelayAiVaultServiceClientOptions = {
hostPlatform: RemoteHostPlatform
}
now?: () => number
idleTimeoutMs?: number
}
export class RelayAiVaultIdleRetirement {
private timer: NodeJS.Timeout | null = null
clear(): void {
if (this.timer) {
clearTimeout(this.timer)
this.timer = null
}
}
schedule(busy: boolean, timeoutMs: number, retire: () => void): void {
if (busy || this.timer) {
return
}
this.timer = setTimeout(() => {
this.timer = null
retire()
}, timeoutMs)
this.timer.unref?.()
}
}
export function retireRelayAiVaultServiceChild(child: ChildProcess): void {
const timer = setTimeout(() => child.kill(), 2_000)
timer.unref?.()
child.once('exit', () => clearTimeout(timer))
child.send({ type: 'shutdown' }, () => undefined)
}
+34 -2
View File
@@ -7,7 +7,10 @@ import {
import { RelayAiVaultServiceClient } from './ai-vault-service-client'
import { relayAiVaultServiceEntryPath } from './ai-vault-service-spawn'
function createClient(children: AiVaultServiceTestChild[]): RelayAiVaultServiceClient {
function createClient(
children: AiVaultServiceTestChild[],
idleTimeoutMs?: number
): RelayAiVaultServiceClient {
return new RelayAiVaultServiceClient({
init: {
remoteHome: '/home/ada',
@@ -17,7 +20,8 @@ function createClient(children: AiVaultServiceTestChild[]): RelayAiVaultServiceC
const child = new AiVaultServiceTestChild(20_000 + children.length)
children.push(child)
return child.asChildProcess()
}
},
idleTimeoutMs
})
}
@@ -122,6 +126,34 @@ describe('RelayAiVaultServiceClient', () => {
await disposing
})
it('retires the sidecar after the idle bound', async () => {
vi.useFakeTimers()
const children: AiVaultServiceTestChild[] = []
const client = createClient(children, 100)
const list = client.listSessions({})
const child = children[0]!
readyAiVaultServiceChild(child)
await Promise.resolve()
const request = child.sent.find(
(message) => (message as { operation?: string }).operation === 'list'
) as { id: number }
child.emit('message', {
type: 'result',
id: request.id,
operation: 'list',
value: { sessions: [], issues: [], scannedAt: '2026-08-09T00:00:00.000Z' }
})
await list
vi.advanceTimersByTime(99)
expect(child.sent).not.toContainEqual({ type: 'shutdown' })
vi.advanceTimersByTime(1)
expect(child.sent).toContainEqual({ type: 'shutdown' })
vi.advanceTimersByTime(2_000)
expect(child.killed).toBe(true)
await client.dispose()
})
it('resolves the sidecar beside each bundled relay', () => {
expect(relayAiVaultServiceEntryPath('/opt/orca/relay')).toBe(
'/opt/orca/relay/relay-ai-vault-service.js'
+32 -28
View File
@@ -7,12 +7,16 @@ import type {
import type { SshAiVaultRelayListParams } from '../shared/ssh-ai-vault-relay'
import {
RELAY_AI_VAULT_MAX_CALLS,
RELAY_AI_VAULT_IDLE_TIMEOUT_MS,
RELAY_AI_VAULT_READY_TIMEOUT_MS,
RELAY_AI_VAULT_SCAN_TIMEOUT_MS,
RELAY_AI_VAULT_TITLE_TIMEOUT_MS,
armRelayAiVaultCancellationTimeout,
relayAiVaultAbortError,
relayAiVaultError,
RelayAiVaultIdleRetirement,
retireRelayAiVaultServiceChild,
settleRelayAiVaultServiceCall,
type RelayAiVaultServiceApi,
type RelayAiVaultServiceCall,
type RelayAiVaultServiceClientOptions
@@ -39,6 +43,7 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
private faults: number[] = []
private circuitUntil = 0
private restartTimer: NodeJS.Timeout | null = null
private readonly idleRetirement = new RelayAiVaultIdleRetirement()
private disposed = false
constructor(private readonly options: RelayAiVaultServiceClientOptions) {}
@@ -62,17 +67,18 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
async dispose(): Promise<void> {
this.disposed = true
this.idleRetirement.clear()
if (this.restartTimer) {
clearTimeout(this.restartTimer)
this.restartTimer = null
}
const error = new Error('Relay AI Vault service was disposed.')
if (this.active) {
this.settle(this.active, error)
settleRelayAiVaultServiceCall(this.active, error)
this.active = null
}
for (const call of this.queue.splice(0)) {
this.settle(call, error)
settleRelayAiVaultServiceCall(call, error)
}
const child = this.detachChild()
if (!child) {
@@ -121,6 +127,7 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
signal.addEventListener('abort', call.onAbort, { once: true })
}
this.queue.push(call)
this.idleRetirement.clear()
this.pump()
})
}
@@ -131,6 +138,7 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
}
const call = this.queue.shift()
if (!call) {
this.scheduleIdleIfNeeded()
return
}
this.active = call
@@ -138,7 +146,7 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
(child) => this.sendCall(child, call),
(error: Error) => {
this.active = null
this.settle(call, error)
settleRelayAiVaultServiceCall(call, error)
this.pump()
}
)
@@ -227,7 +235,10 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
return
}
this.active = null
this.settle(call, message.type === 'error' ? new Error(message.message) : message.value)
settleRelayAiVaultServiceCall(
call,
message.type === 'error' ? new Error(message.message) : message.value
)
this.pump()
}
@@ -235,13 +246,13 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
const index = this.queue.indexOf(call)
if (index >= 0) {
this.queue.splice(index, 1)
this.settle(call, relayAiVaultAbortError())
settleRelayAiVaultServiceCall(call, relayAiVaultAbortError())
this.pump()
return
}
if (this.active === call) {
this.child?.send({ type: 'cancel', id: call.request.id })
this.settle(call, relayAiVaultAbortError())
settleRelayAiVaultServiceCall(call, relayAiVaultAbortError())
armRelayAiVaultCancellationTimeout(call, () =>
this.onFault(new Error('Relay AI Vault service did not cancel within 2000ms.'))
)
@@ -252,12 +263,13 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
if (!this.child) {
return
}
this.idleRetirement.clear()
this.detachChild()?.kill()
this.readyReject?.(error)
this.readyReject = null
this.ready = null
if (this.active) {
this.settle(this.active, error)
settleRelayAiVaultServiceCall(this.active, error)
this.active = null
}
const now = (this.options.now ?? Date.now)()
@@ -275,31 +287,23 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
}
}
private settle(
call: RelayAiVaultServiceCall,
value: Error | AiVaultListResult | AiVaultSessionTitlesResult
): void {
if (call.settled) {
return
}
call.settled = true
if (call.timer) {
clearTimeout(call.timer)
}
if (call.signal && call.onAbort) {
call.signal.removeEventListener('abort', call.onAbort)
}
if (value instanceof Error) {
call.reject(value)
} else {
call.resolve(value)
}
}
private detachChild(): ChildProcess | null {
const child = this.child
this.child = null
child?.removeAllListeners()
return child
}
private scheduleIdleIfNeeded(): void {
this.idleRetirement.schedule(
Boolean(this.active || this.queue.length > 0 || !this.child),
this.options.idleTimeoutMs ?? RELAY_AI_VAULT_IDLE_TIMEOUT_MS,
() => {
const child = this.detachChild()
if (child) {
retireRelayAiVaultServiceChild(child)
}
}
)
}
}