mirror of
https://github.com/stablyai/orca.git
synced 2026-10-22 00:03:47 +00:00
fix(ai-vault): retire idle service processes
This commit is contained in:
@@ -84,3 +84,25 @@ export type AiVaultServiceReadyWaiter = {
|
||||
reject: (error: Error) => void
|
||||
timer: NodeJS.Timeout
|
||||
}
|
||||
|
||||
export class AiVaultServiceIdleRetirement {
|
||||
private timer: NodeJS.Timeout | null = null
|
||||
|
||||
clear(): void {
|
||||
if (this.timer) {
|
||||
clearTimeout(this.timer)
|
||||
this.timer = null
|
||||
}
|
||||
}
|
||||
|
||||
schedule(busy: boolean, timeoutMs: number, retire: () => void): void {
|
||||
if (busy || this.timer) {
|
||||
return
|
||||
}
|
||||
this.timer = setTimeout(() => {
|
||||
this.timer = null
|
||||
retire()
|
||||
}, timeoutMs)
|
||||
this.timer.unref?.()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -113,6 +113,44 @@ describe('AiVaultScannerServiceClient', () => {
|
||||
client.dispose()
|
||||
})
|
||||
|
||||
it('keeps invalidation-only children alive through acknowledgement, then retires them', async () => {
|
||||
vi.useFakeTimers()
|
||||
const children: AiVaultServiceTestChild[] = []
|
||||
const client = new AiVaultScannerServiceClient({
|
||||
processFactory: () => {
|
||||
const child = new AiVaultServiceTestChild(12_345 + children.length)
|
||||
children.push(child)
|
||||
return child.asChildProcess()
|
||||
},
|
||||
init: { sessionParseCache: null },
|
||||
idleTimeoutMs: 100
|
||||
})
|
||||
|
||||
const first = client.request({ type: 'request', operation: 'titles', requests: [] })
|
||||
readyAiVaultServiceChild(children[0]!)
|
||||
await Promise.resolve()
|
||||
children[0]!.emit('message', {
|
||||
type: 'result',
|
||||
id: aiVaultServiceRequestId(children[0]!, 'titles'),
|
||||
operation: 'titles',
|
||||
value: { titles: [] }
|
||||
})
|
||||
await first
|
||||
vi.advanceTimersByTime(100)
|
||||
|
||||
const invalidation = client.invalidate(['/tmp/deleted.jsonl'])
|
||||
readyAiVaultServiceChild(children[1]!)
|
||||
await Promise.resolve()
|
||||
vi.advanceTimersByTime(100)
|
||||
expect(children[1]!.sent).not.toContainEqual({ type: 'shutdown' })
|
||||
children[1]!.emit('message', { type: 'invalidated', generation: 1 })
|
||||
await invalidation
|
||||
vi.advanceTimersByTime(100)
|
||||
|
||||
expect(children[1]!.sent).toContainEqual({ type: 'shutdown' })
|
||||
client.dispose()
|
||||
})
|
||||
|
||||
it('faults the child on malformed output without affecting later processes', async () => {
|
||||
const { child, client } = setup()
|
||||
const request = client.request({ type: 'request', operation: 'titles', requests: [] })
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
AI_VAULT_SERVICE_MAX_CALLS,
|
||||
AI_VAULT_SERVICE_READY_TIMEOUT_MS,
|
||||
AI_VAULT_SERVICE_SCAN_TIMEOUT_MS,
|
||||
AiVaultServiceIdleRetirement,
|
||||
armAiVaultServiceCancellationTimeout,
|
||||
clearAiVaultServiceCall,
|
||||
rejectAiVaultServiceCall,
|
||||
@@ -35,7 +36,7 @@ export class AiVaultScannerServiceClient {
|
||||
private readonly invalidations = new Map<number, AiVaultServiceInvalidation>()
|
||||
private nextId = 1
|
||||
private invalidationGeneration = 0
|
||||
private idleTimer: NodeJS.Timeout | null = null
|
||||
private readonly idleRetirement = new AiVaultServiceIdleRetirement()
|
||||
private readonly restartPolicy = new AiVaultServiceRestartPolicy()
|
||||
private disposed = false
|
||||
|
||||
@@ -68,7 +69,7 @@ export class AiVaultScannerServiceClient {
|
||||
signal.addEventListener('abort', call.onAbort, { once: true })
|
||||
}
|
||||
this.queue.push(call)
|
||||
this.clearIdleTimer()
|
||||
this.idleRetirement.clear()
|
||||
this.pump()
|
||||
})
|
||||
}
|
||||
@@ -77,12 +78,14 @@ export class AiVaultScannerServiceClient {
|
||||
if (paths.length === 0 || this.disposed) {
|
||||
return
|
||||
}
|
||||
this.idleRetirement.clear()
|
||||
const child = await this.ensureChild()
|
||||
const generation = ++this.invalidationGeneration
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
this.invalidations.delete(generation)
|
||||
reject(new Error('AI Vault service cache invalidation timed out.'))
|
||||
this.scheduleIdleIfNeeded()
|
||||
}, AI_VAULT_SERVICE_READY_TIMEOUT_MS)
|
||||
timer.unref?.()
|
||||
this.invalidations.set(generation, { resolve, reject, timer })
|
||||
@@ -96,7 +99,7 @@ export class AiVaultScannerServiceClient {
|
||||
}
|
||||
this.disposed = true
|
||||
this.restartPolicy.dispose()
|
||||
this.clearIdleTimer()
|
||||
this.idleRetirement.clear()
|
||||
const error = new Error('AI Vault service client was disposed.')
|
||||
for (const call of [...this.active.values(), ...this.queue]) {
|
||||
rejectAiVaultServiceCall(call, error)
|
||||
@@ -218,6 +221,7 @@ export class AiVaultScannerServiceClient {
|
||||
clearTimeout(pending.timer)
|
||||
this.invalidations.delete(message.generation)
|
||||
pending.resolve()
|
||||
this.scheduleIdleIfNeeded()
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -284,25 +288,15 @@ export class AiVaultScannerServiceClient {
|
||||
}
|
||||
|
||||
private scheduleIdleIfNeeded(): void {
|
||||
if (this.active.size > 0 || this.queue.length > 0 || !this.child || this.idleTimer) {
|
||||
return
|
||||
}
|
||||
this.idleTimer = setTimeout(
|
||||
() => this.retireChild(),
|
||||
this.options.idleTimeoutMs ?? AI_VAULT_SERVICE_IDLE_TIMEOUT_MS
|
||||
this.idleRetirement.schedule(
|
||||
this.active.size > 0 || this.queue.length > 0 || this.invalidations.size > 0 || !this.child,
|
||||
this.options.idleTimeoutMs ?? AI_VAULT_SERVICE_IDLE_TIMEOUT_MS,
|
||||
() => this.retireChild()
|
||||
)
|
||||
this.idleTimer.unref?.()
|
||||
}
|
||||
|
||||
private clearIdleTimer(): void {
|
||||
if (this.idleTimer) {
|
||||
clearTimeout(this.idleTimer)
|
||||
this.idleTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
private retireChild(): void {
|
||||
this.clearIdleTimer()
|
||||
this.idleRetirement.clear()
|
||||
const child = this.child
|
||||
this.child = null
|
||||
if (!child) {
|
||||
|
||||
@@ -12,6 +12,7 @@ export const RELAY_AI_VAULT_READY_TIMEOUT_MS = 5_000
|
||||
export const RELAY_AI_VAULT_SCAN_TIMEOUT_MS = 130_000
|
||||
export const RELAY_AI_VAULT_TITLE_TIMEOUT_MS = 15_000
|
||||
export const RELAY_AI_VAULT_MAX_CALLS = 16
|
||||
export const RELAY_AI_VAULT_IDLE_TIMEOUT_MS = 10 * 60_000
|
||||
|
||||
export function relayAiVaultAbortError(): Error {
|
||||
const error = new Error('The operation was aborted.')
|
||||
@@ -31,6 +32,27 @@ export function armRelayAiVaultCancellationTimeout(
|
||||
call.timer.unref?.()
|
||||
}
|
||||
|
||||
export function settleRelayAiVaultServiceCall(
|
||||
call: RelayAiVaultServiceCall,
|
||||
value: Error | AiVaultListResult | AiVaultSessionTitlesResult
|
||||
): void {
|
||||
if (call.settled) {
|
||||
return
|
||||
}
|
||||
call.settled = true
|
||||
if (call.timer) {
|
||||
clearTimeout(call.timer)
|
||||
}
|
||||
if (call.signal && call.onAbort) {
|
||||
call.signal.removeEventListener('abort', call.onAbort)
|
||||
}
|
||||
if (value instanceof Error) {
|
||||
call.reject(value)
|
||||
} else {
|
||||
call.resolve(value)
|
||||
}
|
||||
}
|
||||
|
||||
export type RelayAiVaultServiceCall = {
|
||||
request: RelayAiVaultServiceRequest
|
||||
signal?: AbortSignal
|
||||
@@ -57,4 +79,34 @@ export type RelayAiVaultServiceClientOptions = {
|
||||
hostPlatform: RemoteHostPlatform
|
||||
}
|
||||
now?: () => number
|
||||
idleTimeoutMs?: number
|
||||
}
|
||||
|
||||
export class RelayAiVaultIdleRetirement {
|
||||
private timer: NodeJS.Timeout | null = null
|
||||
|
||||
clear(): void {
|
||||
if (this.timer) {
|
||||
clearTimeout(this.timer)
|
||||
this.timer = null
|
||||
}
|
||||
}
|
||||
|
||||
schedule(busy: boolean, timeoutMs: number, retire: () => void): void {
|
||||
if (busy || this.timer) {
|
||||
return
|
||||
}
|
||||
this.timer = setTimeout(() => {
|
||||
this.timer = null
|
||||
retire()
|
||||
}, timeoutMs)
|
||||
this.timer.unref?.()
|
||||
}
|
||||
}
|
||||
|
||||
export function retireRelayAiVaultServiceChild(child: ChildProcess): void {
|
||||
const timer = setTimeout(() => child.kill(), 2_000)
|
||||
timer.unref?.()
|
||||
child.once('exit', () => clearTimeout(timer))
|
||||
child.send({ type: 'shutdown' }, () => undefined)
|
||||
}
|
||||
|
||||
@@ -7,7 +7,10 @@ import {
|
||||
import { RelayAiVaultServiceClient } from './ai-vault-service-client'
|
||||
import { relayAiVaultServiceEntryPath } from './ai-vault-service-spawn'
|
||||
|
||||
function createClient(children: AiVaultServiceTestChild[]): RelayAiVaultServiceClient {
|
||||
function createClient(
|
||||
children: AiVaultServiceTestChild[],
|
||||
idleTimeoutMs?: number
|
||||
): RelayAiVaultServiceClient {
|
||||
return new RelayAiVaultServiceClient({
|
||||
init: {
|
||||
remoteHome: '/home/ada',
|
||||
@@ -17,7 +20,8 @@ function createClient(children: AiVaultServiceTestChild[]): RelayAiVaultServiceC
|
||||
const child = new AiVaultServiceTestChild(20_000 + children.length)
|
||||
children.push(child)
|
||||
return child.asChildProcess()
|
||||
}
|
||||
},
|
||||
idleTimeoutMs
|
||||
})
|
||||
}
|
||||
|
||||
@@ -122,6 +126,34 @@ describe('RelayAiVaultServiceClient', () => {
|
||||
await disposing
|
||||
})
|
||||
|
||||
it('retires the sidecar after the idle bound', async () => {
|
||||
vi.useFakeTimers()
|
||||
const children: AiVaultServiceTestChild[] = []
|
||||
const client = createClient(children, 100)
|
||||
const list = client.listSessions({})
|
||||
const child = children[0]!
|
||||
readyAiVaultServiceChild(child)
|
||||
await Promise.resolve()
|
||||
const request = child.sent.find(
|
||||
(message) => (message as { operation?: string }).operation === 'list'
|
||||
) as { id: number }
|
||||
child.emit('message', {
|
||||
type: 'result',
|
||||
id: request.id,
|
||||
operation: 'list',
|
||||
value: { sessions: [], issues: [], scannedAt: '2026-08-09T00:00:00.000Z' }
|
||||
})
|
||||
await list
|
||||
|
||||
vi.advanceTimersByTime(99)
|
||||
expect(child.sent).not.toContainEqual({ type: 'shutdown' })
|
||||
vi.advanceTimersByTime(1)
|
||||
expect(child.sent).toContainEqual({ type: 'shutdown' })
|
||||
vi.advanceTimersByTime(2_000)
|
||||
expect(child.killed).toBe(true)
|
||||
await client.dispose()
|
||||
})
|
||||
|
||||
it('resolves the sidecar beside each bundled relay', () => {
|
||||
expect(relayAiVaultServiceEntryPath('/opt/orca/relay')).toBe(
|
||||
'/opt/orca/relay/relay-ai-vault-service.js'
|
||||
|
||||
@@ -7,12 +7,16 @@ import type {
|
||||
import type { SshAiVaultRelayListParams } from '../shared/ssh-ai-vault-relay'
|
||||
import {
|
||||
RELAY_AI_VAULT_MAX_CALLS,
|
||||
RELAY_AI_VAULT_IDLE_TIMEOUT_MS,
|
||||
RELAY_AI_VAULT_READY_TIMEOUT_MS,
|
||||
RELAY_AI_VAULT_SCAN_TIMEOUT_MS,
|
||||
RELAY_AI_VAULT_TITLE_TIMEOUT_MS,
|
||||
armRelayAiVaultCancellationTimeout,
|
||||
relayAiVaultAbortError,
|
||||
relayAiVaultError,
|
||||
RelayAiVaultIdleRetirement,
|
||||
retireRelayAiVaultServiceChild,
|
||||
settleRelayAiVaultServiceCall,
|
||||
type RelayAiVaultServiceApi,
|
||||
type RelayAiVaultServiceCall,
|
||||
type RelayAiVaultServiceClientOptions
|
||||
@@ -39,6 +43,7 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
|
||||
private faults: number[] = []
|
||||
private circuitUntil = 0
|
||||
private restartTimer: NodeJS.Timeout | null = null
|
||||
private readonly idleRetirement = new RelayAiVaultIdleRetirement()
|
||||
private disposed = false
|
||||
|
||||
constructor(private readonly options: RelayAiVaultServiceClientOptions) {}
|
||||
@@ -62,17 +67,18 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
|
||||
|
||||
async dispose(): Promise<void> {
|
||||
this.disposed = true
|
||||
this.idleRetirement.clear()
|
||||
if (this.restartTimer) {
|
||||
clearTimeout(this.restartTimer)
|
||||
this.restartTimer = null
|
||||
}
|
||||
const error = new Error('Relay AI Vault service was disposed.')
|
||||
if (this.active) {
|
||||
this.settle(this.active, error)
|
||||
settleRelayAiVaultServiceCall(this.active, error)
|
||||
this.active = null
|
||||
}
|
||||
for (const call of this.queue.splice(0)) {
|
||||
this.settle(call, error)
|
||||
settleRelayAiVaultServiceCall(call, error)
|
||||
}
|
||||
const child = this.detachChild()
|
||||
if (!child) {
|
||||
@@ -121,6 +127,7 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
|
||||
signal.addEventListener('abort', call.onAbort, { once: true })
|
||||
}
|
||||
this.queue.push(call)
|
||||
this.idleRetirement.clear()
|
||||
this.pump()
|
||||
})
|
||||
}
|
||||
@@ -131,6 +138,7 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
|
||||
}
|
||||
const call = this.queue.shift()
|
||||
if (!call) {
|
||||
this.scheduleIdleIfNeeded()
|
||||
return
|
||||
}
|
||||
this.active = call
|
||||
@@ -138,7 +146,7 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
|
||||
(child) => this.sendCall(child, call),
|
||||
(error: Error) => {
|
||||
this.active = null
|
||||
this.settle(call, error)
|
||||
settleRelayAiVaultServiceCall(call, error)
|
||||
this.pump()
|
||||
}
|
||||
)
|
||||
@@ -227,7 +235,10 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
|
||||
return
|
||||
}
|
||||
this.active = null
|
||||
this.settle(call, message.type === 'error' ? new Error(message.message) : message.value)
|
||||
settleRelayAiVaultServiceCall(
|
||||
call,
|
||||
message.type === 'error' ? new Error(message.message) : message.value
|
||||
)
|
||||
this.pump()
|
||||
}
|
||||
|
||||
@@ -235,13 +246,13 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
|
||||
const index = this.queue.indexOf(call)
|
||||
if (index >= 0) {
|
||||
this.queue.splice(index, 1)
|
||||
this.settle(call, relayAiVaultAbortError())
|
||||
settleRelayAiVaultServiceCall(call, relayAiVaultAbortError())
|
||||
this.pump()
|
||||
return
|
||||
}
|
||||
if (this.active === call) {
|
||||
this.child?.send({ type: 'cancel', id: call.request.id })
|
||||
this.settle(call, relayAiVaultAbortError())
|
||||
settleRelayAiVaultServiceCall(call, relayAiVaultAbortError())
|
||||
armRelayAiVaultCancellationTimeout(call, () =>
|
||||
this.onFault(new Error('Relay AI Vault service did not cancel within 2000ms.'))
|
||||
)
|
||||
@@ -252,12 +263,13 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
|
||||
if (!this.child) {
|
||||
return
|
||||
}
|
||||
this.idleRetirement.clear()
|
||||
this.detachChild()?.kill()
|
||||
this.readyReject?.(error)
|
||||
this.readyReject = null
|
||||
this.ready = null
|
||||
if (this.active) {
|
||||
this.settle(this.active, error)
|
||||
settleRelayAiVaultServiceCall(this.active, error)
|
||||
this.active = null
|
||||
}
|
||||
const now = (this.options.now ?? Date.now)()
|
||||
@@ -275,31 +287,23 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi {
|
||||
}
|
||||
}
|
||||
|
||||
private settle(
|
||||
call: RelayAiVaultServiceCall,
|
||||
value: Error | AiVaultListResult | AiVaultSessionTitlesResult
|
||||
): void {
|
||||
if (call.settled) {
|
||||
return
|
||||
}
|
||||
call.settled = true
|
||||
if (call.timer) {
|
||||
clearTimeout(call.timer)
|
||||
}
|
||||
if (call.signal && call.onAbort) {
|
||||
call.signal.removeEventListener('abort', call.onAbort)
|
||||
}
|
||||
if (value instanceof Error) {
|
||||
call.reject(value)
|
||||
} else {
|
||||
call.resolve(value)
|
||||
}
|
||||
}
|
||||
|
||||
private detachChild(): ChildProcess | null {
|
||||
const child = this.child
|
||||
this.child = null
|
||||
child?.removeAllListeners()
|
||||
return child
|
||||
}
|
||||
|
||||
private scheduleIdleIfNeeded(): void {
|
||||
this.idleRetirement.schedule(
|
||||
Boolean(this.active || this.queue.length > 0 || !this.child),
|
||||
this.options.idleTimeoutMs ?? RELAY_AI_VAULT_IDLE_TIMEOUT_MS,
|
||||
() => {
|
||||
const child = this.detachChild()
|
||||
if (child) {
|
||||
retireRelayAiVaultServiceChild(child)
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user