diff --git a/src/main/ai-vault/session-scanner-service-client-state.ts b/src/main/ai-vault/session-scanner-service-client-state.ts index afa5b06e5cb..cbd778817dd 100644 --- a/src/main/ai-vault/session-scanner-service-client-state.ts +++ b/src/main/ai-vault/session-scanner-service-client-state.ts @@ -84,3 +84,25 @@ export type AiVaultServiceReadyWaiter = { reject: (error: Error) => void timer: NodeJS.Timeout } + +export class AiVaultServiceIdleRetirement { + private timer: NodeJS.Timeout | null = null + + clear(): void { + if (this.timer) { + clearTimeout(this.timer) + this.timer = null + } + } + + schedule(busy: boolean, timeoutMs: number, retire: () => void): void { + if (busy || this.timer) { + return + } + this.timer = setTimeout(() => { + this.timer = null + retire() + }, timeoutMs) + this.timer.unref?.() + } +} diff --git a/src/main/ai-vault/session-scanner-service-client.test.ts b/src/main/ai-vault/session-scanner-service-client.test.ts index d10df1d01ba..2624baca06f 100644 --- a/src/main/ai-vault/session-scanner-service-client.test.ts +++ b/src/main/ai-vault/session-scanner-service-client.test.ts @@ -113,6 +113,44 @@ describe('AiVaultScannerServiceClient', () => { client.dispose() }) + it('keeps invalidation-only children alive through acknowledgement, then retires them', async () => { + vi.useFakeTimers() + const children: AiVaultServiceTestChild[] = [] + const client = new AiVaultScannerServiceClient({ + processFactory: () => { + const child = new AiVaultServiceTestChild(12_345 + children.length) + children.push(child) + return child.asChildProcess() + }, + init: { sessionParseCache: null }, + idleTimeoutMs: 100 + }) + + const first = client.request({ type: 'request', operation: 'titles', requests: [] }) + readyAiVaultServiceChild(children[0]!) + await Promise.resolve() + children[0]!.emit('message', { + type: 'result', + id: aiVaultServiceRequestId(children[0]!, 'titles'), + operation: 'titles', + value: { titles: [] } + }) + await first + vi.advanceTimersByTime(100) + + const invalidation = client.invalidate(['/tmp/deleted.jsonl']) + readyAiVaultServiceChild(children[1]!) + await Promise.resolve() + vi.advanceTimersByTime(100) + expect(children[1]!.sent).not.toContainEqual({ type: 'shutdown' }) + children[1]!.emit('message', { type: 'invalidated', generation: 1 }) + await invalidation + vi.advanceTimersByTime(100) + + expect(children[1]!.sent).toContainEqual({ type: 'shutdown' }) + client.dispose() + }) + it('faults the child on malformed output without affecting later processes', async () => { const { child, client } = setup() const request = client.request({ type: 'request', operation: 'titles', requests: [] }) diff --git a/src/main/ai-vault/session-scanner-service-client.ts b/src/main/ai-vault/session-scanner-service-client.ts index dc334e888e8..c79694cf888 100644 --- a/src/main/ai-vault/session-scanner-service-client.ts +++ b/src/main/ai-vault/session-scanner-service-client.ts @@ -6,6 +6,7 @@ import { AI_VAULT_SERVICE_MAX_CALLS, AI_VAULT_SERVICE_READY_TIMEOUT_MS, AI_VAULT_SERVICE_SCAN_TIMEOUT_MS, + AiVaultServiceIdleRetirement, armAiVaultServiceCancellationTimeout, clearAiVaultServiceCall, rejectAiVaultServiceCall, @@ -35,7 +36,7 @@ export class AiVaultScannerServiceClient { private readonly invalidations = new Map() private nextId = 1 private invalidationGeneration = 0 - private idleTimer: NodeJS.Timeout | null = null + private readonly idleRetirement = new AiVaultServiceIdleRetirement() private readonly restartPolicy = new AiVaultServiceRestartPolicy() private disposed = false @@ -68,7 +69,7 @@ export class AiVaultScannerServiceClient { signal.addEventListener('abort', call.onAbort, { once: true }) } this.queue.push(call) - this.clearIdleTimer() + this.idleRetirement.clear() this.pump() }) } @@ -77,12 +78,14 @@ export class AiVaultScannerServiceClient { if (paths.length === 0 || this.disposed) { return } + this.idleRetirement.clear() const child = await this.ensureChild() const generation = ++this.invalidationGeneration return new Promise((resolve, reject) => { const timer = setTimeout(() => { this.invalidations.delete(generation) reject(new Error('AI Vault service cache invalidation timed out.')) + this.scheduleIdleIfNeeded() }, AI_VAULT_SERVICE_READY_TIMEOUT_MS) timer.unref?.() this.invalidations.set(generation, { resolve, reject, timer }) @@ -96,7 +99,7 @@ export class AiVaultScannerServiceClient { } this.disposed = true this.restartPolicy.dispose() - this.clearIdleTimer() + this.idleRetirement.clear() const error = new Error('AI Vault service client was disposed.') for (const call of [...this.active.values(), ...this.queue]) { rejectAiVaultServiceCall(call, error) @@ -218,6 +221,7 @@ export class AiVaultScannerServiceClient { clearTimeout(pending.timer) this.invalidations.delete(message.generation) pending.resolve() + this.scheduleIdleIfNeeded() } return } @@ -284,25 +288,15 @@ export class AiVaultScannerServiceClient { } private scheduleIdleIfNeeded(): void { - if (this.active.size > 0 || this.queue.length > 0 || !this.child || this.idleTimer) { - return - } - this.idleTimer = setTimeout( - () => this.retireChild(), - this.options.idleTimeoutMs ?? AI_VAULT_SERVICE_IDLE_TIMEOUT_MS + this.idleRetirement.schedule( + this.active.size > 0 || this.queue.length > 0 || this.invalidations.size > 0 || !this.child, + this.options.idleTimeoutMs ?? AI_VAULT_SERVICE_IDLE_TIMEOUT_MS, + () => this.retireChild() ) - this.idleTimer.unref?.() - } - - private clearIdleTimer(): void { - if (this.idleTimer) { - clearTimeout(this.idleTimer) - this.idleTimer = null - } } private retireChild(): void { - this.clearIdleTimer() + this.idleRetirement.clear() const child = this.child this.child = null if (!child) { diff --git a/src/relay/ai-vault-service-client-state.ts b/src/relay/ai-vault-service-client-state.ts index e38ddb2111c..91a4733b44e 100644 --- a/src/relay/ai-vault-service-client-state.ts +++ b/src/relay/ai-vault-service-client-state.ts @@ -12,6 +12,7 @@ export const RELAY_AI_VAULT_READY_TIMEOUT_MS = 5_000 export const RELAY_AI_VAULT_SCAN_TIMEOUT_MS = 130_000 export const RELAY_AI_VAULT_TITLE_TIMEOUT_MS = 15_000 export const RELAY_AI_VAULT_MAX_CALLS = 16 +export const RELAY_AI_VAULT_IDLE_TIMEOUT_MS = 10 * 60_000 export function relayAiVaultAbortError(): Error { const error = new Error('The operation was aborted.') @@ -31,6 +32,27 @@ export function armRelayAiVaultCancellationTimeout( call.timer.unref?.() } +export function settleRelayAiVaultServiceCall( + call: RelayAiVaultServiceCall, + value: Error | AiVaultListResult | AiVaultSessionTitlesResult +): void { + if (call.settled) { + return + } + call.settled = true + if (call.timer) { + clearTimeout(call.timer) + } + if (call.signal && call.onAbort) { + call.signal.removeEventListener('abort', call.onAbort) + } + if (value instanceof Error) { + call.reject(value) + } else { + call.resolve(value) + } +} + export type RelayAiVaultServiceCall = { request: RelayAiVaultServiceRequest signal?: AbortSignal @@ -57,4 +79,34 @@ export type RelayAiVaultServiceClientOptions = { hostPlatform: RemoteHostPlatform } now?: () => number + idleTimeoutMs?: number +} + +export class RelayAiVaultIdleRetirement { + private timer: NodeJS.Timeout | null = null + + clear(): void { + if (this.timer) { + clearTimeout(this.timer) + this.timer = null + } + } + + schedule(busy: boolean, timeoutMs: number, retire: () => void): void { + if (busy || this.timer) { + return + } + this.timer = setTimeout(() => { + this.timer = null + retire() + }, timeoutMs) + this.timer.unref?.() + } +} + +export function retireRelayAiVaultServiceChild(child: ChildProcess): void { + const timer = setTimeout(() => child.kill(), 2_000) + timer.unref?.() + child.once('exit', () => clearTimeout(timer)) + child.send({ type: 'shutdown' }, () => undefined) } diff --git a/src/relay/ai-vault-service-client.test.ts b/src/relay/ai-vault-service-client.test.ts index 53399d6601d..15cc6eec0ae 100644 --- a/src/relay/ai-vault-service-client.test.ts +++ b/src/relay/ai-vault-service-client.test.ts @@ -7,7 +7,10 @@ import { import { RelayAiVaultServiceClient } from './ai-vault-service-client' import { relayAiVaultServiceEntryPath } from './ai-vault-service-spawn' -function createClient(children: AiVaultServiceTestChild[]): RelayAiVaultServiceClient { +function createClient( + children: AiVaultServiceTestChild[], + idleTimeoutMs?: number +): RelayAiVaultServiceClient { return new RelayAiVaultServiceClient({ init: { remoteHome: '/home/ada', @@ -17,7 +20,8 @@ function createClient(children: AiVaultServiceTestChild[]): RelayAiVaultServiceC const child = new AiVaultServiceTestChild(20_000 + children.length) children.push(child) return child.asChildProcess() - } + }, + idleTimeoutMs }) } @@ -122,6 +126,34 @@ describe('RelayAiVaultServiceClient', () => { await disposing }) + it('retires the sidecar after the idle bound', async () => { + vi.useFakeTimers() + const children: AiVaultServiceTestChild[] = [] + const client = createClient(children, 100) + const list = client.listSessions({}) + const child = children[0]! + readyAiVaultServiceChild(child) + await Promise.resolve() + const request = child.sent.find( + (message) => (message as { operation?: string }).operation === 'list' + ) as { id: number } + child.emit('message', { + type: 'result', + id: request.id, + operation: 'list', + value: { sessions: [], issues: [], scannedAt: '2026-08-09T00:00:00.000Z' } + }) + await list + + vi.advanceTimersByTime(99) + expect(child.sent).not.toContainEqual({ type: 'shutdown' }) + vi.advanceTimersByTime(1) + expect(child.sent).toContainEqual({ type: 'shutdown' }) + vi.advanceTimersByTime(2_000) + expect(child.killed).toBe(true) + await client.dispose() + }) + it('resolves the sidecar beside each bundled relay', () => { expect(relayAiVaultServiceEntryPath('/opt/orca/relay')).toBe( '/opt/orca/relay/relay-ai-vault-service.js' diff --git a/src/relay/ai-vault-service-client.ts b/src/relay/ai-vault-service-client.ts index 3f1235612fe..df4ee5292d6 100644 --- a/src/relay/ai-vault-service-client.ts +++ b/src/relay/ai-vault-service-client.ts @@ -7,12 +7,16 @@ import type { import type { SshAiVaultRelayListParams } from '../shared/ssh-ai-vault-relay' import { RELAY_AI_VAULT_MAX_CALLS, + RELAY_AI_VAULT_IDLE_TIMEOUT_MS, RELAY_AI_VAULT_READY_TIMEOUT_MS, RELAY_AI_VAULT_SCAN_TIMEOUT_MS, RELAY_AI_VAULT_TITLE_TIMEOUT_MS, armRelayAiVaultCancellationTimeout, relayAiVaultAbortError, relayAiVaultError, + RelayAiVaultIdleRetirement, + retireRelayAiVaultServiceChild, + settleRelayAiVaultServiceCall, type RelayAiVaultServiceApi, type RelayAiVaultServiceCall, type RelayAiVaultServiceClientOptions @@ -39,6 +43,7 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi { private faults: number[] = [] private circuitUntil = 0 private restartTimer: NodeJS.Timeout | null = null + private readonly idleRetirement = new RelayAiVaultIdleRetirement() private disposed = false constructor(private readonly options: RelayAiVaultServiceClientOptions) {} @@ -62,17 +67,18 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi { async dispose(): Promise { this.disposed = true + this.idleRetirement.clear() if (this.restartTimer) { clearTimeout(this.restartTimer) this.restartTimer = null } const error = new Error('Relay AI Vault service was disposed.') if (this.active) { - this.settle(this.active, error) + settleRelayAiVaultServiceCall(this.active, error) this.active = null } for (const call of this.queue.splice(0)) { - this.settle(call, error) + settleRelayAiVaultServiceCall(call, error) } const child = this.detachChild() if (!child) { @@ -121,6 +127,7 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi { signal.addEventListener('abort', call.onAbort, { once: true }) } this.queue.push(call) + this.idleRetirement.clear() this.pump() }) } @@ -131,6 +138,7 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi { } const call = this.queue.shift() if (!call) { + this.scheduleIdleIfNeeded() return } this.active = call @@ -138,7 +146,7 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi { (child) => this.sendCall(child, call), (error: Error) => { this.active = null - this.settle(call, error) + settleRelayAiVaultServiceCall(call, error) this.pump() } ) @@ -227,7 +235,10 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi { return } this.active = null - this.settle(call, message.type === 'error' ? new Error(message.message) : message.value) + settleRelayAiVaultServiceCall( + call, + message.type === 'error' ? new Error(message.message) : message.value + ) this.pump() } @@ -235,13 +246,13 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi { const index = this.queue.indexOf(call) if (index >= 0) { this.queue.splice(index, 1) - this.settle(call, relayAiVaultAbortError()) + settleRelayAiVaultServiceCall(call, relayAiVaultAbortError()) this.pump() return } if (this.active === call) { this.child?.send({ type: 'cancel', id: call.request.id }) - this.settle(call, relayAiVaultAbortError()) + settleRelayAiVaultServiceCall(call, relayAiVaultAbortError()) armRelayAiVaultCancellationTimeout(call, () => this.onFault(new Error('Relay AI Vault service did not cancel within 2000ms.')) ) @@ -252,12 +263,13 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi { if (!this.child) { return } + this.idleRetirement.clear() this.detachChild()?.kill() this.readyReject?.(error) this.readyReject = null this.ready = null if (this.active) { - this.settle(this.active, error) + settleRelayAiVaultServiceCall(this.active, error) this.active = null } const now = (this.options.now ?? Date.now)() @@ -275,31 +287,23 @@ export class RelayAiVaultServiceClient implements RelayAiVaultServiceApi { } } - private settle( - call: RelayAiVaultServiceCall, - value: Error | AiVaultListResult | AiVaultSessionTitlesResult - ): void { - if (call.settled) { - return - } - call.settled = true - if (call.timer) { - clearTimeout(call.timer) - } - if (call.signal && call.onAbort) { - call.signal.removeEventListener('abort', call.onAbort) - } - if (value instanceof Error) { - call.reject(value) - } else { - call.resolve(value) - } - } - private detachChild(): ChildProcess | null { const child = this.child this.child = null child?.removeAllListeners() return child } + + private scheduleIdleIfNeeded(): void { + this.idleRetirement.schedule( + Boolean(this.active || this.queue.length > 0 || !this.child), + this.options.idleTimeoutMs ?? RELAY_AI_VAULT_IDLE_TIMEOUT_MS, + () => { + const child = this.detachChild() + if (child) { + retireRelayAiVaultServiceChild(child) + } + } + ) + } }