From b6b07f73d0beb737db8237585fd74b3ebb39e2fd Mon Sep 17 00:00:00 2001 From: Jinjing <6427696+AmethystLiang@users.noreply.github.com> Date: Sun, 1 Nov 2026 18:07:54 -0700 Subject: [PATCH] Bound agent journal queue during startup (#37247) * Bound agent journal queue before reading control binds Limit startup queue growth by bytes and operation count, while preserving backpressure and allowing final Pi events to drain after exit. * Test read pause and resume for large Claude messages --- ...ructured-agent-session-event-sink-queue.ts | 27 ++- ...agent-session-pre-control-capacity.test.ts | 155 ++++++++++++++++++ ...ured-agent-session-received-output.test.ts | 6 +- src/main/pi/rpc-session-adapter.test.ts | 18 +- src/main/pi/rpc-session-startup.test.ts | 33 +++- src/main/pi/rpc-session.ts | 7 +- 6 files changed, 233 insertions(+), 13 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-pre-control-capacity.test.ts diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink-queue.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink-queue.ts index d248f96c2c1..a6f7c5368a4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink-queue.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink-queue.ts @@ -21,15 +21,18 @@ export type StructuredAgentSessionSinkOperation = { type Admitted = StructuredAgentSessionSinkOperation & { superseded?: boolean } +const MAX_PRE_CONTROL_OPERATIONS = 1_024 +const MAX_PRE_CONTROL_BYTES = 32 * 1024 * 1024 + export type StructuredAgentSessionDrainWaiter = { through: number resolve: (result: StructuredAgentSessionSinkBarrier) => void } -/** Buffer only before bind, then hand writes directly to the journal in issuance order. - * Pressure pauses reading but never refuses output a provider already delivered. */ +/** Hand writes to the journal in issuance order; bound startup work until reading can pause. */ export class StructuredAgentSessionSinkQueue { private readingControl: StructuredAgentSessionReadingControl | undefined + private hasBoundReadingControl: boolean private target: StructuredAgentSessionEventTarget | null = null private closed = false private failure: { error: unknown } | null = null @@ -57,6 +60,7 @@ export class StructuredAgentSessionSinkQueue { } ) { this.readingControl = deps.readingControl + this.hasBoundReadingControl = deps.readingControl !== undefined } state = (): StructuredAgentSessionSinkState => ({ @@ -79,6 +83,7 @@ export class StructuredAgentSessionSinkQueue { bindReadingControl(control: StructuredAgentSessionReadingControl): () => void { this.readingControl = control + this.hasBoundReadingControl = true if (this.backpressured) { control.pauseReading() } @@ -144,6 +149,16 @@ export class StructuredAgentSessionSinkQueue { : key === undefined ? undefined : this.waitingPublications.get(key) + const queuedBytes = + this.queuedBytes + operation.bytes - (replaceAt >= 0 ? (replaced?.bytes ?? 0) : 0) + const queuedOperations = this.queuedOperations + (replaceAt >= 0 ? 0 : 1) + if ( + !this.hasBoundReadingControl && + (queuedBytes > MAX_PRE_CONTROL_BYTES || queuedOperations > MAX_PRE_CONTROL_OPERATIONS) + ) { + this.fail(new Error('Agent journal queue capacity exceeded before reading control bound')) + return { accepted: false, reason: 'failed' } + } const accepted: Admitted = { ...operation, sequence: ++this.acceptedSequence } if (replaceAt >= 0) { this.buffered.splice(replaceAt, 1) @@ -151,8 +166,8 @@ export class StructuredAgentSessionSinkQueue { // Superseded publications still settle in order as no-ops. replaced.superseded = true } - this.queuedBytes += operation.bytes - (replaced?.bytes ?? 0) - this.queuedOperations += replaced ? 0 : 1 + this.queuedBytes = queuedBytes + this.queuedOperations = queuedOperations if (this.target) { this.handOver(accepted, this.target) } else { @@ -189,9 +204,7 @@ export class StructuredAgentSessionSinkQueue { } private settle(operation: Admitted): void { - if (!operation.superseded) { - this.release(operation) - } + this.release(operation) this.settledSequence = Math.max(this.settledSequence, operation.sequence) this.updateBackpressure() this.settleWaiters() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-pre-control-capacity.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-pre-control-capacity.test.ts new file mode 100644 index 00000000000..8a69af16f70 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-pre-control-capacity.test.ts @@ -0,0 +1,155 @@ +import { afterEach, expect, it, vi } from 'vitest' +import { + closeProviderTimelineRigs, + openProviderTimelineRig +} from '../agent-session-timeline/provider-timeline-assembler-test-support' +import { StructuredAgentSessionSinkQueue } from './structured-agent-session-event-sink-queue' + +function queue(onFailed?: (error: unknown) => void) { + return new StructuredAgentSessionSinkQueue({ + watermarks: { + pauseQueuedBytes: 16 * 1024 * 1024, + lowQueuedBytes: 8 * 1024 * 1024, + pauseQueuedOperations: 512, + lowQueuedOperations: 256 + }, + onFailed + }) +} + +afterEach(closeProviderTimelineRigs) + +it('fails before retaining more than the pre-control byte limit and fails its barriers', async () => { + const failures: unknown[] = [] + const sink = queue((error) => failures.push(error)) + expect(sink.submit({ bytes: 32 * 1024 * 1024, run: () => {} })).toEqual({ accepted: true }) + const barrier = sink.barrier() + expect(sink.submit({ bytes: 1, run: () => {} })).toEqual({ + accepted: false, + reason: 'failed' + }) + expect(sink.state()).toMatchObject({ failed: true, queuedBytes: 0, queuedOperations: 0 }) + await expect(barrier).resolves.toMatchObject({ ok: false }) + expect(sink.submit({ bytes: 1, run: () => {} }).accepted).toBe(false) + expect(failures).toHaveLength(1) +}) + +it('releases replaced buffered publications from the pre-control budget', async () => { + const sink = queue() + for (let index = 0; index < 2_048; index++) { + expect( + sink.submit({ bytes: 32 * 1024 * 1024, publicationKey: 'status', run: () => {} }) + ).toEqual({ accepted: true }) + } + expect(sink.state()).toMatchObject({ failed: false, queuedOperations: 1 }) + sink.close() + await expect(sink.barrier()).resolves.toEqual({ ok: true }) +}) + +it('bounds superseded publications still retained on a blocked journal lane', async () => { + const rig = await openProviderTimelineRig() + const sink = queue() + sink.bind({ journal: rig.journal, fence: 1, publish: () => {} }) + const gate = Promise.withResolvers() + const blocked = rig.journal.readInOrder(() => { + for (let index = 0; index < 1_024; index++) { + expect(sink.submit({ bytes: 1, publicationKey: 'status', run: () => {} })).toEqual({ + accepted: true + }) + } + expect(sink.state()).toMatchObject({ queuedOperations: 1_024, failed: false }) + expect(sink.submit({ bytes: 1, publicationKey: 'status', run: () => {} })).toEqual({ + accepted: false, + reason: 'failed' + }) + return gate.promise + }) + try { + gate.resolve() + await blocked + } finally { + gate.resolve() + await sink.barrier() + } + expect(sink.state()).toMatchObject({ queuedOperations: 0, queuedBytes: 0, failed: true }) +}) + +it('paces superseded publications after reading control binds and publishes only the latest', async () => { + const rig = await openProviderTimelineRig() + const sink = queue() + const control = { pauseReading: vi.fn(), resumeReading: vi.fn() } + sink.bindReadingControl(control) + sink.bind({ journal: rig.journal, fence: 1, publish: () => {} }) + const published: number[] = [] + const gate = Promise.withResolvers() + const blocked = rig.journal.readInOrder(() => { + for (let index = 0; index < 1_025; index++) { + expect( + sink.submit({ bytes: 1, publicationKey: 'status', run: () => void published.push(index) }) + ).toEqual({ accepted: true }) + } + return gate.promise + }) + try { + expect(control.pauseReading).toHaveBeenCalledOnce() + expect(control.resumeReading).not.toHaveBeenCalled() + expect(sink.state()).toMatchObject({ queuedOperations: 1_025, failed: false }) + gate.resolve() + await blocked + await expect(sink.barrier()).resolves.toEqual({ ok: true }) + expect(published).toEqual([1_024]) + expect(sink.state()).toMatchObject({ queuedBytes: 0, queuedOperations: 0 }) + expect(control.resumeReading).toHaveBeenCalledOnce() + } finally { + gate.resolve() + await sink.barrier() + sink.close() + } +}) + +it.each([ + { limit: 'operations', count: 1_025, bytes: 1 }, + { limit: 'bytes', count: 1, bytes: 33 * 1024 * 1024 } +])( + 'drains final writes past the startup $limit limit after control unbinds', + async ({ count, bytes }) => { + const rig = await openProviderTimelineRig() + const sink = queue() + const control = { pauseReading: vi.fn(), resumeReading: vi.fn() } + const unbind = sink.bindReadingControl(control) + const written: number[] = [] + for (let index = 0; index < count; index++) { + expect(sink.submit({ bytes, run: () => void written.push(index) })).toEqual({ + accepted: true + }) + } + expect(control.pauseReading).toHaveBeenCalledOnce() + unbind() + + expect(sink.submit({ bytes: 1, run: () => void written.push(count) })).toEqual({ + accepted: true + }) + sink.bind({ journal: rig.journal, fence: 1, publish: () => {} }) + await expect(sink.written()).resolves.toEqual({ ok: true }) + expect(written).toEqual(Array.from({ length: count + 1 }, (_, index) => index)) + expect(sink.state()).toMatchObject({ failed: false, queuedBytes: 0, queuedOperations: 0 }) + expect(control.resumeReading).not.toHaveBeenCalled() + sink.close() + } +) + +it('reclaims the pre-control budget after writes settle', async () => { + const rig = await openProviderTimelineRig() + const sink = queue() + sink.bind({ journal: rig.journal, fence: 1, publish: () => {} }) + const written: number[] = [] + for (let index = 0; index < 2; index++) { + expect(sink.submit({ bytes: 32 * 1024 * 1024, run: () => void written.push(index) })).toEqual({ + accepted: true + }) + await expect(sink.barrier()).resolves.toEqual({ ok: true }) + expect(sink.state()).toMatchObject({ queuedBytes: 0, queuedOperations: 0 }) + } + expect(written).toEqual([0, 1]) + sink.close() +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-received-output.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-received-output.test.ts index 890168ab553..de783ee366e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-received-output.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-received-output.test.ts @@ -27,9 +27,11 @@ it('saves every tool result past 1,024 in one received Claude message, through r database: openTestJournalHostDatabase(root) }) const onFailed = vi.fn() + const readingControl = { pauseReading: vi.fn(), resumeReading: vi.fn() } const deferred = createDeferredStructuredAgentSessionEventSink({ ...testEventSinkLogging(), - onFailed + onFailed, + readingControl }) deferred.bind({ journal, fence: 1, publish: () => {} }) const translator = createClaudeJournalTranslator({ sink: deferred.sink }) @@ -54,6 +56,8 @@ it('saves every tool result past 1,024 in one received Claude message, through r }) await expect(deferred.drained()).resolves.toEqual({ ok: true }) expect(onFailed).not.toHaveBeenCalled() + expect(readingControl.pauseReading).toHaveBeenCalledOnce() + expect(readingControl.resumeReading).toHaveBeenCalledOnce() await journal.close() await closeTestJournalHostDatabase(root) journal = await openAgentSessionJournal({ diff --git a/src/main/pi/rpc-session-adapter.test.ts b/src/main/pi/rpc-session-adapter.test.ts index c4437bc35c4..abd43ddda31 100644 --- a/src/main/pi/rpc-session-adapter.test.ts +++ b/src/main/pi/rpc-session-adapter.test.ts @@ -181,9 +181,13 @@ describe('Pi RPC session ownership and delivery', () => { expect(h.connections).toHaveLength(2) }) - it('persists final events after exit while journal writes are blocked', async () => { + it('persists a large final message and interrupts its turn after exit while journal writes are blocked', async () => { const h = await setup() const gate = Promise.withResolvers() + const content = Array.from({ length: 1_025 }, (_, index) => ({ + type: 'text', + text: `final block ${index}` + })) try { const blocked = h.rig.journal.readInOrder(() => { expect( @@ -196,6 +200,8 @@ describe('Pi RPC session ownership and delivery', () => { return gate.promise }) vi.useFakeTimers() + h.connection.receive({ type: 'agent_start' }) + h.connection.receive({ type: 'message_end', message: { role: 'assistant', content } }) h.connection.receive({ type: 'extension_ui_request', id: 'last-dialog', @@ -221,6 +227,16 @@ describe('Pi RPC session ownership and delivery', () => { (row) => row.body.kind === 'question' && row.body.question === 'Final tail' ) ).toBe(true) + const reopened = await h.rig.reopenJournal() + const texts = reopened + .snapshot() + .items.flatMap(({ body }) => + body.kind === 'message' && body.role === 'assistant' + ? body.blocks.flatMap((block) => (block.type === 'text' ? [block.text] : [])) + : [] + ) + expect(texts).toEqual(content.map((block) => block.text)) + expect(await h.rig.turns()).toMatchObject([{ state: 'interrupted' }]) } finally { gate.resolve() vi.useRealTimers() diff --git a/src/main/pi/rpc-session-startup.test.ts b/src/main/pi/rpc-session-startup.test.ts index 88da63c9d05..438671c19e6 100644 --- a/src/main/pi/rpc-session-startup.test.ts +++ b/src/main/pi/rpc-session-startup.test.ts @@ -270,7 +270,7 @@ it('paces Pi reading from journal pressure only after startup, and keeps every r { turnScope: { kind: 'thread' } } ) try { - for (let index = 0; index < 1025; index++) { + for (let index = 0; index < 1024; index++) { status(`received-${index}`) } reply.resolve(state) @@ -307,7 +307,7 @@ it('paces Pi reading from journal pressure only after startup, and keeps every r const texts = reopened .snapshot() .items.flatMap((row) => (row.body.kind === 'status' ? [row.body.text] : [])) - expect(texts.filter((text) => text.startsWith('received-'))).toHaveLength(1025) + expect(texts.filter((text) => text.startsWith('received-'))).toHaveLength(1024) expect(texts.filter((text) => text.startsWith('held-'))).toHaveLength(1025) } finally { gate.resolve() @@ -317,3 +317,32 @@ it('paces Pi reading from journal pressure only after startup, and keeps every r deferred.close() } }) + +it('bounds live status transitions while Pi startup has no journal reading control', async () => { + const h = await setup() + const deferred = createDeferredStructuredAgentSessionEventSink(testEventSinkLogging()) + const reply = Promise.withResolvers() + const { acquired, child } = await restart( + h, + (connection) => { + connection.requestOverride = (command) => + command === 'get_state' ? reply.promise : undefined + }, + { events: deferred.sink } + ) + const opening = openAt8(h, acquired.acquisitionGeneration) + try { + for (let index = 0; index < 1_025; index++) { + child.receive({ type: 'auto_compaction_start' }) + } + expect(deferred.state().failed).toBe(true) + expect(deferred.state().queuedOperations).toBe(0) + reply.resolve(state) + await expect(opening).rejects.toThrow('failed') + await expect(deferred.drained()).resolves.toMatchObject({ ok: false }) + } finally { + reply.resolve(state) + await opening.catch(() => {}) + deferred.close() + } +}) diff --git a/src/main/pi/rpc-session.ts b/src/main/pi/rpc-session.ts index 0ffb2d3c1b4..9aae0b4b97d 100644 --- a/src/main/pi/rpc-session.ts +++ b/src/main/pi/rpc-session.ts @@ -262,7 +262,6 @@ export class PiRpcSession { return } this.publishedExit = true - this.unbindReadingControl?.() const observedAt = Date.now() this.input.events?.observeExit?.(this.heldSends, observedAt) const cause = (this.failedCause ??= error) @@ -286,7 +285,11 @@ export class PiRpcSession { }, 2_000) timer.unref() }) - ]).finally(() => clearTimeout(timer)) + ]).finally(() => { + clearTimeout(timer) + // Final transfer must not treat an already received backlog as startup work. + this.unbindReadingControl?.() + }) this.deps.onLifecycle({ type: 'ended', ...(this.connection.lastCloseResult?.killDelivered && this.connection.rootVerdict !== 'exited'