From 17de2ba983bee5cb7df700476ec8a0169c97ecbd Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 21 Oct 2026 18:02:18 +0200 Subject: [PATCH] fix: compact the AI chat history between tool iterations (#11911) * fix: compact the AI chat history between tool iterations Co-Authored-By: Claude Opus 5.5 * fix: budget the in-flight turn when compacting mid-turn Co-Authored-By: Claude Opus 5.5 * test: pin mid-turn compaction off a provider usage report Co-Authored-By: Claude Opus 5.5 * fix: leave unreplayed reasoning out of the mid-turn size check Co-Authored-By: Claude Opus 5.5 * fix: count replayed anthropic thinking in the mid-turn size check Co-Authored-By: Claude Opus 5.5 * fix: leave room for replayed thinking when compacting mid-turn Co-Authored-By: Claude Opus 5.5 * test: assert the second mid-turn summary folds in the first Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../copilot/chat/AIChatManager.svelte.ts | 199 ++++++++++++------ .../copilot/chat/AIChatManager.test.ts | 180 ++++++++++++++++ .../lib/components/copilot/chat/chatLoop.ts | 11 +- 3 files changed, 327 insertions(+), 63 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 6c26adb3ae..ecac52cd0c 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -119,6 +119,7 @@ import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core' import { closeInterruptedToolBatch, runChatLoop, + type ChatLoopConfig, sendsWebSearch, truncateToToolPairedPrefix } from './chatLoop' @@ -1719,8 +1720,15 @@ export class AIChatManager implements ChatViewHost { * Returns true on success. Returns false — caller falls back to drop-oldest — * when summarization isn't worthwhile or fails (user abort, empty summary, or * the circuit breaker being tripped). + * + * `inFlightTokens` is the running turn's output, which is not in `messages` but + * rides on every request after it: the tail shrinks by that much, or the next + * tool result re-triggers at once and drop-oldest discards this summary. */ - private summarizeAndCompact = async (contextWindow: number): Promise => { + private summarizeAndCompact = async ( + contextWindow: number, + inFlightTokens = 0 + ): Promise => { if (this.consecutiveCompactionFailures >= MAX_CONSECUTIVE_COMPACTION_FAILURES) { return false } @@ -1729,13 +1737,14 @@ export class AIChatManager implements ChatViewHost { return false } - const tailBudget = + const historyBudget = contextWindow * COMPACTION_TARGET_RATIO - SUMMARY_OUTPUT_RESERVE_TOKENS - this.estimateOverheadTokens() - if (tailBudget <= 0) { + if (historyBudget <= 0) { return false } + const tailBudget = historyBudget - inFlightTokens const last = this.messages.length - 1 if (last < 1) { @@ -1769,7 +1778,10 @@ export class AIChatManager implements ChatViewHost { const prefix = this.messages.slice(0, keepFrom) const tail = this.messages.slice(keepFrom) - if (prefix.length < MIN_PREFIX_MESSAGES_TO_SUMMARIZE || tail.length === 0) { + // Mid-turn, drop-oldest would cut context the running turn may rely on, so any + // prefix beyond a lone previous summary is worth the round-trip. + const minPrefix = inFlightTokens > 0 ? 2 : MIN_PREFIX_MESSAGES_TO_SUMMARIZE + if (prefix.length < minPrefix || tail.length === 0) { return false } @@ -1802,6 +1814,65 @@ export class AIChatManager implements ChatViewHost { return false } + /** + * Automatic compaction trigger, checked when a turn is sent and again before + * each tool-loop iteration, since one turn's tool results can carry the request + * past the window on their own. Rewrites the stored history (not the turn in + * flight) once `projectedTokens` reaches the trigger ratio of `model`'s window. + * Returns whether it rewrote; callers persist. An unrecognized model still gets + * the conservative assumed window rather than no limit: without one the context + * grows unbounded until the provider (or a proxy in front of it) times out. + * Guessing low only compacts earlier, which is always recoverable. + */ + private compactIfOverTrigger = async ( + projectedTokens: number, + model: ChatLoopConfig['modelProvider'] | undefined, + inFlightTokens = 0 + ): Promise => { + const contextWindow = model + ? getEffectiveModelContextWindow( + model.provider, + model.model, + get(copilotInfo).contextWindowPerModel + ) + : undefined + if (contextWindow === undefined || projectedTokens < contextWindow * COMPACTION_TRIGGER_RATIO) { + return false + } + // Preferred path: summarize the older prefix, keep the recent tail. + const summarized = await this.summarizeAndCompact(contextWindow, inFlightTokens) + // A Stop during the in-flight summary aborts this turn's controller; + // summarizeAndCompact then returns false without touching history. Skip + // the drop-oldest fallback — it would destructively compact a conversation + // the user only meant to cancel, and the request can't run on an aborted + // controller anyway. The cancel path rolls the turn back cleanly on its own. + if (this.abortController?.signal.aborted) { + return false + } + if (!summarized) { + // Fallback when summarization isn't worthwhile or fails: drop the + // oldest messages. A report stays meaningful only debited by what + // was dropped; the estimate path needs no bookkeeping — the next + // read re-estimates the compacted history. chars/4 can + // underestimate the freed tokens, which errs toward compacting + // again — never toward overflowing. + const lengthBefore = this.messages.length + const freed = this.compactOldestMessages( + projectedTokens - contextWindow * COMPACTION_TARGET_RATIO + ) + if (this.messages.length === lengthBefore) { + return false + } + if (this.contextUsage !== undefined) { + this.contextUsage = Math.max(0, this.contextUsage - freed) + } + } + // Reconcile file registrations with the compacted transcript — the + // summary message carries the folded-away turns' files forward. + this.#syncMessageFiles() + return true + } + /** * Manual compaction (the `/compact` session command): summarize the ENTIRE * stored history into a single summary message and keep nothing verbatim, so @@ -3155,7 +3226,8 @@ export class AIChatManager implements ChatViewHost { callbacks, addedMessages, systemMessage: systemMessageOverride, - onWebSearchUnavailable + onWebSearchUnavailable, + onBeforeIteration }: { messages: ChatCompletionMessageParam[] abortController: AbortController @@ -3167,6 +3239,7 @@ export class AIChatManager implements ChatViewHost { addedMessages?: ChatCompletionMessageParam[] systemMessage?: ChatCompletionSystemMessageParam onWebSearchUnavailable?: () => void + onBeforeIteration?: ChatLoopConfig['onBeforeIteration'] }) => { // Fresh batch for this turn — drop any images an aborted prior turn left buffered. this.pendingToolImages.clear() @@ -3269,8 +3342,9 @@ export class AIChatManager implements ChatViewHost { console.error('Failed to record AI usage', e) } }, - onBeforeIteration: async (modelProvider) => { + onBeforeIteration: async (modelProvider, lastIterationUsage) => { this.lastIterationModel = modelProvider + await onBeforeIteration?.(modelProvider, lastIterationUsage) } }) if (this.isSessionChat && this.sessionId && result.tokenUsage.total > 0) { @@ -4113,61 +4187,24 @@ export class AIChatManager implements ChatViewHost { this.currentReasoningActive = false this.resetReasoningTiming() - // Compaction trigger. An unrecognized model still gets the conservative - // assumed window rather than no limit: without one the context grows - // unbounded until the provider (or a proxy in front of it) times out. - // Guessing low only compacts earlier, which is always recoverable. - const contextWindow = model - ? getEffectiveModelContextWindow( - model.provider, - model.model, - get(copilotInfo).contextWindowPerModel - ) - : undefined - if ( - contextWindow !== undefined && - projectedContextTokens >= contextWindow * COMPACTION_TRIGGER_RATIO - ) { - // Preferred path: summarize the older prefix, keep the recent tail. - const summarized = await this.summarizeAndCompact(contextWindow) - // A Stop during the in-flight summary aborts this turn's controller; - // summarizeAndCompact then returns false without touching history. Skip - // the drop-oldest fallback (and its save) — it would destructively - // compact a conversation the user only meant to cancel, and the request - // can't run on an aborted controller anyway. The cancel path below rolls - // the pushed turn back cleanly on its own. - if (!this.abortController?.signal.aborted) { - if (!summarized) { - // Fallback when summarization isn't worthwhile or fails: drop the - // oldest messages. A report stays meaningful only debited by what - // was dropped; the estimate path needs no bookkeeping — the next - // read re-estimates the compacted history. chars/4 can - // underestimate the freed tokens, which errs toward compacting - // again — never toward overflowing. - const freed = this.compactOldestMessages( - projectedContextTokens - contextWindow * COMPACTION_TARGET_RATIO - ) - if (this.contextUsage !== undefined) { - this.contextUsage = Math.max(0, this.contextUsage - freed) - } - } - // Reconcile file registrations with the compacted transcript — the - // summary message carries the folded-away turns' files forward. - this.#syncMessageFiles() - await this.historyManager.saveChat( - this.displayMessages, - this.messages, - this.contextUsage, - this.modifiedItems ? [...this.modifiedItems] : undefined - ) - } + if (await this.compactIfOverTrigger(projectedContextTokens, model)) { + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) } // Rollback anchors for restoreUnsentTurn: captured after compaction so // they index into the (possibly compacted) stored history. The summary // path shrinks displayMessages too, so the display anchor must also be // read here, not before compaction. + // Mid-turn compaction leaves these stale, but it runs only once the turn has + // tool-paired output, which keeps the turn off the restore path. const modelLenAfterUser = this.messages.length const displayLenAfterUser = this.displayMessages.length + // Set when a mid-turn compaction makes the latest provider report stale. + let usageBeforeCompaction: ChatTokenUsage | null = null const params: { messages: ChatCompletionMessageParam[] @@ -4295,6 +4332,46 @@ export class AIChatManager implements ChatViewHost { addedMessages: collectedMessages, onWebSearchUnavailable: () => { webSearchUnavailable = true + }, + onBeforeIteration: async (iterationModel, lastIterationUsage) => { + // The first iteration sends exactly what the send-time check measured. + if (collectedMessages.length === 0) return + // The last report's prompt covers everything before its assistant message. + // That message counts as its `completion` only on Anthropic, which replays + // its thinking verbatim (`_anthropicContent`); other providers bill reasoning + // the next request does not replay, so it is estimated. The estimate is + // the floor in case a later iteration's report never arrived. A report + // from before a compaction measured the history it replaced. + const lastAssistant = collectedMessages.findLastIndex((m) => m.role === 'assistant') + const replaysOutput = !!(collectedMessages[lastAssistant] as any)?._anthropicContent + const reported = + lastIterationUsage && lastIterationUsage !== usageBeforeCompaction + ? lastIterationUsage.prompt + + (replaysOutput ? lastIterationUsage.completion : 0) + + this.estimateMessagesTokens( + collectedMessages.slice(replaysOutput ? lastAssistant + 1 : lastAssistant) + ) + : 0 + const historyTokens = this.estimateWholeContextTokens() + // Whatever the report sees beyond the stored history is the turn's, replayed + // thinking included, so the summary leaves room for it. + const inFlightTokens = Math.max( + this.estimateMessagesTokens(collectedMessages), + reported - historyTokens + ) + const projected = Math.max(reported, historyTokens + inFlightTokens) + if (await this.compactIfOverTrigger(projected, iterationModel, inFlightTokens)) { + usageBeforeCompaction = lastIterationUsage + // The loop sends from this array, which mirrors the stored history + // plus the turn so far. + params.messages.splice( + 0, + params.messages.length, + ...this.messages, + ...collectedMessages + ) + await checkpointTurn(true) + } } }) stopCheckpoints() @@ -4316,9 +4393,10 @@ export class AIChatManager implements ChatViewHost { // stored history it was sent with (the kept partial tail is a small // undercount the trigger headroom absorbs). Without one, clear the // stale value — readers estimate via contextTokens. - this.contextUsage = result?.lastIterationUsage - ? result.lastIterationUsage.prompt + result.lastIterationUsage.completion - : undefined + this.contextUsage = + result?.lastIterationUsage && result.lastIterationUsage !== usageBeforeCompaction + ? result.lastIterationUsage.prompt + result.lastIterationUsage.completion + : undefined await this.historyManager.saveChat( this.displayMessages, this.messages, @@ -4387,9 +4465,10 @@ export class AIChatManager implements ChatViewHost { // stored — no anchoring or index bookkeeping needed. Without a // report, clear the now-stale value — readers estimate via // contextTokens. - this.contextUsage = result?.lastIterationUsage - ? result.lastIterationUsage.prompt + result.lastIterationUsage.completion - : undefined + this.contextUsage = + result?.lastIterationUsage && result.lastIterationUsage !== usageBeforeCompaction + ? result.lastIterationUsage.prompt + result.lastIterationUsage.completion + : undefined if (this.autoAcceptEditsActive) { this.acceptPendingFlowEdits() } diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index 1a87b03024..1fd92aa67d 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -3403,6 +3403,186 @@ describe('AIChatManager context compaction', () => { manager.instructions = 'next question' } + it('compacts between tool iterations once tool results carry the turn over the trigger', async () => { + mocks.getCurrentModel.mockReturnValue(gpt4oModel) + mocks.tryGetCurrentModel.mockReturnValue(gpt4oModel) + mocks.getStreamedCompletionText + .mockResolvedValueOnce('SUMMARY ONE') + .mockResolvedValueOnce('SUMMARY TWO') + const step = (id: string, chars: number) => [ + { + role: 'assistant' as const, + content: '', + tool_calls: [{ id, type: 'function', function: { name: 'x', arguments: '{}' } }] + }, + { role: 'tool' as const, tool_call_id: id, content: 'r'.repeat(chars) } + ] + // ~25k estimated tokens: enough to push a ~60% history past the 0.8 * 128k trigger. + const first = step('t1', 100_000) + // ~70k more: crosses the trigger again within the same turn. + const second = step('t2', 280_000) + let secondIteration: any[] = [] + let thirdIteration: any[] = [] + mocks.runChatLoop.mockImplementation(async (config: any) => { + const push = (ms: any[]) => { + for (const m of ms) { + config.messages.push(m) + config.addedMessages.push(m) + } + } + await config.onBeforeIteration(gpt4oModel, null) + push(first) + await config.onBeforeIteration(gpt4oModel, null) + secondIteration = [...config.messages] + push(second) + await config.onBeforeIteration(gpt4oModel, null) + thirdIteration = [...config.messages] + const reply = { role: 'assistant' as const, content: 'done' } + push([reply]) + return { + addedMessages: [...first, ...second, reply], + tokenUsage: { prompt: 0, completion: 0, total: 0 }, + lastIterationUsage: null, + hitMaxIterations: false + } + }) + const manager = new AIChatManager() + // ~80k estimated tokens over ten messages: under the trigger at send time. + manager.messages = Array.from({ length: 10 }, (_, i) => ({ + role: i % 2 === 0 ? ('user' as const) : ('assistant' as const), + content: `m${i}` + 'x'.repeat(32_000) + })) + manager.displayMessages = manager.messages.map((m, i) => + m.role === 'user' + ? { role: 'user', content: `m${i}`, index: i } + : { role: 'assistant', content: `m${i}` } + ) + manager.instructions = 'next question' + + await manager.sendRequest() + + // Not at send time (under the trigger), only once the tool result landed. + expect(secondIteration[0].content).toContain('SUMMARY ONE') + expect(secondIteration.slice(-2)).toEqual(first) + // The second trigger folds the first summary into a new one instead of + // dropping it to make room for the turn. + expect(mocks.getStreamedCompletionText).toHaveBeenCalledTimes(2) + expect(mocks.getStreamedCompletionText.mock.calls[1][0][0].content).toContain('SUMMARY ONE') + expect(thirdIteration[0].content).toContain('SUMMARY TWO') + expect(thirdIteration.slice(-4)).toEqual([...first, ...second]) + expect(manager.messages[0].content).toContain('SUMMARY TWO') + expect(manager.displayMessages[0].role).toBe('summary') + }) + + it('compacts mid-turn off the provider report, then stops trusting the report it replaced', async () => { + mocks.getCurrentModel.mockReturnValue(gpt4oModel) + mocks.tryGetCurrentModel.mockReturnValue(gpt4oModel) + mocks.getStreamedCompletionText.mockResolvedValue('SUMMARY ONE') + const step = (id: string) => [ + { + role: 'assistant' as const, + content: '', + tool_calls: [{ id, type: 'function', function: { name: 'x', arguments: '{}' } }] + }, + { role: 'tool' as const, tool_call_id: id, content: 'r'.repeat(8_000) } + ] + // The provider counts far more than chars/4 sees: over the 0.8 * 128k trigger. + const report = { prompt: 101_000, completion: 1_000, total: 102_000 } + let thirdIteration: any[] = [] + mocks.runChatLoop.mockImplementation(async (config: any) => { + const push = (ms: any[]) => { + for (const m of ms) { + config.messages.push(m) + config.addedMessages.push(m) + } + } + await config.onBeforeIteration(gpt4oModel, null) + push(step('t1')) + await config.onBeforeIteration(gpt4oModel, report) + push(step('t2')) + // The iteration after the compaction reported nothing, so the loop hands the + // pre-compaction report back. + await config.onBeforeIteration(gpt4oModel, report) + thirdIteration = [...config.messages] + const reply = { role: 'assistant' as const, content: 'done' } + push([reply]) + return { + addedMessages: [...step('t1'), ...step('t2'), reply], + tokenUsage: report, + lastIterationUsage: report, + hitMaxIterations: false + } + }) + const manager = new AIChatManager() + // ~75k estimated tokens: under the trigger by the estimate alone. + manager.messages = Array.from({ length: 10 }, (_, i) => ({ + role: i % 2 === 0 ? ('user' as const) : ('assistant' as const), + content: `m${i}` + 'x'.repeat(30_000) + })) + manager.displayMessages = manager.messages.map((m, i) => + m.role === 'user' + ? { role: 'user', content: `m${i}`, index: i } + : { role: 'assistant', content: `m${i}` } + ) + manager.instructions = 'next question' + + await manager.sendRequest() + + expect(mocks.getStreamedCompletionText).toHaveBeenCalledTimes(1) + expect(thirdIteration[0].content).toContain('SUMMARY ONE') + // Nor does the stale report become the stored usage at commit. + expect(manager.contextUsage).toBeUndefined() + }) + + it('counts replayed Anthropic thinking toward the mid-turn projection', async () => { + mocks.getCurrentModel.mockReturnValue(gpt4oModel) + mocks.tryGetCurrentModel.mockReturnValue(gpt4oModel) + mocks.getStreamedCompletionText.mockResolvedValue('SUMMARY ONE') + // Under the 0.8 * 128k trigger without the thinking the next request replays. + const report = { prompt: 80_000, completion: 25_000, total: 105_000 } + let historyAfter: any[] = [] + mocks.runChatLoop.mockImplementation(async (config: any) => { + await config.onBeforeIteration(gpt4oModel, null) + const assistant = { + role: 'assistant' as const, + content: '', + tool_calls: [{ id: 't1', type: 'function', function: { name: 'x', arguments: '{}' } }], + _anthropicContent: [{ type: 'thinking', thinking: '...', signature: 'sig' }] + } + const tool = { role: 'tool' as const, tool_call_id: 't1', content: 'r'.repeat(4_000) } + for (const m of [assistant, tool]) { + config.messages.push(m) + config.addedMessages.push(m) + } + await config.onBeforeIteration(gpt4oModel, report) + historyAfter = config.messages.slice(0, -2) + return { + addedMessages: [assistant, tool], + tokenUsage: report, + lastIterationUsage: report, + hitMaxIterations: false + } + }) + const manager = new AIChatManager() + manager.messages = Array.from({ length: 10 }, (_, i) => ({ + role: i % 2 === 0 ? ('user' as const) : ('assistant' as const), + content: `m${i}` + 'x'.repeat(30_000) + })) + manager.displayMessages = manager.messages.map((m, i) => + m.role === 'user' + ? { role: 'user', content: `m${i}`, index: i } + : { role: 'assistant', content: `m${i}` } + ) + manager.instructions = 'next question' + + await manager.sendRequest() + + expect(mocks.getStreamedCompletionText).toHaveBeenCalledTimes(1) + // The kept history leaves room for the replayed thinking within the 0.5 target. + const keptTokens = historyAfter.reduce((acc, m) => acc + String(m.content).length / 4, 0) + expect(keptTokens + report.completion).toBeLessThan(128_000 * 0.5) + }) + it('summarizes the older prefix and keeps the recent tail verbatim', async () => { mocks.getCurrentModel.mockReturnValue(gpt4oModel) mocks.tryGetCurrentModel.mockReturnValue(gpt4oModel) diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.ts b/frontend/src/lib/components/copilot/chat/chatLoop.ts index 343dd268e7..09232debbb 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.ts @@ -73,8 +73,13 @@ export interface ChatLoopConfig { * lets the caller recover partial output if the loop throws or is aborted. */ addedMessages?: ChatCompletionMessageParam[] - /** Called before each iteration (e.g. to record which model it is about to use). */ - onBeforeIteration?: (modelProvider: ReasoningProviderModel) => Promise + /** Called before each iteration (e.g. to record which model it is about to use), with + * the last usage report that carried a prompt count, which can predate the latest + * iteration. It may rewrite `messages` in place. */ + onBeforeIteration?: ( + modelProvider: ReasoningProviderModel, + lastIterationUsage: ChatTokenUsage | null + ) => Promise /** Fired for each completed provider response, before the loop continues. The * loop can fail or be aborted at any iteration, so spend has to be handed over * as it happens — a callback only at the end would discard everything the @@ -420,7 +425,7 @@ export async function runChatLoop(config: ChatLoopConfig): Promise (t.schemaFor ? { ...t, def: await t.schemaFor(helpers) } : t))