fix: compact the AI chat history between tool iterations (#11911)

* fix: compact the AI chat history between tool iterations

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: budget the in-flight turn when compacting mid-turn

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: pin mid-turn compaction off a provider usage report

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: leave unreplayed reasoning out of the mid-turn size check

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: count replayed anthropic thinking in the mid-turn size check

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: leave room for replayed thinking when compacting mid-turn

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: assert the second mid-turn summary folds in the first

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts

@@ -119,6 +119,7 @@ import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core'
 import {
 	closeInterruptedToolBatch,
 	runChatLoop,
+	type ChatLoopConfig,
 	sendsWebSearch,
 	truncateToToolPairedPrefix
 } from './chatLoop'
@@ -1719,8 +1720,15 @@ export class AIChatManager implements ChatViewHost {
 	 * Returns true on success. Returns false — caller falls back to drop-oldest —
 	 * when summarization isn't worthwhile or fails (user abort, empty summary, or
 	 * the circuit breaker being tripped).
+	 *
+	 * `inFlightTokens` is the running turn's output, which is not in `messages` but
+	 * rides on every request after it: the tail shrinks by that much, or the next
+	 * tool result re-triggers at once and drop-oldest discards this summary.
 	 */
-	private summarizeAndCompact = async (contextWindow: number): Promise<boolean> => {
+	private summarizeAndCompact = async (
+		contextWindow: number,
+		inFlightTokens = 0
+	): Promise<boolean> => {
 		if (this.consecutiveCompactionFailures >= MAX_CONSECUTIVE_COMPACTION_FAILURES) {
 			return false
 		}
@@ -1729,13 +1737,14 @@ export class AIChatManager implements ChatViewHost {
 			return false
 		}
 
-		const tailBudget =
+		const historyBudget =
 			contextWindow * COMPACTION_TARGET_RATIO -
 			SUMMARY_OUTPUT_RESERVE_TOKENS -
 			this.estimateOverheadTokens()
-		if (tailBudget <= 0) {
+		if (historyBudget <= 0) {
 			return false
 		}
+		const tailBudget = historyBudget - inFlightTokens
 
 		const last = this.messages.length - 1
 		if (last < 1) {
@@ -1769,7 +1778,10 @@ export class AIChatManager implements ChatViewHost {
 
 		const prefix = this.messages.slice(0, keepFrom)
 		const tail = this.messages.slice(keepFrom)
-		if (prefix.length < MIN_PREFIX_MESSAGES_TO_SUMMARIZE || tail.length === 0) {
+		// Mid-turn, drop-oldest would cut context the running turn may rely on, so any
+		// prefix beyond a lone previous summary is worth the round-trip.
+		const minPrefix = inFlightTokens > 0 ? 2 : MIN_PREFIX_MESSAGES_TO_SUMMARIZE
+		if (prefix.length < minPrefix || tail.length === 0) {
 			return false
 		}
 
@@ -1802,6 +1814,65 @@ export class AIChatManager implements ChatViewHost {
 		return false
 	}
 
+	/**
+	 * Automatic compaction trigger, checked when a turn is sent and again before
+	 * each tool-loop iteration, since one turn's tool results can carry the request
+	 * past the window on their own. Rewrites the stored history (not the turn in
+	 * flight) once `projectedTokens` reaches the trigger ratio of `model`'s window.
+	 * Returns whether it rewrote; callers persist. An unrecognized model still gets
+	 * the conservative assumed window rather than no limit: without one the context
+	 * grows unbounded until the provider (or a proxy in front of it) times out.
+	 * Guessing low only compacts earlier, which is always recoverable.
+	 */
+	private compactIfOverTrigger = async (
+		projectedTokens: number,
+		model: ChatLoopConfig['modelProvider'] | undefined,
+		inFlightTokens = 0
+	): Promise<boolean> => {
+		const contextWindow = model
+			? getEffectiveModelContextWindow(
+					model.provider,
+					model.model,
+					get(copilotInfo).contextWindowPerModel
+				)
+			: undefined
+		if (contextWindow === undefined || projectedTokens < contextWindow * COMPACTION_TRIGGER_RATIO) {
+			return false
+		}
+		// Preferred path: summarize the older prefix, keep the recent tail.
+		const summarized = await this.summarizeAndCompact(contextWindow, inFlightTokens)
+		// A Stop during the in-flight summary aborts this turn's controller;
+		// summarizeAndCompact then returns false without touching history. Skip
+		// the drop-oldest fallback — it would destructively compact a conversation
+		// the user only meant to cancel, and the request can't run on an aborted
+		// controller anyway. The cancel path rolls the turn back cleanly on its own.
+		if (this.abortController?.signal.aborted) {
+			return false
+		}
+		if (!summarized) {
+			// Fallback when summarization isn't worthwhile or fails: drop the
+			// oldest messages. A report stays meaningful only debited by what
+			// was dropped; the estimate path needs no bookkeeping — the next
+			// read re-estimates the compacted history. chars/4 can
+			// underestimate the freed tokens, which errs toward compacting
+			// again — never toward overflowing.
+			const lengthBefore = this.messages.length
+			const freed = this.compactOldestMessages(
+				projectedTokens - contextWindow * COMPACTION_TARGET_RATIO
+			)
+			if (this.messages.length === lengthBefore) {
+				return false
+			}
+			if (this.contextUsage !== undefined) {
+				this.contextUsage = Math.max(0, this.contextUsage - freed)
+			}
+		}
+		// Reconcile file registrations with the compacted transcript — the
+		// summary message carries the folded-away turns' files forward.
+		this.#syncMessageFiles()
+		return true
+	}
+
 	/**
 	 * Manual compaction (the `/compact` session command): summarize the ENTIRE
 	 * stored history into a single summary message and keep nothing verbatim, so
@@ -3155,7 +3226,8 @@ export class AIChatManager implements ChatViewHost {
 		callbacks,
 		addedMessages,
 		systemMessage: systemMessageOverride,
-		onWebSearchUnavailable
+		onWebSearchUnavailable,
+		onBeforeIteration
 	}: {
 		messages: ChatCompletionMessageParam[]
 		abortController: AbortController
@@ -3167,6 +3239,7 @@ export class AIChatManager implements ChatViewHost {
 		addedMessages?: ChatCompletionMessageParam[]
 		systemMessage?: ChatCompletionSystemMessageParam
 		onWebSearchUnavailable?: () => void
+		onBeforeIteration?: ChatLoopConfig['onBeforeIteration']
 	}) => {
 		// Fresh batch for this turn — drop any images an aborted prior turn left buffered.
 		this.pendingToolImages.clear()
@@ -3269,8 +3342,9 @@ export class AIChatManager implements ChatViewHost {
 						console.error('Failed to record AI usage', e)
 					}
 				},
-				onBeforeIteration: async (modelProvider) => {
+				onBeforeIteration: async (modelProvider, lastIterationUsage) => {
 					this.lastIterationModel = modelProvider
+					await onBeforeIteration?.(modelProvider, lastIterationUsage)
 				}
 			})
 			if (this.isSessionChat && this.sessionId && result.tokenUsage.total > 0) {
@@ -4113,61 +4187,24 @@ export class AIChatManager implements ChatViewHost {
 			this.currentReasoningActive = false
 			this.resetReasoningTiming()
 
-			// Compaction trigger. An unrecognized model still gets the conservative
-			// assumed window rather than no limit: without one the context grows
-			// unbounded until the provider (or a proxy in front of it) times out.
-			// Guessing low only compacts earlier, which is always recoverable.
-			const contextWindow = model
-				? getEffectiveModelContextWindow(
-						model.provider,
-						model.model,
-						get(copilotInfo).contextWindowPerModel
-					)
-				: undefined
-			if (
-				contextWindow !== undefined &&
-				projectedContextTokens >= contextWindow * COMPACTION_TRIGGER_RATIO
-			) {
-				// Preferred path: summarize the older prefix, keep the recent tail.
-				const summarized = await this.summarizeAndCompact(contextWindow)
-				// A Stop during the in-flight summary aborts this turn's controller;
-				// summarizeAndCompact then returns false without touching history. Skip
-				// the drop-oldest fallback (and its save) — it would destructively
-				// compact a conversation the user only meant to cancel, and the request
-				// can't run on an aborted controller anyway. The cancel path below rolls
-				// the pushed turn back cleanly on its own.
-				if (!this.abortController?.signal.aborted) {
-					if (!summarized) {
-						// Fallback when summarization isn't worthwhile or fails: drop the
-						// oldest messages. A report stays meaningful only debited by what
-						// was dropped; the estimate path needs no bookkeeping — the next
-						// read re-estimates the compacted history. chars/4 can
-						// underestimate the freed tokens, which errs toward compacting
-						// again — never toward overflowing.
-						const freed = this.compactOldestMessages(
-							projectedContextTokens - contextWindow * COMPACTION_TARGET_RATIO
-						)
-						if (this.contextUsage !== undefined) {
-							this.contextUsage = Math.max(0, this.contextUsage - freed)
-						}
-					}
-					// Reconcile file registrations with the compacted transcript — the
-					// summary message carries the folded-away turns' files forward.
-					this.#syncMessageFiles()
-					await this.historyManager.saveChat(
-						this.displayMessages,
-						this.messages,
-						this.contextUsage,
-						this.modifiedItems ? [...this.modifiedItems] : undefined
-					)
-				}
+			if (await this.compactIfOverTrigger(projectedContextTokens, model)) {
+				await this.historyManager.saveChat(
+					this.displayMessages,
+					this.messages,
+					this.contextUsage,
+					this.modifiedItems ? [...this.modifiedItems] : undefined
+				)
 			}
 			// Rollback anchors for restoreUnsentTurn: captured after compaction so
 			// they index into the (possibly compacted) stored history. The summary
 			// path shrinks displayMessages too, so the display anchor must also be
 			// read here, not before compaction.
+			// Mid-turn compaction leaves these stale, but it runs only once the turn has
+			// tool-paired output, which keeps the turn off the restore path.
 			const modelLenAfterUser = this.messages.length
 			const displayLenAfterUser = this.displayMessages.length
+			// Set when a mid-turn compaction makes the latest provider report stale.
+			let usageBeforeCompaction: ChatTokenUsage | null = null
 
 			const params: {
 				messages: ChatCompletionMessageParam[]
@@ -4295,6 +4332,46 @@ export class AIChatManager implements ChatViewHost {
 				addedMessages: collectedMessages,
 				onWebSearchUnavailable: () => {
 					webSearchUnavailable = true
+				},
+				onBeforeIteration: async (iterationModel, lastIterationUsage) => {
+					// The first iteration sends exactly what the send-time check measured.
+					if (collectedMessages.length === 0) return
+					// The last report's prompt covers everything before its assistant message.
+					// That message counts as its `completion` only on Anthropic, which replays
+					// its thinking verbatim (`_anthropicContent`); other providers bill reasoning
+					// the next request does not replay, so it is estimated. The estimate is
+					// the floor in case a later iteration's report never arrived. A report
+					// from before a compaction measured the history it replaced.
+					const lastAssistant = collectedMessages.findLastIndex((m) => m.role === 'assistant')
+					const replaysOutput = !!(collectedMessages[lastAssistant] as any)?._anthropicContent
+					const reported =
+						lastIterationUsage && lastIterationUsage !== usageBeforeCompaction
+							? lastIterationUsage.prompt +
+								(replaysOutput ? lastIterationUsage.completion : 0) +
+								this.estimateMessagesTokens(
+									collectedMessages.slice(replaysOutput ? lastAssistant + 1 : lastAssistant)
+								)
+							: 0
+					const historyTokens = this.estimateWholeContextTokens()
+					// Whatever the report sees beyond the stored history is the turn's, replayed
+					// thinking included, so the summary leaves room for it.
+					const inFlightTokens = Math.max(
+						this.estimateMessagesTokens(collectedMessages),
+						reported - historyTokens
+					)
+					const projected = Math.max(reported, historyTokens + inFlightTokens)
+					if (await this.compactIfOverTrigger(projected, iterationModel, inFlightTokens)) {
+						usageBeforeCompaction = lastIterationUsage
+						// The loop sends from this array, which mirrors the stored history
+						// plus the turn so far.
+						params.messages.splice(
+							0,
+							params.messages.length,
+							...this.messages,
+							...collectedMessages
+						)
+						await checkpointTurn(true)
+					}
 				}
 			})
 			stopCheckpoints()
@@ -4316,9 +4393,10 @@ export class AIChatManager implements ChatViewHost {
 				// stored history it was sent with (the kept partial tail is a small
 				// undercount the trigger headroom absorbs). Without one, clear the
 				// stale value — readers estimate via contextTokens.
-				this.contextUsage = result?.lastIterationUsage
-					? result.lastIterationUsage.prompt + result.lastIterationUsage.completion
-					: undefined
+				this.contextUsage =
+					result?.lastIterationUsage && result.lastIterationUsage !== usageBeforeCompaction
+						? result.lastIterationUsage.prompt + result.lastIterationUsage.completion
+						: undefined
 				await this.historyManager.saveChat(
 					this.displayMessages,
 					this.messages,
@@ -4387,9 +4465,10 @@ export class AIChatManager implements ChatViewHost {
 				// stored — no anchoring or index bookkeeping needed. Without a
 				// report, clear the now-stale value — readers estimate via
 				// contextTokens.
-				this.contextUsage = result?.lastIterationUsage
-					? result.lastIterationUsage.prompt + result.lastIterationUsage.completion
-					: undefined
+				this.contextUsage =
+					result?.lastIterationUsage && result.lastIterationUsage !== usageBeforeCompaction
+						? result.lastIterationUsage.prompt + result.lastIterationUsage.completion
+						: undefined
 				if (this.autoAcceptEditsActive) {
 					this.acceptPendingFlowEdits()
 				}

frontend/src/lib/components/copilot/chat/AIChatManager.test.ts

@@ -3403,6 +3403,186 @@ describe('AIChatManager context compaction', () => {
 		manager.instructions = 'next question'
 	}
 
+	it('compacts between tool iterations once tool results carry the turn over the trigger', async () => {
+		mocks.getCurrentModel.mockReturnValue(gpt4oModel)
+		mocks.tryGetCurrentModel.mockReturnValue(gpt4oModel)
+		mocks.getStreamedCompletionText
+			.mockResolvedValueOnce('<summary>SUMMARY ONE</summary>')
+			.mockResolvedValueOnce('<summary>SUMMARY TWO</summary>')
+		const step = (id: string, chars: number) => [
+			{
+				role: 'assistant' as const,
+				content: '',
+				tool_calls: [{ id, type: 'function', function: { name: 'x', arguments: '{}' } }]
+			},
+			{ role: 'tool' as const, tool_call_id: id, content: 'r'.repeat(chars) }
+		]
+		// ~25k estimated tokens: enough to push a ~60% history past the 0.8 * 128k trigger.
+		const first = step('t1', 100_000)
+		// ~70k more: crosses the trigger again within the same turn.
+		const second = step('t2', 280_000)
+		let secondIteration: any[] = []
+		let thirdIteration: any[] = []
+		mocks.runChatLoop.mockImplementation(async (config: any) => {
+			const push = (ms: any[]) => {
+				for (const m of ms) {
+					config.messages.push(m)
+					config.addedMessages.push(m)
+				}
+			}
+			await config.onBeforeIteration(gpt4oModel, null)
+			push(first)
+			await config.onBeforeIteration(gpt4oModel, null)
+			secondIteration = [...config.messages]
+			push(second)
+			await config.onBeforeIteration(gpt4oModel, null)
+			thirdIteration = [...config.messages]
+			const reply = { role: 'assistant' as const, content: 'done' }
+			push([reply])
+			return {
+				addedMessages: [...first, ...second, reply],
+				tokenUsage: { prompt: 0, completion: 0, total: 0 },
+				lastIterationUsage: null,
+				hitMaxIterations: false
+			}
+		})
+		const manager = new AIChatManager()
+		// ~80k estimated tokens over ten messages: under the trigger at send time.
+		manager.messages = Array.from({ length: 10 }, (_, i) => ({
+			role: i % 2 === 0 ? ('user' as const) : ('assistant' as const),
+			content: `m${i}` + 'x'.repeat(32_000)
+		}))
+		manager.displayMessages = manager.messages.map((m, i) =>
+			m.role === 'user'
+				? { role: 'user', content: `m${i}`, index: i }
+				: { role: 'assistant', content: `m${i}` }
+		)
+		manager.instructions = 'next question'
+
+		await manager.sendRequest()
+
+		// Not at send time (under the trigger), only once the tool result landed.
+		expect(secondIteration[0].content).toContain('SUMMARY ONE')
+		expect(secondIteration.slice(-2)).toEqual(first)
+		// The second trigger folds the first summary into a new one instead of
+		// dropping it to make room for the turn.
+		expect(mocks.getStreamedCompletionText).toHaveBeenCalledTimes(2)
+		expect(mocks.getStreamedCompletionText.mock.calls[1][0][0].content).toContain('SUMMARY ONE')
+		expect(thirdIteration[0].content).toContain('SUMMARY TWO')
+		expect(thirdIteration.slice(-4)).toEqual([...first, ...second])
+		expect(manager.messages[0].content).toContain('SUMMARY TWO')
+		expect(manager.displayMessages[0].role).toBe('summary')
+	})
+
+	it('compacts mid-turn off the provider report, then stops trusting the report it replaced', async () => {
+		mocks.getCurrentModel.mockReturnValue(gpt4oModel)
+		mocks.tryGetCurrentModel.mockReturnValue(gpt4oModel)
+		mocks.getStreamedCompletionText.mockResolvedValue('<summary>SUMMARY ONE</summary>')
+		const step = (id: string) => [
+			{
+				role: 'assistant' as const,
+				content: '',
+				tool_calls: [{ id, type: 'function', function: { name: 'x', arguments: '{}' } }]
+			},
+			{ role: 'tool' as const, tool_call_id: id, content: 'r'.repeat(8_000) }
+		]
+		// The provider counts far more than chars/4 sees: over the 0.8 * 128k trigger.
+		const report = { prompt: 101_000, completion: 1_000, total: 102_000 }
+		let thirdIteration: any[] = []
+		mocks.runChatLoop.mockImplementation(async (config: any) => {
+			const push = (ms: any[]) => {
+				for (const m of ms) {
+					config.messages.push(m)
+					config.addedMessages.push(m)
+				}
+			}
+			await config.onBeforeIteration(gpt4oModel, null)
+			push(step('t1'))
+			await config.onBeforeIteration(gpt4oModel, report)
+			push(step('t2'))
+			// The iteration after the compaction reported nothing, so the loop hands the
+			// pre-compaction report back.
+			await config.onBeforeIteration(gpt4oModel, report)
+			thirdIteration = [...config.messages]
+			const reply = { role: 'assistant' as const, content: 'done' }
+			push([reply])
+			return {
+				addedMessages: [...step('t1'), ...step('t2'), reply],
+				tokenUsage: report,
+				lastIterationUsage: report,
+				hitMaxIterations: false
+			}
+		})
+		const manager = new AIChatManager()
+		// ~75k estimated tokens: under the trigger by the estimate alone.
+		manager.messages = Array.from({ length: 10 }, (_, i) => ({
+			role: i % 2 === 0 ? ('user' as const) : ('assistant' as const),
+			content: `m${i}` + 'x'.repeat(30_000)
+		}))
+		manager.displayMessages = manager.messages.map((m, i) =>
+			m.role === 'user'
+				? { role: 'user', content: `m${i}`, index: i }
+				: { role: 'assistant', content: `m${i}` }
+		)
+		manager.instructions = 'next question'
+
+		await manager.sendRequest()
+
+		expect(mocks.getStreamedCompletionText).toHaveBeenCalledTimes(1)
+		expect(thirdIteration[0].content).toContain('SUMMARY ONE')
+		// Nor does the stale report become the stored usage at commit.
+		expect(manager.contextUsage).toBeUndefined()
+	})
+
+	it('counts replayed Anthropic thinking toward the mid-turn projection', async () => {
+		mocks.getCurrentModel.mockReturnValue(gpt4oModel)
+		mocks.tryGetCurrentModel.mockReturnValue(gpt4oModel)
+		mocks.getStreamedCompletionText.mockResolvedValue('<summary>SUMMARY ONE</summary>')
+		// Under the 0.8 * 128k trigger without the thinking the next request replays.
+		const report = { prompt: 80_000, completion: 25_000, total: 105_000 }
+		let historyAfter: any[] = []
+		mocks.runChatLoop.mockImplementation(async (config: any) => {
+			await config.onBeforeIteration(gpt4oModel, null)
+			const assistant = {
+				role: 'assistant' as const,
+				content: '',
+				tool_calls: [{ id: 't1', type: 'function', function: { name: 'x', arguments: '{}' } }],
+				_anthropicContent: [{ type: 'thinking', thinking: '...', signature: 'sig' }]
+			}
+			const tool = { role: 'tool' as const, tool_call_id: 't1', content: 'r'.repeat(4_000) }
+			for (const m of [assistant, tool]) {
+				config.messages.push(m)
+				config.addedMessages.push(m)
+			}
+			await config.onBeforeIteration(gpt4oModel, report)
+			historyAfter = config.messages.slice(0, -2)
+			return {
+				addedMessages: [assistant, tool],
+				tokenUsage: report,
+				lastIterationUsage: report,
+				hitMaxIterations: false
+			}
+		})
+		const manager = new AIChatManager()
+		manager.messages = Array.from({ length: 10 }, (_, i) => ({
+			role: i % 2 === 0 ? ('user' as const) : ('assistant' as const),
+			content: `m${i}` + 'x'.repeat(30_000)
+		}))
+		manager.displayMessages = manager.messages.map((m, i) =>
+			m.role === 'user'
+				? { role: 'user', content: `m${i}`, index: i }
+				: { role: 'assistant', content: `m${i}` }
+		)
+		manager.instructions = 'next question'
+
+		await manager.sendRequest()
+
+		expect(mocks.getStreamedCompletionText).toHaveBeenCalledTimes(1)
+		// The kept history leaves room for the replayed thinking within the 0.5 target.
+		const keptTokens = historyAfter.reduce((acc, m) => acc + String(m.content).length / 4, 0)
+		expect(keptTokens + report.completion).toBeLessThan(128_000 * 0.5)
+	})
+
 	it('summarizes the older prefix and keeps the recent tail verbatim', async () => {
 		mocks.getCurrentModel.mockReturnValue(gpt4oModel)
 		mocks.tryGetCurrentModel.mockReturnValue(gpt4oModel)

frontend/src/lib/components/copilot/chat/chatLoop.ts

@@ -73,8 +73,13 @@ export interface ChatLoopConfig {
 	 * lets the caller recover partial output if the loop throws or is aborted.
 	 */
 	addedMessages?: ChatCompletionMessageParam[]
-	/** Called before each iteration (e.g. to record which model it is about to use). */
-	onBeforeIteration?: (modelProvider: ReasoningProviderModel) => Promise<void>
+	/** Called before each iteration (e.g. to record which model it is about to use), with
+	 * the last usage report that carried a prompt count, which can predate the latest
+	 * iteration. It may rewrite `messages` in place. */
+	onBeforeIteration?: (
+		modelProvider: ReasoningProviderModel,
+		lastIterationUsage: ChatTokenUsage | null
+	) => Promise<void>
 	/** Fired for each completed provider response, before the loop continues. The
 	 * loop can fail or be aborted at any iteration, so spend has to be handed over
 	 * as it happens — a callback only at the end would discard everything the
@@ -420,7 +425,7 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
 		const webSearch = sendsWebSearch(workspace, modelProvider, config.webSearch ?? true)
 
 		if (onBeforeIteration) {
-			await onBeforeIteration(modelProvider)
+			await onBeforeIteration(modelProvider, lastIterationUsage)
 		}
 		const tools = await Promise.all(
 			config.tools.map(async (t) => (t.schemaFor ? { ...t, def: await t.schemaFor(helpers) } : t))