feat: display openai reasoning summaries in ai chat (#10147)

* feat(frontend): display openai reasoning summaries in ai chat with unverified-org fallback Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(frontend): scope hidden-thinking hint per workspace/provider and skip summary on explicit reasoning-off Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(frontend): compose responses fallbacks in either error order and track all unavailable summary keys Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-09-11 08:07:15 +00:00 · 2026-07-16 13:31:10 +02:00
parent 0694b84da7
commit 4ee1d32101
5 changed files with 324 additions and 33 deletions
@@ -643,7 +643,7 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. -->
 											: aiChatManager.currentReasoningActive &&
 												  !aiChatManager.currentReply &&
 												  !aiChatManager.currentReasoning
-												? 'Thinking'
+												? (aiChatManager.reasoningHiddenIndicatorLabel ?? 'Thinking')
 												: undefined}
 								/>
 							{/if}
@@ -154,6 +154,21 @@ const AI_AUTONOMY_MODE_STORAGE_KEY = 'ai-chat-autonomy-mode'
 const LEGACY_AUTO_ACCEPT_TOOL_CONFIRMATIONS_STORAGE_KEY = 'ai-chat-yolo-mode'
 const WEB_SEARCH_ERROR_HINT =
 	'Web search is unavailable for this provider/model/key. Disable web search in workspace settings and try again.'
+// The full explanation is shown once per browser; afterwards the hidden
+// thinking is only hinted at discreetly in the typing indicator.
+const REASONING_SUMMARY_WARNED_STORAGE_KEY = 'ai-chat-reasoning-summary-unverified-warned'
+
+function providerDisplayName(provider: string): string {
+	return provider === 'azure_openai' ? 'Azure OpenAI' : 'OpenAI'
+}
+
+function reasoningSummaryUnavailableMessage(provider: string): string {
+	const verifyHint =
+		provider === 'azure_openai'
+			? 'To display it, verify your organization with your provider, then reload this page.'
+			: 'To display it, verify your organization in the OpenAI platform settings (Settings > General), then reload this page.'
+	return `This model is reasoning, but your ${providerDisplayName(provider)} organization is not verified to generate reasoning summaries, so its thinking stays hidden. ${verifyHint}`
+}

 export enum AIMode {
 	SCRIPT = 'script',
@@ -326,6 +341,30 @@ export class AIChatManager {
 	currentReply = $state<string>('')
 	currentReasoning = $state<string>('')
 	currentReasoningActive = $state<boolean>(false)
+	// The provider reasons but refuses to stream summaries (unverified OpenAI
+	// organization) — drives the discreet "Thinking (hidden)" indicator. Keyed
+	// by workspace:provider like the chat-loop fallback cache, so the hint never
+	// carries over to a provider or workspace whose summaries work. A list, not
+	// a scalar: several workspace/provider pairs can be unavailable at once, and
+	// the chat loop only notifies on first detection per pair.
+	private reasoningSummaryUnavailableFor = $state<string[]>([])
+
+	private reasoningSummaryKey(provider: string): string {
+		return `${this.operatingWorkspace ?? ''}:${provider}`
+	}
+
+	/** Label for the live "Thinking" indicator when thinking stays hidden for
+	 * the current workspace/provider, undefined otherwise. */
+	get reasoningHiddenIndicatorLabel(): string | undefined {
+		if (this.reasoningSummaryUnavailableFor.length === 0) {
+			return undefined
+		}
+		const provider = getCurrentModel().provider
+		if (!this.reasoningSummaryUnavailableFor.includes(this.reasoningSummaryKey(provider))) {
+			return undefined
+		}
+		return `Thinking (hidden, ${providerDisplayName(provider)} org not verified)`
+	}
 	// Smooths the provider's bursty delivery into continuous typing by revealing
 	// buffered text a slice per frame. The reply and the reasoning/thinking stream
 	// each get their own reveal (independent buffers, both append to their own
@@ -1716,6 +1755,18 @@ export class AIChatManager {
 		}
 	}

+	private notifyReasoningSummaryUnavailable = () => {
+		const provider = getCurrentModel().provider
+		const key = this.reasoningSummaryKey(provider)
+		if (!this.reasoningSummaryUnavailableFor.includes(key)) {
+			this.reasoningSummaryUnavailableFor = [...this.reasoningSummaryUnavailableFor, key]
+		}
+		if (getLocalSetting(REASONING_SUMMARY_WARNED_STORAGE_KEY) !== 'true') {
+			storeLocalSetting(REASONING_SUMMARY_WARNED_STORAGE_KEY, 'true')
+			sendUserToast(reasoningSummaryUnavailableMessage(provider), 'warning', [], undefined, 10000)
+		}
+	}
+
 	private chatRequest = async ({
 		messages,
 		abortController,
@@ -1735,6 +1786,7 @@ export class AIChatManager {
 		systemMessage?: ChatCompletionSystemMessageParam
 		onWebSearchUnavailable?: () => void
 	}) => {
+		const onReasoningSummaryUnavailable = () => this.notifyReasoningSummaryUnavailable()
 		try {
 			// Use JS getters so runChatLoop re-reads tools/helpers/systemMessage/modelProvider
 			// on each iteration. This is critical for changeModeTool (Navigator → Script/Flow)
@@ -1784,6 +1836,7 @@ export class AIChatManager {
 					this.skipResponsesApi = true
 				},
 				onWebSearchUnavailable,
+				onReasoningSummaryUnavailable,
 				getPendingUserMessage: () => {
 					const pendingPrompt = this.pendingPrompt
 					if (!pendingPrompt) return undefined
@@ -12,7 +12,8 @@ const mocks = vi.hoisted(() => ({
 	parseOpenAIResponsesCompletion: vi.fn(),
 	getAnthropicCompletion: vi.fn(),
 	parseAnthropicCompletion: vi.fn(),
-	resolveRequestReasoning: vi.fn()
+	resolveRequestReasoning: vi.fn(),
+	resolveEffectiveReasoning: vi.fn()
 }))

 vi.mock('../lib', () => ({
@@ -22,7 +23,8 @@ vi.mock('../lib', () => ({
 }))

 vi.mock('../reasoningRegistry', () => ({
-	resolveRequestReasoning: mocks.resolveRequestReasoning
+	resolveRequestReasoning: mocks.resolveRequestReasoning,
+	resolveEffectiveReasoning: mocks.resolveEffectiveReasoning
 }))

 vi.mock('./openai-responses', () => ({
@@ -50,12 +52,14 @@ function createConfig({
 	workspace,
 	modelProvider = { provider: 'openai', model: 'gpt-4.1' },
 	callbacks = createCallbacks(),
-	onWebSearchUnavailable
+	onWebSearchUnavailable,
+	onReasoningSummaryUnavailable
 }: {
 	workspace: string
 	modelProvider?: ReasoningProviderModel
 	callbacks?: ChatLoopConfig['callbacks']
 	onWebSearchUnavailable?: ChatLoopConfig['onWebSearchUnavailable']
+	onReasoningSummaryUnavailable?: ChatLoopConfig['onReasoningSummaryUnavailable']
 }): ChatLoopConfig {
 	const messages: ChatCompletionMessageParam[] = [{ role: 'user', content: 'search this' }]

@@ -73,7 +77,8 @@ function createConfig({
 		},
 		workspace,
 		maxIterations: 1,
-		onWebSearchUnavailable
+		onWebSearchUnavailable,
+		onReasoningSummaryUnavailable
 	}
 }

@@ -291,6 +296,133 @@ describe('runChatLoop web search fallback', () => {
 	})
 })

+describe('runChatLoop reasoning summary fallback', () => {
+	beforeEach(() => {
+		vi.resetAllMocks()
+		mocks.providerSupportsWebSearch.mockReturnValue(false)
+		mocks.resolveRequestReasoning.mockReturnValue('high')
+		mocks.resolveEffectiveReasoning.mockReturnValue('high')
+		mocks.parseOpenAIResponsesCompletion.mockResolvedValue({
+			shouldContinue: false,
+			tokenUsage
+		})
+	})
+
+	it('retries once without the summary on the unverified-org error and caches per workspace/provider', async () => {
+		const onReasoningSummaryUnavailable = vi.fn()
+		const workspace = `workspace-${randomUUID()}`
+		const modelProvider: ReasoningProviderModel = { provider: 'openai', model: 'gpt-5.1' }
+
+		mocks.getOpenAIResponsesCompletion
+			.mockRejectedValueOnce(
+				Object.assign(
+					new Error('Your organization must be verified to generate reasoning summaries.'),
+					{
+						status: 400,
+						param: 'reasoning.summary',
+						code: 'unsupported_value',
+						error: { type: 'invalid_request_error' }
+					}
+				)
+			)
+			.mockResolvedValue({})
+
+		await runChatLoop(createConfig({ workspace, modelProvider, onReasoningSummaryUnavailable }))
+
+		expect(mocks.getOpenAIResponsesCompletion).toHaveBeenCalledTimes(2)
+		expect(mocks.getOpenAIResponsesCompletion.mock.calls[0][3]).toEqual(
+			expect.objectContaining({ reasoningSummary: true })
+		)
+		expect(mocks.getOpenAIResponsesCompletion.mock.calls[1][3]).toEqual(
+			expect.objectContaining({ reasoningSummary: false })
+		)
+		expect(onReasoningSummaryUnavailable).toHaveBeenCalledTimes(1)
+		expect(mocks.getCompletion).not.toHaveBeenCalled()
+
+		// Cached: a later run in the same workspace skips the summary outright,
+		// but a different model on the same provider shares the cache entry.
+		await runChatLoop(
+			createConfig({
+				workspace,
+				modelProvider: { provider: 'openai', model: 'o3' }
+			})
+		)
+
+		expect(mocks.getOpenAIResponsesCompletion).toHaveBeenCalledTimes(3)
+		expect(mocks.getOpenAIResponsesCompletion.mock.calls[2][3]).toEqual(
+			expect.objectContaining({ reasoningSummary: false })
+		)
+	})
+
+	it('composes with the web-search fallback when the errors arrive web-search first', async () => {
+		mocks.providerSupportsWebSearch.mockReturnValue(true)
+		const onReasoningSummaryUnavailable = vi.fn()
+		const onWebSearchUnavailable = vi.fn()
+		const workspace = `workspace-${randomUUID()}`
+		const modelProvider: ReasoningProviderModel = { provider: 'openai', model: 'gpt-5.1' }
+
+		mocks.getOpenAIResponsesCompletion
+			.mockRejectedValueOnce(
+				Object.assign(new Error("Hosted tool 'web_search' is not supported with this model"), {
+					status: 400,
+					error: { type: 'invalid_request_error' }
+				})
+			)
+			.mockRejectedValueOnce(
+				Object.assign(
+					new Error('Your organization must be verified to generate reasoning summaries.'),
+					{
+						status: 400,
+						param: 'reasoning.summary',
+						code: 'unsupported_value',
+						error: { type: 'invalid_request_error' }
+					}
+				)
+			)
+			.mockResolvedValue({})
+
+		await runChatLoop(
+			createConfig({
+				workspace,
+				modelProvider,
+				onReasoningSummaryUnavailable,
+				onWebSearchUnavailable
+			})
+		)
+
+		expect(mocks.getOpenAIResponsesCompletion).toHaveBeenCalledTimes(3)
+		expect(mocks.getOpenAIResponsesCompletion.mock.calls[0][3]).toEqual(
+			expect.objectContaining({ webSearch: true, reasoningSummary: true })
+		)
+		expect(mocks.getOpenAIResponsesCompletion.mock.calls[1][3]).toEqual(
+			expect.objectContaining({ webSearch: false, reasoningSummary: true })
+		)
+		expect(mocks.getOpenAIResponsesCompletion.mock.calls[2][3]).toEqual(
+			expect.objectContaining({ webSearch: false, reasoningSummary: false })
+		)
+		expect(onWebSearchUnavailable).toHaveBeenCalledTimes(1)
+		expect(onReasoningSummaryUnavailable).toHaveBeenCalledTimes(1)
+		expect(mocks.getCompletion).not.toHaveBeenCalled()
+	})
+
+	it('does not request a summary when reasoning is explicitly off via a disable token', async () => {
+		// gpt-5.1+ reasoning is turned off with the explicit 'none' effort on the
+		// wire, while the effective reasoning resolves to undefined.
+		mocks.resolveRequestReasoning.mockReturnValue('none')
+		mocks.resolveEffectiveReasoning.mockReturnValue(undefined)
+		mocks.getOpenAIResponsesCompletion.mockResolvedValue({})
+		const workspace = `workspace-${randomUUID()}`
+
+		await runChatLoop(
+			createConfig({ workspace, modelProvider: { provider: 'openai', model: 'gpt-5.1' } })
+		)
+
+		expect(mocks.getOpenAIResponsesCompletion.mock.calls[0][3]).toEqual(
+			expect.objectContaining({ reasoningEffort: 'none', reasoningSummary: false })
+		)
+	})
+})
+
 describe('runChatLoop lastIterationUsage', () => {
 	beforeEach(() => {
 		vi.resetAllMocks()
@@ -6,7 +6,11 @@ import type {
 	ChatCompletionUserMessageParam
 } from 'openai/resources/chat/completions.mjs'
 import { getCompletion, parseOpenAICompletion, providerSupportsWebSearch } from '../lib'
-import { resolveRequestReasoning, type ReasoningProviderModel } from '../reasoningRegistry'
+import {
+	resolveEffectiveReasoning,
+	resolveRequestReasoning,
+	type ReasoningProviderModel
+} from '../reasoningRegistry'
 import { getAnthropicCompletion, parseAnthropicCompletion } from './anthropic'
 import { usesAnthropicMessagesApi } from '../modelConfig'
 import { getOpenAIResponsesCompletion, parseOpenAIResponsesCompletion } from './openai-responses'
@@ -48,6 +52,12 @@ export interface ChatLoopConfig {
 	skipResponsesApi?: boolean
 	onSkipResponsesApi?: () => void
 	onWebSearchUnavailable?: () => void
+	/**
+	 * Called when the provider refuses to generate reasoning summaries (OpenAI
+	 * gates them behind organization verification). The request is retried
+	 * without summaries, so reasoning happens but stays hidden.
+	 */
+	onReasoningSummaryUnavailable?: () => void
 	/** Return a pending user message to inject between iterations, or undefined. */
 	getPendingUserMessage?: () => ChatCompletionUserMessageParam | undefined
 	/**
@@ -107,10 +117,24 @@ export function truncateToToolPairedPrefix(
 const unsupportedWebSearchCache = new Set<string>()
 const WEB_SEARCH_UNAVAILABLE_STATUS_CODES = new Set([400, 403, 404])

+// Reasoning-summary availability is an org-level property of the provider
+// credentials (OpenAI organization verification), not of the model — key by
+// workspace + provider. In-memory on purpose: a page reload re-probes, so a
+// freshly verified organization starts getting summaries again.
+const unsupportedReasoningSummaryCache = new Set<string>()
+const REASONING_SUMMARY_UNAVAILABLE_STATUS_CODES = new Set([400, 403])
+
 function getWebSearchCacheKey(workspace: string, modelProvider: ReasoningProviderModel): string {
 	return [workspace, modelProvider.provider, modelProvider.model].join(':')
 }

+function getReasoningSummaryCacheKey(
+	workspace: string,
+	modelProvider: ReasoningProviderModel
+): string {
+	return [workspace, modelProvider.provider].join(':')
+}
+
 function isRecord(value: unknown): value is Record<string, unknown> {
 	return typeof value === 'object' && value !== null
 }
@@ -193,6 +217,44 @@ function shouldRetryWithoutWebSearch(err: unknown): boolean {
 	return status === undefined || WEB_SEARCH_UNAVAILABLE_STATUS_CODES.has(status)
 }

+function getErrorParam(err: unknown): string | undefined {
+	if (!isRecord(err)) {
+		return undefined
+	}
+	const candidates = [err.param]
+	if (isRecord(err.error)) {
+		candidates.push(err.error.param)
+	}
+	return candidates.find((param): param is string => typeof param === 'string')
+}
+
+// Unverified OpenAI organizations get a 400 on the reasoning.summary param
+// ("Your organization must be verified to generate reasoning summaries").
+function shouldRetryWithoutReasoningSummary(err: unknown): boolean {
+	const status = getErrorStatus(err)
+	if (status !== undefined && !REASONING_SUMMARY_UNAVAILABLE_STATUS_CODES.has(status)) {
+		return false
+	}
+	if (getErrorParam(err) === 'reasoning.summary') {
+		return true
+	}
+	const message = getErrorText(err).toLowerCase()
+	return (
+		message.includes('reasoning.summary') ||
+		/verified to (?:generate|stream) reasoning summar/.test(message)
+	)
+}
+
+function markReasoningSummaryUnsupported(
+	cacheKey: string,
+	err: unknown,
+	onReasoningSummaryUnavailable?: () => void
+) {
+	unsupportedReasoningSummaryCache.add(cacheKey)
+	console.warn('Reasoning summaries unavailable; retrying without them:', err)
+	onReasoningSummaryUnavailable?.()
+}
+
 function markWebSearchUnsupported(
 	cacheKey: string,
 	err: unknown,
@@ -212,6 +274,7 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
 		workspace,
 		maxIterations,
 		onSkipResponsesApi,
+		onReasoningSummaryUnavailable,
 		getPendingUserMessage,
 		onBeforeIteration
 	} = config
@@ -275,6 +338,14 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
 		const parseOptions = { workspace, provider: modelProvider.provider }

 		if (isOpenAI) {
+			const reasoningSummaryCacheKey = getReasoningSummaryCacheKey(workspace, modelProvider)
+			// Gate on the effective (not request) reasoning: an explicit off resolves
+			// to a truthy disable token like 'none' on the request side, and asking
+			// for a summary on a non-reasoning request would 400 on unverified orgs.
+			let reasoningSummary =
+				resolveEffectiveReasoning(modelProvider) !== undefined &&
+				!unsupportedReasoningSummaryCache.has(reasoningSummaryCacheKey)
+
 			const runOpenAIResponses = async (useWebSearch: boolean): Promise<boolean> => {
 				const completion = await getOpenAIResponsesCompletion(
 					messageParams,
@@ -284,7 +355,8 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
 						forceModelProvider: modelProvider,
 						openaiClient: clients.openai,
 						webSearch: useWebSearch,
-						reasoningEffort
+						reasoningEffort,
+						reasoningSummary
 					}
 				)
 				const continueCompletion = await parseOpenAIResponsesCompletion(
@@ -302,35 +374,47 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul

 			let useCompletionsApi = skipResponsesApi
 			if (!skipResponsesApi) {
-				try {
-					if (!(await runOpenAIResponses(webSearch))) {
-						break
-					}
-				} catch (err) {
-					let fallbackError = err
-					if (webSearch && shouldRetryWithoutWebSearch(err)) {
-						markWebSearchUnsupported(webSearchCacheKey, err, config.onWebSearchUnavailable)
-						try {
-							if (!(await runOpenAIResponses(false))) {
-								break
-							}
-							continue
-						} catch (retryErr) {
-							fallbackError = retryErr
+				// Retry the Responses call disabling whichever optional feature the
+				// provider rejected (reasoning summary, web search) in the order the
+				// errors arrive — a turn can hit both, either one first. Each retry
+				// permanently disables one feature, so this loops at most twice.
+				let useWebSearch = webSearch
+				let outcome: 'break' | 'continue' | undefined
+				let fallbackError: unknown
+				while (outcome === undefined) {
+					try {
+						outcome = (await runOpenAIResponses(useWebSearch)) ? 'continue' : 'break'
+					} catch (err) {
+						if (reasoningSummary && shouldRetryWithoutReasoningSummary(err)) {
+							markReasoningSummaryUnsupported(
+								reasoningSummaryCacheKey,
+								err,
+								onReasoningSummaryUnavailable
+							)
+							reasoningSummary = false
+						} else if (useWebSearch && shouldRetryWithoutWebSearch(err)) {
+							markWebSearchUnsupported(webSearchCacheKey, err, config.onWebSearchUnavailable)
+							useWebSearch = false
+						} else {
+							fallbackError = err
+							break
 						}
 					}
-
-					console.warn(
-						'OpenAI Responses API failed, falling back to Completions API:',
-						fallbackError
-					)
-					const errorMessage = getErrorText(fallbackError)
-					if (errorMessage.includes('Responses API is not enabled')) {
-						skipResponsesApi = true
-						onSkipResponsesApi?.()
-					}
-					useCompletionsApi = true
 				}
+				if (outcome === 'break') {
+					break
+				}
+				if (outcome === 'continue') {
+					continue
+				}
+
+				console.warn('OpenAI Responses API failed, falling back to Completions API:', fallbackError)
+				const errorMessage = getErrorText(fallbackError)
+				if (errorMessage.includes('Responses API is not enabled')) {
+					skipResponsesApi = true
+					onSkipResponsesApi?.()
+				}
+				useCompletionsApi = true
 			}

 			if (useCompletionsApi) {
@@ -159,6 +159,7 @@ export async function getOpenAIResponsesCompletion(
 		openaiClient?: OpenAI
 		webSearch?: boolean
 		reasoningEffort?: string
+		reasoningSummary?: boolean
 	}
 ) {
 	const { provider, config } = getProviderAndCompletionConfig({
@@ -174,6 +175,13 @@ export async function getOpenAIResponsesCompletion(
 		options?.reasoningEffort
 	)

+	// Reasoning summaries make the model's thinking renderable in the chat, but
+	// OpenAI rejects the request (400 on reasoning.summary) for organizations
+	// that haven't completed verification — callers opt in and fall back.
+	if (options?.reasoningSummary && responsesConfig.reasoning) {
+		responsesConfig.reasoning = { ...responsesConfig.reasoning, summary: 'auto' }
+	}
+
 	// Enable OpenAI's built-in web search tool. The proxy forwards the body
 	// verbatim, so this reaches OpenAI as a native server-side tool.
 	if (options?.webSearch && providerSupportsWebSearch(provider)) {
@@ -291,6 +299,20 @@ export async function parseOpenAIResponsesCompletion(
 		textContent += event.delta
 	})

+	// Stream the reasoning summary (present when the request asked for
+	// reasoning.summary) into the thinking display. Summaries arrive as
+	// separate parts; join them as paragraphs.
+	let reasoningSummaryParts = 0
+	runner.on('response.reasoning_summary_part.added', () => {
+		reasoningSummaryParts++
+		if (reasoningSummaryParts > 1) {
+			callbacks.onReasoningDelta?.('\n\n')
+		}
+	})
+	runner.on('response.reasoning_summary_text.delta', (event) => {
+		callbacks.onReasoningDelta?.(event.delta)
+	})
+
 	// Handle new output items (including function calls)
 	runner.on('response.output_item.added', (event) => {
 		const item = event.item