mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-10-19 16:03:29 +00:00
fix(ai-chat): replay anthropic turns verbatim to keep thinking valid
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
hugocasa
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
96c0ff65bd
commit
12b9a291b9
@@ -0,0 +1,164 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs'
|
||||
import { convertOpenAIToAnthropicMessages } from './anthropic'
|
||||
|
||||
// anthropic.ts pulls in the chat client/registry layer at import time; the
|
||||
// converter under test is pure, so stub those side-effecting modules away.
|
||||
vi.mock('../lib', () => ({
|
||||
getProviderAndCompletionConfig: vi.fn(),
|
||||
workspaceAIClients: {}
|
||||
}))
|
||||
|
||||
vi.mock('../reasoningRegistry', () => ({
|
||||
applyReasoningToConfig: vi.fn()
|
||||
}))
|
||||
|
||||
vi.mock('./shared', () => ({
|
||||
processToolCall: vi.fn()
|
||||
}))
|
||||
|
||||
describe('convertOpenAIToAnthropicMessages', () => {
|
||||
it('replays a captured assistant turn verbatim, skips the standalone text copy, and leaves the turn untouched', () => {
|
||||
const anthropicContent = [
|
||||
{ type: 'thinking', thinking: 'first', signature: 'sig-1' },
|
||||
{
|
||||
type: 'server_tool_use',
|
||||
id: 'srv_1',
|
||||
name: 'web_search',
|
||||
input: { query: 'nist password length' }
|
||||
},
|
||||
{
|
||||
type: 'web_search_tool_result',
|
||||
tool_use_id: 'srv_1',
|
||||
content: [{ type: 'web_search_result', title: 'NIST', url: 'https://nist.gov' }]
|
||||
},
|
||||
{ type: 'thinking', thinking: 'second', signature: 'sig-2' },
|
||||
{ type: 'tool_use', id: 'tool_1', name: 'list_resources', input: {} }
|
||||
]
|
||||
// Snapshot to assert the stored content is never mutated by the converter.
|
||||
const anthropicContentSnapshot = JSON.parse(JSON.stringify(anthropicContent))
|
||||
|
||||
const messages: ChatCompletionMessageParam[] = [
|
||||
{ role: 'user', content: 'find the nist length then list resources' },
|
||||
// Standalone text the streamer emits before the tool-call message.
|
||||
{ role: 'assistant', content: 'Let me search the web.' },
|
||||
{
|
||||
role: 'assistant',
|
||||
tool_calls: [
|
||||
{
|
||||
id: 'tool_1',
|
||||
type: 'function',
|
||||
function: { name: 'list_resources', arguments: '{}' }
|
||||
}
|
||||
],
|
||||
_anthropicContent: anthropicContent
|
||||
} as any,
|
||||
{ role: 'tool', tool_call_id: 'tool_1', content: 'resource A, resource B' }
|
||||
]
|
||||
|
||||
const { messages: out } = convertOpenAIToAnthropicMessages(messages)
|
||||
|
||||
// user + the verbatim assistant turn + the tool result; the standalone text is dropped.
|
||||
expect(out).toHaveLength(3)
|
||||
expect(out[0]).toEqual({ role: 'user', content: 'find the nist length then list resources' })
|
||||
|
||||
// Assistant turn replayed in original order, no reordering or dropped blocks.
|
||||
expect(out[1].role).toBe('assistant')
|
||||
expect((out[1].content as any[]).map((b) => b.type)).toEqual([
|
||||
'thinking',
|
||||
'server_tool_use',
|
||||
'web_search_tool_result',
|
||||
'thinking',
|
||||
'tool_use'
|
||||
])
|
||||
// The verbatim turn must stay byte-identical — no cache_control injected into it,
|
||||
// or a thinking-block signature would no longer validate.
|
||||
expect(out[1].content).toEqual(anthropicContentSnapshot)
|
||||
expect((out[1].content as any[]).some((b) => 'cache_control' in b)).toBe(false)
|
||||
expect(anthropicContent).toEqual(anthropicContentSnapshot)
|
||||
|
||||
// The cache breakpoint lands on the trailing tool result, not the assistant turn.
|
||||
expect(out[2]).toEqual({
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'tool_result',
|
||||
tool_use_id: 'tool_1',
|
||||
content: 'resource A, resource B',
|
||||
cache_control: { type: 'ephemeral' }
|
||||
}
|
||||
]
|
||||
})
|
||||
})
|
||||
|
||||
it('converts a plain text assistant turn (no tools) and caches the trailing text block', () => {
|
||||
const messages: ChatCompletionMessageParam[] = [
|
||||
{ role: 'user', content: 'hello' },
|
||||
{ role: 'assistant', content: 'hi there' }
|
||||
]
|
||||
|
||||
const { messages: out } = convertOpenAIToAnthropicMessages(messages)
|
||||
|
||||
expect(out).toHaveLength(2)
|
||||
expect(out[0]).toEqual({ role: 'user', content: 'hello' })
|
||||
expect(out[1].role).toBe('assistant')
|
||||
expect(out[1].content).toEqual([
|
||||
{ type: 'text', text: 'hi there', cache_control: { type: 'ephemeral' } }
|
||||
])
|
||||
})
|
||||
|
||||
it('falls back to _anthropicThinkingBlocks for turns persisted before _anthropicContent', () => {
|
||||
const thinkingBlocks = [{ type: 'thinking', thinking: 'reasoning', signature: 'sig-old' }]
|
||||
|
||||
const messages: ChatCompletionMessageParam[] = [
|
||||
{ role: 'user', content: 'do a thing' },
|
||||
// The standalone text persisted alongside an old-style turn must NOT be skipped
|
||||
// (the fallback reconstruction relies on it for the assistant text).
|
||||
{ role: 'assistant', content: 'working on it' },
|
||||
{
|
||||
role: 'assistant',
|
||||
tool_calls: [
|
||||
{
|
||||
id: 'tool_old',
|
||||
type: 'function',
|
||||
function: { name: 'list_resources', arguments: '{}' }
|
||||
}
|
||||
],
|
||||
_anthropicThinkingBlocks: thinkingBlocks
|
||||
} as any
|
||||
]
|
||||
|
||||
const { messages: out } = convertOpenAIToAnthropicMessages(messages)
|
||||
|
||||
expect(out).toHaveLength(3)
|
||||
expect(out[1]).toEqual({ role: 'assistant', content: 'working on it' })
|
||||
const content = out[2].content as any[]
|
||||
// Thinking block re-injected first, then the tool_use.
|
||||
expect(content.map((b) => b.type)).toEqual(['thinking', 'tool_use'])
|
||||
expect(content[0]).toEqual(thinkingBlocks[0])
|
||||
expect(content[1]).toMatchObject({ type: 'tool_use', id: 'tool_old', name: 'list_resources' })
|
||||
})
|
||||
|
||||
it('caches a trailing tool result even when the prior turn used no captured content', () => {
|
||||
const messages: ChatCompletionMessageParam[] = [
|
||||
{ role: 'user', content: 'q' },
|
||||
{
|
||||
role: 'assistant',
|
||||
tool_calls: [
|
||||
{ id: 't1', type: 'function', function: { name: 'list_resources', arguments: '{}' } }
|
||||
]
|
||||
} as any,
|
||||
{ role: 'tool', tool_call_id: 't1', content: 'done' }
|
||||
]
|
||||
|
||||
const { messages: out } = convertOpenAIToAnthropicMessages(messages)
|
||||
|
||||
const last = out[out.length - 1]
|
||||
expect(last.role).toBe('user')
|
||||
expect((last.content as any[])[0]).toMatchObject({
|
||||
type: 'tool_result',
|
||||
tool_use_id: 't1',
|
||||
cache_control: { type: 'ephemeral' }
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -286,15 +286,15 @@ export async function parseAnthropicCompletion(
|
||||
role: 'assistant',
|
||||
tool_calls: toolCallsToProcess
|
||||
}
|
||||
// Preserve thinking blocks (with signatures) so the next request keeps the
|
||||
// reasoning chain — Anthropic requires this when thinking is combined with tool
|
||||
// use. They are re-injected by convertOpenAIToAnthropicMessages.
|
||||
const thinkingBlocks = finalMessage.content.filter(
|
||||
(b) => b.type === 'thinking' || b.type === 'redacted_thinking'
|
||||
)
|
||||
if (thinkingBlocks.length > 0) {
|
||||
;(assistantWithTools as any)._anthropicThinkingBlocks = thinkingBlocks
|
||||
}
|
||||
// Preserve the assistant turn verbatim (thinking/redacted_thinking with their
|
||||
// signatures, server_tool_use + web_search_tool_result, text and tool_use) in
|
||||
// original order. Anthropic binds each thinking block's signature to the blocks
|
||||
// that precede it in the latest assistant message, so when this turn is replayed
|
||||
// to continue past its tool call it must be byte-identical: reordering thinking to
|
||||
// the front or dropping the web-search blocks invalidates a later block's
|
||||
// signature and the request 400s with "thinking blocks ... cannot be modified".
|
||||
// convertOpenAIToAnthropicMessages replays this content as-is.
|
||||
;(assistantWithTools as any)._anthropicContent = finalMessage.content
|
||||
messages.push(assistantWithTools)
|
||||
addedMessages.push(assistantWithTools)
|
||||
|
||||
@@ -323,7 +323,33 @@ export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessage
|
||||
let system: TextBlockParam[] | undefined
|
||||
const anthropicMessages: MessageParam[] = []
|
||||
|
||||
for (const message of messages) {
|
||||
// A streamed assistant turn that ends in tool calls is persisted as one or more
|
||||
// standalone text messages followed by the tool-call message carrying
|
||||
// _anthropicContent. That text is already inside _anthropicContent (replayed verbatim
|
||||
// below), so drop the standalone copies — otherwise the text is duplicated and
|
||||
// emitted ahead of the turn's thinking blocks. The scan stops at the preceding
|
||||
// user/tool message, so only the current turn's own text is skipped.
|
||||
const skipStandaloneText = new Set<number>()
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
if (!(messages[i] as any)._anthropicContent) continue
|
||||
for (let j = i - 1; j >= 0; j--) {
|
||||
const m = messages[j]
|
||||
if (
|
||||
m.role === 'assistant' &&
|
||||
typeof m.content === 'string' &&
|
||||
!m.tool_calls &&
|
||||
!(m as any)._anthropicContent
|
||||
) {
|
||||
skipStandaloneText.add(j)
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
const message = messages[i]
|
||||
if (skipStandaloneText.has(i)) continue
|
||||
if (message.role === 'system') {
|
||||
const systemText =
|
||||
typeof message.content === 'string' ? message.content : JSON.stringify(message.content)
|
||||
@@ -345,10 +371,19 @@ export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessage
|
||||
typeof message.content === 'string' ? message.content : JSON.stringify(message.content)
|
||||
})
|
||||
} else if (message.role === 'assistant') {
|
||||
// Replay a captured assistant turn verbatim so its thinking-block signatures
|
||||
// stay valid (see the _anthropicContent note where the streamed turn is stored).
|
||||
const anthropicContent = (message as any)._anthropicContent
|
||||
if (Array.isArray(anthropicContent) && anthropicContent.length > 0) {
|
||||
anthropicMessages.push({ role: 'assistant', content: anthropicContent as any })
|
||||
continue
|
||||
}
|
||||
|
||||
const content: any[] = []
|
||||
|
||||
// Re-inject preserved thinking blocks first (Anthropic requires thinking to
|
||||
// precede tool_use in the same assistant turn when thinking is enabled).
|
||||
// Fallback for sessions persisted before _anthropicContent existed: re-inject
|
||||
// the preserved thinking blocks first (Anthropic requires thinking to precede
|
||||
// tool_use in the same assistant turn when thinking is enabled).
|
||||
const thinkingBlocks = (message as any)._anthropicThinkingBlocks
|
||||
if (Array.isArray(thinkingBlocks) && thinkingBlocks.length > 0) {
|
||||
content.push(...thinkingBlocks)
|
||||
@@ -404,26 +439,29 @@ export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessage
|
||||
}
|
||||
}
|
||||
|
||||
// Add cache_control to the last message content blocks
|
||||
// Cache the conversation prefix: put an ephemeral breakpoint on the last content
|
||||
// block of the last message. Each continuation only appends a tool result plus the
|
||||
// next turn, so everything up to here is read from cache — which is what keeps
|
||||
// replaying assistant turns verbatim (web-search results included) affordable.
|
||||
// cache_control is valid on text/tool_use/tool_result blocks, but a thinking or
|
||||
// redacted_thinking block must never be modified, so skip the breakpoint there.
|
||||
if (anthropicMessages.length > 0) {
|
||||
const lastMessage = anthropicMessages[anthropicMessages.length - 1]
|
||||
if (Array.isArray(lastMessage.content)) {
|
||||
// Add cache_control to the last content block
|
||||
if (lastMessage.content.length > 0) {
|
||||
const lastBlock = lastMessage.content[lastMessage.content.length - 1]
|
||||
if (lastBlock.type === 'text') {
|
||||
lastBlock.cache_control = { type: 'ephemeral' }
|
||||
}
|
||||
}
|
||||
} else if (typeof lastMessage.content === 'string') {
|
||||
// Convert string content to array format with cache_control
|
||||
if (typeof lastMessage.content === 'string') {
|
||||
lastMessage.content = [
|
||||
{
|
||||
type: 'text',
|
||||
text: lastMessage.content,
|
||||
cache_control: { type: 'ephemeral' }
|
||||
}
|
||||
{ type: 'text', text: lastMessage.content, cache_control: { type: 'ephemeral' } }
|
||||
]
|
||||
} else if (Array.isArray(lastMessage.content) && lastMessage.content.length > 0) {
|
||||
const lastIndex = lastMessage.content.length - 1
|
||||
const lastBlock = lastMessage.content[lastIndex] as any
|
||||
if (lastBlock.type !== 'thinking' && lastBlock.type !== 'redacted_thinking') {
|
||||
// Clone the block instead of mutating in place: the array may be a verbatim
|
||||
// _anthropicContent turn that must stay unaltered for later requests.
|
||||
lastMessage.content = [
|
||||
...lastMessage.content.slice(0, lastIndex),
|
||||
{ ...lastBlock, cache_control: { type: 'ephemeral' } }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user