mirror of
https://github.com/stablyai/orca.git
synced 2026-10-28 16:02:55 +00:00
fix(native-chat): stop rendering a tool result whose call is outside the window
A tool result carries no call id, so it can only be attributed to a tool call loaded alongside it. Both chat views read a windowed transcript tail (mobile 40 messages, desktop 300), and the window regularly opens between an assistant's `tool_use` record and the user-role record that answers it. Claude also re-emits already-answered `tool_result` records at a `/compact` boundary, long after their call scrolled out of the window. `foldToolMessages` had no rule for those: with no assistant predecessor in the output they were pushed through as standalone messages and rendered as a bare, unowned block of raw tool output with no tool name — reading as a message from nowhere mid-conversation. Sampling real Claude transcripts, 176 of 400 sessions (44%) produced one in a mobile-sized first page. Drop a result no loaded call can own, before folding. It is not lost: it comes back attached to its call as soon as the owning turn pages in.
This commit is contained in:
@@ -125,10 +125,9 @@ describe('foldToolMessages', () => {
|
||||
expect(folded.map((m) => m.id)).toEqual(['a0', 'a2'])
|
||||
})
|
||||
|
||||
it('keeps a tool message standalone when no assistant precedes it', () => {
|
||||
it('drops a tool message whose call is outside the loaded window', () => {
|
||||
const folded = foldToolMessages([msg('tool', [{ type: 'tool-result', output: 'x' }])])
|
||||
expect(folded).toHaveLength(1)
|
||||
expect(folded[0]!.role).toBe('tool')
|
||||
expect(folded).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('does not merge across a user message', () => {
|
||||
@@ -142,12 +141,13 @@ describe('foldToolMessages', () => {
|
||||
|
||||
it('folds a long tool run without mutating the source assistant', () => {
|
||||
const assistant = msg('assistant', [{ type: 'text', text: 'working' }], 'a')
|
||||
const tools = Array.from({ length: 1_000 }, (_unused, index) =>
|
||||
const tools = Array.from({ length: 1_000 }, (_unused, index) => [
|
||||
msg('assistant', [{ type: 'tool-call', name: 'Bash', input: {} }], `c-${index}`),
|
||||
msg('tool', [{ type: 'tool-result', output: String(index) }], `t-${index}`)
|
||||
)
|
||||
]).flat()
|
||||
const folded = foldToolMessages([assistant, ...tools])
|
||||
|
||||
expect(folded[0].blocks).toHaveLength(1_001)
|
||||
expect(folded[0].blocks).toHaveLength(2_001)
|
||||
expect(assistant.blocks).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -173,3 +173,52 @@ describe('buildMobileNativeChatTransientData', () => {
|
||||
expect(data.some((message) => message.id === 'streaming')).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('foldMobileNativeChatMessages', () => {
|
||||
function toolCall(id: string): NativeChatMessage {
|
||||
return {
|
||||
id,
|
||||
role: 'assistant',
|
||||
blocks: [{ type: 'tool-call', name: 'Bash', input: { command: 'command -v orca-ide' } }],
|
||||
timestamp: 0,
|
||||
source: 'transcript'
|
||||
}
|
||||
}
|
||||
|
||||
function toolResult(id: string, output: string): NativeChatMessage {
|
||||
return {
|
||||
id,
|
||||
role: 'tool',
|
||||
blocks: [{ type: 'tool-result', output }],
|
||||
timestamp: 0,
|
||||
source: 'transcript'
|
||||
}
|
||||
}
|
||||
|
||||
// The chat reads a 40-message tail, so the window regularly opens on a result
|
||||
// whose `tool_use` is older than the window. It used to render as its own
|
||||
// bubble of raw shell output with no owning call.
|
||||
it('drops a leading tool result whose call is older than the read window', () => {
|
||||
const folded = foldMobileNativeChatMessages([
|
||||
toolResult('orphan', 'Exit code 1\norca-ide not found'),
|
||||
assistant('a1', 'Falling back to the installed binary.')
|
||||
])
|
||||
|
||||
expect(folded.map((message) => message.id)).toEqual(['a1'])
|
||||
})
|
||||
|
||||
it('still folds a result whose call is inside the window', () => {
|
||||
const folded = foldMobileNativeChatMessages([
|
||||
assistant('a1', 'Checking which binary is on PATH.'),
|
||||
toolCall('c1'),
|
||||
toolResult('r1', 'orca-ide not found')
|
||||
])
|
||||
|
||||
expect(folded).toHaveLength(1)
|
||||
expect(folded[0]?.blocks.map((block) => block.type)).toEqual([
|
||||
'text',
|
||||
'tool-call',
|
||||
'tool-result'
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { stripNoiseMessages } from '../../shared/native-chat-noise'
|
||||
import { foldToolMessages, pairToolBlocks } from '../../shared/native-chat-tool-fold'
|
||||
import { isToolCallBlock, isToolResultBlock } from '../../shared/native-chat-types'
|
||||
import type { NativeChatMessage } from '../../shared/native-chat-types'
|
||||
import { decodeClaudeTranscriptLine } from './transcript-line-decoders-claude'
|
||||
import { readNativeChatTranscriptTailFile } from './transcript-tail-reader'
|
||||
|
||||
let tempRoots: string[] = []
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(tempRoots.map((root) => rm(root, { recursive: true, force: true })))
|
||||
tempRoots = []
|
||||
})
|
||||
|
||||
async function writeTranscript(records: unknown[]): Promise<string> {
|
||||
const root = await mkdtemp(join(tmpdir(), 'orca-native-chat-attribution-'))
|
||||
tempRoots.push(root)
|
||||
const filePath = join(root, 'transcript.jsonl')
|
||||
await writeFile(filePath, records.map((record) => JSON.stringify(record)).join('\n'))
|
||||
return filePath
|
||||
}
|
||||
|
||||
/** One Claude turn: the assistant's `tool_use` record, then the user-role record
|
||||
* carrying its `tool_result` — the shape every Claude transcript writes. */
|
||||
function toolTurn(n: number): unknown[] {
|
||||
return [
|
||||
{
|
||||
type: 'assistant',
|
||||
uuid: `call-${n}`,
|
||||
timestamp: `2026-08-20T10:00:${String(n).padStart(2, '0')}.000Z`,
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: [{ type: 'tool_use', name: 'Bash', input: { command: `echo ${n}` } }]
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'user',
|
||||
uuid: `result-${n}`,
|
||||
timestamp: `2026-08-20T10:00:${String(n).padStart(2, '0')}.500Z`,
|
||||
message: {
|
||||
role: 'user',
|
||||
content: [{ type: 'tool_result', content: `output ${n}`, tool_use_id: `toolu_${n}` }]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
/** The rendered defect: a tool row with output but no call to attribute it to.
|
||||
* Mirrors what both chat views build from a folded message's tool blocks. */
|
||||
function unattributedResults(messages: readonly NativeChatMessage[]): string[] {
|
||||
const stray: string[] = []
|
||||
for (const message of messages) {
|
||||
const tools = message.blocks.filter(
|
||||
(block) => isToolCallBlock(block) || isToolResultBlock(block)
|
||||
)
|
||||
if (tools.length === 0) {
|
||||
continue
|
||||
}
|
||||
for (const pair of pairToolBlocks(tools)) {
|
||||
if (pair.call === undefined && pair.result) {
|
||||
stray.push(pair.result.output)
|
||||
}
|
||||
}
|
||||
}
|
||||
return stray
|
||||
}
|
||||
|
||||
async function renderedMessages(filePath: string, limit: number): Promise<NativeChatMessage[]> {
|
||||
const page = await readNativeChatTranscriptTailFile(
|
||||
filePath,
|
||||
limit,
|
||||
decodeClaudeTranscriptLine,
|
||||
true
|
||||
)
|
||||
return foldToolMessages(stripNoiseMessages(page.messages))
|
||||
}
|
||||
|
||||
describe('windowed transcript tool attribution', () => {
|
||||
it('drops a tool result whose call falls outside the read window', async () => {
|
||||
const filePath = await writeTranscript([...toolTurn(1), ...toolTurn(2), ...toolTurn(3)])
|
||||
|
||||
// A 3-message window cuts turn 2 in half: its result is the oldest record in
|
||||
// the window, its `tool_use` is not.
|
||||
const windowed = await renderedMessages(filePath, 3)
|
||||
|
||||
expect(unattributedResults(windowed)).toEqual([])
|
||||
// The turn that survives whole still renders its output.
|
||||
expect(
|
||||
windowed.flatMap((message) =>
|
||||
message.blocks.filter(isToolResultBlock).map((block) => block.output)
|
||||
)
|
||||
).toEqual(['output 3'])
|
||||
})
|
||||
|
||||
it('keeps every result when the window contains its call', async () => {
|
||||
const filePath = await writeTranscript([...toolTurn(1), ...toolTurn(2)])
|
||||
|
||||
const whole = await renderedMessages(filePath, 100)
|
||||
|
||||
expect(unattributedResults(whole)).toEqual([])
|
||||
expect(
|
||||
whole.flatMap((message) =>
|
||||
message.blocks.filter(isToolResultBlock).map((block) => block.output)
|
||||
)
|
||||
).toEqual(['output 1', 'output 2'])
|
||||
})
|
||||
|
||||
it('drops the answered result records Claude re-emits at a /compact boundary', async () => {
|
||||
// Claude appends byte-identical copies of already-answered tool_result
|
||||
// records after a compaction, with no `tool_use` in between.
|
||||
const [, answeredResult] = toolTurn(1)
|
||||
const filePath = await writeTranscript([
|
||||
...toolTurn(1),
|
||||
{
|
||||
type: 'user',
|
||||
uuid: 'compact-prompt',
|
||||
timestamp: '2026-08-20T10:05:00.000Z',
|
||||
message: { role: 'user', content: '/compact' }
|
||||
},
|
||||
answeredResult
|
||||
])
|
||||
|
||||
const whole = await renderedMessages(filePath, 100)
|
||||
|
||||
expect(unattributedResults(whole)).toEqual([])
|
||||
expect(
|
||||
whole.flatMap((message) =>
|
||||
message.blocks.filter(isToolResultBlock).map((block) => block.output)
|
||||
)
|
||||
).toEqual(['output 1'])
|
||||
})
|
||||
})
|
||||
@@ -17,13 +17,21 @@ function msg(
|
||||
describe('foldToolMessages', () => {
|
||||
it('merges a tool-only message into the preceding assistant turn', () => {
|
||||
const folded = foldToolMessages([
|
||||
msg({ id: 'a', role: 'assistant', blocks: [{ type: 'text', text: 'running it' }] }),
|
||||
msg({
|
||||
id: 'a',
|
||||
role: 'assistant',
|
||||
blocks: [
|
||||
{ type: 'text', text: 'running it' },
|
||||
{ type: 'tool-call', name: 'Bash', input: {} }
|
||||
]
|
||||
}),
|
||||
msg({ id: 't', role: 'tool', blocks: [{ type: 'tool-result', output: 'done' }] })
|
||||
])
|
||||
expect(folded).toHaveLength(1)
|
||||
expect(folded[0]?.id).toBe('a')
|
||||
expect(folded[0]?.blocks).toEqual([
|
||||
{ type: 'text', text: 'running it' },
|
||||
{ type: 'tool-call', name: 'Bash', input: {} },
|
||||
{ type: 'tool-result', output: 'done' }
|
||||
])
|
||||
})
|
||||
@@ -38,12 +46,21 @@ describe('foldToolMessages', () => {
|
||||
expect(folded[0]?.blocks).toHaveLength(3)
|
||||
})
|
||||
|
||||
it('leaves an orphan tool message standalone when no assistant precedes it', () => {
|
||||
it('drops a tool result no loaded call can own instead of leaving it standalone', () => {
|
||||
const folded = foldToolMessages([
|
||||
msg({ id: 'u', role: 'user', blocks: [{ type: 'text', text: 'hi' }] }),
|
||||
msg({ id: 't', role: 'tool', blocks: [{ type: 'tool-result', output: 'x' }] })
|
||||
])
|
||||
expect(folded.map((m) => m.id)).toEqual(['u', 't'])
|
||||
expect(folded.map((m) => m.id)).toEqual(['u'])
|
||||
})
|
||||
|
||||
it('keeps a result answering a call that a user turn interrupted', () => {
|
||||
const folded = foldToolMessages([
|
||||
msg({ id: 'c', role: 'assistant', blocks: [{ type: 'tool-call', name: 'Bash', input: {} }] }),
|
||||
msg({ id: 'u', role: 'user', blocks: [{ type: 'text', text: 'stop' }] }),
|
||||
msg({ id: 't', role: 'tool', blocks: [{ type: 'tool-result', output: 'x' }] })
|
||||
])
|
||||
expect(folded.map((m) => m.id)).toEqual(['c', 'u', 't'])
|
||||
})
|
||||
|
||||
it('does not fold a message carrying prose alongside a tool block', () => {
|
||||
|
||||
@@ -14,11 +14,46 @@ function isToolOnlyMessage(message: NativeChatMessage): boolean {
|
||||
)
|
||||
}
|
||||
|
||||
/** Fold consecutive tool-only messages into their preceding assistant turn. */
|
||||
/** A tool result carries no call id, so it can only be attributed to a tool call
|
||||
* that is loaded alongside it. A windowed transcript read starts wherever the
|
||||
* window falls — often between an assistant's `tool_use` and the result that
|
||||
* answers it — and Claude re-emits already-answered result records at a
|
||||
* `/compact` boundary, long after their call scrolled out. Either way the
|
||||
* result renders as a bare, unowned block that reads as a message from nowhere,
|
||||
* so drop it here; it comes back attached once the owning turn pages in. */
|
||||
function dropUnattributableToolResults(
|
||||
messages: readonly NativeChatMessage[]
|
||||
): NativeChatMessage[] {
|
||||
const output: NativeChatMessage[] = []
|
||||
let unansweredCalls = 0
|
||||
for (const message of messages) {
|
||||
const blocks: NativeChatBlock[] = []
|
||||
for (const block of message.blocks) {
|
||||
if (isToolCallBlock(block)) {
|
||||
unansweredCalls += 1
|
||||
} else if (isToolResultBlock(block)) {
|
||||
if (unansweredCalls === 0) {
|
||||
continue
|
||||
}
|
||||
unansweredCalls -= 1
|
||||
}
|
||||
blocks.push(block)
|
||||
}
|
||||
if (blocks.length === message.blocks.length) {
|
||||
output.push(message)
|
||||
} else if (blocks.length > 0) {
|
||||
output.push({ ...message, blocks })
|
||||
}
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
/** Fold consecutive tool-only messages into their preceding assistant turn,
|
||||
* after dropping results no loaded call can own. */
|
||||
export function foldToolMessages(messages: readonly NativeChatMessage[]): NativeChatMessage[] {
|
||||
const output: NativeChatMessage[] = []
|
||||
let mutableAssistantIndex = -1
|
||||
for (const message of messages) {
|
||||
for (const message of dropUnattributableToolResults(messages)) {
|
||||
const previous = output.at(-1)
|
||||
if (isToolOnlyMessage(message) && previous?.role === 'assistant') {
|
||||
const index = output.length - 1
|
||||
|
||||
Reference in New Issue
Block a user