From be1ae8798eb2f420c4f018375464fca06fac5ec9 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:30:37 +0000 Subject: [PATCH] fix(chat): clear the sent image bubble when Claude takes an image in mid-turn (#40975) Claude records a prompt it absorbs mid-turn only as a queued_command attachment whose images are inline base64 with no path. The decoder dropped those images, so an image-only send had no transcript row to match and its pending bubble stayed at the bottom of the chat. Decode queued inline images as data URLs. Pathless image-only rows cannot name their send, so each retires one delivered image-only send after its boundary, oldest first, and pruning hands the claimed row to the remaining sends so none reuses it. Failed or uncertain writes never claim one. Keep inline image bytes out of handoff and worker text. Co-authored-by: Xulin Wu --- ...line-decoders-claude-queued-prompt.test.ts | 14 ++++ .../transcript-line-decoders-claude.ts | 21 +++++- .../native-chat-rpc-image-block.ts | 9 +-- src/shared/agent-session-handoff-context.ts | 3 +- .../native-chat-inline-data-url.ts | 8 +++ .../native-chat-pending-occurrence.test.ts | 43 +++++++++++ .../transcript/native-chat-pending-sends.ts | 72 +++++++++++++++++-- src/shared/worker-transcript-text.ts | 3 +- 8 files changed, 158 insertions(+), 15 deletions(-) create mode 100644 src/shared/native-chat/native-chat-inline-data-url.ts diff --git a/src/main/native-chat/transcript-line-decoders-claude-queued-prompt.test.ts b/src/main/native-chat/transcript-line-decoders-claude-queued-prompt.test.ts index 543d923f845..392b5533a74 100644 --- a/src/main/native-chat/transcript-line-decoders-claude-queued-prompt.test.ts +++ b/src/main/native-chat/transcript-line-decoders-claude-queued-prompt.test.ts @@ -37,4 +37,18 @@ describe('Claude queued_command attachment', () => { parentId: 'tool-result-row' }) }) + + it('keeps inline images from a prompt absorbed mid-turn, which have no source row', () => { + const line = queuedCommand({ + prompt: [ + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'iVBORw0K' } }, + { type: 'text', text: '[Image #3]' } + ], + commandMode: 'prompt' + }) + expect(decodeClaudeTranscriptLine(line, 'fallback')?.blocks).toEqual([ + { type: 'image-ref', url: 'data:image/png;base64,iVBORw0K' }, + { type: 'text', text: '[Image #3]' } + ]) + }) }) diff --git a/src/main/native-chat/transcript-line-decoders-claude.ts b/src/main/native-chat/transcript-line-decoders-claude.ts index 2e256af18e4..095e183e1d4 100644 --- a/src/main/native-chat/transcript-line-decoders-claude.ts +++ b/src/main/native-chat/transcript-line-decoders-claude.ts @@ -13,6 +13,7 @@ import { } from '../ai-vault/session-scanner-values' import { imageSourcePathFromText } from '../../shared/native-chat/transcript/native-chat-image-transcript-markers' import { claudeContentBlocks } from './transcript-record-blocks' +import { isKnownRasterImageMimeType } from '../../shared/file-preview/raster-image-preview-limits' import { unwrapClaudePastedContentBlock } from '../../shared/claude-pasted-content' import { unwrapClaudeBashInputBlock } from '../../shared/claude-bash-input' import { claudeInterruptedMessageId } from './transcript-turn-markers' @@ -161,7 +162,25 @@ function claudeQueuedPromptAsUserRecord( if (typeof prompt !== 'string' && !Array.isArray(prompt)) { return null } - return { ...record, type: 'user', message: { role: 'user', content: prompt } } + const content = Array.isArray(prompt) ? prompt.map(queuedPromptImageAsDataUrl) : prompt + return { ...record, type: 'user', message: { role: 'user', content } } +} + +// Queued prompts get no image-source companion row; their inline base64 is the only image record. +function queuedPromptImageAsDataUrl(item: unknown): unknown { + const record = asRecord(item) + const source = asRecord(record?.source) + const mediaType = extractString(source?.media_type) ?? undefined + const data = extractString(source?.data) + if ( + record?.type !== 'image' || + source?.type !== 'base64' || + !data || + !isKnownRasterImageMimeType(mediaType) + ) { + return item + } + return { ...record, source: { url: `data:${mediaType};base64,${data}` } } } function claudeStopHookFeedbackMessage( diff --git a/src/main/runtime/rpc/methods/native-chat/native-chat-rpc-image-block.ts b/src/main/runtime/rpc/methods/native-chat/native-chat-rpc-image-block.ts index 20ec8d05196..6dfe708b4d9 100644 --- a/src/main/runtime/rpc/methods/native-chat/native-chat-rpc-image-block.ts +++ b/src/main/runtime/rpc/methods/native-chat/native-chat-rpc-image-block.ts @@ -1,4 +1,5 @@ import type { NativeChatImageRefBlock } from '../../../../../shared/native-chat/native-chat-types' +import { isInlineDataUrl } from '../../../../../shared/native-chat/native-chat-inline-data-url' // Image refs are labels over RPC; inline bytes stay local and metadata cannot consume the frame. const NATIVE_CHAT_RPC_IMAGE_METADATA_CAP = 512 @@ -21,11 +22,3 @@ export function sanitizeNativeChatRpcImageBlock( function boundedImageMetadata(value: string | undefined): string | undefined { return value && value.length <= NATIVE_CHAT_RPC_IMAGE_METADATA_CAP ? value : undefined } - -function isInlineDataUrl(value: string): boolean { - let schemeStart = 0 - while (schemeStart < value.length && value.charCodeAt(schemeStart) <= 0x20) { - schemeStart += 1 - } - return /^data:/i.test(value.slice(schemeStart)) -} diff --git a/src/shared/agent-session-handoff-context.ts b/src/shared/agent-session-handoff-context.ts index 0bdf19963c8..f7fbc891669 100644 --- a/src/shared/agent-session-handoff-context.ts +++ b/src/shared/agent-session-handoff-context.ts @@ -11,6 +11,7 @@ import { MAX_CONTEXT_CHARS } from './agent-session/requests/agent-session-contex import { agentJournalToolCallLifecycle } from './agent-journal-tool-call-lifecycle' import { withoutNativeChatVisualDirectiveLines } from './native-chat/native-chat-visual-directive' import { workerTranscriptBlockFallbacks } from './worker-transcript-text' +import { isInlineDataUrl } from './native-chat/native-chat-inline-data-url' const MAX_TOOL_CHARS = 8_000 const MAX_MESSAGE_CHARS = 8_000 @@ -74,7 +75,7 @@ function prioritizeConversation(messages: readonly NativeChatMessage[]): Convers text = message.role === 'user' ? block.text : withoutNativeChatVisualDirectiveLines(block.text) } else if (block.type === 'image-ref') { - text = `[image] ${excerpt(block.path ?? block.url ?? block.alt ?? 'image omitted', MAX_MESSAGE_CHARS / 2)}` + text = `[image] ${excerpt(block.path ?? (block.url && !isInlineDataUrl(block.url) ? block.url : null) ?? block.alt ?? 'image omitted', MAX_MESSAGE_CHARS / 2)}` } return text.trim() ? [{ messageIndex, blockIndex, text }] : [] }) diff --git a/src/shared/native-chat/native-chat-inline-data-url.ts b/src/shared/native-chat/native-chat-inline-data-url.ts new file mode 100644 index 00000000000..c582410d203 --- /dev/null +++ b/src/shared/native-chat/native-chat-inline-data-url.ts @@ -0,0 +1,8 @@ +/** Inline image bytes; stay local and out of RPC frames and text exports. */ +export function isInlineDataUrl(value: string): boolean { + let schemeStart = 0 + while (schemeStart < value.length && value.charCodeAt(schemeStart) <= 0x20) { + schemeStart += 1 + } + return /^data:/i.test(value.slice(schemeStart)) +} diff --git a/src/shared/native-chat/transcript/native-chat-pending-occurrence.test.ts b/src/shared/native-chat/transcript/native-chat-pending-occurrence.test.ts index 79ff232a823..2390c2c8dcb 100644 --- a/src/shared/native-chat/transcript/native-chat-pending-occurrence.test.ts +++ b/src/shared/native-chat/transcript/native-chat-pending-occurrence.test.ts @@ -76,4 +76,47 @@ describe('pending sends typed through an agent TUI', () => { expect(prunePendingSends(pending, transcript)).toEqual([]) expect(pendingSendsAsMessages(pending, transcript)).toEqual([]) }) + + it('lets each pathless image row retire one delivered image-only send, once, oldest first', () => { + const send = ( + id: string, + extra: Partial = {} + ): NativeChatPendingSend => ({ + id, + text: '', + imagePaths: [`/tmp/orca-paste-${id}.png`], + sentAt: 100, + afterMessageId: 'working', + ...extra + }) + const pending = [send('a', { delivery: 'rejected' }), send('b'), send('c')] + const imageRow = (id: string, timestamp: number): NativeChatMessage => ({ + id, + role: 'user', + blocks: [{ type: 'image-ref', url: 'data:image/png;base64,iVBORw0K' }], + timestamp, + source: 'transcript' + }) + const before = [imageRow('older-image', 50), message('working', 'assistant', 'busy', 90)] + expect(pendingSendsAsMessages(pending, before).map((row) => row.id)).toEqual([ + 'pending:a', + 'pending:b', + 'pending:c' + ]) + + const landed = [...before, imageRow('queued-image', 110)] + expect(pendingSendsAsMessages(pending, landed).map((row) => row.id)).toEqual([ + 'pending:a', + 'pending:c' + ]) + expect(prunePendingSends(pending, landed)).toBe(pending) + const replied = [...landed, message('reply', 'assistant', 'ok', 120)] + const remaining = prunePendingSends(pending, replied) + expect(remaining.map((entry) => entry.id)).toEqual(['a', 'c']) + expect(pendingSendsAsMessages(remaining, replied).map((row) => row.id)).toEqual([ + 'pending:a', + 'pending:c' + ]) + expect(prunePendingSends(remaining, replied).map((entry) => entry.id)).toEqual(['a', 'c']) + }) }) diff --git a/src/shared/native-chat/transcript/native-chat-pending-sends.ts b/src/shared/native-chat/transcript/native-chat-pending-sends.ts index 579605e0bc6..d08f1baa080 100644 --- a/src/shared/native-chat/transcript/native-chat-pending-sends.ts +++ b/src/shared/native-chat/transcript/native-chat-pending-sends.ts @@ -1,5 +1,6 @@ import { nativeChatContinuationSendText } from './native-chat-continuation-send' -import type { NativeChatMessage } from '../native-chat-types' +import { isImageRefBlock, type NativeChatMessage } from '../native-chat-types' +import { normalizedNativeChatUserMessageText } from './native-chat-image-transcript-markers' import { advancedNativeChatUserContentCounts, advancedNativeChatUserRows, @@ -9,6 +10,7 @@ import { nativeChatPendingMatchKey, nativeChatPendingMatchingAfter, nativeChatPendingOccurrence, + normalizeNativeChatPendingText, selectPendingIndicesRepresentedByUserRows, type NativeChatGluedUserRow, type NativeChatUserRow @@ -40,6 +42,8 @@ export type NativeChatPendingSend = { matchingOccurrence?: number /** Shared time boundary when that message boundary is unavailable. */ matchingAfterTimestamp?: number + /** Pathless image rows already claimed by pruned sends, so this send cannot reclaim them. */ + claimedImageRowIds?: readonly string[] } function messagesAfterPendingBoundary( @@ -132,6 +136,52 @@ function gluedCandidateRows( }) } +// Pathless image rows (Claude queued prompts) cannot name their send, so each is claimed once, oldest send first. +// Failed or uncertain writes never claim one, so another send's image cannot hide them. +function claimPathlessImageRow( + entry: NativeChatPendingSend, + allMessages: readonly NativeChatMessage[], + claimed: Set, + advancedOnly: boolean +): boolean { + const count = entry.imagePaths?.filter(Boolean).length ?? 0 + if ( + count === 0 || + // Only a real row boundary or a proven-empty session; client-clock boundaries could bind old images. + (typeof entry.afterMessageId !== 'string' && !entry.afterEmptyTranscriptSessionId) || + entry.delivery !== undefined || + entry.writeUnconfirmedAt !== undefined || + normalizeNativeChatPendingText(entry.text) + ) { + return false + } + const messages = messagesAfterPendingBoundary(allMessages, entry) + const lastNonUser = messages.findLastIndex((message) => message.role !== 'user') + const row = messages.find((message, index) => { + if ( + message.role !== 'user' || + claimed.has(message.id) || + (advancedOnly && index > lastNonUser) + ) { + return false + } + const images = message.blocks.filter(isImageRefBlock) + return ( + images.length === count && + images.every((image) => !image.path) && + !normalizedNativeChatUserMessageText(message) + ) + }) + if (row) { + claimed.add(row.id) + } + return Boolean(row) +} + +function carriedImageRowClaims(pending: readonly NativeChatPendingSend[]): Set { + return new Set(pending.flatMap((entry) => entry.claimedImageRowIds ?? [])) +} + /** Keep echoes until a later non-user turn to avoid first-turn flashes during transient empty reads. */ export function prunePendingSends( pending: T[], @@ -141,6 +191,8 @@ export function prunePendingSends( return pending } const consumed = new Map() + const claimedImageRows = carriedImageRowClaims(pending) + const carriedClaimCount = claimedImageRows.size const exactKeep = pending.map((entry) => { const contentKey = nativeChatPendingContentKey(entry) const key = nativeChatPendingMatchKey(entry) @@ -152,7 +204,7 @@ export function prunePendingSends( const used = consumed.get(key) ?? 0 const occurrence = nativeChatPendingOccurrence(entry, used) consumed.set(key, Math.max(used, occurrence)) - return occurrence > available + return occurrence > available && !claimPathlessImageRow(entry, messages, claimedImageRows, true) }) // Lost Enter can glue sends into one transcript row; retire them after the assistant advances. const stillOpen = pending.filter((_, index) => exactKeep[index]) @@ -167,7 +219,15 @@ export function prunePendingSends( const openIndex = stillOpen.indexOf(entry) return openIndex === -1 || !gluedRepresented.has(openIndex) }) - return next.length === pending.length ? pending : next + if (next.length === pending.length) { + return pending + } + // Claims here retire their sends; hand them to the remaining image sends so none reuses the row. + return claimedImageRows.size === carriedClaimCount + ? next + : next.map((entry) => + entry.imagePaths?.length ? { ...entry, claimedImageRowIds: [...claimedImageRows] } : entry + ) } /** Sends whose transcript row has not landed yet; exact and glued matches both hide an echo. */ @@ -179,6 +239,7 @@ export function visibleNativeChatPendingSends( return pending } const consumed = new Map() + const claimedImageRows = carriedImageRowClaims(pending) const exactVisible = pending.map((entry) => { const contentKey = nativeChatPendingContentKey(entry) const key = nativeChatPendingMatchKey(entry) @@ -190,7 +251,10 @@ export function visibleNativeChatPendingSends( const used = consumed.get(key) ?? 0 const occurrence = nativeChatPendingOccurrence(entry, used) consumed.set(key, Math.max(used, occurrence)) - return occurrence > represented + return ( + occurrence > represented && + !claimPathlessImageRow(entry, existingMessages, claimedImageRows, false) + ) }) // Hide glued echoes once their user row exists, before assistant advancement permits pruning. const stillVisible = pending.filter((_, index) => exactVisible[index]) diff --git a/src/shared/worker-transcript-text.ts b/src/shared/worker-transcript-text.ts index 038f451e1ff..895a387ba38 100644 --- a/src/shared/worker-transcript-text.ts +++ b/src/shared/worker-transcript-text.ts @@ -7,6 +7,7 @@ import { } from './native-chat/transcript/native-chat-subagent-summary' import { subagentFallbackText } from './native-chat/transcript/native-chat-subagent-items' import type { NativeChatMessage } from './native-chat/native-chat-types' +import { isInlineDataUrl } from './native-chat/native-chat-inline-data-url' export function formatWorkerTranscriptMessage(message: NativeChatMessage): string { const fallbacks = workerTranscriptBlockFallbacks(message.blocks) @@ -21,7 +22,7 @@ export function formatWorkerTranscriptMessage(message: NativeChatMessage): strin return `[tool result${block.isError ? ' error' : ''}] ${block.output}` } if (block.type === 'image-ref') { - return block.url ? `[image] ${block.url}` : `[image omitted]` + return block.url && !isInlineDataUrl(block.url) ? `[image] ${block.url}` : `[image omitted]` } if ( block.type === 'background-task' ||