fix(chat): clear the sent image bubble when Claude takes an image in mid-turn (#40975)

Claude records a prompt it absorbs mid-turn only as a queued_command
attachment whose images are inline base64 with no path. The decoder
dropped those images, so an image-only send had no transcript row to
match and its pending bubble stayed at the bottom of the chat.

Decode queued inline images as data URLs. Pathless image-only rows
cannot name their send, so each retires one delivered image-only send
after its boundary, oldest first, and pruning hands the claimed row to
the remaining sends so none reuses it. Failed or uncertain writes never
claim one. Keep inline image bytes out of handoff and worker text.

Co-authored-by: Xulin Wu <wuxulin@bytedance.com>
This commit is contained in:
Kelvin Amoaba
2026-11-11 02:30:37 +00:00
committed by GitHub
co-authored by Xulin Wu
parent a94ead49d0
commit be1ae8798e
8 changed files with 158 additions and 15 deletions
@@ -37,4 +37,18 @@ describe('Claude queued_command attachment', () => {
parentId: 'tool-result-row'
})
})
it('keeps inline images from a prompt absorbed mid-turn, which have no source row', () => {
const line = queuedCommand({
prompt: [
{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'iVBORw0K' } },
{ type: 'text', text: '[Image #3]' }
],
commandMode: 'prompt'
})
expect(decodeClaudeTranscriptLine(line, 'fallback')?.blocks).toEqual([
{ type: 'image-ref', url: 'data:image/png;base64,iVBORw0K' },
{ type: 'text', text: '[Image #3]' }
])
})
})
@@ -13,6 +13,7 @@ import {
} from '../ai-vault/session-scanner-values'
import { imageSourcePathFromText } from '../../shared/native-chat/transcript/native-chat-image-transcript-markers'
import { claudeContentBlocks } from './transcript-record-blocks'
import { isKnownRasterImageMimeType } from '../../shared/file-preview/raster-image-preview-limits'
import { unwrapClaudePastedContentBlock } from '../../shared/claude-pasted-content'
import { unwrapClaudeBashInputBlock } from '../../shared/claude-bash-input'
import { claudeInterruptedMessageId } from './transcript-turn-markers'
@@ -161,7 +162,25 @@ function claudeQueuedPromptAsUserRecord(
if (typeof prompt !== 'string' && !Array.isArray(prompt)) {
return null
}
return { ...record, type: 'user', message: { role: 'user', content: prompt } }
const content = Array.isArray(prompt) ? prompt.map(queuedPromptImageAsDataUrl) : prompt
return { ...record, type: 'user', message: { role: 'user', content } }
}
// Queued prompts get no image-source companion row; their inline base64 is the only image record.
function queuedPromptImageAsDataUrl(item: unknown): unknown {
const record = asRecord(item)
const source = asRecord(record?.source)
const mediaType = extractString(source?.media_type) ?? undefined
const data = extractString(source?.data)
if (
record?.type !== 'image' ||
source?.type !== 'base64' ||
!data ||
!isKnownRasterImageMimeType(mediaType)
) {
return item
}
return { ...record, source: { url: `data:${mediaType};base64,${data}` } }
}
function claudeStopHookFeedbackMessage(
@@ -1,4 +1,5 @@
import type { NativeChatImageRefBlock } from '../../../../../shared/native-chat/native-chat-types'
import { isInlineDataUrl } from '../../../../../shared/native-chat/native-chat-inline-data-url'
// Image refs are labels over RPC; inline bytes stay local and metadata cannot consume the frame.
const NATIVE_CHAT_RPC_IMAGE_METADATA_CAP = 512
@@ -21,11 +22,3 @@ export function sanitizeNativeChatRpcImageBlock(
function boundedImageMetadata(value: string | undefined): string | undefined {
return value && value.length <= NATIVE_CHAT_RPC_IMAGE_METADATA_CAP ? value : undefined
}
function isInlineDataUrl(value: string): boolean {
let schemeStart = 0
while (schemeStart < value.length && value.charCodeAt(schemeStart) <= 0x20) {
schemeStart += 1
}
return /^data:/i.test(value.slice(schemeStart))
}
+2 -1
View File
@@ -11,6 +11,7 @@ import { MAX_CONTEXT_CHARS } from './agent-session/requests/agent-session-contex
import { agentJournalToolCallLifecycle } from './agent-journal-tool-call-lifecycle'
import { withoutNativeChatVisualDirectiveLines } from './native-chat/native-chat-visual-directive'
import { workerTranscriptBlockFallbacks } from './worker-transcript-text'
import { isInlineDataUrl } from './native-chat/native-chat-inline-data-url'
const MAX_TOOL_CHARS = 8_000
const MAX_MESSAGE_CHARS = 8_000
@@ -74,7 +75,7 @@ function prioritizeConversation(messages: readonly NativeChatMessage[]): Convers
text =
message.role === 'user' ? block.text : withoutNativeChatVisualDirectiveLines(block.text)
} else if (block.type === 'image-ref') {
text = `[image] ${excerpt(block.path ?? block.url ?? block.alt ?? 'image omitted', MAX_MESSAGE_CHARS / 2)}`
text = `[image] ${excerpt(block.path ?? (block.url && !isInlineDataUrl(block.url) ? block.url : null) ?? block.alt ?? 'image omitted', MAX_MESSAGE_CHARS / 2)}`
}
return text.trim() ? [{ messageIndex, blockIndex, text }] : []
})
@@ -0,0 +1,8 @@
/** Inline image bytes; stay local and out of RPC frames and text exports. */
export function isInlineDataUrl(value: string): boolean {
let schemeStart = 0
while (schemeStart < value.length && value.charCodeAt(schemeStart) <= 0x20) {
schemeStart += 1
}
return /^data:/i.test(value.slice(schemeStart))
}
@@ -76,4 +76,47 @@ describe('pending sends typed through an agent TUI', () => {
expect(prunePendingSends(pending, transcript)).toEqual([])
expect(pendingSendsAsMessages(pending, transcript)).toEqual([])
})
it('lets each pathless image row retire one delivered image-only send, once, oldest first', () => {
const send = (
id: string,
extra: Partial<NativeChatPendingSend> = {}
): NativeChatPendingSend => ({
id,
text: '',
imagePaths: [`/tmp/orca-paste-${id}.png`],
sentAt: 100,
afterMessageId: 'working',
...extra
})
const pending = [send('a', { delivery: 'rejected' }), send('b'), send('c')]
const imageRow = (id: string, timestamp: number): NativeChatMessage => ({
id,
role: 'user',
blocks: [{ type: 'image-ref', url: 'data:image/png;base64,iVBORw0K' }],
timestamp,
source: 'transcript'
})
const before = [imageRow('older-image', 50), message('working', 'assistant', 'busy', 90)]
expect(pendingSendsAsMessages(pending, before).map((row) => row.id)).toEqual([
'pending:a',
'pending:b',
'pending:c'
])
const landed = [...before, imageRow('queued-image', 110)]
expect(pendingSendsAsMessages(pending, landed).map((row) => row.id)).toEqual([
'pending:a',
'pending:c'
])
expect(prunePendingSends(pending, landed)).toBe(pending)
const replied = [...landed, message('reply', 'assistant', 'ok', 120)]
const remaining = prunePendingSends(pending, replied)
expect(remaining.map((entry) => entry.id)).toEqual(['a', 'c'])
expect(pendingSendsAsMessages(remaining, replied).map((row) => row.id)).toEqual([
'pending:a',
'pending:c'
])
expect(prunePendingSends(remaining, replied).map((entry) => entry.id)).toEqual(['a', 'c'])
})
})
@@ -1,5 +1,6 @@
import { nativeChatContinuationSendText } from './native-chat-continuation-send'
import type { NativeChatMessage } from '../native-chat-types'
import { isImageRefBlock, type NativeChatMessage } from '../native-chat-types'
import { normalizedNativeChatUserMessageText } from './native-chat-image-transcript-markers'
import {
advancedNativeChatUserContentCounts,
advancedNativeChatUserRows,
@@ -9,6 +10,7 @@ import {
nativeChatPendingMatchKey,
nativeChatPendingMatchingAfter,
nativeChatPendingOccurrence,
normalizeNativeChatPendingText,
selectPendingIndicesRepresentedByUserRows,
type NativeChatGluedUserRow,
type NativeChatUserRow
@@ -40,6 +42,8 @@ export type NativeChatPendingSend = {
matchingOccurrence?: number
/** Shared time boundary when that message boundary is unavailable. */
matchingAfterTimestamp?: number
/** Pathless image rows already claimed by pruned sends, so this send cannot reclaim them. */
claimedImageRowIds?: readonly string[]
}
function messagesAfterPendingBoundary(
@@ -132,6 +136,52 @@ function gluedCandidateRows(
})
}
// Pathless image rows (Claude queued prompts) cannot name their send, so each is claimed once, oldest send first.
// Failed or uncertain writes never claim one, so another send's image cannot hide them.
function claimPathlessImageRow(
entry: NativeChatPendingSend,
allMessages: readonly NativeChatMessage[],
claimed: Set<string>,
advancedOnly: boolean
): boolean {
const count = entry.imagePaths?.filter(Boolean).length ?? 0
if (
count === 0 ||
// Only a real row boundary or a proven-empty session; client-clock boundaries could bind old images.
(typeof entry.afterMessageId !== 'string' && !entry.afterEmptyTranscriptSessionId) ||
entry.delivery !== undefined ||
entry.writeUnconfirmedAt !== undefined ||
normalizeNativeChatPendingText(entry.text)
) {
return false
}
const messages = messagesAfterPendingBoundary(allMessages, entry)
const lastNonUser = messages.findLastIndex((message) => message.role !== 'user')
const row = messages.find((message, index) => {
if (
message.role !== 'user' ||
claimed.has(message.id) ||
(advancedOnly && index > lastNonUser)
) {
return false
}
const images = message.blocks.filter(isImageRefBlock)
return (
images.length === count &&
images.every((image) => !image.path) &&
!normalizedNativeChatUserMessageText(message)
)
})
if (row) {
claimed.add(row.id)
}
return Boolean(row)
}
function carriedImageRowClaims(pending: readonly NativeChatPendingSend[]): Set<string> {
return new Set(pending.flatMap((entry) => entry.claimedImageRowIds ?? []))
}
/** Keep echoes until a later non-user turn to avoid first-turn flashes during transient empty reads. */
export function prunePendingSends<T extends NativeChatPendingSend>(
pending: T[],
@@ -141,6 +191,8 @@ export function prunePendingSends<T extends NativeChatPendingSend>(
return pending
}
const consumed = new Map<string, number>()
const claimedImageRows = carriedImageRowClaims(pending)
const carriedClaimCount = claimedImageRows.size
const exactKeep = pending.map((entry) => {
const contentKey = nativeChatPendingContentKey(entry)
const key = nativeChatPendingMatchKey(entry)
@@ -152,7 +204,7 @@ export function prunePendingSends<T extends NativeChatPendingSend>(
const used = consumed.get(key) ?? 0
const occurrence = nativeChatPendingOccurrence(entry, used)
consumed.set(key, Math.max(used, occurrence))
return occurrence > available
return occurrence > available && !claimPathlessImageRow(entry, messages, claimedImageRows, true)
})
// Lost Enter can glue sends into one transcript row; retire them after the assistant advances.
const stillOpen = pending.filter((_, index) => exactKeep[index])
@@ -167,7 +219,15 @@ export function prunePendingSends<T extends NativeChatPendingSend>(
const openIndex = stillOpen.indexOf(entry)
return openIndex === -1 || !gluedRepresented.has(openIndex)
})
return next.length === pending.length ? pending : next
if (next.length === pending.length) {
return pending
}
// Claims here retire their sends; hand them to the remaining image sends so none reuses the row.
return claimedImageRows.size === carriedClaimCount
? next
: next.map((entry) =>
entry.imagePaths?.length ? { ...entry, claimedImageRowIds: [...claimedImageRows] } : entry
)
}
/** Sends whose transcript row has not landed yet; exact and glued matches both hide an echo. */
@@ -179,6 +239,7 @@ export function visibleNativeChatPendingSends<T extends NativeChatPendingSend>(
return pending
}
const consumed = new Map<string, number>()
const claimedImageRows = carriedImageRowClaims(pending)
const exactVisible = pending.map((entry) => {
const contentKey = nativeChatPendingContentKey(entry)
const key = nativeChatPendingMatchKey(entry)
@@ -190,7 +251,10 @@ export function visibleNativeChatPendingSends<T extends NativeChatPendingSend>(
const used = consumed.get(key) ?? 0
const occurrence = nativeChatPendingOccurrence(entry, used)
consumed.set(key, Math.max(used, occurrence))
return occurrence > represented
return (
occurrence > represented &&
!claimPathlessImageRow(entry, existingMessages, claimedImageRows, false)
)
})
// Hide glued echoes once their user row exists, before assistant advancement permits pruning.
const stillVisible = pending.filter((_, index) => exactVisible[index])
+2 -1
View File
@@ -7,6 +7,7 @@ import {
} from './native-chat/transcript/native-chat-subagent-summary'
import { subagentFallbackText } from './native-chat/transcript/native-chat-subagent-items'
import type { NativeChatMessage } from './native-chat/native-chat-types'
import { isInlineDataUrl } from './native-chat/native-chat-inline-data-url'
export function formatWorkerTranscriptMessage(message: NativeChatMessage): string {
const fallbacks = workerTranscriptBlockFallbacks(message.blocks)
@@ -21,7 +22,7 @@ export function formatWorkerTranscriptMessage(message: NativeChatMessage): strin
return `[tool result${block.isError ? ' error' : ''}] ${block.output}`
}
if (block.type === 'image-ref') {
return block.url ? `[image] ${block.url}` : `[image omitted]`
return block.url && !isInlineDataUrl(block.url) ? `[image] ${block.url}` : `[image omitted]`
}
if (
block.type === 'background-task' ||