diff --git a/src/main/native-chat/transcript-line-decoders-claude-queued-prompt.test.ts b/src/main/native-chat/transcript-line-decoders-claude-queued-prompt.test.ts index 543d923f845..392b5533a74 100644 --- a/src/main/native-chat/transcript-line-decoders-claude-queued-prompt.test.ts +++ b/src/main/native-chat/transcript-line-decoders-claude-queued-prompt.test.ts @@ -37,4 +37,18 @@ describe('Claude queued_command attachment', () => { parentId: 'tool-result-row' }) }) + + it('keeps inline images from a prompt absorbed mid-turn, which have no source row', () => { + const line = queuedCommand({ + prompt: [ + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'iVBORw0K' } }, + { type: 'text', text: '[Image #3]' } + ], + commandMode: 'prompt' + }) + expect(decodeClaudeTranscriptLine(line, 'fallback')?.blocks).toEqual([ + { type: 'image-ref', url: 'data:image/png;base64,iVBORw0K' }, + { type: 'text', text: '[Image #3]' } + ]) + }) }) diff --git a/src/main/native-chat/transcript-line-decoders-claude.ts b/src/main/native-chat/transcript-line-decoders-claude.ts index 2e256af18e4..095e183e1d4 100644 --- a/src/main/native-chat/transcript-line-decoders-claude.ts +++ b/src/main/native-chat/transcript-line-decoders-claude.ts @@ -13,6 +13,7 @@ import { } from '../ai-vault/session-scanner-values' import { imageSourcePathFromText } from '../../shared/native-chat/transcript/native-chat-image-transcript-markers' import { claudeContentBlocks } from './transcript-record-blocks' +import { isKnownRasterImageMimeType } from '../../shared/file-preview/raster-image-preview-limits' import { unwrapClaudePastedContentBlock } from '../../shared/claude-pasted-content' import { unwrapClaudeBashInputBlock } from '../../shared/claude-bash-input' import { claudeInterruptedMessageId } from './transcript-turn-markers' @@ -161,7 +162,25 @@ function claudeQueuedPromptAsUserRecord( if (typeof prompt !== 'string' && !Array.isArray(prompt)) { return null } - return { ...record, type: 'user', message: { role: 'user', content: prompt } } + const content = Array.isArray(prompt) ? prompt.map(queuedPromptImageAsDataUrl) : prompt + return { ...record, type: 'user', message: { role: 'user', content } } +} + +// Queued prompts get no image-source companion row; their inline base64 is the only image record. +function queuedPromptImageAsDataUrl(item: unknown): unknown { + const record = asRecord(item) + const source = asRecord(record?.source) + const mediaType = extractString(source?.media_type) ?? undefined + const data = extractString(source?.data) + if ( + record?.type !== 'image' || + source?.type !== 'base64' || + !data || + !isKnownRasterImageMimeType(mediaType) + ) { + return item + } + return { ...record, source: { url: `data:${mediaType};base64,${data}` } } } function claudeStopHookFeedbackMessage( diff --git a/src/main/runtime/rpc/methods/native-chat/native-chat-rpc-image-block.ts b/src/main/runtime/rpc/methods/native-chat/native-chat-rpc-image-block.ts index 20ec8d05196..6dfe708b4d9 100644 --- a/src/main/runtime/rpc/methods/native-chat/native-chat-rpc-image-block.ts +++ b/src/main/runtime/rpc/methods/native-chat/native-chat-rpc-image-block.ts @@ -1,4 +1,5 @@ import type { NativeChatImageRefBlock } from '../../../../../shared/native-chat/native-chat-types' +import { isInlineDataUrl } from '../../../../../shared/native-chat/native-chat-inline-data-url' // Image refs are labels over RPC; inline bytes stay local and metadata cannot consume the frame. const NATIVE_CHAT_RPC_IMAGE_METADATA_CAP = 512 @@ -21,11 +22,3 @@ export function sanitizeNativeChatRpcImageBlock( function boundedImageMetadata(value: string | undefined): string | undefined { return value && value.length <= NATIVE_CHAT_RPC_IMAGE_METADATA_CAP ? value : undefined } - -function isInlineDataUrl(value: string): boolean { - let schemeStart = 0 - while (schemeStart < value.length && value.charCodeAt(schemeStart) <= 0x20) { - schemeStart += 1 - } - return /^data:/i.test(value.slice(schemeStart)) -} diff --git a/src/shared/agent-session-handoff-context.ts b/src/shared/agent-session-handoff-context.ts index 0bdf19963c8..f7fbc891669 100644 --- a/src/shared/agent-session-handoff-context.ts +++ b/src/shared/agent-session-handoff-context.ts @@ -11,6 +11,7 @@ import { MAX_CONTEXT_CHARS } from './agent-session/requests/agent-session-contex import { agentJournalToolCallLifecycle } from './agent-journal-tool-call-lifecycle' import { withoutNativeChatVisualDirectiveLines } from './native-chat/native-chat-visual-directive' import { workerTranscriptBlockFallbacks } from './worker-transcript-text' +import { isInlineDataUrl } from './native-chat/native-chat-inline-data-url' const MAX_TOOL_CHARS = 8_000 const MAX_MESSAGE_CHARS = 8_000 @@ -74,7 +75,7 @@ function prioritizeConversation(messages: readonly NativeChatMessage[]): Convers text = message.role === 'user' ? block.text : withoutNativeChatVisualDirectiveLines(block.text) } else if (block.type === 'image-ref') { - text = `[image] ${excerpt(block.path ?? block.url ?? block.alt ?? 'image omitted', MAX_MESSAGE_CHARS / 2)}` + text = `[image] ${excerpt(block.path ?? (block.url && !isInlineDataUrl(block.url) ? block.url : null) ?? block.alt ?? 'image omitted', MAX_MESSAGE_CHARS / 2)}` } return text.trim() ? [{ messageIndex, blockIndex, text }] : [] }) diff --git a/src/shared/native-chat/native-chat-inline-data-url.ts b/src/shared/native-chat/native-chat-inline-data-url.ts new file mode 100644 index 00000000000..c582410d203 --- /dev/null +++ b/src/shared/native-chat/native-chat-inline-data-url.ts @@ -0,0 +1,8 @@ +/** Inline image bytes; stay local and out of RPC frames and text exports. */ +export function isInlineDataUrl(value: string): boolean { + let schemeStart = 0 + while (schemeStart < value.length && value.charCodeAt(schemeStart) <= 0x20) { + schemeStart += 1 + } + return /^data:/i.test(value.slice(schemeStart)) +} diff --git a/src/shared/native-chat/transcript/native-chat-pending-occurrence.test.ts b/src/shared/native-chat/transcript/native-chat-pending-occurrence.test.ts index 79ff232a823..2390c2c8dcb 100644 --- a/src/shared/native-chat/transcript/native-chat-pending-occurrence.test.ts +++ b/src/shared/native-chat/transcript/native-chat-pending-occurrence.test.ts @@ -76,4 +76,47 @@ describe('pending sends typed through an agent TUI', () => { expect(prunePendingSends(pending, transcript)).toEqual([]) expect(pendingSendsAsMessages(pending, transcript)).toEqual([]) }) + + it('lets each pathless image row retire one delivered image-only send, once, oldest first', () => { + const send = ( + id: string, + extra: Partial = {} + ): NativeChatPendingSend => ({ + id, + text: '', + imagePaths: [`/tmp/orca-paste-${id}.png`], + sentAt: 100, + afterMessageId: 'working', + ...extra + }) + const pending = [send('a', { delivery: 'rejected' }), send('b'), send('c')] + const imageRow = (id: string, timestamp: number): NativeChatMessage => ({ + id, + role: 'user', + blocks: [{ type: 'image-ref', url: 'data:image/png;base64,iVBORw0K' }], + timestamp, + source: 'transcript' + }) + const before = [imageRow('older-image', 50), message('working', 'assistant', 'busy', 90)] + expect(pendingSendsAsMessages(pending, before).map((row) => row.id)).toEqual([ + 'pending:a', + 'pending:b', + 'pending:c' + ]) + + const landed = [...before, imageRow('queued-image', 110)] + expect(pendingSendsAsMessages(pending, landed).map((row) => row.id)).toEqual([ + 'pending:a', + 'pending:c' + ]) + expect(prunePendingSends(pending, landed)).toBe(pending) + const replied = [...landed, message('reply', 'assistant', 'ok', 120)] + const remaining = prunePendingSends(pending, replied) + expect(remaining.map((entry) => entry.id)).toEqual(['a', 'c']) + expect(pendingSendsAsMessages(remaining, replied).map((row) => row.id)).toEqual([ + 'pending:a', + 'pending:c' + ]) + expect(prunePendingSends(remaining, replied).map((entry) => entry.id)).toEqual(['a', 'c']) + }) }) diff --git a/src/shared/native-chat/transcript/native-chat-pending-sends.ts b/src/shared/native-chat/transcript/native-chat-pending-sends.ts index 579605e0bc6..d08f1baa080 100644 --- a/src/shared/native-chat/transcript/native-chat-pending-sends.ts +++ b/src/shared/native-chat/transcript/native-chat-pending-sends.ts @@ -1,5 +1,6 @@ import { nativeChatContinuationSendText } from './native-chat-continuation-send' -import type { NativeChatMessage } from '../native-chat-types' +import { isImageRefBlock, type NativeChatMessage } from '../native-chat-types' +import { normalizedNativeChatUserMessageText } from './native-chat-image-transcript-markers' import { advancedNativeChatUserContentCounts, advancedNativeChatUserRows, @@ -9,6 +10,7 @@ import { nativeChatPendingMatchKey, nativeChatPendingMatchingAfter, nativeChatPendingOccurrence, + normalizeNativeChatPendingText, selectPendingIndicesRepresentedByUserRows, type NativeChatGluedUserRow, type NativeChatUserRow @@ -40,6 +42,8 @@ export type NativeChatPendingSend = { matchingOccurrence?: number /** Shared time boundary when that message boundary is unavailable. */ matchingAfterTimestamp?: number + /** Pathless image rows already claimed by pruned sends, so this send cannot reclaim them. */ + claimedImageRowIds?: readonly string[] } function messagesAfterPendingBoundary( @@ -132,6 +136,52 @@ function gluedCandidateRows( }) } +// Pathless image rows (Claude queued prompts) cannot name their send, so each is claimed once, oldest send first. +// Failed or uncertain writes never claim one, so another send's image cannot hide them. +function claimPathlessImageRow( + entry: NativeChatPendingSend, + allMessages: readonly NativeChatMessage[], + claimed: Set, + advancedOnly: boolean +): boolean { + const count = entry.imagePaths?.filter(Boolean).length ?? 0 + if ( + count === 0 || + // Only a real row boundary or a proven-empty session; client-clock boundaries could bind old images. + (typeof entry.afterMessageId !== 'string' && !entry.afterEmptyTranscriptSessionId) || + entry.delivery !== undefined || + entry.writeUnconfirmedAt !== undefined || + normalizeNativeChatPendingText(entry.text) + ) { + return false + } + const messages = messagesAfterPendingBoundary(allMessages, entry) + const lastNonUser = messages.findLastIndex((message) => message.role !== 'user') + const row = messages.find((message, index) => { + if ( + message.role !== 'user' || + claimed.has(message.id) || + (advancedOnly && index > lastNonUser) + ) { + return false + } + const images = message.blocks.filter(isImageRefBlock) + return ( + images.length === count && + images.every((image) => !image.path) && + !normalizedNativeChatUserMessageText(message) + ) + }) + if (row) { + claimed.add(row.id) + } + return Boolean(row) +} + +function carriedImageRowClaims(pending: readonly NativeChatPendingSend[]): Set { + return new Set(pending.flatMap((entry) => entry.claimedImageRowIds ?? [])) +} + /** Keep echoes until a later non-user turn to avoid first-turn flashes during transient empty reads. */ export function prunePendingSends( pending: T[], @@ -141,6 +191,8 @@ export function prunePendingSends( return pending } const consumed = new Map() + const claimedImageRows = carriedImageRowClaims(pending) + const carriedClaimCount = claimedImageRows.size const exactKeep = pending.map((entry) => { const contentKey = nativeChatPendingContentKey(entry) const key = nativeChatPendingMatchKey(entry) @@ -152,7 +204,7 @@ export function prunePendingSends( const used = consumed.get(key) ?? 0 const occurrence = nativeChatPendingOccurrence(entry, used) consumed.set(key, Math.max(used, occurrence)) - return occurrence > available + return occurrence > available && !claimPathlessImageRow(entry, messages, claimedImageRows, true) }) // Lost Enter can glue sends into one transcript row; retire them after the assistant advances. const stillOpen = pending.filter((_, index) => exactKeep[index]) @@ -167,7 +219,15 @@ export function prunePendingSends( const openIndex = stillOpen.indexOf(entry) return openIndex === -1 || !gluedRepresented.has(openIndex) }) - return next.length === pending.length ? pending : next + if (next.length === pending.length) { + return pending + } + // Claims here retire their sends; hand them to the remaining image sends so none reuses the row. + return claimedImageRows.size === carriedClaimCount + ? next + : next.map((entry) => + entry.imagePaths?.length ? { ...entry, claimedImageRowIds: [...claimedImageRows] } : entry + ) } /** Sends whose transcript row has not landed yet; exact and glued matches both hide an echo. */ @@ -179,6 +239,7 @@ export function visibleNativeChatPendingSends( return pending } const consumed = new Map() + const claimedImageRows = carriedImageRowClaims(pending) const exactVisible = pending.map((entry) => { const contentKey = nativeChatPendingContentKey(entry) const key = nativeChatPendingMatchKey(entry) @@ -190,7 +251,10 @@ export function visibleNativeChatPendingSends( const used = consumed.get(key) ?? 0 const occurrence = nativeChatPendingOccurrence(entry, used) consumed.set(key, Math.max(used, occurrence)) - return occurrence > represented + return ( + occurrence > represented && + !claimPathlessImageRow(entry, existingMessages, claimedImageRows, false) + ) }) // Hide glued echoes once their user row exists, before assistant advancement permits pruning. const stillVisible = pending.filter((_, index) => exactVisible[index]) diff --git a/src/shared/worker-transcript-text.ts b/src/shared/worker-transcript-text.ts index 038f451e1ff..895a387ba38 100644 --- a/src/shared/worker-transcript-text.ts +++ b/src/shared/worker-transcript-text.ts @@ -7,6 +7,7 @@ import { } from './native-chat/transcript/native-chat-subagent-summary' import { subagentFallbackText } from './native-chat/transcript/native-chat-subagent-items' import type { NativeChatMessage } from './native-chat/native-chat-types' +import { isInlineDataUrl } from './native-chat/native-chat-inline-data-url' export function formatWorkerTranscriptMessage(message: NativeChatMessage): string { const fallbacks = workerTranscriptBlockFallbacks(message.blocks) @@ -21,7 +22,7 @@ export function formatWorkerTranscriptMessage(message: NativeChatMessage): strin return `[tool result${block.isError ? ' error' : ''}] ${block.output}` } if (block.type === 'image-ref') { - return block.url ? `[image] ${block.url}` : `[image omitted]` + return block.url && !isInlineDataUrl(block.url) ? `[image] ${block.url}` : `[image omitted]` } if ( block.type === 'background-task' ||