From 0f51d0b3bbb68dab7b2b3e5b44fb9df2cd3e149b Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 13 Aug 2026 01:18:08 -0700 Subject: [PATCH] Fix native chat image marker position handling (#14162) * fix(native-chat): handle image markers in any position * fix(native-chat): preserve image caption whitespace * test(native-chat): cover marker boundary spacing * fix(mobile): normalize image echo reconciliation * fix(mobile): use idiomatic tail access * refactor(native-chat): share image echo matching * perf(native-chat): avoid unchanged block copies --- ...mobile-native-chat-draft-reconcile.test.ts | 97 +++++++++++++++++- .../mobile-native-chat-draft-reconcile.ts | 27 ++--- ...le-native-chat-image-transcript-markers.ts | 18 +--- .../mobile-native-chat-render-data.test.ts | 18 +++- .../use-mobile-native-chat-drafts.test.ts | 35 ++++++- .../session/use-mobile-native-chat-drafts.ts | 24 ++--- .../native-chat-pending-occurrence.ts | 38 +++---- .../native-chat/native-chat-pending.test.ts | 28 ++++++ ...e-chat-preassembled-session-parity.test.ts | 23 +++++ .../native-chat-session-assembler.test.ts | 28 ++++++ ...tive-chat-image-transcript-markers.test.ts | 93 +++++++++++++++++- .../native-chat-image-transcript-markers.ts | 98 +++++++++++++------ 12 files changed, 418 insertions(+), 109 deletions(-) diff --git a/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts b/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts index 4c0cee804c1..b7453e7a67d 100644 --- a/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts +++ b/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts @@ -2,8 +2,10 @@ import { describe, expect, it } from 'vitest' import type { NativeChatMessage } from '../../../src/shared/native-chat-types' import { findLandedImagePreviewEchoes, + findLandedUnconfirmedSends, migrateImagePreviewMessageIds, - type PendingImagePreviewEcho + type PendingImagePreviewEcho, + type UnconfirmedSend } from './mobile-native-chat-draft-reconcile' function userText(id: string, text: string): NativeChatMessage { @@ -21,6 +23,67 @@ function pending(id: string, images: string[], expectedOccurrence = 1): PendingI } describe('mobile native chat image preview reconciliation', () => { + it('reconciles a trailing-marker echo and hands its preview to that echo', () => { + const messages = [ + userText('source', '[Image: source: /tmp/a.png]'), + userText('prompt', 'look at this[Image #1]') + ] + const preview = { + ...pending('pending', ['file:///a.jpg']), + text: 'look at this' + } + const unconfirmed: UnconfirmedSend = { + draftKey: 'draft', + pendingKey: 'pending-key', + text: 'look at this', + normalizedText: 'look at this', + baselineTailMessageId: null, + deadline: null + } + + expect(findLandedUnconfirmedSends(messages, [unconfirmed])).toEqual([unconfirmed]) + expect(findLandedImagePreviewEchoes(messages, [preview])).toEqual([ + { pendingId: 'pending', messageId: 'prompt', images: ['file:///a.jpg'] } + ]) + }) + + it('reconciles a middle-marker echo without changing its rendered whitespace', () => { + const messages = [ + userText('source', '[Image: source: /tmp/a.png]'), + userText('prompt', 'look [Image #1] here') + ] + const preview = { ...pending('pending', ['file:///a.jpg']), text: 'look here' } + const unconfirmed: UnconfirmedSend = { + draftKey: 'draft', + pendingKey: 'pending-key', + text: 'look here', + normalizedText: 'look here', + baselineTailMessageId: null, + deadline: null + } + + expect(findLandedUnconfirmedSends(messages, [unconfirmed])).toEqual([unconfirmed]) + expect(findLandedImagePreviewEchoes(messages, [preview])).toEqual([ + { pendingId: 'pending', messageId: 'prompt', images: ['file:///a.jpg'] } + ]) + }) + + it('reconciles multiple transcript text blocks with desktop separators', () => { + const prompt: NativeChatMessage = { + ...userText('prompt', 'unused'), + blocks: [ + { type: 'text', text: 'look' }, + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'text', text: '[Image #1] here' } + ] + } + const preview = { ...pending('pending', ['file:///a.jpg']), text: 'look here' } + + expect(findLandedImagePreviewEchoes([prompt], [preview])).toEqual([ + { pendingId: 'pending', messageId: 'prompt', images: ['file:///a.jpg'] } + ]) + }) + it('keeps separate adjacent image-only sends independently reconcilable', () => { const landed = findLandedImagePreviewEchoes( [ @@ -69,4 +132,36 @@ describe('mobile native chat image preview reconciliation', () => { [sessionKey]: { prompt: ['file:///a.jpg'] } }) }) + + it('moves an early standalone preview to a trailing-marker prompt id', () => { + const sessionKey = 'host\0worktree\0tab\0session' + const previous = { [sessionKey]: { source: ['file:///a.jpg'] } } + const messages = [ + userText('source', '[Image: source: /tmp/a.png]'), + userText('prompt', 'look[Image #1]') + ] + + expect(migrateImagePreviewMessageIds(previous, sessionKey, messages)).toEqual({ + [sessionKey]: { prompt: ['file:///a.jpg'] } + }) + }) + + it('moves a preview when the prompt marker is in a later text block', () => { + const sessionKey = 'host\0worktree\0tab\0session' + const previous = { [sessionKey]: { source: ['file:///a.jpg'] } } + const prompt: NativeChatMessage = { + ...userText('prompt', 'unused'), + blocks: [ + { type: 'text', text: 'look' }, + { type: 'text', text: '[Image #1] here' } + ] + } + + expect( + migrateImagePreviewMessageIds(previous, sessionKey, [ + userText('source', '[Image: source: /tmp/a.png]'), + prompt + ]) + ).toEqual({ [sessionKey]: { prompt: ['file:///a.jpg'] } }) + }) }) diff --git a/mobile/src/session/mobile-native-chat-draft-reconcile.ts b/mobile/src/session/mobile-native-chat-draft-reconcile.ts index a6f599b7458..a68fe5bcf32 100644 --- a/mobile/src/session/mobile-native-chat-draft-reconcile.ts +++ b/mobile/src/session/mobile-native-chat-draft-reconcile.ts @@ -1,9 +1,12 @@ import { isImageRefBlock, type NativeChatMessage } from '../../../src/shared/native-chat-types' import { + hasImagePromptMarker, isImageSourceUserTurn, normalizeImageTranscriptMessages, - stripImagePromptMarker + normalizeNativeChatUserText, + normalizedNativeChatUserMessageText } from './mobile-native-chat-image-transcript-markers' +export { normalizeNativeChatUserText as normalizeReconcileText } from './mobile-native-chat-image-transcript-markers' /** An ack-lost ('unknown' outcome) send held until its transcript echo lands or * the deadline surfaces the uncertainty. */ @@ -17,17 +20,7 @@ export type UnconfirmedSend = { } export function normalizedUserText(message: NativeChatMessage): string | null { - if (message.role !== 'user') { - return null - } - const text = message.blocks - .filter((block) => block.type === 'text') - .map((block) => (block.type === 'text' ? block.text : '')) - .join('') - // Claude echoes a captioned image send as `[Image #1] caption` — the sent - // text must still match its echo, so strip the marker before comparing. - const stripped = stripImagePromptMarker(text).trim() - return stripped || null + return normalizedNativeChatUserMessageText(message) } export function countUserTextOccurrences( @@ -117,17 +110,13 @@ function imagePreviewReplacementMessageId( nextIndex++ } const prompt = messages[nextIndex] - const firstText = prompt?.blocks.find((block) => block.type === 'text') - return prompt?.role === 'user' && - prompt.source === source.source && - firstText?.type === 'text' && - stripImagePromptMarker(firstText.text) !== firstText.text + return prompt?.role === 'user' && prompt.source === source.source && hasImagePromptMarker(prompt) ? prompt.id : null } /** Moves previews forward when a progressive source-only transcript frame later - * folds into the marker-prefixed prompt with a different authoritative id. */ + * folds into the marker-bearing prompt with a different authoritative id. */ export function migrateImagePreviewMessageIds( previous: Record>, sessionKey: string, @@ -171,7 +160,7 @@ export function findLandedImagePreviewEchoes( if (!entry.images?.length) { continue } - const targetText = entry.text.trim() + const targetText = normalizeNativeChatUserText(entry.text) const candidates = normalized.filter((message) => { if (message.role !== 'user') { return false diff --git a/mobile/src/session/mobile-native-chat-image-transcript-markers.ts b/mobile/src/session/mobile-native-chat-image-transcript-markers.ts index 2c1ecc291ff..3c3998349ee 100644 --- a/mobile/src/session/mobile-native-chat-image-transcript-markers.ts +++ b/mobile/src/session/mobile-native-chat-image-transcript-markers.ts @@ -1,21 +1,13 @@ // Single-sources the marker logic (pure functions over shared types): // Claude records an attached image as `[Image: source: /path]` (+ `[Image #N]` -// prefix on the caption turn), and both render and echo reconciliation must +// on the caption turn), and both render and echo reconciliation must // agree with desktop on how those marker turns are interpreted. export { imageSourcePathFromText, + hasImagePromptMarker, + isImageSourceUserTurn, normalizeImageTranscriptMessages, + normalizeNativeChatUserText, + normalizedNativeChatUserMessageText, stripImagePromptMarker } from '../../../src/shared/native-chat-image-transcript-markers' -import { imageSourcePathFromText } from '../../../src/shared/native-chat-image-transcript-markers' -import { isTextBlock, type NativeChatMessage } from '../../../src/shared/native-chat-types' - -/** A raw (un-normalized) transcript user turn that is an image-source marker — - * the echo shape of an image riding along on a send. */ -export function isImageSourceUserTurn(message: NativeChatMessage): boolean { - if (message.role !== 'user' || message.blocks.length !== 1) { - return false - } - const block = message.blocks[0] - return block !== undefined && isTextBlock(block) && imageSourcePathFromText(block.text) !== null -} diff --git a/mobile/src/session/mobile-native-chat-render-data.test.ts b/mobile/src/session/mobile-native-chat-render-data.test.ts index c67257ec24d..1a1811f808d 100644 --- a/mobile/src/session/mobile-native-chat-render-data.test.ts +++ b/mobile/src/session/mobile-native-chat-render-data.test.ts @@ -92,8 +92,8 @@ describe('buildMobileNativeChatTransientData', () => { }) it('folds transcript image marker turns into image-ref blocks (desktop parity)', () => { - // Claude records an attached image as `[Image: source: /path]` + an - // `[Image #1] `-prefixed caption turn; the fold must merge them into one + // Claude records an attached image as `[Image: source: /path]` plus a + // caption turn carrying `[Image #1]`; the fold must merge them into one // user turn with an image-ref block instead of showing raw marker text. const data = build( [ @@ -111,6 +111,20 @@ describe('buildMobileNativeChatTransientData', () => { ]) }) + it('folds a trailing-marker image echo into one user bubble', () => { + const data = build( + [user('u1', '[Image: source: /tmp/a.png]'), user('u2', 'look at this[Image #1]')], + null, + [] + ) + + expect(data).toHaveLength(1) + expect(data[0]?.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'text', text: 'look at this' } + ]) + }) + it('renders a lone image marker turn (no caption) as an image-ref block', () => { const data = build([user('u1', '[Image: source: /tmp/a.png]')], null, []) expect(data[0]?.blocks).toEqual([{ type: 'image-ref', path: '/tmp/a.png' }]) diff --git a/mobile/src/session/use-mobile-native-chat-drafts.test.ts b/mobile/src/session/use-mobile-native-chat-drafts.test.ts index db8e76696d0..daf451e53f9 100644 --- a/mobile/src/session/use-mobile-native-chat-drafts.test.ts +++ b/mobile/src/session/use-mobile-native-chat-drafts.test.ts @@ -316,8 +316,8 @@ describe('useMobileNativeChatDrafts', () => { }) expect(state?.pending).toHaveLength(1) - // Claude echoes a captioned image send as two turns: the source marker and - // the caption prefixed with `[Image #1] ` — the pending must still match. + // Claude echoes a captioned image send as a source turn plus a caption + // carrying `[Image #1]`; the pending must still match. await act(async () => renderer?.update( createElement(Harness, { @@ -334,6 +334,37 @@ describe('useMobileNativeChatDrafts', () => { expect(state?.imagePreviewsByMessageId).toEqual({ u2: ['file:///a.jpg'] }) }) + it('reconciles a captioned image echo with a trailing [Image #N] marker', async () => { + await mount('a') + await act(async () => + renderer?.update( + createElement(Harness, { tabId: 'a', messages: [assistantTextMessage('a1', 'hi')] }) + ) + ) + const origin = state?.captureSendOrigin('look at this') + act(() => { + if (origin) { + state?.acceptSend(origin, 'look at this', ['file:///a.jpg']) + } + }) + expect(state?.pending).toHaveLength(1) + + await act(async () => + renderer?.update( + createElement(Harness, { + tabId: 'a', + messages: [ + assistantTextMessage('a1', 'hi'), + userTextMessage('u1', '[Image: source: /tmp/a.png]'), + userTextMessage('u2', 'look at this[Image #1]') + ] + }) + ) + ) + expect(state?.pending).toEqual([]) + expect(state?.imagePreviewsByMessageId).toEqual({ u2: ['file:///a.jpg'] }) + }) + it('hands a marker-only image preview to the authoritative user bubble', async () => { await mount('a') const origin = state?.captureSendOrigin('') diff --git a/mobile/src/session/use-mobile-native-chat-drafts.ts b/mobile/src/session/use-mobile-native-chat-drafts.ts index 9da4dea6a9b..55fff5b42e3 100644 --- a/mobile/src/session/use-mobile-native-chat-drafts.ts +++ b/mobile/src/session/use-mobile-native-chat-drafts.ts @@ -7,6 +7,7 @@ import { findLandedUnconfirmedSends, mergeLandedImagePreviewEchoes, migrateImagePreviewMessageIds, + normalizeReconcileText, normalizedUserText, type UnconfirmedSend } from './mobile-native-chat-draft-reconcile' @@ -27,8 +28,7 @@ export type { MobileNativeChatPendingMessage, MobileNativeChatSendOrigin } const NO_PENDING_MESSAGES: MobileNativeChatPendingMessage[] = [] const NO_IMAGE_PREVIEWS: Record = {} -// How long an ack-lost send waits for its transcript echo before the UI surfaces -// that delivery remains unconfirmed. +// Ack-lost sends wait for a transcript echo before surfacing as unconfirmed. const UNCONFIRMED_SEND_DEADLINE_MS = 20_000 export function useMobileNativeChatDrafts(args: { @@ -132,14 +132,13 @@ export function useMobileNativeChatDrafts(args: { if (!draftKey) { return null } - const normalizedText = text.trim() - const currentMessages = messagesRef.current + const normalizedText = normalizeReconcileText(text) return { draftKey, pendingKey, normalizedText, - baselineOccurrences: countUserTextOccurrences(currentMessages, normalizedText), - baselineTailMessageId: currentMessages[currentMessages.length - 1]?.id ?? null + baselineOccurrences: countUserTextOccurrences(messagesRef.current, normalizedText), + baselineTailMessageId: messagesRef.current.at(-1)?.id ?? null } }, [draftKey, pendingKey] @@ -299,26 +298,19 @@ export function useMobileNativeChatDrafts(args: { landedCounts.set(text, (landedCounts.get(text) ?? 0) + 1) } } - // Why: compare against the count captured before send; historical equal - // turns cannot clear a new echo, while duplicates land one occurrence each. - // An image-only echo has no text to match, so it reconciles by ORDINAL - // against the count of new `[Image: source: …]` echo turns after its - // baseline tail — text echoes are excluded so an unrelated outstanding - // text send cannot clear it. Ordinal-vs-count stays stable when the effect - // re-runs on the shrunken list, and ignores paginated-in history. + // Image-only source-turn counts stay stable across reruns and ignore paginated history. const next = current.filter((item) => { if (landedImagePendingIds.has(item.id)) { return false } - // Image echoes hand their local URIs to the authoritative message above; - // never drop them through the text-only fallback before that handoff. + // Keep image echoes until their local preview reaches the authoritative message. if (item.images?.length) { return true } return item.text.trim() === '' ? countImageSourceTurnsAfter(messages, item.baselineTailMessageId) < item.expectedOccurrence - : (landedCounts.get(item.text.trim()) ?? 0) < item.expectedOccurrence + : (landedCounts.get(normalizeReconcileText(item.text)) ?? 0) < item.expectedOccurrence }) if (next.length === current.length) { return previous diff --git a/src/renderer/src/components/native-chat/native-chat-pending-occurrence.ts b/src/renderer/src/components/native-chat/native-chat-pending-occurrence.ts index 5a4631421bc..a994ecf8da7 100644 --- a/src/renderer/src/components/native-chat/native-chat-pending-occurrence.ts +++ b/src/renderer/src/components/native-chat/native-chat-pending-occurrence.ts @@ -1,9 +1,8 @@ -import { stripImagePromptMarker } from '../../../../shared/native-chat-image-transcript-markers' import { - isImageRefBlock, - isTextBlock, - type NativeChatMessage -} from '../../../../shared/native-chat-types' + normalizeNativeChatUserText, + normalizedNativeChatUserMessageText +} from '../../../../shared/native-chat-image-transcript-markers' +import { isImageRefBlock, type NativeChatMessage } from '../../../../shared/native-chat-types' export type NativeChatPendingOccurrence = { text: string @@ -16,7 +15,7 @@ export type NativeChatPendingOccurrence = { } export function normalizeNativeChatPendingText(text: string): string { - return stripImagePromptMarker(text).trim().replace(/\s+/g, ' ') + return normalizeNativeChatUserText(text) } export function nativeChatPendingContentKey( @@ -34,15 +33,15 @@ function nativeChatUserMessageContentKey(message: NativeChatMessage): string | n if (message.role !== 'user') { return null } - const text = message.blocks - .filter(isTextBlock) - .map((block) => block.text) - .join(' ') + const text = normalizedNativeChatUserMessageText(message) ?? '' + if (text) { + return `text:${text}` + } const imagePaths = message.blocks .filter(isImageRefBlock) .map((block) => block.path) .filter((path): path is string => Boolean(path)) - const key = nativeChatPendingContentKey({ text, imagePaths }) + const key = nativeChatPendingContentKey({ text: '', imagePaths }) return key === 'empty' ? null : key } @@ -80,19 +79,6 @@ export function advancedNativeChatUserContentCounts( return advanced } -function nativeChatUserMessageNormalizedText(message: NativeChatMessage): string | null { - if (message.role !== 'user') { - return null - } - const text = normalizeNativeChatPendingText( - message.blocks - .filter(isTextBlock) - .map((block) => block.text) - .join(' ') - ) - return text.length > 0 ? text : null -} - /** User texts that already have a later non-user turn (ready to prune echoes). */ export function advancedNativeChatUserTexts( messages: readonly NativeChatMessage[] @@ -101,7 +87,7 @@ export function advancedNativeChatUserTexts( const waiting: string[] = [] for (const message of messages) { if (message.role === 'user') { - const text = nativeChatUserMessageNormalizedText(message) + const text = normalizedNativeChatUserMessageText(message) if (text) { waiting.push(text) } @@ -119,7 +105,7 @@ export function matchingNativeChatUserTexts( ): readonly string[] { const texts: string[] = [] for (const message of messages) { - const text = nativeChatUserMessageNormalizedText(message) + const text = normalizedNativeChatUserMessageText(message) if (text) { texts.push(text) } diff --git a/src/renderer/src/components/native-chat/native-chat-pending.test.ts b/src/renderer/src/components/native-chat/native-chat-pending.test.ts index 9686aa3eb38..75b2c3d34c7 100644 --- a/src/renderer/src/components/native-chat/native-chat-pending.test.ts +++ b/src/renderer/src/components/native-chat/native-chat-pending.test.ts @@ -95,6 +95,34 @@ describe('prunePendingSends', () => { expect(next).toEqual([]) }) + it('drops an attachment pending send once a trailing-marker prompt advances', () => { + const pending = [ + { ...pendingOf('p1', 'what do you see'), imagePaths: ['/Users/me/Downloads/3d.png'] } + ] + const next = prunePendingSends(pending, [ + userMessage('m1', 'what do you see[Image #1]'), + assistantMessage('m2', 'an image') + ]) + expect(next).toEqual([]) + }) + + it('drops a pending send represented by multiple marker-bearing text blocks', () => { + const prompt: NativeChatMessage = { + ...userMessage('m1', 'unused'), + blocks: [ + { type: 'text', text: 'what do' }, + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'text', text: '[Image #1] you see' } + ] + } + const next = prunePendingSends( + [pendingOf('p1', 'what do you see')], + [prompt, assistantMessage('m2', 'an image')] + ) + + expect(next).toEqual([]) + }) + it('drops an attachment-only pending send once its image turn advances', () => { const pending = [{ ...pendingOf('p1', ''), imagePaths: ['/tmp/first.png', '/tmp/second.png'] }] const transcript = [ diff --git a/src/renderer/src/components/native-chat/native-chat-preassembled-session-parity.test.ts b/src/renderer/src/components/native-chat/native-chat-preassembled-session-parity.test.ts index 0575ba0a3f9..eb7f2f0d617 100644 --- a/src/renderer/src/components/native-chat/native-chat-preassembled-session-parity.test.ts +++ b/src/renderer/src/components/native-chat/native-chat-preassembled-session-parity.test.ts @@ -74,6 +74,29 @@ describe('preassembled native-chat live sessions', () => { expect(out[0]).toMatchObject({ id: 'image-prompt', source: 'transcript' }) }) + it('keeps legacy parity for trailing image prompt markers', () => { + const transcript: NativeChatMessage[] = [ + message('image-source', { + role: 'user', + blocks: [{ type: 'text', text: '[Image: source: /tmp/a.png]' }], + timestamp: 1 + }), + message('image-prompt', { + role: 'user', + blocks: [{ type: 'text', text: 'describe this[Image #1]' }], + timestamp: 2 + }) + ] + + const out = expectLegacyMessageParity(transcript) + + expect(out).toHaveLength(1) + expect(out[0]?.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'text', text: 'describe this' } + ]) + }) + it('re-dedupes surfaced skill envelopes that collide across sources', () => { const skill = (id: string, plugin: string, source: 'transcript' | 'scrape') => message(id, { diff --git a/src/renderer/src/components/native-chat/native-chat-session-assembler.test.ts b/src/renderer/src/components/native-chat/native-chat-session-assembler.test.ts index 8c081b12796..78321d61548 100644 --- a/src/renderer/src/components/native-chat/native-chat-session-assembler.test.ts +++ b/src/renderer/src/components/native-chat/native-chat-session-assembler.test.ts @@ -112,6 +112,34 @@ describe('assembleNativeChatSession', () => { ]) }) + it('merges Claude image source markers into a trailing-marker prompt', () => { + const imageSource = msg({ + id: 'u-image-source', + role: 'user', + timestamp: 100, + blocks: [{ type: 'text', text: '[Image: source: /Users/me/Downloads/3d.png]' }] + }) + const prompt = msg({ + id: 'u-prompt', + role: 'user', + timestamp: 101, + blocks: [{ type: 'text', text: 'what do you see[Image #1]' }] + }) + + const session = assembleNativeChatSession({ + sources: { transcript: [imageSource, prompt] }, + sessionId: 's1', + agent: 'claude' + }) + + expect(session.messages).toHaveLength(1) + expect(session.messages[0]).toMatchObject({ id: 'u-prompt', role: 'user' }) + expect(session.messages[0].blocks).toEqual([ + { type: 'image-ref', path: '/Users/me/Downloads/3d.png' }, + { type: 'text', text: 'what do you see' } + ]) + }) + it('drops a scrape duplicate even when scrape is processed first by id', () => { const scrape = msg({ id: 'shared-id', diff --git a/src/shared/native-chat-image-transcript-markers.test.ts b/src/shared/native-chat-image-transcript-markers.test.ts index 7e66532c469..aff2c6f71fe 100644 --- a/src/shared/native-chat-image-transcript-markers.test.ts +++ b/src/shared/native-chat-image-transcript-markers.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from 'vitest' import type { NativeChatMessage } from './native-chat-types' -import { normalizeImageTranscriptMessages } from './native-chat-image-transcript-markers' +import { + isImageSourceUserTurn, + normalizeImageTranscriptMessages, + normalizeNativeChatUserText, + normalizedNativeChatUserMessageText, + stripImagePromptMarker +} from './native-chat-image-transcript-markers' function userText(id: string, text: string): NativeChatMessage { return { @@ -25,6 +31,91 @@ describe('normalizeImageTranscriptMessages', () => { ]) }) + it('merges a source turn into a prompt with a trailing image marker', () => { + const out = normalizeImageTranscriptMessages([ + userText('a', '[Image: source: /tmp/orca-paste-1-2.png]'), + userText('b', 'describe this[Image #1]') + ]) + + expect(out).toHaveLength(1) + expect(out[0]!.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/orca-paste-1-2.png' }, + { type: 'text', text: 'describe this' } + ]) + }) + + it('folds and strips markers in later text blocks', () => { + const prompt: NativeChatMessage = { + ...userText('prompt', 'unused'), + blocks: [ + { type: 'text', text: 'describe' }, + { type: 'image-ref', path: '/tmp/existing.png' }, + { type: 'text', text: '[Image #1] this' } + ] + } + const out = normalizeImageTranscriptMessages([ + userText('source', '[Image: source: /tmp/a.png]'), + prompt + ]) + + expect(out).toHaveLength(1) + expect(out[0]?.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'text', text: 'describe' }, + { type: 'image-ref', path: '/tmp/existing.png' }, + { type: 'text', text: 'this' } + ]) + }) + + it.each([ + ['[Image #1] describe this', 'describe this'], + ['[Image #1]\t describe this', 'describe this'], + [' \t[Image #1] describe this', 'describe this'], + ['describe this [Image #1]', 'describe this'], + ['describe this \t[Image #1]', 'describe this'], + ['describe this [Image #1]\t ', 'describe this'], + ['describe [Image #1] this', 'describe this'], + ['describe [Image #1]\t this', 'describe \t this'], + ['describe[Image #1]\t this', 'describe\t this'], + ['describe\n[Image #1]\nthis', 'describe\n\nthis'], + ['com[Image #1]pare this', 'compare this'], + ['[Image #1] [Image #2]', ''], + ['literal [Image #x] text', 'literal [Image #x] text'] + ])('strips image prompt markers anywhere in text', (text, expected) => { + expect(stripImagePromptMarker(text)).toBe(expected) + }) + + it('returns long marker-free whitespace without regex backtracking', () => { + const text = ' '.repeat(50_000) + expect(stripImagePromptMarker(text)).toBe(text) + }) + + it('shares marker-aware text matching across multiple text blocks', () => { + const message: NativeChatMessage = { + ...userText('prompt', 'unused'), + blocks: [ + { type: 'text', text: 'look' }, + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'text', text: '[Image #1] here' } + ] + } + + expect(normalizeNativeChatUserText(' look [Image #1] here ')).toBe('look here') + expect(normalizedNativeChatUserMessageText(message)).toBe('look here') + }) + + it('recognizes only sole-text image-source user turns', () => { + const source = userText('source', '[Image: source: /tmp/a.png]') + expect(isImageSourceUserTurn(source)).toBe(true) + expect(isImageSourceUserTurn({ ...source, role: 'assistant' })).toBe(false) + expect( + isImageSourceUserTurn({ + ...source, + blocks: [...source.blocks, { type: 'text', text: 'caption' }] + }) + ).toBe(false) + }) + it('converts a lone [Image: source] turn (no prompt) into an image-ref instead of raw text', () => { const out = normalizeImageTranscriptMessages([ userText('a', '[Image: source: /Users/me/Pictures/hero-image-2.png]') diff --git a/src/shared/native-chat-image-transcript-markers.ts b/src/shared/native-chat-image-transcript-markers.ts index 34c2e9ae32d..df52ec8c12c 100644 --- a/src/shared/native-chat-image-transcript-markers.ts +++ b/src/shared/native-chat-image-transcript-markers.ts @@ -1,7 +1,12 @@ import { isTextBlock, type NativeChatBlock, type NativeChatMessage } from './native-chat-types' const IMAGE_SOURCE_MARKER = /^\[Image:\s*source:\s*(.+?)\]\s*$/ -const IMAGE_PROMPT_MARKERS = /^(?:\[Image #\d+\]\s*)+/ +const IMAGE_PROMPT_MARKER = /\[Image #\d+\]/ +const IMAGE_PROMPT_MARKERS = /\[Image #\d+\]/g +const IMAGE_PROMPT_MARKER_AT_START = /^[^\S\r\n]*\[Image #\d+\]/ +const IMAGE_PROMPT_MARKER_AT_END = /\[Image #\d+\][^\S\r\n]*$/ +const HORIZONTAL_WHITESPACE_START = /^[^\S\r\n]+/ +const HORIZONTAL_WHITESPACE_END = /[^\S\r\n]+$/ function soleText(message: NativeChatMessage): string | null { return message.blocks.length === 1 && isTextBlock(message.blocks[0]) @@ -13,40 +18,75 @@ export function imageSourcePathFromText(text: string): string | null { return text.match(IMAGE_SOURCE_MARKER)?.[1]?.trim() ?? null } -export function stripImagePromptMarker(text: string): string { - return text.replace(IMAGE_PROMPT_MARKERS, '') +export function isImageSourceUserTurn(message: NativeChatMessage): boolean { + return message.role === 'user' && imageSourcePathFromText(soleText(message) ?? '') !== null } -function stripImagePromptMarkersFromFirstText( +export function stripImagePromptMarker(text: string): string { + const stripped = text.replace(IMAGE_PROMPT_MARKERS, '') + if (stripped === text) { + return text + } + let result = IMAGE_PROMPT_MARKER_AT_START.test(text) + ? stripped.replace(HORIZONTAL_WHITESPACE_START, '') + : stripped + if (IMAGE_PROMPT_MARKER_AT_END.test(text)) { + result = result.replace(HORIZONTAL_WHITESPACE_END, '') + } + return result +} + +export function normalizeNativeChatUserText(text: string): string { + return stripImagePromptMarker(text).trim().replace(/\s+/g, ' ') +} + +export function normalizedNativeChatUserMessageText(message: NativeChatMessage): string | null { + if (message.role !== 'user') { + return null + } + const normalized = normalizeNativeChatUserText( + message.blocks + .filter(isTextBlock) + .map((block) => block.text) + .join(' ') + ) + return normalized || null +} + +function stripImagePromptMarkersFromTextBlocks( blocks: readonly NativeChatBlock[] ): NativeChatBlock[] { - const textIndex = blocks.findIndex(isTextBlock) - if (textIndex === -1) { - return blocks as NativeChatBlock[] + let sawText = false + let next: NativeChatBlock[] | null = null + for (let index = 0; index < blocks.length; index += 1) { + const block = blocks[index]! + if (!isTextBlock(block)) { + next?.push(block) + continue + } + const isFirstText = !sawText + sawText = true + const text = stripImagePromptMarker(block.text) + if (!text.trim() && (text !== block.text || isFirstText)) { + next ??= blocks.slice(0, index) + continue + } + if (text !== block.text) { + next ??= blocks.slice(0, index) + next.push({ ...block, text }) + continue + } + next?.push(block) } - const block = blocks[textIndex] - if (!block || !isTextBlock(block)) { - return blocks as NativeChatBlock[] - } - const text = stripImagePromptMarker(block.text) - if (text.trim().length === 0) { - return blocks.filter((_, index) => index !== textIndex) - } - if (text === block.text) { - return blocks as NativeChatBlock[] - } - const next = [...blocks] - next[textIndex] = { ...block, text } - return next + return next ?? (blocks as NativeChatBlock[]) } -function imagePromptMarkerStartsMessage(message: NativeChatMessage): boolean { - const firstText = message.blocks.find(isTextBlock) - return firstText ? IMAGE_PROMPT_MARKERS.test(firstText.text) : false +export function hasImagePromptMarker(message: NativeChatMessage): boolean { + return message.blocks.some((block) => isTextBlock(block) && IMAGE_PROMPT_MARKER.test(block.text)) } -/** Claude records image paths as source turns followed by one marker-prefixed - * prompt. Merge the whole run back into one native user turn. */ +/** Claude records image paths as source turns followed by a prompt carrying + * image markers. Merge the whole run back into one native user turn. */ export function normalizeImageTranscriptMessages( messages: readonly NativeChatMessage[] ): NativeChatMessage[] { @@ -75,13 +115,13 @@ export function normalizeImageTranscriptMessages( if ( prompt?.role === 'user' && prompt.source === message.source && - imagePromptMarkerStartsMessage(prompt) + hasImagePromptMarker(prompt) ) { normalized.push({ ...prompt, blocks: [ ...imagePaths.map((path) => ({ type: 'image-ref' as const, path })), - ...stripImagePromptMarkersFromFirstText(prompt.blocks) + ...stripImagePromptMarkersFromTextBlocks(prompt.blocks) ] }) index = nextIndex @@ -93,7 +133,7 @@ export function normalizeImageTranscriptMessages( }) continue } - const blocks = stripImagePromptMarkersFromFirstText(message.blocks) + const blocks = stripImagePromptMarkersFromTextBlocks(message.blocks) if (blocks === message.blocks) { normalized?.push(message) } else {