diff --git a/mobile/src/session/mobile-native-chat-render-data.test.ts b/mobile/src/session/mobile-native-chat-render-data.test.ts index 2bb0ef89305..bbeb953b793 100644 --- a/mobile/src/session/mobile-native-chat-render-data.test.ts +++ b/mobile/src/session/mobile-native-chat-render-data.test.ts @@ -133,6 +133,22 @@ describe('buildMobileNativeChatTransientData', () => { ]) }) + it('keeps a marker beyond the folded run’s image count as literal text', () => { + const data = build( + [ + user('u1', '[Image: source: /tmp/a.png]'), + user('u2', '[Image #1] compare with the [Image #2] I mentioned earlier') + ], + null, + [] + ) + + expect(data[0]?.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'text', text: 'compare with the [Image #2] I mentioned earlier' } + ]) + }) + it('renders a lone image marker turn (no caption) as an image-ref block', () => { const data = build([user('u1', '[Image: source: /tmp/a.png]')], null, []) expect(data[0]?.blocks).toEqual([{ type: 'image-ref', path: '/tmp/a.png' }]) diff --git a/src/shared/native-chat-image-transcript-markers.test.ts b/src/shared/native-chat-image-transcript-markers.test.ts index 477d17d54ae..16ffff4267b 100644 --- a/src/shared/native-chat-image-transcript-markers.test.ts +++ b/src/shared/native-chat-image-transcript-markers.test.ts @@ -259,6 +259,79 @@ describe('normalizeImageTranscriptMessages', () => { ]) }) + it('keeps markers beyond the anchored run’s image count as the user’s own words', () => { + const out = normalizeImageTranscriptMessages([ + userText('s1', '[Image: source: /tmp/a.png]'), + userText('p1', '[Image #1] compare with the [Image #2] I mentioned earlier') + ]) + + expect(out).toHaveLength(1) + expect(out[0]!.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'text', text: 'compare with the [Image #2] I mentioned earlier' } + ]) + }) + + it('strips by document position, not by marker ordinal', () => { + const out = normalizeImageTranscriptMessages([ + userText('s1', '[Image: source: /tmp/a.png]'), + userText('p1', '[Image #2] came first but [Image #1] came second') + ]) + + expect(out[0]!.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'text', text: 'came first but [Image #1] came second' } + ]) + }) + + it('strips a marker whose ordinal exceeds the run size when it is within the count', () => { + const out = normalizeImageTranscriptMessages([ + userText('s1', '[Image: source: /tmp/a.png]'), + userText('p1', '[Image #7] what is this?') + ]) + + expect(out[0]!.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'text', text: 'what is this?' } + ]) + }) + + it('spends the image budget across text blocks in document order', () => { + const prompt: NativeChatMessage = { + ...userText('prompt', 'unused'), + blocks: [ + { type: 'text', text: '[Image #1] look at' }, + { type: 'image-ref', path: '/tmp/existing.png' }, + { type: 'text', text: 'not [Image #2] though' } + ] + } + const out = normalizeImageTranscriptMessages([ + userText('s1', '[Image: source: /tmp/a.png]'), + prompt + ]) + + expect(out[0]!.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'text', text: 'look at' }, + { type: 'image-ref', path: '/tmp/existing.png' }, + { type: 'text', text: 'not [Image #2] though' } + ]) + }) + + it('keeps the surplus marker on a two-image run that mentions a third', () => { + const out = normalizeImageTranscriptMessages([ + userText('s1', '[Image: source: /tmp/a.png]'), + userText('s2', '[Image: source: /tmp/b.png]'), + userText('p1', '[Image #1] [Image #2] versus the [Image #3] from before') + ]) + + expect(out[0]!.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'image-ref', path: '/tmp/b.png' }, + { type: 'text', text: 'versus the [Image #3] from before' } + ]) + }) + it('leaves assistant messages untouched', () => { const assistant: NativeChatMessage = { id: 'a', diff --git a/src/shared/native-chat-image-transcript-markers.ts b/src/shared/native-chat-image-transcript-markers.ts index 4864187cc6b..e27006f0a3a 100644 --- a/src/shared/native-chat-image-transcript-markers.ts +++ b/src/shared/native-chat-image-transcript-markers.ts @@ -27,20 +27,41 @@ export function isImageSourceUserTurn(message: NativeChatMessage): boolean { return message.role === 'user' && imageSourcePathFromText(soleText(message) ?? '') !== null } -export function stripImagePromptMarker(text: string): string { - const stripped = text.replace(IMAGE_PROMPT_MARKERS, '') +function countMarkers(text: string): number { + return text.match(IMAGE_PROMPT_MARKERS)?.length ?? 0 +} + +/** Strips the first `limit` markers in document order; the rest are the user's + * own words, so they stay verbatim. */ +function stripImagePromptMarkersUpTo(text: string, limit: number): string { + if (limit <= 0) { + return text + } + let used = 0 + const stripped = text.replace(IMAGE_PROMPT_MARKERS, (match) => { + if (used >= limit) { + return match + } + used += 1 + return '' + }) if (stripped === text) { return text } let result = IMAGE_PROMPT_MARKER_AT_START.test(text) ? stripped.replace(HORIZONTAL_WHITESPACE_START, '') : stripped - if (IMAGE_PROMPT_MARKER_AT_END.test(text)) { + // Why: a trailing marker left in place still owns the whitespace before it. + if (used === countMarkers(text) && IMAGE_PROMPT_MARKER_AT_END.test(text)) { result = result.replace(HORIZONTAL_WHITESPACE_END, '') } return result } +export function stripImagePromptMarker(text: string): string { + return stripImagePromptMarkersUpTo(text, Number.POSITIVE_INFINITY) +} + export function normalizeNativeChatUserText(text: string): string { return stripImagePromptMarker(text).trim().replace(/\s+/g, ' ') } @@ -78,10 +99,14 @@ export function nativeChatUserMessageMatchText(message: NativeChatMessage): stri return nativeChatUserTextMatchText(joined, message.blocks.some(isImageRefBlock)) || null } +/** `limit` is the number of images the run actually carried. Only that many + * markers can be placeholders; the rest are text the user typed. */ function stripImagePromptMarkersFromTextBlocks( - blocks: readonly NativeChatBlock[] + blocks: readonly NativeChatBlock[], + limit: number ): NativeChatBlock[] { let sawText = false + let remaining = limit let next: NativeChatBlock[] | null = null for (let index = 0; index < blocks.length; index += 1) { const block = blocks[index]! @@ -91,7 +116,8 @@ function stripImagePromptMarkersFromTextBlocks( } const isFirstText = !sawText sawText = true - const text = stripImagePromptMarker(block.text) + const text = stripImagePromptMarkersUpTo(block.text, remaining) + remaining -= Math.min(remaining, countMarkers(block.text)) if (!text.trim() && (text !== block.text || isFirstText)) { next ??= blocks.slice(0, index) continue @@ -173,7 +199,7 @@ export function normalizeImageTranscriptMessages( ...prompt, blocks: [ ...imagePaths.map((path) => ({ type: 'image-ref' as const, path })), - ...stripImagePromptMarkersFromTextBlocks(prompt.blocks) + ...stripImagePromptMarkersFromTextBlocks(prompt.blocks, imagePaths.length) ] }) index = nextIndex