From 3d4658ed12f616c8b2f4e6f76b9f85172a2e0725 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 12 Aug 2026 19:21:12 -0700 Subject: [PATCH] fix(native-chat): handle image markers in any position --- ...mobile-native-chat-draft-reconcile.test.ts | 41 ++++++++++++++++++- .../mobile-native-chat-draft-reconcile.ts | 6 +-- ...le-native-chat-image-transcript-markers.ts | 2 +- .../mobile-native-chat-render-data.test.ts | 18 +++++++- .../use-mobile-native-chat-drafts.test.ts | 35 +++++++++++++++- .../native-chat/native-chat-pending.test.ts | 11 +++++ ...e-chat-preassembled-session-parity.test.ts | 23 +++++++++++ .../native-chat-session-assembler.test.ts | 28 +++++++++++++ ...tive-chat-image-transcript-markers.test.ts | 28 ++++++++++++- .../native-chat-image-transcript-markers.ts | 20 +++++---- 10 files changed, 195 insertions(+), 17 deletions(-) diff --git a/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts b/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts index 4c0cee804c1..f910d23bc3c 100644 --- a/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts +++ b/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts @@ -2,8 +2,10 @@ import { describe, expect, it } from 'vitest' import type { NativeChatMessage } from '../../../src/shared/native-chat-types' import { findLandedImagePreviewEchoes, + findLandedUnconfirmedSends, migrateImagePreviewMessageIds, - type PendingImagePreviewEcho + type PendingImagePreviewEcho, + type UnconfirmedSend } from './mobile-native-chat-draft-reconcile' function userText(id: string, text: string): NativeChatMessage { @@ -21,6 +23,30 @@ function pending(id: string, images: string[], expectedOccurrence = 1): PendingI } describe('mobile native chat image preview reconciliation', () => { + it('reconciles a trailing-marker echo and hands its preview to that echo', () => { + const messages = [ + userText('source', '[Image: source: /tmp/a.png]'), + userText('prompt', 'look at this[Image #1]') + ] + const preview = { + ...pending('pending', ['file:///a.jpg']), + text: 'look at this' + } + const unconfirmed: UnconfirmedSend = { + draftKey: 'draft', + pendingKey: 'pending-key', + text: 'look at this', + normalizedText: 'look at this', + baselineTailMessageId: null, + deadline: null + } + + expect(findLandedUnconfirmedSends(messages, [unconfirmed])).toEqual([unconfirmed]) + expect(findLandedImagePreviewEchoes(messages, [preview])).toEqual([ + { pendingId: 'pending', messageId: 'prompt', images: ['file:///a.jpg'] } + ]) + }) + it('keeps separate adjacent image-only sends independently reconcilable', () => { const landed = findLandedImagePreviewEchoes( [ @@ -69,4 +95,17 @@ describe('mobile native chat image preview reconciliation', () => { [sessionKey]: { prompt: ['file:///a.jpg'] } }) }) + + it('moves an early standalone preview to a trailing-marker prompt id', () => { + const sessionKey = 'host\0worktree\0tab\0session' + const previous = { [sessionKey]: { source: ['file:///a.jpg'] } } + const messages = [ + userText('source', '[Image: source: /tmp/a.png]'), + userText('prompt', 'look[Image #1]') + ] + + expect(migrateImagePreviewMessageIds(previous, sessionKey, messages)).toEqual({ + [sessionKey]: { prompt: ['file:///a.jpg'] } + }) + }) }) diff --git a/mobile/src/session/mobile-native-chat-draft-reconcile.ts b/mobile/src/session/mobile-native-chat-draft-reconcile.ts index a6f599b7458..5a7e0819f0c 100644 --- a/mobile/src/session/mobile-native-chat-draft-reconcile.ts +++ b/mobile/src/session/mobile-native-chat-draft-reconcile.ts @@ -24,8 +24,8 @@ export function normalizedUserText(message: NativeChatMessage): string | null { .filter((block) => block.type === 'text') .map((block) => (block.type === 'text' ? block.text : '')) .join('') - // Claude echoes a captioned image send as `[Image #1] caption` — the sent - // text must still match its echo, so strip the marker before comparing. + // Claude can place `[Image #N]` anywhere in a caption echo, so strip it + // before comparing against the sent text. const stripped = stripImagePromptMarker(text).trim() return stripped || null } @@ -127,7 +127,7 @@ function imagePreviewReplacementMessageId( } /** Moves previews forward when a progressive source-only transcript frame later - * folds into the marker-prefixed prompt with a different authoritative id. */ + * folds into the marker-bearing prompt with a different authoritative id. */ export function migrateImagePreviewMessageIds( previous: Record>, sessionKey: string, diff --git a/mobile/src/session/mobile-native-chat-image-transcript-markers.ts b/mobile/src/session/mobile-native-chat-image-transcript-markers.ts index 2c1ecc291ff..fa9da474cda 100644 --- a/mobile/src/session/mobile-native-chat-image-transcript-markers.ts +++ b/mobile/src/session/mobile-native-chat-image-transcript-markers.ts @@ -1,6 +1,6 @@ // Single-sources the marker logic (pure functions over shared types): // Claude records an attached image as `[Image: source: /path]` (+ `[Image #N]` -// prefix on the caption turn), and both render and echo reconciliation must +// on the caption turn), and both render and echo reconciliation must // agree with desktop on how those marker turns are interpreted. export { imageSourcePathFromText, diff --git a/mobile/src/session/mobile-native-chat-render-data.test.ts b/mobile/src/session/mobile-native-chat-render-data.test.ts index c67257ec24d..1a1811f808d 100644 --- a/mobile/src/session/mobile-native-chat-render-data.test.ts +++ b/mobile/src/session/mobile-native-chat-render-data.test.ts @@ -92,8 +92,8 @@ describe('buildMobileNativeChatTransientData', () => { }) it('folds transcript image marker turns into image-ref blocks (desktop parity)', () => { - // Claude records an attached image as `[Image: source: /path]` + an - // `[Image #1] `-prefixed caption turn; the fold must merge them into one + // Claude records an attached image as `[Image: source: /path]` plus a + // caption turn carrying `[Image #1]`; the fold must merge them into one // user turn with an image-ref block instead of showing raw marker text. const data = build( [ @@ -111,6 +111,20 @@ describe('buildMobileNativeChatTransientData', () => { ]) }) + it('folds a trailing-marker image echo into one user bubble', () => { + const data = build( + [user('u1', '[Image: source: /tmp/a.png]'), user('u2', 'look at this[Image #1]')], + null, + [] + ) + + expect(data).toHaveLength(1) + expect(data[0]?.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'text', text: 'look at this' } + ]) + }) + it('renders a lone image marker turn (no caption) as an image-ref block', () => { const data = build([user('u1', '[Image: source: /tmp/a.png]')], null, []) expect(data[0]?.blocks).toEqual([{ type: 'image-ref', path: '/tmp/a.png' }]) diff --git a/mobile/src/session/use-mobile-native-chat-drafts.test.ts b/mobile/src/session/use-mobile-native-chat-drafts.test.ts index db8e76696d0..daf451e53f9 100644 --- a/mobile/src/session/use-mobile-native-chat-drafts.test.ts +++ b/mobile/src/session/use-mobile-native-chat-drafts.test.ts @@ -316,8 +316,8 @@ describe('useMobileNativeChatDrafts', () => { }) expect(state?.pending).toHaveLength(1) - // Claude echoes a captioned image send as two turns: the source marker and - // the caption prefixed with `[Image #1] ` — the pending must still match. + // Claude echoes a captioned image send as a source turn plus a caption + // carrying `[Image #1]`; the pending must still match. await act(async () => renderer?.update( createElement(Harness, { @@ -334,6 +334,37 @@ describe('useMobileNativeChatDrafts', () => { expect(state?.imagePreviewsByMessageId).toEqual({ u2: ['file:///a.jpg'] }) }) + it('reconciles a captioned image echo with a trailing [Image #N] marker', async () => { + await mount('a') + await act(async () => + renderer?.update( + createElement(Harness, { tabId: 'a', messages: [assistantTextMessage('a1', 'hi')] }) + ) + ) + const origin = state?.captureSendOrigin('look at this') + act(() => { + if (origin) { + state?.acceptSend(origin, 'look at this', ['file:///a.jpg']) + } + }) + expect(state?.pending).toHaveLength(1) + + await act(async () => + renderer?.update( + createElement(Harness, { + tabId: 'a', + messages: [ + assistantTextMessage('a1', 'hi'), + userTextMessage('u1', '[Image: source: /tmp/a.png]'), + userTextMessage('u2', 'look at this[Image #1]') + ] + }) + ) + ) + expect(state?.pending).toEqual([]) + expect(state?.imagePreviewsByMessageId).toEqual({ u2: ['file:///a.jpg'] }) + }) + it('hands a marker-only image preview to the authoritative user bubble', async () => { await mount('a') const origin = state?.captureSendOrigin('') diff --git a/src/renderer/src/components/native-chat/native-chat-pending.test.ts b/src/renderer/src/components/native-chat/native-chat-pending.test.ts index 9686aa3eb38..34e4a4c0389 100644 --- a/src/renderer/src/components/native-chat/native-chat-pending.test.ts +++ b/src/renderer/src/components/native-chat/native-chat-pending.test.ts @@ -95,6 +95,17 @@ describe('prunePendingSends', () => { expect(next).toEqual([]) }) + it('drops an attachment pending send once a trailing-marker prompt advances', () => { + const pending = [ + { ...pendingOf('p1', 'what do you see'), imagePaths: ['/Users/me/Downloads/3d.png'] } + ] + const next = prunePendingSends(pending, [ + userMessage('m1', 'what do you see[Image #1]'), + assistantMessage('m2', 'an image') + ]) + expect(next).toEqual([]) + }) + it('drops an attachment-only pending send once its image turn advances', () => { const pending = [{ ...pendingOf('p1', ''), imagePaths: ['/tmp/first.png', '/tmp/second.png'] }] const transcript = [ diff --git a/src/renderer/src/components/native-chat/native-chat-preassembled-session-parity.test.ts b/src/renderer/src/components/native-chat/native-chat-preassembled-session-parity.test.ts index 0575ba0a3f9..eb7f2f0d617 100644 --- a/src/renderer/src/components/native-chat/native-chat-preassembled-session-parity.test.ts +++ b/src/renderer/src/components/native-chat/native-chat-preassembled-session-parity.test.ts @@ -74,6 +74,29 @@ describe('preassembled native-chat live sessions', () => { expect(out[0]).toMatchObject({ id: 'image-prompt', source: 'transcript' }) }) + it('keeps legacy parity for trailing image prompt markers', () => { + const transcript: NativeChatMessage[] = [ + message('image-source', { + role: 'user', + blocks: [{ type: 'text', text: '[Image: source: /tmp/a.png]' }], + timestamp: 1 + }), + message('image-prompt', { + role: 'user', + blocks: [{ type: 'text', text: 'describe this[Image #1]' }], + timestamp: 2 + }) + ] + + const out = expectLegacyMessageParity(transcript) + + expect(out).toHaveLength(1) + expect(out[0]?.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'text', text: 'describe this' } + ]) + }) + it('re-dedupes surfaced skill envelopes that collide across sources', () => { const skill = (id: string, plugin: string, source: 'transcript' | 'scrape') => message(id, { diff --git a/src/renderer/src/components/native-chat/native-chat-session-assembler.test.ts b/src/renderer/src/components/native-chat/native-chat-session-assembler.test.ts index 8c081b12796..78321d61548 100644 --- a/src/renderer/src/components/native-chat/native-chat-session-assembler.test.ts +++ b/src/renderer/src/components/native-chat/native-chat-session-assembler.test.ts @@ -112,6 +112,34 @@ describe('assembleNativeChatSession', () => { ]) }) + it('merges Claude image source markers into a trailing-marker prompt', () => { + const imageSource = msg({ + id: 'u-image-source', + role: 'user', + timestamp: 100, + blocks: [{ type: 'text', text: '[Image: source: /Users/me/Downloads/3d.png]' }] + }) + const prompt = msg({ + id: 'u-prompt', + role: 'user', + timestamp: 101, + blocks: [{ type: 'text', text: 'what do you see[Image #1]' }] + }) + + const session = assembleNativeChatSession({ + sources: { transcript: [imageSource, prompt] }, + sessionId: 's1', + agent: 'claude' + }) + + expect(session.messages).toHaveLength(1) + expect(session.messages[0]).toMatchObject({ id: 'u-prompt', role: 'user' }) + expect(session.messages[0].blocks).toEqual([ + { type: 'image-ref', path: '/Users/me/Downloads/3d.png' }, + { type: 'text', text: 'what do you see' } + ]) + }) + it('drops a scrape duplicate even when scrape is processed first by id', () => { const scrape = msg({ id: 'shared-id', diff --git a/src/shared/native-chat-image-transcript-markers.test.ts b/src/shared/native-chat-image-transcript-markers.test.ts index 7e66532c469..0308c44951d 100644 --- a/src/shared/native-chat-image-transcript-markers.test.ts +++ b/src/shared/native-chat-image-transcript-markers.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from 'vitest' import type { NativeChatMessage } from './native-chat-types' -import { normalizeImageTranscriptMessages } from './native-chat-image-transcript-markers' +import { + normalizeImageTranscriptMessages, + stripImagePromptMarker +} from './native-chat-image-transcript-markers' function userText(id: string, text: string): NativeChatMessage { return { @@ -25,6 +28,29 @@ describe('normalizeImageTranscriptMessages', () => { ]) }) + it('merges a source turn into a prompt with a trailing image marker', () => { + const out = normalizeImageTranscriptMessages([ + userText('a', '[Image: source: /tmp/orca-paste-1-2.png]'), + userText('b', 'describe this[Image #1]') + ]) + + expect(out).toHaveLength(1) + expect(out[0]!.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/orca-paste-1-2.png' }, + { type: 'text', text: 'describe this' } + ]) + }) + + it.each([ + ['[Image #1] describe this', 'describe this'], + ['describe this [Image #1]', 'describe this'], + ['describe [Image #1] this', 'describe this'], + ['com[Image #1]pare this', 'compare this'], + ['[Image #1] [Image #2]', ''] + ])('strips image prompt markers anywhere in text', (text, expected) => { + expect(stripImagePromptMarker(text)).toBe(expected) + }) + it('converts a lone [Image: source] turn (no prompt) into an image-ref instead of raw text', () => { const out = normalizeImageTranscriptMessages([ userText('a', '[Image: source: /Users/me/Pictures/hero-image-2.png]') diff --git a/src/shared/native-chat-image-transcript-markers.ts b/src/shared/native-chat-image-transcript-markers.ts index 34c2e9ae32d..a14f395ea2d 100644 --- a/src/shared/native-chat-image-transcript-markers.ts +++ b/src/shared/native-chat-image-transcript-markers.ts @@ -1,7 +1,8 @@ import { isTextBlock, type NativeChatBlock, type NativeChatMessage } from './native-chat-types' const IMAGE_SOURCE_MARKER = /^\[Image:\s*source:\s*(.+?)\]\s*$/ -const IMAGE_PROMPT_MARKERS = /^(?:\[Image #\d+\]\s*)+/ +const IMAGE_PROMPT_MARKER = /\[Image #\d+\]/ +const IMAGE_PROMPT_MARKER_RUN = /[^\S\r\n]*(?:\[Image #\d+\][^\S\r\n]*)+/g function soleText(message: NativeChatMessage): string | null { return message.blocks.length === 1 && isTextBlock(message.blocks[0]) @@ -14,7 +15,12 @@ export function imageSourcePathFromText(text: string): string | null { } export function stripImagePromptMarker(text: string): string { - return text.replace(IMAGE_PROMPT_MARKERS, '') + return text.replace(IMAGE_PROMPT_MARKER_RUN, (run, offset: number, source: string) => { + const hasTextBefore = offset > 0 + const hasTextAfter = offset + run.length < source.length + const hadSurroundingWhitespace = run.replace(/\[Image #\d+\]/g, '').length > 0 + return hasTextBefore && hasTextAfter && hadSurroundingWhitespace ? ' ' : '' + }) } function stripImagePromptMarkersFromFirstText( @@ -40,13 +46,13 @@ function stripImagePromptMarkersFromFirstText( return next } -function imagePromptMarkerStartsMessage(message: NativeChatMessage): boolean { +function imagePromptMarkerAppearsInMessage(message: NativeChatMessage): boolean { const firstText = message.blocks.find(isTextBlock) - return firstText ? IMAGE_PROMPT_MARKERS.test(firstText.text) : false + return firstText ? IMAGE_PROMPT_MARKER.test(firstText.text) : false } -/** Claude records image paths as source turns followed by one marker-prefixed - * prompt. Merge the whole run back into one native user turn. */ +/** Claude records image paths as source turns followed by a prompt carrying + * image markers. Merge the whole run back into one native user turn. */ export function normalizeImageTranscriptMessages( messages: readonly NativeChatMessage[] ): NativeChatMessage[] { @@ -75,7 +81,7 @@ export function normalizeImageTranscriptMessages( if ( prompt?.role === 'user' && prompt.source === message.source && - imagePromptMarkerStartsMessage(prompt) + imagePromptMarkerAppearsInMessage(prompt) ) { normalized.push({ ...prompt,