diff --git a/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts b/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts index 5c6776008b1..2b300bf8387 100644 --- a/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts +++ b/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts @@ -37,6 +37,7 @@ describe('mobile native chat image preview reconciliation', () => { pendingKey: 'pending-key', text: 'look at this', normalizedText: 'look at this', + imageCount: 1, baselineTailMessageId: null, deadline: null } @@ -58,6 +59,7 @@ describe('mobile native chat image preview reconciliation', () => { pendingKey: 'pending-key', text: 'look here', normalizedText: 'look here', + imageCount: 1, baselineTailMessageId: null, deadline: null } diff --git a/mobile/src/session/mobile-native-chat-draft-reconcile.ts b/mobile/src/session/mobile-native-chat-draft-reconcile.ts index f07eaffd6a7..3f27a42f72a 100644 --- a/mobile/src/session/mobile-native-chat-draft-reconcile.ts +++ b/mobile/src/session/mobile-native-chat-draft-reconcile.ts @@ -3,15 +3,12 @@ import { countImagePromptMarkers, hasImagePromptMarker, isImageSourceUserTurn, + nativeChatUserMessageImageEvidenceCount, nativeChatUserMessageMatchText, + nativeChatUserTextMatchText, normalizeImageTranscriptMessages, - normalizeNativeChatUserText, normalizedNativeChatUserMessageText } from './mobile-native-chat-image-transcript-markers' -export { - normalizeNativeChatUserText as normalizeReconcileText, - normalizeNativeChatUserTextWithLiteralFallback as normalizeReconcileTextWithLiteralFallback -} from './mobile-native-chat-image-transcript-markers' /** An ack-lost ('unknown' outcome) send held until its transcript echo lands or * the deadline surfaces the uncertainty. */ @@ -20,6 +17,7 @@ export type UnconfirmedSend = { pendingKey: string | null text: string normalizedText: string + imageCount: number baselineTailMessageId: string | null deadline: ReturnType | null } @@ -30,11 +28,15 @@ export function normalizedUserText(message: NativeChatMessage): string | null { export function countUserTextOccurrences( messages: readonly NativeChatMessage[], - text: string + text: string, + imageCount = 0 ): number { let count = 0 for (const message of messages) { - if (nativeChatUserMessageMatchText(message) === text) { + if ( + nativeChatUserMessageMatchText(message) === text && + nativeChatUserMessageImageEvidenceCount(message) >= imageCount + ) { count++ } } @@ -178,20 +180,24 @@ export function findLandedImagePreviewEchoes( if (!entry.images?.length) { continue } - const targetText = normalizeNativeChatUserText(entry.text) + const targetText = nativeChatUserTextMatchText(entry.text, true) const candidates = normalized.filter((message) => { if (message.role !== 'user') { return false } + const imageCount = nativeChatUserMessageImageEvidenceCount(message) if (targetText) { - return normalizedUserText(message) === targetText + return ( + nativeChatUserMessageMatchText(message) === targetText && + imageCount >= entry.images!.length + ) } - const imageCount = message.blocks.filter(isImageRefBlock).length + const imageRefCount = message.blocks.filter(isImageRefBlock).length // Why: a marker-only echo proves delivery per marker, so a partially // rendered turn must not claim (and rebind) previews it cannot account for. return ( message.blocks.length === 0 || - imageCount >= entry.images!.length || + imageRefCount >= entry.images!.length || (countImagePromptMarkers(message) >= entry.images!.length && normalizedUserText(message) === null) ) @@ -228,7 +234,10 @@ export function findLandedUnconfirmedSends( // (`[Image: source: …]` or no text) keys under '' so an empty-text send can // claim it. const messageIndexById = new Map() - const userMessagesByText = new Map>() + const userMessagesByText = new Map< + string, + Array<{ id: string; index: number; imageCount: number }> + >() for (const [index, message] of messages.entries()) { messageIndexById.set(message.id, index) if (message.role !== 'user') { @@ -239,9 +248,10 @@ export function findLandedUnconfirmedSends( // A marker-only turn keys both ways: '' claims it for a caption-less image // send, and its literal text claims it for a send of that same text. const literalKey = isImageSource ? key : (nativeChatUserMessageMatchText(message) ?? key) + const imageCount = nativeChatUserMessageImageEvidenceCount(message) for (const candidateKey of literalKey === key ? [key] : [key, literalKey]) { const current = userMessagesByText.get(candidateKey) ?? [] - current.push({ id: message.id, index }) + current.push({ id: message.id, index, imageCount }) userMessagesByText.set(candidateKey, current) } } @@ -257,7 +267,12 @@ export function findLandedUnconfirmedSends( } const echo = userMessagesByText .get(entry.normalizedText) - ?.find((message) => message.index > tailIndex && !claimedMessageIds.has(message.id)) + ?.find( + (message) => + message.index > tailIndex && + message.imageCount >= entry.imageCount && + !claimedMessageIds.has(message.id) + ) if (echo) { claimedMessageIds.add(echo.id) landed.push(entry) diff --git a/mobile/src/session/mobile-native-chat-image-transcript-markers.ts b/mobile/src/session/mobile-native-chat-image-transcript-markers.ts index 658252d3ff8..6196b6c725c 100644 --- a/mobile/src/session/mobile-native-chat-image-transcript-markers.ts +++ b/mobile/src/session/mobile-native-chat-image-transcript-markers.ts @@ -7,10 +7,11 @@ export { imageSourcePathFromText, hasImagePromptMarker, isImageSourceUserTurn, + nativeChatUserMessageImageEvidenceCount, nativeChatUserMessageMatchText, + nativeChatUserTextMatchText, normalizeImageTranscriptMessages, normalizeNativeChatUserText, - normalizeNativeChatUserTextWithLiteralFallback, normalizedNativeChatUserMessageText, stripImagePromptMarker } from '../../../src/shared/native-chat-image-transcript-markers' diff --git a/mobile/src/session/mobile-native-chat-literal-marker-echo.test.ts b/mobile/src/session/mobile-native-chat-literal-marker-echo.test.ts index c8aef152b3a..77ad99005ba 100644 --- a/mobile/src/session/mobile-native-chat-literal-marker-echo.test.ts +++ b/mobile/src/session/mobile-native-chat-literal-marker-echo.test.ts @@ -3,7 +3,7 @@ // an incomplete multi-image echo is allowed to do to the local previews. import { createElement } from 'react' -import { act, create, type ReactTestRenderer } from 'react-test-renderer' +import { act, create } from 'react-test-renderer' import { afterEach, describe, expect, it } from 'vitest' import { isImageRefBlock, @@ -29,12 +29,14 @@ function userTextMessage(id: string, text: string): NativeChatMessage { } describe('mobile literal image-marker turns', () => { - let renderer: ReactTestRenderer | null = null + let updateRenderer: (messages: NativeChatMessage[]) => void = () => {} + let unmountRenderer = (): void => {} let state: DraftState | null = null afterEach(() => { - act(() => renderer?.unmount()) - renderer = null + act(unmountRenderer) + updateRenderer = () => {} + unmountRenderer = () => {} state = null }) @@ -51,12 +53,14 @@ describe('mobile literal image-marker turns', () => { async function mount(): Promise { await act(async () => { - renderer = create(createElement(Harness, {})) + const renderer = create(createElement(Harness, {})) + updateRenderer = (messages) => renderer.update(createElement(Harness, { messages })) + unmountRenderer = () => renderer.unmount() }) } async function transcript(messages: NativeChatMessage[]): Promise { - await act(async () => renderer?.update(createElement(Harness, { messages }))) + await act(async () => updateRenderer(messages)) } function send(text: string, images?: string[]): void { @@ -158,4 +162,40 @@ describe('mobile literal image-marker turns', () => { expect(rendered.map(rowText)).toEqual(['look']) expect(rendered.map(rowImages)).toEqual([['file:///a.jpg']]) }) + + it('keeps a captioned multi-image echo until the transcript accounts for every image', async () => { + await mount() + send('compare these', ['file:///a.jpg', 'file:///b.jpg', 'file:///c.jpg']) + + const partial = [ + userTextMessage('source-a', '[Image: source: /tmp/a.png]'), + userTextMessage('prompt', 'compare these [Image #1]') + ] + await transcript(partial) + + expect(rows(partial).map(rowImages)).toEqual([ + ['/tmp/a.png'], + ['file:///a.jpg', 'file:///b.jpg', 'file:///c.jpg'] + ]) + expect(rows(partial).map(rowText)).toEqual(['compare these', 'compare these']) + expect(state?.imagePreviewsByMessageId).toEqual({}) + + const complete = [ + userTextMessage('source-a', '[Image: source: /tmp/a.png]'), + userTextMessage('source-b', '[Image: source: /tmp/b.png]'), + userTextMessage('source-c', '[Image: source: /tmp/c.png]'), + userTextMessage('complete-prompt', 'compare these [Image #1] [Image #2] [Image #3]') + ] + await transcript(complete) + + expect(state?.imagePreviewsByMessageId).toEqual({ + 'complete-prompt': ['file:///a.jpg', 'file:///b.jpg', 'file:///c.jpg'] + }) + const completeRows = rows(complete) + expect(completeRows).toHaveLength(1) + expect(completeRows.map(rowText)).toEqual(['compare these']) + expect(completeRows.map(rowImages)).toEqual([ + ['file:///a.jpg', 'file:///b.jpg', 'file:///c.jpg'] + ]) + }) }) diff --git a/mobile/src/session/mobile-native-chat-pending-echo.ts b/mobile/src/session/mobile-native-chat-pending-echo.ts index bcf66250872..b5fb6ab4f90 100644 --- a/mobile/src/session/mobile-native-chat-pending-echo.ts +++ b/mobile/src/session/mobile-native-chat-pending-echo.ts @@ -1,6 +1,6 @@ import { - normalizeNativeChatUserText, - normalizeNativeChatUserTextWithLiteralFallback + nativeChatUserTextMatchText, + normalizeNativeChatUserText } from './mobile-native-chat-image-transcript-markers' export type MobileNativeChatPendingMessage = { @@ -16,6 +16,7 @@ export type MobileNativeChatSendOrigin = { draftKey: string pendingKey: string | null normalizedText: string + imageCount: number baselineOccurrences: number baselineTailMessageId: string | null } @@ -25,9 +26,7 @@ type PendingByKey = Record /** Markers are the agent's only on a send that carried images; otherwise they * are literal text, so `captureSendOrigin` keeps them and so must this. */ function pendingMatchText(pending: MobileNativeChatPendingMessage): string { - return pending.images?.length - ? normalizeNativeChatUserText(pending.text) - : normalizeNativeChatUserTextWithLiteralFallback(pending.text) + return nativeChatUserTextMatchText(pending.text, Boolean(pending.images?.length)) } export function combineMobileNativeChatPending( diff --git a/mobile/src/session/use-mobile-native-chat-drafts.ts b/mobile/src/session/use-mobile-native-chat-drafts.ts index 456ca583ca5..21d70a1a33e 100644 --- a/mobile/src/session/use-mobile-native-chat-drafts.ts +++ b/mobile/src/session/use-mobile-native-chat-drafts.ts @@ -7,8 +7,6 @@ import { findLandedUnconfirmedSends, mergeLandedImagePreviewEchoes, migrateImagePreviewMessageIds, - normalizeReconcileText, - normalizeReconcileTextWithLiteralFallback, userTextOccurrenceCounts, type UnconfirmedSend } from './mobile-native-chat-draft-reconcile' @@ -21,6 +19,7 @@ import { type MobileNativeChatSendOrigin } from './mobile-native-chat-pending-echo' import { mobileNativeChatScopeKey } from './mobile-native-chat-scope-key' +import { nativeChatUserTextMatchText } from './mobile-native-chat-image-transcript-markers' import { useMobileNativeChatLaunchDraftSeed } from './use-mobile-native-chat-launch-draft-seed' import type { MobileNativeChatLaunchDraftSeed } from './use-mobile-native-chat-launch-draft-seed' @@ -135,14 +134,18 @@ export function useMobileNativeChatDrafts(args: { } // Without an attached image an `[Image #n]` run is the user's own text, so // it has to stay in the key the transcript echo will be matched against. - const normalizedText = images?.length - ? normalizeReconcileText(text) - : normalizeReconcileTextWithLiteralFallback(text) + const imageCount = images?.length ?? 0 + const normalizedText = nativeChatUserTextMatchText(text, imageCount > 0) return { draftKey, pendingKey, normalizedText, - baselineOccurrences: countUserTextOccurrences(messagesRef.current, normalizedText), + imageCount, + baselineOccurrences: countUserTextOccurrences( + messagesRef.current, + normalizedText, + imageCount + ), baselineTailMessageId: messagesRef.current.at(-1)?.id ?? null } }, @@ -207,6 +210,7 @@ export function useMobileNativeChatDrafts(args: { pendingKey: origin.pendingKey, text, normalizedText: origin.normalizedText, + imageCount: origin.imageCount, baselineTailMessageId: origin.baselineTailMessageId, deadline: null } @@ -299,17 +303,14 @@ export function useMobileNativeChatDrafts(args: { const landedCounts = userTextOccurrenceCounts(messages) // Image-only source-turn counts stay stable across reruns and ignore paginated history. const next = current.filter((item) => { - if (landedImagePendingIds.has(item.id)) { - return false - } // Keep image echoes until their local preview reaches the authoritative message. if (item.images?.length) { - return true + return !landedImagePendingIds.has(item.id) } return item.text.trim() === '' ? countImageSourceTurnsAfter(messages, item.baselineTailMessageId) < item.expectedOccurrence - : (landedCounts.get(normalizeReconcileTextWithLiteralFallback(item.text)) ?? 0) < + : (landedCounts.get(nativeChatUserTextMatchText(item.text, false)) ?? 0) < item.expectedOccurrence }) if (next.length === current.length) { diff --git a/src/renderer/src/components/native-chat/native-chat-literal-marker-echo.test.ts b/src/renderer/src/components/native-chat/native-chat-literal-marker-echo.test.ts index 65c137f9f0c..d530a3a0059 100644 --- a/src/renderer/src/components/native-chat/native-chat-literal-marker-echo.test.ts +++ b/src/renderer/src/components/native-chat/native-chat-literal-marker-echo.test.ts @@ -124,4 +124,38 @@ describe('literal image-marker turns in native chat', () => { expect(rowText(rows[0]!)).toBe('') expect(rowImagePaths(rows[0]!)).toEqual(['/tmp/a.png']) }) + + it('keeps a captioned multi-image echo until the transcript accounts for every image', () => { + const pending: NativeChatPendingSend[] = [ + { + id: 'p1', + text: 'compare these', + imagePaths: ['/tmp/a.png', '/tmp/b.png', '/tmp/c.png'], + sentAt: 100 + } + ] + const transcript = [ + userMessage('source-a', '[Image: source: /tmp/a.png]'), + userMessage('prompt', 'compare these [Image #1]') + ] + + const rows = renderedUserRows(transcript, pending) + + expect(rows.map(rowText)).toEqual(['compare these', 'compare these']) + expect(rows.map(rowImagePaths)).toEqual([ + ['/tmp/a.png'], + ['/tmp/a.png', '/tmp/b.png', '/tmp/c.png'] + ]) + + const complete = [ + userMessage('source-a', '[Image: source: /tmp/a.png]'), + userMessage('source-b', '[Image: source: /tmp/b.png]'), + userMessage('source-c', '[Image: source: /tmp/c.png]'), + userMessage('complete-prompt', 'compare these [Image #1] [Image #2] [Image #3]') + ] + const completeRows = renderedUserRows(complete, pending) + expect(completeRows).toHaveLength(1) + expect(rowText(completeRows[0]!)).toBe('compare these') + expect(rowImagePaths(completeRows[0]!)).toEqual(['/tmp/a.png', '/tmp/b.png', '/tmp/c.png']) + }) }) diff --git a/src/renderer/src/components/native-chat/native-chat-pending-occurrence.ts b/src/renderer/src/components/native-chat/native-chat-pending-occurrence.ts index b455be57dc3..a3dd2706406 100644 --- a/src/renderer/src/components/native-chat/native-chat-pending-occurrence.ts +++ b/src/renderer/src/components/native-chat/native-chat-pending-occurrence.ts @@ -1,7 +1,9 @@ import { + nativeChatUserMessageImageEvidenceCount, nativeChatUserMessageMatchText, + nativeChatUserTextMatchText, normalizeNativeChatUserText, - normalizeNativeChatUserTextWithLiteralFallback + normalizedNativeChatUserMessageText } from '../../../../shared/native-chat-image-transcript-markers' import { isImageRefBlock, type NativeChatMessage } from '../../../../shared/native-chat-types' @@ -25,36 +27,47 @@ export function normalizeNativeChatPendingText(text: string): string { function nativeChatPendingMatchText( pending: Pick ): string { - return pending.imagePaths?.some(Boolean) - ? normalizeNativeChatPendingText(pending.text) - : normalizeNativeChatUserTextWithLiteralFallback(pending.text) + return nativeChatUserTextMatchText(pending.text, Boolean(pending.imagePaths?.some(Boolean))) +} + +function nativeChatTextContentKey(text: string, imageCount: number): string { + return imageCount > 0 ? `text:${text}\0images:${imageCount}` : `text:${text}` } export function nativeChatPendingContentKey( pending: Pick ): string { const text = nativeChatPendingMatchText(pending) - if (text) { - return `text:${text}` - } const imagePaths = pending.imagePaths?.filter(Boolean) ?? [] + if (text) { + return nativeChatTextContentKey(text, imagePaths.length) + } return imagePaths.length > 0 ? `images:${JSON.stringify(imagePaths)}` : 'empty' } -function nativeChatUserMessageContentKey(message: NativeChatMessage): string | null { +function nativeChatUserMessageContentKeys(message: NativeChatMessage): readonly string[] { if (message.role !== 'user') { - return null + return [] } - const text = nativeChatUserMessageMatchText(message) ?? '' - if (text) { - return `text:${text}` + const keys = new Set() + const matchText = nativeChatUserMessageMatchText(message) ?? '' + if (matchText) { + keys.add(nativeChatTextContentKey(matchText, 0)) } - const imagePaths = message.blocks - .filter(isImageRefBlock) - .map((block) => block.path) - .filter((path): path is string => Boolean(path)) - const key = nativeChatPendingContentKey({ text: '', imagePaths }) - return key === 'empty' ? null : key + const imageCount = nativeChatUserMessageImageEvidenceCount(message) + const imageText = normalizedNativeChatUserMessageText(message) + if (imageText && imageCount > 0) { + keys.add(nativeChatTextContentKey(imageText, imageCount)) + } else if (imageCount > 0) { + const imagePaths = message.blocks + .filter(isImageRefBlock) + .map((block) => block.path) + .filter((path): path is string => Boolean(path)) + if (imagePaths.length > 0) { + keys.add(`images:${JSON.stringify(imagePaths)}`) + } + } + return [...keys] } export function matchingNativeChatUserContentCounts( @@ -62,8 +75,7 @@ export function matchingNativeChatUserContentCounts( ): Map { const counts = new Map() for (const message of messages) { - const key = nativeChatUserMessageContentKey(message) - if (key) { + for (const key of nativeChatUserMessageContentKeys(message)) { counts.set(key, (counts.get(key) ?? 0) + 1) } } @@ -77,8 +89,7 @@ export function advancedNativeChatUserContentCounts( const waiting = new Map() for (const message of messages) { if (message.role === 'user') { - const key = nativeChatUserMessageContentKey(message) - if (key) { + for (const key of nativeChatUserMessageContentKeys(message)) { waiting.set(key, (waiting.get(key) ?? 0) + 1) } continue @@ -180,10 +191,13 @@ export function selectPendingIndicesRepresentedByUserTexts( } const remaining = pending.map((entry, index) => ({ index, - text: nativeChatPendingMatchText(entry) + text: nativeChatPendingMatchText(entry), + hasImages: Boolean(entry.imagePaths?.some(Boolean)) })) for (const userText of userTexts) { - const open = remaining.filter((entry) => !represented.has(entry.index) && entry.text.length > 0) + const open = remaining.filter( + (entry) => !entry.hasImages && !represented.has(entry.index) && entry.text.length > 0 + ) const gluedCount = countLeadingPendingTextsGluedToUserText( open.map((entry) => entry.text), userText diff --git a/src/shared/native-chat-image-transcript-markers.ts b/src/shared/native-chat-image-transcript-markers.ts index 9c28e52cc1b..9a0621a41f6 100644 --- a/src/shared/native-chat-image-transcript-markers.ts +++ b/src/shared/native-chat-image-transcript-markers.ts @@ -48,10 +48,16 @@ export function normalizeNativeChatUserText(text: string): string { /** Marker text that stripping would erase entirely is text the user typed * literally — a real marker always rides on an image turn, which normalization * has already folded away by the time echo matching runs. */ -export function normalizeNativeChatUserTextWithLiteralFallback(text: string): string { +function normalizeNativeChatUserTextWithLiteralFallback(text: string): string { return normalizeNativeChatUserText(text) || text.trim().replace(/\s+/g, ' ') } +export function nativeChatUserTextMatchText(text: string, hasImages: boolean): string { + return hasImages + ? normalizeNativeChatUserText(text) + : normalizeNativeChatUserTextWithLiteralFallback(text) +} + function joinedUserText(message: NativeChatMessage): string { return message.blocks .filter(isTextBlock) @@ -74,11 +80,7 @@ export function nativeChatUserMessageMatchText(message: NativeChatMessage): stri return null } const joined = joinedUserText(message) - const normalized = normalizeNativeChatUserText(joined) - if (normalized || message.blocks.some(isImageRefBlock)) { - return normalized || null - } - return normalizeNativeChatUserTextWithLiteralFallback(joined) || null + return nativeChatUserTextMatchText(joined, message.blocks.some(isImageRefBlock)) || null } function stripImagePromptMarkersFromTextBlocks( @@ -132,6 +134,14 @@ export function countImagePromptMarkers(message: NativeChatMessage): number { ) } +export function nativeChatUserMessageImageEvidenceCount(message: NativeChatMessage): number { + if (message.role !== 'user') { + return 0 + } + const imageRefCount = message.blocks.filter(isImageRefBlock).length + return Math.max(imageRefCount, countImagePromptMarkers(message)) +} + /** Claude records image paths as source turns followed by a prompt carrying * image markers. Merge the whole run back into one native user turn. */ export function normalizeImageTranscriptMessages(