diff --git a/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts b/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts index 8dfce5ebbfa..25255d5743c 100644 --- a/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts +++ b/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import type { NativeChatMessage } from '../../../src/shared/native-chat-types' import { + countUserTextOccurrences, findLandedImagePreviewEchoes, findLandedUnconfirmedSends, migrateImagePreviewMessageIds, @@ -89,6 +90,35 @@ describe('mobile native chat image preview reconciliation', () => { ).toEqual([unconfirmed]) }) + it('does not treat literal marker text as a markerless unknown-ack echo', () => { + const unconfirmed: UnconfirmedSend = { + draftKey: 'draft', + pendingKey: 'pending-key', + text: 'keep literal', + normalizedText: 'keep literal', + imageCount: 0, + baselineTailMessageId: null, + deadline: null + } + + expect( + findLandedUnconfirmedSends([userText('wrong', 'keep [Image #1] literal')], [unconfirmed]) + ).toEqual([]) + expect(findLandedUnconfirmedSends([userText('right', 'keep literal')], [unconfirmed])).toEqual([ + unconfirmed + ]) + }) + + it('keeps no-image occurrence baselines separate from attached-image turns', () => { + const messages = [ + userText('source', '[Image: source: /tmp/a.png]'), + userText('prompt', 'keep [Image #1] literal') + ] + + expect(countUserTextOccurrences(messages, 'keep literal')).toBe(0) + expect(countUserTextOccurrences(messages, 'keep literal', 1)).toBe(1) + }) + it('reconciles multiple transcript text blocks with desktop separators', () => { const prompt: NativeChatMessage = { ...userText('prompt', 'unused'), @@ -114,6 +144,15 @@ describe('mobile native chat image preview reconciliation', () => { ).toEqual([{ pendingId: 'pending', messageId: 'prompt', images: ['file:///a.jpg'] }]) }) + it('hands a captioned preview to a marker-only compatibility echo', () => { + expect( + findLandedImagePreviewEchoes( + [userText('prompt', 'look [Image #1] here')], + [{ ...pending('pending', ['file:///a.jpg']), text: 'look here' }] + ) + ).toEqual([{ pendingId: 'pending', messageId: 'prompt', images: ['file:///a.jpg'] }]) + }) + it('keeps separate adjacent image-only sends independently reconcilable', () => { const landed = findLandedImagePreviewEchoes( [ diff --git a/mobile/src/session/mobile-native-chat-draft-reconcile.ts b/mobile/src/session/mobile-native-chat-draft-reconcile.ts index 19f51d5b095..9e10e6355f2 100644 --- a/mobile/src/session/mobile-native-chat-draft-reconcile.ts +++ b/mobile/src/session/mobile-native-chat-draft-reconcile.ts @@ -34,8 +34,14 @@ export function countUserTextOccurrences( ): number { let count = 0 for (const message of normalizeImageTranscriptMessages(messages)) { + const hasImageRefs = message.blocks.some(isImageRefBlock) + const matchText = + imageCount > 0 + ? normalizedNativeChatUserMessageText(message) + : nativeChatUserMessageMatchText(message) if ( - nativeChatUserMessageMatchText(message) === text && + matchText === text && + (imageCount > 0 || !hasImageRefs) && nativeChatUserMessageImageEvidenceCount(message) >= imageCount ) { count++ @@ -176,7 +182,7 @@ export function findLandedImagePreviewEchoes( const imageCount = nativeChatUserMessageImageEvidenceCount(message) if (targetText) { return ( - nativeChatUserMessageMatchText(message) === targetText && + normalizedNativeChatUserMessageText(message) === targetText && imageCount >= entry.images!.length ) } @@ -222,7 +228,11 @@ export function findLandedUnconfirmedSends( // (`[Image: source: …]` or no text) keys under '' so an empty-text send can // claim it. const messageIndexById = new Map(messages.map((message, index) => [message.id, index])) - const userMessagesByText = new Map() + const literalMessagesByText = new Map< + string, + { id: string; index: number; imageCount: number }[] + >() + const imageMessagesByText = new Map() for (const message of normalizeImageTranscriptMessages(messages)) { const index = messageIndexById.get(message.id) if (index === undefined) { @@ -232,15 +242,19 @@ export function findLandedUnconfirmedSends( continue } const isImageSource = isImageSourceUserTurn(message) - const key = isImageSource ? '' : (normalizedUserText(message) ?? '') - // A marker-only turn keys both ways: '' claims it for a caption-less image - // send, and its literal text claims it for a send of that same text. - const literalKey = isImageSource ? key : (nativeChatUserMessageMatchText(message) ?? key) const imageCount = nativeChatUserMessageImageEvidenceCount(message) - for (const candidateKey of literalKey === key ? [key] : [key, literalKey]) { - const current = userMessagesByText.get(candidateKey) ?? [] - current.push({ id: message.id, index, imageCount }) - userMessagesByText.set(candidateKey, current) + const candidate = { id: message.id, index, imageCount } + if (!message.blocks.some(isImageRefBlock)) { + const literalKey = isImageSource ? '' : (nativeChatUserMessageMatchText(message) ?? '') + const current = literalMessagesByText.get(literalKey) ?? [] + current.push(candidate) + literalMessagesByText.set(literalKey, current) + } + if (imageCount > 0) { + const imageKey = isImageSource ? '' : (normalizedUserText(message) ?? '') + const current = imageMessagesByText.get(imageKey) ?? [] + current.push(candidate) + imageMessagesByText.set(imageKey, current) } } @@ -253,7 +267,8 @@ export function findLandedUnconfirmedSends( if (tailIndex === undefined) { continue } - const echo = userMessagesByText + const messagesByText = entry.imageCount > 0 ? imageMessagesByText : literalMessagesByText + const echo = messagesByText .get(entry.normalizedText) ?.find( (message) => diff --git a/mobile/src/session/mobile-native-chat-literal-marker-echo.test.ts b/mobile/src/session/mobile-native-chat-literal-marker-echo.test.ts index f7d491aef2f..18cb6d2a1a9 100644 --- a/mobile/src/session/mobile-native-chat-literal-marker-echo.test.ts +++ b/mobile/src/session/mobile-native-chat-literal-marker-echo.test.ts @@ -130,6 +130,18 @@ describe('mobile literal image-marker turns', () => { expect(rows(exact).map(rowText)).toEqual(['keep [Image #1] literal']) }) + it('does not let an attached-image send shift a markerless send ordinal', async () => { + await mount() + send('keep [Image #1] literal', ['file:///a.jpg']) + send('keep literal') + + const messages = [userTextMessage('plain', 'keep literal')] + await transcript(messages) + + expect(rows(messages).map(rowText)).toEqual(['keep literal', 'keep [Image #1] literal']) + expect(rows(messages).map(rowImages)).toEqual([[], ['file:///a.jpg']]) + }) + it('holds every preview on the optimistic echo until the marker echo covers them all', async () => { await mount() send('', ['file:///a.jpg', 'file:///b.jpg']) diff --git a/mobile/src/session/mobile-native-chat-pending-echo.ts b/mobile/src/session/mobile-native-chat-pending-echo.ts index 0a02d95c9cb..2766030ca4b 100644 --- a/mobile/src/session/mobile-native-chat-pending-echo.ts +++ b/mobile/src/session/mobile-native-chat-pending-echo.ts @@ -55,7 +55,9 @@ export function appendMobileNativeChatPending( const earlierOutstanding = current.filter( (pending) => pendingMatchText(pending) === origin.normalizedText && - (pending.images?.length ?? 0) >= origin.imageCount && + (origin.imageCount > 0 + ? (pending.images?.length ?? 0) >= origin.imageCount + : !pending.images?.length) && pending.expectedOccurrence > origin.baselineOccurrences ).length // An image send whose caption is only markers still reconciles by image @@ -63,6 +65,7 @@ export function appendMobileNativeChatPending( const expectedImageEchoOrdinal = current.filter( (pending) => + origin.imageCount > 0 && (pending.images?.length ?? 0) >= origin.imageCount && normalizeNativeChatUserText(pending.text) === '' ).length + 1 diff --git a/mobile/src/session/mobile-native-chat-pending-retirement.test.ts b/mobile/src/session/mobile-native-chat-pending-retirement.test.ts index 623f475f578..32231098921 100644 --- a/mobile/src/session/mobile-native-chat-pending-retirement.test.ts +++ b/mobile/src/session/mobile-native-chat-pending-retirement.test.ts @@ -282,4 +282,28 @@ describe('retireLandedMobileNativeChatPending', () => { ).toEqual(['p1']) expect(retireLandedMobileNativeChatPending(messages, pending, new Set(['p1']))).toEqual([]) }) + + it('keeps literal and attached-image turns in separate retirement lanes', () => { + const markerPending = [pendingSend('marker', 'keep [Image #1] literal', null)] + const plainPending = [pendingSend('plain', 'keep literal', null)] + const literalMarker = [userTurn('literal', 'keep [Image #1] literal', 1000)] + const attachedImage = [ + userTurn('source', '[Image: source: /tmp/a.png]', 1000), + userTurn('prompt', 'keep [Image #1] literal', 1001) + ] + + expect( + retireLandedMobileNativeChatPending(literalMarker, plainPending, NO_IMAGE_ECHOES).map( + (item) => item.id + ) + ).toEqual(['plain']) + expect( + retireLandedMobileNativeChatPending(literalMarker, markerPending, NO_IMAGE_ECHOES) + ).toEqual([]) + expect( + retireLandedMobileNativeChatPending(attachedImage, plainPending, NO_IMAGE_ECHOES).map( + (item) => item.id + ) + ).toEqual(['plain']) + }) }) diff --git a/mobile/src/session/mobile-native-chat-pending-retirement.ts b/mobile/src/session/mobile-native-chat-pending-retirement.ts index afa5563a9b4..ba1f2669c39 100644 --- a/mobile/src/session/mobile-native-chat-pending-retirement.ts +++ b/mobile/src/session/mobile-native-chat-pending-retirement.ts @@ -1,4 +1,4 @@ -import type { NativeChatMessage } from '../../../src/shared/native-chat-types' +import { isImageRefBlock, type NativeChatMessage } from '../../../src/shared/native-chat-types' import { countImageSourceTurnsAfter } from './mobile-native-chat-draft-reconcile' import { nativeChatUserMessageMatchText, @@ -33,7 +33,9 @@ export function selectGluedPendingIds( if (index === undefined) { continue } - const text = nativeChatUserMessageMatchText(message) + const text = message.blocks.some(isImageRefBlock) + ? null + : nativeChatUserMessageMatchText(message) if (text) { turns.push({ index, text }) } @@ -120,6 +122,9 @@ export function retireLandedMobileNativeChatPending( ): MobileNativeChatPendingMessage[] { const landedCounts = new Map() for (const message of normalizeImageTranscriptMessages(messages)) { + if (message.blocks.some(isImageRefBlock)) { + continue + } const text = nativeChatUserMessageMatchText(message) if (text) { landedCounts.set(text, (landedCounts.get(text) ?? 0) + 1) diff --git a/mobile/src/session/use-mobile-native-chat-drafts-launch-draft.test.ts b/mobile/src/session/use-mobile-native-chat-drafts-launch-draft.test.ts index 93f03b9ea0c..f709bcba451 100644 --- a/mobile/src/session/use-mobile-native-chat-drafts-launch-draft.test.ts +++ b/mobile/src/session/use-mobile-native-chat-drafts-launch-draft.test.ts @@ -129,7 +129,7 @@ describe('useMobileNativeChatDrafts launch draft', () => { renderer?.update( createElement(Harness, { tabId: 'a', - messages: [userTextMessage('m1', '[Image #1]')], + messages: [userTextMessage('m1', 'keep [Image #1] literal')], launchDraft: 'issue link' }) ) @@ -149,7 +149,7 @@ describe('useMobileNativeChatDrafts launch draft', () => { renderer?.update( createElement(Harness, { tabId: 'a', - messages: [userTextMessage('m1', '[Image #1]')], + messages: [userTextMessage('m1', 'keep [Image #1] literal')], launchDraft: 'issue link' }) ) diff --git a/mobile/src/session/use-mobile-native-chat-drafts.test.ts b/mobile/src/session/use-mobile-native-chat-drafts.test.ts index daf451e53f9..3a5ae07351a 100644 --- a/mobile/src/session/use-mobile-native-chat-drafts.test.ts +++ b/mobile/src/session/use-mobile-native-chat-drafts.test.ts @@ -219,7 +219,7 @@ describe('useMobileNativeChatDrafts', () => { createElement(Harness, { tabId: 'a', messages: [assistantTextMessage('a1', 'hi')] }) ) ) - const origin = state?.captureSendOrigin('') + const origin = state?.captureSendOrigin('', ['file:///a.jpg']) act(() => { if (origin) { state?.acceptSend(origin, '', ['file:///a.jpg']) @@ -265,7 +265,7 @@ describe('useMobileNativeChatDrafts', () => { ) ) const textOrigin = state?.captureSendOrigin('ping') - const imageOrigin = state?.captureSendOrigin('') + const imageOrigin = state?.captureSendOrigin('', ['file:///a.jpg']) act(() => { if (textOrigin && imageOrigin) { state?.acceptSend(textOrigin, 'ping') @@ -308,7 +308,7 @@ describe('useMobileNativeChatDrafts', () => { createElement(Harness, { tabId: 'a', messages: [assistantTextMessage('a1', 'hi')] }) ) ) - const origin = state?.captureSendOrigin('look at this') + const origin = state?.captureSendOrigin('look at this', ['file:///a.jpg']) act(() => { if (origin) { state?.acceptSend(origin, 'look at this', ['file:///a.jpg']) @@ -341,7 +341,7 @@ describe('useMobileNativeChatDrafts', () => { createElement(Harness, { tabId: 'a', messages: [assistantTextMessage('a1', 'hi')] }) ) ) - const origin = state?.captureSendOrigin('look at this') + const origin = state?.captureSendOrigin('look at this', ['file:///a.jpg']) act(() => { if (origin) { state?.acceptSend(origin, 'look at this', ['file:///a.jpg']) @@ -367,7 +367,7 @@ describe('useMobileNativeChatDrafts', () => { it('hands a marker-only image preview to the authoritative user bubble', async () => { await mount('a') - const origin = state?.captureSendOrigin('') + const origin = state?.captureSendOrigin('', ['file:///a.jpg']) act(() => { if (origin) { state?.acceptSend(origin, '', ['file:///a.jpg']) @@ -475,7 +475,7 @@ describe('useMobileNativeChatDrafts', () => { ) ) // Image-only send: empty text, so it can only reconcile against a new user turn. - const origin = state?.captureSendOrigin('') + const origin = state?.captureSendOrigin('', ['file:///a.jpg']) const onUnconfirmed = vi.fn() act(() => { if (origin) { @@ -492,7 +492,7 @@ describe('useMobileNativeChatDrafts', () => { }) ) ) - // ...but the user's own turn landing does, so the deadline never warns. + // ...but the user's own image turn landing does, so the deadline never warns. await act(async () => renderer?.update( createElement(Harness, { @@ -500,7 +500,7 @@ describe('useMobileNativeChatDrafts', () => { messages: [ assistantTextMessage('a1', 'hi'), assistantTextMessage('a2', 'ok'), - userTextMessage('u1', '') + userTextMessage('u1', '[Image #1]') ] }) ) @@ -519,7 +519,7 @@ describe('useMobileNativeChatDrafts', () => { createElement(Harness, { tabId: 'a', messages: [assistantTextMessage('a1', 'hi')] }) ) ) - const origin = state?.captureSendOrigin('') + const origin = state?.captureSendOrigin('', ['file:///a.jpg']) act(() => { if (origin) { state?.acceptSend(origin, '', ['file:///a.jpg']) @@ -768,7 +768,7 @@ describe('useMobileNativeChatDrafts', () => { const images = ['file:///a.jpg', 'file:///b.jpg', 'file:///c.jpg'] act(() => state?.setComposerText('look')) - const origin = state?.captureSendOrigin('look') + const origin = state?.captureSendOrigin('look', images) expect(origin).toMatchObject({ pendingKey: null }) act(() => { if (origin) { diff --git a/src/renderer/src/components/native-chat/native-chat-literal-marker-echo.test.ts b/src/renderer/src/components/native-chat/native-chat-literal-marker-echo.test.ts index d2f4f5eef2d..c36aabd2dfd 100644 --- a/src/renderer/src/components/native-chat/native-chat-literal-marker-echo.test.ts +++ b/src/renderer/src/components/native-chat/native-chat-literal-marker-echo.test.ts @@ -88,6 +88,27 @@ describe('literal image-marker turns in native chat', () => { ]) }) + it('does not let literal or attached-image marker rows hide markerless text', () => { + const pending: NativeChatPendingSend[] = [{ id: 'p1', text: 'keep literal', sentAt: 100 }] + const literalTranscript = [userMessage('literal', 'keep [Image #1] literal')] + const imageTranscript = [ + userMessage('source', '[Image: source: /tmp/a.png]'), + userMessage('prompt', 'keep[Image #1] literal') + ] + + expect(renderedUserRows(literalTranscript, pending).map(rowText)).toEqual([ + 'keep [Image #1] literal', + 'keep literal' + ]) + expect(renderedUserRows(imageTranscript, pending).map(rowText)).toEqual([ + 'keep literal', + 'keep literal' + ]) + expect( + prunePendingSends(pending, assembled([...imageTranscript, assistantMessage('a1', 'done')])) + ).toEqual(pending) + }) + it('prunes the marker-only echo once the assistant answers it', () => { const pending: NativeChatPendingSend[] = [{ id: 'p1', text: '[Image #1]', sentAt: 100 }] const transcript = [userMessage('m1', '[Image #1]'), assistantMessage('m2', 'preserved')] diff --git a/src/renderer/src/components/native-chat/native-chat-pending-occurrence.ts b/src/renderer/src/components/native-chat/native-chat-pending-occurrence.ts index a3dd2706406..a770df39cf5 100644 --- a/src/renderer/src/components/native-chat/native-chat-pending-occurrence.ts +++ b/src/renderer/src/components/native-chat/native-chat-pending-occurrence.ts @@ -51,7 +51,8 @@ function nativeChatUserMessageContentKeys(message: NativeChatMessage): readonly } const keys = new Set() const matchText = nativeChatUserMessageMatchText(message) ?? '' - if (matchText) { + const hasImageRefs = message.blocks.some(isImageRefBlock) + if (matchText && !hasImageRefs) { keys.add(nativeChatTextContentKey(matchText, 0)) } const imageCount = nativeChatUserMessageImageEvidenceCount(message) @@ -110,7 +111,9 @@ export function advancedNativeChatUserTexts( const waiting: string[] = [] for (const message of messages) { if (message.role === 'user') { - const text = nativeChatUserMessageMatchText(message) + const text = message.blocks.some(isImageRefBlock) + ? null + : nativeChatUserMessageMatchText(message) if (text) { waiting.push(text) } @@ -128,7 +131,9 @@ export function matchingNativeChatUserTexts( ): readonly string[] { const texts: string[] = [] for (const message of messages) { - const text = nativeChatUserMessageMatchText(message) + const text = message.blocks.some(isImageRefBlock) + ? null + : nativeChatUserMessageMatchText(message) if (text) { texts.push(text) } diff --git a/src/renderer/src/components/native-chat/native-chat-pending.test.ts b/src/renderer/src/components/native-chat/native-chat-pending.test.ts index 29771fd9507..38559f6e05f 100644 --- a/src/renderer/src/components/native-chat/native-chat-pending.test.ts +++ b/src/renderer/src/components/native-chat/native-chat-pending.test.ts @@ -116,7 +116,7 @@ describe('prunePendingSends', () => { ] } const next = prunePendingSends( - [pendingOf('p1', 'what do you see')], + [{ ...pendingOf('p1', 'what do you see'), imagePaths: ['/tmp/a.png'] }], [prompt, assistantMessage('m2', 'an image')] ) @@ -468,7 +468,7 @@ describe('launchPromptAsMessage', () => { ).toBeNull() }) - it('uses pending-send normalization for large multiline generated prompts', () => { + it('does not strip literal markers from launch-prompt identity', () => { const prompt = [ '[Image #1] Resolve the failing checks:', '', @@ -488,17 +488,17 @@ describe('launchPromptAsMessage', () => { { ...assistantMessage('a1', 'I will fix it'), timestamp: 44 } ] - expect( - shouldPruneLaunchPrompt( - { - tabId: 'tab-1', - agent: 'codex', - text: prompt, - createdAt: 42 - }, - transcript - ) - ).toBe(true) + const entry = { tabId: 'tab-1', agent: 'codex' as const, text: prompt, createdAt: 42 } + + expect(launchPromptAsMessage(entry, transcript)).not.toBeNull() + expect(shouldPruneLaunchPrompt(entry, transcript)).toBe(false) + + const exact = [ + { ...userMessage('u1', prompt), timestamp: 43 }, + { ...assistantMessage('a1', 'I will fix it'), timestamp: 44 } + ] + expect(launchPromptAsMessage(entry, exact)).toBeNull() + expect(shouldPruneLaunchPrompt(entry, exact)).toBe(true) }) it('keeps the launch prompt until the transcript advances past the user turn', () => { diff --git a/src/shared/native-chat-image-transcript-markers.test.ts b/src/shared/native-chat-image-transcript-markers.test.ts index 6a1ebaa6e6b..859aaf6ab43 100644 --- a/src/shared/native-chat-image-transcript-markers.test.ts +++ b/src/shared/native-chat-image-transcript-markers.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import type { NativeChatMessage } from './native-chat-types' import { isImageSourceUserTurn, + nativeChatUserMessageMatchText, nativeChatUserTextMatchText, normalizeImageTranscriptMessages, normalizeNativeChatUserText, @@ -110,6 +111,14 @@ describe('normalizeImageTranscriptMessages', () => { 'keep [Image #1] literal' ) expect(nativeChatUserTextMatchText(' keep [Image #1] literal ', true)).toBe('keep literal') + + const literal = userText('literal', 'keep [Image #1] literal') + const attached = { + ...literal, + blocks: [{ type: 'image-ref' as const, path: '/tmp/a.png' }, ...literal.blocks] + } + expect(nativeChatUserMessageMatchText(literal)).toBe('keep [Image #1] literal') + expect(nativeChatUserMessageMatchText(attached)).toBe('keep literal') }) it('recognizes only sole-text image-source user turns', () => {