fix(native-chat): key optimistic echoes on literal image markers

The render path became provenance-aware for `[Image #n]` markers, but the
pending-key path kept stripping them unconditionally, so the two disagreed:
a standalone marker-only turn rendered its literal text while its content key
computed to empty, leaving the optimistic echo unmatchable and the row
duplicated on desktop and mobile.

Give both sides the same rule via `nativeChatUserMessageMatchText` — a marker
run that stripping would erase, on a turn carrying no image, is the user's own
text and keys verbatim. Mobile's preview reconciliation also accepted a single
marker as proof for any number of pending images, so an incomplete multi-image
echo could clear or rebind every preview; it now requires one marker per image.
This commit is contained in:
Brennan Benson
2026-08-14 16:47:07 -07:00
parent fab320e424
commit 7803de9c1b
9 changed files with 422 additions and 37 deletions
@@ -1,12 +1,17 @@
import { isImageRefBlock, type NativeChatMessage } from '../../../src/shared/native-chat-types'
import {
countImagePromptMarkers,
hasImagePromptMarker,
isImageSourceUserTurn,
nativeChatUserMessageMatchText,
normalizeImageTranscriptMessages,
normalizeNativeChatUserText,
normalizedNativeChatUserMessageText
} from './mobile-native-chat-image-transcript-markers'
export { normalizeNativeChatUserText as normalizeReconcileText } from './mobile-native-chat-image-transcript-markers'
export {
normalizeNativeChatUserText as normalizeReconcileText,
normalizeNativeChatUserTextWithLiteralFallback as normalizeReconcileTextWithLiteralFallback
} from './mobile-native-chat-image-transcript-markers'
/** An ack-lost ('unknown' outcome) send held until its transcript echo lands or
* the deadline surfaces the uncertainty. */
@@ -29,13 +34,26 @@ export function countUserTextOccurrences(
): number {
let count = 0
for (const message of messages) {
if (normalizedUserText(message) === text) {
if (nativeChatUserMessageMatchText(message) === text) {
count++
}
}
return count
}
export function userTextOccurrenceCounts(
messages: readonly NativeChatMessage[]
): Map<string, number> {
const counts = new Map<string, number>()
for (const message of messages) {
const text = nativeChatUserMessageMatchText(message)
if (text) {
counts.set(text, (counts.get(text) ?? 0) + 1)
}
}
return counts
}
/** Number of `[Image: source: …]` echo turns strictly after `tailId` (or the
* whole transcript when the tail was paginated out). An image-only send has no
* caption to match, so it reconciles by ordinal against this count — counting
@@ -169,10 +187,13 @@ export function findLandedImagePreviewEchoes(
return normalizedUserText(message) === targetText
}
const imageCount = message.blocks.filter(isImageRefBlock).length
// Why: a marker-only echo proves delivery per marker, so a partially
// rendered turn must not claim (and rebind) previews it cannot account for.
return (
message.blocks.length === 0 ||
imageCount >= entry.images!.length ||
(hasImagePromptMarker(message) && normalizedUserText(message) === null)
(countImagePromptMarkers(message) >= entry.images!.length &&
normalizedUserText(message) === null)
)
})
const tailIndex = entry.baselineTailMessageId
@@ -213,10 +234,16 @@ export function findLandedUnconfirmedSends(
if (message.role !== 'user') {
continue
}
const key = isImageSourceUserTurn(message) ? '' : (normalizedUserText(message) ?? '')
const current = userMessagesByText.get(key) ?? []
current.push({ id: message.id, index })
userMessagesByText.set(key, current)
const isImageSource = isImageSourceUserTurn(message)
const key = isImageSource ? '' : (normalizedUserText(message) ?? '')
// A marker-only turn keys both ways: '' claims it for a caption-less image
// send, and its literal text claims it for a send of that same text.
const literalKey = isImageSource ? key : (nativeChatUserMessageMatchText(message) ?? key)
for (const candidateKey of literalKey === key ? [key] : [key, literalKey]) {
const current = userMessagesByText.get(candidateKey) ?? []
current.push({ id: message.id, index })
userMessagesByText.set(candidateKey, current)
}
}
const claimedMessageIds = new Set<string>()
@@ -3,11 +3,14 @@
// on the caption turn), and both render and echo reconciliation must
// agree with desktop on how those marker turns are interpreted.
export {
countImagePromptMarkers,
imageSourcePathFromText,
hasImagePromptMarker,
isImageSourceUserTurn,
nativeChatUserMessageMatchText,
normalizeImageTranscriptMessages,
normalizeNativeChatUserText,
normalizeNativeChatUserTextWithLiteralFallback,
normalizedNativeChatUserMessageText,
stripImagePromptMarker
} from '../../../src/shared/native-chat-image-transcript-markers'
@@ -0,0 +1,161 @@
// Outcome-level guard for STA-4363 on mobile: what the chat list ends up
// showing after a send whose text is nothing but `[Image #n]` markers, and what
// an incomplete multi-image echo is allowed to do to the local previews.
import { createElement } from 'react'
import { act, create, type ReactTestRenderer } from 'react-test-renderer'
import { afterEach, describe, expect, it } from 'vitest'
import {
isImageRefBlock,
isTextBlock,
type NativeChatMessage
} from '../../../src/shared/native-chat-types'
import {
buildMobileNativeChatTransientData,
foldMobileNativeChatMessages
} from './mobile-native-chat-render-data'
import { useMobileNativeChatDrafts } from './use-mobile-native-chat-drafts'
type DraftState = ReturnType<typeof useMobileNativeChatDrafts>
function userTextMessage(id: string, text: string): NativeChatMessage {
return {
id,
role: 'user',
blocks: [{ type: 'text', text }],
timestamp: null,
source: 'transcript'
}
}
describe('mobile literal image-marker turns', () => {
let renderer: ReactTestRenderer | null = null
let state: DraftState | null = null
afterEach(() => {
act(() => renderer?.unmount())
renderer = null
state = null
})
function Harness({ messages = [] }: { messages?: NativeChatMessage[] }): null {
state = useMobileNativeChatDrafts({
hostId: 'host',
worktreeId: 'worktree',
tabId: 'a',
sessionId: 'session-a',
messages
})
return null
}
async function mount(): Promise<void> {
await act(async () => {
renderer = create(createElement(Harness, {}))
})
}
async function transcript(messages: NativeChatMessage[]): Promise<void> {
await act(async () => renderer?.update(createElement(Harness, { messages })))
}
function send(text: string, images?: string[]): void {
const origin = state?.captureSendOrigin(text, images)
act(() => {
if (origin) {
state?.acceptSend(origin, text, images)
}
})
}
/** The rows the chat list renders: folded transcript (with any rebound local
* previews) followed by the still-unreconciled optimistic echoes. */
function rows(messages: NativeChatMessage[]): NativeChatMessage[] {
return buildMobileNativeChatTransientData({
folded: foldMobileNativeChatMessages(messages),
streaming: null,
pending: state?.pending ?? [],
imagePreviewsByMessageId: state?.imagePreviewsByMessageId
}).data
}
function rowText(message: NativeChatMessage): string {
return message.blocks
.filter(isTextBlock)
.map((block) => block.text)
.join('')
}
function rowImages(message: NativeChatMessage): string[] {
return message.blocks
.filter(isImageRefBlock)
.map((block) => block.url ?? block.path ?? '')
.filter(Boolean)
}
it('shows a standalone marker-only send exactly once', async () => {
await mount()
send('[Image #1]')
const messages = [userTextMessage('u1', '[Image #1]')]
await transcript(messages)
expect(rows(messages).map(rowText)).toEqual(['[Image #1]'])
})
it('shows a marker-only send with surrounding prose exactly once', async () => {
await mount()
send('keep [Image #1] literal')
const messages = [userTextMessage('u1', 'keep [Image #1] literal')]
await transcript(messages)
expect(rows(messages).map(rowText)).toEqual(['keep [Image #1] literal'])
})
it('holds every preview on the optimistic echo until the marker echo covers them all', async () => {
await mount()
send('', ['file:///a.jpg', 'file:///b.jpg'])
const partial = [userTextMessage('prompt', '[Image #1]')]
await transcript(partial)
const partialRows = rows(partial)
expect(partialRows.map(rowImages)).toEqual([[], ['file:///a.jpg', 'file:///b.jpg']])
expect(state?.imagePreviewsByMessageId).toEqual({})
const complete = [userTextMessage('prompt', '[Image #1] [Image #2]')]
await transcript(complete)
expect(state?.imagePreviewsByMessageId).toEqual({
prompt: ['file:///a.jpg', 'file:///b.jpg']
})
expect(rows(complete).map(rowImages)).toEqual([['file:///a.jpg', 'file:///b.jpg']])
})
it('still binds a single-image preview to its marker-only echo', async () => {
await mount()
send('', ['file:///a.jpg'])
const messages = [userTextMessage('prompt', '[Image #1]')]
await transcript(messages)
expect(state?.imagePreviewsByMessageId).toEqual({ prompt: ['file:///a.jpg'] })
expect(rows(messages).map(rowImages)).toEqual([['file:///a.jpg']])
})
it('still strips a real marker that follows an attached image', async () => {
await mount()
send('look', ['file:///a.jpg'])
const messages = [
userTextMessage('src', '[Image: source: /tmp/a.png]'),
userTextMessage('prompt', 'look [Image #1]')
]
await transcript(messages)
const rendered = rows(messages)
expect(rendered.map(rowText)).toEqual(['look'])
expect(rendered.map(rowImages)).toEqual([['file:///a.jpg']])
})
})
@@ -1,3 +1,8 @@
import {
normalizeNativeChatUserText,
normalizeNativeChatUserTextWithLiteralFallback
} from './mobile-native-chat-image-transcript-markers'
export type MobileNativeChatPendingMessage = {
id: string
text: string
@@ -17,6 +22,14 @@ export type MobileNativeChatSendOrigin = {
type PendingByKey = Record<string, MobileNativeChatPendingMessage[]>
/** Markers are the agent's only on a send that carried images; otherwise they
* are literal text, so `captureSendOrigin` keeps them and so must this. */
function pendingMatchText(pending: MobileNativeChatPendingMessage): string {
return pending.images?.length
? normalizeNativeChatUserText(pending.text)
: normalizeNativeChatUserTextWithLiteralFallback(pending.text)
}
export function combineMobileNativeChatPending(
session: MobileNativeChatPendingMessage[],
waiting: readonly MobileNativeChatPendingMessage[]
@@ -39,11 +52,15 @@ export function appendMobileNativeChatPending(
const current = previous[key] ?? []
const earlierOutstanding = current.filter(
(pending) =>
pending.text.trim() === origin.normalizedText &&
pendingMatchText(pending) === origin.normalizedText &&
pending.expectedOccurrence > origin.baselineOccurrences
).length
// An image send whose caption is only markers still reconciles by image
// ordinal, matching how `captureSendOrigin` normalized it.
const expectedImageEchoOrdinal =
current.filter((pending) => pending.text.trim() === '' && pending.images?.length).length + 1
current.filter(
(pending) => pending.images?.length && normalizeNativeChatUserText(pending.text) === ''
).length + 1
return {
...previous,
[key]: [
@@ -8,7 +8,8 @@ import {
mergeLandedImagePreviewEchoes,
migrateImagePreviewMessageIds,
normalizeReconcileText,
normalizedUserText,
normalizeReconcileTextWithLiteralFallback,
userTextOccurrenceCounts,
type UnconfirmedSend
} from './mobile-native-chat-draft-reconcile'
import {
@@ -54,7 +55,7 @@ export function useMobileNativeChatDrafts(args: {
/** Phone-local previews rebound to the transcript message that replaced the
* optimistic echo, keyed by authoritative message id. */
imagePreviewsByMessageId: Record<string, string[]>
captureSendOrigin: (text: string) => MobileNativeChatSendOrigin | null
captureSendOrigin: (text: string, images?: readonly string[]) => MobileNativeChatSendOrigin | null
/** Launch-context text still believed to be parked on the agent's TUI input
* line, or null once it has been declined or retired. Send paths size their
* pre-clear from it, since one Ctrl+U clears only one logical line. */
@@ -128,11 +129,15 @@ export function useMobileNativeChatDrafts(args: {
)
const captureSendOrigin = useCallback(
(text: string) => {
(text: string, images?: readonly string[]) => {
if (!draftKey) {
return null
}
const normalizedText = normalizeReconcileText(text)
// Without an attached image an `[Image #n]` run is the user's own text, so
// it has to stay in the key the transcript echo will be matched against.
const normalizedText = images?.length
? normalizeReconcileText(text)
: normalizeReconcileTextWithLiteralFallback(text)
return {
draftKey,
pendingKey,
@@ -291,13 +296,7 @@ export function useMobileNativeChatDrafts(args: {
}
setPendingBySession((previous) => {
const current = previous[pendingKey] ?? []
const landedCounts = new Map<string, number>()
for (const message of messages) {
const text = normalizedUserText(message)
if (text) {
landedCounts.set(text, (landedCounts.get(text) ?? 0) + 1)
}
}
const landedCounts = userTextOccurrenceCounts(messages)
// Image-only source-turn counts stay stable across reruns and ignore paginated history.
const next = current.filter((item) => {
if (landedImagePendingIds.has(item.id)) {
@@ -310,7 +309,8 @@ export function useMobileNativeChatDrafts(args: {
return item.text.trim() === ''
? countImageSourceTurnsAfter(messages, item.baselineTailMessageId) <
item.expectedOccurrence
: (landedCounts.get(normalizeReconcileText(item.text)) ?? 0) < item.expectedOccurrence
: (landedCounts.get(normalizeReconcileTextWithLiteralFallback(item.text)) ?? 0) <
item.expectedOccurrence
})
if (next.length === current.length) {
return previous
@@ -53,7 +53,7 @@ export function useMobileNativeChatMessageSend(args: {
/** Captured when a control send starts so a later tab switch cannot record its
* session-option effects against the newly active tab. */
commandSendRef: MutableRefObject<(command: string) => void>
captureSendOrigin: (text: string) => MobileNativeChatSendOrigin | null
captureSendOrigin: (text: string, images?: readonly string[]) => MobileNativeChatSendOrigin | null
/** Launch-context text Orca parked on the agent's TUI input line, or null. Read
* at send time so the pre-clear can be sized to every line it occupies. */
readSeededLaunchDraftSeed: () => MobileNativeChatLaunchDraftSeed | null
@@ -92,7 +92,7 @@ export function useMobileNativeChatMessageSend(args: {
sharedDeadline?: number
): Promise<MobileNativeChatSendOutcome> => {
const handle = handleRef.current
const origin = captureSendOrigin(text)
const origin = captureSendOrigin(text, images)
const agent = agentRef.current
const recordCommand = commandSendRef.current
// Why: the lease collapses one render after `connState`, so a question-card
@@ -0,0 +1,127 @@
// Outcome-level guard for STA-4363: the render path and the pending-echo key
// path must agree on which `[Image #n]` runs are literal text, or the optimistic
// echo never reconciles and the turn renders twice.
import { describe, expect, it } from 'vitest'
import {
isImageRefBlock,
isTextBlock,
type NativeChatMessage
} from '../../../../shared/native-chat-types'
import { pendingSendsAsMessages, prunePendingSends } from './native-chat-pending'
import type { NativeChatPendingSend } from './native-chat-pending'
import { assembleNativeChatSession } from './native-chat-session-assembler'
function userMessage(id: string, text: string): NativeChatMessage {
return { id, role: 'user', blocks: [{ type: 'text', text }], timestamp: 1, source: 'transcript' }
}
function assistantMessage(id: string, text: string): NativeChatMessage {
return {
id,
role: 'assistant',
blocks: [{ type: 'text', text }],
timestamp: 2,
source: 'transcript'
}
}
function assembled(transcript: NativeChatMessage[]): NativeChatMessage[] {
return assembleNativeChatSession({ sources: { transcript }, sessionId: 's1', agent: 'claude' })
.messages
}
/** The user rows NativeChatView actually renders: assembled transcript first,
* then whichever optimistic echoes are still unrepresented. */
function renderedUserRows(
transcript: NativeChatMessage[],
pending: NativeChatPendingSend[]
): NativeChatMessage[] {
const messages = assembled(transcript)
return [...messages, ...pendingSendsAsMessages(pending, messages)].filter(
(message) => message.role === 'user'
)
}
function rowText(message: NativeChatMessage): string {
return message.blocks
.filter(isTextBlock)
.map((block) => block.text)
.join('')
}
function rowImagePaths(message: NativeChatMessage): string[] {
return message.blocks
.filter(isImageRefBlock)
.map((block) => block.path ?? block.url ?? '')
.filter(Boolean)
}
describe('literal image-marker turns in native chat', () => {
it('shows a standalone marker-only send exactly once', () => {
const pending: NativeChatPendingSend[] = [{ id: 'p1', text: '[Image #1]', sentAt: 100 }]
const transcript = [userMessage('m1', '[Image #1]')]
const rows = renderedUserRows(transcript, pending)
expect(rows.map(rowText)).toEqual(['[Image #1]'])
})
it('shows a marker-only send with surrounding prose exactly once', () => {
const pending: NativeChatPendingSend[] = [
{ id: 'p1', text: 'keep [Image #1] literal', sentAt: 100 }
]
const transcript = [userMessage('m1', 'keep [Image #1] literal')]
expect(renderedUserRows(transcript, pending).map(rowText)).toEqual(['keep [Image #1] literal'])
})
it('prunes the marker-only echo once the assistant answers it', () => {
const pending: NativeChatPendingSend[] = [{ id: 'p1', text: '[Image #1]', sentAt: 100 }]
const transcript = [userMessage('m1', '[Image #1]'), assistantMessage('m2', 'preserved')]
expect(prunePendingSends(pending, assembled(transcript))).toEqual([])
})
it('shows two rapid marker-only sends exactly twice', () => {
const pending: NativeChatPendingSend[] = [
{ id: 'p1', text: '[Image #1]', sentAt: 100, matchingOccurrence: 1 },
{ id: 'p2', text: '[Image #1]', sentAt: 101, matchingOccurrence: 2 }
]
const transcript = [userMessage('m1', '[Image #1]'), userMessage('m2', '[Image #1]')]
expect(renderedUserRows(transcript, pending).map(rowText)).toEqual(['[Image #1]', '[Image #1]'])
})
it('still strips a real marker that follows an attached image, and echoes once', () => {
const pending: NativeChatPendingSend[] = [
{ id: 'p1', text: 'look', imagePaths: ['/tmp/a.png'], sentAt: 100 }
]
const transcript = [
userMessage('src', '[Image: source: /tmp/a.png]'),
userMessage('prompt', 'look [Image #1]')
]
const rows = renderedUserRows(transcript, pending)
expect(rows).toHaveLength(1)
expect(rowText(rows[0]!)).toBe('look')
expect(rowImagePaths(rows[0]!)).toEqual(['/tmp/a.png'])
})
it('still reconciles a caption-less image send whose echo is marker-only', () => {
const pending: NativeChatPendingSend[] = [
{ id: 'p1', text: '', imagePaths: ['/tmp/a.png'], sentAt: 100 }
]
const transcript = [
userMessage('src', '[Image: source: /tmp/a.png]'),
userMessage('prompt', '[Image #1]')
]
const rows = renderedUserRows(transcript, pending)
expect(rows).toHaveLength(1)
expect(rowText(rows[0]!)).toBe('')
expect(rowImagePaths(rows[0]!)).toEqual(['/tmp/a.png'])
})
})
@@ -1,6 +1,7 @@
import {
nativeChatUserMessageMatchText,
normalizeNativeChatUserText,
normalizedNativeChatUserMessageText
normalizeNativeChatUserTextWithLiteralFallback
} from '../../../../shared/native-chat-image-transcript-markers'
import { isImageRefBlock, type NativeChatMessage } from '../../../../shared/native-chat-types'
@@ -18,10 +19,21 @@ export function normalizeNativeChatPendingText(text: string): string {
return normalizeNativeChatUserText(text)
}
/** Message-side rule (`nativeChatUserMessageMatchText`) applied to a send: only
* a send that actually carried images can own an `[Image #n]` marker, so on
* every other send the marker is literal text and has to key verbatim. */
function nativeChatPendingMatchText(
pending: Pick<NativeChatPendingOccurrence, 'text' | 'imagePaths'>
): string {
return pending.imagePaths?.some(Boolean)
? normalizeNativeChatPendingText(pending.text)
: normalizeNativeChatUserTextWithLiteralFallback(pending.text)
}
export function nativeChatPendingContentKey(
pending: Pick<NativeChatPendingOccurrence, 'text' | 'imagePaths'>
): string {
const text = normalizeNativeChatPendingText(pending.text)
const text = nativeChatPendingMatchText(pending)
if (text) {
return `text:${text}`
}
@@ -33,7 +45,7 @@ function nativeChatUserMessageContentKey(message: NativeChatMessage): string | n
if (message.role !== 'user') {
return null
}
const text = normalizedNativeChatUserMessageText(message) ?? ''
const text = nativeChatUserMessageMatchText(message) ?? ''
if (text) {
return `text:${text}`
}
@@ -87,7 +99,7 @@ export function advancedNativeChatUserTexts(
const waiting: string[] = []
for (const message of messages) {
if (message.role === 'user') {
const text = normalizedNativeChatUserMessageText(message)
const text = nativeChatUserMessageMatchText(message)
if (text) {
waiting.push(text)
}
@@ -105,7 +117,7 @@ export function matchingNativeChatUserTexts(
): readonly string[] {
const texts: string[] = []
for (const message of messages) {
const text = normalizedNativeChatUserMessageText(message)
const text = nativeChatUserMessageMatchText(message)
if (text) {
texts.push(text)
}
@@ -158,7 +170,7 @@ export function selectPendingIndicesRepresentedByUserTexts(
}
const remaining = pending.map((entry, index) => ({
index,
text: normalizeNativeChatPendingText(entry.text)
text: nativeChatPendingMatchText(entry)
}))
for (const userText of userTexts) {
const open = remaining.filter((entry) => !represented.has(entry.index) && entry.text.length > 0)
@@ -1,4 +1,9 @@
import { isTextBlock, type NativeChatBlock, type NativeChatMessage } from './native-chat-types'
import {
isImageRefBlock,
isTextBlock,
type NativeChatBlock,
type NativeChatMessage
} from './native-chat-types'
const IMAGE_SOURCE_MARKER = /^\[Image:\s*source:\s*(.+?)\]\s*$/
const IMAGE_PROMPT_MARKER = /\[Image #\d+\]/
@@ -40,17 +45,40 @@ export function normalizeNativeChatUserText(text: string): string {
return stripImagePromptMarker(text).trim().replace(/\s+/g, ' ')
}
/** Marker text that stripping would erase entirely is text the user typed
* literally — a real marker always rides on an image turn, which normalization
* has already folded away by the time echo matching runs. */
export function normalizeNativeChatUserTextWithLiteralFallback(text: string): string {
return normalizeNativeChatUserText(text) || text.trim().replace(/\s+/g, ' ')
}
function joinedUserText(message: NativeChatMessage): string {
return message.blocks
.filter(isTextBlock)
.map((block) => block.text)
.join(' ')
}
export function normalizedNativeChatUserMessageText(message: NativeChatMessage): string | null {
if (message.role !== 'user') {
return null
}
const normalized = normalizeNativeChatUserText(
message.blocks
.filter(isTextBlock)
.map((block) => block.text)
.join(' ')
)
return normalized || null
return normalizeNativeChatUserText(joinedUserText(message)) || null
}
/** The text an optimistic echo matches a landed turn on. Keeps literal marker
* text keyable so the pending path and the render path agree about which
* `[Image #n]` runs are the user's own words. */
export function nativeChatUserMessageMatchText(message: NativeChatMessage): string | null {
if (message.role !== 'user') {
return null
}
const joined = joinedUserText(message)
const normalized = normalizeNativeChatUserText(joined)
if (normalized || message.blocks.some(isImageRefBlock)) {
return normalized || null
}
return normalizeNativeChatUserTextWithLiteralFallback(joined) || null
}
function stripImagePromptMarkersFromTextBlocks(
@@ -94,6 +122,16 @@ export function hasImagePromptMarker(message: NativeChatMessage): boolean {
return message.blocks.some((block) => isTextBlock(block) && IMAGE_PROMPT_MARKER.test(block.text))
}
/** Markers carried by a turn. One marker vouches for one image, so a send of N
* images is only echoed by a turn bearing N of them. */
export function countImagePromptMarkers(message: NativeChatMessage): number {
return message.blocks.reduce(
(count, block) =>
count + (isTextBlock(block) ? (block.text.match(IMAGE_PROMPT_MARKERS)?.length ?? 0) : 0),
0
)
}
/** Claude records image paths as source turns followed by a prompt carrying
* image markers. Merge the whole run back into one native user turn. */
export function normalizeImageTranscriptMessages(