fix(native-chat): bound marker stripping by the run's image count

Anchoring decided which turn gets stripped but not how many markers may go,
so a turn attaching one image and mentioning a second marker in prose still
lost the user's literal text. Strip at most one marker per image in the
anchored source run, in document order across the turn's text blocks; markers
past the count stay verbatim.

Position, not ordinal, decides which markers are placeholders, so the rule
holds whether the producing CLI numbers markers per-turn or per-session.
This commit is contained in:
Brennan Benson
2026-08-16 16:40:08 -07:00
parent 0f74ea00cc
commit 1689e2f241
3 changed files with 121 additions and 6 deletions
@@ -133,6 +133,22 @@ describe('buildMobileNativeChatTransientData', () => {
])
})
it('keeps a marker beyond the folded run’s image count as literal text', () => {
const data = build(
[
user('u1', '[Image: source: /tmp/a.png]'),
user('u2', '[Image #1] compare with the [Image #2] I mentioned earlier')
],
null,
[]
)
expect(data[0]?.blocks).toEqual([
{ type: 'image-ref', path: '/tmp/a.png' },
{ type: 'text', text: 'compare with the [Image #2] I mentioned earlier' }
])
})
it('renders a lone image marker turn (no caption) as an image-ref block', () => {
const data = build([user('u1', '[Image: source: /tmp/a.png]')], null, [])
expect(data[0]?.blocks).toEqual([{ type: 'image-ref', path: '/tmp/a.png' }])
@@ -259,6 +259,79 @@ describe('normalizeImageTranscriptMessages', () => {
])
})
it('keeps markers beyond the anchored run’s image count as the user’s own words', () => {
const out = normalizeImageTranscriptMessages([
userText('s1', '[Image: source: /tmp/a.png]'),
userText('p1', '[Image #1] compare with the [Image #2] I mentioned earlier')
])
expect(out).toHaveLength(1)
expect(out[0]!.blocks).toEqual([
{ type: 'image-ref', path: '/tmp/a.png' },
{ type: 'text', text: 'compare with the [Image #2] I mentioned earlier' }
])
})
it('strips by document position, not by marker ordinal', () => {
const out = normalizeImageTranscriptMessages([
userText('s1', '[Image: source: /tmp/a.png]'),
userText('p1', '[Image #2] came first but [Image #1] came second')
])
expect(out[0]!.blocks).toEqual([
{ type: 'image-ref', path: '/tmp/a.png' },
{ type: 'text', text: 'came first but [Image #1] came second' }
])
})
it('strips a marker whose ordinal exceeds the run size when it is within the count', () => {
const out = normalizeImageTranscriptMessages([
userText('s1', '[Image: source: /tmp/a.png]'),
userText('p1', '[Image #7] what is this?')
])
expect(out[0]!.blocks).toEqual([
{ type: 'image-ref', path: '/tmp/a.png' },
{ type: 'text', text: 'what is this?' }
])
})
it('spends the image budget across text blocks in document order', () => {
const prompt: NativeChatMessage = {
...userText('prompt', 'unused'),
blocks: [
{ type: 'text', text: '[Image #1] look at' },
{ type: 'image-ref', path: '/tmp/existing.png' },
{ type: 'text', text: 'not [Image #2] though' }
]
}
const out = normalizeImageTranscriptMessages([
userText('s1', '[Image: source: /tmp/a.png]'),
prompt
])
expect(out[0]!.blocks).toEqual([
{ type: 'image-ref', path: '/tmp/a.png' },
{ type: 'text', text: 'look at' },
{ type: 'image-ref', path: '/tmp/existing.png' },
{ type: 'text', text: 'not [Image #2] though' }
])
})
it('keeps the surplus marker on a two-image run that mentions a third', () => {
const out = normalizeImageTranscriptMessages([
userText('s1', '[Image: source: /tmp/a.png]'),
userText('s2', '[Image: source: /tmp/b.png]'),
userText('p1', '[Image #1] [Image #2] versus the [Image #3] from before')
])
expect(out[0]!.blocks).toEqual([
{ type: 'image-ref', path: '/tmp/a.png' },
{ type: 'image-ref', path: '/tmp/b.png' },
{ type: 'text', text: 'versus the [Image #3] from before' }
])
})
it('leaves assistant messages untouched', () => {
const assistant: NativeChatMessage = {
id: 'a',
@@ -27,20 +27,41 @@ export function isImageSourceUserTurn(message: NativeChatMessage): boolean {
return message.role === 'user' && imageSourcePathFromText(soleText(message) ?? '') !== null
}
export function stripImagePromptMarker(text: string): string {
const stripped = text.replace(IMAGE_PROMPT_MARKERS, '')
function countMarkers(text: string): number {
return text.match(IMAGE_PROMPT_MARKERS)?.length ?? 0
}
/** Strips the first `limit` markers in document order; the rest are the user's
* own words, so they stay verbatim. */
function stripImagePromptMarkersUpTo(text: string, limit: number): string {
if (limit <= 0) {
return text
}
let used = 0
const stripped = text.replace(IMAGE_PROMPT_MARKERS, (match) => {
if (used >= limit) {
return match
}
used += 1
return ''
})
if (stripped === text) {
return text
}
let result = IMAGE_PROMPT_MARKER_AT_START.test(text)
? stripped.replace(HORIZONTAL_WHITESPACE_START, '')
: stripped
if (IMAGE_PROMPT_MARKER_AT_END.test(text)) {
// Why: a trailing marker left in place still owns the whitespace before it.
if (used === countMarkers(text) && IMAGE_PROMPT_MARKER_AT_END.test(text)) {
result = result.replace(HORIZONTAL_WHITESPACE_END, '')
}
return result
}
export function stripImagePromptMarker(text: string): string {
return stripImagePromptMarkersUpTo(text, Number.POSITIVE_INFINITY)
}
export function normalizeNativeChatUserText(text: string): string {
return stripImagePromptMarker(text).trim().replace(/\s+/g, ' ')
}
@@ -78,10 +99,14 @@ export function nativeChatUserMessageMatchText(message: NativeChatMessage): stri
return nativeChatUserTextMatchText(joined, message.blocks.some(isImageRefBlock)) || null
}
/** `limit` is the number of images the run actually carried. Only that many
* markers can be placeholders; the rest are text the user typed. */
function stripImagePromptMarkersFromTextBlocks(
blocks: readonly NativeChatBlock[]
blocks: readonly NativeChatBlock[],
limit: number
): NativeChatBlock[] {
let sawText = false
let remaining = limit
let next: NativeChatBlock[] | null = null
for (let index = 0; index < blocks.length; index += 1) {
const block = blocks[index]!
@@ -91,7 +116,8 @@ function stripImagePromptMarkersFromTextBlocks(
}
const isFirstText = !sawText
sawText = true
const text = stripImagePromptMarker(block.text)
const text = stripImagePromptMarkersUpTo(block.text, remaining)
remaining -= Math.min(remaining, countMarkers(block.text))
if (!text.trim() && (text !== block.text || isFirstText)) {
next ??= blocks.slice(0, index)
continue
@@ -173,7 +199,7 @@ export function normalizeImageTranscriptMessages(
...prompt,
blocks: [
...imagePaths.map((path) => ({ type: 'image-ref' as const, path })),
...stripImagePromptMarkersFromTextBlocks(prompt.blocks)
...stripImagePromptMarkersFromTextBlocks(prompt.blocks, imagePaths.length)
]
})
index = nextIndex