Fix native chat image marker position handling (#14162)

* fix(native-chat): handle image markers in any position

* fix(native-chat): preserve image caption whitespace

* test(native-chat): cover marker boundary spacing

* fix(mobile): normalize image echo reconciliation

* fix(mobile): use idiomatic tail access

* refactor(native-chat): share image echo matching

* perf(native-chat): avoid unchanged block copies
This commit is contained in:
Brennan Benson
2026-08-13 01:18:08 -07:00
committed by GitHub
parent 0ed6db77cf
commit 0f51d0b3bb
12 changed files with 418 additions and 109 deletions
@@ -1,9 +1,8 @@
import { stripImagePromptMarker } from '../../../../shared/native-chat-image-transcript-markers'
import {
isImageRefBlock,
isTextBlock,
type NativeChatMessage
} from '../../../../shared/native-chat-types'
normalizeNativeChatUserText,
normalizedNativeChatUserMessageText
} from '../../../../shared/native-chat-image-transcript-markers'
import { isImageRefBlock, type NativeChatMessage } from '../../../../shared/native-chat-types'
export type NativeChatPendingOccurrence = {
text: string
@@ -16,7 +15,7 @@ export type NativeChatPendingOccurrence = {
}
export function normalizeNativeChatPendingText(text: string): string {
return stripImagePromptMarker(text).trim().replace(/\s+/g, ' ')
return normalizeNativeChatUserText(text)
}
export function nativeChatPendingContentKey(
@@ -34,15 +33,15 @@ function nativeChatUserMessageContentKey(message: NativeChatMessage): string | n
if (message.role !== 'user') {
return null
}
const text = message.blocks
.filter(isTextBlock)
.map((block) => block.text)
.join(' ')
const text = normalizedNativeChatUserMessageText(message) ?? ''
if (text) {
return `text:${text}`
}
const imagePaths = message.blocks
.filter(isImageRefBlock)
.map((block) => block.path)
.filter((path): path is string => Boolean(path))
const key = nativeChatPendingContentKey({ text, imagePaths })
const key = nativeChatPendingContentKey({ text: '', imagePaths })
return key === 'empty' ? null : key
}
@@ -80,19 +79,6 @@ export function advancedNativeChatUserContentCounts(
return advanced
}
function nativeChatUserMessageNormalizedText(message: NativeChatMessage): string | null {
if (message.role !== 'user') {
return null
}
const text = normalizeNativeChatPendingText(
message.blocks
.filter(isTextBlock)
.map((block) => block.text)
.join(' ')
)
return text.length > 0 ? text : null
}
/** User texts that already have a later non-user turn (ready to prune echoes). */
export function advancedNativeChatUserTexts(
messages: readonly NativeChatMessage[]
@@ -101,7 +87,7 @@ export function advancedNativeChatUserTexts(
const waiting: string[] = []
for (const message of messages) {
if (message.role === 'user') {
const text = nativeChatUserMessageNormalizedText(message)
const text = normalizedNativeChatUserMessageText(message)
if (text) {
waiting.push(text)
}
@@ -119,7 +105,7 @@ export function matchingNativeChatUserTexts(
): readonly string[] {
const texts: string[] = []
for (const message of messages) {
const text = nativeChatUserMessageNormalizedText(message)
const text = normalizedNativeChatUserMessageText(message)
if (text) {
texts.push(text)
}
@@ -95,6 +95,34 @@ describe('prunePendingSends', () => {
expect(next).toEqual([])
})
it('drops an attachment pending send once a trailing-marker prompt advances', () => {
const pending = [
{ ...pendingOf('p1', 'what do you see'), imagePaths: ['/Users/me/Downloads/3d.png'] }
]
const next = prunePendingSends(pending, [
userMessage('m1', 'what do you see[Image #1]'),
assistantMessage('m2', 'an image')
])
expect(next).toEqual([])
})
it('drops a pending send represented by multiple marker-bearing text blocks', () => {
const prompt: NativeChatMessage = {
...userMessage('m1', 'unused'),
blocks: [
{ type: 'text', text: 'what do' },
{ type: 'image-ref', path: '/tmp/a.png' },
{ type: 'text', text: '[Image #1] you see' }
]
}
const next = prunePendingSends(
[pendingOf('p1', 'what do you see')],
[prompt, assistantMessage('m2', 'an image')]
)
expect(next).toEqual([])
})
it('drops an attachment-only pending send once its image turn advances', () => {
const pending = [{ ...pendingOf('p1', ''), imagePaths: ['/tmp/first.png', '/tmp/second.png'] }]
const transcript = [
@@ -74,6 +74,29 @@ describe('preassembled native-chat live sessions', () => {
expect(out[0]).toMatchObject({ id: 'image-prompt', source: 'transcript' })
})
it('keeps legacy parity for trailing image prompt markers', () => {
const transcript: NativeChatMessage[] = [
message('image-source', {
role: 'user',
blocks: [{ type: 'text', text: '[Image: source: /tmp/a.png]' }],
timestamp: 1
}),
message('image-prompt', {
role: 'user',
blocks: [{ type: 'text', text: 'describe this[Image #1]' }],
timestamp: 2
})
]
const out = expectLegacyMessageParity(transcript)
expect(out).toHaveLength(1)
expect(out[0]?.blocks).toEqual([
{ type: 'image-ref', path: '/tmp/a.png' },
{ type: 'text', text: 'describe this' }
])
})
it('re-dedupes surfaced skill envelopes that collide across sources', () => {
const skill = (id: string, plugin: string, source: 'transcript' | 'scrape') =>
message(id, {
@@ -112,6 +112,34 @@ describe('assembleNativeChatSession', () => {
])
})
it('merges Claude image source markers into a trailing-marker prompt', () => {
const imageSource = msg({
id: 'u-image-source',
role: 'user',
timestamp: 100,
blocks: [{ type: 'text', text: '[Image: source: /Users/me/Downloads/3d.png]' }]
})
const prompt = msg({
id: 'u-prompt',
role: 'user',
timestamp: 101,
blocks: [{ type: 'text', text: 'what do you see[Image #1]' }]
})
const session = assembleNativeChatSession({
sources: { transcript: [imageSource, prompt] },
sessionId: 's1',
agent: 'claude'
})
expect(session.messages).toHaveLength(1)
expect(session.messages[0]).toMatchObject({ id: 'u-prompt', role: 'user' })
expect(session.messages[0].blocks).toEqual([
{ type: 'image-ref', path: '/Users/me/Downloads/3d.png' },
{ type: 'text', text: 'what do you see' }
])
})
it('drops a scrape duplicate even when scrape is processed first by id', () => {
const scrape = msg({
id: 'shared-id',