Fix native chat image marker position handling (#14162)

* fix(native-chat): handle image markers in any position

* fix(native-chat): preserve image caption whitespace

* test(native-chat): cover marker boundary spacing

* fix(mobile): normalize image echo reconciliation

* fix(mobile): use idiomatic tail access

* refactor(native-chat): share image echo matching

* perf(native-chat): avoid unchanged block copies
This commit is contained in:
Brennan Benson
2026-08-13 01:18:08 -07:00
committed by GitHub
parent 0ed6db77cf
commit 0f51d0b3bb
12 changed files with 418 additions and 109 deletions
@@ -2,8 +2,10 @@ import { describe, expect, it } from 'vitest'
import type { NativeChatMessage } from '../../../src/shared/native-chat-types'
import {
findLandedImagePreviewEchoes,
findLandedUnconfirmedSends,
migrateImagePreviewMessageIds,
type PendingImagePreviewEcho
type PendingImagePreviewEcho,
type UnconfirmedSend
} from './mobile-native-chat-draft-reconcile'
function userText(id: string, text: string): NativeChatMessage {
@@ -21,6 +23,67 @@ function pending(id: string, images: string[], expectedOccurrence = 1): PendingI
}
describe('mobile native chat image preview reconciliation', () => {
it('reconciles a trailing-marker echo and hands its preview to that echo', () => {
const messages = [
userText('source', '[Image: source: /tmp/a.png]'),
userText('prompt', 'look at this[Image #1]')
]
const preview = {
...pending('pending', ['file:///a.jpg']),
text: 'look at this'
}
const unconfirmed: UnconfirmedSend = {
draftKey: 'draft',
pendingKey: 'pending-key',
text: 'look at this',
normalizedText: 'look at this',
baselineTailMessageId: null,
deadline: null
}
expect(findLandedUnconfirmedSends(messages, [unconfirmed])).toEqual([unconfirmed])
expect(findLandedImagePreviewEchoes(messages, [preview])).toEqual([
{ pendingId: 'pending', messageId: 'prompt', images: ['file:///a.jpg'] }
])
})
it('reconciles a middle-marker echo without changing its rendered whitespace', () => {
const messages = [
userText('source', '[Image: source: /tmp/a.png]'),
userText('prompt', 'look [Image #1] here')
]
const preview = { ...pending('pending', ['file:///a.jpg']), text: 'look here' }
const unconfirmed: UnconfirmedSend = {
draftKey: 'draft',
pendingKey: 'pending-key',
text: 'look here',
normalizedText: 'look here',
baselineTailMessageId: null,
deadline: null
}
expect(findLandedUnconfirmedSends(messages, [unconfirmed])).toEqual([unconfirmed])
expect(findLandedImagePreviewEchoes(messages, [preview])).toEqual([
{ pendingId: 'pending', messageId: 'prompt', images: ['file:///a.jpg'] }
])
})
it('reconciles multiple transcript text blocks with desktop separators', () => {
const prompt: NativeChatMessage = {
...userText('prompt', 'unused'),
blocks: [
{ type: 'text', text: 'look' },
{ type: 'image-ref', path: '/tmp/a.png' },
{ type: 'text', text: '[Image #1] here' }
]
}
const preview = { ...pending('pending', ['file:///a.jpg']), text: 'look here' }
expect(findLandedImagePreviewEchoes([prompt], [preview])).toEqual([
{ pendingId: 'pending', messageId: 'prompt', images: ['file:///a.jpg'] }
])
})
it('keeps separate adjacent image-only sends independently reconcilable', () => {
const landed = findLandedImagePreviewEchoes(
[
@@ -69,4 +132,36 @@ describe('mobile native chat image preview reconciliation', () => {
[sessionKey]: { prompt: ['file:///a.jpg'] }
})
})
it('moves an early standalone preview to a trailing-marker prompt id', () => {
const sessionKey = 'host\0worktree\0tab\0session'
const previous = { [sessionKey]: { source: ['file:///a.jpg'] } }
const messages = [
userText('source', '[Image: source: /tmp/a.png]'),
userText('prompt', 'look[Image #1]')
]
expect(migrateImagePreviewMessageIds(previous, sessionKey, messages)).toEqual({
[sessionKey]: { prompt: ['file:///a.jpg'] }
})
})
it('moves a preview when the prompt marker is in a later text block', () => {
const sessionKey = 'host\0worktree\0tab\0session'
const previous = { [sessionKey]: { source: ['file:///a.jpg'] } }
const prompt: NativeChatMessage = {
...userText('prompt', 'unused'),
blocks: [
{ type: 'text', text: 'look' },
{ type: 'text', text: '[Image #1] here' }
]
}
expect(
migrateImagePreviewMessageIds(previous, sessionKey, [
userText('source', '[Image: source: /tmp/a.png]'),
prompt
])
).toEqual({ [sessionKey]: { prompt: ['file:///a.jpg'] } })
})
})
@@ -1,9 +1,12 @@
import { isImageRefBlock, type NativeChatMessage } from '../../../src/shared/native-chat-types'
import {
hasImagePromptMarker,
isImageSourceUserTurn,
normalizeImageTranscriptMessages,
stripImagePromptMarker
normalizeNativeChatUserText,
normalizedNativeChatUserMessageText
} from './mobile-native-chat-image-transcript-markers'
export { normalizeNativeChatUserText as normalizeReconcileText } from './mobile-native-chat-image-transcript-markers'
/** An ack-lost ('unknown' outcome) send held until its transcript echo lands or
* the deadline surfaces the uncertainty. */
@@ -17,17 +20,7 @@ export type UnconfirmedSend = {
}
export function normalizedUserText(message: NativeChatMessage): string | null {
if (message.role !== 'user') {
return null
}
const text = message.blocks
.filter((block) => block.type === 'text')
.map((block) => (block.type === 'text' ? block.text : ''))
.join('')
// Claude echoes a captioned image send as `[Image #1] caption` — the sent
// text must still match its echo, so strip the marker before comparing.
const stripped = stripImagePromptMarker(text).trim()
return stripped || null
return normalizedNativeChatUserMessageText(message)
}
export function countUserTextOccurrences(
@@ -117,17 +110,13 @@ function imagePreviewReplacementMessageId(
nextIndex++
}
const prompt = messages[nextIndex]
const firstText = prompt?.blocks.find((block) => block.type === 'text')
return prompt?.role === 'user' &&
prompt.source === source.source &&
firstText?.type === 'text' &&
stripImagePromptMarker(firstText.text) !== firstText.text
return prompt?.role === 'user' && prompt.source === source.source && hasImagePromptMarker(prompt)
? prompt.id
: null
}
/** Moves previews forward when a progressive source-only transcript frame later
* folds into the marker-prefixed prompt with a different authoritative id. */
* folds into the marker-bearing prompt with a different authoritative id. */
export function migrateImagePreviewMessageIds(
previous: Record<string, Record<string, string[]>>,
sessionKey: string,
@@ -171,7 +160,7 @@ export function findLandedImagePreviewEchoes(
if (!entry.images?.length) {
continue
}
const targetText = entry.text.trim()
const targetText = normalizeNativeChatUserText(entry.text)
const candidates = normalized.filter((message) => {
if (message.role !== 'user') {
return false
@@ -1,21 +1,13 @@
// Single-sources the marker logic (pure functions over shared types):
// Claude records an attached image as `[Image: source: /path]` (+ `[Image #N]`
// prefix on the caption turn), and both render and echo reconciliation must
// on the caption turn), and both render and echo reconciliation must
// agree with desktop on how those marker turns are interpreted.
export {
imageSourcePathFromText,
hasImagePromptMarker,
isImageSourceUserTurn,
normalizeImageTranscriptMessages,
normalizeNativeChatUserText,
normalizedNativeChatUserMessageText,
stripImagePromptMarker
} from '../../../src/shared/native-chat-image-transcript-markers'
import { imageSourcePathFromText } from '../../../src/shared/native-chat-image-transcript-markers'
import { isTextBlock, type NativeChatMessage } from '../../../src/shared/native-chat-types'
/** A raw (un-normalized) transcript user turn that is an image-source marker —
* the echo shape of an image riding along on a send. */
export function isImageSourceUserTurn(message: NativeChatMessage): boolean {
if (message.role !== 'user' || message.blocks.length !== 1) {
return false
}
const block = message.blocks[0]
return block !== undefined && isTextBlock(block) && imageSourcePathFromText(block.text) !== null
}
@@ -92,8 +92,8 @@ describe('buildMobileNativeChatTransientData', () => {
})
it('folds transcript image marker turns into image-ref blocks (desktop parity)', () => {
// Claude records an attached image as `[Image: source: /path]` + an
// `[Image #1] `-prefixed caption turn; the fold must merge them into one
// Claude records an attached image as `[Image: source: /path]` plus a
// caption turn carrying `[Image #1]`; the fold must merge them into one
// user turn with an image-ref block instead of showing raw marker text.
const data = build(
[
@@ -111,6 +111,20 @@ describe('buildMobileNativeChatTransientData', () => {
])
})
it('folds a trailing-marker image echo into one user bubble', () => {
const data = build(
[user('u1', '[Image: source: /tmp/a.png]'), user('u2', 'look at this[Image #1]')],
null,
[]
)
expect(data).toHaveLength(1)
expect(data[0]?.blocks).toEqual([
{ type: 'image-ref', path: '/tmp/a.png' },
{ type: 'text', text: 'look at this' }
])
})
it('renders a lone image marker turn (no caption) as an image-ref block', () => {
const data = build([user('u1', '[Image: source: /tmp/a.png]')], null, [])
expect(data[0]?.blocks).toEqual([{ type: 'image-ref', path: '/tmp/a.png' }])
@@ -316,8 +316,8 @@ describe('useMobileNativeChatDrafts', () => {
})
expect(state?.pending).toHaveLength(1)
// Claude echoes a captioned image send as two turns: the source marker and
// the caption prefixed with `[Image #1] ` — the pending must still match.
// Claude echoes a captioned image send as a source turn plus a caption
// carrying `[Image #1]`; the pending must still match.
await act(async () =>
renderer?.update(
createElement(Harness, {
@@ -334,6 +334,37 @@ describe('useMobileNativeChatDrafts', () => {
expect(state?.imagePreviewsByMessageId).toEqual({ u2: ['file:///a.jpg'] })
})
it('reconciles a captioned image echo with a trailing [Image #N] marker', async () => {
await mount('a')
await act(async () =>
renderer?.update(
createElement(Harness, { tabId: 'a', messages: [assistantTextMessage('a1', 'hi')] })
)
)
const origin = state?.captureSendOrigin('look at this')
act(() => {
if (origin) {
state?.acceptSend(origin, 'look at this', ['file:///a.jpg'])
}
})
expect(state?.pending).toHaveLength(1)
await act(async () =>
renderer?.update(
createElement(Harness, {
tabId: 'a',
messages: [
assistantTextMessage('a1', 'hi'),
userTextMessage('u1', '[Image: source: /tmp/a.png]'),
userTextMessage('u2', 'look at this[Image #1]')
]
})
)
)
expect(state?.pending).toEqual([])
expect(state?.imagePreviewsByMessageId).toEqual({ u2: ['file:///a.jpg'] })
})
it('hands a marker-only image preview to the authoritative user bubble', async () => {
await mount('a')
const origin = state?.captureSendOrigin('')
@@ -7,6 +7,7 @@ import {
findLandedUnconfirmedSends,
mergeLandedImagePreviewEchoes,
migrateImagePreviewMessageIds,
normalizeReconcileText,
normalizedUserText,
type UnconfirmedSend
} from './mobile-native-chat-draft-reconcile'
@@ -27,8 +28,7 @@ export type { MobileNativeChatPendingMessage, MobileNativeChatSendOrigin }
const NO_PENDING_MESSAGES: MobileNativeChatPendingMessage[] = []
const NO_IMAGE_PREVIEWS: Record<string, string[]> = {}
// How long an ack-lost send waits for its transcript echo before the UI surfaces
// that delivery remains unconfirmed.
// Ack-lost sends wait for a transcript echo before surfacing as unconfirmed.
const UNCONFIRMED_SEND_DEADLINE_MS = 20_000
export function useMobileNativeChatDrafts(args: {
@@ -132,14 +132,13 @@ export function useMobileNativeChatDrafts(args: {
if (!draftKey) {
return null
}
const normalizedText = text.trim()
const currentMessages = messagesRef.current
const normalizedText = normalizeReconcileText(text)
return {
draftKey,
pendingKey,
normalizedText,
baselineOccurrences: countUserTextOccurrences(currentMessages, normalizedText),
baselineTailMessageId: currentMessages[currentMessages.length - 1]?.id ?? null
baselineOccurrences: countUserTextOccurrences(messagesRef.current, normalizedText),
baselineTailMessageId: messagesRef.current.at(-1)?.id ?? null
}
},
[draftKey, pendingKey]
@@ -299,26 +298,19 @@ export function useMobileNativeChatDrafts(args: {
landedCounts.set(text, (landedCounts.get(text) ?? 0) + 1)
}
}
// Why: compare against the count captured before send; historical equal
// turns cannot clear a new echo, while duplicates land one occurrence each.
// An image-only echo has no text to match, so it reconciles by ORDINAL
// against the count of new `[Image: source: …]` echo turns after its
// baseline tail — text echoes are excluded so an unrelated outstanding
// text send cannot clear it. Ordinal-vs-count stays stable when the effect
// re-runs on the shrunken list, and ignores paginated-in history.
// Image-only source-turn counts stay stable across reruns and ignore paginated history.
const next = current.filter((item) => {
if (landedImagePendingIds.has(item.id)) {
return false
}
// Image echoes hand their local URIs to the authoritative message above;
// never drop them through the text-only fallback before that handoff.
// Keep image echoes until their local preview reaches the authoritative message.
if (item.images?.length) {
return true
}
return item.text.trim() === ''
? countImageSourceTurnsAfter(messages, item.baselineTailMessageId) <
item.expectedOccurrence
: (landedCounts.get(item.text.trim()) ?? 0) < item.expectedOccurrence
: (landedCounts.get(normalizeReconcileText(item.text)) ?? 0) < item.expectedOccurrence
})
if (next.length === current.length) {
return previous
@@ -1,9 +1,8 @@
import { stripImagePromptMarker } from '../../../../shared/native-chat-image-transcript-markers'
import {
isImageRefBlock,
isTextBlock,
type NativeChatMessage
} from '../../../../shared/native-chat-types'
normalizeNativeChatUserText,
normalizedNativeChatUserMessageText
} from '../../../../shared/native-chat-image-transcript-markers'
import { isImageRefBlock, type NativeChatMessage } from '../../../../shared/native-chat-types'
export type NativeChatPendingOccurrence = {
text: string
@@ -16,7 +15,7 @@ export type NativeChatPendingOccurrence = {
}
export function normalizeNativeChatPendingText(text: string): string {
return stripImagePromptMarker(text).trim().replace(/\s+/g, ' ')
return normalizeNativeChatUserText(text)
}
export function nativeChatPendingContentKey(
@@ -34,15 +33,15 @@ function nativeChatUserMessageContentKey(message: NativeChatMessage): string | n
if (message.role !== 'user') {
return null
}
const text = message.blocks
.filter(isTextBlock)
.map((block) => block.text)
.join(' ')
const text = normalizedNativeChatUserMessageText(message) ?? ''
if (text) {
return `text:${text}`
}
const imagePaths = message.blocks
.filter(isImageRefBlock)
.map((block) => block.path)
.filter((path): path is string => Boolean(path))
const key = nativeChatPendingContentKey({ text, imagePaths })
const key = nativeChatPendingContentKey({ text: '', imagePaths })
return key === 'empty' ? null : key
}
@@ -80,19 +79,6 @@ export function advancedNativeChatUserContentCounts(
return advanced
}
function nativeChatUserMessageNormalizedText(message: NativeChatMessage): string | null {
if (message.role !== 'user') {
return null
}
const text = normalizeNativeChatPendingText(
message.blocks
.filter(isTextBlock)
.map((block) => block.text)
.join(' ')
)
return text.length > 0 ? text : null
}
/** User texts that already have a later non-user turn (ready to prune echoes). */
export function advancedNativeChatUserTexts(
messages: readonly NativeChatMessage[]
@@ -101,7 +87,7 @@ export function advancedNativeChatUserTexts(
const waiting: string[] = []
for (const message of messages) {
if (message.role === 'user') {
const text = nativeChatUserMessageNormalizedText(message)
const text = normalizedNativeChatUserMessageText(message)
if (text) {
waiting.push(text)
}
@@ -119,7 +105,7 @@ export function matchingNativeChatUserTexts(
): readonly string[] {
const texts: string[] = []
for (const message of messages) {
const text = nativeChatUserMessageNormalizedText(message)
const text = normalizedNativeChatUserMessageText(message)
if (text) {
texts.push(text)
}
@@ -95,6 +95,34 @@ describe('prunePendingSends', () => {
expect(next).toEqual([])
})
it('drops an attachment pending send once a trailing-marker prompt advances', () => {
const pending = [
{ ...pendingOf('p1', 'what do you see'), imagePaths: ['/Users/me/Downloads/3d.png'] }
]
const next = prunePendingSends(pending, [
userMessage('m1', 'what do you see[Image #1]'),
assistantMessage('m2', 'an image')
])
expect(next).toEqual([])
})
it('drops a pending send represented by multiple marker-bearing text blocks', () => {
const prompt: NativeChatMessage = {
...userMessage('m1', 'unused'),
blocks: [
{ type: 'text', text: 'what do' },
{ type: 'image-ref', path: '/tmp/a.png' },
{ type: 'text', text: '[Image #1] you see' }
]
}
const next = prunePendingSends(
[pendingOf('p1', 'what do you see')],
[prompt, assistantMessage('m2', 'an image')]
)
expect(next).toEqual([])
})
it('drops an attachment-only pending send once its image turn advances', () => {
const pending = [{ ...pendingOf('p1', ''), imagePaths: ['/tmp/first.png', '/tmp/second.png'] }]
const transcript = [
@@ -74,6 +74,29 @@ describe('preassembled native-chat live sessions', () => {
expect(out[0]).toMatchObject({ id: 'image-prompt', source: 'transcript' })
})
it('keeps legacy parity for trailing image prompt markers', () => {
const transcript: NativeChatMessage[] = [
message('image-source', {
role: 'user',
blocks: [{ type: 'text', text: '[Image: source: /tmp/a.png]' }],
timestamp: 1
}),
message('image-prompt', {
role: 'user',
blocks: [{ type: 'text', text: 'describe this[Image #1]' }],
timestamp: 2
})
]
const out = expectLegacyMessageParity(transcript)
expect(out).toHaveLength(1)
expect(out[0]?.blocks).toEqual([
{ type: 'image-ref', path: '/tmp/a.png' },
{ type: 'text', text: 'describe this' }
])
})
it('re-dedupes surfaced skill envelopes that collide across sources', () => {
const skill = (id: string, plugin: string, source: 'transcript' | 'scrape') =>
message(id, {
@@ -112,6 +112,34 @@ describe('assembleNativeChatSession', () => {
])
})
it('merges Claude image source markers into a trailing-marker prompt', () => {
const imageSource = msg({
id: 'u-image-source',
role: 'user',
timestamp: 100,
blocks: [{ type: 'text', text: '[Image: source: /Users/me/Downloads/3d.png]' }]
})
const prompt = msg({
id: 'u-prompt',
role: 'user',
timestamp: 101,
blocks: [{ type: 'text', text: 'what do you see[Image #1]' }]
})
const session = assembleNativeChatSession({
sources: { transcript: [imageSource, prompt] },
sessionId: 's1',
agent: 'claude'
})
expect(session.messages).toHaveLength(1)
expect(session.messages[0]).toMatchObject({ id: 'u-prompt', role: 'user' })
expect(session.messages[0].blocks).toEqual([
{ type: 'image-ref', path: '/Users/me/Downloads/3d.png' },
{ type: 'text', text: 'what do you see' }
])
})
it('drops a scrape duplicate even when scrape is processed first by id', () => {
const scrape = msg({
id: 'shared-id',
@@ -1,6 +1,12 @@
import { describe, expect, it } from 'vitest'
import type { NativeChatMessage } from './native-chat-types'
import { normalizeImageTranscriptMessages } from './native-chat-image-transcript-markers'
import {
isImageSourceUserTurn,
normalizeImageTranscriptMessages,
normalizeNativeChatUserText,
normalizedNativeChatUserMessageText,
stripImagePromptMarker
} from './native-chat-image-transcript-markers'
function userText(id: string, text: string): NativeChatMessage {
return {
@@ -25,6 +31,91 @@ describe('normalizeImageTranscriptMessages', () => {
])
})
it('merges a source turn into a prompt with a trailing image marker', () => {
const out = normalizeImageTranscriptMessages([
userText('a', '[Image: source: /tmp/orca-paste-1-2.png]'),
userText('b', 'describe this[Image #1]')
])
expect(out).toHaveLength(1)
expect(out[0]!.blocks).toEqual([
{ type: 'image-ref', path: '/tmp/orca-paste-1-2.png' },
{ type: 'text', text: 'describe this' }
])
})
it('folds and strips markers in later text blocks', () => {
const prompt: NativeChatMessage = {
...userText('prompt', 'unused'),
blocks: [
{ type: 'text', text: 'describe' },
{ type: 'image-ref', path: '/tmp/existing.png' },
{ type: 'text', text: '[Image #1] this' }
]
}
const out = normalizeImageTranscriptMessages([
userText('source', '[Image: source: /tmp/a.png]'),
prompt
])
expect(out).toHaveLength(1)
expect(out[0]?.blocks).toEqual([
{ type: 'image-ref', path: '/tmp/a.png' },
{ type: 'text', text: 'describe' },
{ type: 'image-ref', path: '/tmp/existing.png' },
{ type: 'text', text: 'this' }
])
})
it.each([
['[Image #1] describe this', 'describe this'],
['[Image #1]\t describe this', 'describe this'],
[' \t[Image #1] describe this', 'describe this'],
['describe this [Image #1]', 'describe this'],
['describe this \t[Image #1]', 'describe this'],
['describe this [Image #1]\t ', 'describe this'],
['describe [Image #1] this', 'describe this'],
['describe [Image #1]\t this', 'describe \t this'],
['describe[Image #1]\t this', 'describe\t this'],
['describe\n[Image #1]\nthis', 'describe\n\nthis'],
['com[Image #1]pare this', 'compare this'],
['[Image #1] [Image #2]', ''],
['literal [Image #x] text', 'literal [Image #x] text']
])('strips image prompt markers anywhere in text', (text, expected) => {
expect(stripImagePromptMarker(text)).toBe(expected)
})
it('returns long marker-free whitespace without regex backtracking', () => {
const text = ' '.repeat(50_000)
expect(stripImagePromptMarker(text)).toBe(text)
})
it('shares marker-aware text matching across multiple text blocks', () => {
const message: NativeChatMessage = {
...userText('prompt', 'unused'),
blocks: [
{ type: 'text', text: 'look' },
{ type: 'image-ref', path: '/tmp/a.png' },
{ type: 'text', text: '[Image #1] here' }
]
}
expect(normalizeNativeChatUserText(' look [Image #1] here ')).toBe('look here')
expect(normalizedNativeChatUserMessageText(message)).toBe('look here')
})
it('recognizes only sole-text image-source user turns', () => {
const source = userText('source', '[Image: source: /tmp/a.png]')
expect(isImageSourceUserTurn(source)).toBe(true)
expect(isImageSourceUserTurn({ ...source, role: 'assistant' })).toBe(false)
expect(
isImageSourceUserTurn({
...source,
blocks: [...source.blocks, { type: 'text', text: 'caption' }]
})
).toBe(false)
})
it('converts a lone [Image: source] turn (no prompt) into an image-ref instead of raw text', () => {
const out = normalizeImageTranscriptMessages([
userText('a', '[Image: source: /Users/me/Pictures/hero-image-2.png]')
@@ -1,7 +1,12 @@
import { isTextBlock, type NativeChatBlock, type NativeChatMessage } from './native-chat-types'
const IMAGE_SOURCE_MARKER = /^\[Image:\s*source:\s*(.+?)\]\s*$/
const IMAGE_PROMPT_MARKERS = /^(?:\[Image #\d+\]\s*)+/
const IMAGE_PROMPT_MARKER = /\[Image #\d+\]/
const IMAGE_PROMPT_MARKERS = /\[Image #\d+\]/g
const IMAGE_PROMPT_MARKER_AT_START = /^[^\S\r\n]*\[Image #\d+\]/
const IMAGE_PROMPT_MARKER_AT_END = /\[Image #\d+\][^\S\r\n]*$/
const HORIZONTAL_WHITESPACE_START = /^[^\S\r\n]+/
const HORIZONTAL_WHITESPACE_END = /[^\S\r\n]+$/
function soleText(message: NativeChatMessage): string | null {
return message.blocks.length === 1 && isTextBlock(message.blocks[0])
@@ -13,40 +18,75 @@ export function imageSourcePathFromText(text: string): string | null {
return text.match(IMAGE_SOURCE_MARKER)?.[1]?.trim() ?? null
}
export function stripImagePromptMarker(text: string): string {
return text.replace(IMAGE_PROMPT_MARKERS, '')
export function isImageSourceUserTurn(message: NativeChatMessage): boolean {
return message.role === 'user' && imageSourcePathFromText(soleText(message) ?? '') !== null
}
function stripImagePromptMarkersFromFirstText(
export function stripImagePromptMarker(text: string): string {
const stripped = text.replace(IMAGE_PROMPT_MARKERS, '')
if (stripped === text) {
return text
}
let result = IMAGE_PROMPT_MARKER_AT_START.test(text)
? stripped.replace(HORIZONTAL_WHITESPACE_START, '')
: stripped
if (IMAGE_PROMPT_MARKER_AT_END.test(text)) {
result = result.replace(HORIZONTAL_WHITESPACE_END, '')
}
return result
}
export function normalizeNativeChatUserText(text: string): string {
return stripImagePromptMarker(text).trim().replace(/\s+/g, ' ')
}
export function normalizedNativeChatUserMessageText(message: NativeChatMessage): string | null {
if (message.role !== 'user') {
return null
}
const normalized = normalizeNativeChatUserText(
message.blocks
.filter(isTextBlock)
.map((block) => block.text)
.join(' ')
)
return normalized || null
}
function stripImagePromptMarkersFromTextBlocks(
blocks: readonly NativeChatBlock[]
): NativeChatBlock[] {
const textIndex = blocks.findIndex(isTextBlock)
if (textIndex === -1) {
return blocks as NativeChatBlock[]
let sawText = false
let next: NativeChatBlock[] | null = null
for (let index = 0; index < blocks.length; index += 1) {
const block = blocks[index]!
if (!isTextBlock(block)) {
next?.push(block)
continue
}
const isFirstText = !sawText
sawText = true
const text = stripImagePromptMarker(block.text)
if (!text.trim() && (text !== block.text || isFirstText)) {
next ??= blocks.slice(0, index)
continue
}
if (text !== block.text) {
next ??= blocks.slice(0, index)
next.push({ ...block, text })
continue
}
next?.push(block)
}
const block = blocks[textIndex]
if (!block || !isTextBlock(block)) {
return blocks as NativeChatBlock[]
}
const text = stripImagePromptMarker(block.text)
if (text.trim().length === 0) {
return blocks.filter((_, index) => index !== textIndex)
}
if (text === block.text) {
return blocks as NativeChatBlock[]
}
const next = [...blocks]
next[textIndex] = { ...block, text }
return next
return next ?? (blocks as NativeChatBlock[])
}
function imagePromptMarkerStartsMessage(message: NativeChatMessage): boolean {
const firstText = message.blocks.find(isTextBlock)
return firstText ? IMAGE_PROMPT_MARKERS.test(firstText.text) : false
export function hasImagePromptMarker(message: NativeChatMessage): boolean {
return message.blocks.some((block) => isTextBlock(block) && IMAGE_PROMPT_MARKER.test(block.text))
}
/** Claude records image paths as source turns followed by one marker-prefixed
* prompt. Merge the whole run back into one native user turn. */
/** Claude records image paths as source turns followed by a prompt carrying
* image markers. Merge the whole run back into one native user turn. */
export function normalizeImageTranscriptMessages(
messages: readonly NativeChatMessage[]
): NativeChatMessage[] {
@@ -75,13 +115,13 @@ export function normalizeImageTranscriptMessages(
if (
prompt?.role === 'user' &&
prompt.source === message.source &&
imagePromptMarkerStartsMessage(prompt)
hasImagePromptMarker(prompt)
) {
normalized.push({
...prompt,
blocks: [
...imagePaths.map((path) => ({ type: 'image-ref' as const, path })),
...stripImagePromptMarkersFromFirstText(prompt.blocks)
...stripImagePromptMarkersFromTextBlocks(prompt.blocks)
]
})
index = nextIndex
@@ -93,7 +133,7 @@ export function normalizeImageTranscriptMessages(
})
continue
}
const blocks = stripImagePromptMarkersFromFirstText(message.blocks)
const blocks = stripImagePromptMarkersFromTextBlocks(message.blocks)
if (blocks === message.blocks) {
normalized?.push(message)
} else {