mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 00:02:31 +00:00
Fix native chat image marker position handling (#14162)
* fix(native-chat): handle image markers in any position * fix(native-chat): preserve image caption whitespace * test(native-chat): cover marker boundary spacing * fix(mobile): normalize image echo reconciliation * fix(mobile): use idiomatic tail access * refactor(native-chat): share image echo matching * perf(native-chat): avoid unchanged block copies
This commit is contained in:
@@ -2,8 +2,10 @@ import { describe, expect, it } from 'vitest'
|
||||
import type { NativeChatMessage } from '../../../src/shared/native-chat-types'
|
||||
import {
|
||||
findLandedImagePreviewEchoes,
|
||||
findLandedUnconfirmedSends,
|
||||
migrateImagePreviewMessageIds,
|
||||
type PendingImagePreviewEcho
|
||||
type PendingImagePreviewEcho,
|
||||
type UnconfirmedSend
|
||||
} from './mobile-native-chat-draft-reconcile'
|
||||
|
||||
function userText(id: string, text: string): NativeChatMessage {
|
||||
@@ -21,6 +23,67 @@ function pending(id: string, images: string[], expectedOccurrence = 1): PendingI
|
||||
}
|
||||
|
||||
describe('mobile native chat image preview reconciliation', () => {
|
||||
it('reconciles a trailing-marker echo and hands its preview to that echo', () => {
|
||||
const messages = [
|
||||
userText('source', '[Image: source: /tmp/a.png]'),
|
||||
userText('prompt', 'look at this[Image #1]')
|
||||
]
|
||||
const preview = {
|
||||
...pending('pending', ['file:///a.jpg']),
|
||||
text: 'look at this'
|
||||
}
|
||||
const unconfirmed: UnconfirmedSend = {
|
||||
draftKey: 'draft',
|
||||
pendingKey: 'pending-key',
|
||||
text: 'look at this',
|
||||
normalizedText: 'look at this',
|
||||
baselineTailMessageId: null,
|
||||
deadline: null
|
||||
}
|
||||
|
||||
expect(findLandedUnconfirmedSends(messages, [unconfirmed])).toEqual([unconfirmed])
|
||||
expect(findLandedImagePreviewEchoes(messages, [preview])).toEqual([
|
||||
{ pendingId: 'pending', messageId: 'prompt', images: ['file:///a.jpg'] }
|
||||
])
|
||||
})
|
||||
|
||||
it('reconciles a middle-marker echo without changing its rendered whitespace', () => {
|
||||
const messages = [
|
||||
userText('source', '[Image: source: /tmp/a.png]'),
|
||||
userText('prompt', 'look [Image #1] here')
|
||||
]
|
||||
const preview = { ...pending('pending', ['file:///a.jpg']), text: 'look here' }
|
||||
const unconfirmed: UnconfirmedSend = {
|
||||
draftKey: 'draft',
|
||||
pendingKey: 'pending-key',
|
||||
text: 'look here',
|
||||
normalizedText: 'look here',
|
||||
baselineTailMessageId: null,
|
||||
deadline: null
|
||||
}
|
||||
|
||||
expect(findLandedUnconfirmedSends(messages, [unconfirmed])).toEqual([unconfirmed])
|
||||
expect(findLandedImagePreviewEchoes(messages, [preview])).toEqual([
|
||||
{ pendingId: 'pending', messageId: 'prompt', images: ['file:///a.jpg'] }
|
||||
])
|
||||
})
|
||||
|
||||
it('reconciles multiple transcript text blocks with desktop separators', () => {
|
||||
const prompt: NativeChatMessage = {
|
||||
...userText('prompt', 'unused'),
|
||||
blocks: [
|
||||
{ type: 'text', text: 'look' },
|
||||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||||
{ type: 'text', text: '[Image #1] here' }
|
||||
]
|
||||
}
|
||||
const preview = { ...pending('pending', ['file:///a.jpg']), text: 'look here' }
|
||||
|
||||
expect(findLandedImagePreviewEchoes([prompt], [preview])).toEqual([
|
||||
{ pendingId: 'pending', messageId: 'prompt', images: ['file:///a.jpg'] }
|
||||
])
|
||||
})
|
||||
|
||||
it('keeps separate adjacent image-only sends independently reconcilable', () => {
|
||||
const landed = findLandedImagePreviewEchoes(
|
||||
[
|
||||
@@ -69,4 +132,36 @@ describe('mobile native chat image preview reconciliation', () => {
|
||||
[sessionKey]: { prompt: ['file:///a.jpg'] }
|
||||
})
|
||||
})
|
||||
|
||||
it('moves an early standalone preview to a trailing-marker prompt id', () => {
|
||||
const sessionKey = 'host\0worktree\0tab\0session'
|
||||
const previous = { [sessionKey]: { source: ['file:///a.jpg'] } }
|
||||
const messages = [
|
||||
userText('source', '[Image: source: /tmp/a.png]'),
|
||||
userText('prompt', 'look[Image #1]')
|
||||
]
|
||||
|
||||
expect(migrateImagePreviewMessageIds(previous, sessionKey, messages)).toEqual({
|
||||
[sessionKey]: { prompt: ['file:///a.jpg'] }
|
||||
})
|
||||
})
|
||||
|
||||
it('moves a preview when the prompt marker is in a later text block', () => {
|
||||
const sessionKey = 'host\0worktree\0tab\0session'
|
||||
const previous = { [sessionKey]: { source: ['file:///a.jpg'] } }
|
||||
const prompt: NativeChatMessage = {
|
||||
...userText('prompt', 'unused'),
|
||||
blocks: [
|
||||
{ type: 'text', text: 'look' },
|
||||
{ type: 'text', text: '[Image #1] here' }
|
||||
]
|
||||
}
|
||||
|
||||
expect(
|
||||
migrateImagePreviewMessageIds(previous, sessionKey, [
|
||||
userText('source', '[Image: source: /tmp/a.png]'),
|
||||
prompt
|
||||
])
|
||||
).toEqual({ [sessionKey]: { prompt: ['file:///a.jpg'] } })
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
import { isImageRefBlock, type NativeChatMessage } from '../../../src/shared/native-chat-types'
|
||||
import {
|
||||
hasImagePromptMarker,
|
||||
isImageSourceUserTurn,
|
||||
normalizeImageTranscriptMessages,
|
||||
stripImagePromptMarker
|
||||
normalizeNativeChatUserText,
|
||||
normalizedNativeChatUserMessageText
|
||||
} from './mobile-native-chat-image-transcript-markers'
|
||||
export { normalizeNativeChatUserText as normalizeReconcileText } from './mobile-native-chat-image-transcript-markers'
|
||||
|
||||
/** An ack-lost ('unknown' outcome) send held until its transcript echo lands or
|
||||
* the deadline surfaces the uncertainty. */
|
||||
@@ -17,17 +20,7 @@ export type UnconfirmedSend = {
|
||||
}
|
||||
|
||||
export function normalizedUserText(message: NativeChatMessage): string | null {
|
||||
if (message.role !== 'user') {
|
||||
return null
|
||||
}
|
||||
const text = message.blocks
|
||||
.filter((block) => block.type === 'text')
|
||||
.map((block) => (block.type === 'text' ? block.text : ''))
|
||||
.join('')
|
||||
// Claude echoes a captioned image send as `[Image #1] caption` — the sent
|
||||
// text must still match its echo, so strip the marker before comparing.
|
||||
const stripped = stripImagePromptMarker(text).trim()
|
||||
return stripped || null
|
||||
return normalizedNativeChatUserMessageText(message)
|
||||
}
|
||||
|
||||
export function countUserTextOccurrences(
|
||||
@@ -117,17 +110,13 @@ function imagePreviewReplacementMessageId(
|
||||
nextIndex++
|
||||
}
|
||||
const prompt = messages[nextIndex]
|
||||
const firstText = prompt?.blocks.find((block) => block.type === 'text')
|
||||
return prompt?.role === 'user' &&
|
||||
prompt.source === source.source &&
|
||||
firstText?.type === 'text' &&
|
||||
stripImagePromptMarker(firstText.text) !== firstText.text
|
||||
return prompt?.role === 'user' && prompt.source === source.source && hasImagePromptMarker(prompt)
|
||||
? prompt.id
|
||||
: null
|
||||
}
|
||||
|
||||
/** Moves previews forward when a progressive source-only transcript frame later
|
||||
* folds into the marker-prefixed prompt with a different authoritative id. */
|
||||
* folds into the marker-bearing prompt with a different authoritative id. */
|
||||
export function migrateImagePreviewMessageIds(
|
||||
previous: Record<string, Record<string, string[]>>,
|
||||
sessionKey: string,
|
||||
@@ -171,7 +160,7 @@ export function findLandedImagePreviewEchoes(
|
||||
if (!entry.images?.length) {
|
||||
continue
|
||||
}
|
||||
const targetText = entry.text.trim()
|
||||
const targetText = normalizeNativeChatUserText(entry.text)
|
||||
const candidates = normalized.filter((message) => {
|
||||
if (message.role !== 'user') {
|
||||
return false
|
||||
|
||||
@@ -1,21 +1,13 @@
|
||||
// Single-sources the marker logic (pure functions over shared types):
|
||||
// Claude records an attached image as `[Image: source: /path]` (+ `[Image #N]`
|
||||
// prefix on the caption turn), and both render and echo reconciliation must
|
||||
// on the caption turn), and both render and echo reconciliation must
|
||||
// agree with desktop on how those marker turns are interpreted.
|
||||
export {
|
||||
imageSourcePathFromText,
|
||||
hasImagePromptMarker,
|
||||
isImageSourceUserTurn,
|
||||
normalizeImageTranscriptMessages,
|
||||
normalizeNativeChatUserText,
|
||||
normalizedNativeChatUserMessageText,
|
||||
stripImagePromptMarker
|
||||
} from '../../../src/shared/native-chat-image-transcript-markers'
|
||||
import { imageSourcePathFromText } from '../../../src/shared/native-chat-image-transcript-markers'
|
||||
import { isTextBlock, type NativeChatMessage } from '../../../src/shared/native-chat-types'
|
||||
|
||||
/** A raw (un-normalized) transcript user turn that is an image-source marker —
|
||||
* the echo shape of an image riding along on a send. */
|
||||
export function isImageSourceUserTurn(message: NativeChatMessage): boolean {
|
||||
if (message.role !== 'user' || message.blocks.length !== 1) {
|
||||
return false
|
||||
}
|
||||
const block = message.blocks[0]
|
||||
return block !== undefined && isTextBlock(block) && imageSourcePathFromText(block.text) !== null
|
||||
}
|
||||
|
||||
@@ -92,8 +92,8 @@ describe('buildMobileNativeChatTransientData', () => {
|
||||
})
|
||||
|
||||
it('folds transcript image marker turns into image-ref blocks (desktop parity)', () => {
|
||||
// Claude records an attached image as `[Image: source: /path]` + an
|
||||
// `[Image #1] `-prefixed caption turn; the fold must merge them into one
|
||||
// Claude records an attached image as `[Image: source: /path]` plus a
|
||||
// caption turn carrying `[Image #1]`; the fold must merge them into one
|
||||
// user turn with an image-ref block instead of showing raw marker text.
|
||||
const data = build(
|
||||
[
|
||||
@@ -111,6 +111,20 @@ describe('buildMobileNativeChatTransientData', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('folds a trailing-marker image echo into one user bubble', () => {
|
||||
const data = build(
|
||||
[user('u1', '[Image: source: /tmp/a.png]'), user('u2', 'look at this[Image #1]')],
|
||||
null,
|
||||
[]
|
||||
)
|
||||
|
||||
expect(data).toHaveLength(1)
|
||||
expect(data[0]?.blocks).toEqual([
|
||||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||||
{ type: 'text', text: 'look at this' }
|
||||
])
|
||||
})
|
||||
|
||||
it('renders a lone image marker turn (no caption) as an image-ref block', () => {
|
||||
const data = build([user('u1', '[Image: source: /tmp/a.png]')], null, [])
|
||||
expect(data[0]?.blocks).toEqual([{ type: 'image-ref', path: '/tmp/a.png' }])
|
||||
|
||||
@@ -316,8 +316,8 @@ describe('useMobileNativeChatDrafts', () => {
|
||||
})
|
||||
expect(state?.pending).toHaveLength(1)
|
||||
|
||||
// Claude echoes a captioned image send as two turns: the source marker and
|
||||
// the caption prefixed with `[Image #1] ` — the pending must still match.
|
||||
// Claude echoes a captioned image send as a source turn plus a caption
|
||||
// carrying `[Image #1]`; the pending must still match.
|
||||
await act(async () =>
|
||||
renderer?.update(
|
||||
createElement(Harness, {
|
||||
@@ -334,6 +334,37 @@ describe('useMobileNativeChatDrafts', () => {
|
||||
expect(state?.imagePreviewsByMessageId).toEqual({ u2: ['file:///a.jpg'] })
|
||||
})
|
||||
|
||||
it('reconciles a captioned image echo with a trailing [Image #N] marker', async () => {
|
||||
await mount('a')
|
||||
await act(async () =>
|
||||
renderer?.update(
|
||||
createElement(Harness, { tabId: 'a', messages: [assistantTextMessage('a1', 'hi')] })
|
||||
)
|
||||
)
|
||||
const origin = state?.captureSendOrigin('look at this')
|
||||
act(() => {
|
||||
if (origin) {
|
||||
state?.acceptSend(origin, 'look at this', ['file:///a.jpg'])
|
||||
}
|
||||
})
|
||||
expect(state?.pending).toHaveLength(1)
|
||||
|
||||
await act(async () =>
|
||||
renderer?.update(
|
||||
createElement(Harness, {
|
||||
tabId: 'a',
|
||||
messages: [
|
||||
assistantTextMessage('a1', 'hi'),
|
||||
userTextMessage('u1', '[Image: source: /tmp/a.png]'),
|
||||
userTextMessage('u2', 'look at this[Image #1]')
|
||||
]
|
||||
})
|
||||
)
|
||||
)
|
||||
expect(state?.pending).toEqual([])
|
||||
expect(state?.imagePreviewsByMessageId).toEqual({ u2: ['file:///a.jpg'] })
|
||||
})
|
||||
|
||||
it('hands a marker-only image preview to the authoritative user bubble', async () => {
|
||||
await mount('a')
|
||||
const origin = state?.captureSendOrigin('')
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
findLandedUnconfirmedSends,
|
||||
mergeLandedImagePreviewEchoes,
|
||||
migrateImagePreviewMessageIds,
|
||||
normalizeReconcileText,
|
||||
normalizedUserText,
|
||||
type UnconfirmedSend
|
||||
} from './mobile-native-chat-draft-reconcile'
|
||||
@@ -27,8 +28,7 @@ export type { MobileNativeChatPendingMessage, MobileNativeChatSendOrigin }
|
||||
const NO_PENDING_MESSAGES: MobileNativeChatPendingMessage[] = []
|
||||
const NO_IMAGE_PREVIEWS: Record<string, string[]> = {}
|
||||
|
||||
// How long an ack-lost send waits for its transcript echo before the UI surfaces
|
||||
// that delivery remains unconfirmed.
|
||||
// Ack-lost sends wait for a transcript echo before surfacing as unconfirmed.
|
||||
const UNCONFIRMED_SEND_DEADLINE_MS = 20_000
|
||||
|
||||
export function useMobileNativeChatDrafts(args: {
|
||||
@@ -132,14 +132,13 @@ export function useMobileNativeChatDrafts(args: {
|
||||
if (!draftKey) {
|
||||
return null
|
||||
}
|
||||
const normalizedText = text.trim()
|
||||
const currentMessages = messagesRef.current
|
||||
const normalizedText = normalizeReconcileText(text)
|
||||
return {
|
||||
draftKey,
|
||||
pendingKey,
|
||||
normalizedText,
|
||||
baselineOccurrences: countUserTextOccurrences(currentMessages, normalizedText),
|
||||
baselineTailMessageId: currentMessages[currentMessages.length - 1]?.id ?? null
|
||||
baselineOccurrences: countUserTextOccurrences(messagesRef.current, normalizedText),
|
||||
baselineTailMessageId: messagesRef.current.at(-1)?.id ?? null
|
||||
}
|
||||
},
|
||||
[draftKey, pendingKey]
|
||||
@@ -299,26 +298,19 @@ export function useMobileNativeChatDrafts(args: {
|
||||
landedCounts.set(text, (landedCounts.get(text) ?? 0) + 1)
|
||||
}
|
||||
}
|
||||
// Why: compare against the count captured before send; historical equal
|
||||
// turns cannot clear a new echo, while duplicates land one occurrence each.
|
||||
// An image-only echo has no text to match, so it reconciles by ORDINAL
|
||||
// against the count of new `[Image: source: …]` echo turns after its
|
||||
// baseline tail — text echoes are excluded so an unrelated outstanding
|
||||
// text send cannot clear it. Ordinal-vs-count stays stable when the effect
|
||||
// re-runs on the shrunken list, and ignores paginated-in history.
|
||||
// Image-only source-turn counts stay stable across reruns and ignore paginated history.
|
||||
const next = current.filter((item) => {
|
||||
if (landedImagePendingIds.has(item.id)) {
|
||||
return false
|
||||
}
|
||||
// Image echoes hand their local URIs to the authoritative message above;
|
||||
// never drop them through the text-only fallback before that handoff.
|
||||
// Keep image echoes until their local preview reaches the authoritative message.
|
||||
if (item.images?.length) {
|
||||
return true
|
||||
}
|
||||
return item.text.trim() === ''
|
||||
? countImageSourceTurnsAfter(messages, item.baselineTailMessageId) <
|
||||
item.expectedOccurrence
|
||||
: (landedCounts.get(item.text.trim()) ?? 0) < item.expectedOccurrence
|
||||
: (landedCounts.get(normalizeReconcileText(item.text)) ?? 0) < item.expectedOccurrence
|
||||
})
|
||||
if (next.length === current.length) {
|
||||
return previous
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
import { stripImagePromptMarker } from '../../../../shared/native-chat-image-transcript-markers'
|
||||
import {
|
||||
isImageRefBlock,
|
||||
isTextBlock,
|
||||
type NativeChatMessage
|
||||
} from '../../../../shared/native-chat-types'
|
||||
normalizeNativeChatUserText,
|
||||
normalizedNativeChatUserMessageText
|
||||
} from '../../../../shared/native-chat-image-transcript-markers'
|
||||
import { isImageRefBlock, type NativeChatMessage } from '../../../../shared/native-chat-types'
|
||||
|
||||
export type NativeChatPendingOccurrence = {
|
||||
text: string
|
||||
@@ -16,7 +15,7 @@ export type NativeChatPendingOccurrence = {
|
||||
}
|
||||
|
||||
export function normalizeNativeChatPendingText(text: string): string {
|
||||
return stripImagePromptMarker(text).trim().replace(/\s+/g, ' ')
|
||||
return normalizeNativeChatUserText(text)
|
||||
}
|
||||
|
||||
export function nativeChatPendingContentKey(
|
||||
@@ -34,15 +33,15 @@ function nativeChatUserMessageContentKey(message: NativeChatMessage): string | n
|
||||
if (message.role !== 'user') {
|
||||
return null
|
||||
}
|
||||
const text = message.blocks
|
||||
.filter(isTextBlock)
|
||||
.map((block) => block.text)
|
||||
.join(' ')
|
||||
const text = normalizedNativeChatUserMessageText(message) ?? ''
|
||||
if (text) {
|
||||
return `text:${text}`
|
||||
}
|
||||
const imagePaths = message.blocks
|
||||
.filter(isImageRefBlock)
|
||||
.map((block) => block.path)
|
||||
.filter((path): path is string => Boolean(path))
|
||||
const key = nativeChatPendingContentKey({ text, imagePaths })
|
||||
const key = nativeChatPendingContentKey({ text: '', imagePaths })
|
||||
return key === 'empty' ? null : key
|
||||
}
|
||||
|
||||
@@ -80,19 +79,6 @@ export function advancedNativeChatUserContentCounts(
|
||||
return advanced
|
||||
}
|
||||
|
||||
function nativeChatUserMessageNormalizedText(message: NativeChatMessage): string | null {
|
||||
if (message.role !== 'user') {
|
||||
return null
|
||||
}
|
||||
const text = normalizeNativeChatPendingText(
|
||||
message.blocks
|
||||
.filter(isTextBlock)
|
||||
.map((block) => block.text)
|
||||
.join(' ')
|
||||
)
|
||||
return text.length > 0 ? text : null
|
||||
}
|
||||
|
||||
/** User texts that already have a later non-user turn (ready to prune echoes). */
|
||||
export function advancedNativeChatUserTexts(
|
||||
messages: readonly NativeChatMessage[]
|
||||
@@ -101,7 +87,7 @@ export function advancedNativeChatUserTexts(
|
||||
const waiting: string[] = []
|
||||
for (const message of messages) {
|
||||
if (message.role === 'user') {
|
||||
const text = nativeChatUserMessageNormalizedText(message)
|
||||
const text = normalizedNativeChatUserMessageText(message)
|
||||
if (text) {
|
||||
waiting.push(text)
|
||||
}
|
||||
@@ -119,7 +105,7 @@ export function matchingNativeChatUserTexts(
|
||||
): readonly string[] {
|
||||
const texts: string[] = []
|
||||
for (const message of messages) {
|
||||
const text = nativeChatUserMessageNormalizedText(message)
|
||||
const text = normalizedNativeChatUserMessageText(message)
|
||||
if (text) {
|
||||
texts.push(text)
|
||||
}
|
||||
|
||||
@@ -95,6 +95,34 @@ describe('prunePendingSends', () => {
|
||||
expect(next).toEqual([])
|
||||
})
|
||||
|
||||
it('drops an attachment pending send once a trailing-marker prompt advances', () => {
|
||||
const pending = [
|
||||
{ ...pendingOf('p1', 'what do you see'), imagePaths: ['/Users/me/Downloads/3d.png'] }
|
||||
]
|
||||
const next = prunePendingSends(pending, [
|
||||
userMessage('m1', 'what do you see[Image #1]'),
|
||||
assistantMessage('m2', 'an image')
|
||||
])
|
||||
expect(next).toEqual([])
|
||||
})
|
||||
|
||||
it('drops a pending send represented by multiple marker-bearing text blocks', () => {
|
||||
const prompt: NativeChatMessage = {
|
||||
...userMessage('m1', 'unused'),
|
||||
blocks: [
|
||||
{ type: 'text', text: 'what do' },
|
||||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||||
{ type: 'text', text: '[Image #1] you see' }
|
||||
]
|
||||
}
|
||||
const next = prunePendingSends(
|
||||
[pendingOf('p1', 'what do you see')],
|
||||
[prompt, assistantMessage('m2', 'an image')]
|
||||
)
|
||||
|
||||
expect(next).toEqual([])
|
||||
})
|
||||
|
||||
it('drops an attachment-only pending send once its image turn advances', () => {
|
||||
const pending = [{ ...pendingOf('p1', ''), imagePaths: ['/tmp/first.png', '/tmp/second.png'] }]
|
||||
const transcript = [
|
||||
|
||||
+23
@@ -74,6 +74,29 @@ describe('preassembled native-chat live sessions', () => {
|
||||
expect(out[0]).toMatchObject({ id: 'image-prompt', source: 'transcript' })
|
||||
})
|
||||
|
||||
it('keeps legacy parity for trailing image prompt markers', () => {
|
||||
const transcript: NativeChatMessage[] = [
|
||||
message('image-source', {
|
||||
role: 'user',
|
||||
blocks: [{ type: 'text', text: '[Image: source: /tmp/a.png]' }],
|
||||
timestamp: 1
|
||||
}),
|
||||
message('image-prompt', {
|
||||
role: 'user',
|
||||
blocks: [{ type: 'text', text: 'describe this[Image #1]' }],
|
||||
timestamp: 2
|
||||
})
|
||||
]
|
||||
|
||||
const out = expectLegacyMessageParity(transcript)
|
||||
|
||||
expect(out).toHaveLength(1)
|
||||
expect(out[0]?.blocks).toEqual([
|
||||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||||
{ type: 'text', text: 'describe this' }
|
||||
])
|
||||
})
|
||||
|
||||
it('re-dedupes surfaced skill envelopes that collide across sources', () => {
|
||||
const skill = (id: string, plugin: string, source: 'transcript' | 'scrape') =>
|
||||
message(id, {
|
||||
|
||||
@@ -112,6 +112,34 @@ describe('assembleNativeChatSession', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('merges Claude image source markers into a trailing-marker prompt', () => {
|
||||
const imageSource = msg({
|
||||
id: 'u-image-source',
|
||||
role: 'user',
|
||||
timestamp: 100,
|
||||
blocks: [{ type: 'text', text: '[Image: source: /Users/me/Downloads/3d.png]' }]
|
||||
})
|
||||
const prompt = msg({
|
||||
id: 'u-prompt',
|
||||
role: 'user',
|
||||
timestamp: 101,
|
||||
blocks: [{ type: 'text', text: 'what do you see[Image #1]' }]
|
||||
})
|
||||
|
||||
const session = assembleNativeChatSession({
|
||||
sources: { transcript: [imageSource, prompt] },
|
||||
sessionId: 's1',
|
||||
agent: 'claude'
|
||||
})
|
||||
|
||||
expect(session.messages).toHaveLength(1)
|
||||
expect(session.messages[0]).toMatchObject({ id: 'u-prompt', role: 'user' })
|
||||
expect(session.messages[0].blocks).toEqual([
|
||||
{ type: 'image-ref', path: '/Users/me/Downloads/3d.png' },
|
||||
{ type: 'text', text: 'what do you see' }
|
||||
])
|
||||
})
|
||||
|
||||
it('drops a scrape duplicate even when scrape is processed first by id', () => {
|
||||
const scrape = msg({
|
||||
id: 'shared-id',
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { NativeChatMessage } from './native-chat-types'
|
||||
import { normalizeImageTranscriptMessages } from './native-chat-image-transcript-markers'
|
||||
import {
|
||||
isImageSourceUserTurn,
|
||||
normalizeImageTranscriptMessages,
|
||||
normalizeNativeChatUserText,
|
||||
normalizedNativeChatUserMessageText,
|
||||
stripImagePromptMarker
|
||||
} from './native-chat-image-transcript-markers'
|
||||
|
||||
function userText(id: string, text: string): NativeChatMessage {
|
||||
return {
|
||||
@@ -25,6 +31,91 @@ describe('normalizeImageTranscriptMessages', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('merges a source turn into a prompt with a trailing image marker', () => {
|
||||
const out = normalizeImageTranscriptMessages([
|
||||
userText('a', '[Image: source: /tmp/orca-paste-1-2.png]'),
|
||||
userText('b', 'describe this[Image #1]')
|
||||
])
|
||||
|
||||
expect(out).toHaveLength(1)
|
||||
expect(out[0]!.blocks).toEqual([
|
||||
{ type: 'image-ref', path: '/tmp/orca-paste-1-2.png' },
|
||||
{ type: 'text', text: 'describe this' }
|
||||
])
|
||||
})
|
||||
|
||||
it('folds and strips markers in later text blocks', () => {
|
||||
const prompt: NativeChatMessage = {
|
||||
...userText('prompt', 'unused'),
|
||||
blocks: [
|
||||
{ type: 'text', text: 'describe' },
|
||||
{ type: 'image-ref', path: '/tmp/existing.png' },
|
||||
{ type: 'text', text: '[Image #1] this' }
|
||||
]
|
||||
}
|
||||
const out = normalizeImageTranscriptMessages([
|
||||
userText('source', '[Image: source: /tmp/a.png]'),
|
||||
prompt
|
||||
])
|
||||
|
||||
expect(out).toHaveLength(1)
|
||||
expect(out[0]?.blocks).toEqual([
|
||||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||||
{ type: 'text', text: 'describe' },
|
||||
{ type: 'image-ref', path: '/tmp/existing.png' },
|
||||
{ type: 'text', text: 'this' }
|
||||
])
|
||||
})
|
||||
|
||||
it.each([
|
||||
['[Image #1] describe this', 'describe this'],
|
||||
['[Image #1]\t describe this', 'describe this'],
|
||||
[' \t[Image #1] describe this', 'describe this'],
|
||||
['describe this [Image #1]', 'describe this'],
|
||||
['describe this \t[Image #1]', 'describe this'],
|
||||
['describe this [Image #1]\t ', 'describe this'],
|
||||
['describe [Image #1] this', 'describe this'],
|
||||
['describe [Image #1]\t this', 'describe \t this'],
|
||||
['describe[Image #1]\t this', 'describe\t this'],
|
||||
['describe\n[Image #1]\nthis', 'describe\n\nthis'],
|
||||
['com[Image #1]pare this', 'compare this'],
|
||||
['[Image #1] [Image #2]', ''],
|
||||
['literal [Image #x] text', 'literal [Image #x] text']
|
||||
])('strips image prompt markers anywhere in text', (text, expected) => {
|
||||
expect(stripImagePromptMarker(text)).toBe(expected)
|
||||
})
|
||||
|
||||
it('returns long marker-free whitespace without regex backtracking', () => {
|
||||
const text = ' '.repeat(50_000)
|
||||
expect(stripImagePromptMarker(text)).toBe(text)
|
||||
})
|
||||
|
||||
it('shares marker-aware text matching across multiple text blocks', () => {
|
||||
const message: NativeChatMessage = {
|
||||
...userText('prompt', 'unused'),
|
||||
blocks: [
|
||||
{ type: 'text', text: 'look' },
|
||||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||||
{ type: 'text', text: '[Image #1] here' }
|
||||
]
|
||||
}
|
||||
|
||||
expect(normalizeNativeChatUserText(' look [Image #1] here ')).toBe('look here')
|
||||
expect(normalizedNativeChatUserMessageText(message)).toBe('look here')
|
||||
})
|
||||
|
||||
it('recognizes only sole-text image-source user turns', () => {
|
||||
const source = userText('source', '[Image: source: /tmp/a.png]')
|
||||
expect(isImageSourceUserTurn(source)).toBe(true)
|
||||
expect(isImageSourceUserTurn({ ...source, role: 'assistant' })).toBe(false)
|
||||
expect(
|
||||
isImageSourceUserTurn({
|
||||
...source,
|
||||
blocks: [...source.blocks, { type: 'text', text: 'caption' }]
|
||||
})
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('converts a lone [Image: source] turn (no prompt) into an image-ref instead of raw text', () => {
|
||||
const out = normalizeImageTranscriptMessages([
|
||||
userText('a', '[Image: source: /Users/me/Pictures/hero-image-2.png]')
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
import { isTextBlock, type NativeChatBlock, type NativeChatMessage } from './native-chat-types'
|
||||
|
||||
const IMAGE_SOURCE_MARKER = /^\[Image:\s*source:\s*(.+?)\]\s*$/
|
||||
const IMAGE_PROMPT_MARKERS = /^(?:\[Image #\d+\]\s*)+/
|
||||
const IMAGE_PROMPT_MARKER = /\[Image #\d+\]/
|
||||
const IMAGE_PROMPT_MARKERS = /\[Image #\d+\]/g
|
||||
const IMAGE_PROMPT_MARKER_AT_START = /^[^\S\r\n]*\[Image #\d+\]/
|
||||
const IMAGE_PROMPT_MARKER_AT_END = /\[Image #\d+\][^\S\r\n]*$/
|
||||
const HORIZONTAL_WHITESPACE_START = /^[^\S\r\n]+/
|
||||
const HORIZONTAL_WHITESPACE_END = /[^\S\r\n]+$/
|
||||
|
||||
function soleText(message: NativeChatMessage): string | null {
|
||||
return message.blocks.length === 1 && isTextBlock(message.blocks[0])
|
||||
@@ -13,40 +18,75 @@ export function imageSourcePathFromText(text: string): string | null {
|
||||
return text.match(IMAGE_SOURCE_MARKER)?.[1]?.trim() ?? null
|
||||
}
|
||||
|
||||
export function stripImagePromptMarker(text: string): string {
|
||||
return text.replace(IMAGE_PROMPT_MARKERS, '')
|
||||
export function isImageSourceUserTurn(message: NativeChatMessage): boolean {
|
||||
return message.role === 'user' && imageSourcePathFromText(soleText(message) ?? '') !== null
|
||||
}
|
||||
|
||||
function stripImagePromptMarkersFromFirstText(
|
||||
export function stripImagePromptMarker(text: string): string {
|
||||
const stripped = text.replace(IMAGE_PROMPT_MARKERS, '')
|
||||
if (stripped === text) {
|
||||
return text
|
||||
}
|
||||
let result = IMAGE_PROMPT_MARKER_AT_START.test(text)
|
||||
? stripped.replace(HORIZONTAL_WHITESPACE_START, '')
|
||||
: stripped
|
||||
if (IMAGE_PROMPT_MARKER_AT_END.test(text)) {
|
||||
result = result.replace(HORIZONTAL_WHITESPACE_END, '')
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export function normalizeNativeChatUserText(text: string): string {
|
||||
return stripImagePromptMarker(text).trim().replace(/\s+/g, ' ')
|
||||
}
|
||||
|
||||
export function normalizedNativeChatUserMessageText(message: NativeChatMessage): string | null {
|
||||
if (message.role !== 'user') {
|
||||
return null
|
||||
}
|
||||
const normalized = normalizeNativeChatUserText(
|
||||
message.blocks
|
||||
.filter(isTextBlock)
|
||||
.map((block) => block.text)
|
||||
.join(' ')
|
||||
)
|
||||
return normalized || null
|
||||
}
|
||||
|
||||
function stripImagePromptMarkersFromTextBlocks(
|
||||
blocks: readonly NativeChatBlock[]
|
||||
): NativeChatBlock[] {
|
||||
const textIndex = blocks.findIndex(isTextBlock)
|
||||
if (textIndex === -1) {
|
||||
return blocks as NativeChatBlock[]
|
||||
let sawText = false
|
||||
let next: NativeChatBlock[] | null = null
|
||||
for (let index = 0; index < blocks.length; index += 1) {
|
||||
const block = blocks[index]!
|
||||
if (!isTextBlock(block)) {
|
||||
next?.push(block)
|
||||
continue
|
||||
}
|
||||
const isFirstText = !sawText
|
||||
sawText = true
|
||||
const text = stripImagePromptMarker(block.text)
|
||||
if (!text.trim() && (text !== block.text || isFirstText)) {
|
||||
next ??= blocks.slice(0, index)
|
||||
continue
|
||||
}
|
||||
if (text !== block.text) {
|
||||
next ??= blocks.slice(0, index)
|
||||
next.push({ ...block, text })
|
||||
continue
|
||||
}
|
||||
next?.push(block)
|
||||
}
|
||||
const block = blocks[textIndex]
|
||||
if (!block || !isTextBlock(block)) {
|
||||
return blocks as NativeChatBlock[]
|
||||
}
|
||||
const text = stripImagePromptMarker(block.text)
|
||||
if (text.trim().length === 0) {
|
||||
return blocks.filter((_, index) => index !== textIndex)
|
||||
}
|
||||
if (text === block.text) {
|
||||
return blocks as NativeChatBlock[]
|
||||
}
|
||||
const next = [...blocks]
|
||||
next[textIndex] = { ...block, text }
|
||||
return next
|
||||
return next ?? (blocks as NativeChatBlock[])
|
||||
}
|
||||
|
||||
function imagePromptMarkerStartsMessage(message: NativeChatMessage): boolean {
|
||||
const firstText = message.blocks.find(isTextBlock)
|
||||
return firstText ? IMAGE_PROMPT_MARKERS.test(firstText.text) : false
|
||||
export function hasImagePromptMarker(message: NativeChatMessage): boolean {
|
||||
return message.blocks.some((block) => isTextBlock(block) && IMAGE_PROMPT_MARKER.test(block.text))
|
||||
}
|
||||
|
||||
/** Claude records image paths as source turns followed by one marker-prefixed
|
||||
* prompt. Merge the whole run back into one native user turn. */
|
||||
/** Claude records image paths as source turns followed by a prompt carrying
|
||||
* image markers. Merge the whole run back into one native user turn. */
|
||||
export function normalizeImageTranscriptMessages(
|
||||
messages: readonly NativeChatMessage[]
|
||||
): NativeChatMessage[] {
|
||||
@@ -75,13 +115,13 @@ export function normalizeImageTranscriptMessages(
|
||||
if (
|
||||
prompt?.role === 'user' &&
|
||||
prompt.source === message.source &&
|
||||
imagePromptMarkerStartsMessage(prompt)
|
||||
hasImagePromptMarker(prompt)
|
||||
) {
|
||||
normalized.push({
|
||||
...prompt,
|
||||
blocks: [
|
||||
...imagePaths.map((path) => ({ type: 'image-ref' as const, path })),
|
||||
...stripImagePromptMarkersFromFirstText(prompt.blocks)
|
||||
...stripImagePromptMarkersFromTextBlocks(prompt.blocks)
|
||||
]
|
||||
})
|
||||
index = nextIndex
|
||||
@@ -93,7 +133,7 @@ export function normalizeImageTranscriptMessages(
|
||||
})
|
||||
continue
|
||||
}
|
||||
const blocks = stripImagePromptMarkersFromFirstText(message.blocks)
|
||||
const blocks = stripImagePromptMarkersFromTextBlocks(message.blocks)
|
||||
if (blocks === message.blocks) {
|
||||
normalized?.push(message)
|
||||
} else {
|
||||
|
||||
Reference in New Issue
Block a user