mirror of
https://github.com/stablyai/orca.git
synced 2026-10-23 08:03:39 +00:00
fix(native-chat): bound marker stripping by the run's image count
Anchoring decided which turn gets stripped but not how many markers may go, so a turn attaching one image and mentioning a second marker in prose still lost the user's literal text. Strip at most one marker per image in the anchored source run, in document order across the turn's text blocks; markers past the count stay verbatim. Position, not ordinal, decides which markers are placeholders, so the rule holds whether the producing CLI numbers markers per-turn or per-session.
This commit is contained in:
@@ -133,6 +133,22 @@ describe('buildMobileNativeChatTransientData', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('keeps a marker beyond the folded run’s image count as literal text', () => {
|
||||
const data = build(
|
||||
[
|
||||
user('u1', '[Image: source: /tmp/a.png]'),
|
||||
user('u2', '[Image #1] compare with the [Image #2] I mentioned earlier')
|
||||
],
|
||||
null,
|
||||
[]
|
||||
)
|
||||
|
||||
expect(data[0]?.blocks).toEqual([
|
||||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||||
{ type: 'text', text: 'compare with the [Image #2] I mentioned earlier' }
|
||||
])
|
||||
})
|
||||
|
||||
it('renders a lone image marker turn (no caption) as an image-ref block', () => {
|
||||
const data = build([user('u1', '[Image: source: /tmp/a.png]')], null, [])
|
||||
expect(data[0]?.blocks).toEqual([{ type: 'image-ref', path: '/tmp/a.png' }])
|
||||
|
||||
@@ -259,6 +259,79 @@ describe('normalizeImageTranscriptMessages', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('keeps markers beyond the anchored run’s image count as the user’s own words', () => {
|
||||
const out = normalizeImageTranscriptMessages([
|
||||
userText('s1', '[Image: source: /tmp/a.png]'),
|
||||
userText('p1', '[Image #1] compare with the [Image #2] I mentioned earlier')
|
||||
])
|
||||
|
||||
expect(out).toHaveLength(1)
|
||||
expect(out[0]!.blocks).toEqual([
|
||||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||||
{ type: 'text', text: 'compare with the [Image #2] I mentioned earlier' }
|
||||
])
|
||||
})
|
||||
|
||||
it('strips by document position, not by marker ordinal', () => {
|
||||
const out = normalizeImageTranscriptMessages([
|
||||
userText('s1', '[Image: source: /tmp/a.png]'),
|
||||
userText('p1', '[Image #2] came first but [Image #1] came second')
|
||||
])
|
||||
|
||||
expect(out[0]!.blocks).toEqual([
|
||||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||||
{ type: 'text', text: 'came first but [Image #1] came second' }
|
||||
])
|
||||
})
|
||||
|
||||
it('strips a marker whose ordinal exceeds the run size when it is within the count', () => {
|
||||
const out = normalizeImageTranscriptMessages([
|
||||
userText('s1', '[Image: source: /tmp/a.png]'),
|
||||
userText('p1', '[Image #7] what is this?')
|
||||
])
|
||||
|
||||
expect(out[0]!.blocks).toEqual([
|
||||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||||
{ type: 'text', text: 'what is this?' }
|
||||
])
|
||||
})
|
||||
|
||||
it('spends the image budget across text blocks in document order', () => {
|
||||
const prompt: NativeChatMessage = {
|
||||
...userText('prompt', 'unused'),
|
||||
blocks: [
|
||||
{ type: 'text', text: '[Image #1] look at' },
|
||||
{ type: 'image-ref', path: '/tmp/existing.png' },
|
||||
{ type: 'text', text: 'not [Image #2] though' }
|
||||
]
|
||||
}
|
||||
const out = normalizeImageTranscriptMessages([
|
||||
userText('s1', '[Image: source: /tmp/a.png]'),
|
||||
prompt
|
||||
])
|
||||
|
||||
expect(out[0]!.blocks).toEqual([
|
||||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||||
{ type: 'text', text: 'look at' },
|
||||
{ type: 'image-ref', path: '/tmp/existing.png' },
|
||||
{ type: 'text', text: 'not [Image #2] though' }
|
||||
])
|
||||
})
|
||||
|
||||
it('keeps the surplus marker on a two-image run that mentions a third', () => {
|
||||
const out = normalizeImageTranscriptMessages([
|
||||
userText('s1', '[Image: source: /tmp/a.png]'),
|
||||
userText('s2', '[Image: source: /tmp/b.png]'),
|
||||
userText('p1', '[Image #1] [Image #2] versus the [Image #3] from before')
|
||||
])
|
||||
|
||||
expect(out[0]!.blocks).toEqual([
|
||||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||||
{ type: 'image-ref', path: '/tmp/b.png' },
|
||||
{ type: 'text', text: 'versus the [Image #3] from before' }
|
||||
])
|
||||
})
|
||||
|
||||
it('leaves assistant messages untouched', () => {
|
||||
const assistant: NativeChatMessage = {
|
||||
id: 'a',
|
||||
|
||||
@@ -27,20 +27,41 @@ export function isImageSourceUserTurn(message: NativeChatMessage): boolean {
|
||||
return message.role === 'user' && imageSourcePathFromText(soleText(message) ?? '') !== null
|
||||
}
|
||||
|
||||
export function stripImagePromptMarker(text: string): string {
|
||||
const stripped = text.replace(IMAGE_PROMPT_MARKERS, '')
|
||||
function countMarkers(text: string): number {
|
||||
return text.match(IMAGE_PROMPT_MARKERS)?.length ?? 0
|
||||
}
|
||||
|
||||
/** Strips the first `limit` markers in document order; the rest are the user's
|
||||
* own words, so they stay verbatim. */
|
||||
function stripImagePromptMarkersUpTo(text: string, limit: number): string {
|
||||
if (limit <= 0) {
|
||||
return text
|
||||
}
|
||||
let used = 0
|
||||
const stripped = text.replace(IMAGE_PROMPT_MARKERS, (match) => {
|
||||
if (used >= limit) {
|
||||
return match
|
||||
}
|
||||
used += 1
|
||||
return ''
|
||||
})
|
||||
if (stripped === text) {
|
||||
return text
|
||||
}
|
||||
let result = IMAGE_PROMPT_MARKER_AT_START.test(text)
|
||||
? stripped.replace(HORIZONTAL_WHITESPACE_START, '')
|
||||
: stripped
|
||||
if (IMAGE_PROMPT_MARKER_AT_END.test(text)) {
|
||||
// Why: a trailing marker left in place still owns the whitespace before it.
|
||||
if (used === countMarkers(text) && IMAGE_PROMPT_MARKER_AT_END.test(text)) {
|
||||
result = result.replace(HORIZONTAL_WHITESPACE_END, '')
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export function stripImagePromptMarker(text: string): string {
|
||||
return stripImagePromptMarkersUpTo(text, Number.POSITIVE_INFINITY)
|
||||
}
|
||||
|
||||
export function normalizeNativeChatUserText(text: string): string {
|
||||
return stripImagePromptMarker(text).trim().replace(/\s+/g, ' ')
|
||||
}
|
||||
@@ -78,10 +99,14 @@ export function nativeChatUserMessageMatchText(message: NativeChatMessage): stri
|
||||
return nativeChatUserTextMatchText(joined, message.blocks.some(isImageRefBlock)) || null
|
||||
}
|
||||
|
||||
/** `limit` is the number of images the run actually carried. Only that many
|
||||
* markers can be placeholders; the rest are text the user typed. */
|
||||
function stripImagePromptMarkersFromTextBlocks(
|
||||
blocks: readonly NativeChatBlock[]
|
||||
blocks: readonly NativeChatBlock[],
|
||||
limit: number
|
||||
): NativeChatBlock[] {
|
||||
let sawText = false
|
||||
let remaining = limit
|
||||
let next: NativeChatBlock[] | null = null
|
||||
for (let index = 0; index < blocks.length; index += 1) {
|
||||
const block = blocks[index]!
|
||||
@@ -91,7 +116,8 @@ function stripImagePromptMarkersFromTextBlocks(
|
||||
}
|
||||
const isFirstText = !sawText
|
||||
sawText = true
|
||||
const text = stripImagePromptMarker(block.text)
|
||||
const text = stripImagePromptMarkersUpTo(block.text, remaining)
|
||||
remaining -= Math.min(remaining, countMarkers(block.text))
|
||||
if (!text.trim() && (text !== block.text || isFirstText)) {
|
||||
next ??= blocks.slice(0, index)
|
||||
continue
|
||||
@@ -173,7 +199,7 @@ export function normalizeImageTranscriptMessages(
|
||||
...prompt,
|
||||
blocks: [
|
||||
...imagePaths.map((path) => ({ type: 'image-ref' as const, path })),
|
||||
...stripImagePromptMarkersFromTextBlocks(prompt.blocks)
|
||||
...stripImagePromptMarkersFromTextBlocks(prompt.blocks, imagePaths.length)
|
||||
]
|
||||
})
|
||||
index = nextIndex
|
||||
|
||||
Reference in New Issue
Block a user