mirror of
https://github.com/stablyai/orca.git
synced 2026-10-23 08:03:39 +00:00
Brennan Benson
1689e2f241
Anchoring decided which turn gets stripped but not how many markers may go, so a turn attaching one image and mentioning a second marker in prose still lost the user's literal text. Strip at most one marker per image in the anchored source run, in document order across the turn's text blocks; markers past the count stay verbatim. Position, not ordinal, decides which markers are placeholders, so the rule holds whether the producing CLI numbers markers per-turn or per-session.
346 lines
12 KiB
TypeScript
346 lines
12 KiB
TypeScript
import { describe, expect, it } from 'vitest'
|
||
import type { NativeChatMessage } from './native-chat-types'
|
||
import {
|
||
isImageSourceUserTurn,
|
||
nativeChatUserMessageMatchText,
|
||
nativeChatUserTextMatchText,
|
||
normalizeImageTranscriptMessages,
|
||
normalizeNativeChatUserText,
|
||
normalizedNativeChatUserMessageText,
|
||
stripImagePromptMarker
|
||
} from './native-chat-image-transcript-markers'
|
||
|
||
function userText(id: string, text: string): NativeChatMessage {
|
||
return {
|
||
id,
|
||
role: 'user',
|
||
blocks: [{ type: 'text', text }],
|
||
timestamp: 1,
|
||
source: 'transcript'
|
||
}
|
||
}
|
||
|
||
describe('normalizeImageTranscriptMessages', () => {
|
||
it('merges the paired [Image: source]/[Image #1] turns into one image-ref turn', () => {
|
||
const out = normalizeImageTranscriptMessages([
|
||
userText('a', '[Image: source: /tmp/orca-paste-1-2.png]'),
|
||
userText('b', '[Image #1] describe this')
|
||
])
|
||
expect(out).toHaveLength(1)
|
||
expect(out[0]!.blocks).toEqual([
|
||
{ type: 'image-ref', path: '/tmp/orca-paste-1-2.png' },
|
||
{ type: 'text', text: 'describe this' }
|
||
])
|
||
})
|
||
|
||
it('merges a source turn into a prompt with a trailing image marker', () => {
|
||
const out = normalizeImageTranscriptMessages([
|
||
userText('a', '[Image: source: /tmp/orca-paste-1-2.png]'),
|
||
userText('b', 'describe this[Image #1]')
|
||
])
|
||
|
||
expect(out).toHaveLength(1)
|
||
expect(out[0]!.blocks).toEqual([
|
||
{ type: 'image-ref', path: '/tmp/orca-paste-1-2.png' },
|
||
{ type: 'text', text: 'describe this' }
|
||
])
|
||
})
|
||
|
||
it('folds and strips markers in later text blocks', () => {
|
||
const prompt: NativeChatMessage = {
|
||
...userText('prompt', 'unused'),
|
||
blocks: [
|
||
{ type: 'text', text: 'describe' },
|
||
{ type: 'image-ref', path: '/tmp/existing.png' },
|
||
{ type: 'text', text: '[Image #1] this' }
|
||
]
|
||
}
|
||
const out = normalizeImageTranscriptMessages([
|
||
userText('source', '[Image: source: /tmp/a.png]'),
|
||
prompt
|
||
])
|
||
|
||
expect(out).toHaveLength(1)
|
||
expect(out[0]?.blocks).toEqual([
|
||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||
{ type: 'text', text: 'describe' },
|
||
{ type: 'image-ref', path: '/tmp/existing.png' },
|
||
{ type: 'text', text: 'this' }
|
||
])
|
||
})
|
||
|
||
it.each([
|
||
['[Image #1] describe this', 'describe this'],
|
||
['[Image #1]\t describe this', 'describe this'],
|
||
[' \t[Image #1] describe this', 'describe this'],
|
||
['describe this [Image #1]', 'describe this'],
|
||
['describe this \t[Image #1]', 'describe this'],
|
||
['describe this [Image #1]\t ', 'describe this'],
|
||
['describe [Image #1] this', 'describe this'],
|
||
['describe [Image #1]\t this', 'describe \t this'],
|
||
['describe[Image #1]\t this', 'describe\t this'],
|
||
['describe\n[Image #1]\nthis', 'describe\n\nthis'],
|
||
['com[Image #1]pare this', 'compare this'],
|
||
['[Image #1] [Image #2]', ''],
|
||
['literal [Image #x] text', 'literal [Image #x] text']
|
||
])('strips image prompt markers anywhere in text', (text, expected) => {
|
||
expect(stripImagePromptMarker(text)).toBe(expected)
|
||
})
|
||
|
||
it('returns long marker-free whitespace without regex backtracking', () => {
|
||
const text = ' '.repeat(50_000)
|
||
expect(stripImagePromptMarker(text)).toBe(text)
|
||
})
|
||
|
||
it('shares marker-aware text matching across multiple text blocks', () => {
|
||
const message: NativeChatMessage = {
|
||
...userText('prompt', 'unused'),
|
||
blocks: [
|
||
{ type: 'text', text: 'look' },
|
||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||
{ type: 'text', text: '[Image #1] here' }
|
||
]
|
||
}
|
||
|
||
expect(normalizeNativeChatUserText(' look [Image #1] here ')).toBe('look here')
|
||
expect(normalizedNativeChatUserMessageText(message)).toBe('look here')
|
||
})
|
||
|
||
it('keeps markers in no-image match keys without changing real image keys', () => {
|
||
expect(nativeChatUserTextMatchText(' keep [Image #1] literal ', false)).toBe(
|
||
'keep [Image #1] literal'
|
||
)
|
||
expect(nativeChatUserTextMatchText(' keep [Image #1] literal ', true)).toBe('keep literal')
|
||
|
||
const literal = userText('literal', 'keep [Image #1] literal')
|
||
const attached = {
|
||
...literal,
|
||
blocks: [{ type: 'image-ref' as const, path: '/tmp/a.png' }, ...literal.blocks]
|
||
}
|
||
expect(nativeChatUserMessageMatchText(literal)).toBe('keep [Image #1] literal')
|
||
expect(nativeChatUserMessageMatchText(attached)).toBe('keep literal')
|
||
})
|
||
|
||
it('recognizes only sole-text image-source user turns', () => {
|
||
const source = userText('source', '[Image: source: /tmp/a.png]')
|
||
expect(isImageSourceUserTurn(source)).toBe(true)
|
||
expect(isImageSourceUserTurn({ ...source, role: 'assistant' })).toBe(false)
|
||
expect(
|
||
isImageSourceUserTurn({
|
||
...source,
|
||
blocks: [...source.blocks, { type: 'text', text: 'caption' }]
|
||
})
|
||
).toBe(false)
|
||
})
|
||
|
||
it('converts a lone [Image: source] turn (no prompt) into an image-ref instead of raw text', () => {
|
||
const out = normalizeImageTranscriptMessages([
|
||
userText('a', '[Image: source: /Users/me/Pictures/hero-image-2.png]')
|
||
])
|
||
expect(out).toHaveLength(1)
|
||
expect(out[0]!.blocks).toEqual([
|
||
{ type: 'image-ref', path: '/Users/me/Pictures/hero-image-2.png' }
|
||
])
|
||
})
|
||
|
||
it('folds every source and strips every prompt marker for a multi-image send', () => {
|
||
const out = normalizeImageTranscriptMessages([
|
||
userText('a', '[Image: source: /tmp/a.png]'),
|
||
userText('b', '[Image: source: /tmp/b.png]'),
|
||
userText('c', '[Image: source: /tmp/c.png]'),
|
||
userText('prompt', '[Image #1] [Image #2] [Image #3] compare these')
|
||
])
|
||
|
||
expect(out).toHaveLength(1)
|
||
expect(out[0]).toMatchObject({ id: 'prompt' })
|
||
expect(out[0]!.blocks).toEqual([
|
||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||
{ type: 'image-ref', path: '/tmp/b.png' },
|
||
{ type: 'image-ref', path: '/tmp/c.png' },
|
||
{ type: 'text', text: 'compare these' }
|
||
])
|
||
})
|
||
|
||
it('keeps all image refs when a multi-image send has no caption', () => {
|
||
const out = normalizeImageTranscriptMessages([
|
||
userText('a', '[Image: source: /tmp/a.png]'),
|
||
userText('b', '[Image: source: /tmp/b.png]'),
|
||
userText('prompt', '[Image #1] [Image #2]')
|
||
])
|
||
|
||
expect(out).toHaveLength(1)
|
||
expect(out[0]!.blocks).toEqual([
|
||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||
{ type: 'image-ref', path: '/tmp/b.png' }
|
||
])
|
||
})
|
||
|
||
it('preserves adjacent standalone image turns without a prompt marker', () => {
|
||
const out = normalizeImageTranscriptMessages([
|
||
userText('a', '[Image: source: /tmp/a.png]'),
|
||
userText('b', '[Image: source: /tmp/b.png]')
|
||
])
|
||
|
||
expect(out).toHaveLength(2)
|
||
expect(out.map((message) => message.id)).toEqual(['a', 'b'])
|
||
expect(out.map((message) => message.blocks)).toEqual([
|
||
[{ type: 'image-ref', path: '/tmp/a.png' }],
|
||
[{ type: 'image-ref', path: '/tmp/b.png' }]
|
||
])
|
||
})
|
||
|
||
it('leaves ordinary user text untouched', () => {
|
||
const message = userText('a', 'keep [Image #1] as literal text')
|
||
const messages = [message]
|
||
const out = normalizeImageTranscriptMessages(messages)
|
||
expect(out).toBe(messages)
|
||
expect(out[0]).toBe(message)
|
||
expect(out[0]!.blocks).toBe(message.blocks)
|
||
})
|
||
|
||
it('removes a whitespace-only first text block', () => {
|
||
const out = normalizeImageTranscriptMessages([userText('a', ' ')])
|
||
|
||
expect(out[0]?.blocks).toEqual([])
|
||
})
|
||
|
||
it('preserves unaffected rows when another row needs normalization', () => {
|
||
const before = userText('before', 'keep this row')
|
||
const marker = userText('marker', '[Image: source: /tmp/image.png]')
|
||
const after = userText('after', 'keep this row too')
|
||
const messages = [before, marker, after]
|
||
|
||
const out = normalizeImageTranscriptMessages(messages)
|
||
|
||
expect(out).not.toBe(messages)
|
||
expect(out[0]).toBe(before)
|
||
expect(out[2]).toBe(after)
|
||
})
|
||
|
||
it('keeps a later literal marker when earlier turns separate it from a real image send', () => {
|
||
const out = normalizeImageTranscriptMessages([
|
||
userText('s1', '[Image: source: /tmp/a.jpg]'),
|
||
userText('p1', '[Image #1] what is this?'),
|
||
userText('u1', 'thanks'),
|
||
userText('u2', 'in my notes I wrote [Image #2] as plain text')
|
||
])
|
||
|
||
expect(out.at(-1)?.blocks).toEqual([
|
||
{ type: 'text', text: 'in my notes I wrote [Image #2] as plain text' }
|
||
])
|
||
})
|
||
|
||
it('breaks the anchor when a non-image user turn separates the source run from the prompt', () => {
|
||
const out = normalizeImageTranscriptMessages([
|
||
userText('s1', '[Image: source: /tmp/a.jpg]'),
|
||
userText('u1', 'here is the picture'),
|
||
userText('u2', 'please preserve [Image #1] literally')
|
||
])
|
||
|
||
expect(out.map((message) => message.blocks)).toEqual([
|
||
[{ type: 'image-ref', path: '/tmp/a.jpg' }],
|
||
[{ type: 'text', text: 'here is the picture' }],
|
||
[{ type: 'text', text: 'please preserve [Image #1] literally' }]
|
||
])
|
||
})
|
||
|
||
// A marker the user typed must not absorb the image of the send that follows it.
|
||
it('does not let a literal marker turn claim a later send’s image source', () => {
|
||
const out = normalizeImageTranscriptMessages([
|
||
userText('u1', 'remember I typed [Image #1] by hand'),
|
||
userText('s1', '[Image: source: /tmp/a.jpg]'),
|
||
userText('p1', '[Image #1] what is this?')
|
||
])
|
||
|
||
expect(out[0]?.blocks).toEqual([{ type: 'text', text: 'remember I typed [Image #1] by hand' }])
|
||
expect(out.at(-1)?.blocks).toEqual([
|
||
{ type: 'image-ref', path: '/tmp/a.jpg' },
|
||
{ type: 'text', text: 'what is this?' }
|
||
])
|
||
})
|
||
|
||
it('keeps markers beyond the anchored run’s image count as the user’s own words', () => {
|
||
const out = normalizeImageTranscriptMessages([
|
||
userText('s1', '[Image: source: /tmp/a.png]'),
|
||
userText('p1', '[Image #1] compare with the [Image #2] I mentioned earlier')
|
||
])
|
||
|
||
expect(out).toHaveLength(1)
|
||
expect(out[0]!.blocks).toEqual([
|
||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||
{ type: 'text', text: 'compare with the [Image #2] I mentioned earlier' }
|
||
])
|
||
})
|
||
|
||
it('strips by document position, not by marker ordinal', () => {
|
||
const out = normalizeImageTranscriptMessages([
|
||
userText('s1', '[Image: source: /tmp/a.png]'),
|
||
userText('p1', '[Image #2] came first but [Image #1] came second')
|
||
])
|
||
|
||
expect(out[0]!.blocks).toEqual([
|
||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||
{ type: 'text', text: 'came first but [Image #1] came second' }
|
||
])
|
||
})
|
||
|
||
it('strips a marker whose ordinal exceeds the run size when it is within the count', () => {
|
||
const out = normalizeImageTranscriptMessages([
|
||
userText('s1', '[Image: source: /tmp/a.png]'),
|
||
userText('p1', '[Image #7] what is this?')
|
||
])
|
||
|
||
expect(out[0]!.blocks).toEqual([
|
||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||
{ type: 'text', text: 'what is this?' }
|
||
])
|
||
})
|
||
|
||
it('spends the image budget across text blocks in document order', () => {
|
||
const prompt: NativeChatMessage = {
|
||
...userText('prompt', 'unused'),
|
||
blocks: [
|
||
{ type: 'text', text: '[Image #1] look at' },
|
||
{ type: 'image-ref', path: '/tmp/existing.png' },
|
||
{ type: 'text', text: 'not [Image #2] though' }
|
||
]
|
||
}
|
||
const out = normalizeImageTranscriptMessages([
|
||
userText('s1', '[Image: source: /tmp/a.png]'),
|
||
prompt
|
||
])
|
||
|
||
expect(out[0]!.blocks).toEqual([
|
||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||
{ type: 'text', text: 'look at' },
|
||
{ type: 'image-ref', path: '/tmp/existing.png' },
|
||
{ type: 'text', text: 'not [Image #2] though' }
|
||
])
|
||
})
|
||
|
||
it('keeps the surplus marker on a two-image run that mentions a third', () => {
|
||
const out = normalizeImageTranscriptMessages([
|
||
userText('s1', '[Image: source: /tmp/a.png]'),
|
||
userText('s2', '[Image: source: /tmp/b.png]'),
|
||
userText('p1', '[Image #1] [Image #2] versus the [Image #3] from before')
|
||
])
|
||
|
||
expect(out[0]!.blocks).toEqual([
|
||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||
{ type: 'image-ref', path: '/tmp/b.png' },
|
||
{ type: 'text', text: 'versus the [Image #3] from before' }
|
||
])
|
||
})
|
||
|
||
it('leaves assistant messages untouched', () => {
|
||
const assistant: NativeChatMessage = {
|
||
id: 'a',
|
||
role: 'assistant',
|
||
blocks: [{ type: 'text', text: '[Image: source: /tmp/x.png]' }],
|
||
timestamp: 1,
|
||
source: 'transcript'
|
||
}
|
||
expect(normalizeImageTranscriptMessages([assistant])).toEqual([assistant])
|
||
})
|
||
})
|