mirror of
https://github.com/stablyai/orca.git
synced 2026-10-29 08:07:18 +00:00
Fix native chat image marker position handling (#14162)
* fix(native-chat): handle image markers in any position * fix(native-chat): preserve image caption whitespace * test(native-chat): cover marker boundary spacing * fix(mobile): normalize image echo reconciliation * fix(mobile): use idiomatic tail access * refactor(native-chat): share image echo matching * perf(native-chat): avoid unchanged block copies
This commit is contained in:
@@ -1,9 +1,8 @@
|
||||
import { stripImagePromptMarker } from '../../../../shared/native-chat-image-transcript-markers'
|
||||
import {
|
||||
isImageRefBlock,
|
||||
isTextBlock,
|
||||
type NativeChatMessage
|
||||
} from '../../../../shared/native-chat-types'
|
||||
normalizeNativeChatUserText,
|
||||
normalizedNativeChatUserMessageText
|
||||
} from '../../../../shared/native-chat-image-transcript-markers'
|
||||
import { isImageRefBlock, type NativeChatMessage } from '../../../../shared/native-chat-types'
|
||||
|
||||
export type NativeChatPendingOccurrence = {
|
||||
text: string
|
||||
@@ -16,7 +15,7 @@ export type NativeChatPendingOccurrence = {
|
||||
}
|
||||
|
||||
export function normalizeNativeChatPendingText(text: string): string {
|
||||
return stripImagePromptMarker(text).trim().replace(/\s+/g, ' ')
|
||||
return normalizeNativeChatUserText(text)
|
||||
}
|
||||
|
||||
export function nativeChatPendingContentKey(
|
||||
@@ -34,15 +33,15 @@ function nativeChatUserMessageContentKey(message: NativeChatMessage): string | n
|
||||
if (message.role !== 'user') {
|
||||
return null
|
||||
}
|
||||
const text = message.blocks
|
||||
.filter(isTextBlock)
|
||||
.map((block) => block.text)
|
||||
.join(' ')
|
||||
const text = normalizedNativeChatUserMessageText(message) ?? ''
|
||||
if (text) {
|
||||
return `text:${text}`
|
||||
}
|
||||
const imagePaths = message.blocks
|
||||
.filter(isImageRefBlock)
|
||||
.map((block) => block.path)
|
||||
.filter((path): path is string => Boolean(path))
|
||||
const key = nativeChatPendingContentKey({ text, imagePaths })
|
||||
const key = nativeChatPendingContentKey({ text: '', imagePaths })
|
||||
return key === 'empty' ? null : key
|
||||
}
|
||||
|
||||
@@ -80,19 +79,6 @@ export function advancedNativeChatUserContentCounts(
|
||||
return advanced
|
||||
}
|
||||
|
||||
function nativeChatUserMessageNormalizedText(message: NativeChatMessage): string | null {
|
||||
if (message.role !== 'user') {
|
||||
return null
|
||||
}
|
||||
const text = normalizeNativeChatPendingText(
|
||||
message.blocks
|
||||
.filter(isTextBlock)
|
||||
.map((block) => block.text)
|
||||
.join(' ')
|
||||
)
|
||||
return text.length > 0 ? text : null
|
||||
}
|
||||
|
||||
/** User texts that already have a later non-user turn (ready to prune echoes). */
|
||||
export function advancedNativeChatUserTexts(
|
||||
messages: readonly NativeChatMessage[]
|
||||
@@ -101,7 +87,7 @@ export function advancedNativeChatUserTexts(
|
||||
const waiting: string[] = []
|
||||
for (const message of messages) {
|
||||
if (message.role === 'user') {
|
||||
const text = nativeChatUserMessageNormalizedText(message)
|
||||
const text = normalizedNativeChatUserMessageText(message)
|
||||
if (text) {
|
||||
waiting.push(text)
|
||||
}
|
||||
@@ -119,7 +105,7 @@ export function matchingNativeChatUserTexts(
|
||||
): readonly string[] {
|
||||
const texts: string[] = []
|
||||
for (const message of messages) {
|
||||
const text = nativeChatUserMessageNormalizedText(message)
|
||||
const text = normalizedNativeChatUserMessageText(message)
|
||||
if (text) {
|
||||
texts.push(text)
|
||||
}
|
||||
|
||||
@@ -95,6 +95,34 @@ describe('prunePendingSends', () => {
|
||||
expect(next).toEqual([])
|
||||
})
|
||||
|
||||
it('drops an attachment pending send once a trailing-marker prompt advances', () => {
|
||||
const pending = [
|
||||
{ ...pendingOf('p1', 'what do you see'), imagePaths: ['/Users/me/Downloads/3d.png'] }
|
||||
]
|
||||
const next = prunePendingSends(pending, [
|
||||
userMessage('m1', 'what do you see[Image #1]'),
|
||||
assistantMessage('m2', 'an image')
|
||||
])
|
||||
expect(next).toEqual([])
|
||||
})
|
||||
|
||||
it('drops a pending send represented by multiple marker-bearing text blocks', () => {
|
||||
const prompt: NativeChatMessage = {
|
||||
...userMessage('m1', 'unused'),
|
||||
blocks: [
|
||||
{ type: 'text', text: 'what do' },
|
||||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||||
{ type: 'text', text: '[Image #1] you see' }
|
||||
]
|
||||
}
|
||||
const next = prunePendingSends(
|
||||
[pendingOf('p1', 'what do you see')],
|
||||
[prompt, assistantMessage('m2', 'an image')]
|
||||
)
|
||||
|
||||
expect(next).toEqual([])
|
||||
})
|
||||
|
||||
it('drops an attachment-only pending send once its image turn advances', () => {
|
||||
const pending = [{ ...pendingOf('p1', ''), imagePaths: ['/tmp/first.png', '/tmp/second.png'] }]
|
||||
const transcript = [
|
||||
|
||||
+23
@@ -74,6 +74,29 @@ describe('preassembled native-chat live sessions', () => {
|
||||
expect(out[0]).toMatchObject({ id: 'image-prompt', source: 'transcript' })
|
||||
})
|
||||
|
||||
it('keeps legacy parity for trailing image prompt markers', () => {
|
||||
const transcript: NativeChatMessage[] = [
|
||||
message('image-source', {
|
||||
role: 'user',
|
||||
blocks: [{ type: 'text', text: '[Image: source: /tmp/a.png]' }],
|
||||
timestamp: 1
|
||||
}),
|
||||
message('image-prompt', {
|
||||
role: 'user',
|
||||
blocks: [{ type: 'text', text: 'describe this[Image #1]' }],
|
||||
timestamp: 2
|
||||
})
|
||||
]
|
||||
|
||||
const out = expectLegacyMessageParity(transcript)
|
||||
|
||||
expect(out).toHaveLength(1)
|
||||
expect(out[0]?.blocks).toEqual([
|
||||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||||
{ type: 'text', text: 'describe this' }
|
||||
])
|
||||
})
|
||||
|
||||
it('re-dedupes surfaced skill envelopes that collide across sources', () => {
|
||||
const skill = (id: string, plugin: string, source: 'transcript' | 'scrape') =>
|
||||
message(id, {
|
||||
|
||||
@@ -112,6 +112,34 @@ describe('assembleNativeChatSession', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('merges Claude image source markers into a trailing-marker prompt', () => {
|
||||
const imageSource = msg({
|
||||
id: 'u-image-source',
|
||||
role: 'user',
|
||||
timestamp: 100,
|
||||
blocks: [{ type: 'text', text: '[Image: source: /Users/me/Downloads/3d.png]' }]
|
||||
})
|
||||
const prompt = msg({
|
||||
id: 'u-prompt',
|
||||
role: 'user',
|
||||
timestamp: 101,
|
||||
blocks: [{ type: 'text', text: 'what do you see[Image #1]' }]
|
||||
})
|
||||
|
||||
const session = assembleNativeChatSession({
|
||||
sources: { transcript: [imageSource, prompt] },
|
||||
sessionId: 's1',
|
||||
agent: 'claude'
|
||||
})
|
||||
|
||||
expect(session.messages).toHaveLength(1)
|
||||
expect(session.messages[0]).toMatchObject({ id: 'u-prompt', role: 'user' })
|
||||
expect(session.messages[0].blocks).toEqual([
|
||||
{ type: 'image-ref', path: '/Users/me/Downloads/3d.png' },
|
||||
{ type: 'text', text: 'what do you see' }
|
||||
])
|
||||
})
|
||||
|
||||
it('drops a scrape duplicate even when scrape is processed first by id', () => {
|
||||
const scrape = msg({
|
||||
id: 'shared-id',
|
||||
|
||||
Reference in New Issue
Block a user