mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 19:34:29 +00:00
fix(chat): clear the sent image bubble when Claude takes an image in mid-turn (#40975)
Claude records a prompt it absorbs mid-turn only as a queued_command attachment whose images are inline base64 with no path. The decoder dropped those images, so an image-only send had no transcript row to match and its pending bubble stayed at the bottom of the chat. Decode queued inline images as data URLs. Pathless image-only rows cannot name their send, so each retires one delivered image-only send after its boundary, oldest first, and pruning hands the claimed row to the remaining sends so none reuses it. Failed or uncertain writes never claim one. Keep inline image bytes out of handoff and worker text. Co-authored-by: Xulin Wu <wuxulin@bytedance.com>
This commit is contained in:
co-authored by
Xulin Wu
parent
a94ead49d0
commit
be1ae8798e
@@ -37,4 +37,18 @@ describe('Claude queued_command attachment', () => {
|
||||
parentId: 'tool-result-row'
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps inline images from a prompt absorbed mid-turn, which have no source row', () => {
|
||||
const line = queuedCommand({
|
||||
prompt: [
|
||||
{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'iVBORw0K' } },
|
||||
{ type: 'text', text: '[Image #3]' }
|
||||
],
|
||||
commandMode: 'prompt'
|
||||
})
|
||||
expect(decodeClaudeTranscriptLine(line, 'fallback')?.blocks).toEqual([
|
||||
{ type: 'image-ref', url: 'data:image/png;base64,iVBORw0K' },
|
||||
{ type: 'text', text: '[Image #3]' }
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
} from '../ai-vault/session-scanner-values'
|
||||
import { imageSourcePathFromText } from '../../shared/native-chat/transcript/native-chat-image-transcript-markers'
|
||||
import { claudeContentBlocks } from './transcript-record-blocks'
|
||||
import { isKnownRasterImageMimeType } from '../../shared/file-preview/raster-image-preview-limits'
|
||||
import { unwrapClaudePastedContentBlock } from '../../shared/claude-pasted-content'
|
||||
import { unwrapClaudeBashInputBlock } from '../../shared/claude-bash-input'
|
||||
import { claudeInterruptedMessageId } from './transcript-turn-markers'
|
||||
@@ -161,7 +162,25 @@ function claudeQueuedPromptAsUserRecord(
|
||||
if (typeof prompt !== 'string' && !Array.isArray(prompt)) {
|
||||
return null
|
||||
}
|
||||
return { ...record, type: 'user', message: { role: 'user', content: prompt } }
|
||||
const content = Array.isArray(prompt) ? prompt.map(queuedPromptImageAsDataUrl) : prompt
|
||||
return { ...record, type: 'user', message: { role: 'user', content } }
|
||||
}
|
||||
|
||||
// Queued prompts get no image-source companion row; their inline base64 is the only image record.
|
||||
function queuedPromptImageAsDataUrl(item: unknown): unknown {
|
||||
const record = asRecord(item)
|
||||
const source = asRecord(record?.source)
|
||||
const mediaType = extractString(source?.media_type) ?? undefined
|
||||
const data = extractString(source?.data)
|
||||
if (
|
||||
record?.type !== 'image' ||
|
||||
source?.type !== 'base64' ||
|
||||
!data ||
|
||||
!isKnownRasterImageMimeType(mediaType)
|
||||
) {
|
||||
return item
|
||||
}
|
||||
return { ...record, source: { url: `data:${mediaType};base64,${data}` } }
|
||||
}
|
||||
|
||||
function claudeStopHookFeedbackMessage(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { NativeChatImageRefBlock } from '../../../../../shared/native-chat/native-chat-types'
|
||||
import { isInlineDataUrl } from '../../../../../shared/native-chat/native-chat-inline-data-url'
|
||||
|
||||
// Image refs are labels over RPC; inline bytes stay local and metadata cannot consume the frame.
|
||||
const NATIVE_CHAT_RPC_IMAGE_METADATA_CAP = 512
|
||||
@@ -21,11 +22,3 @@ export function sanitizeNativeChatRpcImageBlock(
|
||||
function boundedImageMetadata(value: string | undefined): string | undefined {
|
||||
return value && value.length <= NATIVE_CHAT_RPC_IMAGE_METADATA_CAP ? value : undefined
|
||||
}
|
||||
|
||||
function isInlineDataUrl(value: string): boolean {
|
||||
let schemeStart = 0
|
||||
while (schemeStart < value.length && value.charCodeAt(schemeStart) <= 0x20) {
|
||||
schemeStart += 1
|
||||
}
|
||||
return /^data:/i.test(value.slice(schemeStart))
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import { MAX_CONTEXT_CHARS } from './agent-session/requests/agent-session-contex
|
||||
import { agentJournalToolCallLifecycle } from './agent-journal-tool-call-lifecycle'
|
||||
import { withoutNativeChatVisualDirectiveLines } from './native-chat/native-chat-visual-directive'
|
||||
import { workerTranscriptBlockFallbacks } from './worker-transcript-text'
|
||||
import { isInlineDataUrl } from './native-chat/native-chat-inline-data-url'
|
||||
|
||||
const MAX_TOOL_CHARS = 8_000
|
||||
const MAX_MESSAGE_CHARS = 8_000
|
||||
@@ -74,7 +75,7 @@ function prioritizeConversation(messages: readonly NativeChatMessage[]): Convers
|
||||
text =
|
||||
message.role === 'user' ? block.text : withoutNativeChatVisualDirectiveLines(block.text)
|
||||
} else if (block.type === 'image-ref') {
|
||||
text = `[image] ${excerpt(block.path ?? block.url ?? block.alt ?? 'image omitted', MAX_MESSAGE_CHARS / 2)}`
|
||||
text = `[image] ${excerpt(block.path ?? (block.url && !isInlineDataUrl(block.url) ? block.url : null) ?? block.alt ?? 'image omitted', MAX_MESSAGE_CHARS / 2)}`
|
||||
}
|
||||
return text.trim() ? [{ messageIndex, blockIndex, text }] : []
|
||||
})
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
/** Inline image bytes; stay local and out of RPC frames and text exports. */
|
||||
export function isInlineDataUrl(value: string): boolean {
|
||||
let schemeStart = 0
|
||||
while (schemeStart < value.length && value.charCodeAt(schemeStart) <= 0x20) {
|
||||
schemeStart += 1
|
||||
}
|
||||
return /^data:/i.test(value.slice(schemeStart))
|
||||
}
|
||||
@@ -76,4 +76,47 @@ describe('pending sends typed through an agent TUI', () => {
|
||||
expect(prunePendingSends(pending, transcript)).toEqual([])
|
||||
expect(pendingSendsAsMessages(pending, transcript)).toEqual([])
|
||||
})
|
||||
|
||||
it('lets each pathless image row retire one delivered image-only send, once, oldest first', () => {
|
||||
const send = (
|
||||
id: string,
|
||||
extra: Partial<NativeChatPendingSend> = {}
|
||||
): NativeChatPendingSend => ({
|
||||
id,
|
||||
text: '',
|
||||
imagePaths: [`/tmp/orca-paste-${id}.png`],
|
||||
sentAt: 100,
|
||||
afterMessageId: 'working',
|
||||
...extra
|
||||
})
|
||||
const pending = [send('a', { delivery: 'rejected' }), send('b'), send('c')]
|
||||
const imageRow = (id: string, timestamp: number): NativeChatMessage => ({
|
||||
id,
|
||||
role: 'user',
|
||||
blocks: [{ type: 'image-ref', url: 'data:image/png;base64,iVBORw0K' }],
|
||||
timestamp,
|
||||
source: 'transcript'
|
||||
})
|
||||
const before = [imageRow('older-image', 50), message('working', 'assistant', 'busy', 90)]
|
||||
expect(pendingSendsAsMessages(pending, before).map((row) => row.id)).toEqual([
|
||||
'pending:a',
|
||||
'pending:b',
|
||||
'pending:c'
|
||||
])
|
||||
|
||||
const landed = [...before, imageRow('queued-image', 110)]
|
||||
expect(pendingSendsAsMessages(pending, landed).map((row) => row.id)).toEqual([
|
||||
'pending:a',
|
||||
'pending:c'
|
||||
])
|
||||
expect(prunePendingSends(pending, landed)).toBe(pending)
|
||||
const replied = [...landed, message('reply', 'assistant', 'ok', 120)]
|
||||
const remaining = prunePendingSends(pending, replied)
|
||||
expect(remaining.map((entry) => entry.id)).toEqual(['a', 'c'])
|
||||
expect(pendingSendsAsMessages(remaining, replied).map((row) => row.id)).toEqual([
|
||||
'pending:a',
|
||||
'pending:c'
|
||||
])
|
||||
expect(prunePendingSends(remaining, replied).map((entry) => entry.id)).toEqual(['a', 'c'])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { nativeChatContinuationSendText } from './native-chat-continuation-send'
|
||||
import type { NativeChatMessage } from '../native-chat-types'
|
||||
import { isImageRefBlock, type NativeChatMessage } from '../native-chat-types'
|
||||
import { normalizedNativeChatUserMessageText } from './native-chat-image-transcript-markers'
|
||||
import {
|
||||
advancedNativeChatUserContentCounts,
|
||||
advancedNativeChatUserRows,
|
||||
@@ -9,6 +10,7 @@ import {
|
||||
nativeChatPendingMatchKey,
|
||||
nativeChatPendingMatchingAfter,
|
||||
nativeChatPendingOccurrence,
|
||||
normalizeNativeChatPendingText,
|
||||
selectPendingIndicesRepresentedByUserRows,
|
||||
type NativeChatGluedUserRow,
|
||||
type NativeChatUserRow
|
||||
@@ -40,6 +42,8 @@ export type NativeChatPendingSend = {
|
||||
matchingOccurrence?: number
|
||||
/** Shared time boundary when that message boundary is unavailable. */
|
||||
matchingAfterTimestamp?: number
|
||||
/** Pathless image rows already claimed by pruned sends, so this send cannot reclaim them. */
|
||||
claimedImageRowIds?: readonly string[]
|
||||
}
|
||||
|
||||
function messagesAfterPendingBoundary(
|
||||
@@ -132,6 +136,52 @@ function gluedCandidateRows(
|
||||
})
|
||||
}
|
||||
|
||||
// Pathless image rows (Claude queued prompts) cannot name their send, so each is claimed once, oldest send first.
|
||||
// Failed or uncertain writes never claim one, so another send's image cannot hide them.
|
||||
function claimPathlessImageRow(
|
||||
entry: NativeChatPendingSend,
|
||||
allMessages: readonly NativeChatMessage[],
|
||||
claimed: Set<string>,
|
||||
advancedOnly: boolean
|
||||
): boolean {
|
||||
const count = entry.imagePaths?.filter(Boolean).length ?? 0
|
||||
if (
|
||||
count === 0 ||
|
||||
// Only a real row boundary or a proven-empty session; client-clock boundaries could bind old images.
|
||||
(typeof entry.afterMessageId !== 'string' && !entry.afterEmptyTranscriptSessionId) ||
|
||||
entry.delivery !== undefined ||
|
||||
entry.writeUnconfirmedAt !== undefined ||
|
||||
normalizeNativeChatPendingText(entry.text)
|
||||
) {
|
||||
return false
|
||||
}
|
||||
const messages = messagesAfterPendingBoundary(allMessages, entry)
|
||||
const lastNonUser = messages.findLastIndex((message) => message.role !== 'user')
|
||||
const row = messages.find((message, index) => {
|
||||
if (
|
||||
message.role !== 'user' ||
|
||||
claimed.has(message.id) ||
|
||||
(advancedOnly && index > lastNonUser)
|
||||
) {
|
||||
return false
|
||||
}
|
||||
const images = message.blocks.filter(isImageRefBlock)
|
||||
return (
|
||||
images.length === count &&
|
||||
images.every((image) => !image.path) &&
|
||||
!normalizedNativeChatUserMessageText(message)
|
||||
)
|
||||
})
|
||||
if (row) {
|
||||
claimed.add(row.id)
|
||||
}
|
||||
return Boolean(row)
|
||||
}
|
||||
|
||||
function carriedImageRowClaims(pending: readonly NativeChatPendingSend[]): Set<string> {
|
||||
return new Set(pending.flatMap((entry) => entry.claimedImageRowIds ?? []))
|
||||
}
|
||||
|
||||
/** Keep echoes until a later non-user turn to avoid first-turn flashes during transient empty reads. */
|
||||
export function prunePendingSends<T extends NativeChatPendingSend>(
|
||||
pending: T[],
|
||||
@@ -141,6 +191,8 @@ export function prunePendingSends<T extends NativeChatPendingSend>(
|
||||
return pending
|
||||
}
|
||||
const consumed = new Map<string, number>()
|
||||
const claimedImageRows = carriedImageRowClaims(pending)
|
||||
const carriedClaimCount = claimedImageRows.size
|
||||
const exactKeep = pending.map((entry) => {
|
||||
const contentKey = nativeChatPendingContentKey(entry)
|
||||
const key = nativeChatPendingMatchKey(entry)
|
||||
@@ -152,7 +204,7 @@ export function prunePendingSends<T extends NativeChatPendingSend>(
|
||||
const used = consumed.get(key) ?? 0
|
||||
const occurrence = nativeChatPendingOccurrence(entry, used)
|
||||
consumed.set(key, Math.max(used, occurrence))
|
||||
return occurrence > available
|
||||
return occurrence > available && !claimPathlessImageRow(entry, messages, claimedImageRows, true)
|
||||
})
|
||||
// Lost Enter can glue sends into one transcript row; retire them after the assistant advances.
|
||||
const stillOpen = pending.filter((_, index) => exactKeep[index])
|
||||
@@ -167,7 +219,15 @@ export function prunePendingSends<T extends NativeChatPendingSend>(
|
||||
const openIndex = stillOpen.indexOf(entry)
|
||||
return openIndex === -1 || !gluedRepresented.has(openIndex)
|
||||
})
|
||||
return next.length === pending.length ? pending : next
|
||||
if (next.length === pending.length) {
|
||||
return pending
|
||||
}
|
||||
// Claims here retire their sends; hand them to the remaining image sends so none reuses the row.
|
||||
return claimedImageRows.size === carriedClaimCount
|
||||
? next
|
||||
: next.map((entry) =>
|
||||
entry.imagePaths?.length ? { ...entry, claimedImageRowIds: [...claimedImageRows] } : entry
|
||||
)
|
||||
}
|
||||
|
||||
/** Sends whose transcript row has not landed yet; exact and glued matches both hide an echo. */
|
||||
@@ -179,6 +239,7 @@ export function visibleNativeChatPendingSends<T extends NativeChatPendingSend>(
|
||||
return pending
|
||||
}
|
||||
const consumed = new Map<string, number>()
|
||||
const claimedImageRows = carriedImageRowClaims(pending)
|
||||
const exactVisible = pending.map((entry) => {
|
||||
const contentKey = nativeChatPendingContentKey(entry)
|
||||
const key = nativeChatPendingMatchKey(entry)
|
||||
@@ -190,7 +251,10 @@ export function visibleNativeChatPendingSends<T extends NativeChatPendingSend>(
|
||||
const used = consumed.get(key) ?? 0
|
||||
const occurrence = nativeChatPendingOccurrence(entry, used)
|
||||
consumed.set(key, Math.max(used, occurrence))
|
||||
return occurrence > represented
|
||||
return (
|
||||
occurrence > represented &&
|
||||
!claimPathlessImageRow(entry, existingMessages, claimedImageRows, false)
|
||||
)
|
||||
})
|
||||
// Hide glued echoes once their user row exists, before assistant advancement permits pruning.
|
||||
const stillVisible = pending.filter((_, index) => exactVisible[index])
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
} from './native-chat/transcript/native-chat-subagent-summary'
|
||||
import { subagentFallbackText } from './native-chat/transcript/native-chat-subagent-items'
|
||||
import type { NativeChatMessage } from './native-chat/native-chat-types'
|
||||
import { isInlineDataUrl } from './native-chat/native-chat-inline-data-url'
|
||||
|
||||
export function formatWorkerTranscriptMessage(message: NativeChatMessage): string {
|
||||
const fallbacks = workerTranscriptBlockFallbacks(message.blocks)
|
||||
@@ -21,7 +22,7 @@ export function formatWorkerTranscriptMessage(message: NativeChatMessage): strin
|
||||
return `[tool result${block.isError ? ' error' : ''}] ${block.output}`
|
||||
}
|
||||
if (block.type === 'image-ref') {
|
||||
return block.url ? `[image] ${block.url}` : `[image omitted]`
|
||||
return block.url && !isInlineDataUrl(block.url) ? `[image] ${block.url}` : `[image omitted]`
|
||||
}
|
||||
if (
|
||||
block.type === 'background-task' ||
|
||||
|
||||
Reference in New Issue
Block a user