mirror of
https://github.com/stablyai/orca.git
synced 2026-11-15 02:17:31 +00:00
fix(native-chat): show images agents read, and markdown images on mobile (#36953)
A Claude Read of a PNG returns only base64; the path is on the call. A shared rule turns each finished file-read of a raster image into an image-ref under the tool run header on desktop and mobile. Mobile chat prose now draws https, data and absolute-path markdown images inline, every image in a paragraph, through the existing host image read.
This commit is contained in:
@@ -245,3 +245,21 @@ it('renders document images through the image renderer, including reference synt
|
||||
expect(renderImage).toHaveBeenCalledWith('assets/diagram.png', 'Architecture')
|
||||
expect(renderImage).toHaveBeenCalledWith('assets/other.png', 'Reference')
|
||||
})
|
||||
|
||||
it('draws every image in a paragraph and keeps the text around them, leaving declined images as links', () => {
|
||||
const renderImage = vi.fn((href: string, alt: string) =>
|
||||
href.startsWith('/') ? createElement('View', { testID: `image:${href}`, alt }) : null
|
||||
)
|
||||
act(() => {
|
||||
renderer = create(
|
||||
createElement(MobileMarkdown, {
|
||||
content:
|
||||
'Before and after: \n\n\n done',
|
||||
renderImage
|
||||
})
|
||||
)
|
||||
})
|
||||
const images = renderer!.root.findAll((node) => String(node.props.testID).startsWith('image:'))
|
||||
expect(images.map((node) => node.props.testID)).toEqual(['image:/tmp/a.png', 'image:/tmp/b.png'])
|
||||
expect(texts(renderer!)).toEqual(['Before and after:', 'three done'])
|
||||
})
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { Code, List, RootContent, Table } from 'mdast'
|
||||
import type { Code, List, PhrasingContent, RootContent, Table } from 'mdast'
|
||||
import { Fragment, type ReactNode } from 'react'
|
||||
import {
|
||||
Pressable,
|
||||
@@ -39,6 +39,7 @@ export type MarkdownBlockContext = MarkdownInlineContext & {
|
||||
/** Interactive children own their touches and must forward the row action. */
|
||||
rowLongPress?: () => void
|
||||
directives: readonly NativeChatVisualDirective[]
|
||||
/** Null or undefined leaves the image a link. */
|
||||
renderImage?: (href: string, alt: string) => ReactNode
|
||||
renderVisual?: (directive: NativeChatVisualDirective, index: number) => ReactNode
|
||||
mermaidOccurrences: Map<string, number>
|
||||
@@ -130,6 +131,78 @@ function renderTable(node: Table, key: number, ctx: MarkdownBlockContext): React
|
||||
)
|
||||
}
|
||||
|
||||
function isBlankInline(node: PhrasingContent): boolean {
|
||||
return node.type === 'break' || (node.type === 'text' && node.value.trim() === '')
|
||||
}
|
||||
|
||||
/** Images draw as blocks between the paragraph's text runs; an image `renderImage` declines stays a link. */
|
||||
function renderImageParagraph(
|
||||
children: PhrasingContent[],
|
||||
key: number,
|
||||
ctx: MarkdownBlockContext,
|
||||
text: ParagraphText
|
||||
): ReactNode {
|
||||
const { renderImage } = ctx
|
||||
if (!renderImage || !children.some((child) => child.type === 'image')) {
|
||||
return null
|
||||
}
|
||||
const parts: ReactNode[] = []
|
||||
let drewImage = false
|
||||
let run: PhrasingContent[] = []
|
||||
const flushRun = (): void => {
|
||||
if (!run.every(isBlankInline)) {
|
||||
parts.push(
|
||||
<MarkdownText key={parts.length} selectable={text.selectable} style={text.style}>
|
||||
{renderInline(trimRunEdges(run), ctx)}
|
||||
</MarkdownText>
|
||||
)
|
||||
}
|
||||
run = []
|
||||
}
|
||||
for (const child of children) {
|
||||
const image = child.type === 'image' ? renderImage(child.url, child.alt ?? '') : null
|
||||
if (image === null || image === undefined) {
|
||||
run.push(child)
|
||||
continue
|
||||
}
|
||||
flushRun()
|
||||
parts.push(<Fragment key={parts.length}>{image}</Fragment>)
|
||||
drewImage = true
|
||||
}
|
||||
if (!drewImage) {
|
||||
return null
|
||||
}
|
||||
flushRun()
|
||||
// One wrapper whatever the part count, so a growing paragraph never remounts its images.
|
||||
return (
|
||||
<View key={key} style={styles.imageParagraph}>
|
||||
{parts}
|
||||
</View>
|
||||
)
|
||||
}
|
||||
|
||||
// The line breaks beside an image would otherwise leave blank lines in the text around it.
|
||||
function trimRunEdges(run: PhrasingContent[]): PhrasingContent[] {
|
||||
let start = 0
|
||||
let end = run.length
|
||||
while (start < end && run[start]?.type === 'break') {
|
||||
start += 1
|
||||
}
|
||||
while (end > start && run[end - 1]?.type === 'break') {
|
||||
end -= 1
|
||||
}
|
||||
const trimmed = run.slice(start, end)
|
||||
const first = trimmed[0]
|
||||
if (first?.type === 'text') {
|
||||
trimmed[0] = { ...first, value: first.value.trimStart() }
|
||||
}
|
||||
const last = trimmed.at(-1)
|
||||
if (last?.type === 'text') {
|
||||
trimmed[trimmed.length - 1] = { ...last, value: last.value.trimEnd() }
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
function renderBlock(
|
||||
node: RootContent,
|
||||
key: number,
|
||||
@@ -159,8 +232,9 @@ function renderBlock(
|
||||
</Fragment>
|
||||
) : null
|
||||
}
|
||||
if (node.children.length === 1 && only?.type === 'image' && ctx.renderImage) {
|
||||
return <Fragment key={key}>{ctx.renderImage(only.url, only.alt ?? '')}</Fragment>
|
||||
const withImages = renderImageParagraph(node.children, key, ctx, text)
|
||||
if (withImages) {
|
||||
return withImages
|
||||
}
|
||||
if (node.children.length === 1 && only?.type === 'image' && /^https?:\/\//i.test(only.url)) {
|
||||
return (
|
||||
|
||||
@@ -104,6 +104,9 @@ export const styles = themedStyleSheet(() => ({
|
||||
lineHeight: 17,
|
||||
color: colors.textPrimary
|
||||
},
|
||||
imageParagraph: {
|
||||
gap: spacing.sm
|
||||
},
|
||||
imageFrame: {
|
||||
borderWidth: 1,
|
||||
borderColor: colors.borderSubtle,
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
import './mobile-agent-message-renderer.test-fixture'
|
||||
import { createElement, type ReactNode } from 'react'
|
||||
import { act, create, type ReactTestRenderer } from 'react-test-renderer'
|
||||
import { afterEach, expect, it, vi } from 'vitest'
|
||||
import type { NativeChatMessage } from '../../../../../src/shared/native-chat/native-chat-types'
|
||||
import { stubRpcClient } from '../../../transport/rpc-client-test-stub'
|
||||
import {
|
||||
findAllByHostType,
|
||||
findByHostType
|
||||
} from '../../../test-support/react-test-renderer-host-nodes'
|
||||
import { MobileNativeChatRowOverlaysContext } from './mobile-native-chat-row-overlays'
|
||||
import { MobileNativeChatMessage } from './MobileNativeChatMessage'
|
||||
|
||||
vi.mock('@react-native-async-storage/async-storage', () => ({
|
||||
default: { getItem: async () => null, setItem: async () => {}, removeItem: async () => {} }
|
||||
}))
|
||||
|
||||
const PNG =
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=='
|
||||
|
||||
let renderer: ReactTestRenderer | null = null
|
||||
let sendRequest = vi.fn()
|
||||
|
||||
afterEach(() => {
|
||||
act(() => renderer?.unmount())
|
||||
renderer = null
|
||||
})
|
||||
|
||||
async function renderWithHost(node: ReactNode): Promise<ReactTestRenderer> {
|
||||
act(() => renderer?.unmount())
|
||||
sendRequest = vi.fn(async () => ({
|
||||
id: '1',
|
||||
ok: true,
|
||||
result: { content: PNG, isBinary: true, isImage: true, mimeType: 'image/png' }
|
||||
}))
|
||||
const value = {
|
||||
openImage: vi.fn(),
|
||||
openActions: vi.fn(),
|
||||
imageSource: { client: stubRpcClient({ sendRequest }), worktreeId: 'wt-1' }
|
||||
}
|
||||
await act(async () => {
|
||||
renderer = create(createElement(MobileNativeChatRowOverlaysContext.Provider, { value }, node))
|
||||
})
|
||||
return renderer!
|
||||
}
|
||||
|
||||
function assistant(blocks: NativeChatMessage['blocks']) {
|
||||
const message: NativeChatMessage = {
|
||||
id: 'm1',
|
||||
role: 'assistant',
|
||||
source: 'transcript',
|
||||
blocks,
|
||||
timestamp: null
|
||||
}
|
||||
return createElement(MobileNativeChatMessage, { message })
|
||||
}
|
||||
|
||||
const hostRead = (path: string) => [
|
||||
'files.readChatImage',
|
||||
{ worktree: 'id:wt-1', path },
|
||||
expect.anything()
|
||||
]
|
||||
const thumbnails = (tree: ReactTestRenderer) =>
|
||||
findAllByHostType(tree.root, 'Image').map((image) => image.props.source.uri)
|
||||
|
||||
it('draws prose images from the web or the host, and leaves the rest as links', async () => {
|
||||
const tree = await renderWithHost(assistant([{ type: 'text', text: '' }]))
|
||||
const { renderImage } = findByHostType(tree.root, 'MobileMarkdown').props
|
||||
const draw = (href: string) => renderWithHost(renderImage(href, 'shot'))
|
||||
|
||||
for (const [href, path] of [
|
||||
['/tmp/s.png', '/tmp/s.png'],
|
||||
['file:///tmp/s%20b.png', '/tmp/s b.png'],
|
||||
['C:/shots/s.png', 'C:/shots/s.png']
|
||||
]) {
|
||||
const image = await draw(href!)
|
||||
expect(sendRequest).toHaveBeenCalledWith(...hostRead(path!))
|
||||
expect(thumbnails(image)).toEqual([`data:image/png;base64,${PNG}`])
|
||||
}
|
||||
for (const url of ['https://example.com/s.png', 'data:image/png;base64,AA']) {
|
||||
expect(thumbnails(await draw(url))).toEqual([url])
|
||||
expect(sendRequest).not.toHaveBeenCalled()
|
||||
}
|
||||
for (const kept of [
|
||||
'shots/s.png',
|
||||
'http://192.168.1.10:8000/s.png',
|
||||
'data:image/svg+xml;base64,AA',
|
||||
'/repo/logo.svg',
|
||||
'https://img.example/b.svg?v=1',
|
||||
'/repo/out/report.pdf'
|
||||
]) {
|
||||
expect(renderImage(kept, 'shot')).toBeNull()
|
||||
}
|
||||
})
|
||||
|
||||
it('shows the image a Read returned under the closed run', async () => {
|
||||
const tree = await renderWithHost(
|
||||
assistant([
|
||||
{ type: 'tool-call', name: 'Read', callId: 'c1', input: { file_path: '/tmp/s2.png' } },
|
||||
{ type: 'tool-result', callId: 'c1', output: '' }
|
||||
])
|
||||
)
|
||||
expect(sendRequest).toHaveBeenCalledWith(...hostRead('/tmp/s2.png'))
|
||||
expect(thumbnails(tree)).toEqual([`data:image/png;base64,${PNG}`])
|
||||
})
|
||||
@@ -3,6 +3,7 @@ import { Text, View } from 'react-native'
|
||||
import { INLINE_TEXT_SELECTION } from '../../../components/inline-text-selection'
|
||||
import { MobileNativeChatLongPressContent as Content } from '../MobileNativeChatLongPressContent'
|
||||
import { splitNativeChatBlocks } from '../../../../../src/shared/native-chat/tools/native-chat-tool-fold'
|
||||
import { nativeChatToolRunViewedImages } from '../../../../../src/shared/native-chat/tools/native-chat-tool-viewed-images'
|
||||
import { selectActiveToolCall } from '../../../../../src/shared/native-chat/tools/native-chat-tool-activity'
|
||||
import {
|
||||
isImageRefBlock,
|
||||
@@ -154,6 +155,11 @@ function MobileNativeChatMessageImpl({
|
||||
}
|
||||
return { before, after: run.thoughtsAfter.map(thought) }
|
||||
}, [run, fontScale, activeTurnIsWorking, openReasoning, onToggleReasoning, onOpenFile, openSheet])
|
||||
// Asks and prose read no files, so the run's own blocks or this message's give the same images.
|
||||
const viewedImages = useMemo(
|
||||
() => nativeChatToolRunViewedImages(run?.blocks ?? message.blocks),
|
||||
[run, message.blocks]
|
||||
)
|
||||
const drawnProse = isReasoning ? [] : withoutSubagentGroupTwins(prose)
|
||||
const isSystem = message.role === 'system'
|
||||
const showToolRun = tools.length > 0
|
||||
@@ -331,6 +337,20 @@ function MobileNativeChatMessageImpl({
|
||||
previousUpdatePlan={previousUpdatePlan}
|
||||
fontScale={fontScale}
|
||||
onOpenFile={onOpenFile}
|
||||
viewedImages={
|
||||
viewedImages.length > 0
|
||||
? // Keyed by path: a read finishing late inserts above images already loaded.
|
||||
viewedImages.map((image) => (
|
||||
<Prose
|
||||
key={image.path}
|
||||
block={image}
|
||||
text=""
|
||||
fontScale={fontScale}
|
||||
onLongPress={onLongPress}
|
||||
/>
|
||||
))
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
) : null}
|
||||
</Content>
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useContext, useEffect, useState } from 'react'
|
||||
import { useCallback, useContext, useEffect, useState } from 'react'
|
||||
import { Image, Pressable, Text, View } from 'react-native'
|
||||
import { INLINE_TEXT_SELECTION } from '../../../components/inline-text-selection'
|
||||
import { MobileMarkdown } from '../../../components/MobileMarkdown'
|
||||
@@ -27,6 +27,34 @@ import { useMobileColorScheme } from '../../../theme/use-mobile-color-scheme'
|
||||
import { useMobileTranslation } from '../../../i18n/use-mobile-translation'
|
||||
import { nativeChatHostStatusCopy } from './native-chat-notice-copy'
|
||||
import { pressedStyle } from '../../../theme/mobile-theme'
|
||||
import { routeNativeChatHref } from '../../../../../src/shared/native-chat/native-chat-href-routing'
|
||||
import { isRuntimePathAbsolute } from '../../../../../src/shared/cross-platform-path'
|
||||
import { isRasterImagePath } from '../../../../../src/shared/file-preview-kind'
|
||||
import { isKnownRasterImageMimeType } from '../../../../../src/shared/raster-image-preview-limits'
|
||||
import { isSvgFilePath } from '../../../../../src/shared/svg-document'
|
||||
|
||||
/** Raster https, data and absolute-path images draw on the phone (paths through the chat's host); anything else stays a link.
|
||||
* The shell's CSP refuses plain http, and React Native's Image cannot draw SVG. */
|
||||
function markdownImageBlock(href: string, alt: string): NativeChatImageRefBlock | null {
|
||||
const label = alt.trim() ? { alt } : {}
|
||||
const source = href.trim()
|
||||
if (/^data:/i.test(source)) {
|
||||
return isKnownRasterImageMimeType(source.slice(5).split(/[;,]/, 1)[0])
|
||||
? { type: 'image-ref', url: source, ...label }
|
||||
: null
|
||||
}
|
||||
const route = routeNativeChatHref(source)
|
||||
if (route.kind === 'web') {
|
||||
return /^https:/i.test(route.url) && !isSvgFilePath(route.url.split(/[?#]/, 1)[0] ?? '')
|
||||
? { type: 'image-ref', url: route.url, ...label }
|
||||
: null
|
||||
}
|
||||
return route.kind === 'file' &&
|
||||
isRuntimePathAbsolute(route.pathText) &&
|
||||
isRasterImagePath(route.pathText)
|
||||
? { type: 'image-ref', path: route.pathText, ...label }
|
||||
: null
|
||||
}
|
||||
|
||||
export function Prose({
|
||||
block,
|
||||
@@ -54,6 +82,15 @@ export function Prose({
|
||||
}): React.JSX.Element | null {
|
||||
useMobileColorScheme()
|
||||
const t = useMobileTranslation()
|
||||
const renderImage = useCallback(
|
||||
(href: string, alt: string): React.ReactNode => {
|
||||
const image = markdownImageBlock(href, alt)
|
||||
return image ? (
|
||||
<ImageAttachment block={image} fontScale={fontScale} onLongPress={onLongPress} />
|
||||
) : null
|
||||
},
|
||||
[fontScale, onLongPress]
|
||||
)
|
||||
if (isTextBlock(block)) {
|
||||
if (isAgentSessionHostStatusPresentation(block.presentation)) {
|
||||
return (
|
||||
@@ -74,6 +111,7 @@ export function Prose({
|
||||
textScale={CHAT_PROSE_SCALE * fontScale}
|
||||
onOpenFile={onOpenFile}
|
||||
onLongPress={onLongPress}
|
||||
renderImage={renderImage}
|
||||
/>
|
||||
)
|
||||
}
|
||||
@@ -90,6 +128,7 @@ export function Prose({
|
||||
textScale={CHAT_PROSE_SCALE * fontScale}
|
||||
onOpenFile={onOpenFile}
|
||||
onLongPress={onLongPress}
|
||||
renderImage={renderImage}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -175,7 +175,8 @@ export function ToolRun({
|
||||
previousTodoWrite,
|
||||
previousUpdatePlan,
|
||||
fontScale = 1,
|
||||
onOpenFile
|
||||
onOpenFile,
|
||||
viewedImages
|
||||
}: MobileNativeChatToolRunProps): React.JSX.Element {
|
||||
useMobileColorScheme()
|
||||
const t = useMobileTranslation()
|
||||
@@ -245,6 +246,8 @@ export function ToolRun({
|
||||
</Text>
|
||||
) : null}
|
||||
</Pressable>
|
||||
{/* A read's result is the image the agent saw, so it shows without opening the run. */}
|
||||
{viewedImages ? <View style={styles.viewedImages}>{viewedImages}</View> : null}
|
||||
{open ? (
|
||||
<ToolRunMembers
|
||||
followKey={blocks.length + asideCount}
|
||||
|
||||
@@ -24,4 +24,6 @@ export type MobileNativeChatToolRunProps = {
|
||||
awaitingAnswer?: boolean
|
||||
fontScale?: number
|
||||
onOpenFile?: (path: string) => void
|
||||
/** Thumbnails of the images the run's reads returned, drawn under its header. */
|
||||
viewedImages?: React.ReactNode
|
||||
}
|
||||
|
||||
@@ -18,6 +18,12 @@ export const toolStyles = themedStyleSheet(() => ({
|
||||
run: {
|
||||
marginTop: spacing.xs
|
||||
},
|
||||
viewedImages: {
|
||||
flexDirection: 'row',
|
||||
flexWrap: 'wrap',
|
||||
gap: spacing.sm,
|
||||
marginTop: spacing.xs
|
||||
},
|
||||
header: {
|
||||
flexDirection: 'row',
|
||||
alignItems: 'center',
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { foldToolMessages } from '../../shared/native-chat/tools/native-chat-tool-fold'
|
||||
import { nativeChatToolRunViewedImages } from '../../shared/native-chat/tools/native-chat-tool-viewed-images'
|
||||
import type { NativeChatMessage } from '../../shared/native-chat/native-chat-types'
|
||||
import { decodeClaudeTranscriptLine } from './transcript-line-decoders-claude'
|
||||
|
||||
// Claude 2.x Read of an image: the result holds only base64, the path is on the call.
|
||||
function readTurn(filePath: string, result: Record<string, unknown>): string[] {
|
||||
return [
|
||||
JSON.stringify({
|
||||
type: 'assistant',
|
||||
uuid: 'call-row',
|
||||
timestamp: '2026-10-29T14:09:20.000Z',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: [{ type: 'tool_use', id: 'toolu_1', name: 'Read', input: { file_path: filePath } }]
|
||||
}
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: 'user',
|
||||
uuid: 'result-row',
|
||||
timestamp: '2026-10-29T14:09:23.939Z',
|
||||
message: {
|
||||
role: 'user',
|
||||
content: [{ tool_use_id: 'toolu_1', type: 'tool_result', ...result }]
|
||||
},
|
||||
toolUseResult: { type: 'image', file: { base64: 'iVBORw0KGgo=', type: 'image/png' } }
|
||||
})
|
||||
]
|
||||
}
|
||||
|
||||
const IMAGE_RESULT = {
|
||||
content: [
|
||||
{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'iVBORw0KGgo=' } }
|
||||
]
|
||||
}
|
||||
|
||||
function viewedImages(lines: string[]): ReturnType<typeof nativeChatToolRunViewedImages> {
|
||||
const decoded = lines
|
||||
.map((line, index) => decodeClaudeTranscriptLine(line, `fallback-${index}`))
|
||||
.filter((message): message is NativeChatMessage => message !== null)
|
||||
const folded = foldToolMessages(decoded)
|
||||
expect(folded).toHaveLength(1)
|
||||
return nativeChatToolRunViewedImages(folded[0]!.blocks)
|
||||
}
|
||||
|
||||
describe('Claude Read of an image', () => {
|
||||
it('shows the screenshot the agent read', () => {
|
||||
expect(viewedImages(readTurn('/tmp/orca-annot/s2.png', IMAGE_RESULT))).toEqual([
|
||||
{ type: 'image-ref', path: '/tmp/orca-annot/s2.png' }
|
||||
])
|
||||
})
|
||||
|
||||
it('shows nothing for a read that failed or returned text', () => {
|
||||
const failed = { content: 'File does not exist.', is_error: true }
|
||||
expect(viewedImages(readTurn('/tmp/gone.png', failed))).toEqual([])
|
||||
expect(viewedImages(readTurn('/repo/src/a.ts', { content: '1\tconst a = 1' }))).toEqual([])
|
||||
expect(viewedImages(readTurn('/repo/icons/logo.svg', { content: '1\t<svg/>' }))).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
it('shows nothing for a shell read, which gave the agent bytes as text', () => {
|
||||
const codexCat = {
|
||||
type: 'tool-call',
|
||||
name: 'read',
|
||||
callId: 'c1',
|
||||
input: { command: 'cat /repo/images/shot.png', cwd: '/repo', path: '/repo/images/shot.png' }
|
||||
} as const
|
||||
expect(
|
||||
nativeChatToolRunViewedImages([codexCat, { type: 'tool-result', callId: 'c1', output: '' }])
|
||||
).toEqual([])
|
||||
})
|
||||
@@ -132,6 +132,7 @@ function toolRunProps(
|
||||
activeTurnIsWorking: props.activeTurnIsWorking,
|
||||
trailing: props.trailingRun,
|
||||
disclosureId: props.message.id,
|
||||
asides: run?.asides
|
||||
asides: run?.asides,
|
||||
runtimeContext: props.runtimeContext
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,6 +31,9 @@ import { NativeChatToolRunCallCounts } from './NativeChatToolRunCallCounts'
|
||||
import { NativeChatToolRunMembers, revealNativeChatToolRunMember } from './NativeChatToolRunMembers'
|
||||
import { NativeChatToolRunMemberList } from './NativeChatToolRunMemberList'
|
||||
import { NativeChatBrowserRunPageButton } from './NativeChatBrowserPageButton'
|
||||
import { NativeChatImageAttachments } from './NativeChatTranscriptChrome'
|
||||
import { nativeChatToolRunViewedImages } from '../../../../shared/native-chat/tools/native-chat-tool-viewed-images'
|
||||
import type { RuntimeFileOperationArgs } from '@/runtime/runtime-file-client-types'
|
||||
|
||||
/** Insert keyed asides before their block or after the final block. */
|
||||
export type NativeChatToolRunAsides = {
|
||||
@@ -58,7 +61,8 @@ export function NativeChatToolRun({
|
||||
trailing,
|
||||
disclosureId,
|
||||
onLinkClick,
|
||||
asides
|
||||
asides,
|
||||
runtimeContext
|
||||
}: {
|
||||
blocks: NativeChatBlock[]
|
||||
previousTodoWrite?: NativeChatToolCallBlock
|
||||
@@ -84,6 +88,7 @@ export function NativeChatToolRun({
|
||||
disclosureId?: string
|
||||
onLinkClick?: CommentMarkdownLinkClickHandler
|
||||
asides?: NativeChatToolRunAsides
|
||||
runtimeContext?: RuntimeFileOperationArgs | null
|
||||
}): React.JSX.Element | null {
|
||||
// This row owns the language subscription for its tool, diff, and task labels.
|
||||
useTranslation()
|
||||
@@ -142,6 +147,7 @@ export function NativeChatToolRun({
|
||||
() => nativeChatToolRunHeader(blocks, { activeTurnIsWorking, trailing }),
|
||||
[activeTurnIsWorking, blocks, trailing]
|
||||
)
|
||||
const viewedImages = useMemo(() => nativeChatToolRunViewedImages(headerBlocks), [headerBlocks])
|
||||
const hasAskCall = asks.length > 0
|
||||
const askSubject = hasAskCall ? nativeChatAskRunSubject(asks) : null
|
||||
const showsHeader = !hasAskCall || callCount > 0
|
||||
@@ -231,6 +237,12 @@ export function NativeChatToolRun({
|
||||
<NativeChatBrowserRunPageButton calls={headerBlocks.filter(isToolCallBlock)} />
|
||||
</div>
|
||||
)}
|
||||
{/* A read's result is the image the agent saw, so it shows without opening the run. */}
|
||||
<NativeChatImageAttachments
|
||||
blocks={viewedImages}
|
||||
runtimeContext={runtimeContext}
|
||||
className="mb-0 mt-1"
|
||||
/>
|
||||
<NativeChatExpandable open={open && showsHeader}>
|
||||
<NativeChatToolRunMembers
|
||||
followKey={blocks.length + asideCount}
|
||||
|
||||
@@ -208,12 +208,14 @@ function TranscriptImagePreview({
|
||||
export function NativeChatImageAttachments({
|
||||
blocks,
|
||||
runtimeContext,
|
||||
enablePreview = runtimeContext !== undefined
|
||||
enablePreview = runtimeContext !== undefined,
|
||||
className
|
||||
}: {
|
||||
blocks: NativeChatBlock[]
|
||||
runtimeContext?: RuntimeFileOperationArgs | null
|
||||
/** Callers without an image owner get label-only chips. */
|
||||
enablePreview?: boolean
|
||||
className?: string
|
||||
}): React.JSX.Element | null {
|
||||
const images = blocks.filter((block) => block.type === 'image-ref')
|
||||
if (images.length === 0) {
|
||||
@@ -222,7 +224,7 @@ export function NativeChatImageAttachments({
|
||||
const imageKeyCounts = new Map<string, number>()
|
||||
if (!enablePreview) {
|
||||
return (
|
||||
<div className="mb-2 flex flex-wrap gap-1.5">
|
||||
<div className={cn('mb-2 flex flex-wrap gap-1.5', className)}>
|
||||
{images.map((image) => {
|
||||
const label =
|
||||
image.alt ??
|
||||
@@ -253,7 +255,7 @@ export function NativeChatImageAttachments({
|
||||
)
|
||||
}
|
||||
return (
|
||||
<div className="mb-2 flex flex-wrap gap-1.5">
|
||||
<div className={cn('mb-2 flex flex-wrap gap-1.5', className)}>
|
||||
{images.map((image) => {
|
||||
const label =
|
||||
image.alt ??
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { IMAGE_FILE_MIME_TYPES } from './image-file-extensions'
|
||||
import { MEDIA_FILE_MIME_TYPES } from './media-file-extensions'
|
||||
import { isKnownRasterImageMimeType } from './raster-image-preview-limits'
|
||||
|
||||
export type FilePreviewKind = 'image' | 'pdf' | 'audio' | 'video'
|
||||
export type FilePreview = { kind: FilePreviewKind; mimeType: string }
|
||||
@@ -32,6 +33,12 @@ export function filePreviewForPath(filePath: string): FilePreview | null {
|
||||
return null
|
||||
}
|
||||
|
||||
/** Images an image element without an SVG engine (React Native's) can draw. */
|
||||
export function isRasterImagePath(filePath: string): boolean {
|
||||
const preview = filePreviewForPath(filePath)
|
||||
return preview?.kind === 'image' && isKnownRasterImageMimeType(preview.mimeType)
|
||||
}
|
||||
|
||||
/** Audio and video always stream by range; their players can't take base64. */
|
||||
export function streamsPreviewByRange(preview: FilePreview | null): boolean {
|
||||
return preview?.kind === 'audio' || preview?.kind === 'video'
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import { isRuntimePathAbsolute } from '../../cross-platform-path'
|
||||
import { isRasterImagePath } from '../../file-preview-kind'
|
||||
import {
|
||||
isToolCallBlock,
|
||||
type NativeChatBlock,
|
||||
type NativeChatImageRefBlock,
|
||||
type NativeChatToolCallBlock
|
||||
} from '../native-chat-types'
|
||||
import { nativeChatToolCategory } from './native-chat-tool-icon'
|
||||
import { nativeChatFullCommand } from './native-chat-tool-input-text'
|
||||
import { nativeChatToolLineLabel } from './native-chat-tool-line-label'
|
||||
import { pairToolBlocks } from './native-chat-tool-pairs'
|
||||
import { toolFilePath } from './native-chat-tool-summary'
|
||||
|
||||
const NO_IMAGES: NativeChatImageRefBlock[] = []
|
||||
|
||||
/** The raster image a file-read tool names. A shell read (Codex's `cat x.png`) and an SVG read hand
|
||||
* the agent text, not a picture; hosts read only absolute paths. */
|
||||
function imageReadPath(call: NativeChatToolCallBlock): string | null {
|
||||
if (
|
||||
nativeChatToolCategory(call.name, call.mcpIdentity) !== 'read' ||
|
||||
nativeChatFullCommand(call.input) !== null
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const path = toolFilePath(call.input)
|
||||
return path !== null && isRuntimePathAbsolute(path) && isRasterImagePath(path) ? path : null
|
||||
}
|
||||
|
||||
/** Image files a run's tools read; their results hold only the bytes, so each call's path names its image. */
|
||||
export function nativeChatToolRunViewedImages(
|
||||
blocks: readonly NativeChatBlock[]
|
||||
): NativeChatImageRefBlock[] {
|
||||
// Most runs read no images, so they skip pairing.
|
||||
if (!blocks.some((block) => isToolCallBlock(block) && imageReadPath(block) !== null)) {
|
||||
return NO_IMAGES
|
||||
}
|
||||
const paths = new Set<string>()
|
||||
for (const { call, result } of pairToolBlocks(blocks)) {
|
||||
const path = call ? imageReadPath(call) : null
|
||||
// Only a read that finished without error showed the agent its image.
|
||||
if (call && path !== null && nativeChatToolLineLabel(call, result).verb === 'read') {
|
||||
paths.add(path)
|
||||
}
|
||||
}
|
||||
return Array.from(paths, (path) => ({ type: 'image-ref', path }))
|
||||
}
|
||||
Reference in New Issue
Block a user