diff --git a/mobile/src/components/MobileMarkdown.rendering.test.tsx b/mobile/src/components/MobileMarkdown.rendering.test.tsx index 3c75366bcb7..a291531ce88 100644 --- a/mobile/src/components/MobileMarkdown.rendering.test.tsx +++ b/mobile/src/components/MobileMarkdown.rendering.test.tsx @@ -245,3 +245,21 @@ it('renders document images through the image renderer, including reference synt expect(renderImage).toHaveBeenCalledWith('assets/diagram.png', 'Architecture') expect(renderImage).toHaveBeenCalledWith('assets/other.png', 'Reference') }) + +it('draws every image in a paragraph and keeps the text around them, leaving declined images as links', () => { + const renderImage = vi.fn((href: string, alt: string) => + href.startsWith('/') ? createElement('View', { testID: `image:${href}`, alt }) : null + ) + act(() => { + renderer = create( + createElement(MobileMarkdown, { + content: + 'Before and after: \n![one](/tmp/a.png)\n![two](/tmp/b.png)\n![three](rel.png) done', + renderImage + }) + ) + }) + const images = renderer!.root.findAll((node) => String(node.props.testID).startsWith('image:')) + expect(images.map((node) => node.props.testID)).toEqual(['image:/tmp/a.png', 'image:/tmp/b.png']) + expect(texts(renderer!)).toEqual(['Before and after:', 'three done']) +}) diff --git a/mobile/src/components/mobile-markdown-blocks.tsx b/mobile/src/components/mobile-markdown-blocks.tsx index 3b7f226a746..454131d7d67 100644 --- a/mobile/src/components/mobile-markdown-blocks.tsx +++ b/mobile/src/components/mobile-markdown-blocks.tsx @@ -1,4 +1,4 @@ -import type { Code, List, RootContent, Table } from 'mdast' +import type { Code, List, PhrasingContent, RootContent, Table } from 'mdast' import { Fragment, type ReactNode } from 'react' import { Pressable, @@ -39,6 +39,7 @@ export type MarkdownBlockContext = MarkdownInlineContext & { /** Interactive children own their touches and must forward the row action. */ rowLongPress?: () => void directives: readonly NativeChatVisualDirective[] + /** Null or undefined leaves the image a link. */ renderImage?: (href: string, alt: string) => ReactNode renderVisual?: (directive: NativeChatVisualDirective, index: number) => ReactNode mermaidOccurrences: Map @@ -130,6 +131,78 @@ function renderTable(node: Table, key: number, ctx: MarkdownBlockContext): React ) } +function isBlankInline(node: PhrasingContent): boolean { + return node.type === 'break' || (node.type === 'text' && node.value.trim() === '') +} + +/** Images draw as blocks between the paragraph's text runs; an image `renderImage` declines stays a link. */ +function renderImageParagraph( + children: PhrasingContent[], + key: number, + ctx: MarkdownBlockContext, + text: ParagraphText +): ReactNode { + const { renderImage } = ctx + if (!renderImage || !children.some((child) => child.type === 'image')) { + return null + } + const parts: ReactNode[] = [] + let drewImage = false + let run: PhrasingContent[] = [] + const flushRun = (): void => { + if (!run.every(isBlankInline)) { + parts.push( + + {renderInline(trimRunEdges(run), ctx)} + + ) + } + run = [] + } + for (const child of children) { + const image = child.type === 'image' ? renderImage(child.url, child.alt ?? '') : null + if (image === null || image === undefined) { + run.push(child) + continue + } + flushRun() + parts.push({image}) + drewImage = true + } + if (!drewImage) { + return null + } + flushRun() + // One wrapper whatever the part count, so a growing paragraph never remounts its images. + return ( + + {parts} + + ) +} + +// The line breaks beside an image would otherwise leave blank lines in the text around it. +function trimRunEdges(run: PhrasingContent[]): PhrasingContent[] { + let start = 0 + let end = run.length + while (start < end && run[start]?.type === 'break') { + start += 1 + } + while (end > start && run[end - 1]?.type === 'break') { + end -= 1 + } + const trimmed = run.slice(start, end) + const first = trimmed[0] + if (first?.type === 'text') { + trimmed[0] = { ...first, value: first.value.trimStart() } + } + const last = trimmed.at(-1) + if (last?.type === 'text') { + trimmed[trimmed.length - 1] = { ...last, value: last.value.trimEnd() } + } + return trimmed +} + function renderBlock( node: RootContent, key: number, @@ -159,8 +232,9 @@ function renderBlock( ) : null } - if (node.children.length === 1 && only?.type === 'image' && ctx.renderImage) { - return {ctx.renderImage(only.url, only.alt ?? '')} + const withImages = renderImageParagraph(node.children, key, ctx, text) + if (withImages) { + return withImages } if (node.children.length === 1 && only?.type === 'image' && /^https?:\/\//i.test(only.url)) { return ( diff --git a/mobile/src/components/mobile-markdown-styles.ts b/mobile/src/components/mobile-markdown-styles.ts index 4568fb6829b..ddd6ab92b58 100644 --- a/mobile/src/components/mobile-markdown-styles.ts +++ b/mobile/src/components/mobile-markdown-styles.ts @@ -104,6 +104,9 @@ export const styles = themedStyleSheet(() => ({ lineHeight: 17, color: colors.textPrimary }, + imageParagraph: { + gap: spacing.sm + }, imageFrame: { borderWidth: 1, borderColor: colors.borderSubtle, diff --git a/mobile/src/session/native-chat/transcript/MobileNativeChatMessage.images.test.ts b/mobile/src/session/native-chat/transcript/MobileNativeChatMessage.images.test.ts new file mode 100644 index 00000000000..a0771f61332 --- /dev/null +++ b/mobile/src/session/native-chat/transcript/MobileNativeChatMessage.images.test.ts @@ -0,0 +1,105 @@ +import './mobile-agent-message-renderer.test-fixture' +import { createElement, type ReactNode } from 'react' +import { act, create, type ReactTestRenderer } from 'react-test-renderer' +import { afterEach, expect, it, vi } from 'vitest' +import type { NativeChatMessage } from '../../../../../src/shared/native-chat/native-chat-types' +import { stubRpcClient } from '../../../transport/rpc-client-test-stub' +import { + findAllByHostType, + findByHostType +} from '../../../test-support/react-test-renderer-host-nodes' +import { MobileNativeChatRowOverlaysContext } from './mobile-native-chat-row-overlays' +import { MobileNativeChatMessage } from './MobileNativeChatMessage' + +vi.mock('@react-native-async-storage/async-storage', () => ({ + default: { getItem: async () => null, setItem: async () => {}, removeItem: async () => {} } +})) + +const PNG = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==' + +let renderer: ReactTestRenderer | null = null +let sendRequest = vi.fn() + +afterEach(() => { + act(() => renderer?.unmount()) + renderer = null +}) + +async function renderWithHost(node: ReactNode): Promise { + act(() => renderer?.unmount()) + sendRequest = vi.fn(async () => ({ + id: '1', + ok: true, + result: { content: PNG, isBinary: true, isImage: true, mimeType: 'image/png' } + })) + const value = { + openImage: vi.fn(), + openActions: vi.fn(), + imageSource: { client: stubRpcClient({ sendRequest }), worktreeId: 'wt-1' } + } + await act(async () => { + renderer = create(createElement(MobileNativeChatRowOverlaysContext.Provider, { value }, node)) + }) + return renderer! +} + +function assistant(blocks: NativeChatMessage['blocks']) { + const message: NativeChatMessage = { + id: 'm1', + role: 'assistant', + source: 'transcript', + blocks, + timestamp: null + } + return createElement(MobileNativeChatMessage, { message }) +} + +const hostRead = (path: string) => [ + 'files.readChatImage', + { worktree: 'id:wt-1', path }, + expect.anything() +] +const thumbnails = (tree: ReactTestRenderer) => + findAllByHostType(tree.root, 'Image').map((image) => image.props.source.uri) + +it('draws prose images from the web or the host, and leaves the rest as links', async () => { + const tree = await renderWithHost(assistant([{ type: 'text', text: '![shot](/tmp/s.png)' }])) + const { renderImage } = findByHostType(tree.root, 'MobileMarkdown').props + const draw = (href: string) => renderWithHost(renderImage(href, 'shot')) + + for (const [href, path] of [ + ['/tmp/s.png', '/tmp/s.png'], + ['file:///tmp/s%20b.png', '/tmp/s b.png'], + ['C:/shots/s.png', 'C:/shots/s.png'] + ]) { + const image = await draw(href!) + expect(sendRequest).toHaveBeenCalledWith(...hostRead(path!)) + expect(thumbnails(image)).toEqual([`data:image/png;base64,${PNG}`]) + } + for (const url of ['https://example.com/s.png', 'data:image/png;base64,AA']) { + expect(thumbnails(await draw(url))).toEqual([url]) + expect(sendRequest).not.toHaveBeenCalled() + } + for (const kept of [ + 'shots/s.png', + 'http://192.168.1.10:8000/s.png', + 'data:image/svg+xml;base64,AA', + '/repo/logo.svg', + 'https://img.example/b.svg?v=1', + '/repo/out/report.pdf' + ]) { + expect(renderImage(kept, 'shot')).toBeNull() + } +}) + +it('shows the image a Read returned under the closed run', async () => { + const tree = await renderWithHost( + assistant([ + { type: 'tool-call', name: 'Read', callId: 'c1', input: { file_path: '/tmp/s2.png' } }, + { type: 'tool-result', callId: 'c1', output: '' } + ]) + ) + expect(sendRequest).toHaveBeenCalledWith(...hostRead('/tmp/s2.png')) + expect(thumbnails(tree)).toEqual([`data:image/png;base64,${PNG}`]) +}) diff --git a/mobile/src/session/native-chat/transcript/MobileNativeChatMessage.tsx b/mobile/src/session/native-chat/transcript/MobileNativeChatMessage.tsx index 0a1af7b0c68..b32b6adedae 100644 --- a/mobile/src/session/native-chat/transcript/MobileNativeChatMessage.tsx +++ b/mobile/src/session/native-chat/transcript/MobileNativeChatMessage.tsx @@ -3,6 +3,7 @@ import { Text, View } from 'react-native' import { INLINE_TEXT_SELECTION } from '../../../components/inline-text-selection' import { MobileNativeChatLongPressContent as Content } from '../MobileNativeChatLongPressContent' import { splitNativeChatBlocks } from '../../../../../src/shared/native-chat/tools/native-chat-tool-fold' +import { nativeChatToolRunViewedImages } from '../../../../../src/shared/native-chat/tools/native-chat-tool-viewed-images' import { selectActiveToolCall } from '../../../../../src/shared/native-chat/tools/native-chat-tool-activity' import { isImageRefBlock, @@ -154,6 +155,11 @@ function MobileNativeChatMessageImpl({ } return { before, after: run.thoughtsAfter.map(thought) } }, [run, fontScale, activeTurnIsWorking, openReasoning, onToggleReasoning, onOpenFile, openSheet]) + // Asks and prose read no files, so the run's own blocks or this message's give the same images. + const viewedImages = useMemo( + () => nativeChatToolRunViewedImages(run?.blocks ?? message.blocks), + [run, message.blocks] + ) const drawnProse = isReasoning ? [] : withoutSubagentGroupTwins(prose) const isSystem = message.role === 'system' const showToolRun = tools.length > 0 @@ -331,6 +337,20 @@ function MobileNativeChatMessageImpl({ previousUpdatePlan={previousUpdatePlan} fontScale={fontScale} onOpenFile={onOpenFile} + viewedImages={ + viewedImages.length > 0 + ? // Keyed by path: a read finishing late inserts above images already loaded. + viewedImages.map((image) => ( + + )) + : undefined + } /> ) : null} diff --git a/mobile/src/session/native-chat/transcript/MobileNativeChatProse.tsx b/mobile/src/session/native-chat/transcript/MobileNativeChatProse.tsx index 5e3d19457a1..a9c0ca47e0f 100644 --- a/mobile/src/session/native-chat/transcript/MobileNativeChatProse.tsx +++ b/mobile/src/session/native-chat/transcript/MobileNativeChatProse.tsx @@ -1,4 +1,4 @@ -import { useContext, useEffect, useState } from 'react' +import { useCallback, useContext, useEffect, useState } from 'react' import { Image, Pressable, Text, View } from 'react-native' import { INLINE_TEXT_SELECTION } from '../../../components/inline-text-selection' import { MobileMarkdown } from '../../../components/MobileMarkdown' @@ -27,6 +27,34 @@ import { useMobileColorScheme } from '../../../theme/use-mobile-color-scheme' import { useMobileTranslation } from '../../../i18n/use-mobile-translation' import { nativeChatHostStatusCopy } from './native-chat-notice-copy' import { pressedStyle } from '../../../theme/mobile-theme' +import { routeNativeChatHref } from '../../../../../src/shared/native-chat/native-chat-href-routing' +import { isRuntimePathAbsolute } from '../../../../../src/shared/cross-platform-path' +import { isRasterImagePath } from '../../../../../src/shared/file-preview-kind' +import { isKnownRasterImageMimeType } from '../../../../../src/shared/raster-image-preview-limits' +import { isSvgFilePath } from '../../../../../src/shared/svg-document' + +/** Raster https, data and absolute-path images draw on the phone (paths through the chat's host); anything else stays a link. + * The shell's CSP refuses plain http, and React Native's Image cannot draw SVG. */ +function markdownImageBlock(href: string, alt: string): NativeChatImageRefBlock | null { + const label = alt.trim() ? { alt } : {} + const source = href.trim() + if (/^data:/i.test(source)) { + return isKnownRasterImageMimeType(source.slice(5).split(/[;,]/, 1)[0]) + ? { type: 'image-ref', url: source, ...label } + : null + } + const route = routeNativeChatHref(source) + if (route.kind === 'web') { + return /^https:/i.test(route.url) && !isSvgFilePath(route.url.split(/[?#]/, 1)[0] ?? '') + ? { type: 'image-ref', url: route.url, ...label } + : null + } + return route.kind === 'file' && + isRuntimePathAbsolute(route.pathText) && + isRasterImagePath(route.pathText) + ? { type: 'image-ref', path: route.pathText, ...label } + : null +} export function Prose({ block, @@ -54,6 +82,15 @@ export function Prose({ }): React.JSX.Element | null { useMobileColorScheme() const t = useMobileTranslation() + const renderImage = useCallback( + (href: string, alt: string): React.ReactNode => { + const image = markdownImageBlock(href, alt) + return image ? ( + + ) : null + }, + [fontScale, onLongPress] + ) if (isTextBlock(block)) { if (isAgentSessionHostStatusPresentation(block.presentation)) { return ( @@ -74,6 +111,7 @@ export function Prose({ textScale={CHAT_PROSE_SCALE * fontScale} onOpenFile={onOpenFile} onLongPress={onLongPress} + renderImage={renderImage} /> ) } @@ -90,6 +128,7 @@ export function Prose({ textScale={CHAT_PROSE_SCALE * fontScale} onOpenFile={onOpenFile} onLongPress={onLongPress} + renderImage={renderImage} /> ) } diff --git a/mobile/src/session/native-chat/transcript/MobileNativeChatToolRun.tsx b/mobile/src/session/native-chat/transcript/MobileNativeChatToolRun.tsx index e20370dcaf5..4d143ff2d26 100644 --- a/mobile/src/session/native-chat/transcript/MobileNativeChatToolRun.tsx +++ b/mobile/src/session/native-chat/transcript/MobileNativeChatToolRun.tsx @@ -175,7 +175,8 @@ export function ToolRun({ previousTodoWrite, previousUpdatePlan, fontScale = 1, - onOpenFile + onOpenFile, + viewedImages }: MobileNativeChatToolRunProps): React.JSX.Element { useMobileColorScheme() const t = useMobileTranslation() @@ -245,6 +246,8 @@ export function ToolRun({ ) : null} + {/* A read's result is the image the agent saw, so it shows without opening the run. */} + {viewedImages ? {viewedImages} : null} {open ? ( void + /** Thumbnails of the images the run's reads returned, drawn under its header. */ + viewedImages?: React.ReactNode } diff --git a/mobile/src/session/native-chat/transcript/mobile-native-chat-tool-styles.ts b/mobile/src/session/native-chat/transcript/mobile-native-chat-tool-styles.ts index 591f36ae8bd..68be83a6c66 100644 --- a/mobile/src/session/native-chat/transcript/mobile-native-chat-tool-styles.ts +++ b/mobile/src/session/native-chat/transcript/mobile-native-chat-tool-styles.ts @@ -18,6 +18,12 @@ export const toolStyles = themedStyleSheet(() => ({ run: { marginTop: spacing.xs }, + viewedImages: { + flexDirection: 'row', + flexWrap: 'wrap', + gap: spacing.sm, + marginTop: spacing.xs + }, header: { flexDirection: 'row', alignItems: 'center', diff --git a/src/main/native-chat/transcript-claude-read-image.test.ts b/src/main/native-chat/transcript-claude-read-image.test.ts new file mode 100644 index 00000000000..658c6d71bff --- /dev/null +++ b/src/main/native-chat/transcript-claude-read-image.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from 'vitest' +import { foldToolMessages } from '../../shared/native-chat/tools/native-chat-tool-fold' +import { nativeChatToolRunViewedImages } from '../../shared/native-chat/tools/native-chat-tool-viewed-images' +import type { NativeChatMessage } from '../../shared/native-chat/native-chat-types' +import { decodeClaudeTranscriptLine } from './transcript-line-decoders-claude' + +// Claude 2.x Read of an image: the result holds only base64, the path is on the call. +function readTurn(filePath: string, result: Record): string[] { + return [ + JSON.stringify({ + type: 'assistant', + uuid: 'call-row', + timestamp: '2026-10-29T14:09:20.000Z', + message: { + role: 'assistant', + content: [{ type: 'tool_use', id: 'toolu_1', name: 'Read', input: { file_path: filePath } }] + } + }), + JSON.stringify({ + type: 'user', + uuid: 'result-row', + timestamp: '2026-10-29T14:09:23.939Z', + message: { + role: 'user', + content: [{ tool_use_id: 'toolu_1', type: 'tool_result', ...result }] + }, + toolUseResult: { type: 'image', file: { base64: 'iVBORw0KGgo=', type: 'image/png' } } + }) + ] +} + +const IMAGE_RESULT = { + content: [ + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'iVBORw0KGgo=' } } + ] +} + +function viewedImages(lines: string[]): ReturnType { + const decoded = lines + .map((line, index) => decodeClaudeTranscriptLine(line, `fallback-${index}`)) + .filter((message): message is NativeChatMessage => message !== null) + const folded = foldToolMessages(decoded) + expect(folded).toHaveLength(1) + return nativeChatToolRunViewedImages(folded[0]!.blocks) +} + +describe('Claude Read of an image', () => { + it('shows the screenshot the agent read', () => { + expect(viewedImages(readTurn('/tmp/orca-annot/s2.png', IMAGE_RESULT))).toEqual([ + { type: 'image-ref', path: '/tmp/orca-annot/s2.png' } + ]) + }) + + it('shows nothing for a read that failed or returned text', () => { + const failed = { content: 'File does not exist.', is_error: true } + expect(viewedImages(readTurn('/tmp/gone.png', failed))).toEqual([]) + expect(viewedImages(readTurn('/repo/src/a.ts', { content: '1\tconst a = 1' }))).toEqual([]) + expect(viewedImages(readTurn('/repo/icons/logo.svg', { content: '1\t' }))).toEqual([]) + }) +}) + +it('shows nothing for a shell read, which gave the agent bytes as text', () => { + const codexCat = { + type: 'tool-call', + name: 'read', + callId: 'c1', + input: { command: 'cat /repo/images/shot.png', cwd: '/repo', path: '/repo/images/shot.png' } + } as const + expect( + nativeChatToolRunViewedImages([codexCat, { type: 'tool-result', callId: 'c1', output: '' }]) + ).toEqual([]) +}) diff --git a/src/renderer/src/components/native-chat/NativeChatAssistantMessageRow.tsx b/src/renderer/src/components/native-chat/NativeChatAssistantMessageRow.tsx index 90186440ecc..d06e71c69fb 100644 --- a/src/renderer/src/components/native-chat/NativeChatAssistantMessageRow.tsx +++ b/src/renderer/src/components/native-chat/NativeChatAssistantMessageRow.tsx @@ -132,6 +132,7 @@ function toolRunProps( activeTurnIsWorking: props.activeTurnIsWorking, trailing: props.trailingRun, disclosureId: props.message.id, - asides: run?.asides + asides: run?.asides, + runtimeContext: props.runtimeContext } } diff --git a/src/renderer/src/components/native-chat/NativeChatToolRun.tsx b/src/renderer/src/components/native-chat/NativeChatToolRun.tsx index 327f886d04c..07857e68cd6 100644 --- a/src/renderer/src/components/native-chat/NativeChatToolRun.tsx +++ b/src/renderer/src/components/native-chat/NativeChatToolRun.tsx @@ -31,6 +31,9 @@ import { NativeChatToolRunCallCounts } from './NativeChatToolRunCallCounts' import { NativeChatToolRunMembers, revealNativeChatToolRunMember } from './NativeChatToolRunMembers' import { NativeChatToolRunMemberList } from './NativeChatToolRunMemberList' import { NativeChatBrowserRunPageButton } from './NativeChatBrowserPageButton' +import { NativeChatImageAttachments } from './NativeChatTranscriptChrome' +import { nativeChatToolRunViewedImages } from '../../../../shared/native-chat/tools/native-chat-tool-viewed-images' +import type { RuntimeFileOperationArgs } from '@/runtime/runtime-file-client-types' /** Insert keyed asides before their block or after the final block. */ export type NativeChatToolRunAsides = { @@ -58,7 +61,8 @@ export function NativeChatToolRun({ trailing, disclosureId, onLinkClick, - asides + asides, + runtimeContext }: { blocks: NativeChatBlock[] previousTodoWrite?: NativeChatToolCallBlock @@ -84,6 +88,7 @@ export function NativeChatToolRun({ disclosureId?: string onLinkClick?: CommentMarkdownLinkClickHandler asides?: NativeChatToolRunAsides + runtimeContext?: RuntimeFileOperationArgs | null }): React.JSX.Element | null { // This row owns the language subscription for its tool, diff, and task labels. useTranslation() @@ -142,6 +147,7 @@ export function NativeChatToolRun({ () => nativeChatToolRunHeader(blocks, { activeTurnIsWorking, trailing }), [activeTurnIsWorking, blocks, trailing] ) + const viewedImages = useMemo(() => nativeChatToolRunViewedImages(headerBlocks), [headerBlocks]) const hasAskCall = asks.length > 0 const askSubject = hasAskCall ? nativeChatAskRunSubject(asks) : null const showsHeader = !hasAskCall || callCount > 0 @@ -231,6 +237,12 @@ export function NativeChatToolRun({ )} + {/* A read's result is the image the agent saw, so it shows without opening the run. */} + block.type === 'image-ref') if (images.length === 0) { @@ -222,7 +224,7 @@ export function NativeChatImageAttachments({ const imageKeyCounts = new Map() if (!enablePreview) { return ( -
+
{images.map((image) => { const label = image.alt ?? @@ -253,7 +255,7 @@ export function NativeChatImageAttachments({ ) } return ( -
+
{images.map((image) => { const label = image.alt ?? diff --git a/src/shared/file-preview-kind.ts b/src/shared/file-preview-kind.ts index ce438712441..682d3d82706 100644 --- a/src/shared/file-preview-kind.ts +++ b/src/shared/file-preview-kind.ts @@ -1,5 +1,6 @@ import { IMAGE_FILE_MIME_TYPES } from './image-file-extensions' import { MEDIA_FILE_MIME_TYPES } from './media-file-extensions' +import { isKnownRasterImageMimeType } from './raster-image-preview-limits' export type FilePreviewKind = 'image' | 'pdf' | 'audio' | 'video' export type FilePreview = { kind: FilePreviewKind; mimeType: string } @@ -32,6 +33,12 @@ export function filePreviewForPath(filePath: string): FilePreview | null { return null } +/** Images an image element without an SVG engine (React Native's) can draw. */ +export function isRasterImagePath(filePath: string): boolean { + const preview = filePreviewForPath(filePath) + return preview?.kind === 'image' && isKnownRasterImageMimeType(preview.mimeType) +} + /** Audio and video always stream by range; their players can't take base64. */ export function streamsPreviewByRange(preview: FilePreview | null): boolean { return preview?.kind === 'audio' || preview?.kind === 'video' diff --git a/src/shared/native-chat/tools/native-chat-tool-viewed-images.ts b/src/shared/native-chat/tools/native-chat-tool-viewed-images.ts new file mode 100644 index 00000000000..35d77e75f1a --- /dev/null +++ b/src/shared/native-chat/tools/native-chat-tool-viewed-images.ts @@ -0,0 +1,47 @@ +import { isRuntimePathAbsolute } from '../../cross-platform-path' +import { isRasterImagePath } from '../../file-preview-kind' +import { + isToolCallBlock, + type NativeChatBlock, + type NativeChatImageRefBlock, + type NativeChatToolCallBlock +} from '../native-chat-types' +import { nativeChatToolCategory } from './native-chat-tool-icon' +import { nativeChatFullCommand } from './native-chat-tool-input-text' +import { nativeChatToolLineLabel } from './native-chat-tool-line-label' +import { pairToolBlocks } from './native-chat-tool-pairs' +import { toolFilePath } from './native-chat-tool-summary' + +const NO_IMAGES: NativeChatImageRefBlock[] = [] + +/** The raster image a file-read tool names. A shell read (Codex's `cat x.png`) and an SVG read hand + * the agent text, not a picture; hosts read only absolute paths. */ +function imageReadPath(call: NativeChatToolCallBlock): string | null { + if ( + nativeChatToolCategory(call.name, call.mcpIdentity) !== 'read' || + nativeChatFullCommand(call.input) !== null + ) { + return null + } + const path = toolFilePath(call.input) + return path !== null && isRuntimePathAbsolute(path) && isRasterImagePath(path) ? path : null +} + +/** Image files a run's tools read; their results hold only the bytes, so each call's path names its image. */ +export function nativeChatToolRunViewedImages( + blocks: readonly NativeChatBlock[] +): NativeChatImageRefBlock[] { + // Most runs read no images, so they skip pairing. + if (!blocks.some((block) => isToolCallBlock(block) && imageReadPath(block) !== null)) { + return NO_IMAGES + } + const paths = new Set() + for (const { call, result } of pairToolBlocks(blocks)) { + const path = call ? imageReadPath(call) : null + // Only a read that finished without error showed the agent its image. + if (call && path !== null && nativeChatToolLineLabel(call, result).verb === 'read') { + paths.add(path) + } + } + return Array.from(paths, (path) => ({ type: 'image-ref', path })) +}