fix(mobile): show embedded images in Markdown previews (#31304)

Render standalone embedded images in Markdown file previews and open them in the existing full-screen viewer. Resolve relative paths through authorized paired-host reads.

Thanks to @dudupii for reporting and diagnosing #24468 and implementing the broader solution in #24469; related editor integration is preserved in #25594. This independently implemented preview fix leaves editor image serialization, inline images, and pinch zoom as separate work.

Validation: focused tests, iPhone simulator proof, and all CI checks passed.
This commit is contained in:
Neil
2026-10-20 19:38:23 -07:00
committed by GitHub
parent 356bda10d1
commit ecc4a6cacf
11 changed files with 322 additions and 1 deletions
@@ -224,3 +224,18 @@ describe('MobileMarkdown tone', () => {
expect(styleOf(renderer!.root, 'Cell').fontSize).toBe(24)
})
})
it('renders document images through the image renderer, including reference syntax', () => {
const renderImage = vi.fn((href: string, alt: string) => createElement('View', { href, alt }))
act(() => {
renderer = create(
createElement(MobileMarkdown, {
content:
'![Architecture](assets/diagram.png)\n\n![Reference][diagram]\n\n[diagram]: assets/other.png',
renderImage
})
)
})
expect(renderImage).toHaveBeenCalledWith('assets/diagram.png', 'Architecture')
expect(renderImage).toHaveBeenCalledWith('assets/other.png', 'Reference')
})
+3
View File
@@ -25,6 +25,7 @@ type Props = {
onOpenFile?: (pathText: string) => void
/** Native-chat assistant prose only: renders `::orca-visual{...}` directive lines. Without it,
* a directive line is ordinary text. Must be referentially stable (this component is memoized). */
renderImage?: (href: string, alt: string) => ReactNode
renderVisual?: (directive: NativeChatVisualDirective, index: number) => ReactNode
/** The text is still being appended to: markup its end leaves open is closed, as on desktop,
* and a diagram stays code until the text stops. */
@@ -40,6 +41,7 @@ function MobileMarkdownContent({
textScale = 1,
onOpenFile,
renderVisual,
renderImage,
growing = false,
tone: toneName = 'default'
}: Props) {
@@ -69,6 +71,7 @@ function MobileMarkdownContent({
rowLongPress: setup.androidTranscript ? setup.onLongPress : undefined,
directives,
renderVisual,
renderImage,
mermaidOccurrences: new Map()
}
// Native-chat range selection is set on each block; nested inline spans inherit it.
@@ -35,6 +35,7 @@ export type MarkdownBlockContext = MarkdownInlineContext & {
/** Interactive children own their touches and must forward the row action. */
rowLongPress?: () => void
directives: readonly NativeChatVisualDirective[]
renderImage?: (href: string, alt: string) => ReactNode
renderVisual?: (directive: NativeChatVisualDirective, index: number) => ReactNode
mermaidOccurrences: Map<string, number>
}
@@ -153,6 +154,9 @@ function renderBlock(
</Fragment>
) : null
}
if (node.children.length === 1 && only?.type === 'image' && ctx.renderImage) {
return <Fragment key={key}>{ctx.renderImage(only.url, only.alt ?? '')}</Fragment>
}
if (node.children.length === 1 && only?.type === 'image' && /^https?:\/\//i.test(only.url)) {
return (
<Pressable
@@ -23,6 +23,10 @@ vi.mock('lucide-react-native', () => ({
const { writeText, alert } = vi.hoisted(() => ({ writeText: vi.fn(), alert: vi.fn() }))
vi.mock('../platform/clipboard', () => ({ useClipboardWriter: () => ({ writeText }) }))
vi.mock('./MobileMarkdownPreviewImage', () => ({
MobileMarkdownPreviewImage: 'MobileMarkdownPreviewImage'
}))
vi.mock('../components/MobileMarkdown', () => ({
MobileMarkdown: 'MobileMarkdown'
}))
+24 -1
View File
@@ -1,3 +1,5 @@
import { MobileMarkdownPreviewImage } from './MobileMarkdownPreviewImage'
import type { MobileFilePreviewRpcSender } from './mobile-file-preview-operations'
import { useState } from 'react'
import { Pressable, ScrollView, View } from 'react-native'
import { Code, Pencil } from 'lucide-react-native'
@@ -15,6 +17,8 @@ type Props = {
content: string
truncated: boolean
byteLength: number
client?: MobileFilePreviewRpcSender | null
worktreeId?: string
initialLine?: number
onOpenFile?: (pathText: string) => void
}
@@ -25,6 +29,8 @@ export function MobileFileMarkdownPreview({
truncated,
byteLength,
initialLine,
client,
worktreeId,
onOpenFile
}: Props) {
const [mode, setMode] = useState<'preview' | 'source'>(() => (initialLine ? 'source' : 'preview'))
@@ -79,7 +85,24 @@ export function MobileFileMarkdownPreview({
{mode === 'preview' ? (
<ScrollView style={styles.scroll} contentContainerStyle={styles.markdownContent}>
{truncated ? <MobileFilePreviewTruncatedNote byteLength={byteLength} /> : null}
<MobileMarkdown content={content} onOpenFile={onOpenFile} />
<MobileMarkdown
content={content}
onOpenFile={onOpenFile}
renderImage={
worktreeId
? (href, alt) => (
<MobileMarkdownPreviewImage
key={`${worktreeId}:${relativePath}:${href}`}
client={client ?? null}
worktreeId={worktreeId}
documentPath={relativePath}
href={href}
alt={alt}
/>
)
: undefined
}
/>
</ScrollView>
) : (
<MobileFilePreviewSourceText
@@ -13,6 +13,7 @@ import { MobileLoadErrorState } from '../components/MobileLoadErrorState'
type Props = {
preview: MobileFilePreviewResult
client?: MobileFilePreviewRpcSender | null
worktreeId?: string
relativePath: string
title: string
editable: boolean
@@ -85,6 +86,8 @@ export function MobileFilePreviewBody({ preview, ...options }: Props) {
) : (
<MobileFileMarkdownPreview
relativePath={options.relativePath}
client={options.client}
worktreeId={options.worktreeId}
content={preview.content}
truncated={preview.truncated}
byteLength={preview.byteLength}
@@ -265,6 +265,7 @@ export function MobileFilePreviewScreen({ route }: Props) {
<MobileFilePreviewBody
preview={preview}
client={client}
worktreeId={previewSource?.source === 'worktree' ? previewSource.worktreeId : undefined}
relativePath={displayPath}
title={title || 'File'}
editable={isEditableTerminalArtifact}
@@ -0,0 +1,71 @@
import { createElement } from 'react'
import type { ImageProps } from 'react-native'
import { act, create, type ReactTestRenderer } from 'react-test-renderer'
import { afterEach, expect, it, vi } from 'vitest'
import { MobileMarkdownPreviewImage } from './MobileMarkdownPreviewImage'
vi.mock('react-native', () => ({
ActivityIndicator: 'ActivityIndicator',
Image: Object.assign((props: ImageProps) => createElement('Image', props), {
getSize: (_uri: string, loaded: (width: number, height: number) => void) => loaded(600, 220)
}),
Pressable: 'Pressable',
StyleSheet: { create: (styles: unknown) => styles },
Text: 'Text',
View: 'View'
}))
vi.mock('../components/MobileLoadErrorState', () => ({ MobileLoadErrorState: 'LoadError' }))
vi.mock('../session/MobileNativeChatImageViewer', () => ({
MobileNativeChatImageViewer: 'ImageViewer'
}))
const loader = vi.hoisted(() => vi.fn())
vi.mock('./mobile-markdown-image-source', () => ({ loadMarkdownPreviewImage: loader }))
let tree: ReactTestRenderer | null = null
afterEach(() => {
act(() => tree?.unmount())
tree = null
vi.resetAllMocks()
})
const props = {
client: null,
worktreeId: 'wt',
documentPath: 'notes.md',
href: 'a.png',
alt: 'Architecture'
}
it('shows the loaded image and expands it in the existing image viewer', async () => {
loader.mockResolvedValue('data:image/png;base64,cG5n')
await act(async () => {
tree = create(createElement(MobileMarkdownPreviewImage, props))
})
const image = tree!.root.find((node) => String(node.type) === 'Image')
expect(image.props.source.uri).toBe('data:image/png;base64,cG5n')
expect(tree!.root.find((node) => String(node.type) === 'Image').props.style.aspectRatio).toBe(
600 / 220
)
await act(async () =>
tree!.root.find((node) => String(node.type) === 'Pressable').props.onPress()
)
expect(tree!.root.find((node) => String(node.type) === 'ImageViewer').props.uri).toBe(
'data:image/png;base64,cG5n'
)
})
it('retries a refused image without hiding the surrounding document', async () => {
loader
.mockRejectedValueOnce(new Error('Unable to reach the desktop filesystem'))
.mockResolvedValueOnce('data:image/png;base64,cG5n')
await act(async () => {
tree = create(createElement(MobileMarkdownPreviewImage, props))
})
expect(tree!.root.find((node) => String(node.type) === 'LoadError').props.message).toContain(
'Unable to reach'
)
await act(async () =>
tree!.root.find((node) => String(node.type) === 'LoadError').props.onRetry()
)
expect(tree!.root.find((node) => String(node.type) === 'Image').props.source.uri).toBe(
'data:image/png;base64,cG5n'
)
})
@@ -0,0 +1,92 @@
import { useEffect, useState } from 'react'
import { ActivityIndicator, Image, Pressable, StyleSheet, Text, View } from 'react-native'
import { colors, spacing, typography } from '../theme/mobile-theme'
import { MobileLoadErrorState } from '../components/MobileLoadErrorState'
import { MobileNativeChatImageViewer } from '../session/MobileNativeChatImageViewer'
import type { MobileFilePreviewRpcSender } from './mobile-file-preview-operations'
import { loadMarkdownPreviewImage } from './mobile-markdown-image-source'
type Props = {
client: MobileFilePreviewRpcSender | null
worktreeId: string
documentPath: string
href: string
alt: string
}
export function MobileMarkdownPreviewImage({ client, worktreeId, documentPath, href, alt }: Props) {
const [load, setLoad] = useState<{ uri?: string; error?: string }>({})
const [attempt, setAttempt] = useState(0)
const [aspectRatio, setAspectRatio] = useState(16 / 9)
const [expanded, setExpanded] = useState(false)
useEffect(() => {
let cancelled = false
void loadMarkdownPreviewImage(client, worktreeId, documentPath, href).then(
(uri) => {
if (!cancelled) {
setLoad({ uri })
Image.getSize(
uri,
(width, height) => {
if (!cancelled && width > 0 && height > 0) {
setAspectRatio(width / height)
}
},
() => {}
)
}
},
(error: unknown) => {
if (!cancelled) {
setLoad({ error: error instanceof Error ? error.message : 'Unable to load image' })
}
}
)
return () => {
cancelled = true
}
}, [client, worktreeId, documentPath, href, attempt])
return (
<View style={styles.frame}>
{load.error ? (
<MobileLoadErrorState
message={load.error}
onRetry={() => {
setLoad({})
setAttempt((value) => value + 1)
}}
/>
) : load.uri ? (
<Pressable
onPress={() => setExpanded(true)}
accessibilityRole="button"
accessibilityLabel={`Expand ${alt || 'image'}`}
>
<Image
source={{ uri: load.uri }}
accessibilityLabel={alt || 'Markdown image'}
resizeMode="contain"
style={{ width: '100%', aspectRatio }}
onError={() => setLoad({ error: 'Unable to load image' })}
/>
</Pressable>
) : (
<ActivityIndicator color={colors.textSecondary} />
)}
{alt ? <Text style={styles.caption}>{alt}</Text> : null}
{expanded && load.uri ? (
<MobileNativeChatImageViewer
uri={load.uri}
label={alt || 'Markdown image'}
onClose={() => setExpanded(false)}
/>
) : null}
</View>
)
}
const styles = StyleSheet.create({
frame: { gap: spacing.xs },
caption: { color: colors.textSecondary, fontSize: typography.metaSize }
})
@@ -0,0 +1,60 @@
import { describe, expect, it, vi } from 'vitest'
import { loadMarkdownPreviewImage } from './mobile-markdown-image-source'
function host() {
return {
sendRequest: vi.fn(async () => ({
id: '1',
ok: true as const,
_meta: { runtimeId: 'remote' },
result: { content: 'cG5n', isBinary: true, isImage: true, mimeType: 'image/png' }
}))
}
}
describe('Markdown preview images', () => {
it.each([
['architecture.png', 'docs/architecture.png'],
['../assets/a%20b.png', 'assets/a b.png'],
['nested\\image.png', 'docs/nested/image.png']
])(
'reads %s relative to the document through the workspace client',
async (href, relativePath) => {
const client = host()
await expect(
loadMarkdownPreviewImage(client, 'folder-workspace', 'docs/readme.md', href)
).resolves.toBe('data:image/png;base64,cG5n')
expect(client.sendRequest).toHaveBeenCalledWith('files.readPreview', {
worktree: 'id:folder-workspace',
relativePath
})
}
)
it.each([
'../../private.png',
'/private.png',
'C:\\private.png',
'file:///tmp/private.png',
'javascript:alert(1)',
'notes.txt'
])('does not read unauthorized or unsupported image locations: %s', async (href) => {
const client = host()
await expect(loadMarkdownPreviewImage(client, 'wt', 'docs/readme.md', href)).rejects.toThrow()
expect(client.sendRequest).not.toHaveBeenCalled()
})
it('lets native Image load public URLs without sending them to the workspace host', async () => {
const client = host()
await expect(
loadMarkdownPreviewImage(client, 'wt', 'readme.md', 'https://example.com/a.png')
).resolves.toBe('https://example.com/a.png')
expect(client.sendRequest).not.toHaveBeenCalled()
})
it('reports a disconnected host instead of trying to read phone-local files', async () => {
await expect(loadMarkdownPreviewImage(null, 'wt', 'readme.md', 'a.png')).rejects.toThrow(
'Waiting for desktop'
)
})
})
@@ -0,0 +1,45 @@
import {
isWindowsAbsolutePathLike,
relativePathInsideRoot,
resolveRuntimePath
} from '../../../src/shared/cross-platform-path'
import { routeMarkdownHref } from '../components/markdown-href-routing'
import type { MobileFilePreviewRpcSender } from './mobile-file-preview-operations'
import { loadMobileFilePreview } from './mobile-file-preview-request'
import { classifyMobileArtifact } from '../session/mobile-artifact-kind'
export async function loadMarkdownPreviewImage(
client: MobileFilePreviewRpcSender | null,
worktreeId: string,
documentPath: string,
href: string
): Promise<string> {
if (/^https?:\/\//i.test(href)) {
return href
}
const link = routeMarkdownHref(href)
if (link.kind !== 'file') {
throw new Error('Unsupported image location')
}
const path = link.pathText
if (/^[\\/]/.test(path) || isWindowsAbsolutePathLike(path) || path.includes('\0')) {
throw new Error('Image must be inside this workspace')
}
const source = documentPath.replaceAll('\\', '/')
const directory = source.slice(0, source.lastIndexOf('/') + 1)
const relativePath = relativePathInsideRoot(
'/workspace',
resolveRuntimePath(`/workspace/${directory}`, path)
)
if (!relativePath || classifyMobileArtifact(relativePath) !== 'image') {
throw new Error('Unsupported image location')
}
if (!client) {
throw new Error('Waiting for desktop...')
}
const result = await loadMobileFilePreview(client, worktreeId, relativePath)
if (result.status === 'ready' && result.kind === 'image') {
return result.dataUri
}
throw new Error('message' in result ? result.message : 'Unable to load image')
}