feat(browser): paste images into annotation comments and send them to the agent (#36004)

* feat(browser): paste images into annotation comments and send their paths to the agent

* fix(paste): hand an image-only clipboard to native paste so Cmd+V reaches the annotation field

* fix(browser): let an open image preview keep Escape instead of exiting Design Mode

* test(paste): drop the ownership-event test the menu-claim test already covers

* refactor(browser): reuse chat preview lifetimes and the terminal's paste host for annotation images

* fix(browser): release pasted edit images when the edited note disappears
This commit is contained in:
Kelvin Amoaba
2026-10-29 20:48:45 +00:00
committed by GitHub
parent 0cdb7b35cd
commit cc6da802a5
15 changed files with 260 additions and 61 deletions
@@ -0,0 +1,21 @@
import { NativeChatImageAttachmentPreview } from '@/components/native-chat/NativeChatImageAttachmentPreview'
import type { NativeChatComposerImageAttachment } from '@/components/native-chat/NativeChatComposerField'
export function BrowserAnnotationImageStrip({
images,
onRemove
}: {
images: NativeChatComposerImageAttachment[]
onRemove: (id: string) => void
}): React.JSX.Element | null {
if (images.length === 0) {
return null
}
return (
<div className="mt-2 flex flex-wrap gap-2 pt-1.5 pr-1.5">
{images.map((image) => (
<NativeChatImageAttachmentPreview key={image.id} attachment={image} onRemove={onRemove} />
))}
</div>
)
}
@@ -164,6 +164,17 @@ describe('formatBrowserAnnotationsAsMarkdown', () => {
expect(markdown).not.toContain('\n## injected')
})
it('lists pasted image paths after the feedback so the agent can open them', () => {
const markdown = formatBrowserAnnotationsAsMarkdown([
makeAnnotation({ imagePaths: ['/tmp/orca-paste-1.png', 'C:\\Users\\A B\\orca-paste-2.png'] })
])
expect(markdown.split('\n').slice(-3)).toEqual([
'**Images (open these files):**',
'- `/tmp/orca-paste-1.png`',
'- `C:\\Users\\A B\\orca-paste-2.png`'
])
})
it('returns an empty string when no annotations exist', () => {
expect(formatBrowserAnnotationsAsMarkdown([])).toBe('')
})
@@ -239,6 +239,12 @@ export function formatBrowserAnnotationsAsMarkdown(annotations: BrowserPageAnnot
lines.push(...fence('html', target.htmlSnippet))
}
lines.push(`**Feedback:** ${inlineText(annotation.comment)}`)
if (annotation.imagePaths?.length) {
lines.push('**Images (open these files):**')
for (const imagePath of annotation.imagePaths) {
lines.push(`- ${inlineCode(imagePath)}`)
}
}
lines.push('')
})
}
@@ -95,6 +95,7 @@ export function BrowserGuestAnnotateOverlays({
{pendingAnnotationPayload ? (
<PendingBrowserAnnotationCard
payload={pendingAnnotationPayload}
worktreeId={worktreeId}
resolveAnchor={(cardHeight) =>
getBrowserOverlayAnchor(
pendingAnnotationPayload,
@@ -139,7 +139,8 @@ describe('BrowserPageAnnotationTray edit mode', () => {
expect(handleUpdateBrowserAnnotation).toHaveBeenCalledWith(
'annotation-1',
'Updated comment',
'question'
'question',
[]
)
})
@@ -208,7 +209,8 @@ describe('BrowserPageAnnotationTray collapse', () => {
expect(handleUpdateBrowserAnnotation).toHaveBeenCalledWith(
'annotation-1',
'Unsaved draft',
'question'
'question',
[]
)
})
})
@@ -1,6 +1,15 @@
import { ImeTextarea } from '@/lib/ime-text-field'
import { useEffect, useState } from 'react'
import { ChevronDown, ChevronRight, CircleCheck, Copy, Pencil, Send, Trash2 } from 'lucide-react'
import {
ChevronDown,
ChevronRight,
CircleCheck,
Copy,
Image as ImageIcon,
Pencil,
Send,
Trash2
} from 'lucide-react'
import { Button } from '@/components/ui/button'
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible'
import {
@@ -21,6 +30,8 @@ import { BROWSER_ANNOTATION_INTENT_OPTIONS } from '../describe-page/browser-anno
import { BrowserAnnotationSendMenuContent } from './BrowserAnnotationSendMenuContent'
import { preventAgentSendTargetOutsideDismiss } from './prevent-agent-send-target-outside-dismiss'
import { browserAnnotationMatchesPageUrl } from './browser-annotation-page-url'
import { useBrowserAnnotationImages } from './use-browser-annotation-images'
import { BrowserAnnotationImageStrip } from './browser-annotation-image-strip'
export function BrowserPageAnnotationTray({
browserAnnotations,
@@ -54,12 +65,15 @@ export function BrowserPageAnnotationTray({
handleUpdateBrowserAnnotation: (
annotationId: string,
comment: string,
intent: BrowserAnnotationIntent
intent: BrowserAnnotationIntent,
imagePaths: string[]
) => void
}): React.JSX.Element {
const [editingAnnotationId, setEditingAnnotationId] = useState<string | null>(null)
const [editComment, setEditComment] = useState('')
const [editIntent, setEditIntent] = useState<BrowserAnnotationIntent>('change')
const editImages = useBrowserAnnotationImages(worktreeId)
const resetEditImages = editImages.resetImages
const [expanded, setExpanded] = useState(true)
const ToggleChevron = expanded ? ChevronDown : ChevronRight
@@ -70,26 +84,34 @@ export function BrowserPageAnnotationTray({
!browserAnnotations.some((annotation) => annotation.id === editingAnnotationId)
) {
setEditingAnnotationId(null)
resetEditImages()
}
}, [browserAnnotations, editingAnnotationId])
}, [browserAnnotations, editingAnnotationId, resetEditImages])
const handleStartEdit = (annotation: BrowserPageAnnotation): void => {
setEditingAnnotationId(annotation.id)
setEditComment(annotation.comment)
setEditIntent(annotation.intent)
editImages.resetImages(annotation.imagePaths)
}
const handleCancelEdit = (): void => {
setEditingAnnotationId(null)
editImages.resetImages()
}
const canSaveEdit = Boolean(editComment.trim()) && !editImages.saving
const handleSaveEdit = (): void => {
const trimmed = editComment.trim()
if (!trimmed || !editingAnnotationId) {
if (!canSaveEdit || !editingAnnotationId) {
return
}
handleUpdateBrowserAnnotation(editingAnnotationId, trimmed, editIntent)
setEditingAnnotationId(null)
handleUpdateBrowserAnnotation(
editingAnnotationId,
editComment.trim(),
editIntent,
editImages.imagePaths
)
handleCancelEdit()
}
return (
@@ -230,8 +252,12 @@ export function BrowserPageAnnotationTray({
className="min-w-0 flex-1"
data-slot="annotation-edit"
onKeyDown={(event) => {
// Why: bail-out selector in useGrabMode lets this reach any focused child, not just the textarea.
if (event.key === 'Escape') {
// Why: useGrabMode's bail-out lets this reach any focused child; an image preview's portal only closes itself.
if (
event.key === 'Escape' &&
event.target instanceof Node &&
event.currentTarget.contains(event.target)
) {
event.preventDefault()
event.stopPropagation()
handleCancelEdit()
@@ -241,6 +267,7 @@ export function BrowserPageAnnotationTray({
<ImeTextarea
value={editComment}
onChange={(event) => setEditComment(event.target.value)}
onPaste={editImages.handlePaste}
maxLength={GRAB_BUDGET.annotationCommentMaxLength}
className="h-16 w-full resize-none rounded-md border border-input bg-background px-2 py-1.5 text-xs outline-none ring-offset-background focus-visible:ring-2 focus-visible:ring-ring"
autoFocus
@@ -256,6 +283,10 @@ export function BrowserPageAnnotationTray({
}
}}
/>
<BrowserAnnotationImageStrip
images={editImages.images}
onRemove={editImages.removeImage}
/>
<ToggleGroup
type="single"
size="sm"
@@ -294,7 +325,7 @@ export function BrowserPageAnnotationTray({
'Cancel'
)}
</Button>
<Button size="xs" disabled={!editComment.trim()} onClick={handleSaveEdit}>
<Button size="xs" disabled={!canSaveEdit} onClick={handleSaveEdit}>
{translate(
'auto.components.browser.pane.annotate.browser.page.annotation.tray.c16f932cb3',
'Save'
@@ -315,6 +346,12 @@ export function BrowserPageAnnotationTray({
</div>
<div className="mt-1 flex items-center gap-1.5 text-[11px] text-muted-foreground">
<span>{annotation.intent}</span>
{annotation.imagePaths?.length ? (
<span className="flex items-center gap-0.5">
<ImageIcon className="size-3" />
{annotation.imagePaths.length}
</span>
) : null}
{currentUrl !== undefined &&
!browserAnnotationMatchesPageUrl(
annotation.payload.page.sanitizedUrl,
@@ -16,19 +16,23 @@ import {
BROWSER_ANNOTATION_INTENT_OPTIONS,
type BrowserOverlayAnchor
} from '../describe-page/browser-annotation-geometry'
import { useBrowserAnnotationImages } from './use-browser-annotation-images'
import { BrowserAnnotationImageStrip } from './browser-annotation-image-strip'
export function PendingBrowserAnnotationCard({
payload,
worktreeId,
resolveAnchor,
portalContainer,
onAdd,
onCancel
}: {
payload: BrowserGrabPayload
worktreeId: string
/** Anchor for a card of the given measured height; null until the card has rendered once. */
resolveAnchor: (cardHeight: number | null) => BrowserOverlayAnchor
portalContainer: HTMLElement | null
onAdd: (comment: string, intent: BrowserAnnotationIntent) => void
onAdd: (comment: string, intent: BrowserAnnotationIntent, imagePaths: string[]) => void
onCancel: () => void
}): React.JSX.Element {
const [card, setCard] = useState<HTMLDivElement | null>(null)
@@ -45,7 +49,10 @@ export function PendingBrowserAnnotationCard({
const anchor = resolveAnchor(cardHeight)
const [comment, setComment] = useState('')
const [intent, setIntent] = useState<BrowserAnnotationIntent>('change')
const { images, imagePaths, saving, handlePaste, removeImage } =
useBrowserAnnotationImages(worktreeId)
const trimmed = comment.trim()
const canAdd = Boolean(trimmed) && !saving
const submitModifierLabel = getScreenSubmitModifierLabel()
return (
@@ -99,6 +106,7 @@ export function PendingBrowserAnnotationCard({
id="browser-annotation-comment"
value={comment}
onChange={(event) => setComment(event.target.value)}
onPaste={handlePaste}
placeholder={translate(
'auto.components.browser.pane.BrowserPane.532bac48c5',
'Describe what the agent should change here...'
@@ -116,12 +124,13 @@ export function PendingBrowserAnnotationCard({
if (isScreenSubmitShortcut(event)) {
event.preventDefault()
event.stopPropagation()
if (trimmed) {
onAdd(trimmed, intent)
if (canAdd) {
onAdd(trimmed, intent, imagePaths)
}
}
}}
/>
<BrowserAnnotationImageStrip images={images} onRemove={removeImage} />
<ToggleGroup
type="single"
size="sm"
@@ -160,8 +169,8 @@ export function PendingBrowserAnnotationCard({
<Button
size="sm"
className="h-8 gap-1.5"
disabled={!trimmed}
onClick={() => onAdd(trimmed, intent)}
disabled={!canAdd}
onClick={() => onAdd(trimmed, intent, imagePaths)}
>
<MessageSquarePlus className="size-3.5" />
{translate('auto.components.browser.pane.BrowserPane.90d021f2ad', 'Add')}
@@ -0,0 +1,89 @@
import { useCallback, useState } from 'react'
import { toast } from 'sonner'
import { useAppStore } from '@/store'
import { getConnectionId } from '@/lib/connection-context'
import { useMountedRef } from '@/hooks/useMountedRef'
import { createBrowserUuid } from '@/lib/browser-uuid'
import { formatClipboardImagePasteError } from '@/components/terminal-pane/terminal-pane-paste-execution'
import { resolveTerminalPasteRuntimeEnvironmentId } from '@/components/terminal-pane/terminal-paste-runtime'
import { clipboardEventImageFile } from '@/components/native-chat/native-chat-clipboard-payload'
import { useNativeChatComposerAttachmentPreviews } from '@/components/native-chat/use-native-chat-composer-attachment-previews'
import type { NativeChatComposerImageAttachment } from '@/components/native-chat/NativeChatComposerField'
const NO_PENDING: readonly NativeChatComposerImageAttachment[] = []
/** Save where a terminal paste in this worktree would, so the agent can open the path. */
function saveAnnotationClipboardImage(worktreeId: string): Promise<string | null> {
return window.api.ui.saveClipboardImageAsTempFile({
connectionId: getConnectionId(worktreeId) ?? null,
runtimeEnvironmentId: resolveTerminalPasteRuntimeEnvironmentId(
useAppStore.getState(),
worktreeId,
null
)
})
}
export function useBrowserAnnotationImages(worktreeId: string): {
images: NativeChatComposerImageAttachment[]
imagePaths: string[]
saving: boolean
handlePaste: (event: React.ClipboardEvent) => void
removeImage: (id: string) => void
resetImages: (paths?: readonly string[]) => void
} {
const mountedRef = useMountedRef()
const [images, setImages] = useState<NativeChatComposerImageAttachment[]>([])
// Releases each clipboard preview once its image leaves the list or the field unmounts.
const { previews, setPreview } = useNativeChatComposerAttachmentPreviews(images, NO_PENDING)
const removeImage = useCallback((id: string): void => {
setImages((current) => current.filter((image) => image.id !== id))
}, [])
const handlePaste = useCallback(
(event: React.ClipboardEvent): void => {
// Text wins, as in a terminal: rich copies often carry an image rendition of their text.
const file = event.clipboardData.getData('text/plain') ? null : clipboardEventImageFile(event)
if (!file) {
return
}
event.preventDefault()
const id = createBrowserUuid()
setImages((current) => [...current, { id, path: '', pending: true }])
setPreview(id, URL.createObjectURL(file))
const settle = (path: string | null): void => {
if (!mountedRef.current) {
return
}
if (!path) {
removeImage(id)
return
}
setImages((current) =>
current.map((image) => (image.id === id ? { ...image, path, pending: false } : image))
)
}
saveAnnotationClipboardImage(worktreeId).then(settle, (error: unknown) => {
settle(null)
toast.error(formatClipboardImagePasteError(error))
})
},
[mountedRef, removeImage, setPreview, worktreeId]
)
const resetImages = useCallback((paths: readonly string[] = []): void => {
setImages(paths.map((path) => ({ id: path, path })))
}, [])
return {
images: images.map((image) =>
previews.has(image.id) ? { ...image, previewUrl: previews.get(image.id) } : image
),
imagePaths: images.flatMap((image) => (image.pending ? [] : [image.path])),
saving: images.some((image) => image.pending),
handlePaste,
removeImage,
resetImages
}
}
@@ -33,7 +33,8 @@ export function useBrowserPageAnnotationSend({
handleUpdateBrowserAnnotation: (
annotationId: string,
comment: string,
intent: BrowserAnnotationIntent
intent: BrowserAnnotationIntent,
imagePaths: string[]
) => void
handleBrowserAnnotationsSentToAgent: () => void
handleBrowserAnnotationsHandedOff: (delivered: Promise<unknown>) => void
@@ -187,8 +188,13 @@ export function useBrowserPageAnnotationSend({
)
const handleUpdateBrowserAnnotation = useCallback(
(annotationId: string, comment: string, intent: BrowserAnnotationIntent): void => {
updateBrowserPageAnnotation(browserTabId, annotationId, { comment, intent })
(
annotationId: string,
comment: string,
intent: BrowserAnnotationIntent,
imagePaths: string[]
): void => {
updateBrowserPageAnnotation(browserTabId, annotationId, { comment, intent, imagePaths })
recordFeatureInteraction('browser-annotations')
},
[browserTabId, recordFeatureInteraction, updateBrowserPageAnnotation]
@@ -70,7 +70,11 @@ export function useBrowserPageGrabAnnotations({
handleGrabCopy: () => void
handleGrabCopyScreenshot: () => void
grabMenuActionTakenRef: MutableRefObject<boolean>
handleAddBrowserAnnotation: (comment: string, intent: BrowserAnnotationIntent) => void
handleAddBrowserAnnotation: (
comment: string,
intent: BrowserAnnotationIntent,
imagePaths?: string[]
) => void
handleCancelPendingBrowserAnnotation: () => void
cancelPendingBrowserCapture: () => void
handleGrabActionShortcut: (key: 'c' | 's') => void
@@ -263,7 +267,7 @@ export function useBrowserPageGrabAnnotations({
}, [grab, recordFeatureInteraction, showGrabToast])
const handleAddBrowserAnnotation = useCallback(
(comment: string, intent: BrowserAnnotationIntent): void => {
(comment: string, intent: BrowserAnnotationIntent, imagePaths: string[] = []): void => {
const payload = pendingAnnotationPayloadRef.current
if (!payload) {
return
@@ -275,7 +279,8 @@ export function useBrowserPageGrabAnnotations({
intent,
priority: DEFAULT_BROWSER_ANNOTATION_PRIORITY,
createdAt: new Date().toISOString(),
payload: createBrowserAnnotationPayload(payload)
payload: createBrowserAnnotationPayload(payload),
...(imagePaths.length > 0 ? { imagePaths } : {})
})
recordFeatureInteraction('browser-annotations')
pendingAnnotationPayloadRef.current = null
@@ -235,7 +235,7 @@ export function useGrabMode(browserPageId: string): GrabModeHook {
if (
isEditableKeyboardTarget(e.target) ||
element?.closest(
'[data-slot="select-trigger"], [data-slot="select-content"], [data-slot="annotation-edit"]'
'[data-slot="select-trigger"], [data-slot="select-content"], [data-slot="annotation-edit"], [data-slot="dialog-content"]'
)
) {
return
+16 -16
View File
@@ -1,11 +1,7 @@
// @vitest-environment happy-dom
import { afterEach, describe, expect, it, vi } from 'vitest'
import {
APP_MENU_PASTE_EVENT,
dispatchAppMenuPasteEvent,
handleAppMenuPasteRequest
} from './app-menu-paste'
import { APP_MENU_PASTE_EVENT, handleAppMenuPasteRequest } from './app-menu-paste'
function appendTextarea(value = ''): HTMLTextAreaElement {
const textarea = document.createElement('textarea')
@@ -148,6 +144,21 @@ describe('app menu paste', () => {
expect(performNativePaste).toHaveBeenCalledWith({ mode: 'paste' })
})
it('native-pastes an image-only clipboard so the control can take the image', async () => {
const textarea = appendTextarea()
textarea.focus()
const performNativePaste = vi.fn()
const result = await handleAppMenuPasteRequest({
readClipboardText: vi.fn(async () => ''),
performNativePaste,
dispatchOwnedPasteEvent: () => false
})
expect(result).toEqual({ status: 'native-fallback', reason: 'no-clipboard-text' })
expect(performNativePaste).toHaveBeenCalledWith({ mode: 'paste' })
})
it('does not native-fallback when clipboard read fails after focus moves', async () => {
const textarea = appendTextarea('unchanged')
const other = appendTextarea('other')
@@ -204,15 +215,4 @@ describe('app menu paste', () => {
expect(textarea.value).toBe('unchanged')
expect(performNativePaste).not.toHaveBeenCalled()
})
it('exposes a cancellable ownership event', () => {
const onPaste = (event: Event): void => {
event.preventDefault()
}
window.addEventListener(APP_MENU_PASTE_EVENT, onPaste)
expect(dispatchAppMenuPasteEvent()).toBe(true)
window.removeEventListener(APP_MENU_PASTE_EVENT, onPaste)
})
})
+29 -16
View File
@@ -10,7 +10,10 @@ export const APP_MENU_PASTE_EVENT = 'orca-app-menu-paste'
export type AppMenuPasteRequestResult =
| { status: 'handled'; target: 'terminal' | 'text-control' }
| { status: 'native-fallback'; reason: 'no-owned-target' | 'clipboard-read-failed' }
| {
status: 'native-fallback'
reason: 'no-owned-target' | 'clipboard-read-failed' | 'no-clipboard-text'
}
| {
status: 'rejected'
target: 'text-control'
@@ -82,6 +85,26 @@ export async function handleAppMenuPasteRequest({
return { status: 'native-fallback', reason: 'no-owned-target' }
}
const nativePasteIntoResolvedTarget = (
reason: 'clipboard-read-failed' | 'no-clipboard-text'
): AppMenuPasteRequestResult => {
// After the async read, native paste could target newly focused controls instead of the resolved owner.
if (target.ownerDocument.activeElement !== target) {
const rejectedResult = createTextControlRejectedResult(
'target-unavailable',
0,
'app-menu',
getNowMs() - startedAtMs
)
return createAppMenuTextControlRejectedResult({
reason: 'target-unavailable',
redactedDiagnostic: rejectedResult.redactedDiagnostic
})
}
performNativePaste({ mode: nativePasteMode })
return { status: 'native-fallback', reason }
}
let text: string
try {
text = await readClipboardText({ maxBytes: TEXT_CONTROL_PASTE_MAX_BYTES })
@@ -98,21 +121,11 @@ export async function handleAppMenuPasteRequest({
redactedDiagnostic: rejectedResult.redactedDiagnostic
})
}
// After async failure, native paste could target newly focused controls instead of the resolved owner.
if (target.ownerDocument.activeElement !== target) {
const rejectedResult = createTextControlRejectedResult(
'target-unavailable',
0,
'app-menu',
getNowMs() - startedAtMs
)
return createAppMenuTextControlRejectedResult({
reason: 'target-unavailable',
redactedDiagnostic: rejectedResult.redactedDiagnostic
})
}
performNativePaste({ mode: nativePasteMode })
return { status: 'native-fallback', reason: 'clipboard-read-failed' }
return nativePasteIntoResolvedTarget('clipboard-read-failed')
}
// Image-only clipboards read as no text; a native paste hands the image to the control's own paste handler.
if (!text) {
return nativePasteIntoResolvedTarget('no-clipboard-text')
}
const result = await pasteTextIntoTextControl(target, text, {
@@ -18,10 +18,7 @@ import type {
} from '../../../../../shared/browser-workspace-types'
import type { WorkspaceSessionState } from '../../../../../shared/workspace-session-state-types'
import type { WorkspaceDocHistoryEntry } from '../../../../../shared/workspace-doc-history'
import type {
BrowserAnnotationIntent,
BrowserPageAnnotation
} from '../../../../../shared/browser-grab-types'
import type { BrowserPageAnnotation } from '../../../../../shared/browser-grab-types'
import type {
BrowserPageConversionLeg,
BrowserPageConversionTarget
@@ -208,7 +205,7 @@ export type BrowserSlice = {
updateBrowserPageAnnotation: (
pageId: string,
annotationId: string,
patch: { comment: string; intent: BrowserAnnotationIntent }
patch: Pick<BrowserPageAnnotation, 'comment' | 'intent' | 'imagePaths'>
) => void
deleteBrowserPageAnnotation: (pageId: string, annotationId: string) => void
clearBrowserPageAnnotations: (pageId: string) => void
+2
View File
@@ -155,6 +155,8 @@ export type BrowserPageAnnotation = {
priority: BrowserAnnotationPriority
createdAt: string
payload: BrowserAnnotationPayload
/** Pasted images, saved on the worktree's host like a terminal image paste. */
imagePaths?: string[]
}
// Payload budgets are enforced in guest and main.