From cc6da802a57fa2f8ffef16b5197658faa123366f Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:48:45 +0000 Subject: [PATCH] feat(browser): paste images into annotation comments and send them to the agent (#36004) * feat(browser): paste images into annotation comments and send their paths to the agent * fix(paste): hand an image-only clipboard to native paste so Cmd+V reaches the annotation field * fix(browser): let an open image preview keep Escape instead of exiting Design Mode * test(paste): drop the ownership-event test the menu-claim test already covers * refactor(browser): reuse chat preview lifetimes and the terminal's paste host for annotation images * fix(browser): release pasted edit images when the edited note disappears --- .../browser-annotation-image-strip.tsx | 21 +++++ .../browser-annotation-output.test.ts | 11 +++ .../annotate/browser-annotation-output.ts | 6 ++ .../browser-guest-annotate-overlays.tsx | 1 + .../browser-page-annotation-tray.test.tsx | 6 +- .../annotate/browser-page-annotation-tray.tsx | 57 +++++++++--- .../pending-browser-annotation-card.tsx | 19 ++-- .../annotate/use-browser-annotation-images.ts | 89 +++++++++++++++++++ .../use-browser-page-annotation-send.ts | 12 ++- .../use-browser-page-grab-annotations.ts | 11 ++- .../browser-pane/annotate/useGrabMode.ts | 2 +- src/renderer/src/lib/app-menu-paste.test.ts | 32 +++---- src/renderer/src/lib/app-menu-paste.ts | 45 ++++++---- .../slices/browser/browser-slice-contract.ts | 7 +- src/shared/browser-grab-types.ts | 2 + 15 files changed, 260 insertions(+), 61 deletions(-) create mode 100644 src/renderer/src/components/browser-pane/annotate/browser-annotation-image-strip.tsx create mode 100644 src/renderer/src/components/browser-pane/annotate/use-browser-annotation-images.ts diff --git a/src/renderer/src/components/browser-pane/annotate/browser-annotation-image-strip.tsx b/src/renderer/src/components/browser-pane/annotate/browser-annotation-image-strip.tsx new file mode 100644 index 00000000000..457fe1fa8bd --- /dev/null +++ b/src/renderer/src/components/browser-pane/annotate/browser-annotation-image-strip.tsx @@ -0,0 +1,21 @@ +import { NativeChatImageAttachmentPreview } from '@/components/native-chat/NativeChatImageAttachmentPreview' +import type { NativeChatComposerImageAttachment } from '@/components/native-chat/NativeChatComposerField' + +export function BrowserAnnotationImageStrip({ + images, + onRemove +}: { + images: NativeChatComposerImageAttachment[] + onRemove: (id: string) => void +}): React.JSX.Element | null { + if (images.length === 0) { + return null + } + return ( +
+ {images.map((image) => ( + + ))} +
+ ) +} diff --git a/src/renderer/src/components/browser-pane/annotate/browser-annotation-output.test.ts b/src/renderer/src/components/browser-pane/annotate/browser-annotation-output.test.ts index a6a1a507227..e27974638dd 100644 --- a/src/renderer/src/components/browser-pane/annotate/browser-annotation-output.test.ts +++ b/src/renderer/src/components/browser-pane/annotate/browser-annotation-output.test.ts @@ -164,6 +164,17 @@ describe('formatBrowserAnnotationsAsMarkdown', () => { expect(markdown).not.toContain('\n## injected') }) + it('lists pasted image paths after the feedback so the agent can open them', () => { + const markdown = formatBrowserAnnotationsAsMarkdown([ + makeAnnotation({ imagePaths: ['/tmp/orca-paste-1.png', 'C:\\Users\\A B\\orca-paste-2.png'] }) + ]) + expect(markdown.split('\n').slice(-3)).toEqual([ + '**Images (open these files):**', + '- `/tmp/orca-paste-1.png`', + '- `C:\\Users\\A B\\orca-paste-2.png`' + ]) + }) + it('returns an empty string when no annotations exist', () => { expect(formatBrowserAnnotationsAsMarkdown([])).toBe('') }) diff --git a/src/renderer/src/components/browser-pane/annotate/browser-annotation-output.ts b/src/renderer/src/components/browser-pane/annotate/browser-annotation-output.ts index 7474b1d8876..5c95ebc8cf8 100644 --- a/src/renderer/src/components/browser-pane/annotate/browser-annotation-output.ts +++ b/src/renderer/src/components/browser-pane/annotate/browser-annotation-output.ts @@ -239,6 +239,12 @@ export function formatBrowserAnnotationsAsMarkdown(annotations: BrowserPageAnnot lines.push(...fence('html', target.htmlSnippet)) } lines.push(`**Feedback:** ${inlineText(annotation.comment)}`) + if (annotation.imagePaths?.length) { + lines.push('**Images (open these files):**') + for (const imagePath of annotation.imagePaths) { + lines.push(`- ${inlineCode(imagePath)}`) + } + } lines.push('') }) } diff --git a/src/renderer/src/components/browser-pane/annotate/browser-guest-annotate-overlays.tsx b/src/renderer/src/components/browser-pane/annotate/browser-guest-annotate-overlays.tsx index 27b7f043ede..75f3312a45a 100644 --- a/src/renderer/src/components/browser-pane/annotate/browser-guest-annotate-overlays.tsx +++ b/src/renderer/src/components/browser-pane/annotate/browser-guest-annotate-overlays.tsx @@ -95,6 +95,7 @@ export function BrowserGuestAnnotateOverlays({ {pendingAnnotationPayload ? ( getBrowserOverlayAnchor( pendingAnnotationPayload, diff --git a/src/renderer/src/components/browser-pane/annotate/browser-page-annotation-tray.test.tsx b/src/renderer/src/components/browser-pane/annotate/browser-page-annotation-tray.test.tsx index 13d16337a5a..27a09fa576a 100644 --- a/src/renderer/src/components/browser-pane/annotate/browser-page-annotation-tray.test.tsx +++ b/src/renderer/src/components/browser-pane/annotate/browser-page-annotation-tray.test.tsx @@ -139,7 +139,8 @@ describe('BrowserPageAnnotationTray edit mode', () => { expect(handleUpdateBrowserAnnotation).toHaveBeenCalledWith( 'annotation-1', 'Updated comment', - 'question' + 'question', + [] ) }) @@ -208,7 +209,8 @@ describe('BrowserPageAnnotationTray collapse', () => { expect(handleUpdateBrowserAnnotation).toHaveBeenCalledWith( 'annotation-1', 'Unsaved draft', - 'question' + 'question', + [] ) }) }) diff --git a/src/renderer/src/components/browser-pane/annotate/browser-page-annotation-tray.tsx b/src/renderer/src/components/browser-pane/annotate/browser-page-annotation-tray.tsx index f8f5beb0ff4..4cbceb70a37 100644 --- a/src/renderer/src/components/browser-pane/annotate/browser-page-annotation-tray.tsx +++ b/src/renderer/src/components/browser-pane/annotate/browser-page-annotation-tray.tsx @@ -1,6 +1,15 @@ import { ImeTextarea } from '@/lib/ime-text-field' import { useEffect, useState } from 'react' -import { ChevronDown, ChevronRight, CircleCheck, Copy, Pencil, Send, Trash2 } from 'lucide-react' +import { + ChevronDown, + ChevronRight, + CircleCheck, + Copy, + Image as ImageIcon, + Pencil, + Send, + Trash2 +} from 'lucide-react' import { Button } from '@/components/ui/button' import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible' import { @@ -21,6 +30,8 @@ import { BROWSER_ANNOTATION_INTENT_OPTIONS } from '../describe-page/browser-anno import { BrowserAnnotationSendMenuContent } from './BrowserAnnotationSendMenuContent' import { preventAgentSendTargetOutsideDismiss } from './prevent-agent-send-target-outside-dismiss' import { browserAnnotationMatchesPageUrl } from './browser-annotation-page-url' +import { useBrowserAnnotationImages } from './use-browser-annotation-images' +import { BrowserAnnotationImageStrip } from './browser-annotation-image-strip' export function BrowserPageAnnotationTray({ browserAnnotations, @@ -54,12 +65,15 @@ export function BrowserPageAnnotationTray({ handleUpdateBrowserAnnotation: ( annotationId: string, comment: string, - intent: BrowserAnnotationIntent + intent: BrowserAnnotationIntent, + imagePaths: string[] ) => void }): React.JSX.Element { const [editingAnnotationId, setEditingAnnotationId] = useState(null) const [editComment, setEditComment] = useState('') const [editIntent, setEditIntent] = useState('change') + const editImages = useBrowserAnnotationImages(worktreeId) + const resetEditImages = editImages.resetImages const [expanded, setExpanded] = useState(true) const ToggleChevron = expanded ? ChevronDown : ChevronRight @@ -70,26 +84,34 @@ export function BrowserPageAnnotationTray({ !browserAnnotations.some((annotation) => annotation.id === editingAnnotationId) ) { setEditingAnnotationId(null) + resetEditImages() } - }, [browserAnnotations, editingAnnotationId]) + }, [browserAnnotations, editingAnnotationId, resetEditImages]) const handleStartEdit = (annotation: BrowserPageAnnotation): void => { setEditingAnnotationId(annotation.id) setEditComment(annotation.comment) setEditIntent(annotation.intent) + editImages.resetImages(annotation.imagePaths) } const handleCancelEdit = (): void => { setEditingAnnotationId(null) + editImages.resetImages() } + const canSaveEdit = Boolean(editComment.trim()) && !editImages.saving const handleSaveEdit = (): void => { - const trimmed = editComment.trim() - if (!trimmed || !editingAnnotationId) { + if (!canSaveEdit || !editingAnnotationId) { return } - handleUpdateBrowserAnnotation(editingAnnotationId, trimmed, editIntent) - setEditingAnnotationId(null) + handleUpdateBrowserAnnotation( + editingAnnotationId, + editComment.trim(), + editIntent, + editImages.imagePaths + ) + handleCancelEdit() } return ( @@ -230,8 +252,12 @@ export function BrowserPageAnnotationTray({ className="min-w-0 flex-1" data-slot="annotation-edit" onKeyDown={(event) => { - // Why: bail-out selector in useGrabMode lets this reach any focused child, not just the textarea. - if (event.key === 'Escape') { + // Why: useGrabMode's bail-out lets this reach any focused child; an image preview's portal only closes itself. + if ( + event.key === 'Escape' && + event.target instanceof Node && + event.currentTarget.contains(event.target) + ) { event.preventDefault() event.stopPropagation() handleCancelEdit() @@ -241,6 +267,7 @@ export function BrowserPageAnnotationTray({ setEditComment(event.target.value)} + onPaste={editImages.handlePaste} maxLength={GRAB_BUDGET.annotationCommentMaxLength} className="h-16 w-full resize-none rounded-md border border-input bg-background px-2 py-1.5 text-xs outline-none ring-offset-background focus-visible:ring-2 focus-visible:ring-ring" autoFocus @@ -256,6 +283,10 @@ export function BrowserPageAnnotationTray({ } }} /> + -