From ab91f910f328ca7062aed22677247e653a43378c Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 16 Jul 2026 23:10:27 +0800 Subject: [PATCH] feat(terminal): paste clipboard images as a file path off macOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pasting a screenshot into a coding-agent pane (Claude Code &co.) did nothing on Windows/Linux. The clipboard-image branch only forwarded SYN (0x16), relying on the agent to read the OS clipboard itself when it sees Ctrl+V. That works on macOS, but Claude Code silently drops raw screenshots off macOS (anthropics/claude-code#26679), so nothing landed. gpui already hands us the image bytes via ClipboardEntry::Image, so off macOS we now stage the image to a temp file and paste its shell-escaped path — the same route drag-and-drop uses, which agents attach reliably. Windows screenshots arrive as BMP (CF_DIB), which agent vision rejects, so those are transcoded to PNG; PNG/JPEG/GIF/WebP pass through untouched. The temp filename is keyed on gpui's content hash so re-pasting one image reuses a single file. macOS keeps the higher-fidelity SYN path, and a staging failure there falls back to SYN too. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 1 + Cargo.toml | 8 +++ src/terminal/view.rs | 118 ++++++++++++++++++++++++++++++++++++++----- 3 files changed, 115 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 119d2e80..246c256f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9031,6 +9031,7 @@ dependencies = [ "gpui-component", "gpui-component-assets", "gpui_platform", + "image", "keyring", "ksni", "libc", diff --git a/Cargo.toml b/Cargo.toml index bbffd1fc..6a6a282f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,14 @@ keyring = "4" serde_yaml = "0.9" plist = "1" +# Clipboard-image paste (`terminal::view`). When the clipboard holds a screenshot +# and a coding-agent TUI (Claude Code &co.) is in the pane, off macOS we stage the +# image to a temp file and paste its path — agents attach an image path the same way +# they do a drag-drop. Windows screenshots arrive as BMP (`CF_DIB`), which agent +# vision won't accept, so we transcode to PNG. `image` is already in the tree via +# gpui's own clipboard code, so pinning it here adds no new native code. +image = "0.25" + # HTTP client for the startup update check (`core::update`): one GET to the # GitHub releases API to see if a newer version has shipped. `reqwest_client` # wraps Zed's `zed-reqwest` fork behind gpui's `http_client` trait (re-exported diff --git a/src/terminal/view.rs b/src/terminal/view.rs index 538d631f..40e75e36 100644 --- a/src/terminal/view.rs +++ b/src/terminal/view.rs @@ -629,6 +629,57 @@ fn clipboard_paste_text(item: &ClipboardItem) -> Option { item.text() } +/// Stage a clipboard image as a temp file so [`paste_clipboard_image`] can paste +/// its path. Web-friendly formats a coding agent's vision accepts (PNG/JPEG/GIF/ +/// WebP) are written through untouched; anything else — notably the BMP that +/// Windows screenshots (`CF_DIB`) arrive as — is transcoded to PNG, since agent +/// vision rejects those. Returns the path, or `None` if decoding/writing failed. +/// +/// The filename is keyed on gpui's content hash of the bytes, so re-pasting the +/// same screenshot reuses one file instead of accumulating temp copies (this +/// crate has no `Date`/random to mint a unique name with anyway). +#[cfg(not(target_os = "macos"))] +fn write_clipboard_image(img: &gpui::Image) -> Option { + use gpui::ImageFormat; + let dir = std::env::temp_dir().join("tty7-clipboard"); + std::fs::create_dir_all(&dir).ok()?; + let (ext, transcoded) = match img.format { + ImageFormat::Png => ("png", None), + ImageFormat::Jpeg => ("jpg", None), + ImageFormat::Gif => ("gif", None), + ImageFormat::Webp => ("webp", None), + other => ("png", Some(transcode_to_png(&img.bytes, other)?)), + }; + let data: &[u8] = transcoded.as_deref().unwrap_or(&img.bytes); + let path = dir.join(format!("paste-{:016x}.{ext}", img.id)); + std::fs::write(&path, data).ok()?; + Some(path) +} + +/// Decode `bytes` (in `format`) and re-encode as PNG. SVG can't be rasterized by +/// the `image` crate, so it — and any decode/encode failure — yields `None`. +#[cfg(not(target_os = "macos"))] +fn transcode_to_png(bytes: &[u8], format: gpui::ImageFormat) -> Option> { + use gpui::ImageFormat as G; + let src = match format { + G::Png => image::ImageFormat::Png, + G::Jpeg => image::ImageFormat::Jpeg, + G::Webp => image::ImageFormat::WebP, + G::Gif => image::ImageFormat::Gif, + G::Bmp => image::ImageFormat::Bmp, + G::Tiff => image::ImageFormat::Tiff, + G::Ico => image::ImageFormat::Ico, + G::Pnm => image::ImageFormat::Pnm, + G::Svg => return None, + }; + let decoded = image::load_from_memory_with_format(bytes, src).ok()?; + let mut out = Vec::new(); + decoded + .write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png) + .ok()?; + Some(out) +} + /// The font fallback chain: the user's configured list with the bundled "Hack" /// pinned to the end. Hack ships inside the binary (`register_bundled_fonts`) /// and covers the symbols prompt themes lean on — `❯`, `➜`, box drawing, the @@ -1414,18 +1465,15 @@ impl TerminalView { if let Some(item) = cx.read_from_clipboard() { if let Some(text) = clipboard_paste_text(&item) { self.paste(text, cx); - } else if !self.input_active() - && item - .entries() - .iter() - .any(|e| matches!(e, ClipboardEntry::Image(_))) - { - // Clipboard holds an image (e.g. a screenshot) with no text. - // A foreground TUI like Claude Code reads the image from the OS - // clipboard itself when it sees Ctrl+V (SYN, 0x16), so forward - // that byte instead of trying to send image data — matching how - // GUI terminals route Cmd+V image pastes to CLI agents. - self.terminal.write(vec![0x16]); + } else if !self.input_active() { + if let Some(img) = item.entries().iter().find_map(|e| match e { + ClipboardEntry::Image(img) => Some(img), + _ => None, + }) { + // Clipboard holds an image (e.g. a screenshot) with no text, + // and a foreground TUI (a coding agent) owns the pane. + self.paste_clipboard_image(img, cx); + } } } CmdKey::Consumed @@ -2266,6 +2314,27 @@ impl TerminalView { self.paste(format!("{text} "), cx); } + /// Paste a clipboard image (e.g. a screenshot) into a foreground coding-agent + /// TUI. Agents like Claude Code attach an image typed as a *file path* at the + /// prompt — the same route drag-and-drop uses — so off macOS we stage the image + /// to a temp file and paste its shell-escaped path, mirroring [`drop_files`]. + /// + /// On macOS the agent can instead read the image straight from the pasteboard + /// when it sees Ctrl+V, so we forward SYN (`0x16`) and let it do that + /// higher-fidelity read. That same read is unreliable off macOS — Claude Code on + /// Windows silently drops raw screenshots (anthropics/claude-code#26679) — which + /// is why we materialize a file there. If staging fails, we fall back to SYN. + fn paste_clipboard_image(&mut self, img: &gpui::Image, cx: &mut Context) { + #[cfg(not(target_os = "macos"))] + if let Some(path) = write_clipboard_image(img) { + let text = shell_escape_path(&path.to_string_lossy()); + self.paste(format!("{text} "), cx); + return; + } + let _ = (img, cx); + self.terminal.write(vec![0x16]); + } + /// Clear the terminal (right-click "Clear"), like Cmd+K / the `clear` /// command: purge the scrollback history *and* wipe the visible screen. /// We drop the history directly, then send Ctrl+L so the shell/TUI repaints @@ -5238,6 +5307,31 @@ mod tests { use gpui_component::IconName; use std::path::PathBuf; + #[cfg(not(target_os = "macos"))] + #[test] + fn clipboard_image_transcodes_bmp_to_png_and_passes_png_through() { + use gpui::{Image, ImageFormat}; + + // A BMP (what a Windows screenshot lands as) must be re-encoded to PNG, + // since agent vision rejects BMP. Build one with the image crate. + let pixel = image::RgbaImage::from_pixel(1, 1, image::Rgba([1, 2, 3, 255])); + let mut bmp = Vec::new(); + image::DynamicImage::ImageRgba8(pixel) + .write_to(&mut std::io::Cursor::new(&mut bmp), image::ImageFormat::Bmp) + .unwrap(); + let path = super::write_clipboard_image(&Image::from_bytes(ImageFormat::Bmp, bmp)).unwrap(); + assert_eq!(path.extension().unwrap(), "png"); + // PNG magic number: the staged file is genuinely a PNG, not renamed BMP. + assert_eq!(&std::fs::read(&path).unwrap()[..8], b"\x89PNG\r\n\x1a\n"); + + // A format agents already accept is written through byte-for-byte. + let png = std::fs::read(&path).unwrap(); + let out = super::write_clipboard_image(&Image::from_bytes(ImageFormat::Png, png.clone())) + .unwrap(); + assert_eq!(out.extension().unwrap(), "png"); + assert_eq!(std::fs::read(&out).unwrap(), png); + } + /// The bundled Hack always anchors the fallback chain so prompt symbols /// (`➜`, `❯`, powerline wedges) never fall through to the OS cascade — /// unless the user already covers it as primary or in their own list.