From 40a8ba711b60c566bd5f020da90b1a8342c67748 Mon Sep 17 00:00:00 2001 From: l0ng-ai Date: Fri, 31 Jul 2026 12:11:24 +0800 Subject: [PATCH 1/2] fix(input): stop a Tab on a detached remote pane from eating the line (#273) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(input): stop a Tab on a detached remote pane from eating the line Tab reaches the view through the SendTab/SendBackTab actions, which bypass the read-only guard in on_key_down. On a remote workspace whose link is not attached, that let a failed completion run handoff_line_to_shell: it cleared the editor's line and wrote it to a writer nobody was reading, so the command vanished, the editor stood down for the rest of the prompt cycle, and every later keystroke was swallowed by the guard the Tab had skipped. Guard tab_pressed, handoff_line_to_shell and submit_command with accepts_input, so a pane that cannot reach its shell keeps the line instead of handing it to nowhere. A late SFTP listing now also checks that the editor still owns the line before acting on it - that round trip can take seconds. A listing that fails still hands the line to the shell, same as locally. * fix(input): name the reason a detached link turns a Tab away Fold the link check into a `link_inactive_reason` so the Tab path logs why it did nothing, and let `remote_path_results` consult the link as well as the editor instead of leaning on `handoff_line_to_shell` to catch it. Reword the failed-listing warning: with the drop guard in place the line does not always reach the shell. Dispatch the Tab in the disconnected-pane test through the real `SendTab` action, since the bug was that the action skips `on_key_down`, and add a test that a Tab on a detached pane never starts an SFTP listing — the one thing only the Tab-path guard prevents. --------- Co-authored-by: l0ng-ai <24760907+l0ng-ai@users.noreply.github.com> --- src/terminal/view.rs | 186 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 174 insertions(+), 12 deletions(-) diff --git a/src/terminal/view.rs b/src/terminal/view.rs index f270fd37..f3cad284 100644 --- a/src/terminal/view.rs +++ b/src/terminal/view.rs @@ -2472,6 +2472,10 @@ impl TerminalView { None } + fn link_inactive_reason(&self, cx: &gpui::App) -> Option<&'static str> { + (!self.accepts_input(cx)).then_some("the remote link is not attached") + } + fn shell_vi_prompt(&self) -> bool { self.terminal.shell_vi_mode() && self.terminal.at_prompt() && !self.on_alt_screen() } @@ -2596,7 +2600,7 @@ impl TerminalView { } fn submit_command(&mut self, cx: &mut Context) { - if self.terminal.exited { + if self.terminal.exited || !self.accepts_input(cx) { return; } if let Some(net) = self.hold.engage() { @@ -2855,6 +2859,9 @@ impl TerminalView { } fn handoff_line_to_shell(&mut self, chord: &[u8], cx: &mut Context) { + if !self.accepts_input(cx) { + return; + } if let Some(net) = self.hold.engage() { self.cmd.prepend_str(&net); } @@ -2882,6 +2889,10 @@ impl TerminalView { cx.propagate(); return; } + if let Some(reason) = self.link_inactive_reason(cx) { + log::debug!(target: "tty7::completion", "Tab does nothing and the line stays: {reason}"); + return; + } if let Some(reason) = self.input_inactive_reason() { log::debug!(target: "tty7::completion", "Tab goes straight to the PTY: {reason}"); let bytes = self.tab_bytes(!forward); @@ -3040,19 +3051,16 @@ impl TerminalView { log::debug!(target: "tty7::completion", "listing {dir} over the remote's own connection"); cx.spawn(async move |this, cx| { let listed = cx.background_spawn(async move { route.list(&dir) }).await; - if let Err(e) = &listed { - log::warn!(target: "tty7::completion", "remote listing failed: {e}"); - } + let entries = listed.unwrap_or_else(|e| { + log::warn!( + target: "tty7::completion", + "remote listing failed, treating it as no candidates: {e}" + ); + Vec::new() + }); let _ = this.update(cx, |view, cx| { view.remote_completion_inflight = false; - view.remote_path_results( - req, - &line, - cursor, - listed.unwrap_or_default(), - forward, - cx, - ) + view.remote_path_results(req, &line, cursor, entries, forward, cx); }); }) .detach(); @@ -3068,6 +3076,16 @@ impl TerminalView { forward: bool, cx: &mut Context, ) { + if let Some(reason) = self + .link_inactive_reason(cx) + .or_else(|| self.input_inactive_reason()) + { + log::debug!( + target: "tty7::completion", + "dropping a remote listing for {line:?}: {reason}" + ); + return; + } if self.cmd.text() != line || self.cmd.cursor() != cursor { log::debug!( target: "tty7::completion", @@ -5940,6 +5958,45 @@ mod gpui_tests { wait_for_input_active(&window, cx); } + #[gpui::test] + fn a_late_remote_listing_leaves_a_line_the_editor_no_longer_owns_alone( + cx: &mut TestAppContext, + ) { + crate::core::config::pin_test_config_dir(); + let (window, mut daemon) = harness(cx); + DaemonMsg::Prompt { + active: true, + at_prompt: true, + last_exit: None, + } + .encode(&mut daemon) + .unwrap(); + wait_for_input_active(&window, cx); + + window + .update(cx, |view, _, cx| { + view.cmd.set("ls /nope/"); + view.editor_handoff = Some(view.terminal.prompt_cycle()); + assert!(!view.input_active(), "the shell owns this prompt already"); + + let req = + super::completion::remote_path_request("ls /nope/", 9, "/home/u").unwrap(); + view.remote_path_results(req, "ls /nope/", 9, Vec::new(), true, cx); + + assert_eq!( + view.cmd.text(), + "ls /nope/", + "an empty listing must not hand off a line the editor no longer drives" + ); + }) + .unwrap(); + assert_eq!( + next_input_until_timeout(&mut daemon), + None, + "not one byte reached the wire" + ); + } + #[gpui::test] fn tab_completion_off_sends_every_tab_to_the_shell(cx: &mut TestAppContext) { let (window, mut daemon) = harness(cx); @@ -6931,6 +6988,111 @@ mod gpui_tests { id } + #[gpui::test] + fn a_disconnected_remote_pane_keeps_the_line_instead_of_handing_it_to_nowhere( + cx: &mut TestAppContext, + ) { + crate::core::config::pin_test_config_dir(); + let (window, mut daemon) = harness(cx); + cx.update(|cx| crate::ui::keymap::init(cx)); + DaemonMsg::Prompt { + active: true, + at_prompt: true, + last_exit: None, + } + .encode(&mut daemon) + .unwrap(); + wait_for_input_active(&window, cx); + + window + .update(cx, |view, window, cx| { + window.activate_window(); + view.focus_handle.focus(window, cx); + view.cmd.set("zzqqx"); + bind_to_a_disconnected_remote_workspace(view, cx); + }) + .unwrap(); + + let mut vcx = gpui::VisualTestContext::from_window(window.into(), cx); + vcx.simulate_keystrokes("tab"); + + window + .update(cx, |view, _, cx| { + assert_eq!( + view.cmd.text(), + "zzqqx", + "a Tab dispatched through SendTab must not empty the line" + ); + assert!( + view.editor_handoff.is_none(), + "nothing was handed off, so the editor keeps the prompt" + ); + + view.submit_command(cx); + assert_eq!( + view.cmd.text(), + "zzqqx", + "submit_command guards the link too, even though on_key_down already does" + ); + }) + .unwrap(); + assert_eq!( + next_input_until_timeout(&mut daemon), + None, + "not one byte reached the wire" + ); + } + + #[gpui::test] + fn a_tab_on_a_detached_remote_pane_never_asks_for_a_listing(cx: &mut TestAppContext) { + use std::io::Write as _; + crate::core::config::pin_test_config_dir(); + let (window, mut daemon) = harness(cx); + DaemonMsg::Prompt { + active: true, + at_prompt: true, + last_exit: None, + } + .encode(&mut daemon) + .unwrap(); + DaemonMsg::Cwd(std::path::PathBuf::from("/home/me/proj")) + .encode(&mut daemon) + .unwrap(); + daemon.flush().unwrap(); + wait_for_input_active(&window, cx); + for _ in 0..200 { + if window + .update(cx, |view, _, _| view.cwd().is_some()) + .unwrap() + { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + + window + .update(cx, |view, _, cx| { + view.cmd.set("ls /home/me/"); + bind_to_a_disconnected_remote_workspace(view, cx); + assert!( + view.remote_ssh_cwd().is_some(), + "the pane has to look remote enough to want a listing at all" + ); + + view.tab_pressed(true, cx); + assert!( + !view.remote_completion_inflight, + "a Tab must not send an SFTP listing down a link that is not attached" + ); + assert_eq!( + view.cmd.text(), + "ls /home/me/", + "and the line stays where it was" + ); + }) + .unwrap(); + } + #[gpui::test] fn a_disconnected_remote_pane_swallows_every_kind_of_typing(cx: &mut TestAppContext) { let (window, mut daemon) = harness(cx); From 83d9a1c5475810e8c448c8113139f41a6a70187d Mon Sep 17 00:00:00 2001 From: ayamir <61657399+ayamir@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:47:59 +0800 Subject: [PATCH 2/2] feat(graphics): render kitty graphics with shared-memory transport (#272) Adds kitty graphics protocol support: a daemon-side APC tokenizer lifts image transmissions out of the PTY stream before the replay ring, forwards them out-of-band as compact binary frames interleaved in stream order, and the client decodes off-thread with newest-wins coalescing per image id. Local panes take the file/shm fast path; remote panes keep pixels compressed in-tunnel. Cell size is now reported to children in device pixels so pixel-aware senders render at native resolution. Closes #213. --- Cargo.lock | 1 + crates/tty7-core/Cargo.toml | 6 + crates/tty7-core/src/core/kitty_graphics.rs | 1937 +++++++++++++++++++ crates/tty7-core/src/core/mod.rs | 1 + crates/tty7-core/src/daemon/pane.rs | 321 ++- crates/tty7-core/src/daemon/protocol.rs | 23 + crates/tty7-core/src/daemon/server.rs | 4 + src/terminal/element.rs | 91 +- src/terminal/images.rs | 573 ++++++ src/terminal/mod.rs | 1 + src/terminal/remote.rs | 105 +- src/terminal/view.rs | 24 +- 12 files changed, 3067 insertions(+), 20 deletions(-) create mode 100644 crates/tty7-core/src/core/kitty_graphics.rs create mode 100644 src/terminal/images.rs diff --git a/Cargo.lock b/Cargo.lock index 635a2cf0..2cab2782 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9587,6 +9587,7 @@ dependencies = [ "libgssapi", "log", "memchr", + "miniz_oxide", "notify 8.2.0", "portable-pty", "russh", diff --git a/crates/tty7-core/Cargo.toml b/crates/tty7-core/Cargo.toml index bc262270..b4a73620 100644 --- a/crates/tty7-core/Cargo.toml +++ b/crates/tty7-core/Cargo.toml @@ -112,6 +112,12 @@ memchr = "2" # roughly 4 MB. Base64 costs 1.33× instead. Already in the tree via russh. base64 = "0.22" +# zlib inflate for the kitty graphics protocol's `o=z` (compressed) pixel +# payloads (`core::kitty_graphics`). Pure-Rust, no new native code, and already +# in the tree transitively — the same crate `terminal-browser` uses for the +# deflate side, so the two ends agree on the wire format. +miniz_oxide = "0.8" + [target.'cfg(unix)'.dependencies] libc = "0.2" # GSSAPI/Kerberos SSH auth — see the `gssapi` feature below for why it is diff --git a/crates/tty7-core/src/core/kitty_graphics.rs b/crates/tty7-core/src/core/kitty_graphics.rs new file mode 100644 index 00000000..ff7fa7a5 --- /dev/null +++ b/crates/tty7-core/src/core/kitty_graphics.rs @@ -0,0 +1,1937 @@ +//! Streaming kitty-graphics-protocol extractor (APC `ESC _ G … ST`). +//! +//! The daemon-side counterpart to [`crate::core::osc`]: a tiny, resumable state +//! machine that pulls kitty graphics commands out of the raw PTY byte stream +//! *without* running a full VT parser. It exists for the same reason the OSC +//! sniffer does — the client's `alacritty_terminal` fork silently discards APC +//! sequences (`ESC _` routes to vte's `SosPmApcString`, whose payload bytes hit +//! a `_ => ()` arm), so an image transmitted this way never surfaces as a `Term` +//! event. We tap the bytes here instead. +//! +//! This lives daemon-side so the pixel payload can be lifted out of the stream +//! *before* it enters the replay ring — a full-window RGBA frame is hundreds of +//! KB, and letting it accumulate in the ring would make reattach replay +//! catastrophic. The daemon decodes here and forwards a compact out-of-band +//! frame to the client; the base64 text never rides the socket, and the VT +//! parser never has to chew through it. +//! +//! Scope: this handles the subset the wire actually carries in practice +//! (transmit-and-display, query, delete). Which transmission media we accept +//! depends on where the pane lives: +//! +//! - Direct (`t=d`, base64 inline) is always honored — it is the only medium +//! that survives tty7's socket + SSH topology. +//! - File (`t=f`/`t=t`) and shared memory (`t=s`) are honored only on a *local* +//! unix pane, where the name resolves on this host and the daemon can read it +//! without anything crossing a tunnel. Everywhere else — a remote pane, or a +//! non-unix host where [`MediumTransfer::resolve`] can't do the work — the +//! probe is answered *unsupported* so a sender like `terminal-browser` falls +//! back to `t=d` on its own rather than transmitting into a black hole. +//! +//! Protocol reference: + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use serde::{Deserialize, Serialize}; + +/// Cap on a single APC payload (one chunk) we'll buffer before abandoning it. +/// `terminal-browser` chunks direct transmissions into 4 KiB base64 pieces, but +/// the spec only *recommends* chunking — a sender is free to put a whole frame +/// in one command, and an uncompressed 4K RGBA frame is ~44 MB of base64. Size +/// this to admit a one-shot frame that still fits [`MAX_TRANSMISSION_BASE64`]; +/// [`ApcTokenizer::push_graphics`] logs whatever it has to drop. +const MAX_APC_PAYLOAD: usize = MAX_TRANSMISSION_BASE64; + +/// Cap on the reassembled base64 of one chunked transmission. +/// +/// This has to stay under [`crate::daemon::protocol::MAX_FRAME`] once decoded: +/// base64 shrinks by 3/4, and [`Image::encode_frame`] prepends [`HEADER_LEN`] +/// bytes, so the decoded payload must leave room for the header inside the wire +/// frame. Blowing past `MAX_FRAME` would make `write_frame` fail, which the +/// daemon's writer loop treats as fatal — one oversized image would drop the +/// client's whole connection instead of just that frame. 48 MiB of base64 is +/// ~36 MB of pixels, comfortably more than a 4K full-window RGBA frame (~33 MB) +/// and comfortably under the 64 MiB wire ceiling. +const MAX_TRANSMISSION_BASE64: usize = 48 << 20; // 48 MiB + +/// Cap on the *resolved* pixel bytes of one image, whatever medium carried it. +/// Bounds the file/shm fast path (where the sender names an object whose size we +/// don't control) and backstops the direct path, keeping every frame we hand the +/// daemon inside [`crate::daemon::protocol::MAX_FRAME`]. +pub const MAX_IMAGE_BYTES: usize = crate::daemon::protocol::MAX_FRAME - HEADER_LEN; + +/// A streaming tokenizer that splits raw output into a *passthrough* byte stream +/// and the kitty graphics commands lifted out of it. +/// +/// Feed it raw output bytes. It invokes `on_passthrough` with every byte that is +/// **not** part of a `_G` graphics sequence — that is the exact input with each +/// `ESC _ G … ST` removed, and it is what the daemon appends to the replay ring +/// and forwards to the client. It invokes `on_command` with the complete payload +/// of each `_G` command (the bytes after `ESC _`, terminator excluded — e.g. +/// `Ga=T,f=32,…;`). Non-graphics APC sequences (any payload not starting +/// with `G`) are passed through verbatim, since only kitty graphics is ours to +/// intercept; everything else must reach the client's VT parser unchanged. State +/// persists across `feed` calls, so a sequence split over several reads is still +/// handled. +pub struct ApcTokenizer { + /// Bytes of a `_G` command accumulated after `ESC _ G` while it can still + /// terminate. Cleared whenever a command finishes or is abandoned. + buf: Vec, + state: State, +} + +#[derive(Default, Clone, Copy)] +enum State { + /// Not inside an escape sequence; bytes are passthrough. + #[default] + Ground, + /// Held one `ESC` in ground state; a following `_` opens an APC. + Esc, + /// Held `ESC _`; the next byte decides graphics (`G`) vs. passthrough APC. + ApcStart, + /// Buffering a `_G` graphics command (stripped from passthrough). + ApcGraphics, + /// Saw `ESC` inside a graphics command — a following `\` is the terminator. + ApcGraphicsEsc, + /// Forwarding a non-graphics APC as passthrough (bytes kept verbatim). + PassApc, + /// Saw `ESC` inside a passthrough APC — a following `\` is the terminator. + PassApcEsc, + /// Dropping an abandoned/oversized graphics APC (emitted nowhere). + ApcDrop, + /// Saw `ESC` inside a dropped APC — a following `\` is the terminator. + ApcDropEsc, +} + +impl Default for ApcTokenizer { + fn default() -> Self { + Self::new() + } +} + +impl ApcTokenizer { + pub fn new() -> Self { + Self { + buf: Vec::new(), + state: State::Ground, + } + } + + /// Feed one chunk of output. `on_passthrough` receives runs of non-graphics + /// bytes (the input minus every `ESC _ G … ST`); `on_command` receives the + /// `G…` payload of each graphics command that terminates within the chunk. + /// + /// Like the OSC tokenizer, the states that dominate a real stream — `Ground` + /// between sequences, and the scan-to-terminator states inside an APC — skip + /// ahead with SIMD `memchr` rather than stepping per byte; only the few + /// escape-decision states step one byte at a time. APC has no `BEL` + /// terminator (kitty always closes with `ST`), so only `ESC` can end a + /// payload. A held `ESC`/`ESC _` that later proves to be passthrough is + /// re-emitted as a constant, so nothing needs to survive across `feed` calls + /// but the small state tag and the graphics buffer. + pub fn feed( + &mut self, + bytes: &[u8], + mut on_passthrough: impl FnMut(&[u8]), + mut on_command: impl FnMut(&[u8]), + ) { + let mut i = 0; + while i < bytes.len() { + match self.state { + State::Ground => match memchr::memchr(0x1b, &bytes[i..]) { + Some(off) => { + if off > 0 { + on_passthrough(&bytes[i..i + off]); + } + self.state = State::Esc; + i += off + 1; + } + None => { + on_passthrough(&bytes[i..]); + return; + } + }, + State::PassApc => match memchr::memchr(0x1b, &bytes[i..]) { + Some(off) => { + if off > 0 { + on_passthrough(&bytes[i..i + off]); + } + self.state = State::PassApcEsc; + i += off + 1; + } + None => { + on_passthrough(&bytes[i..]); + return; + } + }, + State::ApcDrop => match memchr::memchr(0x1b, &bytes[i..]) { + Some(off) => { + self.state = State::ApcDropEsc; + i += off + 1; + } + None => return, + }, + State::ApcGraphics => match memchr::memchr(0x1b, &bytes[i..]) { + Some(off) => { + if self.push_graphics(&bytes[i..i + off]) { + self.state = State::ApcGraphicsEsc; + i += off + 1; + } else { + // Oversized: abandon and let the drop state consume + // the terminator we just found (leave `i` on the ESC). + self.state = State::ApcDrop; + i += off; + } + } + None => { + if !self.push_graphics(&bytes[i..]) { + self.state = State::ApcDrop; + } + return; + } + }, + State::Esc => match bytes[i] { + b'_' => { + self.buf.clear(); + self.state = State::ApcStart; + i += 1; + } + // A run of ESCs: the earlier one was a lone ESC (passthrough); + // keep the newest one held. + 0x1b => { + on_passthrough(b"\x1b"); + i += 1; + } + // The ESC began some other escape: emit it and re-examine this + // byte from ground (it is not an ESC, so it joins the run). + _ => { + on_passthrough(b"\x1b"); + self.state = State::Ground; + } + }, + State::ApcStart => { + if bytes[i] == b'G' { + self.buf.clear(); + self.buf.push(b'G'); + self.state = State::ApcGraphics; + i += 1; + } else { + // Not ours: emit the held `ESC _` and forward the rest of + // the APC verbatim (re-examine this byte in `PassApc`). + on_passthrough(b"\x1b_"); + self.state = State::PassApc; + } + } + State::ApcGraphicsEsc => match bytes[i] { + b'\\' => { + on_command(&self.buf); + self.buf.clear(); + self.state = State::Ground; + i += 1; + } + 0x1b => i += 1, // a run of ESCs; stay poised for `\` + // A bare `ESC _` re-opens: the next byte decides graphics again. + b'_' => { + self.buf.clear(); + self.state = State::ApcStart; + i += 1; + } + // ESC began some other escape: abandon this graphics command. + // Its bytes were meant as graphics, so they stay stripped — + // but the escape itself belongs to the terminal, so forward + // it and re-examine this byte from Ground rather than eating + // both. Swallowing them would turn a `\x1b[31m` that follows + // an unterminated graphics command into literal `31m`. + _ => { + self.buf.clear(); + on_passthrough(b"\x1b"); + self.state = State::Ground; + } + }, + State::PassApcEsc => match bytes[i] { + b'\\' => { + on_passthrough(b"\x1b\\"); // ST is part of the forwarded APC + self.state = State::Ground; + i += 1; + } + 0x1b => { + on_passthrough(b"\x1b"); // a lone ESC in APC data + i += 1; + } + // A bare `ESC _` re-opens, same as the two graphics states + // do. Without this, a foreign APC that never sends its ST + // swallows every `ESC _G …` after it — the graphics get + // forwarded as APC text that the client's vte then discards, + // so the image is simply lost. + b'_' => { + self.buf.clear(); + self.state = State::ApcStart; + i += 1; + } + _ => { + on_passthrough(b"\x1b"); + self.state = State::PassApc; // re-examine this byte + } + }, + State::ApcDropEsc => match bytes[i] { + b'\\' => { + self.state = State::Ground; + i += 1; + } + 0x1b => i += 1, + b'_' => { + self.buf.clear(); + self.state = State::ApcStart; + i += 1; + } + // As in `ApcGraphicsEsc`: the dropped command's bytes stay + // stripped, but the escape that interrupted it is the + // terminal's and has to reach the client intact. + _ => { + on_passthrough(b"\x1b"); + self.state = State::Ground; + } + }, + } + } + } + + /// Append a run to the graphics buffer; returns `false` (and clears the + /// buffer) if it would exceed [`MAX_APC_PAYLOAD`], signalling the caller to + /// abandon the command. + fn push_graphics(&mut self, run: &[u8]) -> bool { + if self.buf.len() + run.len() > MAX_APC_PAYLOAD { + log::debug!( + "kitty graphics: dropping a command whose payload passed {MAX_APC_PAYLOAD} bytes" + ); + self.buf.clear(); + return false; + } + self.buf.extend_from_slice(run); + true + } +} + +/// Daemon-side splitter: the [`ApcTokenizer`] wired to a [`GraphicsParser`]. +/// +/// This is the one type the PTY reader loop touches. Feed it raw output and a +/// passthrough sink; it strips every kitty graphics sequence, funnels the +/// passthrough bytes to the sink (for the replay ring and the client), and +/// returns the decoded graphics [`Event`]s — query replies to write back to the +/// PTY, and images to forward out-of-band. +#[derive(Default)] +pub struct GraphicsSniffer { + tokenizer: ApcTokenizer, + parser: GraphicsParser, +} + +impl GraphicsSniffer { + pub fn new() -> Self { + Self::default() + } + + /// A sniffer whose parser may honor file/shm transfer (local pane only). + /// See [`GraphicsParser::new_local`]. + pub fn new_local(local: bool) -> Self { + Self { + tokenizer: ApcTokenizer::new(), + parser: GraphicsParser::new_local(local), + } + } + + /// Update whether the sender currently shares this host's filesystem. A pane + /// is local when tty7 spawned a local shell and no foreground `ssh` owns the + /// PTY; that can flip mid-session, and it gates whether the next `a=q` probe + /// is answered `OK` for file/shm transfer. Cheap enough to call per chunk. + pub fn set_local(&mut self, local: bool) { + self.parser.local = local; + } + + /// Feed one chunk of raw PTY output. `on_passthrough` receives the byte + /// stream with all graphics sequences removed; the returned events are the + /// queries/images/deletes that completed within this chunk, in order. + /// + /// This drops the *relative order* of passthrough vs. events; for the daemon + /// loop, prefer [`sniff`](Self::sniff), which preserves it. Retained as the + /// low-level primitive the unit tests drive. + pub fn feed(&mut self, bytes: &[u8], on_passthrough: impl FnMut(&[u8])) -> Vec { + let Self { tokenizer, parser } = self; + let mut events = Vec::new(); + tokenizer.feed(bytes, on_passthrough, |cmd| { + if let Some(ev) = parser.feed(cmd) { + events.push(ev); + } + }); + events + } + + /// Feed a chunk and get back its content **in stream order**, taking a + /// zero-copy fast path on the overwhelmingly common chunk that carries no + /// graphics at all. + /// + /// Order matters: a kitty image is placed at the cursor cell as it stood + /// *when its command appeared in the stream*, so the client must apply the + /// text before an image before resolving that image's anchor. Returning + /// ordered [`Segment`]s lets the daemon forward `Output`/`Image` frames + /// interleaved exactly as they arrived, so the client's mirror cursor is + /// correct at each placement. + /// + /// The fast path fires when the tokenizer is between sequences *and* the + /// chunk holds no APC opener (`ESC _`) — then the input **is** one big + /// output segment, returned borrowed with no allocation. A build's worth of + /// colored stdout (full of CSI escapes but no APC) stays on this path. The + /// trailing-`ESC` guard keeps an `ESC _` straddling a chunk boundary on the + /// slow path, where the held-`ESC` state carries it across correctly. + pub fn sniff<'a>(&mut self, bytes: &'a [u8]) -> Sniffed<'a> { + if matches!(self.tokenizer.state, State::Ground) + && bytes.last() != Some(&0x1b) + && memchr::memmem::find(bytes, b"\x1b_").is_none() + { + return Sniffed::Plain(bytes); + } + let Self { tokenizer, parser } = self; + // Both tokenizer callbacks push onto the same segment list; `RefCell` + // lets the passthrough and command sinks share it without either taking + // a lasting mutable borrow the borrow checker would reject. + let segments: std::cell::RefCell> = std::cell::RefCell::new(Vec::new()); + tokenizer.feed( + bytes, + |run| { + // Coalesce adjacent passthrough runs into one output segment so + // the client applies them in a single `advance`. + let mut segs = segments.borrow_mut(); + if let Some(Segment::Output(buf)) = segs.last_mut() { + buf.extend_from_slice(run); + } else { + segs.push(Segment::Output(run.to_vec())); + } + }, + |cmd| { + if let Some(ev) = parser.feed(cmd) { + segments.borrow_mut().push(Segment::from(ev)); + } + }, + ); + Sniffed::Segments(segments.into_inner()) + } +} + +/// One ordered piece of a sniffed chunk — see [`GraphicsSniffer::sniff`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Segment { + /// Non-graphics bytes to append to the ring / forward as `Output`. + Output(Vec), + /// An `a=q` reply to write back to the PTY. + Query(Vec), + /// An image to forward out-of-band. + Image(Image), + /// A file/shm transmission the daemon must resolve into pixels before + /// forwarding (local panes only). See [`MediumTransfer`]. + ImageFromMedium(MediumTransfer), + /// A delete to forward out-of-band. + Delete(ImageDelete), +} + +impl From for Segment { + fn from(ev: Event) -> Self { + match ev { + Event::Query { reply, .. } => Segment::Query(reply), + Event::Image(img) => Segment::Image(img), + Event::ImageFromMedium(t) => Segment::ImageFromMedium(t), + Event::Delete(c) => Segment::Delete(ImageDelete::from_control(&c)), + } + } +} + +/// The result of [`GraphicsSniffer::sniff`]: either the whole chunk borrowed as +/// output (the graphics-free fast path), or ordered [`Segment`]s. +pub enum Sniffed<'a> { + /// No graphics in this chunk: the input is output, verbatim and borrowed. + Plain(&'a [u8]), + /// Graphics present: apply these in order. + Segments(Vec), +} + +/// The kitty action key (`a=`). Only the variants tty7 acts on are named; the +/// rest of the protocol (frames, animation, compose) folds into `Other`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Action { + /// `a=q` — query whether a transmission would succeed. We must reply. + Query, + /// `a=t` — transmit only (no display). Rare from our target sender. + Transmit, + /// `a=T` — transmit and display. + TransmitAndDisplay, + /// `a=p` — display a previously transmitted image. + Display, + /// `a=d` — delete image(s)/placement(s). + Delete, + /// Any action we don't specifically handle. + Other, +} + +/// The pixel format key (`f=`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum WireFormat { + /// `f=24` — 3 bytes/pixel RGB. + Rgb, + /// `f=32` — 4 bytes/pixel RGBA (the default, and what `terminal-browser` uses). + Rgba, + /// `f=100` — a PNG file; pixels stay encoded (decoded client-side). + Png, +} + +/// The transmission medium key (`t=`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Medium { + /// `t=d` — direct: the payload is (base64) the image data itself. + Direct, + /// `t=f` — a filesystem path to the data. + File, + /// `t=s` — a POSIX shared-memory object name. + Shared, + /// `t=t` — a temporary file (deleted after reading). + TempFile, +} + +/// Parsed kitty graphics control keys (the `k=v,k=v` list before the `;`). +/// +/// Only the keys tty7 needs are surfaced; unknown keys are ignored so forward +/// protocol additions degrade quietly rather than failing the whole command. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Control { + pub action: Action, + pub format: WireFormat, + pub medium: Medium, + /// `o=z` — payload is zlib-compressed. + pub compressed: bool, + /// `i=` — client-assigned image id (0 = unset). + pub id: u32, + /// `I=` — client-assigned image number (0 = unset). + pub number: u32, + /// `p=` — placement id (0 = unset). + pub placement: u32, + /// `s=` — source pixel width. + pub width: u32, + /// `v=` — source pixel height. + pub height: u32, + /// `c=` — columns to display across (0 = derive from pixels). + pub cols: u32, + /// `r=` — rows to display down (0 = derive from pixels). + pub rows: u32, + /// `m=1` — more chunks follow. + pub more: bool, + /// `d=` — delete target (only meaningful for `a=d`). + pub delete: u8, + /// `q=` — suppress responses (1 = errors only, 2 = all). + pub quiet: u8, + /// `O=` — byte offset into a file/shm object (file/shm mediums only). + pub offset: u32, + /// `S=` — number of bytes to read from a file/shm object (0 = to end). + pub size: u32, +} + +impl Default for Control { + fn default() -> Self { + // Protocol defaults: a=t, f=32, t=d, no compression. + Self { + action: Action::Transmit, + format: WireFormat::Rgba, + medium: Medium::Direct, + compressed: false, + id: 0, + number: 0, + placement: 0, + width: 0, + height: 0, + cols: 0, + rows: 0, + more: false, + delete: 0, + quiet: 0, + offset: 0, + size: 0, + } + } +} + +impl Control { + /// Parse the control section of a `_G` command — the `G…` payload up to (but + /// not including) the first `;`. The leading `G` is part of the first key + /// name's position, i.e. the payload is `Ga=T,f=32,…;`; we accept the + /// whole `G…;…` slice and split on the `;`. + pub fn parse(payload: &[u8]) -> Option { + // Strip the leading `G` identifier. + let rest = payload.strip_prefix(b"G")?; + let control = match rest.iter().position(|&b| b == b';') { + Some(pos) => &rest[..pos], + None => rest, // no payload section (e.g. `a=d`) + }; + let mut c = Control::default(); + for pair in control.split(|&b| b == b',') { + if pair.is_empty() { + continue; + } + let mut kv = pair.splitn(2, |&b| b == b'='); + let key = kv.next().unwrap_or(b""); + let val = kv.next().unwrap_or(b""); + let num = || parse_u32(val); + match key { + b"a" => { + c.action = match val { + b"q" => Action::Query, + b"t" => Action::Transmit, + b"T" => Action::TransmitAndDisplay, + b"p" => Action::Display, + b"d" => Action::Delete, + _ => Action::Other, + } + } + b"f" => { + c.format = match val { + b"24" => WireFormat::Rgb, + b"100" => WireFormat::Png, + _ => WireFormat::Rgba, // 32 and the default + } + } + b"t" => { + c.medium = match val { + b"f" => Medium::File, + b"s" => Medium::Shared, + b"t" => Medium::TempFile, + _ => Medium::Direct, + } + } + b"o" => c.compressed = val == b"z", + b"i" => c.id = num(), + b"I" => c.number = num(), + b"p" => c.placement = num(), + b"s" => c.width = num(), + b"v" => c.height = num(), + b"c" => c.cols = num(), + b"r" => c.rows = num(), + b"m" => c.more = val == b"1", + b"d" => c.delete = val.first().copied().unwrap_or(0), + // Clamp rather than truncate: `q=256` must not wrap to 0 and + // turn a request for silence into a request for chatter. + b"q" => c.quiet = num().min(u8::MAX as u32) as u8, + b"O" => c.offset = num(), + b"S" => c.size = num(), + _ => {} // unknown key: ignore + } + } + Some(c) + } +} + +fn parse_u32(bytes: &[u8]) -> u32 { + let mut n: u32 = 0; + for &b in bytes { + if !b.is_ascii_digit() { + return 0; + } + n = n.saturating_mul(10).saturating_add((b - b'0') as u32); + } + n +} + +/// The `;`-separated data section of a `_G` command (may be empty). +fn payload_data(command: &[u8]) -> &[u8] { + match command.iter().position(|&b| b == b';') { + Some(pos) => &command[pos + 1..], + None => &[], + } +} + +/// A reassembled image transmission, base64-decoded but *not yet* inflated. +/// +/// The daemon does only the base64 decode (cheap, and it strips the 33% text +/// inflation) and forwards this over the socket; the client calls [`to_rgba8`] +/// to inflate and normalize to RGBA8. Keeping [`data`] compressed on the wire is +/// what makes a *remote* pane viable — inflating daemon-side would push tens of +/// MB per frame back through the SSH tunnel. +/// +/// [`data`]: Image::data +/// [`to_rgba8`]: Image::to_rgba8 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Image { + pub id: u32, + pub number: u32, + pub placement: u32, + /// Source pixel dimensions. + pub width: u32, + pub height: u32, + /// Requested display span in cells (0 = derive from pixels / cell size). + pub cols: u32, + pub rows: u32, + /// The base64-decoded payload: zlib-compressed pixels when [`compressed`], + /// otherwise raw pixels ([`WireFormat::Rgb`]/[`WireFormat::Rgba`]) or an + /// encoded PNG file ([`WireFormat::Png`]). + /// + /// [`compressed`]: Image::compressed + pub data: Vec, + pub format: WireFormat, + /// Whether [`data`](Image::data) is still zlib-compressed (`o=z`). + pub compressed: bool, +} + +impl Image { + /// Inflate and normalize to tightly packed RGBA8 (`width*height*4`), the + /// form the renderer wants. Runs client-side. Returns `None` for a PNG + /// (whose decode needs the `image` crate the GUI owns, not this core) or on + /// a malformed payload; PNG callers should decode [`data`](Image::data) + /// themselves after checking [`format`](Image::format). + /// + /// The inflate is bounded by what `width`/`height` claim the pixels are: + /// deflate expands ~1000:1 in the limit, so an unbounded `decompress_to_vec` + /// here would let a payload well inside [`MAX_TRANSMISSION_BASE64`] balloon + /// into tens of GB and OOM the client — and this runs in the GUI process, so + /// it would take every pane down, not just the one that received the escape. + /// The caller checks the inflated length against `width*height*4` anyway, so + /// capping it up front costs nothing. + pub fn to_rgba8(&self) -> Option> { + if self.format == WireFormat::Png { + return None; + } + let raw = if self.compressed { + // Bound the inflate by the pixels the sender *claims* to be sending + // (`width * height` at this format's bytes-per-pixel). Without a cap, + // a high-ratio deflate payload well inside `MAX_TRANSMISSION_BASE64` + // (which bounds only the *compressed* bytes) inflates to tens of GB + // and OOMs the whole GUI process. A payload that decompresses past + // its own declared dimensions is malformed, so we drop it. + // + // The declared size is itself attacker-chosen, so clamp it too: + // `s=65535,v=65535` alone works out to 17 GB, which would hand the + // bomb right back its allocation. No real frame comes near + // `MAX_IMAGE_BYTES`, which is what the wire can carry anyway. + let limit = self.decoded_len()?.min(MAX_IMAGE_BYTES); + miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(&self.data, limit).ok()? + } else { + self.data.clone() + }; + Some(match self.format { + WireFormat::Rgb => rgb_to_rgba(&raw), + _ => raw, // Rgba + }) + } + + /// The byte length of this image's *decoded* (pre-`rgb_to_rgba`) pixels — + /// `width * height * bytes_per_pixel` for the wire format. `None` on overflow + /// or a zero-sized/PNG image, which have no fixed raw length. + fn decoded_len(&self) -> Option { + let bpp = match self.format { + WireFormat::Rgb => 3usize, + WireFormat::Rgba => 4usize, + WireFormat::Png => return None, + }; + (self.width as usize) + .checked_mul(self.height as usize)? + .checked_mul(bpp) + .filter(|&n| n != 0) + } + + /// Encode for the daemon→client [`crate::daemon::protocol::DaemonMsg::Image`] + /// frame: a fixed 30-byte header carrying the metadata, then the raw `data` + /// bytes appended verbatim. A JSON envelope would base64-inflate the pixel + /// payload ~1.33×; this keeps it byte-for-byte, which matters at video frame + /// rates. See [`decode_frame`](Image::decode_frame). + pub fn encode_frame(&self) -> Vec { + let mut out = Vec::with_capacity(HEADER_LEN + self.data.len()); + out.extend_from_slice(&self.id.to_le_bytes()); + out.extend_from_slice(&self.number.to_le_bytes()); + out.extend_from_slice(&self.placement.to_le_bytes()); + out.extend_from_slice(&self.width.to_le_bytes()); + out.extend_from_slice(&self.height.to_le_bytes()); + out.extend_from_slice(&self.cols.to_le_bytes()); + out.extend_from_slice(&self.rows.to_le_bytes()); + out.push(match self.format { + WireFormat::Rgb => 24, + WireFormat::Rgba => 32, + WireFormat::Png => 100, + }); + out.push(u8::from(self.compressed)); + out.extend_from_slice(&self.data); + out + } + + /// Reconstruct from an [`encode_frame`](Image::encode_frame) payload. + pub fn decode_frame(bytes: &[u8]) -> Option { + if bytes.len() < HEADER_LEN { + return None; + } + let u32_at = |o: usize| u32::from_le_bytes(bytes[o..o + 4].try_into().unwrap()); + let format = match bytes[28] { + 24 => WireFormat::Rgb, + 100 => WireFormat::Png, + _ => WireFormat::Rgba, + }; + Some(Image { + id: u32_at(0), + number: u32_at(4), + placement: u32_at(8), + width: u32_at(12), + height: u32_at(16), + cols: u32_at(20), + rows: u32_at(24), + format, + compressed: bytes[29] != 0, + data: bytes[HEADER_LEN..].to_vec(), + }) + } +} + +/// Byte length of the [`Image::encode_frame`] header: seven `u32` fields, then +/// the format and compression bytes. +const HEADER_LEN: usize = 7 * 4 + 1 + 1; + +impl MediumTransfer { + /// Resolve the referenced bytes into an [`Image`] by reading the file or + /// `mmap`ing the POSIX shm object this transfer names, then unlinking it (a + /// [`Medium::TempFile`] and any shm object are one-shot handoffs the sender + /// expects the terminal to consume and remove; a plain [`Medium::File`] is + /// left in place). Runs on the daemon reader thread of a *local* pane, where + /// the name resolves on this host. Returns `None` if the name is unusable or + /// the object can't be read. + /// + /// The pixels come back *uncompressed* unless the sender set `o=z`: a sender + /// that reaches for shared memory does so precisely to avoid the zlib the + /// inline path forces, so this is the fast path that skips the client-side + /// inflate entirely. + #[cfg(unix)] + pub fn resolve(&self) -> Option { + let data = match self.medium { + Medium::Shared => self.read_shared()?, + Medium::File | Medium::TempFile => self.read_file()?, + // Direct never becomes a MediumTransfer. + Medium::Direct => return None, + }; + Some(Image { + id: self.id, + number: self.number, + placement: self.placement, + width: self.width, + height: self.height, + cols: self.cols, + rows: self.rows, + data, + format: self.format, + compressed: self.compressed, + }) + } + + #[cfg(not(unix))] + pub fn resolve(&self) -> Option { + None + } + + /// The `offset..offset+size` slice of `buf` (or `offset..` when `size == 0`), + /// copied out. `None` if `offset` itself falls outside the object; an `S=` + /// that runs past the end is *truncated* to what's there rather than + /// discarding the frame, which is what kitty does. + #[cfg(unix)] + fn slice_region(&self, buf: &[u8]) -> Option> { + let start = self.offset as usize; + if start > buf.len() { + return None; + } + let end = if self.size == 0 { + buf.len() + } else { + start.saturating_add(self.size as usize).min(buf.len()) + }; + buf.get(start..end).map(<[u8]>::to_vec) + } + + #[cfg(unix)] + fn read_file(&self) -> Option> { + use std::io::Read as _; + use std::os::unix::ffi::OsStringExt; + let path = std::path::PathBuf::from(std::ffi::OsString::from_vec(self.name.clone())); + + // Read through a bounded reader, not `fs::read`. The name is attacker + // -reachable (any program that can write to the pty picks it), and + // `fs::read` on `/dev/zero` never returns — on the daemon *reader* + // thread, which would wedge the pane's whole output path. Refusing + // anything that isn't a regular file also keeps us off fifos and + // devices, where the open itself can block. + let file = std::fs::File::open(&path).ok()?; + let meta = file.metadata().ok()?; + if !meta.is_file() || meta.len() as usize > MAX_IMAGE_BYTES { + return None; + } + let mut bytes = Vec::with_capacity(meta.len() as usize); + file.take(MAX_IMAGE_BYTES as u64 + 1) + .read_to_end(&mut bytes) + .ok()?; + if bytes.len() > MAX_IMAGE_BYTES { + return None; + } + + let out = self.slice_region(&bytes)?; + // A temp file is the sender's one-shot handoff: remove it after reading + // so the browser's per-frame temp files don't pile up. A named `t=f` + // file is the sender's to manage; leave it. + // + // Only unlink inside a known temp directory. `name` is an arbitrary path + // out of an escape sequence — `cat`ing a hostile file is enough to send + // one — so an unqualified `remove_file` here would delete anything the + // user can, `~/.ssh/id_ed25519` included. The spec requires this check + // for exactly that reason. + if self.medium == Medium::TempFile && path_is_in_temp_dir(&path) { + let _ = std::fs::remove_file(&path); + } + Some(out) + } + + /// `shm_open` + `mmap` the object, copy out the requested region, then + /// `shm_unlink` it (the sender allocates a fresh object per frame and + /// expects the terminal to reclaim it — matching kitty/ghostty). + #[cfg(unix)] + fn read_shared(&self) -> Option> { + use std::os::raw::c_void; + // A POSIX shm name is a single `/`-prefixed component — no embedded + // separators, no `..`. Some platforms resolve `shm_open` against the + // filesystem, where a name like `/../../etc/passwd` would escape the shm + // namespace and reach a real path we'd then `shm_unlink`. + if !shm_name_is_wellformed(&self.name) { + return None; + } + // The name must be a C string; kitty shm names look like `/px-…`. + let cname = std::ffi::CString::new(self.name.clone()).ok()?; + // SAFETY: FFI. `shm_open` with O_RDONLY on a name the sender created; + // we only ever read, mmap read-only, and always munmap/close/unlink on + // every exit path below. + unsafe { + let fd = libc::shm_open(cname.as_ptr(), libc::O_RDONLY, 0); + if fd < 0 { + return None; + } + // Size the mapping from the object itself; the sender may not send + // `S=`, and mapping past the end would fault on access. + // Refuse an object bigger than a frame can carry rather than mapping + // and copying it out only for the send to fail downstream. + let mut st: libc::stat = std::mem::zeroed(); + if libc::fstat(fd, &mut st) != 0 + || st.st_size <= 0 + || st.st_size as u64 > MAX_IMAGE_BYTES as u64 + { + libc::close(fd); + libc::shm_unlink(cname.as_ptr()); + return None; + } + let len = st.st_size as usize; + let addr = libc::mmap( + std::ptr::null_mut(), + len, + libc::PROT_READ, + libc::MAP_SHARED, + fd, + 0, + ); + libc::close(fd); + if addr == libc::MAP_FAILED { + libc::shm_unlink(cname.as_ptr()); + return None; + } + let mapped = std::slice::from_raw_parts(addr as *const u8, len); + let out = self.slice_region(mapped); + libc::munmap(addr as *mut c_void, len); + // One-shot: the sender expects us to reclaim the object. + libc::shm_unlink(cname.as_ptr()); + out + } + } +} + +/// Whether `path` sits inside a directory the platform hands out for temp files, +/// which is the only place a `t=t` handoff may be unlinked. +/// +/// Compares *canonicalized* paths so `/tmp/../home/me/.ssh/id_ed25519` — which +/// has the right prefix textually — doesn't pass. On macOS `TMPDIR` is a +/// per-user path under `/var/folders/…` that canonicalizes through the +/// `/private` symlink, so `std::env::temp_dir()` is canonicalized too rather +/// than compared raw. +#[cfg(unix)] +fn path_is_in_temp_dir(path: &std::path::Path) -> bool { + let Ok(real) = path.canonicalize() else { + return false; + }; + // `/dev/shm` is where a `t=t` sender that wanted shm-like semantics without + // `shm_open` puts its handoff; kitty accepts it alongside the temp dirs. + let candidates = [ + std::env::temp_dir(), + std::path::PathBuf::from("/tmp"), + std::path::PathBuf::from("/dev/shm"), + ]; + candidates.iter().any(|dir| { + dir.canonicalize() + .is_ok_and(|d| real.starts_with(&d) && real != d) + }) +} + +/// Whether `name` is a well-formed POSIX shm object name: a leading `/` followed +/// by one non-empty component with no further separators and no `.`/`..`. +#[cfg(unix)] +fn shm_name_is_wellformed(name: &[u8]) -> bool { + // POSIX caps the name at NAME_MAX; 255 is the floor every platform we build + // for meets, and no real sender comes close. + if name.len() < 2 || name.len() > 255 || name[0] != b'/' { + return false; + } + let rest = &name[1..]; + !rest.contains(&b'/') && !rest.contains(&0) && rest != b"." && rest != b".." +} + +/// A delete request distilled from an `a=d` command, in the compact form the +/// daemon forwards to the client's image store. The full kitty delete grammar is +/// rich (by id, by placement, by cell, by z-index, …); the client only needs the +/// target selector plus the id/placement it may scope to, which is all any sender +/// tty7 targets — most often `d=A` (delete everything) — actually uses. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ImageDelete { + /// The `d=` selector byte (e.g. `A`/`a` = all, `i` = by id, `p` = by + /// placement). Uppercase variants also free the image data in kitty; the + /// client frees unconditionally, so case only affects which images match. + pub target: u8, + pub id: u32, + pub placement: u32, +} + +impl ImageDelete { + pub fn from_control(c: &Control) -> Self { + Self { + // A bare `a=d` with no `d=` means "delete all visible placements", + // which kitty spells `a` — normalize the unset case to it. + target: if c.delete == 0 { b'a' } else { c.delete }, + id: c.id, + placement: c.placement, + } + } + + /// A fixed 9-byte frame: the selector byte then id and placement (LE). + pub fn encode(&self) -> Vec { + let mut out = Vec::with_capacity(9); + out.push(self.target); + out.extend_from_slice(&self.id.to_le_bytes()); + out.extend_from_slice(&self.placement.to_le_bytes()); + out + } + + pub fn decode(bytes: &[u8]) -> Option { + if bytes.len() < 9 { + return None; + } + Some(Self { + target: bytes[0], + id: u32::from_le_bytes(bytes[1..5].try_into().unwrap()), + placement: u32::from_le_bytes(bytes[5..9].try_into().unwrap()), + }) + } +} + +/// What a fed command turned into. Query/Delete carry no pixels; the caller +/// writes the query reply back to the PTY and applies deletes to its store. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Event { + /// An `a=q` probe. `reply` is the bytes to write to the PTY; `honored` says + /// whether we accepted the requested medium (for logging/metrics only). + Query { reply: Vec, honored: bool }, + /// A complete image transmission (`a=T`/`a=t`). + Image(Image), + /// A transmission whose pixels live in a file or POSIX shm object rather + /// than inline in the escape. The parser can't read the filesystem (it must + /// stay pure and host-agnostic), so it hands the reference to the daemon + /// pane — which is co-located with the sender on a *local* pane — to `mmap` + /// / read and unlink. This is the zero-copy, zero-inflate path a sender like + /// `terminal-browser` prefers; see [`MediumTransfer`]. + ImageFromMedium(MediumTransfer), + /// An `a=d` delete request. + Delete(Control), +} + +/// A file/shm transmission the daemon must resolve into pixels. The `name` is +/// the (base64-decoded) payload the sender put after the `;`: a filesystem path +/// for [`Medium::File`]/[`Medium::TempFile`], or a POSIX shm object name for +/// [`Medium::Shared`]. `offset`/`size` bound the region to read (`size == 0` +/// means "to end of object"). All the metadata needed to build the final +/// [`Image`] rides along so the daemon does no parsing of its own. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MediumTransfer { + pub medium: Medium, + pub name: Vec, + pub offset: u32, + pub size: u32, + pub id: u32, + pub number: u32, + pub placement: u32, + pub width: u32, + pub height: u32, + pub cols: u32, + pub rows: u32, + pub format: WireFormat, + /// Whether the referenced bytes are zlib-compressed (`o=z`). A sender that + /// picks shm/file for speed sends raw pixels, but honor the flag regardless. + pub compressed: bool, +} + +/// State for reassembling a chunked (`m=1`) direct transmission. Kitty +/// serializes chunked transmissions — only one is in flight at a time — so a +/// single pending accumulator suffices. +#[derive(Default)] +struct Pending { + control: Control, + /// Accumulated *base64* text across chunks (decoded once, at the end). + base64: Vec, +} + +/// Reassembles and decodes kitty graphics commands emitted by [`ApcTokenizer`]. +/// +/// Feed it each `_G` command payload; it returns zero or more [`Event`]s. It +/// owns the chunk-reassembly buffer and the base64/zlib decode, so the daemon +/// pane just forwards the resulting [`Image`] out-of-band and writes any +/// [`Event::Query`] reply to the PTY. +#[derive(Default)] +pub struct GraphicsParser { + pending: Option, + /// Whether the sender shares this host's filesystem (a local pane). Only + /// then can we honor file/shm transfer, whose names are host-local; a pane + /// running over SSH must stay on inline `t=d` so the pixels ride the tunnel. + local: bool, +} + +impl GraphicsParser { + pub fn new() -> Self { + Self::default() + } + + /// A parser that may honor file/shm transfer because the sender is on this + /// host (see [`GraphicsParser::local`]). + pub fn new_local(local: bool) -> Self { + Self { + local, + ..Self::default() + } + } + + /// Whether a `t=f`/`t=t`/`t=s` transfer can actually be resolved here: the + /// sender has to share this host's filesystem (a local pane), and + /// [`MediumTransfer::resolve`] has to have a real implementation on this + /// platform (it is unix-only). Both `query_reply` and `finalize` route + /// through this so what we advertise and what we accept can't drift apart. + fn honors_indirect_media(&self) -> bool { + self.local && cfg!(unix) + } + + /// Feed one complete `_G` command payload (the bytes [`ApcTokenizer`] + /// delivers). Returns an [`Event`] when a command completes (a query, a + /// finished image, or a delete); returns `None` for an intermediate chunk of + /// a still-incomplete transmission. + pub fn feed(&mut self, command: &[u8]) -> Option { + let control = Control::parse(command)?; + let data = payload_data(command); + + // A continuation chunk of a chunked transmission carries only `m=…` — no + // `a=` — so its action defaults to `Transmit`. Route anything while a + // transmission is pending straight to the chunk accumulator, which keeps + // the *first* chunk's real control keys. + if self.pending.is_some() { + return self.accept_chunk(control, data); + } + + match control.action { + Action::Query => self.query_reply(&control), + Action::Delete => Some(Event::Delete(control)), + Action::TransmitAndDisplay | Action::Transmit => self.accept_chunk(control, data), + // `a=p` places an image transmitted by an *earlier* command, which + // needs the stored-image table this parser deliberately doesn't + // keep. Routing it through `accept_chunk` would emit an `Image` with + // an empty payload — a frame the client can only throw away — so + // drop it here instead. Senders that split transmit from placement + // get nothing; `a=T` (the shape every sender tty7 targets uses) + // is unaffected. + Action::Display => None, + // An action we don't handle, with nothing in flight: drop it. + Action::Other => None, + } + } + + /// Append a (possibly first) chunk; finalize when `m` is not set. + fn accept_chunk(&mut self, control: Control, data: &[u8]) -> Option { + // The first chunk carries the real control keys; later chunks repeat + // only `m=`. Preserve the first chunk's control across the transmission. + let is_first = self.pending.is_none(); + if is_first { + self.pending = Some(Pending { + control, + base64: Vec::new(), + }); + } + let pending = self.pending.as_mut()?; + pending.base64.extend_from_slice(data); + if pending.base64.len() > MAX_TRANSMISSION_BASE64 { + log::debug!( + "kitty graphics: abandoning a transmission past {MAX_TRANSMISSION_BASE64} base64 bytes" + ); + self.pending = None; // abandon an oversized transmission + return None; + } + if control.more { + return None; // more chunks to come + } + // Complete: decode and emit. + let Pending { control, base64 } = self.pending.take()?; + self.finalize(control, &base64) + } + + fn finalize(&self, control: Control, base64: &[u8]) -> Option { + // Base64-decode the payload once. For direct transmission this *is* the + // (still-compressed) pixels; for file/shm it's the path/object name. + let data = BASE64.decode(base64).ok()?; + + // File/shm transfer: the payload names bytes on this host. Hand the + // reference to the daemon to resolve — but only on a local pane, where + // the name is meaningful and reading it can't leak across an SSH tunnel. + // A `query_reply` earlier already refused these mediums on a remote + // pane, so a well-behaved sender never reaches here; the guard is + // belt-and-suspenders. + if control.medium != Medium::Direct { + if !self.honors_indirect_media() { + return None; + } + return Some(Event::ImageFromMedium(MediumTransfer { + medium: control.medium, + name: data, + offset: control.offset, + size: control.size, + id: control.id, + number: control.number, + placement: control.placement, + width: control.width, + height: control.height, + cols: control.cols, + rows: control.rows, + format: control.format, + compressed: control.compressed, + })); + } + + // Direct: inflation is deferred to the client (`to_rgba8`) so the + // payload rides the socket — and any SSH tunnel — compressed. + Some(Event::Image(Image { + id: control.id, + number: control.number, + placement: control.placement, + width: control.width, + height: control.height, + cols: control.cols, + rows: control.rows, + data, + format: control.format, + compressed: control.compressed, + })) + } + + /// Build the `a=q` reply. On a local unix pane we honor direct, file, and + /// shm transfer, so any of those probes gets `OK`; otherwise only direct is + /// honored and a `t=f`/`t=s`/`t=t` probe gets an error — which makes a + /// sender like `terminal-browser` fall back to inline `t=d` on its own. + /// + /// The answer has to track what [`MediumTransfer::resolve`] can actually do, + /// including on the platform axis: it is unix-only, so replying `OK` to a + /// file/shm probe on Windows would talk a sender into a medium whose every + /// frame we then silently discard, leaving the pane blank. + fn query_reply(&self, control: &Control) -> Option { + let honored = control.medium == Medium::Direct || self.honors_indirect_media(); + // `q=` asks us to stay quiet: 1 suppresses success replies, 2 suppresses + // failures too. This matters because the reply is written back to the + // *PTY* — it arrives as if the user had typed it. A sender that asked + // for silence and gets `\x1b_Gi=1;OK\x1b\` anyway has those bytes land + // in its stdin, or, if it already exited, on the shell's input line. + if control.quiet >= 2 || (control.quiet == 1 && honored) { + return None; + } + let status: &[u8] = if honored { b"OK" } else { b"ENOTSUPPORTED" }; + // Echo back the id (preferred) or number the sender used, exactly as + // kitty does, so the sender can correlate the reply. + let mut reply = Vec::with_capacity(32); + reply.extend_from_slice(b"\x1b_G"); + if control.id != 0 { + reply.extend_from_slice(format!("i={}", control.id).as_bytes()); + } else if control.number != 0 { + reply.extend_from_slice(format!("I={}", control.number).as_bytes()); + } else { + reply.extend_from_slice(b"i=0"); + } + reply.push(b';'); + reply.extend_from_slice(status); + reply.extend_from_slice(b"\x1b\\"); + Some(Event::Query { reply, honored }) + } +} + +/// Expand tightly packed RGB into RGBA with an opaque alpha channel. +fn rgb_to_rgba(rgb: &[u8]) -> Vec { + let mut out = Vec::with_capacity(rgb.len() / 3 * 4); + for px in rgb.chunks_exact(3) { + out.extend_from_slice(px); + out.push(0xff); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn collect(chunks: &[&[u8]]) -> Vec> { + let mut tok = ApcTokenizer::new(); + let mut out = Vec::new(); + for c in chunks { + tok.feed(c, |_| {}, |cmd| out.push(cmd.to_vec())); + } + out + } + + /// Feed `chunks` and return the concatenated passthrough stream (input with + /// every `_G` sequence stripped) alongside the extracted commands. + fn split(chunks: &[&[u8]]) -> (Vec, Vec>) { + let mut tok = ApcTokenizer::new(); + let mut pass = Vec::new(); + let mut cmds = Vec::new(); + for c in chunks { + tok.feed( + c, + |b| pass.extend_from_slice(b), + |cmd| cmds.push(cmd.to_vec()), + ); + } + (pass, cmds) + } + + #[test] + fn extracts_a_graphics_command_between_esc_underscore_and_st() { + assert_eq!( + collect(&[b"\x1b_Ga=T,f=32;AAAA\x1b\\"]), + vec![b"Ga=T,f=32;AAAA".to_vec()] + ); + } + + #[test] + fn non_graphics_apc_is_ignored() { + // An APC that doesn't start with `G` (e.g. some other program's APC) is + // discarded, and a graphics command right after is still caught. + assert_eq!( + collect(&[b"\x1b_Xhello\x1b\\\x1b_Ga=q;AAAA\x1b\\"]), + vec![b"Ga=q;AAAA".to_vec()] + ); + } + + #[test] + fn command_split_across_reads_is_reassembled() { + assert_eq!( + collect(&[b"\x1b_Ga=T,", b"f=32;AA", b"AA\x1b", b"\\"]), + vec![b"Ga=T,f=32;AAAA".to_vec()] + ); + } + + #[test] + fn byte_at_a_time_delivery_crosses_every_state() { + let stream = b"plain\x1b_Ga=q;AAAA\x1b\\more"; + let chunks: Vec<&[u8]> = stream.chunks(1).collect(); + assert_eq!(collect(&chunks), vec![b"Ga=q;AAAA".to_vec()]); + } + + #[test] + fn passthrough_is_input_with_graphics_stripped() { + let (pass, cmds) = split(&[b"before\x1b_Ga=T;AAAA\x1b\\after"]); + assert_eq!(pass, b"beforeafter".to_vec()); + assert_eq!(cmds, vec![b"Ga=T;AAAA".to_vec()]); + } + + #[test] + fn non_graphics_apc_passes_through_verbatim() { + // A foreign APC (e.g. tmux) must reach the client's VT parser unchanged, + // while a graphics command in the same stream is still stripped. + let (pass, cmds) = split(&[b"\x1b_Xhello\x1b\\\x1b_Ga=q;AAAA\x1b\\end"]); + assert_eq!(pass, b"\x1b_Xhello\x1b\\end".to_vec()); + assert_eq!(cmds, vec![b"Ga=q;AAAA".to_vec()]); + } + + #[test] + fn lone_esc_and_other_escapes_are_preserved_in_passthrough() { + // A CSI sequence and a bare ESC must survive; only `_G` is removed. + let (pass, cmds) = split(&[b"a\x1b[31mred\x1b_Ga=d\x1b\\z"]); + assert_eq!(pass, b"a\x1b[31mredz".to_vec()); + assert_eq!(cmds, vec![b"Ga=d".to_vec()]); + } + + #[test] + fn passthrough_survives_being_split_mid_sequence() { + // Feed the same stream one byte at a time; the passthrough must still be + // exactly the input minus the graphics command. + let stream = b"x\x1b_Ga=T;QUJD\x1b\\y\x1b_Zother\x1b\\z"; + let chunks: Vec<&[u8]> = stream.chunks(1).collect(); + let (pass, cmds) = split(&chunks); + assert_eq!(pass, b"xy\x1b_Zother\x1b\\z".to_vec()); + assert_eq!(cmds, vec![b"Ga=T;QUJD".to_vec()]); + } + + #[test] + fn sniffer_ties_tokenizer_and_parser_together() { + let mut s = GraphicsSniffer::new(); + let mut pass = Vec::new(); + let events = s.feed(b"hi\x1b_Gi=1,a=q,t=d;AAAA\x1b\\bye", |b| { + pass.extend_from_slice(b) + }); + assert_eq!(pass, b"hibye".to_vec()); + assert_eq!(events.len(), 1); + match &events[0] { + Event::Query { reply, honored } => { + assert!(honored); + assert_eq!(reply, b"\x1b_Gi=1;OK\x1b\\"); + } + _ => panic!("expected query"), + } + } + + #[test] + fn sniff_takes_zero_copy_fast_path_without_graphics() { + let mut s = GraphicsSniffer::new(); + // A chunk full of CSI escapes but no APC borrows straight through. + match s.sniff(b"\x1b[31mred\x1b[0m plain") { + Sniffed::Plain(b) => assert_eq!(b, b"\x1b[31mred\x1b[0m plain"), + Sniffed::Segments(_) => panic!("expected the borrowed fast path"), + } + } + + #[test] + fn sniff_preserves_stream_order_of_text_and_images() { + let pixel = [0xffu8, 0x00, 0x00, 0xff]; + let b64 = BASE64.encode(pixel); + let stream = format!("A\x1b_Ga=T,f=32,t=d,s=1,v=1,i=1;{b64}\x1b\\B"); + let mut s = GraphicsSniffer::new(); + let segs = match s.sniff(stream.as_bytes()) { + Sniffed::Segments(segs) => segs, + Sniffed::Plain(_) => panic!("graphics present"), + }; + // Text "A", then the image, then text "B" — in that order. + assert_eq!(segs.len(), 3); + assert_eq!(segs[0], Segment::Output(b"A".to_vec())); + assert!(matches!(&segs[1], Segment::Image(img) if img.id == 1)); + assert_eq!(segs[2], Segment::Output(b"B".to_vec())); + } + + #[test] + fn sniff_emits_query_and_delete_segments() { + let mut s = GraphicsSniffer::new(); + let segs = match s.sniff(b"\x1b_Gi=7,a=q,t=d;AAAA\x1b\\x\x1b_Ga=d,d=A\x1b\\") { + Sniffed::Segments(segs) => segs, + Sniffed::Plain(_) => panic!("graphics present"), + }; + assert_eq!(segs[0], Segment::Query(b"\x1b_Gi=7;OK\x1b\\".to_vec())); + assert_eq!(segs[1], Segment::Output(b"x".to_vec())); + assert!(matches!(&segs[2], Segment::Delete(d) if d.target == b'A')); + } + + #[test] + fn sniff_coalesces_adjacent_passthrough_runs() { + // A foreign APC between two text runs is passthrough, so the whole thing + // collapses to a single output segment (no graphics segment at all). + let mut s = GraphicsSniffer::new(); + match s.sniff(b"a\x1b_Zother\x1b\\b") { + Sniffed::Segments(segs) => { + assert_eq!(segs, vec![Segment::Output(b"a\x1b_Zother\x1b\\b".to_vec())]); + } + Sniffed::Plain(_) => panic!("an APC opener forces the slow path"), + } + } + + #[test] + fn resyncs_on_new_apc_after_an_unterminated_one() { + assert_eq!( + collect(&[b"\x1b_Ga=T;dropped\x1b_Ga=q;AAAA\x1b\\"]), + vec![b"Ga=q;AAAA".to_vec()] + ); + } + + #[test] + fn oversized_chunk_is_abandoned_and_stream_recovers() { + let mut big = b"\x1b_G".to_vec(); + big.extend(std::iter::repeat_n(b'x', MAX_APC_PAYLOAD + 1)); + big.extend_from_slice(b"\x1b\\\x1b_Ga=q;AAAA\x1b\\"); + assert_eq!(collect(&[&big]), vec![b"Ga=q;AAAA".to_vec()]); + } + + #[test] + fn control_parses_the_keys_terminal_browser_sends() { + let c = + Control::parse(b"Ga=T,f=32,o=z,s=1920,v=1080,t=d,i=42,p=1,C=1,q=2,m=1;xxxx").unwrap(); + assert_eq!(c.action, Action::TransmitAndDisplay); + assert_eq!(c.format, WireFormat::Rgba); + assert!(c.compressed); + assert_eq!(c.width, 1920); + assert_eq!(c.height, 1080); + assert_eq!(c.medium, Medium::Direct); + assert_eq!(c.id, 42); + assert!(c.more); + assert_eq!(c.quiet, 2); + } + + #[test] + fn query_reply_refuses_shm_file_on_remote_pane() { + // The default parser is non-local (the SSH-safe posture): direct is + // honored, file/shm are refused so the sender falls back to inline. + let mut p = GraphicsParser::new(); + // The exact probe terminal-browser's graphics.ts sends. + let ev = p.feed(b"Gi=4207,a=q,t=d,f=24,s=1,v=1;AAAA").unwrap(); + match ev { + Event::Query { reply, honored } => { + assert!(honored); + assert_eq!(reply, b"\x1b_Gi=4207;OK\x1b\\".to_vec()); + } + _ => panic!("expected query"), + } + // The shm/file medium probes must be refused so the sender falls back. + let ev = p.feed(b"Gi=299,a=q,t=s,f=32,s=1,v=1;L3B4LXE").unwrap(); + match ev { + Event::Query { reply, honored } => { + assert!(!honored); + assert_eq!(reply, b"\x1b_Gi=299;ENOTSUPPORTED\x1b\\".to_vec()); + } + _ => panic!("expected query"), + } + } + + #[test] + #[cfg(unix)] + fn query_reply_ok_for_shm_file_on_local_pane() { + // A local pane shares the sender's filesystem, so file/shm are honored: + // this is what unlocks terminal-browser's zero-inflate fast path. + let mut p = GraphicsParser::new_local(true); + for probe in [ + &b"Gi=299,a=q,t=s,f=32,s=1,v=1;L3B4LXE"[..], + &b"Gi=300,a=q,t=f,f=32,s=1,v=1;L3RtcC94"[..], + &b"Gi=301,a=q,t=t,f=32,s=1,v=1;L3RtcC94"[..], + ] { + match p.feed(probe).unwrap() { + Event::Query { honored, reply } => { + assert!(honored, "local pane should honor {probe:?}"); + assert!(reply.ends_with(b";OK\x1b\\"), "reply was {reply:?}"); + } + _ => panic!("expected query"), + } + } + } + + #[test] + #[cfg(unix)] + fn shared_transmission_surfaces_medium_transfer_on_local() { + // `terminal-browser`'s shm transmit template: raw f=32, no o=z, the + // shm object name base64'd after the `;`. + let name = b"/px-abc123"; + let b64 = BASE64.encode(name); + let cmd = format!("Ga=T,f=32,t=s,s=64,v=1,i=7,S=256;{b64}"); + let mut p = GraphicsParser::new_local(true); + match p.feed(cmd.as_bytes()).unwrap() { + Event::ImageFromMedium(t) => { + assert_eq!(t.medium, Medium::Shared); + assert_eq!(t.name, name); + assert_eq!(t.id, 7); + assert_eq!((t.width, t.height), (64, 1)); + assert_eq!(t.size, 256); + assert!(!t.compressed); + } + _ => panic!("expected medium transfer"), + } + } + + #[test] + fn shared_transmission_dropped_on_remote() { + // A non-local parser must never surface a file/shm transfer even if a + // misbehaving sender ignored the refusal and sent one anyway. + let b64 = BASE64.encode(b"/px-abc123"); + let cmd = format!("Ga=T,f=32,t=s,s=64,v=1,i=7;{b64}"); + let mut p = GraphicsParser::new(); + assert_eq!(p.feed(cmd.as_bytes()), None); + } + + #[test] + #[cfg(unix)] + fn file_transfer_resolves_and_temp_file_is_removed() { + // A temp-file transfer: write raw RGBA to a temp path, resolve it, and + // confirm the file is deleted afterward (the one-shot handoff contract). + let rgba = [0x11u8, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + let dir = std::env::temp_dir(); + let path = dir.join(format!("tty7-kitty-test-{}.rgba", std::process::id())); + std::fs::write(&path, rgba).unwrap(); + let t = MediumTransfer { + medium: Medium::TempFile, + name: path.clone().into_os_string().into_encoded_bytes(), + offset: 0, + size: 0, + id: 1, + number: 0, + placement: 0, + width: 2, + height: 1, + cols: 0, + rows: 0, + format: WireFormat::Rgba, + compressed: false, + }; + let img = t.resolve().expect("resolve temp file"); + assert_eq!(img.data, rgba); + assert_eq!(img.to_rgba8().unwrap(), rgba); + assert!(!path.exists(), "temp file should be removed after read"); + } + + /// Build a `t=t` transfer naming `path`, the shape a hostile escape uses. + #[cfg(unix)] + fn temp_file_transfer(path: &std::path::Path) -> MediumTransfer { + MediumTransfer { + medium: Medium::TempFile, + name: path.to_path_buf().into_os_string().into_encoded_bytes(), + offset: 0, + size: 0, + id: 1, + number: 0, + placement: 0, + width: 2, + height: 1, + cols: 0, + rows: 0, + format: WireFormat::Rgba, + compressed: false, + } + } + + #[test] + #[cfg(unix)] + fn a_temp_file_transfer_outside_the_temp_dir_is_read_but_not_deleted() { + // `t=t` names an arbitrary path out of an escape sequence — `cat`ing a + // hostile file is enough to send one. Unlinking whatever it points at + // would delete e.g. `~/.ssh/id_ed25519`. Read it, leave it. + // + // `/etc/hosts` stands in for the victim: a real, readable file that is + // never under a temp dir on any host we build for. Deliberately *not* + // something derived from `CARGO_MANIFEST_DIR` — a checkout can itself + // live under `/tmp`, which would make the test assert the opposite of + // what it means to. + let victim = std::path::Path::new("/etc/hosts"); + let img = temp_file_transfer(victim).resolve().expect("still reads"); + assert!(!img.data.is_empty()); + assert!( + victim.exists(), + "a path outside the temp dir must survive a t=t handoff" + ); + } + + #[test] + #[cfg(unix)] + fn the_temp_dir_check_resolves_symlinks_and_dotdot_before_comparing() { + // The prefix has to be checked on the *canonicalized* path: `/tmp/../etc` + // starts with `/tmp` textually but lands nowhere near it. macOS also + // routes both `/tmp` and `TMPDIR` through `/private`, so a raw string + // compare would reject the legitimate case too. + let dir = tempfile::tempdir().unwrap(); + let inside = dir.path().join("frame.rgba"); + std::fs::write(&inside, [0u8; 4]).unwrap(); + assert!(path_is_in_temp_dir(&inside)); + + assert!(!path_is_in_temp_dir(std::path::Path::new( + "/tmp/../etc/hosts" + ))); + assert!(!path_is_in_temp_dir(std::path::Path::new("/etc/hosts"))); + // A path that doesn't resolve at all can't be vouched for. + assert!(!path_is_in_temp_dir(std::path::Path::new( + "/nonexistent-tty7-test-path" + ))); + // The temp dir itself is not a file we'd ever unlink. + assert!(!path_is_in_temp_dir(&std::env::temp_dir())); + } + + #[test] + #[cfg(unix)] + fn a_transfer_naming_a_character_device_is_refused() { + // `fs::read` on `/dev/zero` never returns, and this runs on the daemon's + // reader thread — the pane's whole output path would wedge. + let t = temp_file_transfer(std::path::Path::new("/dev/zero")); + assert_eq!(t.resolve(), None); + } + + #[test] + #[cfg(unix)] + fn a_malformed_shm_name_is_refused_before_shm_open() { + for name in [ + &b"/../../etc/passwd"[..], + &b"/sub/dir"[..], + &b"no-leading-slash"[..], + &b"/"[..], + &b"/.."[..], + ] { + assert!( + !shm_name_is_wellformed(name), + "{:?} should be refused", + String::from_utf8_lossy(name) + ); + } + assert!(shm_name_is_wellformed(b"/px-abc123")); + } + + #[test] + fn a_bomb_that_declares_huge_dimensions_is_still_bounded() { + // The declared size is the sender's to choose, so bounding the inflate + // by it alone isn't enough: `s=65535,v=65535` works out to a 17 GB + // budget, which hands a bomb back exactly the allocation the bound was + // meant to deny. The absolute `MAX_IMAGE_BYTES` clamp is what closes it. + // (`to_rgba8_bounds_the_inflate_by_declared_dimensions` covers the + // ordinary case, where the declared size is the tighter of the two.) + let img = Image { + id: 1, + number: 0, + placement: 0, + width: 65535, + height: 65535, + cols: 0, + rows: 0, + // Has to inflate past `MAX_IMAGE_BYTES` to exercise the clamp at + // all — anything smaller decodes on its merits and proves nothing. + // Zeros deflate to a few KB, so the payload on the wire stays tiny, + // which is the whole point of the attack. + data: miniz_oxide::deflate::compress_to_vec_zlib(&vec![0u8; MAX_IMAGE_BYTES + 1], 6), + format: WireFormat::Rgba, + compressed: true, + }; + assert!( + img.decoded_len().unwrap() > MAX_IMAGE_BYTES, + "the declared budget must be the looser bound for this to test anything" + ); + assert!( + img.data.len() < 1 << 20, + "the compressed payload must stay small: {}", + img.data.len() + ); + assert_eq!(img.to_rgba8(), None, "the clamp has to reject it"); + } + + #[test] + fn a_transmission_cap_leaves_room_for_the_wire_frame() { + // `MAX_TRANSMISSION_BASE64` has to decode to something that still fits + // in a `MAX_FRAME` wire frame with the header on it — otherwise + // `write_frame` fails, and the daemon's writer treats that as fatal and + // drops the client's whole connection over one image. + let decoded_max = MAX_TRANSMISSION_BASE64 / 4 * 3; + assert!( + decoded_max + HEADER_LEN <= crate::daemon::protocol::MAX_FRAME, + "{decoded_max} + {HEADER_LEN} must fit in {}", + crate::daemon::protocol::MAX_FRAME + ); + assert!(MAX_IMAGE_BYTES + HEADER_LEN <= crate::daemon::protocol::MAX_FRAME); + } + + #[test] + fn a_quiet_sender_gets_no_reply_written_back_to_its_pty() { + // The reply is written to the *PTY* — it arrives as if typed. `q=1` + // suppresses success, `q=2` suppresses failures too. + let mut p = GraphicsParser::new_local(true); + assert_eq!(p.feed(b"Gi=1,a=q,t=d,q=1;AAAA"), None); + assert_eq!(p.feed(b"Gi=2,a=q,t=d,q=2;AAAA"), None); + // A refusal still reaches a `q=1` sender, which needs to hear it to fall + // back, and an unquiet probe is answered as before. + let mut remote = GraphicsParser::new(); + assert!(matches!( + remote.feed(b"Gi=3,a=q,t=s,q=1;AAAA"), + Some(Event::Query { honored: false, .. }) + )); + assert!(matches!( + p.feed(b"Gi=4,a=q,t=d;AAAA"), + Some(Event::Query { honored: true, .. }) + )); + } + + #[test] + fn a_quiet_key_past_a_byte_does_not_wrap_to_chatty() { + assert_eq!(Control::parse(b"Ga=q,q=256;").unwrap().quiet, 255); + } + + #[test] + fn graphics_after_an_unterminated_foreign_apc_are_still_lifted() { + // A foreign APC that never sends its ST used to swallow every `ESC _G` + // after it: the graphics were forwarded as APC text, which the client's + // vte then discards, so the image vanished with no trace. + let input = b"\x1b_somevendor\x1b_Ga=q;AAAA\x1b\\"; + assert_eq!(collect(&[&input[..]]), vec![b"Ga=q;AAAA".to_vec()]); + } + + #[test] + fn an_escape_interrupting_a_graphics_command_still_reaches_the_client() { + // The abandoned command's bytes stay stripped, but the escape that + // interrupted it belongs to the terminal — swallowing it turned a + // following `\x1b[31m` into literal `31m` on screen. + let mut t = ApcTokenizer::new(); + let mut out = Vec::new(); + t.feed(b"\x1b_Gxx\x1b[31mred", |b| out.extend_from_slice(b), |_| {}); + assert_eq!(out, b"\x1b[31mred"); + } + + #[test] + fn a_place_only_command_emits_nothing_rather_than_an_empty_image() { + // `a=p` places an image transmitted earlier, which needs a stored-image + // table this parser doesn't keep. It used to fall through the chunk + // accumulator and emit an `Image` with no payload. + let mut p = GraphicsParser::new_local(true); + assert_eq!(p.feed(b"Ga=p,i=42,p=1;"), None); + } + + #[test] + #[cfg(unix)] + fn an_oversized_size_key_truncates_instead_of_dropping_the_frame() { + // kitty truncates an `S=` that runs past the object; discarding the + // whole frame loses an image a real sender would have seen rendered. + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("frame.rgba"); + std::fs::write(&path, [1u8, 2, 3, 4]).unwrap(); + let mut t = temp_file_transfer(&path); + t.medium = Medium::File; // leave the file in place + t.size = 4096; + assert_eq!(t.resolve().expect("truncated, not dropped").data.len(), 4); + } + + #[test] + fn single_shot_rgba_transmission_decodes() { + // One 1x1 opaque-red RGBA pixel, uncompressed, direct. + let pixel = [0xffu8, 0x00, 0x00, 0xff]; + let b64 = BASE64.encode(pixel); + let cmd = format!("Ga=T,f=32,t=d,s=1,v=1,i=7;{b64}"); + let mut p = GraphicsParser::new(); + let ev = p.feed(cmd.as_bytes()).unwrap(); + match ev { + Event::Image(img) => { + assert_eq!(img.id, 7); + assert_eq!((img.width, img.height), (1, 1)); + assert!(!img.compressed); + assert_eq!(img.to_rgba8().unwrap(), pixel); + assert_eq!(img.format, WireFormat::Rgba); + } + _ => panic!("expected image"), + } + } + + #[test] + fn chunked_compressed_transmission_reassembles_and_inflates() { + // 2x1 RGBA (red, green), zlib-compressed, split into two direct chunks + // exactly the way terminal-browser frames a `t=d,o=z` transmission. + let rgba = [0xffu8, 0, 0, 0xff, 0x00, 0xff, 0x00, 0xff]; + let z = miniz_oxide::deflate::compress_to_vec_zlib(&rgba, 1); + let b64 = BASE64.encode(&z); + let mid = b64.len() / 2; + let first = format!("Ga=T,f=32,o=z,t=d,s=2,v=1,i=9,m=1;{}", &b64[..mid]); + let second = format!("Gm=0;{}", &b64[mid..]); + + let mut p = GraphicsParser::new(); + assert_eq!(p.feed(first.as_bytes()), None); // more chunks pending + let ev = p.feed(second.as_bytes()).unwrap(); + match ev { + Event::Image(img) => { + assert_eq!(img.id, 9); + assert_eq!((img.width, img.height), (2, 1)); + assert!(img.compressed); + // Wire payload stays compressed; the client inflates. + assert_eq!(img.data, z); + assert_eq!(img.to_rgba8().unwrap(), rgba); + } + _ => panic!("expected image"), + } + } + + #[test] + fn rgb_transmission_is_expanded_to_opaque_rgba() { + let rgb = [0x10u8, 0x20, 0x30]; // one pixel + let b64 = BASE64.encode(rgb); + let cmd = format!("Ga=T,f=24,t=d,s=1,v=1;{b64}"); + let mut p = GraphicsParser::new(); + match p.feed(cmd.as_bytes()).unwrap() { + Event::Image(img) => { + assert_eq!(img.to_rgba8().unwrap(), [0x10, 0x20, 0x30, 0xff]) + } + _ => panic!("expected image"), + } + } + + #[test] + fn delete_is_surfaced() { + let mut p = GraphicsParser::new(); + match p.feed(b"Ga=d,d=A").unwrap() { + Event::Delete(c) => { + assert_eq!(c.action, Action::Delete); + assert_eq!(c.delete, b'A'); + } + _ => panic!("expected delete"), + } + } + + #[test] + fn image_delete_normalizes_and_roundtrips() { + // A bare `a=d` (no selector) means "all visible placements" (`a`). + let bare = ImageDelete::from_control(&Control::parse(b"Ga=d").unwrap()); + assert_eq!(bare.target, b'a'); + // A scoped delete keeps its selector and id. + let scoped = ImageDelete::from_control(&Control::parse(b"Ga=d,d=i,i=5").unwrap()); + assert_eq!((scoped.target, scoped.id), (b'i', 5)); + assert_eq!(ImageDelete::decode(&scoped.encode()), Some(scoped)); + assert_eq!(ImageDelete::decode(&[b'a', 0, 0]), None); // too short + } + + #[test] + fn image_frame_roundtrips_without_touching_the_payload() { + let img = Image { + id: 42, + number: 3, + placement: 1, + width: 1920, + height: 1080, + cols: 80, + rows: 24, + data: vec![1, 2, 3, 4, 5, 6, 7], + format: WireFormat::Rgba, + compressed: true, + }; + let frame = img.encode_frame(); + // Header + payload, payload byte-identical (no base64 inflation). + assert_eq!(&frame[HEADER_LEN..], &img.data[..]); + assert_eq!(Image::decode_frame(&frame), Some(img)); + // A truncated frame is rejected, not panicked on. + assert_eq!(Image::decode_frame(&frame[..HEADER_LEN - 1]), None); + } + + #[test] + fn to_rgba8_bounds_the_inflate_by_declared_dimensions() { + // A hostile payload: a tiny compressed blob that inflates far past the + // 1x1 image it claims to be. Without a cap this would balloon into a + // multi-MB (in the wild, multi-GB) allocation; bounded by the declared + // `width * height * 4` it must be rejected instead. + let bomb = vec![0u8; 4 * 1024 * 1024]; // 4 MiB of zeros → tiny deflate + let z = miniz_oxide::deflate::compress_to_vec_zlib(&bomb, 9); + assert!(z.len() < bomb.len(), "payload must actually compress"); + let img = Image { + id: 1, + number: 0, + placement: 0, + width: 1, + height: 1, + cols: 0, + rows: 0, + data: z, + format: WireFormat::Rgba, + compressed: true, + }; + assert_eq!(img.to_rgba8(), None, "an over-budget inflate is dropped"); + + // A payload that inflates to exactly its declared size still decodes. + let pixels = vec![0xabu8; 4]; // 1x1 RGBA + let z = miniz_oxide::deflate::compress_to_vec_zlib(&pixels, 9); + let ok = Image { data: z, ..img }; + assert_eq!(ok.to_rgba8().as_deref(), Some(&pixels[..])); + } +} diff --git a/crates/tty7-core/src/core/mod.rs b/crates/tty7-core/src/core/mod.rs index 1f0d2dee..a72ca565 100644 --- a/crates/tty7-core/src/core/mod.rs +++ b/crates/tty7-core/src/core/mod.rs @@ -6,6 +6,7 @@ pub mod git; pub mod gitignore; #[allow(dead_code)] pub mod keychain; +pub mod kitty_graphics; pub mod logfile; pub mod machine; pub mod osc; diff --git a/crates/tty7-core/src/daemon/pane.rs b/crates/tty7-core/src/daemon/pane.rs index 33a07dfa..817190d6 100644 --- a/crates/tty7-core/src/daemon/pane.rs +++ b/crates/tty7-core/src/daemon/pane.rs @@ -9,9 +9,11 @@ use std::time::Duration; use portable_pty::{Child, CommandBuilder, MasterPty, PtySize, native_pty_system}; +use crate::core::kitty_graphics::{GraphicsSniffer, Segment, Sniffed}; use crate::core::osc::OscTokenizer; use crate::daemon::protocol::{ - AuthResponse, DaemonMsg, NativeSshSpec, PaneInfo, RemoteContext, RemoteKind, ShellSpec, WinSize, + AuthResponse, DaemonMsg, MAX_FRAME, NativeSshSpec, PaneInfo, RemoteContext, RemoteKind, + ShellSpec, WinSize, }; use crate::daemon::shell_integration; @@ -416,7 +418,13 @@ pub struct DaemonPane { pub id: u64, owner: Option, backend: PaneBackend, - writer: Mutex>, + /// The input side (keyboard input / pasted text): the PTY writer, or the + /// native-SSH channel writer. Behind a `Mutex` because writes can arrive from + /// different connection threads. `Arc`-shared so the reader thread can write + /// kitty-graphics query replies (`\x1b_G…;OK\x1b\\`) back to the PTY inline + /// with the sniff, without routing through `write_input`. + writer: Arc>>, + /// Set during teardown so the reader doesn't emit a spurious exit. shutting_down: Arc, gate: Arc, state: Arc>, @@ -464,6 +472,32 @@ impl DeathReporter { } } +/// One out-of-band frame the reader forwards to the subscriber, kept in stream +/// order so a kitty image lands at the cursor cell the sender drew it at: a chunk +/// carrying graphics splits into `Output` runs interleaved with `Image`/`Delete` +/// frames, and they must reach the client in that same order. `Output` becomes a +/// `DaemonMsg::Output`, `Image` an encoded image frame, `Delete` a compact +/// selector — see the reader loop's `sniff` handling. +enum GraphicsFrame { + Output(Vec), + Image(Vec), + Delete(Vec), +} + +/// Queue an encoded image frame for the subscriber, dropping any frame larger +/// than [`MAX_FRAME`]. The writer's `write_frame` rejects an oversize payload +/// with an error the writer loop treats as *fatal* — it would tear the client +/// off an otherwise-healthy pane. A single image that won't fit is not worth +/// that: drop it here so the rest of the stream keeps flowing. The inline +/// `t=d` path is bounded by `MAX_TRANSMISSION_BASE64` but that still admits +/// ~72 MiB of raw pixels, and the local shm/file path has no cap of its own, +/// so both can land here over the limit. +fn push_image_frame(frames: &mut Vec, frame: Vec) { + if frame.len() <= MAX_FRAME { + frames.push(GraphicsFrame::Image(frame)); + } +} + impl DaemonPane { pub fn spawn( id: u64, @@ -484,7 +518,7 @@ impl DaemonPane { drop(pair.slave); let reader_handle = pair.master.try_clone_reader()?; - let writer = pair.master.take_writer()?; + let writer = Arc::new(Mutex::new(pair.master.take_writer()?)); let state = Arc::new(Mutex::new(PaneState { id, @@ -513,7 +547,7 @@ impl DaemonPane { shell_pid, integration_dir: spawn.integration_dir, }), - writer: Mutex::new(writer), + writer: writer.clone(), shutting_down: shutting_down.clone(), gate: gate.clone(), state: state.clone(), @@ -540,6 +574,7 @@ impl DaemonPane { shutting_down, gate, reader_handle, + writer.clone(), move || foreground_command_running(&fg_master, shell_pid), ForegroundProbes { remote: Box::new(move || foreground_remote_context(&remote_master)), @@ -561,7 +596,10 @@ impl DaemonPane { ) -> anyhow::Result> { let bridge = crate::daemon::ssh::session::make_bridge(); let reader_handle: Box = Box::new(bridge.reader); - let writer: Box = Box::new(bridge.writer); + let writer: Arc>> = + Arc::new(Mutex::new(Box::new(bridge.writer))); + // The connect task fills this once authenticated; the pane exposes it to + // WS4/WS5 via `ssh_connection()`. let connection: crate::daemon::ssh::SharedConnection = Arc::new(Mutex::new(Weak::new())); let target = spec @@ -607,7 +645,7 @@ impl DaemonPane { handle: bridge.handle, connection: connection.clone(), }), - writer: Mutex::new(writer), + writer: writer.clone(), shutting_down: shutting_down.clone(), gate: gate.clone(), state: state.clone(), @@ -622,6 +660,7 @@ impl DaemonPane { shutting_down, gate, reader_handle, + writer.clone(), || false, ForegroundProbes { remote: Box::new(|| None), @@ -664,6 +703,10 @@ impl DaemonPane { shutting_down: Arc, gate: Arc, mut reader: Box, + // The pane's input side, shared so the reader can write kitty-graphics + // query replies (`\x1b_G…;OK\x1b\\`) straight back to the PTY — see the + // graphics sniff in the loop below. + writer: Arc>>, foreground_running: impl Fn() -> bool + Send + 'static, probes: ForegroundProbes, death: Arc, @@ -678,6 +721,20 @@ impl DaemonPane { .spawn(move || { crate::core::threads::promote_to_user_interactive(); let mut sniffer = OscSniffer::new(); + // Kitty graphics interception (issue #213): lifts image + // sequences out of the stream *before* the ring/subscriber see + // them, so the base64 pixels never enter replay and the client's + // VT parser never chews through them. Zero-copy on the common + // no-graphics chunk — see [`GraphicsSniffer::sniff`]. + // + // File/shm transfer (`t=s`/`t=f`/`t=t`) is honored only while the + // pane is *local*: the object/path names resolve on this host, and + // reading them can't leak across an SSH tunnel. A pane that starts + // local can `ssh` out mid-session, so the flag is refreshed from + // the same remote-context poll below. Seed it from the pane's + // current context so the first probe answers correctly. + let starts_local = state.lock().unwrap().remote.is_none(); + let mut graphics = GraphicsSniffer::new_local(starts_local); let mut buf = [0u8; 65536]; let trace = std::env::var("TTY7_TRACE").is_ok_and(|v| !v.is_empty() && v != "0"); @@ -714,7 +771,71 @@ impl DaemonPane { tr_reads += 1; tr_bytes += n as u64; } - let bytes = &buf[..n]; + let raw = &buf[..n]; + // Kitty graphics: strip any image sequences out of the + // stream before anything else sees them. On the common + // no-graphics chunk this borrows `raw` unchanged; only a + // chunk that actually carries `\x1b_G…` allocates. Query + // replies are written straight back to the PTY here, and + // any images/deletes are forwarded to the subscriber + // out-of-band below — none of it enters the replay ring. + // + // `frames` is the ordered list of out-of-band frames to + // forward *in stream position*: a kitty image anchors to + // the cursor cell as it stood when its command appeared, + // so the client must apply the text before an image, then + // the image, then the text after it, in that order. On + // the no-graphics fast path `frames` stays empty and the + // whole chunk is sent as one `Output`; only a chunk with + // graphics splits into interleaved frames. + let mut frames: Vec = Vec::new(); + let passthrough: std::borrow::Cow<[u8]> = match graphics.sniff(raw) { + Sniffed::Plain(b) => std::borrow::Cow::Borrowed(b), + Sniffed::Segments(segs) => { + let mut pass = Vec::new(); + for seg in segs { + match seg { + Segment::Output(b) => { + pass.extend_from_slice(&b); + frames.push(GraphicsFrame::Output(b)); + } + Segment::Query(reply) => { + if let Ok(mut w) = writer.lock() { + let _ = w.write_all(&reply); + let _ = w.flush(); + } + } + Segment::Image(img) => { + push_image_frame(&mut frames, img.encode_frame()); + } + Segment::ImageFromMedium(transfer) => { + // File/shm handoff on a local pane: + // read (and unlink) the object here, + // then forward the raw pixels exactly + // like an inline image. This is the + // fast path that skips the client-side + // inflate the compressed-inline `t=d` + // fallback would force. A failed read + // just drops the frame — the sender + // reclaims its own object. + if let Some(img) = transfer.resolve() { + push_image_frame( + &mut frames, + img.encode_frame(), + ); + } + } + Segment::Delete(d) => { + frames.push(GraphicsFrame::Delete(d.encode())); + } + } + } + std::borrow::Cow::Owned(pass) + } + }; + let bytes: &[u8] = &passthrough; + // Sniff first (cheap, over the same bytes); collect any + // cwd/prompt change to emit while we hold the lock. let mut signals = sniffer.feed(bytes); if signals.shell.iter().any(|s| s.at_prompt) && foreground_running() { @@ -755,14 +876,59 @@ impl DaemonPane { let facts_before = may_change_facts.then(|| observed_facts(&st)); st.ring.append(bytes); if let Some(sub) = &st.subscriber { - if sub.send(DaemonMsg::Output(bytes.to_vec())).is_ok() { - gate.add(n); + // A send error just means the client is gone; ignore + // it and let the next attach install a new sender. + // Successful sends are counted against the gate; the + // connection's writer thread credits them back. + // + // On the no-graphics fast path `frames` is empty and + // the whole passthrough goes as one `Output`. When a + // chunk carried graphics, the frames are forwarded in + // stream order instead, so an image lands at the same + // cursor cell the sender drew it at. Each `Output` + // frame is gated on its own length; an image frame is + // gated too (it rode the same PTY read); a delete is + // tiny and ungated, matching the drain accounting in + // `server.rs`. + if frames.is_empty() { + if !bytes.is_empty() + && sub.send(DaemonMsg::Output(bytes.to_vec())).is_ok() + { + gate.add(bytes.len()); + } + } else { + for frame in frames { + match frame { + GraphicsFrame::Output(b) => { + let len = b.len(); + if !b.is_empty() + && sub.send(DaemonMsg::Output(b)).is_ok() + { + gate.add(len); + } + } + GraphicsFrame::Image(frame) => { + let len = frame.len(); + if sub.send(DaemonMsg::Image(frame)).is_ok() { + gate.add(len); + } + } + GraphicsFrame::Delete(sel) => { + let _ = sub.send(DaemonMsg::DeleteImage(sel)); + } + } + } } } apply_signals(&mut st, signals); if let Some(remote) = remote { apply_remote_context(&mut st, remote); } + // Keep kitty file/shm transfer gated on the pane's + // *current* locality: an `ssh` that just took the PTY + // must stop us honoring host-local object names. Cheap + // and only meaningful when a probe follows. + graphics.set_local(st.remote.is_none()); if let Some(agent) = agent { apply_agent(&mut st, agent); } @@ -1706,6 +1872,7 @@ fn proc_name(pid: i32) -> Option { #[cfg(test)] mod tests { use super::*; + use crate::core::kitty_graphics::ImageDelete; use std::path::Path; #[test] @@ -2495,6 +2662,12 @@ mod tests { assert_eq!(d.shell.last().unwrap().last_exit_code, Some(-1)); } + /// A discard writer for `spawn_reader` tests that don't inspect the PTY + /// write-back path (graphics query replies). + fn null_writer() -> Arc>> { + Arc::new(Mutex::new(Box::new(std::io::sink()))) + } + fn test_state(alive: bool) -> PaneState { PaneState { id: 0, @@ -2595,6 +2768,7 @@ mod tests { Arc::new(AtomicBool::new(shutting_down)), Arc::new(OutputGate::new()), Box::new(std::io::Cursor::new(b"\x1b]133;D;0\x07".to_vec())), + null_writer(), || false, ForegroundProbes { remote: Box::new(|| None), @@ -2856,6 +3030,7 @@ mod tests { Arc::new(AtomicBool::new(false)), Arc::new(OutputGate::new()), Box::new(std::io::Cursor::new(b"tail".to_vec())), + null_writer(), || false, ForegroundProbes { remote: Box::new(|| None), @@ -2881,6 +3056,131 @@ mod tests { ); } + /// Issue #213 end-to-end at the reader: a chunk carrying text plus a + /// kitty graphics query and a transmit-and-display must (1) keep only the + /// text in the replay ring and the `Output` frame, (2) write the `a=q` reply + /// back to the PTY writer, and (3) forward the image out-of-band as an + /// `Image` frame the client can decode. + #[test] + fn reader_strips_graphics_and_forwards_them_out_of_band() { + use crate::core::kitty_graphics::Image; + use base64::Engine as _; + + let state = Arc::new(Mutex::new(test_state(true))); + let (sub_tx, sub_rx) = mpsc::channel(); + state.lock().unwrap().subscriber = Some(sub_tx); + + // A writer we can read back, to prove the query reply reached the PTY. + #[derive(Clone)] + struct SharedBuf(Arc>>); + impl Write for SharedBuf { + fn write(&mut self, b: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(b); + Ok(b.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let sink = Arc::new(Mutex::new(Vec::new())); + let writer: Arc>> = + Arc::new(Mutex::new(Box::new(SharedBuf(sink.clone())))); + + // One 1x1 opaque-red RGBA pixel, transmitted-and-displayed inline. + let pixel = [0xffu8, 0x00, 0x00, 0xff]; + let b64 = base64::engine::general_purpose::STANDARD.encode(pixel); + let stream = format!( + "before\x1b_Gi=1,a=q,t=d,f=32,s=1,v=1;AAAA\x1b\\\ + mid\x1b_Ga=T,f=32,t=d,s=1,v=1,i=1;{b64}\x1b\\after" + ); + + let handle = DaemonPane::spawn_reader( + state.clone(), + Arc::new(AtomicBool::new(false)), + Arc::new(OutputGate::new()), + Box::new(std::io::Cursor::new(stream.into_bytes())), + writer, + || false, + ForegroundProbes { + remote: Box::new(|| None), + agent: Box::new(|| None), + cwd: Box::new(|| None), + }, + Arc::new(DeathReporter::new(|| {})), + ); + handle.join().unwrap(); + + // (1) The ring holds the text, none of the graphics bytes. + assert_eq!(state.lock().unwrap().ring.flatten(), b"beforemidafter"); + // (2) The query reply went to the PTY writer. + assert_eq!(sink.lock().unwrap().as_slice(), b"\x1b_Gi=1;OK\x1b\\"); + // (3) The subscriber sees the passthrough and the image *in stream order*: + // the text before the image, then the image at its cursor cell, then the + // text after it. The `a=q` reply splits "before" from "mid" into two + // Output frames; the image sits between "mid" and "after". + assert!(matches!(sub_rx.try_recv(), Ok(DaemonMsg::Output(b)) if b == b"before")); + assert!(matches!(sub_rx.try_recv(), Ok(DaemonMsg::Output(b)) if b == b"mid")); + match sub_rx.try_recv() { + Ok(DaemonMsg::Image(frame)) => { + let img = Image::decode_frame(&frame).expect("decodable image frame"); + assert_eq!(img.id, 1); + assert_eq!((img.width, img.height), (1, 1)); + assert_eq!(img.to_rgba8().unwrap(), pixel); + } + other => panic!("expected Image frame, got {other:?}"), + } + assert!(matches!(sub_rx.try_recv(), Ok(DaemonMsg::Output(b)) if b == b"after")); + } + + /// A kitty delete (`a=d`) is lifted out and forwarded as a `DeleteImage` + /// selector, leaving the surrounding text intact in the ring. + #[test] + fn reader_forwards_graphics_deletes() { + let state = Arc::new(Mutex::new(test_state(true))); + let (sub_tx, sub_rx) = mpsc::channel(); + state.lock().unwrap().subscriber = Some(sub_tx); + + let handle = DaemonPane::spawn_reader( + state.clone(), + Arc::new(AtomicBool::new(false)), + Arc::new(OutputGate::new()), + Box::new(std::io::Cursor::new(b"x\x1b_Ga=d,d=A\x1b\\y".to_vec())), + null_writer(), + || false, + ForegroundProbes { + remote: Box::new(|| None), + agent: Box::new(|| None), + cwd: Box::new(|| None), + }, + Arc::new(DeathReporter::new(|| {})), + ); + handle.join().unwrap(); + + assert_eq!(state.lock().unwrap().ring.flatten(), b"xy"); + // In stream order: text before the delete, the delete, then text after. + assert!(matches!(sub_rx.try_recv(), Ok(DaemonMsg::Output(b)) if b == b"x")); + match sub_rx.try_recv() { + Ok(DaemonMsg::DeleteImage(sel)) => { + let d = ImageDelete::decode(&sel).unwrap(); + assert_eq!(d.target, b'A'); + } + other => panic!("expected DeleteImage, got {other:?}"), + } + assert!(matches!(sub_rx.try_recv(), Ok(DaemonMsg::Output(b)) if b == b"y")); + } + + #[test] + fn push_image_frame_drops_over_max_frame() { + let mut frames = Vec::new(); + // At the limit: queued. + push_image_frame(&mut frames, vec![0u8; MAX_FRAME]); + // Over the limit: dropped, so the writer's fatal `write_frame` error + // (which would disconnect the client) is never reached. + push_image_frame(&mut frames, vec![0u8; MAX_FRAME + 1]); + assert_eq!(frames.len(), 1, "only the in-budget frame is queued"); + assert!(matches!(&frames[0], GraphicsFrame::Image(f) if f.len() == MAX_FRAME)); + } + #[test] fn reader_poll_applies_the_probed_cwd() { let state = Arc::new(Mutex::new(test_state(true))); @@ -2896,6 +3196,7 @@ mod tests { Arc::new(AtomicBool::new(false)), Arc::new(OutputGate::new()), Box::new(std::io::Cursor::new(b"alice@host ~ % ".to_vec())), + null_writer(), || false, ForegroundProbes { remote: Box::new(|| None), @@ -2926,6 +3227,7 @@ mod tests { Arc::new(AtomicBool::new(false)), Arc::new(OutputGate::new()), Box::new(std::io::Cursor::new(Vec::new())), + null_writer(), || false, ForegroundProbes { remote: Box::new(|| None), @@ -2951,6 +3253,7 @@ mod tests { Arc::new(AtomicBool::new(true)), Arc::new(OutputGate::new()), Box::new(std::io::Cursor::new(Vec::new())), + null_writer(), || false, ForegroundProbes { remote: Box::new(|| None), diff --git a/crates/tty7-core/src/daemon/protocol.rs b/crates/tty7-core/src/daemon/protocol.rs index 1d55c557..8a4a9523 100644 --- a/crates/tty7-core/src/daemon/protocol.rs +++ b/crates/tty7-core/src/daemon/protocol.rs @@ -641,6 +641,18 @@ pub enum DaemonMsg { Size(WinSize), Snapshot(Vec), Output(Vec), + /// A kitty graphics image lifted out of the PTY stream daemon-side (issue + /// #213). Carried out-of-band as a compact binary frame + /// ([`crate::core::kitty_graphics::Image::encode_frame`]) so the base64 text + /// never rides the socket and the client's VT parser never sees it. The + /// pixel payload stays *compressed* on the wire — the client inflates — so a + /// remote pane's frames don't balloon across the SSH tunnel. + Image(Vec), + /// A kitty graphics delete (`a=d`) lifted out of the PTY stream daemon-side. + /// Payload is a compact selector frame + /// ([`crate::core::kitty_graphics::ImageDelete::encode`]) telling the client + /// which stored image(s)/placement(s) to drop. + DeleteImage(Vec), Cwd(PathBuf), Prompt { active: bool, @@ -730,6 +742,13 @@ mod kind { pub const AGENT_STATUS: u8 = 22; pub const VERSION_REPLY: u8 = 40; pub const PROCS: u8 = 50; + /// `Image` — a kitty graphics frame lifted out of the PTY stream (issue + /// #213). 60 sits clear of every range above; the payload is the compact + /// binary encoding, not JSON, so it stays outside the `to_json` arms. + pub const IMAGE: u8 = 60; + /// `DeleteImage` — a kitty graphics `a=d` delete lifted out of the stream + /// (issue #213). Compact binary selector, like `IMAGE`. + pub const DELETE_IMAGE: u8 = 61; } pub fn write_frame(w: &mut W, kind: u8, payload: &[u8]) -> io::Result<()> { @@ -1019,6 +1038,8 @@ impl DaemonMsg { DaemonMsg::Size(size) => write_frame(w, kind::SIZE, &to_json(size)?), DaemonMsg::Snapshot(bytes) => write_frame(w, kind::SNAPSHOT, bytes), DaemonMsg::Output(bytes) => write_frame(w, kind::OUTPUT, bytes), + DaemonMsg::Image(frame) => write_frame(w, kind::IMAGE, frame), + DaemonMsg::DeleteImage(sel) => write_frame(w, kind::DELETE_IMAGE, sel), DaemonMsg::Cwd(path) => write_frame(w, kind::CWD, &to_json(path)?), DaemonMsg::Prompt { active, @@ -1072,6 +1093,8 @@ impl DaemonMsg { kind::SIZE => DaemonMsg::Size(from_json(&payload)?), kind::SNAPSHOT => DaemonMsg::Snapshot(payload), kind::OUTPUT => DaemonMsg::Output(payload), + kind::IMAGE => DaemonMsg::Image(payload), + kind::DELETE_IMAGE => DaemonMsg::DeleteImage(payload), kind::CWD => DaemonMsg::Cwd(from_json(&payload)?), kind::PROMPT => { let (active, at_prompt, last_exit) = from_json(&payload)?; diff --git a/crates/tty7-core/src/daemon/server.rs b/crates/tty7-core/src/daemon/server.rs index b8a39583..f62a4957 100644 --- a/crates/tty7-core/src/daemon/server.rs +++ b/crates/tty7-core/src/daemon/server.rs @@ -662,6 +662,10 @@ fn spawn_writer( }; let drained = match &msg { DaemonMsg::Output(b) => b.len(), + // Image frames are lifted from the same PTY read the gate + // credits, so they must debit it too or the reader stays + // throttled against bytes that already left the queue. + DaemonMsg::Image(b) => b.len(), _ => 0, }; let write_ok = msg.encode(&mut write_stream).is_ok(); diff --git a/src/terminal/element.rs b/src/terminal/element.rs index 912faff8..3fea8fed 100644 --- a/src/terminal/element.rs +++ b/src/terminal/element.rs @@ -7,10 +7,10 @@ use alacritty_terminal::selection::SelectionRange; use alacritty_terminal::term::cell::{Cell, Flags}; use alacritty_terminal::vte::ansi::{Color as AnsiColor, CursorShape, NamedColor, Rgb}; use gpui::{ - App, BorderStyle, Bounds, ContentMask, CursorStyle, Element, ElementId, Font, FontStyle, - FontWeight, GlobalElementId, Hitbox, HitboxBehavior, HitboxId, Hsla, IntoElement, LayoutId, - MouseButton, MouseDownEvent, MouseMoveEvent, MouseUpEvent, Pixels, Point, Rgba, SharedString, - Style, TextAlign, TextRun, Window, fill, outline, point, px, relative, size, + App, BorderStyle, Bounds, ContentMask, Corners, CursorStyle, Element, ElementId, Font, + FontStyle, FontWeight, GlobalElementId, Hitbox, HitboxBehavior, HitboxId, Hsla, IntoElement, + LayoutId, MouseButton, MouseDownEvent, MouseMoveEvent, MouseUpEvent, Pixels, Point, Rgba, + SharedString, Style, TextAlign, TextRun, Window, fill, outline, point, px, relative, size, }; use gpui_component::ActiveTheme as _; @@ -823,6 +823,11 @@ struct GridSnapshot { any_selected: bool, any_match: bool, any_current: bool, + /// Scrollback state at snapshot time, so the paint pass can map a kitty + /// image's absolute anchor row back to a screen row: screen_row = + /// anchor_row - history_size + display_offset. + display_offset: i32, + history_size: usize, } impl TerminalElement { @@ -841,6 +846,7 @@ impl TerminalElement { let mut sliver: Option> = None; let mut any_selected = false; let display_offset; + let history_size; { let mut palette = self.view.read(cx).terminal.palette; if let Some(active) = cx.try_global::() { @@ -850,6 +856,7 @@ impl TerminalElement { let term = term.lock(); let content = term.renderable_content(); display_offset = content.display_offset as i32; + history_size = term.grid().history_size(); let selection = content.selection; for cell in content.display_iter { @@ -906,6 +913,8 @@ impl TerminalElement { any_selected, any_match, any_current, + display_offset, + history_size, } } @@ -1175,7 +1184,7 @@ impl Element for TerminalElement { .max(1.0) as usize; self.view.update(cx, |view, _cx| { - view.set_grid_size(cols, rows, cell_width, line_height); + view.set_grid_size(cols, rows, cell_width, line_height, window.scale_factor()); }); let hitbox = window.insert_hitbox(bounds, HitboxBehavior::Normal); @@ -1243,6 +1252,20 @@ impl Element for TerminalElement { ); let marked = self.view.read(cx).marked_text.clone(); + // Kitty-graphics placements for this pane. Each image anchors to an + // absolute scrollback row (recorded when its command arrived in-stream); + // map that back to a screen row with the snapshot's scroll state and + // drop anything scrolled out of the viewport. + let image_store = self.view.read(cx).terminal.images(); + // Take the retired list *before* the snapshot. The decode worker runs on + // its own thread and can retire a frame between the two calls; taking + // retired second would hand us a list containing an image the snapshot + // still says to paint, and `sprite_atlas.remove` takes effect + // immediately — so the frame would paint and then vanish. In this order + // the worst case is evicting one paint late, which is invisible. + let retired_images = image_store.take_retired(); + let images = image_store.snapshot(); + window.with_content_mask(Some(ContentMask { bounds }), |window| { paint_backgrounds(window, &geom, &buf); if snap.any_selected { @@ -1275,6 +1298,64 @@ impl Element for TerminalElement { bold_font.as_ref(), italic_font.as_ref(), ); + // Kitty-graphics images, painted over the placeholder cells they + // occupy. `anchor_row` is absolute (measured from the top of + // scrollback); convert it to a screen row with the same scroll state + // `build_grid` captured. Rows spanning past the viewport top/bottom + // are clipped by the surrounding content mask. + // + // Sizing: per the kitty spec, an image with no `c=`/`r=` is shown at + // its natural size — one image pixel per terminal *device* pixel. A + // sender like terminal-browser renders its frame to exactly fill the + // pixel area we reported to the child via `ws_xpixel`/`ws_ypixel`, + // which the daemon sets to `cols × round(cell_w × scale)` / + // `rows × round(cell_h × scale)` — the cell size in *device* pixels + // (see `set_grid_size`). So the frame is at device resolution; to map + // it back to a cell span we divide by that *same* device cell size, + // `round(cell_logical × scale)`. Painting the resulting cell-span + // bounds (in logical px) lets gpui blit the device-resolution bitmap + // ~1:1 on the framebuffer — sharp, and the right size — instead of + // upscaling a half-resolution one. Deriving here, not at placement, + // keeps it correct across font-size / zoom / display-scale changes. + let scale = window.scale_factor(); + let scale = if scale.is_finite() && scale > 0. { + scale + } else { + 1. + }; + for img in &images { + let round_w = (geom.cell_width.as_f32() * scale).round().max(1.); + let round_h = (geom.line_height.as_f32() * scale).round().max(1.); + let span_cols = if img.cols > 0 { + img.cols as f32 + } else { + (img.width_px as f32 / round_w).round().max(1.) + }; + let span_rows = if img.rows > 0 { + img.rows as f32 + } else { + (img.height_px as f32 / round_h).round().max(1.) + }; + let screen_row = + img.anchor_row - snap.history_size as i64 + snap.display_offset as i64; + // Fully above or below the viewport: nothing visible to paint. + if screen_row + span_rows as i64 <= 0 || screen_row >= geom.rows as i64 { + continue; + } + let top = geom.origin.y + geom.line_height * screen_row as f32; + let left = geom.origin.x + geom.cell_width * img.anchor_col as f32; + let bounds = Bounds { + origin: point(left, top), + size: size(geom.cell_width * span_cols, geom.line_height * span_rows), + }; + let _ = window.paint_image(bounds, Corners::default(), img.data.clone(), 0, false); + } + // Evict superseded / deleted frames from the sprite atlas. Without + // this a browser re-transmitting at 60fps would leak one GPU tile per + // frame, growing the atlas without bound until the compositor stalls. + for retired in &retired_images { + let _ = window.drop_image(retired.clone()); + } if let Some(row) = sliver { let sg = CellGeom { origin: point(geom.origin.x, geom.origin.y - geom.line_height), diff --git a/src/terminal/images.rs b/src/terminal/images.rs new file mode 100644 index 00000000..8205379b --- /dev/null +++ b/src/terminal/images.rs @@ -0,0 +1,573 @@ +//! Client-side kitty-graphics image store (issue #213). +//! +//! The daemon lifts image transmissions out of the PTY byte stream and forwards +//! them out-of-band as compact frames (`DaemonMsg::Image` / `DaemonMsg::DeleteImage` +//! — see [`tty7_core::core::kitty_graphics`]). This module is the client end: it +//! decodes each frame into a GPUI [`RenderImage`], anchors it to the grid cell the +//! cursor sat on when the command arrived, and hands the placed images to the +//! paint path so the element can blit them over the character grid. +//! +//! # Why anchor by absolute row +//! +//! Like a command [`mark`](crate::terminal::marks), an image's position has to +//! survive scrolling. A kitty image is placed at the cursor cell as it stood when +//! its command appeared in the stream; once recorded, the grid keeps scrolling +//! under it. We store the row as an absolute index from the top of scrollback +//! (`history_size - display_offset + cursor_line`, the exact formula +//! [`record_mark`](crate::terminal::remote) uses) and convert back to a screen +//! row at paint time (`anchor_row - history_size + display_offset`, the inverse +//! [`scroll_to_mark`](crate::terminal::view::TerminalView::scroll_to_mark) +//! applies). Below the scrollback limit — where a pane spends most of its life — +//! this is exact; past it the anchor drifts by the (unobservable) discard count, +//! the same caveat marks carry, and a browser that redraws every frame corrects +//! it on the next transmit anyway. +//! +//! GPUI's sprite atlas expects **BGRA** pixels (it swaps R↔B when caching an +//! `image` crate `RgbaImage` — see `gpui::img`), so [`decode`] does the swap once +//! at ingest; the placed [`RenderImage`] is uploaded verbatim thereafter. + +use std::sync::mpsc::{Receiver, Sender, TryRecvError, channel}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; + +use gpui::RenderImage; +use image::{Frame, RgbaImage}; +use smallvec::SmallVec; + +use tty7_core::core::kitty_graphics::{Image, ImageDelete, WireFormat}; + +/// One image placed on the grid, ready to blit. +#[derive(Clone)] +pub struct PlacedImage { + /// Decoded pixels as a GPUI render image (BGRA, one frame). + pub data: Arc, + /// Row index from the top of scrollback at placement time — the position + /// anchor. See the module docs for when it stops being exact. + pub anchor_row: i64, + /// Column of the top-left cell. + pub anchor_col: usize, + /// Source pixel dimensions, for deriving the cell span when the sender did + /// not give an explicit one. + pub width_px: u32, + pub height_px: u32, + /// Explicit cell span the sender requested (`c=` / `r=`); 0 means "derive + /// from the pixel size and the cell size at paint time". + pub cols: u32, + pub rows: u32, + /// The kitty image id (`i=`) and placement id (`p=`), for targeted deletes. + /// `id == 0` is an anonymous image, removable only by a delete-all. + pub id: u32, + pub placement: u32, +} + +/// A pane's placed images plus the retired render images awaiting atlas +/// eviction, shared between the reader thread (writer) and the paint path +/// (reader), exactly like [`Marks`](crate::terminal::marks::Marks). +/// +/// `retired` is the other half of the fix for a browser that repaints at 60fps: +/// each transmitted frame becomes a fresh [`RenderImage`] with a new atlas id, +/// so the *previous* frame's GPU tile has to be dropped or the sprite atlas +/// grows without bound (see [`take_retired`](ImageStore::take_retired)). The +/// reader can't touch the atlas — that needs `&mut Window` — so it parks the +/// superseded `Arc`s here and the paint path drains and drops them. +#[derive(Default)] +struct StoreInner { + placed: Vec, + retired: Vec>, +} + +/// A pane's placed images, shared between the reader thread (writer) and the +/// paint path (reader), exactly like [`Marks`](crate::terminal::marks::Marks). +#[derive(Clone, Default)] +pub struct ImageStore(Arc>); + +/// Cap on placed images retained at once. A browser deletes-then-transmits every +/// frame, so the live set is tiny; this only bounds a sender that transmits +/// without ever deleting, dropping the oldest rather than growing without limit. +const MAX_IMAGES: usize = 256; + +impl ImageStore { + pub fn new() -> Self { + Self::default() + } + + /// Place a freshly received image at (`anchor_row`, `anchor_col`). A new + /// transmission with the same identity as an existing one replaces it in + /// place (kitty reuses an id to update an image); otherwise it is appended. + /// The replaced frame's render image is retired for atlas eviction. + pub fn place(&self, img: PlacedImage) { + let Ok(mut inner) = self.0.lock() else { return }; + let StoreInner { placed, retired } = &mut *inner; + // Same (id, placement) → replace. An anonymous image (id 0) never + // matches, so each anonymous transmit is a distinct placement. + if img.id != 0 { + placed.retain(|p| { + let same = p.id == img.id && p.placement == img.placement; + if same { + retired.push(p.data.clone()); + } + !same + }); + } + placed.push(img); + let overflow = placed.len().saturating_sub(MAX_IMAGES); + if overflow > 0 { + retired.extend(placed.drain(..overflow).map(|p| p.data)); + } + } + + /// Apply a delete selector. Only the targets a sender tty7 faces actually + /// uses are honored (all / by id / by placement); richer kitty selectors + /// (by cell, by z-index, by number) are left in place rather than guessed. + /// Removed frames' render images are retired for atlas eviction. + pub fn delete(&self, del: &ImageDelete) { + let Ok(mut inner) = self.0.lock() else { return }; + let retire = |p: &PlacedImage, keep: bool, retired: &mut Vec>| { + if !keep { + retired.push(p.data.clone()); + } + keep + }; + let StoreInner { placed, retired } = &mut *inner; + match del.target { + // All visible placements. Case only governs whether kitty also frees + // the image data; the client frees unconditionally, so both clear. + b'a' | b'A' => { + retired.extend(placed.drain(..).map(|p| p.data)); + } + // By image id. + b'i' | b'I' => placed.retain(|p| retire(p, p.id != del.id, retired)), + // By placement id (scoped to its image when an id is also given). + b'p' | b'P' => placed.retain(|p| { + let keep = p.placement != del.placement || (del.id != 0 && p.id != del.id); + retire(p, keep, retired) + }), + // A selector we don't model: leave the store untouched. + _ => {} + } + } + + /// Snapshot the placed images for a paint pass. Cheap: the live set is small + /// and `PlacedImage` is a handful of fields plus an `Arc` clone. + pub fn snapshot(&self) -> Vec { + self.0.lock().map(|s| s.placed.clone()).unwrap_or_default() + } + + /// Take the render images retired since the last call, for the paint path to + /// evict from the sprite atlas (`Window::drop_image`). Draining here — the + /// one place with `&mut Window` — is what stops a 60fps re-transmitting + /// sender from leaking a GPU tile per frame and dragging the compositor down. + pub fn take_retired(&self) -> Vec> { + self.0 + .lock() + .map(|mut s| std::mem::take(&mut s.retired)) + .unwrap_or_default() + } + + /// Drop everything (the grid was cleared, so every anchor is meaningless). + /// Placed frames are retired so their atlas tiles are still evicted. + pub fn clear(&self) { + if let Ok(mut inner) = self.0.lock() { + let gone: Vec<_> = inner.placed.drain(..).map(|p| p.data).collect(); + inner.retired.extend(gone); + } + } +} + +/// A raw (still-compressed) image frame handed to the [`DecodeWorker`], with the +/// grid anchor the reader captured the instant the transmission arrived. Keeping +/// the anchor with the frame lets decoding move off the reader thread without +/// losing the cursor position the image was drawn at. +pub struct PendingFrame { + pub img: Image, + pub anchor_row: i64, + pub anchor_col: usize, +} + +/// Off-thread image decoder with newest-frame-wins coalescing — the crux of the +/// performance story (issue #213). +/// +/// A full-window browser frame is ~28 MB of RGBA after zlib inflate, and the +/// inflate alone measures ~42 ms. Doing that inline on the reader thread (which +/// also services PTY output and scrolling) blocks the whole pane for ~42 ms per +/// frame, and a 60fps sender queues frames faster than they drain — latency +/// grows without bound and scrolling stutters. This is the cost the +/// device-pixel resolution bump quadrupled. +/// +/// The fix mirrors how kitty/ghostty stay smooth: decode off the I/O thread, and +/// when the producer outruns the decoder, **drop stale frames instead of +/// queuing them**. A bounded [`inbox`] of one *replaceable slot per image id* +/// means a re-transmitting browser only ever has its latest frame per image +/// waiting; older undecoded frames are discarded before they cost an inflate. +/// The worker decodes the newest, `place`s it, and wakes the view. +/// +/// [`inbox`]: DecodeWorker::inbox +pub struct DecodeWorker { + tx: Option>, + handle: Option>, +} + +impl DecodeWorker { + /// Spawn the decode thread. `store` is the shared placement store the worker + /// writes decoded frames into; `wake` is called after each successful decode + /// so the view repaints (a cloned `EventProxy::send_event(Wakeup)` in + /// practice). The thread ends when the returned worker is dropped (the + /// channel closes). + pub fn spawn(store: ImageStore, wake: impl Fn() + Send + 'static) -> Self { + let (tx, rx) = channel::(); + let handle = std::thread::Builder::new() + .name("tty7-image-decode".to_string()) + .spawn(move || Self::run(rx, store, wake)) + .ok(); + Self { + tx: Some(tx), + handle, + } + } + + /// Hand a raw frame to the worker. Never blocks the caller (the reader + /// thread): the frame is queued and decoded asynchronously. If the worker + /// has gone away the frame is silently dropped — the pane is tearing down. + pub fn submit(&self, frame: PendingFrame) { + if let Some(tx) = &self.tx { + let _ = tx.send(frame); + } + } + + /// The decode loop. Blocks for the next frame, then **coalesces**: drains + /// everything already queued and keeps only the last frame per image id, so + /// a burst that piled up during a slow inflate collapses to one decode per + /// image. Decodes the survivors newest-first and places them. + fn run(rx: Receiver, store: ImageStore, wake: impl Fn()) { + while let Ok(first) = rx.recv() { + // Collect the blocking frame plus any that arrived while we were + // busy, newest-per-id winning (a later frame with the same id + // supersedes an earlier one — exactly what `place` would do, but + // without paying to decode the ones we'd immediately retire). + let mut latest: Vec = vec![first]; + loop { + match rx.try_recv() { + Ok(next) => { + if next.img.id != 0 + && let Some(slot) = latest.iter_mut().find(|p| p.img.id == next.img.id) + { + *slot = next; // supersede the queued frame for this id + } else { + latest.push(next); + } + } + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => { + // Decode whatever we already gathered, then exit. + Self::place_all(&store, latest, &wake); + return; + } + } + } + Self::place_all(&store, latest, &wake); + } + } + + fn place_all(store: &ImageStore, frames: Vec, wake: &impl Fn()) { + let mut placed_any = false; + for pf in frames { + if let Some((data, w, h)) = decode(&pf.img) { + store.place(PlacedImage { + data, + anchor_row: pf.anchor_row, + anchor_col: pf.anchor_col, + width_px: w, + height_px: h, + cols: pf.img.cols, + rows: pf.img.rows, + id: pf.img.id, + placement: pf.img.placement, + }); + placed_any = true; + } + } + if placed_any { + wake(); + } + } +} + +impl Drop for DecodeWorker { + fn drop(&mut self) { + // Drop the sender first so the worker sees `Disconnected` and returns; + // then join so its store writes are visible and it doesn't outlive the + // pane. Joining before dropping `tx` would deadlock — the loop only ends + // once the channel closes. + self.tx = None; + if let Some(h) = self.handle.take() { + let _ = h.join(); + } + } +} + +/// Decode a daemon [`Image`] frame into GPUI-ready pixels: inflate + expand to +/// RGBA (via the protocol type's own [`Image::to_rgba8`]), or decode a PNG +/// payload, then swap R↔B to the BGRA the sprite atlas wants. Returns the render +/// image and its true pixel dimensions (a PNG carries its own, overriding any +/// `s=`/`v=` the sender may have omitted). `None` if the payload can't be decoded. +pub fn decode(img: &Image) -> Option<(Arc, u32, u32)> { + let (mut rgba, w, h) = match img.format { + WireFormat::Png => { + // The one path pixels stay encoded through: decode with the `image` + // crate (a direct dep, same version gpui uses) rather than the + // protocol type, which declines PNG on purpose. + let dyn_img = image::load_from_memory(&img.data).ok()?; + let buf = dyn_img.into_rgba8(); + let (w, h) = buf.dimensions(); + (buf.into_raw(), w, h) + } + WireFormat::Rgb | WireFormat::Rgba => { + let rgba = img.to_rgba8()?; + (rgba, img.width, img.height) + } + }; + if w == 0 || h == 0 || rgba.len() < (w as usize * h as usize * 4) { + return None; + } + rgba.truncate(w as usize * h as usize * 4); + // RGBA → BGRA for the atlas. A channel swap and nothing else: `RenderImage` + // holds *straight* alpha, not premultiplied. gpui's own producers say so — + // `swap_rgba_pa_to_bgra`, which both the CoreGraphics text rasterizer and + // the SVG renderer run their premultiplied output through, divides the + // color channels back out by alpha on the way in. Premultiplying here would + // darken every translucent pixel of an `f=100` PNG twice over. + for px in rgba.chunks_exact_mut(4) { + px.swap(0, 2); + } + let buffer = RgbaImage::from_raw(w, h, rgba)?; + let render = RenderImage::new(SmallVec::from_elem(Frame::new(buffer), 1)); + Some((Arc::new(render), w, h)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A raw 1x1 opaque-red RGBA image, the minimum a placement needs, built the + /// way the daemon delivers one (base64-decoded, uncompressed). + fn red_pixel() -> Image { + Image { + id: 0, + number: 0, + placement: 0, + width: 1, + height: 1, + cols: 0, + rows: 0, + data: vec![0xff, 0x00, 0x00, 0xff], + format: WireFormat::Rgba, + compressed: false, + } + } + + fn placed(id: u32, placement: u32) -> PlacedImage { + let (data, w, h) = decode(&red_pixel()).unwrap(); + PlacedImage { + data, + anchor_row: 0, + anchor_col: 0, + width_px: w, + height_px: h, + cols: 0, + rows: 0, + id, + placement, + } + } + + #[test] + fn decodes_rgba_and_swaps_to_bgra() { + let img = Image { + data: vec![1, 2, 3, 4], + ..red_pixel() + }; + let (data, w, h) = decode(&img).unwrap(); + assert_eq!((w, h), (1, 1)); + // Red/blue swapped: RGBA [1,2,3,4] → BGRA [3,2,1,4]. + assert_eq!(data.as_bytes(0).unwrap(), &[3, 2, 1, 4]); + } + + #[test] + fn same_id_replaces_in_place() { + let store = ImageStore::new(); + store.place(placed(7, 1)); + store.place(placed(7, 1)); + assert_eq!( + store.snapshot().len(), + 1, + "a re-transmit replaces, not stacks" + ); + } + + #[test] + fn anonymous_images_coexist() { + let store = ImageStore::new(); + store.place(placed(0, 0)); + store.place(placed(0, 0)); + assert_eq!( + store.snapshot().len(), + 2, + "id 0 is a fresh placement each time" + ); + } + + #[test] + fn delete_all_clears_everything() { + let store = ImageStore::new(); + store.place(placed(1, 0)); + store.place(placed(2, 0)); + store.delete(&ImageDelete { + target: b'A', + id: 0, + placement: 0, + }); + assert!(store.snapshot().is_empty()); + } + + #[test] + fn delete_by_id_leaves_others() { + let store = ImageStore::new(); + store.place(placed(1, 0)); + store.place(placed(2, 0)); + store.delete(&ImageDelete { + target: b'i', + id: 1, + placement: 0, + }); + let left = store.snapshot(); + assert_eq!(left.len(), 1); + assert_eq!(left[0].id, 2); + } + + #[test] + fn unknown_selector_is_a_no_op() { + let store = ImageStore::new(); + store.place(placed(1, 0)); + // `z` (by z-index) isn't modeled; the image must survive. + store.delete(&ImageDelete { + target: b'z', + id: 0, + placement: 0, + }); + assert_eq!(store.snapshot().len(), 1); + } + + #[test] + fn overflow_drops_the_oldest() { + let store = ImageStore::new(); + for i in 0..(MAX_IMAGES as u32 + 5) { + store.place(placed(i + 1, 0)); + } + let left = store.snapshot(); + assert_eq!(left.len(), MAX_IMAGES); + assert_eq!(left[0].id, 6, "the five oldest aged out"); + } + + #[test] + fn replacing_a_frame_retires_the_old_render_image() { + let store = ImageStore::new(); + store.place(placed(7, 1)); + // A same-id re-transmit (what a 60fps browser does) must retire the old + // frame's render image so the paint path can evict its atlas tile. + store.place(placed(7, 1)); + assert_eq!(store.snapshot().len(), 1, "still one live placement"); + assert_eq!( + store.take_retired().len(), + 1, + "the superseded frame is retired" + ); + assert!(store.take_retired().is_empty(), "draining is one-shot"); + } + + #[test] + fn deletes_and_clear_retire_render_images() { + let store = ImageStore::new(); + store.place(placed(1, 0)); + store.place(placed(2, 0)); + let _ = store.take_retired(); // drain the (none) from placement + store.delete(&ImageDelete { + target: b'i', + id: 1, + placement: 0, + }); + assert_eq!( + store.take_retired().len(), + 1, + "the deleted frame is retired" + ); + store.clear(); + assert_eq!( + store.take_retired().len(), + 1, + "clear retires the survivor too" + ); + } + + fn frame(id: u32) -> PendingFrame { + PendingFrame { + img: Image { id, ..red_pixel() }, + anchor_row: 0, + anchor_col: 0, + } + } + + /// The worker decodes off-thread and places what it receives. A single frame + /// lands in the store, at the anchor the reader captured. + #[test] + fn worker_decodes_and_places_off_thread() { + let store = ImageStore::new(); + let woken = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let w = woken.clone(); + let worker = DecodeWorker::spawn(store.clone(), move || { + w.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + }); + worker.submit(frame(7)); + drop(worker); // joins the thread, so the decode is finished on return + let placed = store.snapshot(); + assert_eq!(placed.len(), 1); + assert_eq!(placed[0].id, 7); + assert!( + woken.load(std::sync::atomic::Ordering::SeqCst) >= 1, + "a successful decode wakes the view" + ); + } + + /// A burst of same-id frames (what a re-transmitting browser produces faster + /// than the decoder drains) collapses to a single live placement — stale + /// frames are coalesced away rather than each costing an inflate. The store's + /// own same-id replacement guarantees the end state even if the worker only + /// sees them one at a time, so this asserts the invariant the pipeline keeps. + #[test] + fn worker_coalesces_a_same_id_burst_to_one_placement() { + let store = ImageStore::new(); + let worker = DecodeWorker::spawn(store.clone(), || {}); + for _ in 0..50 { + worker.submit(frame(9)); + } + drop(worker); // drains + joins + assert_eq!( + store.snapshot().len(), + 1, + "a same-id burst leaves exactly one live frame" + ); + } + + /// Distinct ids are independent placements — coalescing is per id, so two + /// different images both survive. + #[test] + fn worker_keeps_distinct_ids() { + let store = ImageStore::new(); + let worker = DecodeWorker::spawn(store.clone(), || {}); + worker.submit(frame(1)); + worker.submit(frame(2)); + drop(worker); + let mut ids: Vec = store.snapshot().iter().map(|p| p.id).collect(); + ids.sort_unstable(); + assert_eq!(ids, vec![1, 2]); + } +} diff --git a/src/terminal/mod.rs b/src/terminal/mod.rs index cb0c3e84..52ef2293 100644 --- a/src/terminal/mod.rs +++ b/src/terminal/mod.rs @@ -10,6 +10,7 @@ pub(crate) mod git_status; mod highlight; mod history; mod hold; +pub(crate) mod images; pub mod input; mod loopback; pub(crate) mod marks; diff --git a/src/terminal/remote.rs b/src/terminal/remote.rs index a5002af1..c29207e4 100644 --- a/src/terminal/remote.rs +++ b/src/terminal/remote.rs @@ -76,6 +76,10 @@ struct ReaderSignals { auth: Arc>>, phase: Arc>>, marks: crate::terminal::marks::Marks, + /// Kitty-graphics images the daemon lifted out of the stream (issue #213), + /// anchored to the grid for the paint path to blit. Shared with the reader, + /// which places/deletes them as `DaemonMsg::Image`/`DeleteImage` frames land. + images: crate::terminal::images::ImageStore, } #[derive(Clone, Debug, PartialEq)] @@ -152,6 +156,10 @@ pub struct RemoteTerminal { pub exited: bool, size: TermSize, synced_size: bool, + /// The `(cell_w, cell_h)` last sent to the daemon, in device pixels. Tracked + /// alongside `size` so a display-scale change still reaches the child even + /// when the grid dimensions are unchanged. + synced_cell: (u16, u16), writer: Mutex, cwd: Arc>>, shell_state: Arc>, @@ -167,6 +175,11 @@ pub struct RemoteTerminal { agent: Arc>>, agent_session: Arc>>, marks: crate::terminal::marks::Marks, + /// Kitty-graphics images placed on this pane's grid (issue #213). + /// Written by the reader thread from out-of-band `Image`/`DeleteImage` + /// frames, read by the paint path — same shared-handle discipline as + /// `marks`, since only the client holds the grid the anchors are relative to. + images: crate::terminal::images::ImageStore, route: PaneRoute, proxy: EventProxy, reader_thread: Option>, @@ -333,6 +346,10 @@ impl RemoteTerminal { let mut term = self.term.lock(); term.reset_state(); } + // The grid was just reset, so every image anchor now points nowhere. + // Drop them; the daemon does not replay out-of-band image frames, so a + // browser redraws on its next transmit (see issue #213's reattach note). + self.images.clear(); let reader = Self::spawn_reader( self.term.clone(), @@ -352,6 +369,7 @@ impl RemoteTerminal { auth: self.auth_prompts.clone(), phase: self.ssh_phase.clone(), marks: self.marks.clone(), + images: self.images.clone(), }, ); if let Ok(mut writer) = self.writer.lock() { @@ -400,6 +418,7 @@ impl RemoteTerminal { Arc::new(Mutex::new(VecDeque::new())); let ssh_phase: Arc>> = Arc::new(Mutex::new(None)); let marks = crate::terminal::marks::Marks::new(); + let images = crate::terminal::images::ImageStore::new(); let reader_thread = Self::spawn_reader( term.clone(), @@ -419,6 +438,7 @@ impl RemoteTerminal { auth: auth_prompts.clone(), phase: ssh_phase.clone(), marks: marks.clone(), + images: images.clone(), }, ); @@ -429,6 +449,7 @@ impl RemoteTerminal { exited: false, size, synced_size: false, + synced_cell: (0, 0), writer: Mutex::new(write_half), cwd, shell_state, @@ -444,6 +465,7 @@ impl RemoteTerminal { agent, agent_session, marks, + images, route: PaneRoute::Local, proxy, reader_thread: Some(reader_thread), @@ -489,6 +511,7 @@ impl RemoteTerminal { auth, phase, marks, + images, } = signals; crate::core::threads::promote_to_user_interactive(); let mut stream = read_half; @@ -499,6 +522,19 @@ impl RemoteTerminal { let mut mark_scan = MarkScanner::new(); let mut pending: Vec = buffered; let mut pending_size: Option = None; + // Kitty-graphics decode runs on its own thread with newest-frame + // coalescing (issue #213): inflating a full-window browser frame + // is ~42 ms, and doing it inline here would block PTY output and + // scrolling for that long every frame. The worker owns the inflate + // + BGRA swap + placement; the reader only captures the grid + // anchor and hands off the still-compressed frame. Dropped when + // the loop ends, which joins the thread. + let image_decoder = { + let proxy = proxy.clone(); + crate::terminal::images::DecodeWorker::spawn(images.clone(), move || { + proxy.send_event(AlacEvent::Wakeup); + }) + }; let mut scratch = vec![0u8; 256 * 1024]; let trace = std::env::var("TTY7_TRACE").is_ok_and(|v| !v.is_empty() && v != "0"); @@ -633,6 +669,59 @@ impl RemoteTerminal { out_batch.extend_from_slice(&bytes); tr_frames += 1; } + // Kitty graphics (issue #213): the daemon lifted an + // image out of the stream and forwarded it out-of-band, + // interleaved *in stream order* with the Output frames + // around it. Flush the pending text first so the grid + // cursor sits where the sender drew the image, then + // anchor the placement to that cell in scroll-stable + // absolute-row coordinates (the same formula + // `record_mark` uses), so it tracks scrolling. + DaemonMsg::Image(frame) => { + flush_batch!(); + if let Some(img) = + tty7_core::core::kitty_graphics::Image::decode_frame(&frame) + { + // Capture the anchor *now*, at the cursor cell + // the transmission arrived on; the decode is + // deferred to the worker thread but must land + // at this position, not wherever the cursor has + // scrolled to by the time inflate finishes. + let (anchor_row, anchor_col) = { + use alacritty_terminal::grid::Dimensions as _; + let term = term.lock(); + let grid = term.grid(); + let row = grid.history_size() as i64 + - grid.display_offset() as i64 + + i64::from(grid.cursor.point.line.0); + (row, grid.cursor.point.column.0) + }; + // Hand off without blocking the reader: the + // worker inflates, swaps, places, and wakes the + // view. Stale frames coalesce away there. + image_decoder.submit( + crate::terminal::images::PendingFrame { + img, + anchor_row, + anchor_col, + }, + ); + } + } + // An `a=d` delete, lifted out the same way. Order with + // the surrounding output does not matter for a delete + // (it targets by id/placement, not cursor position), + // but flushing keeps a delete-then-retransmit in the + // same read from racing its own replacement. + DaemonMsg::DeleteImage(sel) => { + flush_batch!(); + if let Some(del) = + tty7_core::core::kitty_graphics::ImageDelete::decode(&sel) + { + images.delete(&del); + proxy.send_event(AlacEvent::Wakeup); + } + } DaemonMsg::Cwd(path) => { flush_batch!(); if let Ok(mut guard) = cwd.lock() { @@ -798,7 +887,13 @@ impl RemoteTerminal { } pub fn resize(&mut self, size: TermSize, cell_w: u16, cell_h: u16) { - if self.synced_size && size == self.size { + // The cell size has to be part of the early-out, not just cols/rows: it + // is reported in *device* pixels, so moving the window between a 2x and + // a 1x display changes `ws_xpixel`/`ws_ypixel` while the grid stays + // exactly the same. Comparing only `size` there would skip the resize + // and leave a pixel-aware child rendering for the old framebuffer. + let cell = (cell_w, cell_h); + if self.synced_size && size == self.size && cell == self.synced_cell { use alacritty_terminal::grid::Dimensions as _; let term = self.term.lock(); if term.columns() == size.cols && term.screen_lines() == size.rows { @@ -807,6 +902,7 @@ impl RemoteTerminal { } self.synced_size = true; self.size = size; + self.synced_cell = cell; self.term.lock().resize(size); let win = win_size(size, cell_w, cell_h); @@ -854,6 +950,13 @@ impl RemoteTerminal { self.marks.clone() } + /// The kitty-graphics image store for this pane. Cheap handle clone — the + /// store is an `Arc>` shared with the reader thread, which places + /// and deletes images as out-of-band frames arrive from the daemon. + pub fn images(&self) -> crate::terminal::images::ImageStore { + self.images.clone() + } + pub fn agent_session(&self) -> Option { self.agent_session.lock().ok().and_then(|g| g.clone()) } diff --git a/src/terminal/view.rs b/src/terminal/view.rs index f3cad284..0f3ffda9 100644 --- a/src/terminal/view.rs +++ b/src/terminal/view.rs @@ -772,6 +772,7 @@ impl TerminalView { rows: usize, cell_width: Pixels, line_height: Pixels, + scale: f32, ) { if (cols, rows) != (self.terminal.size().cols, self.terminal.size().rows) { self.last_hover_cell = None; @@ -779,10 +780,23 @@ impl TerminalView { } self.cell_width = cell_width; self.line_height = line_height; + // Report the cell size to the child in *device* pixels (logical × display + // scale), so `ws_xpixel`/`ws_ypixel` describe the real framebuffer. A + // pixel-aware program like terminal-browser renders its frame at that + // native resolution; painted back into logical-pixel bounds, gpui blits + // it ~1:1 on the framebuffer instead of upscaling a half-resolution + // bitmap (which looked soft and magnified on Retina). This is what + // kitty/ghostty report. `self.cell_width` stays logical — glyph layout + // and mouse mapping work in logical pixels. + let scale = if scale.is_finite() && scale > 0. { + scale + } else { + 1. + }; self.terminal.resize( TermSize::new(cols, rows), - cell_width.as_f32().round() as u16, - line_height.as_f32().round() as u16, + (cell_width.as_f32() * scale).round().max(1.) as u16, + (line_height.as_f32() * scale).round().max(1.) as u16, ); } @@ -5698,16 +5712,16 @@ mod gpui_tests { let (window, _daemon) = harness(cx); window .update(cx, |view, _, cx| { - view.set_grid_size(80, 24, px(8.), px(17.)); + view.set_grid_size(80, 24, px(8.), px(17.), 1.); view.hover_link_at(0, 23, true, cx); assert_eq!(view.last_hover_cell, Some((0, 23))); view.hovered_link = Some(HoveredLink { start: Point::new(Line(23), Column(0)), end: Point::new(Line(23), Column(3)), }); - view.set_grid_size(80, 24, px(8.), px(17.)); + view.set_grid_size(80, 24, px(8.), px(17.), 1.); assert_eq!(view.last_hover_cell, Some((0, 23))); - view.set_grid_size(80, 8, px(8.), px(17.)); + view.set_grid_size(80, 8, px(8.), px(17.), 1.); assert!(view.last_hover_cell.is_none(), "the cell is stale"); assert!(view.hovered_link.is_none(), "so is the link it resolved"); })