starred/tty7
1
0
Fork 0
mirror of https://github.com/l0ng-ai/tty7.git synced 2026-09-21 16:02:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge: main — kitty graphics fans out to observers too

Browse Source 
Two conflicts, both where main's graphics work and this branch's observer work
touched the same lines.

daemon/protocol.rs: both sides appended frame kinds. INPUT_ACK (51) and
IMAGE/DELETE_IMAGE (60/61) do not collide; both kept.

daemon/pane.rs: main taught the reader to forward a chunk as an ordered
GraphicsFrame sequence instead of one Output, so an image lands at the cursor
cell the sender drew it at. This branch had lifted the same send into
fan_out_output, which also feeds read-only observers and holds each to its
budget. fan_out_output now takes the frame sequence: the no-graphics fast path
still sends one Output, and Image frames reach observers as well, gated on their
own length. A Delete selector rides `notify`, which is ungated but still drops
an observer that has stopped draining — matching the drain accounting in
server.rs.

An observer is a read-only mirror of the pane, so it sees images for the same
reason it sees text.
This commit is contained in:
l0ng-ai
2026-07-31 13:59:36 +08:00
parent 3b68a42cb8 83d9a1c547
commit a61bd486d3
12 changed files with 3248 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+1
View File
@@ -9715,6 +9715,7 @@ dependencies = [
"libgssapi",
"log",
"memchr",
"miniz_oxide",
"notify 8.2.0",
"portable-pty",
"russh",
crates/tty7-core/Cargo.toml
+6
View File
@@ -112,6 +112,12 @@ memchr = "2"
# roughly 4 MB. Base64 costs 1.33× instead. Already in the tree via russh.
base64 = "0.22"
# zlib inflate for the kitty graphics protocol's `o=z` (compressed) pixel
# payloads (`core::kitty_graphics`). Pure-Rust, no new native code, and already
# in the tree transitively — the same crate `terminal-browser` uses for the
# deflate side, so the two ends agree on the wire format.
miniz_oxide = "0.8"
[target.'cfg(unix)'.dependencies]
libc = "0.2"
# GSSAPI/Kerberos SSH auth — see the `gssapi` feature below for why it is
crates/tty7-core/src/core/kitty_graphics.rs
+1937
View File
File diff suppressed because it is too large Load Diff
crates/tty7-core/src/core/mod.rs
+1
View File
@@ -6,6 +6,7 @@ pub mod git;
pub mod gitignore;
#[allow(dead_code)]
pub mod keychain;
pub mod kitty_graphics;
pub mod logfile;
pub mod machine;
pub mod osc;
crates/tty7-core/src/daemon/pane.rs
+319 -16
View File
@@ -9,9 +9,11 @@ use std::time::Duration;
use portable_pty::{Child, CommandBuilder, MasterPty, PtySize, native_pty_system};
use crate::core::kitty_graphics::{GraphicsSniffer, Segment, Sniffed};
use crate::core::osc::OscTokenizer;
use crate::daemon::protocol::{
AuthResponse, DaemonMsg, NativeSshSpec, PaneInfo, RemoteContext, RemoteKind, ShellSpec, WinSize,
AuthResponse, DaemonMsg, MAX_FRAME, NativeSshSpec, PaneInfo, RemoteContext, RemoteKind,
ShellSpec, WinSize,
};
use crate::daemon::shell_integration;
@@ -466,24 +468,64 @@ fn notify(st: &mut PaneState, msg: DaemonMsg) {
});
}
fn fan_out_output(st: &mut PaneState, bytes: &[u8], gate: &OutputGate) {
/// One gated message to the controller and to every observer.
///
/// A send error just means that client is gone; ignore it and let the next
/// attach install a new sender. Successful sends are counted against the
/// pane gate, and the connection's writer thread credits them back. Observers
/// each carry their own gate and their own budget: one that has stopped
/// draining is dropped rather than allowed to hold the pane's output forever.
fn fan_out_one(st: &mut PaneState, msg: DaemonMsg, len: usize, gate: &OutputGate) {
if let Some(sub) = &st.subscriber {
if sub.send(DaemonMsg::Output(bytes.to_vec())).is_ok() {
gate.add(bytes.len());
if sub.send(msg.clone()).is_ok() {
gate.add(len);
}
}
st.observers.retain(|obs| {
if obs.gate.queued_bytes() + bytes.len() as i64 > OBSERVER_BUDGET {
if obs.gate.queued_bytes() + len as i64 > OBSERVER_BUDGET {
return false;
}
if obs.tx.send(DaemonMsg::Output(bytes.to_vec())).is_err() {
if obs.tx.send(msg.clone()).is_err() {
return false;
}
obs.gate.add(bytes.len());
obs.gate.add(len);
true
});
}
/// Fan one PTY read out to the controller and every observer.
///
/// On the no-graphics fast path `frames` is empty and the whole passthrough
/// goes as a single `Output`. When a chunk carried graphics the frames are
/// forwarded in stream order instead, so an image lands at the same cursor cell
/// the sender drew it at. Each `Output` is gated on its own length; an image
/// frame is gated too (it rode the same PTY read); a delete is tiny and
/// ungated, matching the drain accounting in `server.rs`.
fn fan_out_output(st: &mut PaneState, bytes: &[u8], frames: Vec<GraphicsFrame>, gate: &OutputGate) {
if frames.is_empty() {
if !bytes.is_empty() {
fan_out_one(st, DaemonMsg::Output(bytes.to_vec()), bytes.len(), gate);
}
return;
}
for frame in frames {
match frame {
GraphicsFrame::Output(b) => {
if !b.is_empty() {
let len = b.len();
fan_out_one(st, DaemonMsg::Output(b), len, gate);
}
}
GraphicsFrame::Image(frame) => {
let len = frame.len();
fan_out_one(st, DaemonMsg::Image(frame), len, gate);
}
// Ungated, and `notify` already holds observers to their budget.
GraphicsFrame::Delete(sel) => notify(st, DaemonMsg::DeleteImage(sel)),
}
}
}
enum PaneBackend {
Pty(PtyBackend),
NativeSsh(NativeSshBackend),
@@ -512,7 +554,13 @@ pub struct DaemonPane {
pub id: u64,
owner: Option<String>,
backend: PaneBackend,
writer: Mutex<Box<dyn Write + Send>>,
/// The input side (keyboard input / pasted text): the PTY writer, or the
/// native-SSH channel writer. Behind a `Mutex` because writes can arrive from
/// different connection threads. `Arc`-shared so the reader thread can write
/// kitty-graphics query replies (`\x1b_G…;OK\x1b\\`) back to the PTY inline
/// with the sniff, without routing through `write_input`.
writer: Arc<Mutex<Box<dyn Write + Send>>>,
/// Set during teardown so the reader doesn't emit a spurious exit.
shutting_down: Arc<AtomicBool>,
gate: Arc<OutputGate>,
state: Arc<Mutex<PaneState>>,
@@ -571,6 +619,32 @@ impl DeathReporter {
}
}
/// One out-of-band frame the reader forwards to the subscriber, kept in stream
/// order so a kitty image lands at the cursor cell the sender drew it at: a chunk
/// carrying graphics splits into `Output` runs interleaved with `Image`/`Delete`
/// frames, and they must reach the client in that same order. `Output` becomes a
/// `DaemonMsg::Output`, `Image` an encoded image frame, `Delete` a compact
/// selector — see the reader loop's `sniff` handling.
enum GraphicsFrame {
Output(Vec<u8>),
Image(Vec<u8>),
Delete(Vec<u8>),
}
/// Queue an encoded image frame for the subscriber, dropping any frame larger
/// than [`MAX_FRAME`]. The writer's `write_frame` rejects an oversize payload
/// with an error the writer loop treats as *fatal* — it would tear the client
/// off an otherwise-healthy pane. A single image that won't fit is not worth
/// that: drop it here so the rest of the stream keeps flowing. The inline
/// `t=d` path is bounded by `MAX_TRANSMISSION_BASE64` but that still admits
/// ~72 MiB of raw pixels, and the local shm/file path has no cap of its own,
/// so both can land here over the limit.
fn push_image_frame(frames: &mut Vec<GraphicsFrame>, frame: Vec<u8>) {
if frame.len() <= MAX_FRAME {
frames.push(GraphicsFrame::Image(frame));
}
}
impl DaemonPane {
pub fn spawn(
id: u64,
@@ -593,7 +667,7 @@ impl DaemonPane {
drop(pair.slave);
let reader_handle = pair.master.try_clone_reader()?;
let writer = pair.master.take_writer()?;
let writer = Arc::new(Mutex::new(pair.master.take_writer()?));
let state = Arc::new(Mutex::new(PaneState {
id,
@@ -625,7 +699,7 @@ impl DaemonPane {
shell_pid,
integration_dir: spawn.integration_dir,
}),
writer: Mutex::new(writer),
writer: writer.clone(),
shutting_down: shutting_down.clone(),
gate: gate.clone(),
state: state.clone(),
@@ -677,6 +751,7 @@ impl DaemonPane {
shutting_down,
gate,
reader_handle,
writer.clone(),
move || foreground_command_running(&fg_master, shell_pid),
ForegroundProbes {
remote: Box::new(move || foreground_remote_context(&remote_master)),
@@ -698,7 +773,10 @@ impl DaemonPane {
) -> anyhow::Result<Arc<Self>> {
let bridge = crate::daemon::ssh::session::make_bridge();
let reader_handle: Box<dyn Read + Send> = Box::new(bridge.reader);
let writer: Box<dyn Write + Send> = Box::new(bridge.writer);
let writer: Arc<Mutex<Box<dyn Write + Send>>> =
Arc::new(Mutex::new(Box::new(bridge.writer)));
// The connect task fills this once authenticated; the pane exposes it to
// WS4/WS5 via `ssh_connection()`.
let connection: crate::daemon::ssh::SharedConnection = Arc::new(Mutex::new(Weak::new()));
let target = spec
@@ -747,7 +825,7 @@ impl DaemonPane {
handle: bridge.handle,
connection: connection.clone(),
}),
writer: Mutex::new(writer),
writer: writer.clone(),
shutting_down: shutting_down.clone(),
gate: gate.clone(),
state: state.clone(),
@@ -762,6 +840,7 @@ impl DaemonPane {
shutting_down,
gate,
reader_handle,
writer.clone(),
|| false,
ForegroundProbes {
remote: Box::new(|| None),
@@ -804,6 +883,10 @@ impl DaemonPane {
shutting_down: Arc<AtomicBool>,
gate: Arc<OutputGate>,
mut reader: Box<dyn Read + Send>,
// The pane's input side, shared so the reader can write kitty-graphics
// query replies (`\x1b_G…;OK\x1b\\`) straight back to the PTY — see the
// graphics sniff in the loop below.
writer: Arc<Mutex<Box<dyn Write + Send>>>,
foreground_running: impl Fn() -> bool + Send + 'static,
probes: ForegroundProbes,
death: Arc<DeathReporter>,
@@ -818,6 +901,20 @@ impl DaemonPane {
.spawn(move || {
crate::core::threads::promote_to_user_interactive();
let mut sniffer = OscSniffer::new();
// Kitty graphics interception (issue #213): lifts image
// sequences out of the stream *before* the ring/subscriber see
// them, so the base64 pixels never enter replay and the client's
// VT parser never chews through them. Zero-copy on the common
// no-graphics chunk — see [`GraphicsSniffer::sniff`].
//
// File/shm transfer (`t=s`/`t=f`/`t=t`) is honored only while the
// pane is *local*: the object/path names resolve on this host, and
// reading them can't leak across an SSH tunnel. A pane that starts
// local can `ssh` out mid-session, so the flag is refreshed from
// the same remote-context poll below. Seed it from the pane's
// current context so the first probe answers correctly.
let starts_local = state.lock().unwrap().remote.is_none();
let mut graphics = GraphicsSniffer::new_local(starts_local);
let mut buf = [0u8; 65536];
let trace = std::env::var("TTY7_TRACE").is_ok_and(|v| !v.is_empty() && v != "0");
@@ -854,7 +951,71 @@ impl DaemonPane {
tr_reads += 1;
tr_bytes += n as u64;
}
let bytes = &buf[..n];
let raw = &buf[..n];
// Kitty graphics: strip any image sequences out of the
// stream before anything else sees them. On the common
// no-graphics chunk this borrows `raw` unchanged; only a
// chunk that actually carries `\x1b_G…` allocates. Query
// replies are written straight back to the PTY here, and
// any images/deletes are forwarded to the subscriber
// out-of-band below — none of it enters the replay ring.
//
// `frames` is the ordered list of out-of-band frames to
// forward *in stream position*: a kitty image anchors to
// the cursor cell as it stood when its command appeared,
// so the client must apply the text before an image, then
// the image, then the text after it, in that order. On
// the no-graphics fast path `frames` stays empty and the
// whole chunk is sent as one `Output`; only a chunk with
// graphics splits into interleaved frames.
let mut frames: Vec<GraphicsFrame> = Vec::new();
let passthrough: std::borrow::Cow<[u8]> = match graphics.sniff(raw) {
Sniffed::Plain(b) => std::borrow::Cow::Borrowed(b),
Sniffed::Segments(segs) => {
let mut pass = Vec::new();
for seg in segs {
match seg {
Segment::Output(b) => {
pass.extend_from_slice(&b);
frames.push(GraphicsFrame::Output(b));
}
Segment::Query(reply) => {
if let Ok(mut w) = writer.lock() {
let _ = w.write_all(&reply);
let _ = w.flush();
}
}
Segment::Image(img) => {
push_image_frame(&mut frames, img.encode_frame());
}
Segment::ImageFromMedium(transfer) => {
// File/shm handoff on a local pane:
// read (and unlink) the object here,
// then forward the raw pixels exactly
// like an inline image. This is the
// fast path that skips the client-side
// inflate the compressed-inline `t=d`
// fallback would force. A failed read
// just drops the frame — the sender
// reclaims its own object.
if let Some(img) = transfer.resolve() {
push_image_frame(
&mut frames,
img.encode_frame(),
);
}
}
Segment::Delete(d) => {
frames.push(GraphicsFrame::Delete(d.encode()));
}
}
}
std::borrow::Cow::Owned(pass)
}
};
let bytes: &[u8] = &passthrough;
// Sniff first (cheap, over the same bytes); collect any
// cwd/prompt change to emit while we hold the lock.
let mut signals = sniffer.feed(bytes);
if signals.shell.iter().any(|s| s.at_prompt) && foreground_running() {
@@ -894,11 +1055,16 @@ impl DaemonPane {
let mut st = state.lock().unwrap();
let facts_before = may_change_facts.then(|| observed_facts(&st));
st.ring.append(bytes);
fan_out_output(&mut st, bytes, &gate);
fan_out_output(&mut st, bytes, frames, &gate);
apply_signals(&mut st, signals);
if let Some(remote) = remote {
apply_remote_context(&mut st, remote);
}
// Keep kitty file/shm transfer gated on the pane's
// *current* locality: an `ssh` that just took the PTY
// must stop us honoring host-local object names. Cheap
// and only meaningful when a probe follows.
graphics.set_local(st.remote.is_none());
if let Some(agent) = agent {
apply_agent(&mut st, agent);
}
@@ -1882,6 +2048,7 @@ fn proc_name(pid: i32) -> Option<String> {
#[cfg(test)]
mod tests {
use super::*;
use crate::core::kitty_graphics::ImageDelete;
use std::path::Path;
#[test]
@@ -2672,6 +2839,12 @@ mod tests {
assert_eq!(d.shell.last().unwrap().last_exit_code, Some(-1));
}
/// A discard writer for `spawn_reader` tests that don't inspect the PTY
/// write-back path (graphics query replies).
fn null_writer() -> Arc<Mutex<Box<dyn Write + Send>>> {
Arc::new(Mutex::new(Box::new(std::io::sink())))
}
fn test_state(alive: bool) -> PaneState {
PaneState {
id: 0,
@@ -2775,6 +2948,7 @@ mod tests {
Arc::new(AtomicBool::new(shutting_down)),
Arc::new(OutputGate::new()),
Box::new(std::io::Cursor::new(b"\x1b]133;D;0\x07".to_vec())),
null_writer(),
|| false,
ForegroundProbes {
remote: Box::new(|| None),
@@ -3138,7 +3312,7 @@ mod tests {
let chunk = vec![b'x'; 1024 * 1024];
let sends = (OBSERVER_BUDGET / chunk.len() as i64) as usize + 2;
for _ in 0..sends {
fan_out_output(&mut st, &chunk, &pane_gate);
fan_out_output(&mut st, &chunk, Vec::new(), &pane_gate);
pane_gate.sub(chunk.len());
}
@@ -3178,7 +3352,7 @@ mod tests {
let sends = (OBSERVER_BUDGET / chunk.len() as i64) as usize * 3;
let mut got = 0usize;
for _ in 0..sends {
fan_out_output(&mut st, &chunk, &pane_gate);
fan_out_output(&mut st, &chunk, Vec::new(), &pane_gate);
pane_gate.sub(chunk.len());
while let Ok(DaemonMsg::Output(b)) = observer_rx.try_recv() {
observer_gate.sub(b.len());
@@ -3282,6 +3456,7 @@ mod tests {
Arc::new(AtomicBool::new(false)),
Arc::new(OutputGate::new()),
Box::new(std::io::Cursor::new(b"tail".to_vec())),
null_writer(),
|| false,
ForegroundProbes {
remote: Box::new(|| None),
@@ -3307,6 +3482,131 @@ mod tests {
);
}
/// Issue #213 end-to-end at the reader: a chunk carrying text plus a
/// kitty graphics query and a transmit-and-display must (1) keep only the
/// text in the replay ring and the `Output` frame, (2) write the `a=q` reply
/// back to the PTY writer, and (3) forward the image out-of-band as an
/// `Image` frame the client can decode.
#[test]
fn reader_strips_graphics_and_forwards_them_out_of_band() {
use crate::core::kitty_graphics::Image;
use base64::Engine as _;
let state = Arc::new(Mutex::new(test_state(true)));
let (sub_tx, sub_rx) = mpsc::channel();
state.lock().unwrap().subscriber = Some(sub_tx);
// A writer we can read back, to prove the query reply reached the PTY.
#[derive(Clone)]
struct SharedBuf(Arc<Mutex<Vec<u8>>>);
impl Write for SharedBuf {
fn write(&mut self, b: &[u8]) -> std::io::Result<usize> {
self.0.lock().unwrap().extend_from_slice(b);
Ok(b.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
let sink = Arc::new(Mutex::new(Vec::new()));
let writer: Arc<Mutex<Box<dyn Write + Send>>> =
Arc::new(Mutex::new(Box::new(SharedBuf(sink.clone()))));
// One 1x1 opaque-red RGBA pixel, transmitted-and-displayed inline.
let pixel = [0xffu8, 0x00, 0x00, 0xff];
let b64 = base64::engine::general_purpose::STANDARD.encode(pixel);
let stream = format!(
"before\x1b_Gi=1,a=q,t=d,f=32,s=1,v=1;AAAA\x1b\\\
mid\x1b_Ga=T,f=32,t=d,s=1,v=1,i=1;{b64}\x1b\\after"
);
let handle = DaemonPane::spawn_reader(
state.clone(),
Arc::new(AtomicBool::new(false)),
Arc::new(OutputGate::new()),
Box::new(std::io::Cursor::new(stream.into_bytes())),
writer,
|| false,
ForegroundProbes {
remote: Box::new(|| None),
agent: Box::new(|| None),
cwd: Box::new(|| None),
},
Arc::new(DeathReporter::new(|| {})),
);
handle.join().unwrap();
// (1) The ring holds the text, none of the graphics bytes.
assert_eq!(state.lock().unwrap().ring.flatten(), b"beforemidafter");
// (2) The query reply went to the PTY writer.
assert_eq!(sink.lock().unwrap().as_slice(), b"\x1b_Gi=1;OK\x1b\\");
// (3) The subscriber sees the passthrough and the image *in stream order*:
// the text before the image, then the image at its cursor cell, then the
// text after it. The `a=q` reply splits "before" from "mid" into two
// Output frames; the image sits between "mid" and "after".
assert!(matches!(sub_rx.try_recv(), Ok(DaemonMsg::Output(b)) if b == b"before"));
assert!(matches!(sub_rx.try_recv(), Ok(DaemonMsg::Output(b)) if b == b"mid"));
match sub_rx.try_recv() {
Ok(DaemonMsg::Image(frame)) => {
let img = Image::decode_frame(&frame).expect("decodable image frame");
assert_eq!(img.id, 1);
assert_eq!((img.width, img.height), (1, 1));
assert_eq!(img.to_rgba8().unwrap(), pixel);
}
other => panic!("expected Image frame, got {other:?}"),
}
assert!(matches!(sub_rx.try_recv(), Ok(DaemonMsg::Output(b)) if b == b"after"));
}
/// A kitty delete (`a=d`) is lifted out and forwarded as a `DeleteImage`
/// selector, leaving the surrounding text intact in the ring.
#[test]
fn reader_forwards_graphics_deletes() {
let state = Arc::new(Mutex::new(test_state(true)));
let (sub_tx, sub_rx) = mpsc::channel();
state.lock().unwrap().subscriber = Some(sub_tx);
let handle = DaemonPane::spawn_reader(
state.clone(),
Arc::new(AtomicBool::new(false)),
Arc::new(OutputGate::new()),
Box::new(std::io::Cursor::new(b"x\x1b_Ga=d,d=A\x1b\\y".to_vec())),
null_writer(),
|| false,
ForegroundProbes {
remote: Box::new(|| None),
agent: Box::new(|| None),
cwd: Box::new(|| None),
},
Arc::new(DeathReporter::new(|| {})),
);
handle.join().unwrap();
assert_eq!(state.lock().unwrap().ring.flatten(), b"xy");
// In stream order: text before the delete, the delete, then text after.
assert!(matches!(sub_rx.try_recv(), Ok(DaemonMsg::Output(b)) if b == b"x"));
match sub_rx.try_recv() {
Ok(DaemonMsg::DeleteImage(sel)) => {
let d = ImageDelete::decode(&sel).unwrap();
assert_eq!(d.target, b'A');
}
other => panic!("expected DeleteImage, got {other:?}"),
}
assert!(matches!(sub_rx.try_recv(), Ok(DaemonMsg::Output(b)) if b == b"y"));
}
#[test]
fn push_image_frame_drops_over_max_frame() {
let mut frames = Vec::new();
// At the limit: queued.
push_image_frame(&mut frames, vec![0u8; MAX_FRAME]);
// Over the limit: dropped, so the writer's fatal `write_frame` error
// (which would disconnect the client) is never reached.
push_image_frame(&mut frames, vec![0u8; MAX_FRAME + 1]);
assert_eq!(frames.len(), 1, "only the in-budget frame is queued");
assert!(matches!(&frames[0], GraphicsFrame::Image(f) if f.len() == MAX_FRAME));
}
#[test]
fn reader_poll_applies_the_probed_cwd() {
let state = Arc::new(Mutex::new(test_state(true)));
@@ -3322,6 +3622,7 @@ mod tests {
Arc::new(AtomicBool::new(false)),
Arc::new(OutputGate::new()),
Box::new(std::io::Cursor::new(b"alice@host ~ % ".to_vec())),
null_writer(),
|| false,
ForegroundProbes {
remote: Box::new(|| None),
@@ -3352,6 +3653,7 @@ mod tests {
Arc::new(AtomicBool::new(false)),
Arc::new(OutputGate::new()),
Box::new(std::io::Cursor::new(Vec::new())),
null_writer(),
|| false,
ForegroundProbes {
remote: Box::new(|| None),
@@ -3377,6 +3679,7 @@ mod tests {
Arc::new(AtomicBool::new(true)),
Arc::new(OutputGate::new()),
Box::new(std::io::Cursor::new(Vec::new())),
null_writer(),
|| false,
ForegroundProbes {
remote: Box::new(|| None),
crates/tty7-core/src/daemon/protocol.rs
+23
View File
@@ -650,6 +650,18 @@ pub enum DaemonMsg {
Size(WinSize),
Snapshot(Vec<u8>),
Output(Vec<u8>),
/// A kitty graphics image lifted out of the PTY stream daemon-side (issue
/// #213). Carried out-of-band as a compact binary frame
/// ([`crate::core::kitty_graphics::Image::encode_frame`]) so the base64 text
/// never rides the socket and the client's VT parser never sees it. The
/// pixel payload stays *compressed* on the wire — the client inflates — so a
/// remote pane's frames don't balloon across the SSH tunnel.
Image(Vec<u8>),
/// A kitty graphics delete (`a=d`) lifted out of the PTY stream daemon-side.
/// Payload is a compact selector frame
/// ([`crate::core::kitty_graphics::ImageDelete::encode`]) telling the client
/// which stored image(s)/placement(s) to drop.
DeleteImage(Vec<u8>),
Cwd(PathBuf),
Prompt {
active: bool,
@@ -745,6 +757,13 @@ mod kind {
pub const VERSION_REPLY: u8 = 40;
pub const PROCS: u8 = 50;
pub const INPUT_ACK: u8 = 51;
/// `Image` — a kitty graphics frame lifted out of the PTY stream (issue
/// #213). 60 sits clear of every range above; the payload is the compact
/// binary encoding, not JSON, so it stays outside the `to_json` arms.
pub const IMAGE: u8 = 60;
/// `DeleteImage` — a kitty graphics `a=d` delete lifted out of the stream
/// (issue #213). Compact binary selector, like `IMAGE`.
pub const DELETE_IMAGE: u8 = 61;
}
pub fn write_frame<W: Write>(w: &mut W, kind: u8, payload: &[u8]) -> io::Result<()> {
@@ -1058,6 +1077,8 @@ impl DaemonMsg {
DaemonMsg::Size(size) => write_frame(w, kind::SIZE, &to_json(size)?),
DaemonMsg::Snapshot(bytes) => write_frame(w, kind::SNAPSHOT, bytes),
DaemonMsg::Output(bytes) => write_frame(w, kind::OUTPUT, bytes),
DaemonMsg::Image(frame) => write_frame(w, kind::IMAGE, frame),
DaemonMsg::DeleteImage(sel) => write_frame(w, kind::DELETE_IMAGE, sel),
DaemonMsg::Cwd(path) => write_frame(w, kind::CWD, &to_json(path)?),
DaemonMsg::Prompt {
active,
@@ -1112,6 +1133,8 @@ impl DaemonMsg {
kind::SIZE => DaemonMsg::Size(from_json(&payload)?),
kind::SNAPSHOT => DaemonMsg::Snapshot(payload),
kind::OUTPUT => DaemonMsg::Output(payload),
kind::IMAGE => DaemonMsg::Image(payload),
kind::DELETE_IMAGE => DaemonMsg::DeleteImage(payload),
kind::CWD => DaemonMsg::Cwd(from_json(&payload)?),
kind::PROMPT => {
let (active, at_prompt, last_exit) = from_json(&payload)?;
crates/tty7-core/src/daemon/server.rs
+4
View File
@@ -790,6 +790,10 @@ fn spawn_writer(
};
let drained = match &msg {
DaemonMsg::Output(b) => b.len(),
// Image frames are lifted from the same PTY read the gate
// credits, so they must debit it too or the reader stays
// throttled against bytes that already left the queue.
DaemonMsg::Image(b) => b.len(),
_ => 0,
};
let write_ok = msg.encode(&mut write_stream).is_ok();
src/terminal/element.rs
+86 -5
View File
@@ -7,10 +7,10 @@ use alacritty_terminal::selection::SelectionRange;
use alacritty_terminal::term::cell::{Cell, Flags};
use alacritty_terminal::vte::ansi::{Color as AnsiColor, CursorShape, NamedColor, Rgb};
use gpui::{
App, BorderStyle, Bounds, ContentMask, CursorStyle, Element, ElementId, Font, FontStyle,
FontWeight, GlobalElementId, Hitbox, HitboxBehavior, HitboxId, Hsla, IntoElement, LayoutId,
MouseButton, MouseDownEvent, MouseMoveEvent, MouseUpEvent, Pixels, Point, Rgba, SharedString,
Style, TextAlign, TextRun, Window, fill, outline, point, px, relative, size,
App, BorderStyle, Bounds, ContentMask, Corners, CursorStyle, Element, ElementId, Font,
FontStyle, FontWeight, GlobalElementId, Hitbox, HitboxBehavior, HitboxId, Hsla, IntoElement,
LayoutId, MouseButton, MouseDownEvent, MouseMoveEvent, MouseUpEvent, Pixels, Point, Rgba,
SharedString, Style, TextAlign, TextRun, Window, fill, outline, point, px, relative, size,
};
use gpui_component::ActiveTheme as _;
@@ -823,6 +823,11 @@ struct GridSnapshot {
any_selected: bool,
any_match: bool,
any_current: bool,
/// Scrollback state at snapshot time, so the paint pass can map a kitty
/// image's absolute anchor row back to a screen row: screen_row =
/// anchor_row - history_size + display_offset.
display_offset: i32,
history_size: usize,
}
impl TerminalElement {
@@ -841,6 +846,7 @@ impl TerminalElement {
let mut sliver: Option<Vec<RenderCell>> = None;
let mut any_selected = false;
let display_offset;
let history_size;
{
let mut palette = self.view.read(cx).terminal.palette;
if let Some(active) = cx.try_global::<crate::terminal::palette::ActivePalette>() {
@@ -850,6 +856,7 @@ impl TerminalElement {
let term = term.lock();
let content = term.renderable_content();
display_offset = content.display_offset as i32;
history_size = term.grid().history_size();
let selection = content.selection;
for cell in content.display_iter {
@@ -906,6 +913,8 @@ impl TerminalElement {
any_selected,
any_match,
any_current,
display_offset,
history_size,
}
}
@@ -1175,7 +1184,7 @@ impl Element for TerminalElement {
.max(1.0) as usize;
self.view.update(cx, |view, _cx| {
view.set_grid_size(cols, rows, cell_width, line_height);
view.set_grid_size(cols, rows, cell_width, line_height, window.scale_factor());
});
let hitbox = window.insert_hitbox(bounds, HitboxBehavior::Normal);
@@ -1243,6 +1252,20 @@ impl Element for TerminalElement {
);
let marked = self.view.read(cx).marked_text.clone();
// Kitty-graphics placements for this pane. Each image anchors to an
// absolute scrollback row (recorded when its command arrived in-stream);
// map that back to a screen row with the snapshot's scroll state and
// drop anything scrolled out of the viewport.
let image_store = self.view.read(cx).terminal.images();
// Take the retired list *before* the snapshot. The decode worker runs on
// its own thread and can retire a frame between the two calls; taking
// retired second would hand us a list containing an image the snapshot
// still says to paint, and `sprite_atlas.remove` takes effect
// immediately — so the frame would paint and then vanish. In this order
// the worst case is evicting one paint late, which is invisible.
let retired_images = image_store.take_retired();
let images = image_store.snapshot();
window.with_content_mask(Some(ContentMask { bounds }), |window| {
paint_backgrounds(window, &geom, &buf);
if snap.any_selected {
@@ -1275,6 +1298,64 @@ impl Element for TerminalElement {
bold_font.as_ref(),
italic_font.as_ref(),
);
// Kitty-graphics images, painted over the placeholder cells they
// occupy. `anchor_row` is absolute (measured from the top of
// scrollback); convert it to a screen row with the same scroll state
// `build_grid` captured. Rows spanning past the viewport top/bottom
// are clipped by the surrounding content mask.
//
// Sizing: per the kitty spec, an image with no `c=`/`r=` is shown at
// its natural size — one image pixel per terminal *device* pixel. A
// sender like terminal-browser renders its frame to exactly fill the
// pixel area we reported to the child via `ws_xpixel`/`ws_ypixel`,
// which the daemon sets to `cols × round(cell_w × scale)` /
// `rows × round(cell_h × scale)` — the cell size in *device* pixels
// (see `set_grid_size`). So the frame is at device resolution; to map
// it back to a cell span we divide by that *same* device cell size,
// `round(cell_logical × scale)`. Painting the resulting cell-span
// bounds (in logical px) lets gpui blit the device-resolution bitmap
// ~1:1 on the framebuffer — sharp, and the right size — instead of
// upscaling a half-resolution one. Deriving here, not at placement,
// keeps it correct across font-size / zoom / display-scale changes.
let scale = window.scale_factor();
let scale = if scale.is_finite() && scale > 0. {
scale
} else {
1.
};
for img in &images {
let round_w = (geom.cell_width.as_f32() * scale).round().max(1.);
let round_h = (geom.line_height.as_f32() * scale).round().max(1.);
let span_cols = if img.cols > 0 {
img.cols as f32
} else {
(img.width_px as f32 / round_w).round().max(1.)
};
let span_rows = if img.rows > 0 {
img.rows as f32
} else {
(img.height_px as f32 / round_h).round().max(1.)
};
let screen_row =
img.anchor_row - snap.history_size as i64 + snap.display_offset as i64;
// Fully above or below the viewport: nothing visible to paint.
if screen_row + span_rows as i64 <= 0 || screen_row >= geom.rows as i64 {
continue;
}
let top = geom.origin.y + geom.line_height * screen_row as f32;
let left = geom.origin.x + geom.cell_width * img.anchor_col as f32;
let bounds = Bounds {
origin: point(left, top),
size: size(geom.cell_width * span_cols, geom.line_height * span_rows),
};
let _ = window.paint_image(bounds, Corners::default(), img.data.clone(), 0, false);
}
// Evict superseded / deleted frames from the sprite atlas. Without
// this a browser re-transmitting at 60fps would leak one GPU tile per
// frame, growing the atlas without bound until the compositor stalls.
for retired in &retired_images {
let _ = window.drop_image(retired.clone());
}
if let Some(row) = sliver {
let sg = CellGeom {
origin: point(geom.origin.x, geom.origin.y - geom.line_height),
src/terminal/images.rs
+573
View File
@@ -0,0 +1,573 @@
//! Client-side kitty-graphics image store (issue #213).
//!
//! The daemon lifts image transmissions out of the PTY byte stream and forwards
//! them out-of-band as compact frames (`DaemonMsg::Image` / `DaemonMsg::DeleteImage`
//! — see [`tty7_core::core::kitty_graphics`]). This module is the client end: it
//! decodes each frame into a GPUI [`RenderImage`], anchors it to the grid cell the
//! cursor sat on when the command arrived, and hands the placed images to the
//! paint path so the element can blit them over the character grid.
//!
//! # Why anchor by absolute row
//!
//! Like a command [`mark`](crate::terminal::marks), an image's position has to
//! survive scrolling. A kitty image is placed at the cursor cell as it stood when
//! its command appeared in the stream; once recorded, the grid keeps scrolling
//! under it. We store the row as an absolute index from the top of scrollback
//! (`history_size - display_offset + cursor_line`, the exact formula
//! [`record_mark`](crate::terminal::remote) uses) and convert back to a screen
//! row at paint time (`anchor_row - history_size + display_offset`, the inverse
//! [`scroll_to_mark`](crate::terminal::view::TerminalView::scroll_to_mark)
//! applies). Below the scrollback limit — where a pane spends most of its life —
//! this is exact; past it the anchor drifts by the (unobservable) discard count,
//! the same caveat marks carry, and a browser that redraws every frame corrects
//! it on the next transmit anyway.
//!
//! GPUI's sprite atlas expects **BGRA** pixels (it swaps R↔B when caching an
//! `image` crate `RgbaImage` — see `gpui::img`), so [`decode`] does the swap once
//! at ingest; the placed [`RenderImage`] is uploaded verbatim thereafter.
use std::sync::mpsc::{Receiver, Sender, TryRecvError, channel};
use std::sync::{Arc, Mutex};
use std::thread::JoinHandle;
use gpui::RenderImage;
use image::{Frame, RgbaImage};
use smallvec::SmallVec;
use tty7_core::core::kitty_graphics::{Image, ImageDelete, WireFormat};
/// One image placed on the grid, ready to blit.
#[derive(Clone)]
pub struct PlacedImage {
/// Decoded pixels as a GPUI render image (BGRA, one frame).
pub data: Arc<RenderImage>,
/// Row index from the top of scrollback at placement time — the position
/// anchor. See the module docs for when it stops being exact.
pub anchor_row: i64,
/// Column of the top-left cell.
pub anchor_col: usize,
/// Source pixel dimensions, for deriving the cell span when the sender did
/// not give an explicit one.
pub width_px: u32,
pub height_px: u32,
/// Explicit cell span the sender requested (`c=` / `r=`); 0 means "derive
/// from the pixel size and the cell size at paint time".
pub cols: u32,
pub rows: u32,
/// The kitty image id (`i=`) and placement id (`p=`), for targeted deletes.
/// `id == 0` is an anonymous image, removable only by a delete-all.
pub id: u32,
pub placement: u32,
}
/// A pane's placed images plus the retired render images awaiting atlas
/// eviction, shared between the reader thread (writer) and the paint path
/// (reader), exactly like [`Marks`](crate::terminal::marks::Marks).
///
/// `retired` is the other half of the fix for a browser that repaints at 60fps:
/// each transmitted frame becomes a fresh [`RenderImage`] with a new atlas id,
/// so the *previous* frame's GPU tile has to be dropped or the sprite atlas
/// grows without bound (see [`take_retired`](ImageStore::take_retired)). The
/// reader can't touch the atlas — that needs `&mut Window` — so it parks the
/// superseded `Arc`s here and the paint path drains and drops them.
#[derive(Default)]
struct StoreInner {
placed: Vec<PlacedImage>,
retired: Vec<Arc<RenderImage>>,
}
/// A pane's placed images, shared between the reader thread (writer) and the
/// paint path (reader), exactly like [`Marks`](crate::terminal::marks::Marks).
#[derive(Clone, Default)]
pub struct ImageStore(Arc<Mutex<StoreInner>>);
/// Cap on placed images retained at once. A browser deletes-then-transmits every
/// frame, so the live set is tiny; this only bounds a sender that transmits
/// without ever deleting, dropping the oldest rather than growing without limit.
const MAX_IMAGES: usize = 256;
impl ImageStore {
pub fn new() -> Self {
Self::default()
}
/// Place a freshly received image at (`anchor_row`, `anchor_col`). A new
/// transmission with the same identity as an existing one replaces it in
/// place (kitty reuses an id to update an image); otherwise it is appended.
/// The replaced frame's render image is retired for atlas eviction.
pub fn place(&self, img: PlacedImage) {
let Ok(mut inner) = self.0.lock() else { return };
let StoreInner { placed, retired } = &mut *inner;
// Same (id, placement) → replace. An anonymous image (id 0) never
// matches, so each anonymous transmit is a distinct placement.
if img.id != 0 {
placed.retain(|p| {
let same = p.id == img.id && p.placement == img.placement;
if same {
retired.push(p.data.clone());
}
!same
});
}
placed.push(img);
let overflow = placed.len().saturating_sub(MAX_IMAGES);
if overflow > 0 {
retired.extend(placed.drain(..overflow).map(|p| p.data));
}
}
/// Apply a delete selector. Only the targets a sender tty7 faces actually
/// uses are honored (all / by id / by placement); richer kitty selectors
/// (by cell, by z-index, by number) are left in place rather than guessed.
/// Removed frames' render images are retired for atlas eviction.
pub fn delete(&self, del: &ImageDelete) {
let Ok(mut inner) = self.0.lock() else { return };
let retire = |p: &PlacedImage, keep: bool, retired: &mut Vec<Arc<RenderImage>>| {
if !keep {
retired.push(p.data.clone());
}
keep
};
let StoreInner { placed, retired } = &mut *inner;
match del.target {
// All visible placements. Case only governs whether kitty also frees
// the image data; the client frees unconditionally, so both clear.
b'a' | b'A' => {
retired.extend(placed.drain(..).map(|p| p.data));
}
// By image id.
b'i' | b'I' => placed.retain(|p| retire(p, p.id != del.id, retired)),
// By placement id (scoped to its image when an id is also given).
b'p' | b'P' => placed.retain(|p| {
let keep = p.placement != del.placement || (del.id != 0 && p.id != del.id);
retire(p, keep, retired)
}),
// A selector we don't model: leave the store untouched.
_ => {}
}
}
/// Snapshot the placed images for a paint pass. Cheap: the live set is small
/// and `PlacedImage` is a handful of fields plus an `Arc` clone.
pub fn snapshot(&self) -> Vec<PlacedImage> {
self.0.lock().map(|s| s.placed.clone()).unwrap_or_default()
}
/// Take the render images retired since the last call, for the paint path to
/// evict from the sprite atlas (`Window::drop_image`). Draining here — the
/// one place with `&mut Window` — is what stops a 60fps re-transmitting
/// sender from leaking a GPU tile per frame and dragging the compositor down.
pub fn take_retired(&self) -> Vec<Arc<RenderImage>> {
self.0
.lock()
.map(|mut s| std::mem::take(&mut s.retired))
.unwrap_or_default()
}
/// Drop everything (the grid was cleared, so every anchor is meaningless).
/// Placed frames are retired so their atlas tiles are still evicted.
pub fn clear(&self) {
if let Ok(mut inner) = self.0.lock() {
let gone: Vec<_> = inner.placed.drain(..).map(|p| p.data).collect();
inner.retired.extend(gone);
}
}
}
/// A raw (still-compressed) image frame handed to the [`DecodeWorker`], with the
/// grid anchor the reader captured the instant the transmission arrived. Keeping
/// the anchor with the frame lets decoding move off the reader thread without
/// losing the cursor position the image was drawn at.
pub struct PendingFrame {
pub img: Image,
pub anchor_row: i64,
pub anchor_col: usize,
}
/// Off-thread image decoder with newest-frame-wins coalescing — the crux of the
/// performance story (issue #213).
///
/// A full-window browser frame is ~28 MB of RGBA after zlib inflate, and the
/// inflate alone measures ~42 ms. Doing that inline on the reader thread (which
/// also services PTY output and scrolling) blocks the whole pane for ~42 ms per
/// frame, and a 60fps sender queues frames faster than they drain — latency
/// grows without bound and scrolling stutters. This is the cost the
/// device-pixel resolution bump quadrupled.
///
/// The fix mirrors how kitty/ghostty stay smooth: decode off the I/O thread, and
/// when the producer outruns the decoder, **drop stale frames instead of
/// queuing them**. A bounded [`inbox`] of one *replaceable slot per image id*
/// means a re-transmitting browser only ever has its latest frame per image
/// waiting; older undecoded frames are discarded before they cost an inflate.
/// The worker decodes the newest, `place`s it, and wakes the view.
///
/// [`inbox`]: DecodeWorker::inbox
pub struct DecodeWorker {
tx: Option<Sender<PendingFrame>>,
handle: Option<JoinHandle<()>>,
}
impl DecodeWorker {
/// Spawn the decode thread. `store` is the shared placement store the worker
/// writes decoded frames into; `wake` is called after each successful decode
/// so the view repaints (a cloned `EventProxy::send_event(Wakeup)` in
/// practice). The thread ends when the returned worker is dropped (the
/// channel closes).
pub fn spawn(store: ImageStore, wake: impl Fn() + Send + 'static) -> Self {
let (tx, rx) = channel::<PendingFrame>();
let handle = std::thread::Builder::new()
.name("tty7-image-decode".to_string())
.spawn(move || Self::run(rx, store, wake))
.ok();
Self {
tx: Some(tx),
handle,
}
}
/// Hand a raw frame to the worker. Never blocks the caller (the reader
/// thread): the frame is queued and decoded asynchronously. If the worker
/// has gone away the frame is silently dropped — the pane is tearing down.
pub fn submit(&self, frame: PendingFrame) {
if let Some(tx) = &self.tx {
let _ = tx.send(frame);
}
}
/// The decode loop. Blocks for the next frame, then **coalesces**: drains
/// everything already queued and keeps only the last frame per image id, so
/// a burst that piled up during a slow inflate collapses to one decode per
/// image. Decodes the survivors newest-first and places them.
fn run(rx: Receiver<PendingFrame>, store: ImageStore, wake: impl Fn()) {
while let Ok(first) = rx.recv() {
// Collect the blocking frame plus any that arrived while we were
// busy, newest-per-id winning (a later frame with the same id
// supersedes an earlier one — exactly what `place` would do, but
// without paying to decode the ones we'd immediately retire).
let mut latest: Vec<PendingFrame> = vec![first];
loop {
match rx.try_recv() {
Ok(next) => {
if next.img.id != 0
&& let Some(slot) = latest.iter_mut().find(|p| p.img.id == next.img.id)
{
*slot = next; // supersede the queued frame for this id
} else {
latest.push(next);
}
}
Err(TryRecvError::Empty) => break,
Err(TryRecvError::Disconnected) => {
// Decode whatever we already gathered, then exit.
Self::place_all(&store, latest, &wake);
return;
}
}
}
Self::place_all(&store, latest, &wake);
}
}
fn place_all(store: &ImageStore, frames: Vec<PendingFrame>, wake: &impl Fn()) {
let mut placed_any = false;
for pf in frames {
if let Some((data, w, h)) = decode(&pf.img) {
store.place(PlacedImage {
data,
anchor_row: pf.anchor_row,
anchor_col: pf.anchor_col,
width_px: w,
height_px: h,
cols: pf.img.cols,
rows: pf.img.rows,
id: pf.img.id,
placement: pf.img.placement,
});
placed_any = true;
}
}
if placed_any {
wake();
}
}
}
impl Drop for DecodeWorker {
fn drop(&mut self) {
// Drop the sender first so the worker sees `Disconnected` and returns;
// then join so its store writes are visible and it doesn't outlive the
// pane. Joining before dropping `tx` would deadlock — the loop only ends
// once the channel closes.
self.tx = None;
if let Some(h) = self.handle.take() {
let _ = h.join();
}
}
}
/// Decode a daemon [`Image`] frame into GPUI-ready pixels: inflate + expand to
/// RGBA (via the protocol type's own [`Image::to_rgba8`]), or decode a PNG
/// payload, then swap R↔B to the BGRA the sprite atlas wants. Returns the render
/// image and its true pixel dimensions (a PNG carries its own, overriding any
/// `s=`/`v=` the sender may have omitted). `None` if the payload can't be decoded.
pub fn decode(img: &Image) -> Option<(Arc<RenderImage>, u32, u32)> {
let (mut rgba, w, h) = match img.format {
WireFormat::Png => {
// The one path pixels stay encoded through: decode with the `image`
// crate (a direct dep, same version gpui uses) rather than the
// protocol type, which declines PNG on purpose.
let dyn_img = image::load_from_memory(&img.data).ok()?;
let buf = dyn_img.into_rgba8();
let (w, h) = buf.dimensions();
(buf.into_raw(), w, h)
}
WireFormat::Rgb | WireFormat::Rgba => {
let rgba = img.to_rgba8()?;
(rgba, img.width, img.height)
}
};
if w == 0 || h == 0 || rgba.len() < (w as usize * h as usize * 4) {
return None;
}
rgba.truncate(w as usize * h as usize * 4);
// RGBA → BGRA for the atlas. A channel swap and nothing else: `RenderImage`
// holds *straight* alpha, not premultiplied. gpui's own producers say so —
// `swap_rgba_pa_to_bgra`, which both the CoreGraphics text rasterizer and
// the SVG renderer run their premultiplied output through, divides the
// color channels back out by alpha on the way in. Premultiplying here would
// darken every translucent pixel of an `f=100` PNG twice over.
for px in rgba.chunks_exact_mut(4) {
px.swap(0, 2);
}
let buffer = RgbaImage::from_raw(w, h, rgba)?;
let render = RenderImage::new(SmallVec::from_elem(Frame::new(buffer), 1));
Some((Arc::new(render), w, h))
}
#[cfg(test)]
mod tests {
use super::*;
/// A raw 1x1 opaque-red RGBA image, the minimum a placement needs, built the
/// way the daemon delivers one (base64-decoded, uncompressed).
fn red_pixel() -> Image {
Image {
id: 0,
number: 0,
placement: 0,
width: 1,
height: 1,
cols: 0,
rows: 0,
data: vec![0xff, 0x00, 0x00, 0xff],
format: WireFormat::Rgba,
compressed: false,
}
}
fn placed(id: u32, placement: u32) -> PlacedImage {
let (data, w, h) = decode(&red_pixel()).unwrap();
PlacedImage {
data,
anchor_row: 0,
anchor_col: 0,
width_px: w,
height_px: h,
cols: 0,
rows: 0,
id,
placement,
}
}
#[test]
fn decodes_rgba_and_swaps_to_bgra() {
let img = Image {
data: vec![1, 2, 3, 4],
..red_pixel()
};
let (data, w, h) = decode(&img).unwrap();
assert_eq!((w, h), (1, 1));
// Red/blue swapped: RGBA [1,2,3,4] → BGRA [3,2,1,4].
assert_eq!(data.as_bytes(0).unwrap(), &[3, 2, 1, 4]);
}
#[test]
fn same_id_replaces_in_place() {
let store = ImageStore::new();
store.place(placed(7, 1));
store.place(placed(7, 1));
assert_eq!(
store.snapshot().len(),
1,
"a re-transmit replaces, not stacks"
);
}
#[test]
fn anonymous_images_coexist() {
let store = ImageStore::new();
store.place(placed(0, 0));
store.place(placed(0, 0));
assert_eq!(
store.snapshot().len(),
2,
"id 0 is a fresh placement each time"
);
}
#[test]
fn delete_all_clears_everything() {
let store = ImageStore::new();
store.place(placed(1, 0));
store.place(placed(2, 0));
store.delete(&ImageDelete {
target: b'A',
id: 0,
placement: 0,
});
assert!(store.snapshot().is_empty());
}
#[test]
fn delete_by_id_leaves_others() {
let store = ImageStore::new();
store.place(placed(1, 0));
store.place(placed(2, 0));
store.delete(&ImageDelete {
target: b'i',
id: 1,
placement: 0,
});
let left = store.snapshot();
assert_eq!(left.len(), 1);
assert_eq!(left[0].id, 2);
}
#[test]
fn unknown_selector_is_a_no_op() {
let store = ImageStore::new();
store.place(placed(1, 0));
// `z` (by z-index) isn't modeled; the image must survive.
store.delete(&ImageDelete {
target: b'z',
id: 0,
placement: 0,
});
assert_eq!(store.snapshot().len(), 1);
}
#[test]
fn overflow_drops_the_oldest() {
let store = ImageStore::new();
for i in 0..(MAX_IMAGES as u32 + 5) {
store.place(placed(i + 1, 0));
}
let left = store.snapshot();
assert_eq!(left.len(), MAX_IMAGES);
assert_eq!(left[0].id, 6, "the five oldest aged out");
}
#[test]
fn replacing_a_frame_retires_the_old_render_image() {
let store = ImageStore::new();
store.place(placed(7, 1));
// A same-id re-transmit (what a 60fps browser does) must retire the old
// frame's render image so the paint path can evict its atlas tile.
store.place(placed(7, 1));
assert_eq!(store.snapshot().len(), 1, "still one live placement");
assert_eq!(
store.take_retired().len(),
1,
"the superseded frame is retired"
);
assert!(store.take_retired().is_empty(), "draining is one-shot");
}
#[test]
fn deletes_and_clear_retire_render_images() {
let store = ImageStore::new();
store.place(placed(1, 0));
store.place(placed(2, 0));
let _ = store.take_retired(); // drain the (none) from placement
store.delete(&ImageDelete {
target: b'i',
id: 1,
placement: 0,
});
assert_eq!(
store.take_retired().len(),
1,
"the deleted frame is retired"
);
store.clear();
assert_eq!(
store.take_retired().len(),
1,
"clear retires the survivor too"
);
}
fn frame(id: u32) -> PendingFrame {
PendingFrame {
img: Image { id, ..red_pixel() },
anchor_row: 0,
anchor_col: 0,
}
}
/// The worker decodes off-thread and places what it receives. A single frame
/// lands in the store, at the anchor the reader captured.
#[test]
fn worker_decodes_and_places_off_thread() {
let store = ImageStore::new();
let woken = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let w = woken.clone();
let worker = DecodeWorker::spawn(store.clone(), move || {
w.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
});
worker.submit(frame(7));
drop(worker); // joins the thread, so the decode is finished on return
let placed = store.snapshot();
assert_eq!(placed.len(), 1);
assert_eq!(placed[0].id, 7);
assert!(
woken.load(std::sync::atomic::Ordering::SeqCst) >= 1,
"a successful decode wakes the view"
);
}
/// A burst of same-id frames (what a re-transmitting browser produces faster
/// than the decoder drains) collapses to a single live placement — stale
/// frames are coalesced away rather than each costing an inflate. The store's
/// own same-id replacement guarantees the end state even if the worker only
/// sees them one at a time, so this asserts the invariant the pipeline keeps.
#[test]
fn worker_coalesces_a_same_id_burst_to_one_placement() {
let store = ImageStore::new();
let worker = DecodeWorker::spawn(store.clone(), || {});
for _ in 0..50 {
worker.submit(frame(9));
}
drop(worker); // drains + joins
assert_eq!(
store.snapshot().len(),
1,
"a same-id burst leaves exactly one live frame"
);
}
/// Distinct ids are independent placements — coalescing is per id, so two
/// different images both survive.
#[test]
fn worker_keeps_distinct_ids() {
let store = ImageStore::new();
let worker = DecodeWorker::spawn(store.clone(), || {});
worker.submit(frame(1));
worker.submit(frame(2));
drop(worker);
let mut ids: Vec<u32> = store.snapshot().iter().map(|p| p.id).collect();
ids.sort_unstable();
assert_eq!(ids, vec![1, 2]);
}
}
src/terminal/mod.rs
+1
View File
@@ -10,6 +10,7 @@ pub(crate) mod git_status;
mod highlight;
mod history;
mod hold;
pub(crate) mod images;
pub mod input;
mod loopback;
pub(crate) mod marks;
src/terminal/remote.rs
+104 -1
View File
@@ -76,6 +76,10 @@ struct ReaderSignals {
auth: Arc<Mutex<VecDeque<(u64, AuthPromptKind)>>>,
phase: Arc<Mutex<Option<SshPhase>>>,
marks: crate::terminal::marks::Marks,
/// Kitty-graphics images the daemon lifted out of the stream (issue #213),
/// anchored to the grid for the paint path to blit. Shared with the reader,
/// which places/deletes them as `DaemonMsg::Image`/`DeleteImage` frames land.
images: crate::terminal::images::ImageStore,
}
#[derive(Clone, Debug, PartialEq)]
@@ -152,6 +156,10 @@ pub struct RemoteTerminal {
pub exited: bool,
size: TermSize,
synced_size: bool,
/// The `(cell_w, cell_h)` last sent to the daemon, in device pixels. Tracked
/// alongside `size` so a display-scale change still reaches the child even
/// when the grid dimensions are unchanged.
synced_cell: (u16, u16),
writer: Mutex<Stream>,
cwd: Arc<Mutex<Option<PathBuf>>>,
shell_state: Arc<Mutex<ShellState>>,
@@ -167,6 +175,11 @@ pub struct RemoteTerminal {
agent: Arc<Mutex<Option<CLIAgent>>>,
agent_session: Arc<Mutex<Option<AgentSessionState>>>,
marks: crate::terminal::marks::Marks,
/// Kitty-graphics images placed on this pane's grid (issue #213).
/// Written by the reader thread from out-of-band `Image`/`DeleteImage`
/// frames, read by the paint path — same shared-handle discipline as
/// `marks`, since only the client holds the grid the anchors are relative to.
images: crate::terminal::images::ImageStore,
route: PaneRoute,
proxy: EventProxy,
reader_thread: Option<JoinHandle<()>>,
@@ -334,6 +347,10 @@ impl RemoteTerminal {
let mut term = self.term.lock();
term.reset_state();
}
// The grid was just reset, so every image anchor now points nowhere.
// Drop them; the daemon does not replay out-of-band image frames, so a
// browser redraws on its next transmit (see issue #213's reattach note).
self.images.clear();
let reader = Self::spawn_reader(
self.term.clone(),
@@ -353,6 +370,7 @@ impl RemoteTerminal {
auth: self.auth_prompts.clone(),
phase: self.ssh_phase.clone(),
marks: self.marks.clone(),
images: self.images.clone(),
},
);
if let Ok(mut writer) = self.writer.lock() {
@@ -401,6 +419,7 @@ impl RemoteTerminal {
Arc::new(Mutex::new(VecDeque::new()));
let ssh_phase: Arc<Mutex<Option<SshPhase>>> = Arc::new(Mutex::new(None));
let marks = crate::terminal::marks::Marks::new();
let images = crate::terminal::images::ImageStore::new();
let reader_thread = Self::spawn_reader(
term.clone(),
@@ -420,6 +439,7 @@ impl RemoteTerminal {
auth: auth_prompts.clone(),
phase: ssh_phase.clone(),
marks: marks.clone(),
images: images.clone(),
},
);
@@ -430,6 +450,7 @@ impl RemoteTerminal {
exited: false,
size,
synced_size: false,
synced_cell: (0, 0),
writer: Mutex::new(write_half),
cwd,
shell_state,
@@ -445,6 +466,7 @@ impl RemoteTerminal {
agent,
agent_session,
marks,
images,
route: PaneRoute::Local,
proxy,
reader_thread: Some(reader_thread),
@@ -490,6 +512,7 @@ impl RemoteTerminal {
auth,
phase,
marks,
images,
} = signals;
crate::core::threads::promote_to_user_interactive();
let mut stream = read_half;
@@ -500,6 +523,19 @@ impl RemoteTerminal {
let mut mark_scan = MarkScanner::new();
let mut pending: Vec<u8> = buffered;
let mut pending_size: Option<WinSize> = None;
// Kitty-graphics decode runs on its own thread with newest-frame
// coalescing (issue #213): inflating a full-window browser frame
// is ~42 ms, and doing it inline here would block PTY output and
// scrolling for that long every frame. The worker owns the inflate
// + BGRA swap + placement; the reader only captures the grid
// anchor and hands off the still-compressed frame. Dropped when
// the loop ends, which joins the thread.
let image_decoder = {
let proxy = proxy.clone();
crate::terminal::images::DecodeWorker::spawn(images.clone(), move || {
proxy.send_event(AlacEvent::Wakeup);
})
};
let mut scratch = vec![0u8; 256 * 1024];
let trace = std::env::var("TTY7_TRACE").is_ok_and(|v| !v.is_empty() && v != "0");
@@ -634,6 +670,59 @@ impl RemoteTerminal {
out_batch.extend_from_slice(&bytes);
tr_frames += 1;
}
// Kitty graphics (issue #213): the daemon lifted an
// image out of the stream and forwarded it out-of-band,
// interleaved *in stream order* with the Output frames
// around it. Flush the pending text first so the grid
// cursor sits where the sender drew the image, then
// anchor the placement to that cell in scroll-stable
// absolute-row coordinates (the same formula
// `record_mark` uses), so it tracks scrolling.
DaemonMsg::Image(frame) => {
flush_batch!();
if let Some(img) =
tty7_core::core::kitty_graphics::Image::decode_frame(&frame)
{
// Capture the anchor *now*, at the cursor cell
// the transmission arrived on; the decode is
// deferred to the worker thread but must land
// at this position, not wherever the cursor has
// scrolled to by the time inflate finishes.
let (anchor_row, anchor_col) = {
use alacritty_terminal::grid::Dimensions as _;
let term = term.lock();
let grid = term.grid();
let row = grid.history_size() as i64
- grid.display_offset() as i64
+ i64::from(grid.cursor.point.line.0);
(row, grid.cursor.point.column.0)
};
// Hand off without blocking the reader: the
// worker inflates, swaps, places, and wakes the
// view. Stale frames coalesce away there.
image_decoder.submit(
crate::terminal::images::PendingFrame {
img,
anchor_row,
anchor_col,
},
);
}
}
// An `a=d` delete, lifted out the same way. Order with
// the surrounding output does not matter for a delete
// (it targets by id/placement, not cursor position),
// but flushing keeps a delete-then-retransmit in the
// same read from racing its own replacement.
DaemonMsg::DeleteImage(sel) => {
flush_batch!();
if let Some(del) =
tty7_core::core::kitty_graphics::ImageDelete::decode(&sel)
{
images.delete(&del);
proxy.send_event(AlacEvent::Wakeup);
}
}
DaemonMsg::Cwd(path) => {
flush_batch!();
if let Ok(mut guard) = cwd.lock() {
@@ -799,7 +888,13 @@ impl RemoteTerminal {
}
pub fn resize(&mut self, size: TermSize, cell_w: u16, cell_h: u16) {
if self.synced_size && size == self.size {
// The cell size has to be part of the early-out, not just cols/rows: it
// is reported in *device* pixels, so moving the window between a 2x and
// a 1x display changes `ws_xpixel`/`ws_ypixel` while the grid stays
// exactly the same. Comparing only `size` there would skip the resize
// and leave a pixel-aware child rendering for the old framebuffer.
let cell = (cell_w, cell_h);
if self.synced_size && size == self.size && cell == self.synced_cell {
use alacritty_terminal::grid::Dimensions as _;
let term = self.term.lock();
if term.columns() == size.cols && term.screen_lines() == size.rows {
@@ -808,6 +903,7 @@ impl RemoteTerminal {
}
self.synced_size = true;
self.size = size;
self.synced_cell = cell;
self.term.lock().resize(size);
let win = win_size(size, cell_w, cell_h);
@@ -855,6 +951,13 @@ impl RemoteTerminal {
self.marks.clone()
}
/// The kitty-graphics image store for this pane. Cheap handle clone — the
/// store is an `Arc<Mutex<..>>` shared with the reader thread, which places
/// and deletes images as out-of-band frames arrive from the daemon.
pub fn images(&self) -> crate::terminal::images::ImageStore {
self.images.clone()
}
pub fn agent_session(&self) -> Option<AgentSessionState> {
self.agent_session.lock().ok().and_then(|g| g.clone())
}
src/terminal/view.rs
+193 -17
View File
@@ -772,6 +772,7 @@ impl TerminalView {
rows: usize,
cell_width: Pixels,
line_height: Pixels,
scale: f32,
) {
if (cols, rows) != (self.terminal.size().cols, self.terminal.size().rows) {
self.last_hover_cell = None;
@@ -779,10 +780,23 @@ impl TerminalView {
}
self.cell_width = cell_width;
self.line_height = line_height;
// Report the cell size to the child in *device* pixels (logical × display
// scale), so `ws_xpixel`/`ws_ypixel` describe the real framebuffer. A
// pixel-aware program like terminal-browser renders its frame at that
// native resolution; painted back into logical-pixel bounds, gpui blits
// it ~1:1 on the framebuffer instead of upscaling a half-resolution
// bitmap (which looked soft and magnified on Retina). This is what
// kitty/ghostty report. `self.cell_width` stays logical — glyph layout
// and mouse mapping work in logical pixels.
let scale = if scale.is_finite() && scale > 0. {
scale
} else {
1.
};
self.terminal.resize(
TermSize::new(cols, rows),
cell_width.as_f32().round() as u16,
line_height.as_f32().round() as u16,
(cell_width.as_f32() * scale).round().max(1.) as u16,
(line_height.as_f32() * scale).round().max(1.) as u16,
);
}
@@ -2472,6 +2486,10 @@ impl TerminalView {
None
}
fn link_inactive_reason(&self, cx: &gpui::App) -> Option<&'static str> {
(!self.accepts_input(cx)).then_some("the remote link is not attached")
}
fn shell_vi_prompt(&self) -> bool {
self.terminal.shell_vi_mode() && self.terminal.at_prompt() && !self.on_alt_screen()
}
@@ -2596,7 +2614,7 @@ impl TerminalView {
}
fn submit_command(&mut self, cx: &mut Context<Self>) {
if self.terminal.exited {
if self.terminal.exited || !self.accepts_input(cx) {
return;
}
if let Some(net) = self.hold.engage() {
@@ -2855,6 +2873,9 @@ impl TerminalView {
}
fn handoff_line_to_shell(&mut self, chord: &[u8], cx: &mut Context<Self>) {
if !self.accepts_input(cx) {
return;
}
if let Some(net) = self.hold.engage() {
self.cmd.prepend_str(&net);
}
@@ -2882,6 +2903,10 @@ impl TerminalView {
cx.propagate();
return;
}
if let Some(reason) = self.link_inactive_reason(cx) {
log::debug!(target: "tty7::completion", "Tab does nothing and the line stays: {reason}");
return;
}
if let Some(reason) = self.input_inactive_reason() {
log::debug!(target: "tty7::completion", "Tab goes straight to the PTY: {reason}");
let bytes = self.tab_bytes(!forward);
@@ -3040,19 +3065,16 @@ impl TerminalView {
log::debug!(target: "tty7::completion", "listing {dir} over the remote's own connection");
cx.spawn(async move |this, cx| {
let listed = cx.background_spawn(async move { route.list(&dir) }).await;
if let Err(e) = &listed {
log::warn!(target: "tty7::completion", "remote listing failed: {e}");
}
let entries = listed.unwrap_or_else(|e| {
log::warn!(
target: "tty7::completion",
"remote listing failed, treating it as no candidates: {e}"
);
Vec::new()
});
let _ = this.update(cx, |view, cx| {
view.remote_completion_inflight = false;
view.remote_path_results(
req,
&line,
cursor,
listed.unwrap_or_default(),
forward,
cx,
)
view.remote_path_results(req, &line, cursor, entries, forward, cx);
});
})
.detach();
@@ -3068,6 +3090,16 @@ impl TerminalView {
forward: bool,
cx: &mut Context<Self>,
) {
if let Some(reason) = self
.link_inactive_reason(cx)
.or_else(|| self.input_inactive_reason())
{
log::debug!(
target: "tty7::completion",
"dropping a remote listing for {line:?}: {reason}"
);
return;
}
if self.cmd.text() != line || self.cmd.cursor() != cursor {
log::debug!(
target: "tty7::completion",
@@ -5680,16 +5712,16 @@ mod gpui_tests {
let (window, _daemon) = harness(cx);
window
.update(cx, |view, _, cx| {
view.set_grid_size(80, 24, px(8.), px(17.));
view.set_grid_size(80, 24, px(8.), px(17.), 1.);
view.hover_link_at(0, 23, true, cx);
assert_eq!(view.last_hover_cell, Some((0, 23)));
view.hovered_link = Some(HoveredLink {
start: Point::new(Line(23), Column(0)),
end: Point::new(Line(23), Column(3)),
});
view.set_grid_size(80, 24, px(8.), px(17.));
view.set_grid_size(80, 24, px(8.), px(17.), 1.);
assert_eq!(view.last_hover_cell, Some((0, 23)));
view.set_grid_size(80, 8, px(8.), px(17.));
view.set_grid_size(80, 8, px(8.), px(17.), 1.);
assert!(view.last_hover_cell.is_none(), "the cell is stale");
assert!(view.hovered_link.is_none(), "so is the link it resolved");
})
@@ -5940,6 +5972,45 @@ mod gpui_tests {
wait_for_input_active(&window, cx);
}
#[gpui::test]
fn a_late_remote_listing_leaves_a_line_the_editor_no_longer_owns_alone(
cx: &mut TestAppContext,
) {
crate::core::config::pin_test_config_dir();
let (window, mut daemon) = harness(cx);
DaemonMsg::Prompt {
active: true,
at_prompt: true,
last_exit: None,
}
.encode(&mut daemon)
.unwrap();
wait_for_input_active(&window, cx);
window
.update(cx, |view, _, cx| {
view.cmd.set("ls /nope/");
view.editor_handoff = Some(view.terminal.prompt_cycle());
assert!(!view.input_active(), "the shell owns this prompt already");
let req =
super::completion::remote_path_request("ls /nope/", 9, "/home/u").unwrap();
view.remote_path_results(req, "ls /nope/", 9, Vec::new(), true, cx);
assert_eq!(
view.cmd.text(),
"ls /nope/",
"an empty listing must not hand off a line the editor no longer drives"
);
})
.unwrap();
assert_eq!(
next_input_until_timeout(&mut daemon),
None,
"not one byte reached the wire"
);
}
#[gpui::test]
fn tab_completion_off_sends_every_tab_to_the_shell(cx: &mut TestAppContext) {
let (window, mut daemon) = harness(cx);
@@ -6931,6 +7002,111 @@ mod gpui_tests {
id
}
#[gpui::test]
fn a_disconnected_remote_pane_keeps_the_line_instead_of_handing_it_to_nowhere(
cx: &mut TestAppContext,
) {
crate::core::config::pin_test_config_dir();
let (window, mut daemon) = harness(cx);
cx.update(|cx| crate::ui::keymap::init(cx));
DaemonMsg::Prompt {
active: true,
at_prompt: true,
last_exit: None,
}
.encode(&mut daemon)
.unwrap();
wait_for_input_active(&window, cx);
window
.update(cx, |view, window, cx| {
window.activate_window();
view.focus_handle.focus(window, cx);
view.cmd.set("zzqqx");
bind_to_a_disconnected_remote_workspace(view, cx);
})
.unwrap();
let mut vcx = gpui::VisualTestContext::from_window(window.into(), cx);
vcx.simulate_keystrokes("tab");
window
.update(cx, |view, _, cx| {
assert_eq!(
view.cmd.text(),
"zzqqx",
"a Tab dispatched through SendTab must not empty the line"
);
assert!(
view.editor_handoff.is_none(),
"nothing was handed off, so the editor keeps the prompt"
);
view.submit_command(cx);
assert_eq!(
view.cmd.text(),
"zzqqx",
"submit_command guards the link too, even though on_key_down already does"
);
})
.unwrap();
assert_eq!(
next_input_until_timeout(&mut daemon),
None,
"not one byte reached the wire"
);
}
#[gpui::test]
fn a_tab_on_a_detached_remote_pane_never_asks_for_a_listing(cx: &mut TestAppContext) {
use std::io::Write as _;
crate::core::config::pin_test_config_dir();
let (window, mut daemon) = harness(cx);
DaemonMsg::Prompt {
active: true,
at_prompt: true,
last_exit: None,
}
.encode(&mut daemon)
.unwrap();
DaemonMsg::Cwd(std::path::PathBuf::from("/home/me/proj"))
.encode(&mut daemon)
.unwrap();
daemon.flush().unwrap();
wait_for_input_active(&window, cx);
for _ in 0..200 {
if window
.update(cx, |view, _, _| view.cwd().is_some())
.unwrap()
{
break;
}
std::thread::sleep(std::time::Duration::from_millis(5));
}
window
.update(cx, |view, _, cx| {
view.cmd.set("ls /home/me/");
bind_to_a_disconnected_remote_workspace(view, cx);
assert!(
view.remote_ssh_cwd().is_some(),
"the pane has to look remote enough to want a listing at all"
);
view.tab_pressed(true, cx);
assert!(
!view.remote_completion_inflight,
"a Tab must not send an SFTP listing down a link that is not attached"
);
assert_eq!(
view.cmd.text(),
"ls /home/me/",
"and the line stays where it was"
);
})
.unwrap();
}
#[gpui::test]
fn a_disconnected_remote_pane_swallows_every_kind_of_typing(cx: &mut TestAppContext) {
let (window, mut daemon) = harness(cx);