diff --git a/src/app/pane_graphics.rs b/src/app/pane_graphics.rs index c1d4841a..7b516116 100644 --- a/src/app/pane_graphics.rs +++ b/src/app/pane_graphics.rs @@ -31,6 +31,7 @@ pub(crate) struct Layer { #[derive(Debug)] pub(crate) struct DirectGate { pub(crate) transfer_id: u64, + pub(crate) image_id: u32, pub(crate) client_id: u64, pub(crate) deadline: std::time::Instant, pub(crate) written: bool, diff --git a/src/client/mod.rs b/src/client/mod.rs index 51d3100f..8c2effb3 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -17,6 +17,8 @@ mod direct_graphics; mod input; mod shell; +#[cfg(unix)] +use std::collections::HashMap; use std::collections::HashSet; #[cfg(unix)] use std::io::IsTerminal as _; @@ -77,6 +79,8 @@ struct ClientState { keyboard_report_all_active: bool, /// The terminal size we reported to the server in our last Hello/Resize. reported_size: (u16, u16), + /// Last exact host cell size used by client-rendered surfaces. + reported_cell_size: (u32, u32), /// Client-local sound playback config, refreshed on server request. sound_config: crate::config::SoundConfig, /// Whether this client may write Kitty graphics bytes to its host terminal. @@ -87,6 +91,9 @@ struct ClientState { /// One server-retired direct transfer to suppress if it was still queued. #[cfg(unix)] retired_direct_graphics: Option<(u64, u32)>, + /// ClientShell assets waiting for the host terminal's direct-upload response. + #[cfg(unix)] + pending_surface_graphics: HashMap, /// Direct attach prefix escape state. None for full-app clients. attach_escape: Option, /// Rows scrolled for one direct-attach wheel notch. @@ -246,6 +253,15 @@ impl ClientState { self.repaint_pending = true; } + fn present_graphics(&mut self, graphics: &[u8]) { + if graphics.is_empty() || !self.kitty_graphics_enabled { + return; + } + let mut stdout = io::stdout(); + let _ = write_encoded_frame_with_graphics(&mut stdout, &[], graphics); + let _ = stdout.flush(); + } + fn present_frame(&mut self, frame_data: FrameData) { let frame_data = if self.draw_host_cursor { render_ansi::frame_with_drawn_cursor(frame_data) @@ -894,6 +910,10 @@ fn do_handshake( requested_encoding, surface_size, pixel_mouse: exact_cell_size && cfg!(unix), + direct_graphics: exact_cell_size + && cell_width_px > 0 + && cell_height_px > 0 + && direct_graphics_profile_allowed(false), } } else { ClientMessage::Hello { @@ -1299,9 +1319,8 @@ fn run_client_with_mode( let host_cursor = loaded_config.config.ui.host_cursor; let direct_attach_requested = attach_request.is_some(); let remote_image_paste_key = client_remote_image_paste_key(&loaded_config.config); - let kitty_graphics_enabled = loaded_config.config.experimental.kitty_graphics - && !direct_attach_requested - && !client_rendered_shell; + let kitty_graphics_enabled = + loaded_config.config.experimental.kitty_graphics && !direct_attach_requested; let loop_config = ClientLoopConfig { sound_config: loaded_config.config.ui.sound, mouse_scroll_lines, @@ -1513,8 +1532,13 @@ fn finish_client_shell_input( } if outcome.resize { let shell = state.shell.as_ref().expect("shell mode remains active"); - let resize = - client_shell_resize_message(shell, state.reported_size.0, state.reported_size.1, 0, 0); + let resize = client_shell_resize_message( + shell, + state.reported_size.0, + state.reported_size.1, + state.reported_cell_size.0, + state.reported_cell_size.1, + ); write_to_server(write_stream, &resize).map_err(ClientError::ConnectionLost)?; } #[cfg(not(windows))] @@ -1559,12 +1583,15 @@ async fn run_client_loop( mouse_capture_active: config.mouse_capture_active, keyboard_report_all_active: false, reported_size: (cols, rows), + reported_cell_size: (initial_cell_width_px, initial_cell_height_px), sound_config: config.sound_config, kitty_graphics_enabled: config.kitty_graphics_enabled, #[cfg(unix)] direct_graphics_response: Arc::new(Mutex::new(direct_graphics::ResponseMatcher::default())), #[cfg(unix)] retired_direct_graphics: None, + #[cfg(unix)] + pending_surface_graphics: HashMap::new(), attach_escape, #[cfg(unix)] mouse_scroll_lines: config.mouse_scroll_lines, @@ -1574,6 +1601,9 @@ async fn run_client_loop( draw_host_cursor, shell: config.shell_config.map(shell::ClientShellState::new), }; + if let Some(shell) = state.shell.as_mut() { + shell.set_graphics_cell_size(initial_cell_width_px, initial_cell_height_px); + } debug!(?negotiated_encoding, "client render encoding active"); let host_mouse_capture_active = Arc::new(AtomicBool::new(state.mouse_capture_active)); // Cell size reported by the host terminal, packed as width<<32 | height. @@ -1856,6 +1886,17 @@ async fn run_client_loop( } #[cfg(unix)] ClientLoopEvent::DirectGraphicsResponse(response) => { + let composed = state + .pending_surface_graphics + .remove(&response.transfer_id) + .filter(|_| response.success) + .and_then(|asset| { + let shell = state.shell.as_mut()?; + shell + .trust_direct_graphics_asset(&asset, response.image_id) + .then(|| shell.compose(state.reported_size.0, state.reported_size.1)) + .flatten() + }); let message = ClientMessage::GraphicsTransmissionResult { transfer_id: response.transfer_id, image_id: response.image_id, @@ -1864,6 +1905,9 @@ async fn run_client_loop( if let Err(err) = write_to_server(&mut write_stream, &message) { return Err(ClientError::ConnectionLost(err)); } + if let Some(frame) = composed { + state.present_frame(frame); + } } #[cfg(unix)] ClientLoopEvent::PixelMouse(data, geometry) => { @@ -1977,9 +2021,11 @@ async fn run_client_loop( } ClientLoopEvent::Resize(new_cols, new_rows, cell_width_px, cell_height_px) => { state.reported_size = (new_cols, new_rows); + state.reported_cell_size = (cell_width_px, cell_height_px); // Resizing invalidates both the host-side blit baseline and pane hit geometry. state.request_repaint(); if let Some(shell) = state.shell.as_mut() { + shell.set_graphics_cell_size(cell_width_px, cell_height_px); shell.invalidate_pane_surface(); } let msg = if let Some(shell) = &state.shell { @@ -2005,10 +2051,12 @@ async fn run_client_loop( ClientLoopEvent::ServerMessage(msg) => match msg { ServerMessage::Frame(frame_data) => state.present_frame(frame_data), ServerMessage::ClientShellSnapshot(snapshot) => { - let (composed, resize) = if let Some(shell) = &mut state.shell { + let (composed, resize, graphics_cleanup) = if let Some(shell) = &mut state.shell + { let previous_size = shell.surface_size(state.reported_size.0, state.reported_size.1); shell.set_snapshot(snapshot); + let graphics_cleanup = shell.take_pending_graphics_cleanup(); let next_size = shell.surface_size(state.reported_size.0, state.reported_size.1); ( @@ -2018,14 +2066,16 @@ async fn run_client_loop( shell, state.reported_size.0, state.reported_size.1, - 0, - 0, + state.reported_cell_size.0, + state.reported_cell_size.1, ) }), + graphics_cleanup, ) } else { - (None, None) + (None, None, Vec::new()) }; + state.present_graphics(&graphics_cleanup); if let Some(resize) = resize { if let Err(err) = write_to_server(&mut write_stream, &resize) { return Err(ClientError::ConnectionLost(err)); @@ -2076,13 +2126,25 @@ async fn run_client_loop( transfer_id, leading, control, + surface_asset, } => { #[cfg(unix)] { if state.retired_direct_graphics.take() == Some((transfer_id, image_id)) { continue; } + let surface_asset_valid = match (state.shell.as_ref(), &surface_asset) { + (Some(shell), Some(asset)) => { + crate::kitty_graphics::surface::host_image_id( + shell.graphics_scope(), + asset, + ) == image_id + } + (None, None) => true, + _ => false, + }; let valid = state.kitty_graphics_enabled + && surface_asset_valid && usize::try_from(expected_len).ok().is_some_and(|len| { crate::pane_graphics_files::validate_direct_source( std::path::Path::new(&path), @@ -2116,6 +2178,9 @@ async fn run_client_loop( false }; if sent { + if let Some(asset) = surface_asset { + state.pending_surface_graphics.insert(transfer_id, asset); + } if let Ok(mut matcher) = state.direct_graphics_response.lock() { matcher.start(transfer_id); } @@ -2127,6 +2192,7 @@ async fn run_client_loop( return Err(ClientError::ConnectionLost(err)); } } else { + state.pending_surface_graphics.remove(&transfer_id); if let Ok(mut matcher) = state.direct_graphics_response.lock() { if valid { matcher.retire(transfer_id); @@ -2145,7 +2211,15 @@ async fn run_client_loop( } } #[cfg(not(unix))] - let _ = (path, expected_len, image_id, transfer_id, leading, control); + let _ = ( + path, + expected_len, + image_id, + transfer_id, + leading, + control, + surface_asset, + ); } ServerMessage::GraphicsTransmissionRetired { transfer_id, @@ -2154,6 +2228,15 @@ async fn run_client_loop( #[cfg(unix)] { state.retired_direct_graphics = Some((transfer_id, image_id)); + state.pending_surface_graphics.remove(&transfer_id); + let cleanup = state.shell.as_mut().map_or_else(Vec::new, |shell| { + shell.retire_direct_graphics_image(image_id); + shell + .compose(state.reported_size.0, state.reported_size.1) + .map(|frame| frame.graphics) + .unwrap_or_else(|| shell.take_pending_graphics_cleanup()) + }); + state.present_graphics(&cleanup); if let Ok(mut matcher) = state.direct_graphics_response.lock() { matcher.retire(transfer_id); } diff --git a/src/client/shell.rs b/src/client/shell.rs index 477a57cd..351ab947 100644 --- a/src/client/shell.rs +++ b/src/client/shell.rs @@ -7,6 +7,7 @@ mod config; mod context_menu; mod copy_mode; mod global_menu; +mod graphics; mod input; mod mobile; mod mouse; @@ -392,6 +393,7 @@ mod tests { }], splits: Vec::new(), popup: None, + graphics: crate::protocol::SurfaceGraphicsScene::default(), } } @@ -499,6 +501,65 @@ mod tests { ); } + #[test] + fn client_shell_graphics_follow_final_shell_origin_and_local_overlay_visibility() { + let mut state = ClientShellState::new(ClientShellConfig::from_config(&Config::default())); + state.set_snapshot(Box::new(snapshot())); + let mut pane_surface = surface(); + let key = crate::protocol::SurfaceGraphicsAssetKey { + source: crate::protocol::SurfaceGraphicsSource::Terminal { + target: crate::protocol::SurfaceGraphicsTarget::Pane { + pane_id: "pane_1".into(), + }, + image_id: 1, + }, + image_width: 1, + image_height: 1, + format: crate::protocol::SurfaceGraphicsFormat::Rgba, + data_len: 4, + data_fingerprint: 17, + }; + pane_surface.graphics = crate::protocol::SurfaceGraphicsScene { + assets: vec![crate::protocol::SurfaceGraphicsAsset { + key: key.clone(), + data: vec![1, 2, 3, 4], + }], + placements: vec![crate::protocol::SurfaceGraphicsPlacement { + asset: key, + logical_placement_id: 1, + x: 0, + y: 0, + cols: 1, + rows: 1, + source_x: 0, + source_y: 0, + source_width: 1, + source_height: 1, + x_offset: 0, + y_offset: 0, + z: 0, + scrollback_offset: 0, + }], + retained_assets: Vec::new(), + }; + state.set_pane_surface(pane_surface); + + let visible = state.compose(106, 20).expect("visible graphics frame"); + let visible = String::from_utf8_lossy(&visible.graphics); + assert!(visible.contains("a=t,t=d")); + assert!(visible.contains("\u{1b}[2;27H")); + + state.overlay = Some(ClientShellOverlay::Onboarding); + let hidden = state.compose(106, 20).expect("overlay frame"); + assert!(String::from_utf8_lossy(&hidden.graphics).contains("a=d,d=i")); + + state.overlay = None; + let restored = state.compose(106, 20).expect("restored graphics frame"); + let restored = String::from_utf8_lossy(&restored.graphics); + assert!(restored.contains("a=p")); + assert!(!restored.contains("a=t,t=d")); + } + #[test] fn endpoint_product_announcement_is_client_rendered_modal_and_dismissed_by_identity() { let mut state = ClientShellState::new(ClientShellConfig::from_config(&Config::default())); diff --git a/src/client/shell/composition.rs b/src/client/shell/composition.rs index 6cb21683..6432f3e1 100644 --- a/src/client/shell/composition.rs +++ b/src/client/shell/composition.rs @@ -452,6 +452,7 @@ impl ClientShellState { .scroll .min(u16::try_from(self.hits.release_notes_max_scroll).unwrap_or(u16::MAX)); } + self.compose_graphics(&mut frame, layout); Some(frame) } } diff --git a/src/client/shell/graphics.rs b/src/client/shell/graphics.rs new file mode 100644 index 00000000..1fc23414 --- /dev/null +++ b/src/client/shell/graphics.rs @@ -0,0 +1,67 @@ +use super::*; + +impl ClientShellState { + #[cfg(unix)] + pub(crate) fn graphics_scope(&self) -> &str { + self.snapshot + .as_deref() + .map(|snapshot| snapshot.boot_id.as_str()) + .unwrap_or_default() + } + + #[cfg(unix)] + pub(crate) fn trust_direct_graphics_asset( + &mut self, + key: &crate::protocol::SurfaceGraphicsAssetKey, + image_id: u32, + ) -> bool { + self.graphics.trust_direct_asset(key, image_id) + } + + #[cfg(unix)] + pub(crate) fn retire_direct_graphics_image(&mut self, image_id: u32) { + self.graphics.retire_direct_image(image_id); + } + + pub(crate) fn take_pending_graphics_cleanup(&mut self) -> Vec { + self.graphics.take_pending_cleanup() + } + + pub(crate) fn set_graphics_cell_size(&mut self, width_px: u32, height_px: u32) { + self.graphics_cell_size = crate::kitty_graphics::HostCellSize { + width_px: width_px.max(1), + height_px: height_px.max(1), + }; + } + + pub(super) fn compose_graphics(&mut self, frame: &mut FrameData, layout: ClientShellLayout) { + let local_cover = self.overlay.is_some() + || self.mode != ClientShellMode::Terminal + || self.endpoint_error.is_some() + || self.config_diagnostic.is_some() + || self.visible_notification.is_some() + || self.copy_feedback.is_some() + || self + .selection + .as_ref() + .is_some_and(|selection| selection.is_visible()); + let visibility = if local_cover { + crate::kitty_graphics::surface::Visibility::Hidden + } else if self.hits.popup.is_some() { + crate::kitty_graphics::surface::Visibility::Popup + } else { + crate::kitty_graphics::surface::Visibility::Main + }; + let popup_origin = self + .hits + .popup + .as_ref() + .map(|popup| (popup.inner_rect.x, popup.inner_rect.y)); + frame.graphics = self.graphics.encode( + visibility, + (layout.pane_surface.x, layout.pane_surface.y), + popup_origin, + self.graphics_cell_size, + ); + } +} diff --git a/src/client/shell/state.rs b/src/client/shell/state.rs index 032de683..a21de1d8 100644 --- a/src/client/shell/state.rs +++ b/src/client/shell/state.rs @@ -757,6 +757,8 @@ pub(crate) struct ClientShellState { pub(super) config: ClientShellConfig, pub(super) snapshot: Option>, pub(super) pane_surface: Option, + pub(super) graphics: crate::kitty_graphics::surface::ClientState, + pub(super) graphics_cell_size: crate::kitty_graphics::HostCellSize, pub(super) popup_terminal_id: Option, pub(super) sidebar_collapsed: bool, pub(super) sidebar_collapsed_manual: bool, @@ -881,6 +883,11 @@ impl ClientShellState { config, snapshot: None, pane_surface: None, + graphics: crate::kitty_graphics::surface::ClientState::default(), + graphics_cell_size: crate::kitty_graphics::HostCellSize { + width_px: 1, + height_px: 1, + }, popup_terminal_id: None, sidebar_collapsed, sidebar_collapsed_manual: preferences.sidebar_collapsed.is_some(), @@ -1011,6 +1018,7 @@ impl ClientShellState { } pub(crate) fn set_snapshot(&mut self, snapshot: Box) { + self.graphics.set_scope(&snapshot.boot_id); self.config_diagnostic = super::config::merged_config_diagnostic( self.local_config_diagnostic.as_deref(), snapshot.config_diagnostic.as_deref(), @@ -1242,7 +1250,7 @@ impl ClientShellState { self.resume_mobile_switcher_if_ready(); } - pub(crate) fn set_pane_surface(&mut self, surface: PaneSurfaceFrame) { + pub(crate) fn set_pane_surface(&mut self, mut surface: PaneSurfaceFrame) { if self .snapshot .as_ref() @@ -1372,6 +1380,8 @@ impl ClientShellState { self.selection_highlight_clear_deadline = None; } self.popup_terminal_id = next_popup; + self.graphics + .set_scene(std::mem::take(&mut surface.graphics)); self.pane_surface = Some(surface); self.resume_mobile_switcher_if_ready(); } diff --git a/src/kitty_graphics.rs b/src/kitty_graphics.rs index eece347c..9af2318d 100644 --- a/src/kitty_graphics.rs +++ b/src/kitty_graphics.rs @@ -17,6 +17,8 @@ use crate::ghostty::{ use crate::layout::{PaneId, PaneInfo}; use crate::terminal::TerminalRuntimeRegistry; +pub(crate) mod surface; + const KITTY_CHUNK_BYTES: usize = 3072; const MAX_OVERSIZED_SOURCES: usize = 256; pub(crate) const HEADLESS_GRAPHICS_TRANSACTION_BUDGET: usize = @@ -87,8 +89,18 @@ struct HostPlacement { #[derive(Debug, Clone, Hash, PartialEq, Eq)] enum HostSourceKey { - Terminal { pane_id: PaneId, image_id: u32 }, - PaneLayer { pane_id: PaneId, layer_id: String }, + Terminal { + pane_id: PaneId, + image_id: u32, + }, + PaneLayer { + pane_id: PaneId, + layer_id: String, + }, + ClientSurface { + scope: String, + source: crate::protocol::SurfaceGraphicsSource, + }, } #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] @@ -703,10 +715,13 @@ fn image_transaction_fits(placement: &HostPlacement, budget: Option) -> b let Some(budget) = budget else { return true; }; - let data = placement.placement.data_len; - let encoded = data.div_ceil(3).saturating_mul(4); - let command_overhead = data.div_ceil(KITTY_CHUNK_BYTES).saturating_mul(16) + 1024; - encoded.saturating_add(command_overhead) <= budget + image_transfer_estimated_size(placement.placement.data_len) <= budget +} + +pub(crate) fn image_transfer_estimated_size(data_len: usize) -> usize { + let encoded = data_len.div_ceil(3).saturating_mul(4); + let command_overhead = data_len.div_ceil(KITTY_CHUNK_BYTES).saturating_mul(16) + 1024; + encoded.saturating_add(command_overhead) } fn placement_identity(placement: &HostPlacement) -> (HostSourceKey, u32) { @@ -720,6 +735,12 @@ fn source_order(source: &HostSourceKey) -> (u32, String) { match source { HostSourceKey::Terminal { pane_id, .. } => (pane_id.raw(), String::new()), HostSourceKey::PaneLayer { pane_id, layer_id } => (pane_id.raw(), layer_id.clone()), + HostSourceKey::ClientSurface { scope, source } => { + let mut hasher = DefaultHasher::new(); + scope.hash(&mut hasher); + source.hash(&mut hasher); + (hasher.finish() as u32, format!("{source:?}")) + } } } @@ -1284,6 +1305,11 @@ fn host_placement_id(source_key: &HostSourceKey, placement: &KittyImagePlacement pane_id.raw().hash(&mut hasher); layer_id.hash(&mut hasher); } + HostSourceKey::ClientSurface { scope, source } => { + "client.surface".hash(&mut hasher); + scope.hash(&mut hasher); + source.hash(&mut hasher); + } } placement.image_id.hash(&mut hasher); placement.placement_id.hash(&mut hasher); diff --git a/src/kitty_graphics/surface.rs b/src/kitty_graphics/surface.rs new file mode 100644 index 00000000..1210dc16 --- /dev/null +++ b/src/kitty_graphics/surface.rs @@ -0,0 +1,1062 @@ +use std::collections::{HashMap, HashSet}; +use std::hash::{Hash, Hasher}; + +use ratatui::layout::Rect; + +use super::{ + clipped_placement, collect_visible_placements, encode_graphics_update_incremental, + HostCellSize, HostGraphicsCache, HostPlacement, HostSourceKey, ImageSignature, +}; +use crate::ghostty::{ + KittyImageDescriptor, KittyImageFormat, KittyImagePlacement, KittyPlacementRenderInfo, +}; +use crate::layout::PaneId; +use crate::protocol::{ + SurfaceGraphicsAsset, SurfaceGraphicsAssetKey, SurfaceGraphicsFormat, SurfaceGraphicsPlacement, + SurfaceGraphicsScene, SurfaceGraphicsSource, SurfaceGraphicsTarget, +}; + +const MAX_SURFACE_GRAPHICS_PLACEMENTS: usize = 4_096; + +#[derive(Clone, Debug, Default)] +pub(crate) struct DeliveryCache { + assets: HashSet, + pending: bool, +} + +impl DeliveryCache { + pub(crate) fn has_pending(&self) -> bool { + self.pending + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Visibility { + Main, + Popup, + Hidden, +} + +#[derive(Debug, Default)] +pub(crate) struct ClientState { + scope: String, + scene: SurfaceGraphicsScene, + assets: HashMap>, + host: HostGraphicsCache, + trusted_direct: HashMap, + reset_pending: bool, + stale_images: Vec, + forced_delete_images: Vec, +} + +impl ClientState { + pub(crate) fn set_scope(&mut self, scope: &str) { + if self.scope == scope { + return; + } + self.scope = scope.to_owned(); + self.scene = SurfaceGraphicsScene::default(); + self.assets.clear(); + self.trusted_direct.clear(); + self.stale_images.clear(); + self.forced_delete_images.clear(); + self.reset_pending = true; + } + + #[cfg(unix)] + pub(crate) fn trust_direct_asset( + &mut self, + key: &SurfaceGraphicsAssetKey, + image_id: u32, + ) -> bool { + if image_id != host_image_id(&self.scope, key) { + return false; + } + self.host + .images + .insert(image_id, image_signature_from_asset(key)); + if self + .scene + .placements + .iter() + .all(|placement| &placement.asset != key) + && !self.scene.retained_assets.contains(key) + { + self.trusted_direct.insert(key.clone(), image_id); + } + true + } + + #[cfg(unix)] + pub(crate) fn retire_direct_image(&mut self, image_id: u32) { + self.trusted_direct + .retain(|_, trusted| *trusted != image_id); + self.forced_delete_images.push(image_id); + } + + pub(crate) fn take_pending_cleanup(&mut self) -> Vec { + let mut bytes = if self.reset_pending { + self.reset_pending = false; + self.stale_images.clear(); + self.host.clear_bytes() + } else { + Vec::new() + }; + self.forced_delete_images.sort_unstable(); + self.forced_delete_images.dedup(); + for image_id in self.forced_delete_images.drain(..) { + self.host.images.remove(&image_id); + self.host.placements.retain(|(id, _), _| *id != image_id); + self.host.sources.retain(|_, id| *id != image_id); + self.host + .replayed_placements + .retain(|(id, _)| *id != image_id); + super::encode_delete_image(&mut bytes, image_id); + } + bytes + } + + pub(crate) fn set_scene(&mut self, mut scene: SurfaceGraphicsScene) { + let desired = scene + .placements + .iter() + .map(|placement| placement.asset.clone()) + .chain(scene.retained_assets.iter().cloned()) + .collect::>(); + let previous = self + .scene + .placements + .iter() + .map(|placement| placement.asset.clone()) + .chain(self.scene.retained_assets.iter().cloned()) + .collect::>(); + self.stale_images.extend( + previous + .difference(&desired) + .map(|key| host_image_id(&self.scope, key)), + ); + let unclaimed = self + .trusted_direct + .iter() + .filter_map(|(key, image_id)| (!desired.contains(key)).then_some(*image_id)) + .collect::>(); + self.stale_images.extend(unclaimed); + self.trusted_direct.clear(); + let placed = scene + .placements + .iter() + .map(|placement| placement.asset.clone()) + .collect::>(); + self.assets.retain(|key, _| placed.contains(key)); + for asset in std::mem::take(&mut scene.assets) { + if asset.data.len() as u64 == asset.key.data_len && placed.contains(&asset.key) { + self.assets.insert(asset.key, asset.data); + } + } + self.scene = scene; + } + + pub(crate) fn encode( + &mut self, + visibility: Visibility, + main_origin: (u16, u16), + popup_origin: Option<(u16, u16)>, + cell_size: HostCellSize, + ) -> Vec { + let mut bytes = self.take_pending_cleanup(); + self.stale_images.sort_unstable(); + self.stale_images.dedup(); + for image_id in self.stale_images.drain(..) { + if self.host.images.remove(&image_id).is_some() { + super::encode_delete_image(&mut bytes, image_id); + } + self.host.placements.retain(|(id, _), _| *id != image_id); + self.host.sources.retain(|_, id| *id != image_id); + self.host + .replayed_placements + .retain(|(id, _)| *id != image_id); + } + if !cell_size.is_known() || self.scope.is_empty() { + bytes.extend(self.host.clear_bytes()); + return bytes; + } + + let placements = self + .scene + .placements + .iter() + .filter_map(|placement| { + client_host_placement( + &self.scope, + placement, + self.assets.get(&placement.asset).map(Vec::as_slice), + visibility, + main_origin, + popup_origin, + cell_size, + ) + }) + .collect::>(); + self.host.request_placement_replay(); + loop { + let encoded = encode_graphics_update_incremental( + &mut self.host, + &placements, + &HashSet::new(), + None, + false, + ); + bytes.extend(encoded.bytes); + if !encoded.incomplete { + return bytes; + } + } + } +} + +pub(crate) fn pane_layer_asset_key( + app: &crate::app::App, + key: &crate::app::pane_graphics::Key, + layer: &crate::app::pane_graphics::Layer, +) -> Option { + let workspace_index = app + .state + .workspaces + .iter() + .position(|workspace| workspace.pane_state(key.0).is_some())?; + Some(SurfaceGraphicsAssetKey { + source: SurfaceGraphicsSource::PaneLayer { + pane_id: app.public_pane_id(workspace_index, key.0)?, + layer_id: key.1.clone(), + }, + image_width: layer.image_width, + image_height: layer.image_height, + format: match layer.format { + crate::api::schema::PaneGraphicsFormat::Rgb => SurfaceGraphicsFormat::Rgb, + crate::api::schema::PaneGraphicsFormat::Rgba + | crate::api::schema::PaneGraphicsFormat::Bgra => SurfaceGraphicsFormat::Rgba, + crate::api::schema::PaneGraphicsFormat::Png => SurfaceGraphicsFormat::Png, + }, + data_len: layer.data_len() as u64, + data_fingerprint: layer.data_fingerprint, + }) +} + +pub(crate) fn host_image_id(scope: &str, key: &SurfaceGraphicsAssetKey) -> u32 { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + scope.hash(&mut hasher); + key.hash(&mut hasher); + 10_000 + ((hasher.finish() as u32) % 900_000) +} + +pub(crate) fn direct_upload_control(scope: &str, key: &SurfaceGraphicsAssetKey) -> (u32, String) { + let image_id = host_image_id(scope, key); + ( + image_id, + format!( + "a=t,f={},s={},v={},i={image_id},q=0", + format_code(key.format), + key.image_width, + key.image_height + ), + ) +} + +pub(crate) fn collect_scene( + app: &crate::app::App, + surface: crate::ui::TabSurfaceView<'_>, + popup_content_size: Option<(u16, u16)>, + cell_size: HostCellSize, + delivered: &DeliveryCache, + client_id: u64, +) -> (SurfaceGraphicsScene, DeliveryCache) { + if !cell_size.is_known() { + return (SurfaceGraphicsScene::default(), DeliveryCache::default()); + } + let workspace_index = app.state.active; + let mut targets = HashMap::new(); + let mut public_panes = HashMap::new(); + if let Some(workspace_index) = workspace_index { + for pane in surface.pane_infos { + if let Some(public_id) = app.public_pane_id(workspace_index, pane.id) { + public_panes.insert(public_id.clone(), pane.id); + targets.insert(pane.id, SurfaceGraphicsTarget::Pane { pane_id: public_id }); + } + } + } + let popup_target = app.state.popup_pane.as_ref().map(|popup| { + ( + popup.pane_id, + SurfaceGraphicsTarget::Popup { + terminal_id: popup.terminal_id.to_string(), + }, + ) + }); + if let Some((pane_id, target)) = popup_target.as_ref() { + targets.insert(*pane_id, target.clone()); + } + + // Reconstruct only the small image-signature index expected by the existing + // collector. This prevents copying already-delivered image payloads on each + // pane-scaled render while keeping Ghostty as the authoritative image store. + let mut uploaded_images = HashMap::new(); + let mut delivered_terminal_images = HashMap::new(); + for key in &delivered.assets { + let signature = image_signature_from_asset(key); + match &key.source { + SurfaceGraphicsSource::Terminal { + target: SurfaceGraphicsTarget::Pane { pane_id }, + image_id, + } => { + if let Some(pane_id) = public_panes.get(pane_id) { + delivered_terminal_images.insert( + HostSourceKey::Terminal { + pane_id: *pane_id, + image_id: *image_id, + }, + signature, + ); + } + } + SurfaceGraphicsSource::PaneLayer { pane_id, layer_id } => { + if let Some(pane_id) = public_panes.get(pane_id) { + if let Some(slot) = app.pane_graphics.slots.get(&(*pane_id, layer_id.clone())) { + uploaded_images.insert(slot.host_image_id, signature); + } + } + } + SurfaceGraphicsSource::Terminal { + target: SurfaceGraphicsTarget::Popup { .. }, + .. + } => {} + } + } + let mut host_placements = collect_visible_placements( + &app.state, + &app.pane_graphics, + &app.terminal_runtimes, + surface, + cell_size, + &uploaded_images, + &delivered_terminal_images, + ); + + if let (Some(popup), Some((width, height)), Some((_, target))) = ( + app.state.popup_pane.as_ref(), + popup_content_size, + popup_target.as_ref(), + ) { + if let Some(runtime) = app.terminal_runtimes.get(&popup.terminal_id) { + let mut requested = HashSet::new(); + for placement in runtime.kitty_image_placements_with_data_filter(|descriptor| { + let key = asset_key_from_descriptor( + SurfaceGraphicsSource::Terminal { + target: target.clone(), + image_id: descriptor.image_id, + }, + descriptor, + ); + !delivered.assets.contains(&key) && requested.insert(key) + }) { + host_placements.push(HostPlacement { + pane_id: popup.pane_id, + host_image_id: None, + area: Rect::new(0, 0, width, height), + cell_size, + source_key: HostSourceKey::Terminal { + pane_id: popup.pane_id, + image_id: placement.image_id, + }, + placement, + scrollback_offset: runtime + .scroll_metrics() + .map(|metrics| metrics.offset_from_bottom as u32) + .unwrap_or(0), + }); + } + } + } + + let mut placements = Vec::new(); + let mut asset_data = HashMap::>::new(); + for mut placement in host_placements { + if placements.len() == MAX_SURFACE_GRAPHICS_PLACEMENTS { + break; + } + let Some(target) = targets.get(&placement.pane_id).cloned() else { + continue; + }; + let source = match &placement.source_key { + HostSourceKey::Terminal { image_id, .. } => SurfaceGraphicsSource::Terminal { + target, + image_id: *image_id, + }, + HostSourceKey::PaneLayer { layer_id, .. } => { + let SurfaceGraphicsTarget::Pane { pane_id } = target else { + continue; + }; + SurfaceGraphicsSource::PaneLayer { + pane_id, + layer_id: layer_id.clone(), + } + } + HostSourceKey::ClientSurface { .. } => continue, + }; + let Some((clipped, _)) = clipped_placement(&placement) else { + continue; + }; + let asset = asset_key(source, &placement.placement); + if !placement.placement.data.is_empty() { + asset_data + .entry(asset.clone()) + .or_insert_with(|| std::mem::take(&mut placement.placement.data)); + } + placements.push(SurfaceGraphicsPlacement { + asset, + logical_placement_id: placement.placement.placement_id, + x: clipped.x, + y: clipped.y, + cols: clipped.cols, + rows: clipped.rows, + source_x: clipped.source_x, + source_y: clipped.source_y, + source_width: clipped.source_width, + source_height: clipped.source_height, + x_offset: clipped.x_offset, + y_offset: clipped.y_offset, + z: placement.placement.z, + scrollback_offset: placement.scrollback_offset, + }); + } + + let desired = placements + .iter() + .map(|placement| placement.asset.clone()) + .collect::>(); + let mut next = DeliveryCache { + assets: delivered.assets.intersection(&desired).cloned().collect(), + pending: false, + }; + let mut assets = Vec::new(); + let mut available = asset_data.into_iter().collect::>(); + available.sort_by_key(|(key, _)| format!("{:?}", key.source)); + let mut payload_bytes = 0usize; + for (key, data) in available { + if next.assets.contains(&key) { + continue; + } + let encoded_size = super::image_transfer_estimated_size(data.len()); + if encoded_size > super::HEADLESS_GRAPHICS_TRANSACTION_BUDGET { + continue; + } + if payload_bytes.saturating_add(encoded_size) > super::HEADLESS_GRAPHICS_TRANSACTION_BUDGET + { + next.pending = true; + continue; + } + payload_bytes = payload_bytes.saturating_add(encoded_size); + assets.push(SurfaceGraphicsAsset { + key: key.clone(), + data, + }); + next.assets.insert(key); + } + assets.sort_by_key(|asset| format!("{:?}", asset.key.source)); + placements.sort_by_key(|placement| { + ( + format!("{:?}", placement.asset.source), + placement.logical_placement_id, + placement.y, + placement.x, + ) + }); + let mut retained_assets = app + .pane_graphics + .slots + .iter() + .filter_map(|(key, slot)| { + let layer = slot.layer.as_ref()?; + let retained_for_client = layer.resident_client() == Some(client_id) + || slot + .direct_gate + .as_ref() + .is_some_and(|gate| gate.client_id == client_id); + retained_for_client.then(|| pane_layer_asset_key(app, key, layer))? + }) + .collect::>(); + retained_assets.sort_by_key(|key| format!("{:?}", key.source)); + ( + SurfaceGraphicsScene { + assets, + placements, + retained_assets, + }, + next, + ) +} + +fn image_signature_from_asset(key: &SurfaceGraphicsAssetKey) -> ImageSignature { + ImageSignature { + image_width: key.image_width, + image_height: key.image_height, + format_code: format_code(key.format), + data_len: usize::try_from(key.data_len).unwrap_or(usize::MAX), + data_fingerprint: key.data_fingerprint, + } +} + +fn asset_key_from_descriptor( + source: SurfaceGraphicsSource, + descriptor: KittyImageDescriptor, +) -> SurfaceGraphicsAssetKey { + SurfaceGraphicsAssetKey { + source, + image_width: descriptor.image_width, + image_height: descriptor.image_height, + format: match descriptor.format { + KittyImageFormat::Rgb => SurfaceGraphicsFormat::Rgb, + KittyImageFormat::Rgba => SurfaceGraphicsFormat::Rgba, + KittyImageFormat::Png => SurfaceGraphicsFormat::Png, + }, + data_len: descriptor.data_len as u64, + data_fingerprint: descriptor.data_fingerprint, + } +} + +fn asset_key( + source: SurfaceGraphicsSource, + placement: &KittyImagePlacement, +) -> SurfaceGraphicsAssetKey { + SurfaceGraphicsAssetKey { + source, + image_width: placement.image_width, + image_height: placement.image_height, + format: match placement.format { + KittyImageFormat::Rgb => SurfaceGraphicsFormat::Rgb, + KittyImageFormat::Rgba => SurfaceGraphicsFormat::Rgba, + KittyImageFormat::Png => SurfaceGraphicsFormat::Png, + }, + data_len: placement.data_len as u64, + data_fingerprint: placement.data_fingerprint, + } +} + +fn client_host_placement( + scope: &str, + placement: &SurfaceGraphicsPlacement, + data: Option<&[u8]>, + visibility: Visibility, + main_origin: (u16, u16), + popup_origin: Option<(u16, u16)>, + cell_size: HostCellSize, +) -> Option { + let origin = match (&placement.asset.source, visibility) { + ( + SurfaceGraphicsSource::Terminal { + target: SurfaceGraphicsTarget::Popup { .. }, + .. + }, + Visibility::Popup, + ) => popup_origin?, + ( + SurfaceGraphicsSource::Terminal { + target: SurfaceGraphicsTarget::Pane { .. }, + .. + }, + Visibility::Main | Visibility::Popup, + ) + | (SurfaceGraphicsSource::PaneLayer { .. }, Visibility::Main | Visibility::Popup) => { + main_origin + } + _ => return None, + }; + let source_key = HostSourceKey::ClientSurface { + scope: scope.to_owned(), + source: placement.asset.source.clone(), + }; + let signature = ImageSignature { + image_width: placement.asset.image_width, + image_height: placement.asset.image_height, + format_code: format_code(placement.asset.format), + data_len: usize::try_from(placement.asset.data_len).unwrap_or(usize::MAX), + data_fingerprint: placement.asset.data_fingerprint, + }; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + scope.hash(&mut hasher); + placement.asset.source.hash(&mut hasher); + signature.hash(&mut hasher); + let raw = hasher.finish(); + let pane_id = PaneId::from_raw((raw as u32).max(1)); + let host_image_id = host_image_id(scope, &placement.asset); + let cols = placement.cols.min(u32::from(u16::MAX)) as u16; + let rows = placement.rows.min(u32::from(u16::MAX)) as u16; + Some(HostPlacement { + pane_id, + host_image_id: Some(host_image_id), + area: Rect::new( + origin.0.saturating_add(placement.x), + origin.1.saturating_add(placement.y), + cols, + rows, + ), + cell_size, + source_key, + placement: KittyImagePlacement { + image_id: 1, + placement_id: placement.logical_placement_id, + z: placement.z, + x_offset: placement.x_offset, + y_offset: placement.y_offset, + image_width: placement.asset.image_width, + image_height: placement.asset.image_height, + format: match placement.asset.format { + SurfaceGraphicsFormat::Rgb => KittyImageFormat::Rgb, + SurfaceGraphicsFormat::Rgba => KittyImageFormat::Rgba, + SurfaceGraphicsFormat::Png => KittyImageFormat::Png, + }, + data_len: usize::try_from(placement.asset.data_len).unwrap_or(usize::MAX), + data_fingerprint: placement.asset.data_fingerprint, + data: data.unwrap_or_default().to_vec(), + render: KittyPlacementRenderInfo { + pixel_width: placement.cols.saturating_mul(cell_size.width_px), + pixel_height: placement.rows.saturating_mul(cell_size.height_px), + grid_cols: placement.cols, + grid_rows: placement.rows, + viewport_col: 0, + viewport_row: 0, + source_x: placement.source_x, + source_y: placement.source_y, + source_width: placement.source_width, + source_height: placement.source_height, + }, + }, + scrollback_offset: placement.scrollback_offset, + }) +} + +fn format_code(format: SurfaceGraphicsFormat) -> u32 { + match format { + SurfaceGraphicsFormat::Rgb => 24, + SurfaceGraphicsFormat::Rgba => 32, + SurfaceGraphicsFormat::Png => 100, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn asset( + target: SurfaceGraphicsTarget, + fingerprint: u64, + data: Vec, + ) -> SurfaceGraphicsAsset { + SurfaceGraphicsAsset { + key: SurfaceGraphicsAssetKey { + source: SurfaceGraphicsSource::Terminal { + target, + image_id: 7, + }, + image_width: 1, + image_height: 1, + format: SurfaceGraphicsFormat::Rgba, + data_len: data.len() as u64, + data_fingerprint: fingerprint, + }, + data, + } + } + + fn scene(asset: SurfaceGraphicsAsset, x: u16, y: u16) -> SurfaceGraphicsScene { + SurfaceGraphicsScene { + placements: vec![SurfaceGraphicsPlacement { + asset: asset.key.clone(), + logical_placement_id: 3, + x, + y, + cols: 1, + rows: 1, + source_x: 0, + source_y: 0, + source_width: 1, + source_height: 1, + x_offset: 0, + y_offset: 0, + z: 0, + scrollback_offset: 0, + }], + assets: vec![asset], + retained_assets: Vec::new(), + } + } + + #[test] + fn client_encodes_final_main_origin_upload_once_and_replays_placement() { + let mut state = ClientState::default(); + state.set_scope("endpoint-a:boot-1"); + let image = asset( + SurfaceGraphicsTarget::Pane { + pane_id: "w1:p1".into(), + }, + 11, + vec![1, 2, 3, 4], + ); + state.set_scene(scene(image, 1, 2)); + + let first = state.encode( + Visibility::Main, + (10, 5), + None, + HostCellSize { + width_px: 8, + height_px: 16, + }, + ); + assert!(String::from_utf8_lossy(&first).contains("a=t,t=d")); + assert!(String::from_utf8_lossy(&first).contains("\u{1b}[8;12H")); + + let second = state.encode( + Visibility::Main, + (10, 5), + None, + HostCellSize { + width_px: 8, + height_px: 16, + }, + ); + let second = String::from_utf8_lossy(&second); + assert!(!second.contains("a=t,t=d")); + assert!(second.contains("a=p")); + assert!(second.contains("\u{1b}[8;12H")); + } + + #[test] + fn client_hides_and_restores_without_reuploading_pixels() { + let mut state = ClientState::default(); + state.set_scope("endpoint-a:boot-1"); + let image = asset( + SurfaceGraphicsTarget::Pane { + pane_id: "w1:p1".into(), + }, + 12, + vec![4, 3, 2, 1], + ); + state.set_scene(scene(image, 0, 0)); + let cell = HostCellSize { + width_px: 8, + height_px: 16, + }; + let _ = state.encode(Visibility::Main, (4, 2), None, cell); + + let hidden = state.encode(Visibility::Hidden, (4, 2), None, cell); + assert!(String::from_utf8_lossy(&hidden).contains("a=d,d=i")); + + let restored = state.encode(Visibility::Main, (4, 2), None, cell); + let restored = String::from_utf8_lossy(&restored); + assert!(restored.contains("a=p")); + assert!(!restored.contains("a=t,t=d")); + } + + #[test] + fn popup_candidates_use_client_resolved_popup_inner_origin() { + let mut state = ClientState::default(); + state.set_scope("endpoint-a:boot-1"); + let image = asset( + SurfaceGraphicsTarget::Popup { + terminal_id: "terminal-popup".into(), + }, + 13, + vec![9, 8, 7, 6], + ); + state.set_scene(scene(image, 2, 1)); + + let bytes = state.encode( + Visibility::Popup, + (20, 4), + Some((30, 10)), + HostCellSize { + width_px: 8, + height_px: 16, + }, + ); + assert!(String::from_utf8_lossy(&bytes).contains("\u{1b}[12;33H")); + } + + #[test] + fn trusted_direct_asset_is_placed_without_inline_reupload() { + let mut state = ClientState::default(); + state.set_scope("endpoint-a:boot-1"); + let _ = state.encode( + Visibility::Hidden, + (0, 0), + None, + HostCellSize { + width_px: 8, + height_px: 16, + }, + ); + let image = asset( + SurfaceGraphicsTarget::Pane { + pane_id: "w1:p1".into(), + }, + 15, + vec![1, 2, 3, 4], + ); + let image_id = host_image_id("endpoint-a:boot-1", &image.key); + assert!(state.trust_direct_asset(&image.key, image_id)); + let mut direct_scene = scene(image, 0, 0); + direct_scene.assets.clear(); + state.set_scene(direct_scene); + + let bytes = state.encode( + Visibility::Main, + (0, 0), + None, + HostCellSize { + width_px: 8, + height_px: 16, + }, + ); + let bytes = String::from_utf8_lossy(&bytes); + assert!(bytes.contains("a=p")); + assert!(!bytes.contains("a=t,t=d")); + } + + #[test] + fn direct_asset_trusted_after_scene_arrival_is_immediately_placeable() { + let mut state = ClientState::default(); + state.set_scope("endpoint-a:boot-1"); + let _ = state.take_pending_cleanup(); + let image = asset( + SurfaceGraphicsTarget::Pane { + pane_id: "w1:p1".into(), + }, + 18, + vec![1, 2, 3, 4], + ); + let image_id = host_image_id("endpoint-a:boot-1", &image.key); + let mut direct_scene = scene(image.clone(), 0, 0); + direct_scene.assets.clear(); + state.set_scene(direct_scene); + assert!(state.trust_direct_asset(&image.key, image_id)); + + let bytes = state.encode( + Visibility::Main, + (0, 0), + None, + HostCellSize { + width_px: 8, + height_px: 16, + }, + ); + let bytes = String::from_utf8_lossy(&bytes); + assert!(bytes.contains("a=p"), "{bytes}"); + assert!(!bytes.contains("a=t,t=d"), "{bytes}"); + } + + #[test] + fn retained_direct_asset_survives_hidden_scene_and_replays_without_upload() { + let mut state = ClientState::default(); + state.set_scope("endpoint-a:boot-1"); + let _ = state.take_pending_cleanup(); + let image = asset( + SurfaceGraphicsTarget::Pane { + pane_id: "w1:p1".into(), + }, + 19, + vec![1, 2, 3, 4], + ); + let image_id = host_image_id("endpoint-a:boot-1", &image.key); + let mut active = scene(image.clone(), 0, 0); + active.assets.clear(); + state.set_scene(active.clone()); + assert!(state.trust_direct_asset(&image.key, image_id)); + let _ = state.encode( + Visibility::Main, + (0, 0), + None, + HostCellSize { + width_px: 8, + height_px: 16, + }, + ); + + state.set_scene(SurfaceGraphicsScene { + retained_assets: vec![image.key.clone()], + ..SurfaceGraphicsScene::default() + }); + let hidden = String::from_utf8(state.encode( + Visibility::Main, + (0, 0), + None, + HostCellSize { + width_px: 8, + height_px: 16, + }, + )) + .unwrap(); + assert!(!hidden.contains(&format!("a=d,d=I,i={image_id}"))); + + active.retained_assets.push(image.key.clone()); + state.set_scene(active); + let restored = String::from_utf8(state.encode( + Visibility::Main, + (0, 0), + None, + HostCellSize { + width_px: 8, + height_px: 16, + }, + )) + .unwrap(); + assert!(restored.contains("a=p"), "{restored}"); + assert!(!restored.contains("a=t,t=d"), "{restored}"); + + state.set_scene(SurfaceGraphicsScene::default()); + let removed = String::from_utf8(state.encode( + Visibility::Main, + (0, 0), + None, + HostCellSize { + width_px: 8, + height_px: 16, + }, + )) + .unwrap(); + assert!(removed.contains(&format!("a=d,d=I,i={image_id}"))); + } + + #[test] + fn popup_visibility_keeps_uncovered_main_scene_placements() { + let mut state = ClientState::default(); + state.set_scope("endpoint-a:boot-1"); + let main = asset( + SurfaceGraphicsTarget::Pane { + pane_id: "w1:p1".into(), + }, + 20, + vec![1, 2, 3, 4], + ); + let popup = asset( + SurfaceGraphicsTarget::Popup { + terminal_id: "popup-1".into(), + }, + 21, + vec![4, 3, 2, 1], + ); + let mut graphics = scene(main, 0, 0); + let popup_scene = scene(popup, 0, 0); + graphics.assets.extend(popup_scene.assets); + graphics.placements.extend(popup_scene.placements); + state.set_scene(graphics); + + let bytes = String::from_utf8(state.encode( + Visibility::Popup, + (2, 1), + Some((20, 10)), + HostCellSize { + width_px: 8, + height_px: 16, + }, + )) + .unwrap(); + assert!(bytes.contains("\u{1b}[2;3H"), "{bytes}"); + assert!(bytes.contains("\u{1b}[11;21H"), "{bytes}"); + } + + #[test] + fn retired_pending_direct_asset_is_deleted_even_before_cache_adoption() { + let mut state = ClientState::default(); + state.set_scope("endpoint-a:boot-1"); + let _ = state.take_pending_cleanup(); + state.retire_direct_image(4242); + let cleanup = String::from_utf8(state.take_pending_cleanup()).unwrap(); + assert!(cleanup.contains("a=d,d=I,i=4242"), "{cleanup}"); + } + + #[test] + fn unclaimed_direct_asset_is_deleted_by_the_next_authoritative_scene() { + let mut state = ClientState::default(); + state.set_scope("endpoint-a:boot-1"); + let _ = state.take_pending_cleanup(); + let image = asset( + SurfaceGraphicsTarget::Pane { + pane_id: "w1:p1".into(), + }, + 16, + vec![1, 2, 3, 4], + ); + let image_id = host_image_id("endpoint-a:boot-1", &image.key); + assert!(state.trust_direct_asset(&image.key, image_id)); + state.set_scene(SurfaceGraphicsScene::default()); + + let bytes = state.encode( + Visibility::Hidden, + (0, 0), + None, + HostCellSize { + width_px: 8, + height_px: 16, + }, + ); + let bytes = String::from_utf8_lossy(&bytes); + assert!(bytes.contains(&format!("a=d,d=I,i={image_id}")), "{bytes}"); + } + + #[test] + fn boot_scope_cleanup_does_not_wait_for_a_coherent_surface() { + let mut state = ClientState::default(); + state.set_scope("endpoint-a:boot-1"); + let image = asset( + SurfaceGraphicsTarget::Pane { + pane_id: "w1:p1".into(), + }, + 17, + vec![1, 2, 3, 4], + ); + state.set_scene(scene(image, 0, 0)); + let _ = state.encode( + Visibility::Main, + (0, 0), + None, + HostCellSize { + width_px: 8, + height_px: 16, + }, + ); + + state.set_scope("endpoint-a:boot-2"); + let cleanup = String::from_utf8(state.take_pending_cleanup()).unwrap(); + assert!(cleanup.contains("a=d,d=I"), "{cleanup}"); + assert!(state.take_pending_cleanup().is_empty()); + } + + #[test] + fn replacement_scene_without_repeated_asset_bytes_keeps_resident_data() { + let mut state = ClientState::default(); + state.set_scope("endpoint-a:boot-1"); + let image = asset( + SurfaceGraphicsTarget::Pane { + pane_id: "w1:p1".into(), + }, + 14, + vec![1, 1, 1, 1], + ); + let first_scene = scene(image, 0, 0); + let mut replacement = first_scene.clone(); + replacement.assets.clear(); + state.set_scene(first_scene); + state.set_scene(replacement); + + let bytes = state.encode( + Visibility::Main, + (0, 0), + None, + HostCellSize { + width_px: 8, + height_px: 16, + }, + ); + assert!(String::from_utf8_lossy(&bytes).contains("a=t,t=d")); + } +} diff --git a/src/protocol/wire.rs b/src/protocol/wire.rs index d9813526..632817d2 100644 --- a/src/protocol/wire.rs +++ b/src/protocol/wire.rs @@ -560,6 +560,7 @@ pub enum ClientMessage { requested_encoding: RenderEncoding, surface_size: ClientSurfaceSize, pixel_mouse: bool, + direct_graphics: bool, }, /// Resize the outer terminal and pane viewport of a client-owned shell. @@ -996,6 +997,78 @@ impl From for SurfaceRect { } } +#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub enum SurfaceGraphicsTarget { + Pane { pane_id: String }, + Popup { terminal_id: String }, +} + +#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub enum SurfaceGraphicsSource { + Terminal { + target: SurfaceGraphicsTarget, + image_id: u32, + }, + PaneLayer { + pane_id: String, + layer_id: String, + }, +} + +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub enum SurfaceGraphicsFormat { + Rgb, + Rgba, + Png, +} + +#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub struct SurfaceGraphicsAssetKey { + pub source: SurfaceGraphicsSource, + pub image_width: u32, + pub image_height: u32, + pub format: SurfaceGraphicsFormat, + pub data_len: u64, + pub data_fingerprint: u64, +} + +/// Image bytes newly needed by this connection's complete desired scene. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SurfaceGraphicsAsset { + pub key: SurfaceGraphicsAssetKey, + pub data: Vec, +} + +/// One already-clipped desired placement relative to its target surface. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SurfaceGraphicsPlacement { + pub asset: SurfaceGraphicsAssetKey, + pub logical_placement_id: u32, + pub x: u16, + pub y: u16, + pub cols: u32, + pub rows: u32, + pub source_x: u32, + pub source_y: u32, + pub source_width: u32, + pub source_height: u32, + pub x_offset: u32, + pub y_offset: u32, + pub z: i32, + pub scrollback_offset: u32, +} + +/// Complete desired placements plus only the image bytes not already sent for +/// the current live scene on this connection. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct SurfaceGraphicsScene { + pub assets: Vec, + pub placements: Vec, + /// Direct-uploaded assets that remain live for this client even while their + /// pane is outside the selected scene. + pub retained_assets: Vec, +} + /// One server-rendered active-tab surface without sidebar, tab bar, or overlays. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct PaneSurfaceFrame { @@ -1005,6 +1078,7 @@ pub struct PaneSurfaceFrame { pub panes: Vec, pub splits: Vec, pub popup: Option>, + pub graphics: SurfaceGraphicsScene, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -1174,6 +1248,8 @@ pub enum ServerMessage { transfer_id: u64, leading: Vec, control: String, + /// ClientShell upload identity. `None` retains the released App path. + surface_asset: Option, }, /// Suppress a direct command that expired before terminal delivery. @@ -2168,6 +2244,7 @@ mod tests { transfer_id: 7, leading: b"\x1b[2;3H".to_vec(), control: "a=T,f=32,i=42,q=0".into(), + surface_asset: None, }; let encoded = bincode::serde::encode_to_vec(&server, bincode::config::standard()).unwrap(); let (decoded, _): (ServerMessage, _) = diff --git a/src/server/client_shell.rs b/src/server/client_shell.rs index ce31b5b3..0349f4a8 100644 --- a/src/server/client_shell.rs +++ b/src/server/client_shell.rs @@ -196,17 +196,23 @@ pub(super) fn snapshot( } } +pub(super) struct RenderedPaneSurface { + pub(super) frame: FrameData, + pub(super) panes: Vec, + pub(super) splits: Vec, + pub(super) popup: Option>, + pub(super) graphics: protocol::SurfaceGraphicsScene, + pub(super) graphics_delivery: crate::kitty_graphics::surface::DeliveryCache, +} + pub(super) fn render_pane_surface( app: &mut app::App, area: Rect, is_foreground: bool, cell_size: crate::kitty_graphics::HostCellSize, -) -> ( - FrameData, - Vec, - Vec, - Option>, -) { + graphics_delivery: &crate::kitty_graphics::surface::DeliveryCache, + client_id: u64, +) -> RenderedPaneSurface { let content_revisions_before = app .state .active @@ -331,12 +337,23 @@ pub(super) fn render_pane_surface( }) .collect(); let popup = render_popup_surface(app, area, is_foreground, cell_size); - ( - FrameData::from_ratatui_buffer_with_hyperlinks(&buffer, cursor, &hyperlinks), + let (graphics, next_graphics_delivery) = crate::server::client_shell_graphics::collect( + app, + &layout.pane_infos, + &layout.split_borders, + popup.as_deref(), + cell_size, + graphics_delivery, + client_id, + ); + RenderedPaneSurface { + frame: FrameData::from_ratatui_buffer_with_hyperlinks(&buffer, cursor, &hyperlinks), panes, splits, popup, - ) + graphics, + graphics_delivery: next_graphics_delivery, + } } fn render_popup_surface( diff --git a/src/server/client_shell_graphics.rs b/src/server/client_shell_graphics.rs new file mode 100644 index 00000000..b16001cf --- /dev/null +++ b/src/server/client_shell_graphics.rs @@ -0,0 +1,25 @@ +use crate::kitty_graphics::surface::DeliveryCache; +use crate::protocol::{ClientShellPopupSurface, SurfaceGraphicsScene}; + +pub(crate) fn collect( + app: &crate::app::App, + pane_infos: &[crate::layout::PaneInfo], + split_borders: &[crate::layout::SplitBorder], + popup: Option<&ClientShellPopupSurface>, + cell_size: crate::kitty_graphics::HostCellSize, + delivered: &DeliveryCache, + client_id: u64, +) -> (SurfaceGraphicsScene, DeliveryCache) { + let popup_content_size = popup.map(|popup| (popup.frame.width, popup.frame.height)); + crate::kitty_graphics::surface::collect_scene( + app, + crate::ui::TabSurfaceView { + pane_infos, + split_borders, + }, + popup_content_size, + cell_size, + delivered, + client_id, + ) +} diff --git a/src/server/client_transport.rs b/src/server/client_transport.rs index dc002c9a..754c4144 100644 --- a/src/server/client_transport.rs +++ b/src/server/client_transport.rs @@ -326,6 +326,7 @@ pub(crate) enum ServerEvent { cell_width_px: u32, cell_height_px: u32, pixel_mouse: bool, + direct_graphics: bool, writer: ClientWriter, }, /// A client sent an input message. @@ -704,6 +705,7 @@ pub(crate) fn handle_client_handshake( requested_encoding, surface_size, pixel_mouse, + direct_graphics, } => { if let protocol::VersionCheck::Incompatible(reason) = protocol::check_client_version(version) @@ -739,7 +741,7 @@ pub(crate) fn handle_client_handshake( requested_encoding, None, false, - false, + direct_graphics, pixel_mouse, true, ) @@ -803,6 +805,7 @@ pub(crate) fn handle_client_handshake( cell_width_px, cell_height_px, pixel_mouse, + direct_graphics, writer, } } else { @@ -1650,6 +1653,7 @@ new_tab = "ctrl+notakey" requested_encoding: RenderEncoding::SemanticFrame, surface_size: crate::protocol::ClientSurfaceSize { cols: 80, rows: 29 }, pixel_mouse: true, + direct_graphics: true, }, ) .expect("write shell hello"); @@ -1675,12 +1679,14 @@ new_tab = "ctrl+notakey" cell_width_px, cell_height_px, pixel_mouse, + direct_graphics, writer, } => { assert_eq!(client_id, 43); assert_eq!((surface_cols, surface_rows), (80, 29)); assert_eq!((cell_width_px, cell_height_px), (8, 16)); assert!(pixel_mouse); + assert!(direct_graphics); drop(writer); } other => panic!("expected ClientShellConnected, got {other:?}"), diff --git a/src/server/clients.rs b/src/server/clients.rs index 37d4e3b2..efd4c879 100644 --- a/src/server/clients.rs +++ b/src/server/clients.rs @@ -54,8 +54,10 @@ pub(crate) struct ClientConnection { pub(crate) last_activity: u64, /// Render baseline for the negotiated client encoding. pub(crate) render_state: ClientRenderState, - /// Client-local host Kitty graphics cache. + /// Client-local host Kitty graphics cache for the legacy server-rendered app path. pub(crate) graphics_cache: crate::kitty_graphics::HostGraphicsCache, + /// Image assets already included in the selected ClientShell scene. + pub(crate) shell_graphics_delivery: crate::kitty_graphics::surface::DeliveryCache, /// Passive eligibility for audited local Kitty regular-file graphics. pub(crate) direct_graphics: bool, /// Whether this frontend preserves exact SGR pixel reports. @@ -133,6 +135,7 @@ impl ClientConnection { last_activity, render_state: ClientRenderState::new(render_encoding), graphics_cache: crate::kitty_graphics::HostGraphicsCache::default(), + shell_graphics_delivery: crate::kitty_graphics::surface::DeliveryCache::default(), direct_graphics: false, pixel_mouse: false, graphics_surface_reset_pending: false, diff --git a/src/server/headless.rs b/src/server/headless.rs index b13df48d..47224588 100644 --- a/src/server/headless.rs +++ b/src/server/headless.rs @@ -1563,7 +1563,9 @@ impl HeadlessServer { self.app_client_count() == 1 && self.foreground_client_id.is_some_and(|id| { self.clients.get(&id).is_some_and(|client| { - client.is_full_app_client() && client.writer.is_some() && client.direct_graphics + client.is_app_surface_client() + && client.writer.is_some() + && client.direct_graphics }) }) } @@ -3256,6 +3258,7 @@ impl HeadlessServer { cell_width_px, cell_height_px, pixel_mouse, + direct_graphics, writer, } => { if self.handoff_in_progress { @@ -3296,6 +3299,7 @@ impl HeadlessServer { Some(writer), ); connection.pixel_mouse = pixel_mouse; + connection.direct_graphics = direct_graphics; connection.shell_projection_revision = 1; let snapshot = client_shell_snapshot( &self.app, @@ -4918,6 +4922,11 @@ impl HeadlessServer { } shell_projection_revision = client.shell_projection_revision; } + let shell_graphics_delivery = self + .clients + .get(&client_id) + .map(|client| client.shell_graphics_delivery.clone()) + .unwrap_or_default(); let mut surface_parts = None; let mut frame = match mode { ClientConnectionMode::App => { @@ -4977,17 +4986,26 @@ impl HeadlessServer { } else { crate::kitty_graphics::HostCellSize::default() }; - let (frame, panes, splits, popup) = render_client_shell_pane_surface( + let crate::server::client_shell::RenderedPaneSurface { + frame, + panes, + splits, + popup, + graphics, + graphics_delivery: next_graphics_delivery, + } = render_client_shell_pane_surface( &mut self.app, area, is_foreground, render_cell_size, + &shell_graphics_delivery, + client_id, ); crate::render_prof::duration_since( "full_render.render_tab_surface_virtual", render_started, ); - surface_parts = Some((panes, splits, popup)); + surface_parts = Some((panes, splits, popup, graphics, next_graphics_delivery)); frame } ClientConnectionMode::TerminalAttach { terminal_id } @@ -5086,8 +5104,17 @@ impl HeadlessServer { commit_graphics_cache = false; encoded.incomplete = false; } - let has_graphics = !frame.graphics.is_empty(); - let prepared = if let Some((panes, splits, popup)) = surface_parts { + let has_graphics = !frame.graphics.is_empty() + || surface_parts + .as_ref() + .is_some_and(|(_, _, _, graphics, _)| { + !graphics.assets.is_empty() + || !graphics.placements.is_empty() + || !graphics.retained_assets.is_empty() + }); + let mut next_shell_graphics_delivery = None; + let prepared = if let Some((panes, splits, popup, graphics, delivery)) = surface_parts { + next_shell_graphics_delivery = Some(delivery); client .render_state .prepare_pane_surface(protocol::PaneSurfaceFrame { @@ -5096,6 +5123,7 @@ impl HeadlessServer { panes, splits, popup, + graphics, }) } else { client.render_state.prepare_frame(frame) @@ -5119,6 +5147,7 @@ impl HeadlessServer { } else { crate::protocol::MAX_FRAME_SIZE }; + let mut shell_assets_deferred = false; let serialized = match Self::frame_server_message_with_max(prepared.message(), max) { Ok(frame) => frame, Err(protocol::FramingError::Oversized { claimed, max }) if has_graphics => { @@ -5126,19 +5155,28 @@ impl HeadlessServer { client_id, claimed, max, "dropping graphics from oversized frame for client" ); - let Some(mut text_only_frame) = prepared.into_frame() else { - crate::render_prof::event("full_render.serialize_error"); - continue; + let framed = if prepared.strip_pane_surface_assets() { + next_shell_graphics_delivery = None; + shell_assets_deferred = true; + Self::frame_server_message(prepared.message()) + } else { + let Some(mut text_only_frame) = prepared.into_frame() else { + crate::render_prof::event("full_render.serialize_error"); + continue; + }; + text_only_frame.graphics.clear(); + let Some(text_only_prepared) = + client.render_state.prepare_frame(text_only_frame) + else { + client.clear_deferred_render(); + crate::render_prof::event("full_render.skip_identical_text_only"); + continue; + }; + let result = Self::frame_server_message(text_only_prepared.message()); + prepared = text_only_prepared; + result }; - text_only_frame.graphics.clear(); - let Some(text_only_prepared) = - client.render_state.prepare_frame(text_only_frame) - else { - client.clear_deferred_render(); - crate::render_prof::event("full_render.skip_identical_text_only"); - continue; - }; - let framed = match Self::frame_server_message(text_only_prepared.message()) { + let framed = match framed { Ok(framed) => framed, Err(err) => { warn!(client_id, err = %err, "failed to serialize text-only frame for client"); @@ -5147,7 +5185,6 @@ impl HeadlessServer { continue; } }; - prepared = text_only_prepared; commit_graphics_cache = false; encoded.incomplete = false; framed @@ -5167,14 +5204,20 @@ impl HeadlessServer { continue; } }; + let shell_graphics_pending = next_shell_graphics_delivery + .as_ref() + .is_some_and(crate::kitty_graphics::surface::DeliveryCache::has_pending); match writer.render.try_send(serialized) { Ok(()) => { if commit_graphics_cache { client.graphics_cache = next_graphics_cache; client.graphics_surface_reset_pending = false; } + if let Some(delivery) = next_shell_graphics_delivery { + client.shell_graphics_delivery = delivery; + } client.render_state.commit_sent_frame(prepared); - if encoded.incomplete { + if encoded.incomplete || shell_graphics_pending || shell_assets_deferred { client.defer_full_render(); deferred_frame = true; } else { @@ -6529,6 +6572,7 @@ mod tests { cell_width_px: 0, cell_height_px: 0, pixel_mouse: false, + direct_graphics: false, writer, }) ); @@ -6576,6 +6620,7 @@ mod tests { cell_width_px: 10, cell_height_px: 20, pixel_mouse: true, + direct_graphics: false, writer, }) ); @@ -6647,6 +6692,7 @@ mod tests { cell_width_px: 0, cell_height_px: 0, pixel_mouse: false, + direct_graphics: false, writer, }) ); @@ -6789,7 +6835,7 @@ mod tests { 40, 12, 0, - b"POPUP_SHELL_LIVE", + b"POPUP_SHELL_LIVE\x1b_Ga=T,f=32,t=d,i=9,p=4,s=1,v=1,c=1,r=1,q=2;/wAA/w==\x1b\\", 4, ); let (_, popup_terminal_id) = server.app.install_test_popup_runtime(popup_runtime); @@ -6800,9 +6846,10 @@ mod tests { client_id: 12, surface_cols: 80, surface_rows: 23, - cell_width_px: 0, - cell_height_px: 0, + cell_width_px: 10, + cell_height_px: 20, pixel_mouse: false, + direct_graphics: false, writer, }) ); @@ -6821,6 +6868,15 @@ mod tests { assert_eq!(popup.terminal_id, popup_terminal_id.as_str()); assert!(frame_text(&popup.frame).contains("POPUP_SHELL_LIVE")); assert_eq!((popup.frame.width, popup.frame.height), (37, 9)); + assert_eq!(surface.graphics.assets.len(), 1); + assert_eq!(surface.graphics.placements.len(), 1); + assert!(matches!( + surface.graphics.placements[0].asset.source, + crate::protocol::SurfaceGraphicsSource::Terminal { + target: crate::protocol::SurfaceGraphicsTarget::Popup { .. }, + image_id: 9, + } + )); assert!( !server.handle_server_event(ServerEvent::ClientShellPaneInput { diff --git a/src/server/headless/pane_graphics.rs b/src/server/headless/pane_graphics.rs index 151b26ee..bcc33bc4 100644 --- a/src/server/headless/pane_graphics.rs +++ b/src/server/headless/pane_graphics.rs @@ -76,7 +76,6 @@ impl HeadlessServer { .and_then(crate::app::pane_graphics::Layer::direct_lease) .map(|lease| { ( - slot.host_image_id, lease.path().to_string_lossy().into_owned(), lease.len() as u64, lease.fingerprint(), @@ -84,21 +83,53 @@ impl HeadlessServer { }) }) }); - if let (Some(key), Some((image_id, path, expected_len, transfer_id))) = + if let (Some(key), Some((path, expected_len, transfer_id))) = (direct_key.clone(), direct_frame) { - let command = self.clients.get(&client_id).and_then(|client| { - crate::kitty_graphics::prepare_direct_file( - &self.app.state, - &self.app.pane_graphics, - self.app.state.view.tab_surface(), - client.cell_size, - !internal_changed, - &client.graphics_cache, - &key, - ) + let prepared = self.clients.get(&client_id).and_then(|client| { + if matches!(client.mode, ClientConnectionMode::ClientShell) { + let layer = self + .app + .pane_graphics + .slots + .get(&key) + .and_then(|slot| slot.layer.as_ref())?; + let asset = crate::kitty_graphics::surface::pane_layer_asset_key( + &self.app, &key, layer, + )?; + let (image_id, control) = + crate::kitty_graphics::surface::direct_upload_control( + &self.client_shell_boot_id, + &asset, + ); + Some(( + crate::kitty_graphics::DirectFileCommand { + leading: Vec::new(), + control, + }, + Some(asset), + image_id, + )) + } else { + let image_id = self + .app + .pane_graphics + .slots + .get(&key) + .map(|slot| slot.host_image_id)?; + crate::kitty_graphics::prepare_direct_file( + &self.app.state, + &self.app.pane_graphics, + self.app.state.view.tab_surface(), + client.cell_size, + !internal_changed, + &client.graphics_cache, + &key, + ) + .map(|command| (command, None, image_id)) + } }); - let Some(command) = command else { + let Some((command, surface_asset, image_id)) = prepared else { if self.install_inline_fallback(&key) { if msg.respond_to.send(response).is_err() { self.retire_direct_gate(&key); @@ -123,6 +154,7 @@ impl HeadlessServer { transfer_id, leading: command.leading, control: command.control, + surface_asset, }; let send = Self::frame_server_message_with_max(&message, MAX_GRAPHICS_FRAME_SIZE) @@ -144,6 +176,7 @@ impl HeadlessServer { if let Some(slot) = self.app.pane_graphics.slots.get_mut(&key) { slot.direct_gate = Some(crate::app::pane_graphics::DirectGate { transfer_id, + image_id, client_id, deadline: std::time::Instant::now() + crate::app::pane_graphics::DIRECT_DELIVERY_TIMEOUT, @@ -267,10 +300,14 @@ impl HeadlessServer { image_id: u32, ) -> bool { if let Some(gate) = self.app.pane_graphics.slots.values_mut().find_map(|slot| { - (slot.host_image_id == image_id && slot.stream_is_active()) + slot.stream_is_active() .then_some(slot.direct_gate.as_mut()) .flatten() - .filter(|gate| gate.client_id == client_id && gate.transfer_id == transfer_id) + .filter(|gate| { + gate.client_id == client_id + && gate.transfer_id == transfer_id + && gate.image_id == image_id + }) }) { gate.written = true; gate.deadline = @@ -297,7 +334,7 @@ impl HeadlessServer { slot.stream_is_active() && gate.client_id == client_id && gate.transfer_id == transfer_id - && slot.host_image_id == image_id + && gate.image_id == image_id && (!success || gate.written) }) .then(|| key.clone()) @@ -335,7 +372,9 @@ impl HeadlessServer { ) }; if let (Some(client), Some(layer)) = ( - self.clients.get_mut(&client_id), + self.clients + .get_mut(&client_id) + .filter(|client| matches!(client.mode, ClientConnectionMode::App)), self.app .pane_graphics .slots @@ -362,8 +401,19 @@ impl HeadlessServer { self.retire_all_direct_graphics(); return true; } - if let Some(client) = self.clients.get_mut(&client_id) { - client.graphics_cache.forget_pane_layer(&key, image_id); + if let Some(client) = self + .clients + .get_mut(&client_id) + .filter(|client| matches!(client.mode, ClientConnectionMode::App)) + { + let host_image_id = self + .app + .pane_graphics + .slots + .get(&key) + .map(|slot| slot.host_image_id) + .unwrap_or(image_id); + client.graphics_cache.forget_pane_layer(&key, host_image_id); } let gate = self .app @@ -404,7 +454,7 @@ impl HeadlessServer { *client_id, ServerMessage::GraphicsTransmissionRetired { transfer_id: gate.transfer_id, - image_id: slot.host_image_id, + image_id: gate.image_id, }, ); } diff --git a/src/server/headless/tests/pane_graphics.rs b/src/server/headless/tests/pane_graphics.rs index bd6e3cd7..8a6f4f45 100644 --- a/src/server/headless/tests/pane_graphics.rs +++ b/src/server/headless/tests/pane_graphics.rs @@ -49,6 +49,173 @@ async fn cold_redraw_advances_one_bounded_layer_after_each_send() { assert_eq!(server.clients[&1].deferred_render(), DeferredRender::None); } +#[tokio::test] +async fn client_shell_surface_sends_complete_placements_and_each_live_asset_once() { + let (mut server, client_rx, pane_id) = retained_test_server(b"client shell graphics"); + let client = server.clients.get_mut(&1).unwrap(); + client.mode = ClientConnectionMode::ClientShell; + client.render_state = + crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); + client.cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + set_graphics_layer(&mut server, pane_id, vec![1, 2, 3, 4]); + + server.render_and_stream(); + let first = read_server_message(receive_render(&client_rx, Duration::from_millis(100))); + let ServerMessage::PaneSurface(first) = first else { + panic!("expected client shell pane surface"); + }; + assert_eq!(first.graphics.placements.len(), 1); + assert_eq!(first.graphics.assets.len(), 1); + assert_eq!(first.graphics.assets[0].data, vec![1, 2, 3, 4]); + assert!(matches!( + first.graphics.placements[0].asset.source, + crate::protocol::SurfaceGraphicsSource::PaneLayer { .. } + )); + + server.clients.get_mut(&1).unwrap().request_repaint(); + server.render_and_stream(); + let second = read_server_message(receive_render(&client_rx, Duration::from_millis(100))); + let ServerMessage::PaneSurface(second) = second else { + panic!("expected replacement client shell pane surface"); + }; + assert_eq!(second.graphics.placements, first.graphics.placements); + assert!(second.graphics.assets.is_empty()); +} + +#[tokio::test] +async fn client_shell_asset_delivery_is_bounded_to_the_current_live_scene() { + let (mut server, client_rx, pane_id) = retained_test_server(b"client shell graphics"); + let client = server.clients.get_mut(&1).unwrap(); + client.mode = ClientConnectionMode::ClientShell; + client.render_state = + crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); + client.cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + set_graphics_layer(&mut server, pane_id, vec![1, 2, 3, 4]); + + server.render_and_stream(); + let _first = receive_render(&client_rx, Duration::from_millis(100)); + server.app.pane_graphics.slots.clear(); + server.app.pane_graphics.mark_changed(); + server.clients.get_mut(&1).unwrap().request_repaint(); + server.render_and_stream(); + let removed = read_server_message(receive_render(&client_rx, Duration::from_millis(100))); + let ServerMessage::PaneSurface(removed) = removed else { + panic!("expected removed client shell scene"); + }; + assert!(removed.graphics.placements.is_empty()); + + set_graphics_layer(&mut server, pane_id, vec![1, 2, 3, 4]); + server.clients.get_mut(&1).unwrap().request_repaint(); + server.render_and_stream(); + let restored = read_server_message(receive_render(&client_rx, Duration::from_millis(100))); + let ServerMessage::PaneSurface(restored) = restored else { + panic!("expected restored client shell scene"); + }; + assert_eq!(restored.graphics.assets.len(), 1); + assert_eq!(restored.graphics.assets[0].data, vec![1, 2, 3, 4]); +} + +#[tokio::test] +async fn client_shell_surface_projects_terminal_kitty_images_from_authoritative_runtime() { + let (mut server, client_rx, _pane_id) = + retained_test_server(b"\x1b_Ga=T,f=32,t=d,i=7,p=3,s=1,v=1,c=1,r=1,q=2;/wAA/w==\x1b\\"); + let client = server.clients.get_mut(&1).unwrap(); + client.mode = ClientConnectionMode::ClientShell; + client.render_state = + crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); + client.cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + + server.render_and_stream(); + let message = read_server_message(receive_render(&client_rx, Duration::from_millis(100))); + let ServerMessage::PaneSurface(surface) = message else { + panic!("expected client shell pane surface"); + }; + assert_eq!(surface.graphics.placements.len(), 1); + assert_eq!(surface.graphics.assets.len(), 1); + assert!(matches!( + surface.graphics.placements[0].asset.source, + crate::protocol::SurfaceGraphicsSource::Terminal { + target: crate::protocol::SurfaceGraphicsTarget::Pane { .. }, + image_id: 7, + } + )); + assert_eq!(surface.graphics.assets[0].data, vec![255, 0, 0, 255]); +} + +#[tokio::test] +async fn client_shell_delivers_equal_pixels_for_distinct_terminal_image_ids() { + let (mut server, client_rx, _pane_id) = retained_test_server( + b"\x1b_Ga=T,f=32,t=d,i=7,p=3,s=1,v=1,c=1,r=1,q=2;/wAA/w==\x1b\\\x1b_Ga=T,f=32,t=d,i=8,p=4,s=1,v=1,c=1,r=1,q=2;/wAA/w==\x1b\\", + ); + let client = server.clients.get_mut(&1).unwrap(); + client.mode = ClientConnectionMode::ClientShell; + client.render_state = + crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); + client.cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + + server.render_and_stream(); + let message = read_server_message(receive_render(&client_rx, Duration::from_millis(100))); + let ServerMessage::PaneSurface(surface) = message else { + panic!("expected client shell pane surface"); + }; + assert_eq!(surface.graphics.placements.len(), 2); + assert_eq!(surface.graphics.assets.len(), 2); + assert_ne!( + surface.graphics.assets[0].key, + surface.graphics.assets[1].key + ); + assert_eq!( + surface.graphics.assets[0].data, + surface.graphics.assets[1].data + ); + + server.clients.get_mut(&1).unwrap().request_repaint(); + server.render_and_stream(); + let message = read_server_message(receive_render(&client_rx, Duration::from_millis(100))); + let ServerMessage::PaneSurface(surface) = message else { + panic!("expected replacement client shell pane surface"); + }; + assert_eq!(surface.graphics.placements.len(), 2); + assert!(surface.graphics.assets.is_empty()); +} + +#[tokio::test] +async fn full_client_shell_render_lane_does_not_commit_graphics_delivery() { + let (mut server, client_rx, pane_id) = retained_test_server(b"client shell graphics"); + let client = server.clients.get_mut(&1).unwrap(); + client.mode = ClientConnectionMode::ClientShell; + client.render_state = + crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); + client.cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + set_graphics_layer(&mut server, pane_id, vec![5, 6, 7, 8]); + + fill_render_lane(&server); + server.render_and_stream(); + let _older = receive_render(&client_rx, Duration::from_millis(100)); + server.render_and_stream(); + let message = read_server_message(receive_render(&client_rx, Duration::from_millis(100))); + let ServerMessage::PaneSurface(surface) = message else { + panic!("expected client shell pane surface"); + }; + assert_eq!(surface.graphics.assets.len(), 1); + assert_eq!(surface.graphics.assets[0].data, vec![5, 6, 7, 8]); +} + fn enable_graphics_and_render( server: &mut HeadlessServer, client_rx: &std::sync::mpsc::Receiver>, @@ -728,6 +895,106 @@ fn rejected_or_stale_requests_do_not_schedule_rendering() { .is_ok()); } +#[cfg(unix)] +#[tokio::test] +async fn client_shell_direct_graphics_uploads_without_server_authored_coordinates() { + let (mut server, client_rx, pane_id) = retained_test_server(b"client shell direct"); + server.app.state.kitty_graphics_enabled = true; + let client = server.clients.get_mut(&1).unwrap(); + client.mode = ClientConnectionMode::ClientShell; + client.render_state = + crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); + client.cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + client.direct_graphics = true; + server.app.direct_graphics_available = true; + set_stream_owner(&mut server, pane_id, "browser"); + let public_pane_id = server.app.public_pane_id(0, pane_id).unwrap(); + let path = sparse_direct_frame(&server, "client-shell-direct.rgba", 1, 1); + let (message, response_rx) = + direct_stream_message("shell-direct", &public_pane_id, "browser", path, 1, 1); + + assert_eq!( + server.handle_pane_graphics_stream_frame(message), + RenderImpact::None + ); + let (transfer_id, image_id, asset) = match read_server_message( + client_rx + .recv_timeout(Duration::from_secs(1)) + .expect("client shell direct upload"), + ) { + ServerMessage::GraphicsFile { + transfer_id, + image_id, + leading, + control, + surface_asset: Some(asset), + .. + } => { + assert!(leading.is_empty()); + assert!(control.starts_with("a=t,"), "{control}"); + assert!(!control.contains("\u{1b}["), "{control}"); + (transfer_id, image_id, asset) + } + other => panic!("expected client shell graphics file, got {other:?}"), + }; + assert_eq!( + image_id, + crate::kitty_graphics::surface::host_image_id(&server.client_shell_boot_id, &asset) + ); + let (pending, _) = crate::server::client_shell_graphics::collect( + &server.app, + &[], + &[], + None, + crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }, + &crate::kitty_graphics::surface::DeliveryCache::default(), + 1, + ); + assert_eq!(pending.retained_assets, vec![asset.clone()]); + + server.start_direct_graphics_response(1, transfer_id, image_id); + assert!(server.complete_direct_graphics(1, transfer_id, image_id, true)); + assert!(serde_json::from_str::( + &response_rx.recv_timeout(Duration::from_secs(1)).unwrap() + ) + .is_ok()); + server.clients.get_mut(&1).unwrap().request_repaint(); + server.render_and_stream(); + let surface = read_server_message( + client_rx + .recv_timeout(Duration::from_secs(1)) + .expect("resident client shell scene"), + ); + let ServerMessage::PaneSurface(surface) = surface else { + panic!("expected client shell pane surface"); + }; + assert_eq!(surface.graphics.placements.len(), 1); + assert!(surface.graphics.assets.is_empty()); + assert_eq!(surface.graphics.placements[0].asset, asset); + assert_eq!(surface.graphics.retained_assets, vec![asset.clone()]); + + let (hidden, _) = crate::server::client_shell_graphics::collect( + &server.app, + &[], + &[], + None, + crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }, + &crate::kitty_graphics::surface::DeliveryCache::default(), + 1, + ); + assert!(hidden.placements.is_empty()); + assert_eq!(hidden.retained_assets, vec![asset]); +} + #[cfg(unix)] #[tokio::test] async fn hidden_large_direct_frame_uploads_then_replays_placement_without_closing_stream() { @@ -1029,6 +1296,7 @@ fn direct_gate_server_with_file( slot.stream_active = Some(active_gate()); slot.direct_gate = Some(crate::app::pane_graphics::DirectGate { transfer_id: lease.fingerprint(), + image_id: (1 << 31) | 900, client_id: 7, deadline: std::time::Instant::now() + Duration::from_secs(1), written: true, @@ -1042,10 +1310,8 @@ fn direct_gate_server_with_file( #[cfg(unix)] fn direct_ids(server: &HeadlessServer, key: &crate::app::pane_graphics::Key) -> (u64, u32) { let slot = &server.app.pane_graphics.slots[key]; - ( - slot.direct_gate.as_ref().unwrap().transfer_id, - slot.host_image_id, - ) + let gate = slot.direct_gate.as_ref().unwrap(); + (gate.transfer_id, gate.image_id) } #[cfg(unix)] diff --git a/src/server/mod.rs b/src/server/mod.rs index ac0b8625..8cc6591e 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -3,6 +3,7 @@ pub mod autodetect; #[cfg(unix)] pub(crate) mod client_accept; pub(crate) mod client_shell; +pub(crate) mod client_shell_graphics; pub(crate) mod client_transport; pub(crate) mod clients; pub(crate) mod clipboard_image; diff --git a/src/server/render_stream.rs b/src/server/render_stream.rs index 07523475..ff7d871b 100644 --- a/src/server/render_stream.rs +++ b/src/server/render_stream.rs @@ -19,6 +19,8 @@ pub(crate) enum ClientRenderState { last_frame: Option, last_surface_panes: Option>, last_surface_popup: Option>, + last_surface_graphics_placements: Option>, + last_surface_graphics_retained: Option>, last_surface_projection_revision: Option, }, /// Terminal-ANSI clients keep a terminal diff encoder and sequence number. @@ -36,6 +38,8 @@ impl ClientRenderState { last_frame: None, last_surface_panes: None, last_surface_popup: None, + last_surface_graphics_placements: None, + last_surface_graphics_retained: None, last_surface_projection_revision: None, }, RenderEncoding::TerminalAnsi => Self::TerminalAnsi { @@ -52,11 +56,15 @@ impl ClientRenderState { last_frame, last_surface_panes, last_surface_popup, + last_surface_graphics_placements, + last_surface_graphics_retained, last_surface_projection_revision, } => { *last_frame = None; *last_surface_panes = None; *last_surface_popup = None; + *last_surface_graphics_placements = None; + *last_surface_graphics_retained = None; *last_surface_projection_revision = None; } Self::TerminalAnsi { @@ -76,11 +84,15 @@ impl ClientRenderState { last_frame, last_surface_panes, last_surface_popup, + last_surface_graphics_placements, + last_surface_graphics_retained, last_surface_projection_revision, } => { *last_frame = None; *last_surface_panes = None; *last_surface_popup = None; + *last_surface_graphics_placements = None; + *last_surface_graphics_retained = None; *last_surface_projection_revision = None; } Self::TerminalAnsi { @@ -94,12 +106,16 @@ impl ClientRenderState { last_frame, last_surface_panes, last_surface_popup, + last_surface_graphics_placements, + last_surface_graphics_retained, last_surface_projection_revision, } = self { *last_frame = None; *last_surface_panes = None; *last_surface_popup = None; + *last_surface_graphics_placements = None; + *last_surface_graphics_retained = None; *last_surface_projection_revision = None; } } @@ -165,14 +181,19 @@ impl ClientRenderState { last_frame, last_surface_panes, last_surface_popup, + last_surface_graphics_placements, + last_surface_graphics_retained, last_surface_projection_revision, } = self else { return None; }; - if last_frame.as_ref() == Some(&surface.frame) + if surface.graphics.assets.is_empty() + && last_frame.as_ref() == Some(&surface.frame) && last_surface_panes.as_ref() == Some(&surface.panes) && *last_surface_popup == surface.popup + && last_surface_graphics_placements.as_ref() == Some(&surface.graphics.placements) + && last_surface_graphics_retained.as_ref() == Some(&surface.graphics.retained_assets) && *last_surface_projection_revision == Some(surface.projection_revision) { return None; @@ -196,6 +217,8 @@ impl ClientRenderState { last_frame, last_surface_panes, last_surface_popup, + last_surface_graphics_placements, + last_surface_graphics_retained, last_surface_projection_revision, }, PreparedRender::Semantic { @@ -205,6 +228,8 @@ impl ClientRenderState { *last_frame = Some(frame); *last_surface_panes = None; *last_surface_popup = None; + *last_surface_graphics_placements = None; + *last_surface_graphics_retained = None; *last_surface_projection_revision = None; } ( @@ -212,6 +237,8 @@ impl ClientRenderState { last_frame, last_surface_panes, last_surface_popup, + last_surface_graphics_placements, + last_surface_graphics_retained, last_surface_projection_revision, }, PreparedRender::Semantic { @@ -222,6 +249,8 @@ impl ClientRenderState { *last_frame = Some(surface.frame); *last_surface_panes = Some(surface.panes); *last_surface_popup = surface.popup; + *last_surface_graphics_placements = Some(surface.graphics.placements); + *last_surface_graphics_retained = Some(surface.graphics.retained_assets); } ( Self::TerminalAnsi { @@ -275,6 +304,7 @@ mod tests { pixel_width: 0, pixel_height: 0, })), + graphics: crate::protocol::SurfaceGraphicsScene::default(), } } @@ -323,6 +353,20 @@ impl PreparedRender { } } + pub(crate) fn strip_pane_surface_assets(&mut self) -> bool { + let Self::Semantic { + message: ServerMessage::PaneSurface(surface), + } = self + else { + return false; + }; + if surface.graphics.assets.is_empty() { + return false; + } + surface.graphics.assets.clear(); + true + } + pub(crate) fn into_frame(self) -> Option { match self { Self::Semantic {