starred/herdr
1
0
Fork 0
mirror of https://github.com/herdrdev/herdr.git synced 2026-09-22 00:01:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: render kitty graphics in client shell

Browse Source
This commit is contained in:
Ogulcan Celik
2026-09-01 15:39:57 +03:00
parent 0bc6fbae67
commit c0d4bfaca7
18 changed files with 1931 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app/pane_graphics.rs
+1
View File
@@ -31,6 +31,7 @@ pub(crate) struct Layer {
#[derive(Debug)]
pub(crate) struct DirectGate {
pub(crate) transfer_id: u64,
pub(crate) image_id: u32,
pub(crate) client_id: u64,
pub(crate) deadline: std::time::Instant,
pub(crate) written: bool,
src/client/mod.rs
+93 -10
View File
@@ -17,6 +17,8 @@ mod direct_graphics;
mod input;
mod shell;
#[cfg(unix)]
use std::collections::HashMap;
use std::collections::HashSet;
#[cfg(unix)]
use std::io::IsTerminal as _;
@@ -77,6 +79,8 @@ struct ClientState {
keyboard_report_all_active: bool,
/// The terminal size we reported to the server in our last Hello/Resize.
reported_size: (u16, u16),
/// Last exact host cell size used by client-rendered surfaces.
reported_cell_size: (u32, u32),
/// Client-local sound playback config, refreshed on server request.
sound_config: crate::config::SoundConfig,
/// Whether this client may write Kitty graphics bytes to its host terminal.
@@ -87,6 +91,9 @@ struct ClientState {
/// One server-retired direct transfer to suppress if it was still queued.
#[cfg(unix)]
retired_direct_graphics: Option<(u64, u32)>,
/// ClientShell assets waiting for the host terminal's direct-upload response.
#[cfg(unix)]
pending_surface_graphics: HashMap<u64, crate::protocol::SurfaceGraphicsAssetKey>,
/// Direct attach prefix escape state. None for full-app clients.
attach_escape: Option<AttachEscapeState>,
/// Rows scrolled for one direct-attach wheel notch.
@@ -246,6 +253,15 @@ impl ClientState {
self.repaint_pending = true;
}
fn present_graphics(&mut self, graphics: &[u8]) {
if graphics.is_empty() || !self.kitty_graphics_enabled {
return;
}
let mut stdout = io::stdout();
let _ = write_encoded_frame_with_graphics(&mut stdout, &[], graphics);
let _ = stdout.flush();
}
fn present_frame(&mut self, frame_data: FrameData) {
let frame_data = if self.draw_host_cursor {
render_ansi::frame_with_drawn_cursor(frame_data)
@@ -894,6 +910,10 @@ fn do_handshake(
requested_encoding,
surface_size,
pixel_mouse: exact_cell_size && cfg!(unix),
direct_graphics: exact_cell_size
&& cell_width_px > 0
&& cell_height_px > 0
&& direct_graphics_profile_allowed(false),
}
} else {
ClientMessage::Hello {
@@ -1299,9 +1319,8 @@ fn run_client_with_mode(
let host_cursor = loaded_config.config.ui.host_cursor;
let direct_attach_requested = attach_request.is_some();
let remote_image_paste_key = client_remote_image_paste_key(&loaded_config.config);
let kitty_graphics_enabled = loaded_config.config.experimental.kitty_graphics
&& !direct_attach_requested
&& !client_rendered_shell;
let kitty_graphics_enabled =
loaded_config.config.experimental.kitty_graphics && !direct_attach_requested;
let loop_config = ClientLoopConfig {
sound_config: loaded_config.config.ui.sound,
mouse_scroll_lines,
@@ -1513,8 +1532,13 @@ fn finish_client_shell_input(
}
if outcome.resize {
let shell = state.shell.as_ref().expect("shell mode remains active");
let resize =
client_shell_resize_message(shell, state.reported_size.0, state.reported_size.1, 0, 0);
let resize = client_shell_resize_message(
shell,
state.reported_size.0,
state.reported_size.1,
state.reported_cell_size.0,
state.reported_cell_size.1,
);
write_to_server(write_stream, &resize).map_err(ClientError::ConnectionLost)?;
}
#[cfg(not(windows))]
@@ -1559,12 +1583,15 @@ async fn run_client_loop(
mouse_capture_active: config.mouse_capture_active,
keyboard_report_all_active: false,
reported_size: (cols, rows),
reported_cell_size: (initial_cell_width_px, initial_cell_height_px),
sound_config: config.sound_config,
kitty_graphics_enabled: config.kitty_graphics_enabled,
#[cfg(unix)]
direct_graphics_response: Arc::new(Mutex::new(direct_graphics::ResponseMatcher::default())),
#[cfg(unix)]
retired_direct_graphics: None,
#[cfg(unix)]
pending_surface_graphics: HashMap::new(),
attach_escape,
#[cfg(unix)]
mouse_scroll_lines: config.mouse_scroll_lines,
@@ -1574,6 +1601,9 @@ async fn run_client_loop(
draw_host_cursor,
shell: config.shell_config.map(shell::ClientShellState::new),
};
if let Some(shell) = state.shell.as_mut() {
shell.set_graphics_cell_size(initial_cell_width_px, initial_cell_height_px);
}
debug!(?negotiated_encoding, "client render encoding active");
let host_mouse_capture_active = Arc::new(AtomicBool::new(state.mouse_capture_active));
// Cell size reported by the host terminal, packed as width<<32 | height.
@@ -1856,6 +1886,17 @@ async fn run_client_loop(
}
#[cfg(unix)]
ClientLoopEvent::DirectGraphicsResponse(response) => {
let composed = state
.pending_surface_graphics
.remove(&response.transfer_id)
.filter(|_| response.success)
.and_then(|asset| {
let shell = state.shell.as_mut()?;
shell
.trust_direct_graphics_asset(&asset, response.image_id)
.then(|| shell.compose(state.reported_size.0, state.reported_size.1))
.flatten()
});
let message = ClientMessage::GraphicsTransmissionResult {
transfer_id: response.transfer_id,
image_id: response.image_id,
@@ -1864,6 +1905,9 @@ async fn run_client_loop(
if let Err(err) = write_to_server(&mut write_stream, &message) {
return Err(ClientError::ConnectionLost(err));
}
if let Some(frame) = composed {
state.present_frame(frame);
}
}
#[cfg(unix)]
ClientLoopEvent::PixelMouse(data, geometry) => {
@@ -1977,9 +2021,11 @@ async fn run_client_loop(
}
ClientLoopEvent::Resize(new_cols, new_rows, cell_width_px, cell_height_px) => {
state.reported_size = (new_cols, new_rows);
state.reported_cell_size = (cell_width_px, cell_height_px);
// Resizing invalidates both the host-side blit baseline and pane hit geometry.
state.request_repaint();
if let Some(shell) = state.shell.as_mut() {
shell.set_graphics_cell_size(cell_width_px, cell_height_px);
shell.invalidate_pane_surface();
}
let msg = if let Some(shell) = &state.shell {
@@ -2005,10 +2051,12 @@ async fn run_client_loop(
ClientLoopEvent::ServerMessage(msg) => match msg {
ServerMessage::Frame(frame_data) => state.present_frame(frame_data),
ServerMessage::ClientShellSnapshot(snapshot) => {
let (composed, resize) = if let Some(shell) = &mut state.shell {
let (composed, resize, graphics_cleanup) = if let Some(shell) = &mut state.shell
{
let previous_size =
shell.surface_size(state.reported_size.0, state.reported_size.1);
shell.set_snapshot(snapshot);
let graphics_cleanup = shell.take_pending_graphics_cleanup();
let next_size =
shell.surface_size(state.reported_size.0, state.reported_size.1);
(
@@ -2018,14 +2066,16 @@ async fn run_client_loop(
shell,
state.reported_size.0,
state.reported_size.1,
0,
0,
state.reported_cell_size.0,
state.reported_cell_size.1,
)
}),
graphics_cleanup,
)
} else {
(None, None)
(None, None, Vec::new())
};
state.present_graphics(&graphics_cleanup);
if let Some(resize) = resize {
if let Err(err) = write_to_server(&mut write_stream, &resize) {
return Err(ClientError::ConnectionLost(err));
@@ -2076,13 +2126,25 @@ async fn run_client_loop(
transfer_id,
leading,
control,
surface_asset,
} => {
#[cfg(unix)]
{
if state.retired_direct_graphics.take() == Some((transfer_id, image_id)) {
continue;
}
let surface_asset_valid = match (state.shell.as_ref(), &surface_asset) {
(Some(shell), Some(asset)) => {
crate::kitty_graphics::surface::host_image_id(
shell.graphics_scope(),
asset,
) == image_id
}
(None, None) => true,
_ => false,
};
let valid = state.kitty_graphics_enabled
&& surface_asset_valid
&& usize::try_from(expected_len).ok().is_some_and(|len| {
crate::pane_graphics_files::validate_direct_source(
std::path::Path::new(&path),
@@ -2116,6 +2178,9 @@ async fn run_client_loop(
false
};
if sent {
if let Some(asset) = surface_asset {
state.pending_surface_graphics.insert(transfer_id, asset);
}
if let Ok(mut matcher) = state.direct_graphics_response.lock() {
matcher.start(transfer_id);
}
@@ -2127,6 +2192,7 @@ async fn run_client_loop(
return Err(ClientError::ConnectionLost(err));
}
} else {
state.pending_surface_graphics.remove(&transfer_id);
if let Ok(mut matcher) = state.direct_graphics_response.lock() {
if valid {
matcher.retire(transfer_id);
@@ -2145,7 +2211,15 @@ async fn run_client_loop(
}
}
#[cfg(not(unix))]
let _ = (path, expected_len, image_id, transfer_id, leading, control);
let _ = (
path,
expected_len,
image_id,
transfer_id,
leading,
control,
surface_asset,
);
}
ServerMessage::GraphicsTransmissionRetired {
transfer_id,
@@ -2154,6 +2228,15 @@ async fn run_client_loop(
#[cfg(unix)]
{
state.retired_direct_graphics = Some((transfer_id, image_id));
state.pending_surface_graphics.remove(&transfer_id);
let cleanup = state.shell.as_mut().map_or_else(Vec::new, |shell| {
shell.retire_direct_graphics_image(image_id);
shell
.compose(state.reported_size.0, state.reported_size.1)
.map(|frame| frame.graphics)
.unwrap_or_else(|| shell.take_pending_graphics_cleanup())
});
state.present_graphics(&cleanup);
if let Ok(mut matcher) = state.direct_graphics_response.lock() {
matcher.retire(transfer_id);
}
src/client/shell.rs
+61
View File
@@ -7,6 +7,7 @@ mod config;
mod context_menu;
mod copy_mode;
mod global_menu;
mod graphics;
mod input;
mod mobile;
mod mouse;
@@ -392,6 +393,7 @@ mod tests {
}],
splits: Vec::new(),
popup: None,
graphics: crate::protocol::SurfaceGraphicsScene::default(),
}
}
@@ -499,6 +501,65 @@ mod tests {
);
}
#[test]
fn client_shell_graphics_follow_final_shell_origin_and_local_overlay_visibility() {
let mut state = ClientShellState::new(ClientShellConfig::from_config(&Config::default()));
state.set_snapshot(Box::new(snapshot()));
let mut pane_surface = surface();
let key = crate::protocol::SurfaceGraphicsAssetKey {
source: crate::protocol::SurfaceGraphicsSource::Terminal {
target: crate::protocol::SurfaceGraphicsTarget::Pane {
pane_id: "pane_1".into(),
},
image_id: 1,
},
image_width: 1,
image_height: 1,
format: crate::protocol::SurfaceGraphicsFormat::Rgba,
data_len: 4,
data_fingerprint: 17,
};
pane_surface.graphics = crate::protocol::SurfaceGraphicsScene {
assets: vec![crate::protocol::SurfaceGraphicsAsset {
key: key.clone(),
data: vec![1, 2, 3, 4],
}],
placements: vec![crate::protocol::SurfaceGraphicsPlacement {
asset: key,
logical_placement_id: 1,
x: 0,
y: 0,
cols: 1,
rows: 1,
source_x: 0,
source_y: 0,
source_width: 1,
source_height: 1,
x_offset: 0,
y_offset: 0,
z: 0,
scrollback_offset: 0,
}],
retained_assets: Vec::new(),
};
state.set_pane_surface(pane_surface);
let visible = state.compose(106, 20).expect("visible graphics frame");
let visible = String::from_utf8_lossy(&visible.graphics);
assert!(visible.contains("a=t,t=d"));
assert!(visible.contains("\u{1b}[2;27H"));
state.overlay = Some(ClientShellOverlay::Onboarding);
let hidden = state.compose(106, 20).expect("overlay frame");
assert!(String::from_utf8_lossy(&hidden.graphics).contains("a=d,d=i"));
state.overlay = None;
let restored = state.compose(106, 20).expect("restored graphics frame");
let restored = String::from_utf8_lossy(&restored.graphics);
assert!(restored.contains("a=p"));
assert!(!restored.contains("a=t,t=d"));
}
#[test]
fn endpoint_product_announcement_is_client_rendered_modal_and_dismissed_by_identity() {
let mut state = ClientShellState::new(ClientShellConfig::from_config(&Config::default()));
src/client/shell/composition.rs
+1
View File
@@ -452,6 +452,7 @@ impl ClientShellState {
.scroll
.min(u16::try_from(self.hits.release_notes_max_scroll).unwrap_or(u16::MAX));
}
self.compose_graphics(&mut frame, layout);
Some(frame)
}
}
src/client/shell/graphics.rs
+67
View File
@@ -0,0 +1,67 @@
use super::*;
impl ClientShellState {
#[cfg(unix)]
pub(crate) fn graphics_scope(&self) -> &str {
self.snapshot
.as_deref()
.map(|snapshot| snapshot.boot_id.as_str())
.unwrap_or_default()
}
#[cfg(unix)]
pub(crate) fn trust_direct_graphics_asset(
&mut self,
key: &crate::protocol::SurfaceGraphicsAssetKey,
image_id: u32,
) -> bool {
self.graphics.trust_direct_asset(key, image_id)
}
#[cfg(unix)]
pub(crate) fn retire_direct_graphics_image(&mut self, image_id: u32) {
self.graphics.retire_direct_image(image_id);
}
pub(crate) fn take_pending_graphics_cleanup(&mut self) -> Vec<u8> {
self.graphics.take_pending_cleanup()
}
pub(crate) fn set_graphics_cell_size(&mut self, width_px: u32, height_px: u32) {
self.graphics_cell_size = crate::kitty_graphics::HostCellSize {
width_px: width_px.max(1),
height_px: height_px.max(1),
};
}
pub(super) fn compose_graphics(&mut self, frame: &mut FrameData, layout: ClientShellLayout) {
let local_cover = self.overlay.is_some()
|| self.mode != ClientShellMode::Terminal
|| self.endpoint_error.is_some()
|| self.config_diagnostic.is_some()
|| self.visible_notification.is_some()
|| self.copy_feedback.is_some()
|| self
.selection
.as_ref()
.is_some_and(|selection| selection.is_visible());
let visibility = if local_cover {
crate::kitty_graphics::surface::Visibility::Hidden
} else if self.hits.popup.is_some() {
crate::kitty_graphics::surface::Visibility::Popup
} else {
crate::kitty_graphics::surface::Visibility::Main
};
let popup_origin = self
.hits
.popup
.as_ref()
.map(|popup| (popup.inner_rect.x, popup.inner_rect.y));
frame.graphics = self.graphics.encode(
visibility,
(layout.pane_surface.x, layout.pane_surface.y),
popup_origin,
self.graphics_cell_size,
);
}
}
src/client/shell/state.rs
+11 -1
View File
@@ -757,6 +757,8 @@ pub(crate) struct ClientShellState {
pub(super) config: ClientShellConfig,
pub(super) snapshot: Option<Box<ClientShellSnapshot>>,
pub(super) pane_surface: Option<PaneSurfaceFrame>,
pub(super) graphics: crate::kitty_graphics::surface::ClientState,
pub(super) graphics_cell_size: crate::kitty_graphics::HostCellSize,
pub(super) popup_terminal_id: Option<String>,
pub(super) sidebar_collapsed: bool,
pub(super) sidebar_collapsed_manual: bool,
@@ -881,6 +883,11 @@ impl ClientShellState {
config,
snapshot: None,
pane_surface: None,
graphics: crate::kitty_graphics::surface::ClientState::default(),
graphics_cell_size: crate::kitty_graphics::HostCellSize {
width_px: 1,
height_px: 1,
},
popup_terminal_id: None,
sidebar_collapsed,
sidebar_collapsed_manual: preferences.sidebar_collapsed.is_some(),
@@ -1011,6 +1018,7 @@ impl ClientShellState {
}
pub(crate) fn set_snapshot(&mut self, snapshot: Box<ClientShellSnapshot>) {
self.graphics.set_scope(&snapshot.boot_id);
self.config_diagnostic = super::config::merged_config_diagnostic(
self.local_config_diagnostic.as_deref(),
snapshot.config_diagnostic.as_deref(),
@@ -1242,7 +1250,7 @@ impl ClientShellState {
self.resume_mobile_switcher_if_ready();
}
pub(crate) fn set_pane_surface(&mut self, surface: PaneSurfaceFrame) {
pub(crate) fn set_pane_surface(&mut self, mut surface: PaneSurfaceFrame) {
if self
.snapshot
.as_ref()
@@ -1372,6 +1380,8 @@ impl ClientShellState {
self.selection_highlight_clear_deadline = None;
}
self.popup_terminal_id = next_popup;
self.graphics
.set_scene(std::mem::take(&mut surface.graphics));
self.pane_surface = Some(surface);
self.resume_mobile_switcher_if_ready();
}
src/kitty_graphics.rs
+32 -6
View File
@@ -17,6 +17,8 @@ use crate::ghostty::{
use crate::layout::{PaneId, PaneInfo};
use crate::terminal::TerminalRuntimeRegistry;
pub(crate) mod surface;
const KITTY_CHUNK_BYTES: usize = 3072;
const MAX_OVERSIZED_SOURCES: usize = 256;
pub(crate) const HEADLESS_GRAPHICS_TRANSACTION_BUDGET: usize =
@@ -87,8 +89,18 @@ struct HostPlacement {
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
enum HostSourceKey {
Terminal { pane_id: PaneId, image_id: u32 },
PaneLayer { pane_id: PaneId, layer_id: String },
Terminal {
pane_id: PaneId,
image_id: u32,
},
PaneLayer {
pane_id: PaneId,
layer_id: String,
},
ClientSurface {
scope: String,
source: crate::protocol::SurfaceGraphicsSource,
},
}
#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
@@ -703,10 +715,13 @@ fn image_transaction_fits(placement: &HostPlacement, budget: Option<usize>) -> b
let Some(budget) = budget else {
return true;
};
let data = placement.placement.data_len;
let encoded = data.div_ceil(3).saturating_mul(4);
let command_overhead = data.div_ceil(KITTY_CHUNK_BYTES).saturating_mul(16) + 1024;
encoded.saturating_add(command_overhead) <= budget
image_transfer_estimated_size(placement.placement.data_len) <= budget
}
pub(crate) fn image_transfer_estimated_size(data_len: usize) -> usize {
let encoded = data_len.div_ceil(3).saturating_mul(4);
let command_overhead = data_len.div_ceil(KITTY_CHUNK_BYTES).saturating_mul(16) + 1024;
encoded.saturating_add(command_overhead)
}
fn placement_identity(placement: &HostPlacement) -> (HostSourceKey, u32) {
@@ -720,6 +735,12 @@ fn source_order(source: &HostSourceKey) -> (u32, String) {
match source {
HostSourceKey::Terminal { pane_id, .. } => (pane_id.raw(), String::new()),
HostSourceKey::PaneLayer { pane_id, layer_id } => (pane_id.raw(), layer_id.clone()),
HostSourceKey::ClientSurface { scope, source } => {
let mut hasher = DefaultHasher::new();
scope.hash(&mut hasher);
source.hash(&mut hasher);
(hasher.finish() as u32, format!("{source:?}"))
}
}
}
@@ -1284,6 +1305,11 @@ fn host_placement_id(source_key: &HostSourceKey, placement: &KittyImagePlacement
pane_id.raw().hash(&mut hasher);
layer_id.hash(&mut hasher);
}
HostSourceKey::ClientSurface { scope, source } => {
"client.surface".hash(&mut hasher);
scope.hash(&mut hasher);
source.hash(&mut hasher);
}
}
placement.image_id.hash(&mut hasher);
placement.placement_id.hash(&mut hasher);
src/kitty_graphics/surface.rs
+1062
View File
File diff suppressed because it is too large Load Diff
src/protocol/wire.rs
+77
View File
@@ -560,6 +560,7 @@ pub enum ClientMessage {
requested_encoding: RenderEncoding,
surface_size: ClientSurfaceSize,
pixel_mouse: bool,
direct_graphics: bool,
},
/// Resize the outer terminal and pane viewport of a client-owned shell.
@@ -996,6 +997,78 @@ impl From<ratatui::layout::Rect> for SurfaceRect {
}
}
#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)]
pub enum SurfaceGraphicsTarget {
Pane { pane_id: String },
Popup { terminal_id: String },
}
#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)]
pub enum SurfaceGraphicsSource {
Terminal {
target: SurfaceGraphicsTarget,
image_id: u32,
},
PaneLayer {
pane_id: String,
layer_id: String,
},
}
#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize)]
pub enum SurfaceGraphicsFormat {
Rgb,
Rgba,
Png,
}
#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)]
pub struct SurfaceGraphicsAssetKey {
pub source: SurfaceGraphicsSource,
pub image_width: u32,
pub image_height: u32,
pub format: SurfaceGraphicsFormat,
pub data_len: u64,
pub data_fingerprint: u64,
}
/// Image bytes newly needed by this connection's complete desired scene.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SurfaceGraphicsAsset {
pub key: SurfaceGraphicsAssetKey,
pub data: Vec<u8>,
}
/// One already-clipped desired placement relative to its target surface.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SurfaceGraphicsPlacement {
pub asset: SurfaceGraphicsAssetKey,
pub logical_placement_id: u32,
pub x: u16,
pub y: u16,
pub cols: u32,
pub rows: u32,
pub source_x: u32,
pub source_y: u32,
pub source_width: u32,
pub source_height: u32,
pub x_offset: u32,
pub y_offset: u32,
pub z: i32,
pub scrollback_offset: u32,
}
/// Complete desired placements plus only the image bytes not already sent for
/// the current live scene on this connection.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct SurfaceGraphicsScene {
pub assets: Vec<SurfaceGraphicsAsset>,
pub placements: Vec<SurfaceGraphicsPlacement>,
/// Direct-uploaded assets that remain live for this client even while their
/// pane is outside the selected scene.
pub retained_assets: Vec<SurfaceGraphicsAssetKey>,
}
/// One server-rendered active-tab surface without sidebar, tab bar, or overlays.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PaneSurfaceFrame {
@@ -1005,6 +1078,7 @@ pub struct PaneSurfaceFrame {
pub panes: Vec<PaneSurfacePane>,
pub splits: Vec<PaneSurfaceSplit>,
pub popup: Option<Box<ClientShellPopupSurface>>,
pub graphics: SurfaceGraphicsScene,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -1174,6 +1248,8 @@ pub enum ServerMessage {
transfer_id: u64,
leading: Vec<u8>,
control: String,
/// ClientShell upload identity. `None` retains the released App path.
surface_asset: Option<SurfaceGraphicsAssetKey>,
},
/// Suppress a direct command that expired before terminal delivery.
@@ -2168,6 +2244,7 @@ mod tests {
transfer_id: 7,
leading: b"\x1b[2;3H".to_vec(),
control: "a=T,f=32,i=42,q=0".into(),
surface_asset: None,
};
let encoded = bincode::serde::encode_to_vec(&server, bincode::config::standard()).unwrap();
let (decoded, _): (ServerMessage, _) =
src/server/client_shell.rs
+26 -9
View File
@@ -196,17 +196,23 @@ pub(super) fn snapshot(
}
}
pub(super) struct RenderedPaneSurface {
pub(super) frame: FrameData,
pub(super) panes: Vec<protocol::PaneSurfacePane>,
pub(super) splits: Vec<protocol::PaneSurfaceSplit>,
pub(super) popup: Option<Box<protocol::ClientShellPopupSurface>>,
pub(super) graphics: protocol::SurfaceGraphicsScene,
pub(super) graphics_delivery: crate::kitty_graphics::surface::DeliveryCache,
}
pub(super) fn render_pane_surface(
app: &mut app::App,
area: Rect,
is_foreground: bool,
cell_size: crate::kitty_graphics::HostCellSize,
) -> (
FrameData,
Vec<protocol::PaneSurfacePane>,
Vec<protocol::PaneSurfaceSplit>,
Option<Box<protocol::ClientShellPopupSurface>>,
) {
graphics_delivery: &crate::kitty_graphics::surface::DeliveryCache,
client_id: u64,
) -> RenderedPaneSurface {
let content_revisions_before = app
.state
.active
@@ -331,12 +337,23 @@ pub(super) fn render_pane_surface(
})
.collect();
let popup = render_popup_surface(app, area, is_foreground, cell_size);
(
FrameData::from_ratatui_buffer_with_hyperlinks(&buffer, cursor, &hyperlinks),
let (graphics, next_graphics_delivery) = crate::server::client_shell_graphics::collect(
app,
&layout.pane_infos,
&layout.split_borders,
popup.as_deref(),
cell_size,
graphics_delivery,
client_id,
);
RenderedPaneSurface {
frame: FrameData::from_ratatui_buffer_with_hyperlinks(&buffer, cursor, &hyperlinks),
panes,
splits,
popup,
)
graphics,
graphics_delivery: next_graphics_delivery,
}
}
fn render_popup_surface(
src/server/client_shell_graphics.rs
+25
View File
@@ -0,0 +1,25 @@
use crate::kitty_graphics::surface::DeliveryCache;
use crate::protocol::{ClientShellPopupSurface, SurfaceGraphicsScene};
pub(crate) fn collect(
app: &crate::app::App,
pane_infos: &[crate::layout::PaneInfo],
split_borders: &[crate::layout::SplitBorder],
popup: Option<&ClientShellPopupSurface>,
cell_size: crate::kitty_graphics::HostCellSize,
delivered: &DeliveryCache,
client_id: u64,
) -> (SurfaceGraphicsScene, DeliveryCache) {
let popup_content_size = popup.map(|popup| (popup.frame.width, popup.frame.height));
crate::kitty_graphics::surface::collect_scene(
app,
crate::ui::TabSurfaceView {
pane_infos,
split_borders,
},
popup_content_size,
cell_size,
delivered,
client_id,
)
}
src/server/client_transport.rs
+7 -1
View File
@@ -326,6 +326,7 @@ pub(crate) enum ServerEvent {
cell_width_px: u32,
cell_height_px: u32,
pixel_mouse: bool,
direct_graphics: bool,
writer: ClientWriter,
},
/// A client sent an input message.
@@ -704,6 +705,7 @@ pub(crate) fn handle_client_handshake(
requested_encoding,
surface_size,
pixel_mouse,
direct_graphics,
} => {
if let protocol::VersionCheck::Incompatible(reason) =
protocol::check_client_version(version)
@@ -739,7 +741,7 @@ pub(crate) fn handle_client_handshake(
requested_encoding,
None,
false,
false,
direct_graphics,
pixel_mouse,
true,
)
@@ -803,6 +805,7 @@ pub(crate) fn handle_client_handshake(
cell_width_px,
cell_height_px,
pixel_mouse,
direct_graphics,
writer,
}
} else {
@@ -1650,6 +1653,7 @@ new_tab = "ctrl+notakey"
requested_encoding: RenderEncoding::SemanticFrame,
surface_size: crate::protocol::ClientSurfaceSize { cols: 80, rows: 29 },
pixel_mouse: true,
direct_graphics: true,
},
)
.expect("write shell hello");
@@ -1675,12 +1679,14 @@ new_tab = "ctrl+notakey"
cell_width_px,
cell_height_px,
pixel_mouse,
direct_graphics,
writer,
} => {
assert_eq!(client_id, 43);
assert_eq!((surface_cols, surface_rows), (80, 29));
assert_eq!((cell_width_px, cell_height_px), (8, 16));
assert!(pixel_mouse);
assert!(direct_graphics);
drop(writer);
}
other => panic!("expected ClientShellConnected, got {other:?}"),
src/server/clients.rs
+4 -1
View File
@@ -54,8 +54,10 @@ pub(crate) struct ClientConnection {
pub(crate) last_activity: u64,
/// Render baseline for the negotiated client encoding.
pub(crate) render_state: ClientRenderState,
/// Client-local host Kitty graphics cache.
/// Client-local host Kitty graphics cache for the legacy server-rendered app path.
pub(crate) graphics_cache: crate::kitty_graphics::HostGraphicsCache,
/// Image assets already included in the selected ClientShell scene.
pub(crate) shell_graphics_delivery: crate::kitty_graphics::surface::DeliveryCache,
/// Passive eligibility for audited local Kitty regular-file graphics.
pub(crate) direct_graphics: bool,
/// Whether this frontend preserves exact SGR pixel reports.
@@ -133,6 +135,7 @@ impl ClientConnection {
last_activity,
render_state: ClientRenderState::new(render_encoding),
graphics_cache: crate::kitty_graphics::HostGraphicsCache::default(),
shell_graphics_delivery: crate::kitty_graphics::surface::DeliveryCache::default(),
direct_graphics: false,
pixel_mouse: false,
graphics_surface_reset_pending: false,
src/server/headless.rs
+78 -22
View File
@@ -1563,7 +1563,9 @@ impl HeadlessServer {
self.app_client_count() == 1
&& self.foreground_client_id.is_some_and(|id| {
self.clients.get(&id).is_some_and(|client| {
client.is_full_app_client() && client.writer.is_some() && client.direct_graphics
client.is_app_surface_client()
&& client.writer.is_some()
&& client.direct_graphics
})
})
}
@@ -3256,6 +3258,7 @@ impl HeadlessServer {
cell_width_px,
cell_height_px,
pixel_mouse,
direct_graphics,
writer,
} => {
if self.handoff_in_progress {
@@ -3296,6 +3299,7 @@ impl HeadlessServer {
Some(writer),
);
connection.pixel_mouse = pixel_mouse;
connection.direct_graphics = direct_graphics;
connection.shell_projection_revision = 1;
let snapshot = client_shell_snapshot(
&self.app,
@@ -4918,6 +4922,11 @@ impl HeadlessServer {
}
shell_projection_revision = client.shell_projection_revision;
}
let shell_graphics_delivery = self
.clients
.get(&client_id)
.map(|client| client.shell_graphics_delivery.clone())
.unwrap_or_default();
let mut surface_parts = None;
let mut frame = match mode {
ClientConnectionMode::App => {
@@ -4977,17 +4986,26 @@ impl HeadlessServer {
} else {
crate::kitty_graphics::HostCellSize::default()
};
let (frame, panes, splits, popup) = render_client_shell_pane_surface(
let crate::server::client_shell::RenderedPaneSurface {
frame,
panes,
splits,
popup,
graphics,
graphics_delivery: next_graphics_delivery,
} = render_client_shell_pane_surface(
&mut self.app,
area,
is_foreground,
render_cell_size,
&shell_graphics_delivery,
client_id,
);
crate::render_prof::duration_since(
"full_render.render_tab_surface_virtual",
render_started,
);
surface_parts = Some((panes, splits, popup));
surface_parts = Some((panes, splits, popup, graphics, next_graphics_delivery));
frame
}
ClientConnectionMode::TerminalAttach { terminal_id }
@@ -5086,8 +5104,17 @@ impl HeadlessServer {
commit_graphics_cache = false;
encoded.incomplete = false;
}
let has_graphics = !frame.graphics.is_empty();
let prepared = if let Some((panes, splits, popup)) = surface_parts {
let has_graphics = !frame.graphics.is_empty()
|| surface_parts
.as_ref()
.is_some_and(|(_, _, _, graphics, _)| {
!graphics.assets.is_empty()
|| !graphics.placements.is_empty()
|| !graphics.retained_assets.is_empty()
});
let mut next_shell_graphics_delivery = None;
let prepared = if let Some((panes, splits, popup, graphics, delivery)) = surface_parts {
next_shell_graphics_delivery = Some(delivery);
client
.render_state
.prepare_pane_surface(protocol::PaneSurfaceFrame {
@@ -5096,6 +5123,7 @@ impl HeadlessServer {
panes,
splits,
popup,
graphics,
})
} else {
client.render_state.prepare_frame(frame)
@@ -5119,6 +5147,7 @@ impl HeadlessServer {
} else {
crate::protocol::MAX_FRAME_SIZE
};
let mut shell_assets_deferred = false;
let serialized = match Self::frame_server_message_with_max(prepared.message(), max) {
Ok(frame) => frame,
Err(protocol::FramingError::Oversized { claimed, max }) if has_graphics => {
@@ -5126,19 +5155,28 @@ impl HeadlessServer {
client_id,
claimed, max, "dropping graphics from oversized frame for client"
);
let Some(mut text_only_frame) = prepared.into_frame() else {
crate::render_prof::event("full_render.serialize_error");
continue;
let framed = if prepared.strip_pane_surface_assets() {
next_shell_graphics_delivery = None;
shell_assets_deferred = true;
Self::frame_server_message(prepared.message())
} else {
let Some(mut text_only_frame) = prepared.into_frame() else {
crate::render_prof::event("full_render.serialize_error");
continue;
};
text_only_frame.graphics.clear();
let Some(text_only_prepared) =
client.render_state.prepare_frame(text_only_frame)
else {
client.clear_deferred_render();
crate::render_prof::event("full_render.skip_identical_text_only");
continue;
};
let result = Self::frame_server_message(text_only_prepared.message());
prepared = text_only_prepared;
result
};
text_only_frame.graphics.clear();
let Some(text_only_prepared) =
client.render_state.prepare_frame(text_only_frame)
else {
client.clear_deferred_render();
crate::render_prof::event("full_render.skip_identical_text_only");
continue;
};
let framed = match Self::frame_server_message(text_only_prepared.message()) {
let framed = match framed {
Ok(framed) => framed,
Err(err) => {
warn!(client_id, err = %err, "failed to serialize text-only frame for client");
@@ -5147,7 +5185,6 @@ impl HeadlessServer {
continue;
}
};
prepared = text_only_prepared;
commit_graphics_cache = false;
encoded.incomplete = false;
framed
@@ -5167,14 +5204,20 @@ impl HeadlessServer {
continue;
}
};
let shell_graphics_pending = next_shell_graphics_delivery
.as_ref()
.is_some_and(crate::kitty_graphics::surface::DeliveryCache::has_pending);
match writer.render.try_send(serialized) {
Ok(()) => {
if commit_graphics_cache {
client.graphics_cache = next_graphics_cache;
client.graphics_surface_reset_pending = false;
}
if let Some(delivery) = next_shell_graphics_delivery {
client.shell_graphics_delivery = delivery;
}
client.render_state.commit_sent_frame(prepared);
if encoded.incomplete {
if encoded.incomplete || shell_graphics_pending || shell_assets_deferred {
client.defer_full_render();
deferred_frame = true;
} else {
@@ -6529,6 +6572,7 @@ mod tests {
cell_width_px: 0,
cell_height_px: 0,
pixel_mouse: false,
direct_graphics: false,
writer,
})
);
@@ -6576,6 +6620,7 @@ mod tests {
cell_width_px: 10,
cell_height_px: 20,
pixel_mouse: true,
direct_graphics: false,
writer,
})
);
@@ -6647,6 +6692,7 @@ mod tests {
cell_width_px: 0,
cell_height_px: 0,
pixel_mouse: false,
direct_graphics: false,
writer,
})
);
@@ -6789,7 +6835,7 @@ mod tests {
40,
12,
0,
b"POPUP_SHELL_LIVE",
b"POPUP_SHELL_LIVE\x1b_Ga=T,f=32,t=d,i=9,p=4,s=1,v=1,c=1,r=1,q=2;/wAA/w==\x1b\\",
4,
);
let (_, popup_terminal_id) = server.app.install_test_popup_runtime(popup_runtime);
@@ -6800,9 +6846,10 @@ mod tests {
client_id: 12,
surface_cols: 80,
surface_rows: 23,
cell_width_px: 0,
cell_height_px: 0,
cell_width_px: 10,
cell_height_px: 20,
pixel_mouse: false,
direct_graphics: false,
writer,
})
);
@@ -6821,6 +6868,15 @@ mod tests {
assert_eq!(popup.terminal_id, popup_terminal_id.as_str());
assert!(frame_text(&popup.frame).contains("POPUP_SHELL_LIVE"));
assert_eq!((popup.frame.width, popup.frame.height), (37, 9));
assert_eq!(surface.graphics.assets.len(), 1);
assert_eq!(surface.graphics.placements.len(), 1);
assert!(matches!(
surface.graphics.placements[0].asset.source,
crate::protocol::SurfaceGraphicsSource::Terminal {
target: crate::protocol::SurfaceGraphicsTarget::Popup { .. },
image_id: 9,
}
));
assert!(
!server.handle_server_event(ServerEvent::ClientShellPaneInput {
src/server/headless/pane_graphics.rs
+70 -20
View File
@@ -76,7 +76,6 @@ impl HeadlessServer {
.and_then(crate::app::pane_graphics::Layer::direct_lease)
.map(|lease| {
(
slot.host_image_id,
lease.path().to_string_lossy().into_owned(),
lease.len() as u64,
lease.fingerprint(),
@@ -84,21 +83,53 @@ impl HeadlessServer {
})
})
});
if let (Some(key), Some((image_id, path, expected_len, transfer_id))) =
if let (Some(key), Some((path, expected_len, transfer_id))) =
(direct_key.clone(), direct_frame)
{
let command = self.clients.get(&client_id).and_then(|client| {
crate::kitty_graphics::prepare_direct_file(
&self.app.state,
&self.app.pane_graphics,
self.app.state.view.tab_surface(),
client.cell_size,
!internal_changed,
&client.graphics_cache,
&key,
)
let prepared = self.clients.get(&client_id).and_then(|client| {
if matches!(client.mode, ClientConnectionMode::ClientShell) {
let layer = self
.app
.pane_graphics
.slots
.get(&key)
.and_then(|slot| slot.layer.as_ref())?;
let asset = crate::kitty_graphics::surface::pane_layer_asset_key(
&self.app, &key, layer,
)?;
let (image_id, control) =
crate::kitty_graphics::surface::direct_upload_control(
&self.client_shell_boot_id,
&asset,
);
Some((
crate::kitty_graphics::DirectFileCommand {
leading: Vec::new(),
control,
},
Some(asset),
image_id,
))
} else {
let image_id = self
.app
.pane_graphics
.slots
.get(&key)
.map(|slot| slot.host_image_id)?;
crate::kitty_graphics::prepare_direct_file(
&self.app.state,
&self.app.pane_graphics,
self.app.state.view.tab_surface(),
client.cell_size,
!internal_changed,
&client.graphics_cache,
&key,
)
.map(|command| (command, None, image_id))
}
});
let Some(command) = command else {
let Some((command, surface_asset, image_id)) = prepared else {
if self.install_inline_fallback(&key) {
if msg.respond_to.send(response).is_err() {
self.retire_direct_gate(&key);
@@ -123,6 +154,7 @@ impl HeadlessServer {
transfer_id,
leading: command.leading,
control: command.control,
surface_asset,
};
let send =
Self::frame_server_message_with_max(&message, MAX_GRAPHICS_FRAME_SIZE)
@@ -144,6 +176,7 @@ impl HeadlessServer {
if let Some(slot) = self.app.pane_graphics.slots.get_mut(&key) {
slot.direct_gate = Some(crate::app::pane_graphics::DirectGate {
transfer_id,
image_id,
client_id,
deadline: std::time::Instant::now()
+ crate::app::pane_graphics::DIRECT_DELIVERY_TIMEOUT,
@@ -267,10 +300,14 @@ impl HeadlessServer {
image_id: u32,
) -> bool {
if let Some(gate) = self.app.pane_graphics.slots.values_mut().find_map(|slot| {
(slot.host_image_id == image_id && slot.stream_is_active())
slot.stream_is_active()
.then_some(slot.direct_gate.as_mut())
.flatten()
.filter(|gate| gate.client_id == client_id && gate.transfer_id == transfer_id)
.filter(|gate| {
gate.client_id == client_id
&& gate.transfer_id == transfer_id
&& gate.image_id == image_id
})
}) {
gate.written = true;
gate.deadline =
@@ -297,7 +334,7 @@ impl HeadlessServer {
slot.stream_is_active()
&& gate.client_id == client_id
&& gate.transfer_id == transfer_id
&& slot.host_image_id == image_id
&& gate.image_id == image_id
&& (!success || gate.written)
})
.then(|| key.clone())
@@ -335,7 +372,9 @@ impl HeadlessServer {
)
};
if let (Some(client), Some(layer)) = (
self.clients.get_mut(&client_id),
self.clients
.get_mut(&client_id)
.filter(|client| matches!(client.mode, ClientConnectionMode::App)),
self.app
.pane_graphics
.slots
@@ -362,8 +401,19 @@ impl HeadlessServer {
self.retire_all_direct_graphics();
return true;
}
if let Some(client) = self.clients.get_mut(&client_id) {
client.graphics_cache.forget_pane_layer(&key, image_id);
if let Some(client) = self
.clients
.get_mut(&client_id)
.filter(|client| matches!(client.mode, ClientConnectionMode::App))
{
let host_image_id = self
.app
.pane_graphics
.slots
.get(&key)
.map(|slot| slot.host_image_id)
.unwrap_or(image_id);
client.graphics_cache.forget_pane_layer(&key, host_image_id);
}
let gate = self
.app
@@ -404,7 +454,7 @@ impl HeadlessServer {
*client_id,
ServerMessage::GraphicsTransmissionRetired {
transfer_id: gate.transfer_id,
image_id: slot.host_image_id,
image_id: gate.image_id,
},
);
}
src/server/headless/tests/pane_graphics.rs
+270 -4
View File
@@ -49,6 +49,173 @@ async fn cold_redraw_advances_one_bounded_layer_after_each_send() {
assert_eq!(server.clients[&1].deferred_render(), DeferredRender::None);
}
#[tokio::test]
async fn client_shell_surface_sends_complete_placements_and_each_live_asset_once() {
let (mut server, client_rx, pane_id) = retained_test_server(b"client shell graphics");
let client = server.clients.get_mut(&1).unwrap();
client.mode = ClientConnectionMode::ClientShell;
client.render_state =
crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame);
client.cell_size = crate::kitty_graphics::HostCellSize {
width_px: 10,
height_px: 20,
};
set_graphics_layer(&mut server, pane_id, vec![1, 2, 3, 4]);
server.render_and_stream();
let first = read_server_message(receive_render(&client_rx, Duration::from_millis(100)));
let ServerMessage::PaneSurface(first) = first else {
panic!("expected client shell pane surface");
};
assert_eq!(first.graphics.placements.len(), 1);
assert_eq!(first.graphics.assets.len(), 1);
assert_eq!(first.graphics.assets[0].data, vec![1, 2, 3, 4]);
assert!(matches!(
first.graphics.placements[0].asset.source,
crate::protocol::SurfaceGraphicsSource::PaneLayer { .. }
));
server.clients.get_mut(&1).unwrap().request_repaint();
server.render_and_stream();
let second = read_server_message(receive_render(&client_rx, Duration::from_millis(100)));
let ServerMessage::PaneSurface(second) = second else {
panic!("expected replacement client shell pane surface");
};
assert_eq!(second.graphics.placements, first.graphics.placements);
assert!(second.graphics.assets.is_empty());
}
#[tokio::test]
async fn client_shell_asset_delivery_is_bounded_to_the_current_live_scene() {
let (mut server, client_rx, pane_id) = retained_test_server(b"client shell graphics");
let client = server.clients.get_mut(&1).unwrap();
client.mode = ClientConnectionMode::ClientShell;
client.render_state =
crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame);
client.cell_size = crate::kitty_graphics::HostCellSize {
width_px: 10,
height_px: 20,
};
set_graphics_layer(&mut server, pane_id, vec![1, 2, 3, 4]);
server.render_and_stream();
let _first = receive_render(&client_rx, Duration::from_millis(100));
server.app.pane_graphics.slots.clear();
server.app.pane_graphics.mark_changed();
server.clients.get_mut(&1).unwrap().request_repaint();
server.render_and_stream();
let removed = read_server_message(receive_render(&client_rx, Duration::from_millis(100)));
let ServerMessage::PaneSurface(removed) = removed else {
panic!("expected removed client shell scene");
};
assert!(removed.graphics.placements.is_empty());
set_graphics_layer(&mut server, pane_id, vec![1, 2, 3, 4]);
server.clients.get_mut(&1).unwrap().request_repaint();
server.render_and_stream();
let restored = read_server_message(receive_render(&client_rx, Duration::from_millis(100)));
let ServerMessage::PaneSurface(restored) = restored else {
panic!("expected restored client shell scene");
};
assert_eq!(restored.graphics.assets.len(), 1);
assert_eq!(restored.graphics.assets[0].data, vec![1, 2, 3, 4]);
}
#[tokio::test]
async fn client_shell_surface_projects_terminal_kitty_images_from_authoritative_runtime() {
let (mut server, client_rx, _pane_id) =
retained_test_server(b"\x1b_Ga=T,f=32,t=d,i=7,p=3,s=1,v=1,c=1,r=1,q=2;/wAA/w==\x1b\\");
let client = server.clients.get_mut(&1).unwrap();
client.mode = ClientConnectionMode::ClientShell;
client.render_state =
crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame);
client.cell_size = crate::kitty_graphics::HostCellSize {
width_px: 10,
height_px: 20,
};
server.render_and_stream();
let message = read_server_message(receive_render(&client_rx, Duration::from_millis(100)));
let ServerMessage::PaneSurface(surface) = message else {
panic!("expected client shell pane surface");
};
assert_eq!(surface.graphics.placements.len(), 1);
assert_eq!(surface.graphics.assets.len(), 1);
assert!(matches!(
surface.graphics.placements[0].asset.source,
crate::protocol::SurfaceGraphicsSource::Terminal {
target: crate::protocol::SurfaceGraphicsTarget::Pane { .. },
image_id: 7,
}
));
assert_eq!(surface.graphics.assets[0].data, vec![255, 0, 0, 255]);
}
#[tokio::test]
async fn client_shell_delivers_equal_pixels_for_distinct_terminal_image_ids() {
let (mut server, client_rx, _pane_id) = retained_test_server(
b"\x1b_Ga=T,f=32,t=d,i=7,p=3,s=1,v=1,c=1,r=1,q=2;/wAA/w==\x1b\\\x1b_Ga=T,f=32,t=d,i=8,p=4,s=1,v=1,c=1,r=1,q=2;/wAA/w==\x1b\\",
);
let client = server.clients.get_mut(&1).unwrap();
client.mode = ClientConnectionMode::ClientShell;
client.render_state =
crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame);
client.cell_size = crate::kitty_graphics::HostCellSize {
width_px: 10,
height_px: 20,
};
server.render_and_stream();
let message = read_server_message(receive_render(&client_rx, Duration::from_millis(100)));
let ServerMessage::PaneSurface(surface) = message else {
panic!("expected client shell pane surface");
};
assert_eq!(surface.graphics.placements.len(), 2);
assert_eq!(surface.graphics.assets.len(), 2);
assert_ne!(
surface.graphics.assets[0].key,
surface.graphics.assets[1].key
);
assert_eq!(
surface.graphics.assets[0].data,
surface.graphics.assets[1].data
);
server.clients.get_mut(&1).unwrap().request_repaint();
server.render_and_stream();
let message = read_server_message(receive_render(&client_rx, Duration::from_millis(100)));
let ServerMessage::PaneSurface(surface) = message else {
panic!("expected replacement client shell pane surface");
};
assert_eq!(surface.graphics.placements.len(), 2);
assert!(surface.graphics.assets.is_empty());
}
#[tokio::test]
async fn full_client_shell_render_lane_does_not_commit_graphics_delivery() {
let (mut server, client_rx, pane_id) = retained_test_server(b"client shell graphics");
let client = server.clients.get_mut(&1).unwrap();
client.mode = ClientConnectionMode::ClientShell;
client.render_state =
crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame);
client.cell_size = crate::kitty_graphics::HostCellSize {
width_px: 10,
height_px: 20,
};
set_graphics_layer(&mut server, pane_id, vec![5, 6, 7, 8]);
fill_render_lane(&server);
server.render_and_stream();
let _older = receive_render(&client_rx, Duration::from_millis(100));
server.render_and_stream();
let message = read_server_message(receive_render(&client_rx, Duration::from_millis(100)));
let ServerMessage::PaneSurface(surface) = message else {
panic!("expected client shell pane surface");
};
assert_eq!(surface.graphics.assets.len(), 1);
assert_eq!(surface.graphics.assets[0].data, vec![5, 6, 7, 8]);
}
fn enable_graphics_and_render(
server: &mut HeadlessServer,
client_rx: &std::sync::mpsc::Receiver<Vec<u8>>,
@@ -728,6 +895,106 @@ fn rejected_or_stale_requests_do_not_schedule_rendering() {
.is_ok());
}
#[cfg(unix)]
#[tokio::test]
async fn client_shell_direct_graphics_uploads_without_server_authored_coordinates() {
let (mut server, client_rx, pane_id) = retained_test_server(b"client shell direct");
server.app.state.kitty_graphics_enabled = true;
let client = server.clients.get_mut(&1).unwrap();
client.mode = ClientConnectionMode::ClientShell;
client.render_state =
crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame);
client.cell_size = crate::kitty_graphics::HostCellSize {
width_px: 10,
height_px: 20,
};
client.direct_graphics = true;
server.app.direct_graphics_available = true;
set_stream_owner(&mut server, pane_id, "browser");
let public_pane_id = server.app.public_pane_id(0, pane_id).unwrap();
let path = sparse_direct_frame(&server, "client-shell-direct.rgba", 1, 1);
let (message, response_rx) =
direct_stream_message("shell-direct", &public_pane_id, "browser", path, 1, 1);
assert_eq!(
server.handle_pane_graphics_stream_frame(message),
RenderImpact::None
);
let (transfer_id, image_id, asset) = match read_server_message(
client_rx
.recv_timeout(Duration::from_secs(1))
.expect("client shell direct upload"),
) {
ServerMessage::GraphicsFile {
transfer_id,
image_id,
leading,
control,
surface_asset: Some(asset),
..
} => {
assert!(leading.is_empty());
assert!(control.starts_with("a=t,"), "{control}");
assert!(!control.contains("\u{1b}["), "{control}");
(transfer_id, image_id, asset)
}
other => panic!("expected client shell graphics file, got {other:?}"),
};
assert_eq!(
image_id,
crate::kitty_graphics::surface::host_image_id(&server.client_shell_boot_id, &asset)
);
let (pending, _) = crate::server::client_shell_graphics::collect(
&server.app,
&[],
&[],
None,
crate::kitty_graphics::HostCellSize {
width_px: 10,
height_px: 20,
},
&crate::kitty_graphics::surface::DeliveryCache::default(),
1,
);
assert_eq!(pending.retained_assets, vec![asset.clone()]);
server.start_direct_graphics_response(1, transfer_id, image_id);
assert!(server.complete_direct_graphics(1, transfer_id, image_id, true));
assert!(serde_json::from_str::<api::schema::SuccessResponse>(
&response_rx.recv_timeout(Duration::from_secs(1)).unwrap()
)
.is_ok());
server.clients.get_mut(&1).unwrap().request_repaint();
server.render_and_stream();
let surface = read_server_message(
client_rx
.recv_timeout(Duration::from_secs(1))
.expect("resident client shell scene"),
);
let ServerMessage::PaneSurface(surface) = surface else {
panic!("expected client shell pane surface");
};
assert_eq!(surface.graphics.placements.len(), 1);
assert!(surface.graphics.assets.is_empty());
assert_eq!(surface.graphics.placements[0].asset, asset);
assert_eq!(surface.graphics.retained_assets, vec![asset.clone()]);
let (hidden, _) = crate::server::client_shell_graphics::collect(
&server.app,
&[],
&[],
None,
crate::kitty_graphics::HostCellSize {
width_px: 10,
height_px: 20,
},
&crate::kitty_graphics::surface::DeliveryCache::default(),
1,
);
assert!(hidden.placements.is_empty());
assert_eq!(hidden.retained_assets, vec![asset]);
}
#[cfg(unix)]
#[tokio::test]
async fn hidden_large_direct_frame_uploads_then_replays_placement_without_closing_stream() {
@@ -1029,6 +1296,7 @@ fn direct_gate_server_with_file(
slot.stream_active = Some(active_gate());
slot.direct_gate = Some(crate::app::pane_graphics::DirectGate {
transfer_id: lease.fingerprint(),
image_id: (1 << 31) | 900,
client_id: 7,
deadline: std::time::Instant::now() + Duration::from_secs(1),
written: true,
@@ -1042,10 +1310,8 @@ fn direct_gate_server_with_file(
#[cfg(unix)]
fn direct_ids(server: &HeadlessServer, key: &crate::app::pane_graphics::Key) -> (u64, u32) {
let slot = &server.app.pane_graphics.slots[key];
(
slot.direct_gate.as_ref().unwrap().transfer_id,
slot.host_image_id,
)
let gate = slot.direct_gate.as_ref().unwrap();
(gate.transfer_id, gate.image_id)
}
#[cfg(unix)]
src/server/mod.rs
+1
View File
@@ -3,6 +3,7 @@ pub mod autodetect;
#[cfg(unix)]
pub(crate) mod client_accept;
pub(crate) mod client_shell;
pub(crate) mod client_shell_graphics;
pub(crate) mod client_transport;
pub(crate) mod clients;
pub(crate) mod clipboard_image;
src/server/render_stream.rs
+45 -1
View File
@@ -19,6 +19,8 @@ pub(crate) enum ClientRenderState {
last_frame: Option<FrameData>,
last_surface_panes: Option<Vec<PaneSurfacePane>>,
last_surface_popup: Option<Box<ClientShellPopupSurface>>,
last_surface_graphics_placements: Option<Vec<crate::protocol::SurfaceGraphicsPlacement>>,
last_surface_graphics_retained: Option<Vec<crate::protocol::SurfaceGraphicsAssetKey>>,
last_surface_projection_revision: Option<u64>,
},
/// Terminal-ANSI clients keep a terminal diff encoder and sequence number.
@@ -36,6 +38,8 @@ impl ClientRenderState {
last_frame: None,
last_surface_panes: None,
last_surface_popup: None,
last_surface_graphics_placements: None,
last_surface_graphics_retained: None,
last_surface_projection_revision: None,
},
RenderEncoding::TerminalAnsi => Self::TerminalAnsi {
@@ -52,11 +56,15 @@ impl ClientRenderState {
last_frame,
last_surface_panes,
last_surface_popup,
last_surface_graphics_placements,
last_surface_graphics_retained,
last_surface_projection_revision,
} => {
*last_frame = None;
*last_surface_panes = None;
*last_surface_popup = None;
*last_surface_graphics_placements = None;
*last_surface_graphics_retained = None;
*last_surface_projection_revision = None;
}
Self::TerminalAnsi {
@@ -76,11 +84,15 @@ impl ClientRenderState {
last_frame,
last_surface_panes,
last_surface_popup,
last_surface_graphics_placements,
last_surface_graphics_retained,
last_surface_projection_revision,
} => {
*last_frame = None;
*last_surface_panes = None;
*last_surface_popup = None;
*last_surface_graphics_placements = None;
*last_surface_graphics_retained = None;
*last_surface_projection_revision = None;
}
Self::TerminalAnsi {
@@ -94,12 +106,16 @@ impl ClientRenderState {
last_frame,
last_surface_panes,
last_surface_popup,
last_surface_graphics_placements,
last_surface_graphics_retained,
last_surface_projection_revision,
} = self
{
*last_frame = None;
*last_surface_panes = None;
*last_surface_popup = None;
*last_surface_graphics_placements = None;
*last_surface_graphics_retained = None;
*last_surface_projection_revision = None;
}
}
@@ -165,14 +181,19 @@ impl ClientRenderState {
last_frame,
last_surface_panes,
last_surface_popup,
last_surface_graphics_placements,
last_surface_graphics_retained,
last_surface_projection_revision,
} = self
else {
return None;
};
if last_frame.as_ref() == Some(&surface.frame)
if surface.graphics.assets.is_empty()
&& last_frame.as_ref() == Some(&surface.frame)
&& last_surface_panes.as_ref() == Some(&surface.panes)
&& *last_surface_popup == surface.popup
&& last_surface_graphics_placements.as_ref() == Some(&surface.graphics.placements)
&& last_surface_graphics_retained.as_ref() == Some(&surface.graphics.retained_assets)
&& *last_surface_projection_revision == Some(surface.projection_revision)
{
return None;
@@ -196,6 +217,8 @@ impl ClientRenderState {
last_frame,
last_surface_panes,
last_surface_popup,
last_surface_graphics_placements,
last_surface_graphics_retained,
last_surface_projection_revision,
},
PreparedRender::Semantic {
@@ -205,6 +228,8 @@ impl ClientRenderState {
*last_frame = Some(frame);
*last_surface_panes = None;
*last_surface_popup = None;
*last_surface_graphics_placements = None;
*last_surface_graphics_retained = None;
*last_surface_projection_revision = None;
}
(
@@ -212,6 +237,8 @@ impl ClientRenderState {
last_frame,
last_surface_panes,
last_surface_popup,
last_surface_graphics_placements,
last_surface_graphics_retained,
last_surface_projection_revision,
},
PreparedRender::Semantic {
@@ -222,6 +249,8 @@ impl ClientRenderState {
*last_frame = Some(surface.frame);
*last_surface_panes = Some(surface.panes);
*last_surface_popup = surface.popup;
*last_surface_graphics_placements = Some(surface.graphics.placements);
*last_surface_graphics_retained = Some(surface.graphics.retained_assets);
}
(
Self::TerminalAnsi {
@@ -275,6 +304,7 @@ mod tests {
pixel_width: 0,
pixel_height: 0,
})),
graphics: crate::protocol::SurfaceGraphicsScene::default(),
}
}
@@ -323,6 +353,20 @@ impl PreparedRender {
}
}
pub(crate) fn strip_pane_surface_assets(&mut self) -> bool {
let Self::Semantic {
message: ServerMessage::PaneSurface(surface),
} = self
else {
return false;
};
if surface.graphics.assets.is_empty() {
return false;
}
surface.graphics.assets.clear();
true
}
pub(crate) fn into_frame(self) -> Option<FrameData> {
match self {
Self::Semantic {