From ed14b561ab72069e3d9186e09df745e9be76b408 Mon Sep 17 00:00:00 2001 From: l0ng-ai <24760907+l0ng-ai@users.noreply.github.com> Date: Fri, 28 Aug 2026 18:15:54 +0800 Subject: [PATCH] feat(remote): auto-relink dead workspace panes and name their tabs (#757) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(remote): auto-relink dead workspace panes and name their tabs A workspace pane whose stream died while its machine's control link stayed up was invisible to the reconnect supervisor: the tab sat on 'tty7 — disconnected' until the workspace was reopened by hand. - The link supervisor's pump now sweeps for such panes and asks for them back on the existing per-workspace backoff (1s doubling to 30s). A refusal — the machine says the pane is gone — is final for that pane; transient failures keep the clock running. - open_relink waits for the daemon's verdict on the Attach instead of handing an unclassifiable stream to the reader; refusals are typed (AttachRefused) so the retry loop can tell them from transport trouble. - PaneWorkspace carries the workspace's display name, and the pane adopts it as its default title, so a dead link reads 'hummingbot — disconnected' instead of the bare app name (the workspace-pane half of #438). - The reader's teardown logs which way the link died (EOF / read error / protocol error); until now all three were indistinguishable afterwards. Claude-Session: https://claude.ai/code/session_016s4fehNxNDXfJ1AfeTNo6y * fix(remote): keep two relink paths from dialling the same pane at once The daemon keeps one subscriber per pane: a second `Attach` for a pane_id kicks the first off. After a machine-level reconnect, `relink_panes` dials every pane and can sit up to fifteen seconds waiting for the far end's verdict — and the pump's own sweep, which runs every 250 ms and still reads those panes as dead, fired a second `Attach` for each of them. Panes are now claimed for the duration of an attempt. Both askers set the claim before dialling and release it when the attempt reports back, so a pane in flight asks for nothing; the workspace's retry clock likewise survives a sweep that finds no dead panes only because a batch holds them. Claude-Session: https://claude.ai/code/session_016s4fehNxNDXfJ1AfeTNo6y --- src/terminal/mod.rs | 2 +- src/terminal/remote.rs | 121 +++++++++++++++--- src/terminal/view.rs | 135 +++++++++++++++++++- src/ui/remote_workspace.rs | 248 ++++++++++++++++++++++++++++++++++++- 4 files changed, 484 insertions(+), 22 deletions(-) diff --git a/src/terminal/mod.rs b/src/terminal/mod.rs index a09e113a..efea92df 100644 --- a/src/terminal/mod.rs +++ b/src/terminal/mod.rs @@ -30,5 +30,5 @@ mod typeahead; pub mod view; pub(crate) use remote::notify_desktop; -pub use remote::{PaneRoute, PaneWorkspace, RemoteTerminal, attach_unanswered}; +pub use remote::{PaneRoute, PaneWorkspace, RemoteTerminal, attach_refused, attach_unanswered}; pub use size::TermSize; diff --git a/src/terminal/remote.rs b/src/terminal/remote.rs index b29bcf12..df9396d6 100644 --- a/src/terminal/remote.rs +++ b/src/terminal/remote.rs @@ -102,6 +102,12 @@ pub struct PaneWorkspace { pub workspace: crate::core::session::WorkspaceId, pub target: crate::core::session::RemoteTarget, pub spec: Option>, + /// The name the workspace answers to — its own label, or its machine's + /// when it has none. A pane keeps answering to this name when its link + /// dies (#438 gave SSH panes their host here; without this, a workspace + /// pane fell back to the bare app name and the tab read "tty7 — + /// disconnected"). + pub label: Option, /// Whether the daemon serving this workspace's panes echoes a `Size` frame /// when it applies a resize — read off the host's control hello by whoever /// built this value, because this module may not ask the network itself. @@ -741,25 +747,33 @@ impl RemoteTerminal { Ok(term) } + /// Opens the daemon connection a relink will adopt, and waits for the + /// daemon's verdict on the `Attach`. Bytes read past the verdict are + /// returned so the adopted reader loses none of the replay. Waiting here + /// is what makes a relink's failure classifiable: fire-and-forget handed + /// a "no such pane" refusal to the reader thread, which could only die + /// with it — indistinguishable from the link dropping again. pub fn open_relink( route: &PaneRoute, pane_id: u64, size: TermSize, cell_w: u16, cell_h: u16, - ) -> anyhow::Result { + ) -> anyhow::Result<(Stream, Vec)> { let mut stream = connect_routed(route)?; ClientMsg::Attach { pane_id, size: win_size(size, cell_w, cell_h), } .encode(&mut stream)?; - Ok(stream) + let buffered = attach_reply_prefix(&mut stream, pane_id, attach_reply_wait(route))?; + Ok((stream, buffered)) } pub fn adopt_relink( &mut self, stream: Stream, + buffered: Vec, route: &PaneRoute, size: TermSize, cell_w: u16, @@ -791,7 +805,7 @@ impl RemoteTerminal { self.term.clone(), self.proxy.clone(), read_half, - Vec::new(), + buffered, quit.clone(), ReaderSignals { cwd: self.cwd.clone(), @@ -1035,13 +1049,19 @@ impl RemoteTerminal { let mut tr_adv_t = std::time::Duration::ZERO; let mut tr_frames: u32 = 0; - let teardown = || { + let teardown = |reason: Option<&str>| { // A retired reader exits silently: the pane is being // relinked or released, not dying — marking it exited // would wrongly kill the freshly adopted link. if quit.load(Ordering::SeqCst) { return; } + // Said out loud because every way a link dies looks the + // same from the window — a pane suddenly "disconnected" — + // and this line is the only record of which way it was. + if let Some(reason) = reason { + log::warn!("a pane's link died: {reason}"); + } term.lock().exit(); exited_flag.store(true, Ordering::SeqCst); proxy.send_event(AlacEvent::Wakeup); @@ -1172,15 +1192,15 @@ impl RemoteTerminal { let frame = match crate::daemon::protocol::take_frame(&mut pending) { Ok(Some(frame)) => frame, Ok(None) => break, - Err(_) => { - teardown(); + Err(e) => { + teardown(Some(&format!("unframeable bytes on the link: {e}"))); break 'main; } }; let msg = match DaemonMsg::from_frame(frame.0, frame.1) { Ok(msg) => msg, - Err(_) => { - teardown(); + Err(e) => { + teardown(Some(&format!("undecodable frame on the link: {e}"))); break 'main; } }; @@ -1383,7 +1403,8 @@ impl RemoteTerminal { DaemonMsg::Exited { .. } => { flush_batch!(); child_exited.store(true, Ordering::SeqCst); - teardown(); + // Routine — the child ended — so no log line. + teardown(None); break 'main; } _ => {} @@ -1436,7 +1457,7 @@ impl RemoteTerminal { let tr0 = trace.then(std::time::Instant::now); match stream.read(&mut scratch) { Ok(0) => { - teardown(); + teardown(Some("the far end closed it (EOF)")); break; } Ok(n) => { @@ -1453,8 +1474,8 @@ impl RemoteTerminal { std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut ) => {} Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} - Err(_) => { - teardown(); + Err(e) => { + teardown(Some(&format!("read failed: {e}"))); break; } } @@ -2104,6 +2125,30 @@ pub fn attach_unanswered(err: &anyhow::Error) -> bool { .any(|cause| cause.downcast_ref::().is_some()) } +/// An `Attach` the daemon answered with an `Error` frame — it heard us and +/// said no, which for a relink means the pane is gone on its machine. Typed so +/// a retry loop can tell this apart from the failures worth retrying: silence +/// and transport trouble pass, a refusal repeats identically forever. +#[derive(Debug)] +struct AttachRefused { + message: String, +} + +impl std::fmt::Display for AttachRefused { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "daemon refused Attach: {}", self.message) + } +} + +impl std::error::Error for AttachRefused {} + +/// Whether `err` is an `Attach` the daemon explicitly refused. Retrying one of +/// these can only produce the same refusal. +pub fn attach_refused(err: &anyhow::Error) -> bool { + err.chain() + .any(|cause| cause.downcast_ref::().is_some()) +} + /// Reads far enough into the daemon's answer to an `Attach` to classify it, and /// hands back every byte read so the reader thread loses none of the replay. /// @@ -2156,7 +2201,7 @@ fn attach_reply_prefix( } let message = read_error_frame(stream, &mut buffered, wait) .unwrap_or_else(|| format!("no such pane {pane_id}")); - Err(anyhow::anyhow!("daemon refused Attach: {message}")) + Err(anyhow::Error::new(AttachRefused { message })) } fn read_error_frame( @@ -2792,8 +2837,15 @@ mod windows_tests { let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { let mut term = term; - term.adopt_relink(new_client, &PaneRoute::Local, TermSize::new(80, 24), 8, 16) - .unwrap(); + term.adopt_relink( + new_client, + Vec::new(), + &PaneRoute::Local, + TermSize::new(80, 24), + 8, + 16, + ) + .unwrap(); let _ = tx.send(term); }); let term = rx @@ -2961,6 +3013,7 @@ mod tests { ) .unwrap(), )), + label: None, resize_echo: false, } } @@ -3006,6 +3059,7 @@ mod tests { distro: "Ubuntu-22.04".into(), }, spec: None, + label: None, resize_echo: false, }; let route = PaneRoute::for_workspace(Some(&ws)); @@ -3023,6 +3077,7 @@ mod tests { args: vec!["--stdio".into()], }, spec: None, + label: None, resize_echo: false, }; let route = PaneRoute::for_workspace(Some(&ws)); @@ -3045,6 +3100,7 @@ mod tests { alias: "build-box".into(), }, spec: None, + label: None, resize_echo: false, }; let route = PaneRoute::for_workspace(Some(&ws)); @@ -3280,6 +3336,41 @@ mod tests { assert!(!attach_unanswered(&refused)); } + /// The relink retry loop keys off this split: a refusal means the pane is + /// gone on its machine and asking again can only repeat the answer, while + /// silence and a hangup are transport trouble worth another try. + #[test] + fn a_refused_attach_is_final_and_every_other_failure_is_not() { + let wait = std::time::Duration::from_millis(200); + + let (mut client_side, mut daemon_side) = UnixStream::pair().unwrap(); + DaemonMsg::Error("no such pane 7".into()) + .encode(&mut daemon_side) + .unwrap(); + daemon_side.flush().unwrap(); + let refused = attach_reply_prefix(&mut client_side, 7, wait).expect_err("refused"); + assert!(attach_refused(&refused)); + assert!( + format!("{refused:#}").contains("no such pane 7"), + "the daemon's own words survive: {refused:#}" + ); + + let (mut client_side, _held_open) = UnixStream::pair().unwrap(); + let silent = attach_reply_prefix(&mut client_side, 7, wait).expect_err("silent"); + assert!( + !attach_refused(&silent), + "silence is not a verdict on the pane" + ); + + let (mut client_side, daemon_side) = UnixStream::pair().unwrap(); + drop(daemon_side); + let hungup = attach_reply_prefix(&mut client_side, 7, wait).expect_err("hung up"); + assert!( + !attach_refused(&hungup), + "a hangup is not a verdict on the pane" + ); + } + /// The other side of the rule above: the frame a quiet pane does send is /// enough. Nothing else is required of the daemon for the attach to stand. #[test] diff --git a/src/terminal/view.rs b/src/terminal/view.rs index 0f803762..2d3b7feb 100644 --- a/src/terminal/view.rs +++ b/src/terminal/view.rs @@ -258,8 +258,21 @@ pub struct TerminalView { /// What the pane is called before anything running in it says otherwise — /// and what it goes back to when the program resets the title or the /// session ends. "tty7" for a local shell; for an SSH pane it is the host - /// it dialled, so a window full of them is still readable (#438). + /// it dialled, so a window full of them is still readable (#438); for a + /// workspace pane it is the workspace's name, set in `set_workspace`. pub(super) default_title: String, + /// The link supervisor asked this pane's machine for a relink and was + /// refused — the pane is gone at the far end, and asking again can only + /// repeat the answer. Cleared when a relink is adopted anyway (the manual + /// reconnect path), which is the one thing that changes the question. + relink_abandoned: bool, + /// A relink for this pane is already on the wire. Both askers set it — + /// the machine-level reconnect and the pump's own sweep — because the + /// daemon keeps exactly one subscriber per pane and a second `Attach` + /// kicks the first. Without this the pump would join a dial still in + /// flight every 250 ms, and a dial can sit for fifteen seconds waiting + /// for the far end's verdict. + relink_inflight: bool, pub marked_text: String, last_mouse_cell: Option<(usize, usize)>, last_hover_cell: Option<(usize, usize)>, @@ -1291,6 +1304,8 @@ impl TerminalView { title: DEFAULT_TITLE.to_string(), pending_title: None, default_title: DEFAULT_TITLE.to_string(), + relink_abandoned: false, + relink_inflight: false, marked_text: String::new(), last_mouse_cell: None, report_mouse, @@ -1473,6 +1488,15 @@ impl TerminalView { self.host_id = workspace .as_ref() .map_or(crate::ui::host_ops::HostId::LOCAL, |w| w.target.host_id()); + // The pane answers to its workspace's name from here on: untitled tabs + // show it, and a dead link's "— disconnected" suffix hangs off it + // instead of the bare app name. + if let Some(label) = workspace.as_ref().and_then(|w| w.label.clone()) { + if self.title == self.default_title { + self.title = label.clone(); + } + self.default_title = label; + } self.workspace = workspace; } @@ -1499,6 +1523,7 @@ impl TerminalView { pub fn adopt_relink( &mut self, stream: crate::daemon::transport::Stream, + buffered: Vec, route: &crate::terminal::PaneRoute, size: TermSize, cell_w: u16, @@ -1506,7 +1531,9 @@ impl TerminalView { cx: &mut Context, ) -> anyhow::Result<()> { self.terminal - .adopt_relink(stream, route, size, cell_w, cell_h)?; + .adopt_relink(stream, buffered, route, size, cell_w, cell_h)?; + self.relink_abandoned = false; + self.relink_inflight = false; self.title = self.default_title.clone(); cx.notify(); Ok(()) @@ -1712,6 +1739,40 @@ impl TerminalView { self.ssh_spec.is_some() && self.terminal.exited } + /// Whether this pane is a workspace pane whose link died under it — the + /// far-end session should still be alive, so the link supervisor keeps + /// asking for it back. A child that exited is over, not disconnected, and + /// a pane whose machine already refused the relink is past asking. + pub fn wants_relink(&self) -> bool { + self.workspace.is_some() + && self.terminal.exited + && !self.terminal.child_exited() + && !self.relink_abandoned + && !self.relink_inflight + } + + /// Claims this pane for one relink attempt. Every path that dials for a + /// pane calls this first, so the other paths leave it alone until the + /// attempt reports back through `relink_settled` or `adopt_relink`. + pub fn mark_relinking(&mut self) { + self.relink_inflight = true; + } + + /// Releases the claim `mark_relinking` took, for the attempts that end + /// without a stream to adopt. A pane freed this way is up for asking + /// again on the next sweep. + pub fn relink_settled(&mut self) { + self.relink_inflight = false; + } + + /// Records that this pane's machine refused to give the pane back — it is + /// gone at the far end — so the link supervisor stops asking. The pane + /// keeps its disconnected face; only the retrying stops. + pub fn abandon_relink(&mut self) { + self.relink_abandoned = true; + self.relink_inflight = false; + } + /// Takes a title the program set, and gives it to the tab only once it has /// stood for `TITLE_SETTLE`. /// @@ -7298,6 +7359,7 @@ mod tests { .unwrap(), ) }), + label: None, resize_echo: false, } } @@ -8597,6 +8659,7 @@ mod tests { workspace: WorkspaceId::new(), target: target.clone(), spec: None, + label: None, resize_echo: false, }; @@ -8613,6 +8676,7 @@ mod tests { workspace: WorkspaceId::new(), target, spec: None, + label: None, resize_echo: false, }; assert_eq!(sibling.target.host_id(), remote); @@ -11650,6 +11714,7 @@ mod gpui_tests { ) .unwrap(), )), + label: None, resize_echo: false, })); id @@ -11888,6 +11953,71 @@ mod gpui_tests { .unwrap(); } + /// A workspace pane answers to its workspace's name: untitled tabs show + /// it, and a dead link's suffix hangs off it — not off the bare app name, + /// which read "tty7 — disconnected" no matter whose link died. + #[gpui::test] + fn a_workspace_pane_answers_to_its_workspaces_name(cx: &mut TestAppContext) { + use crate::core::session::{RemoteTarget, WorkspaceId}; + use crate::terminal::PaneWorkspace; + let (window, _daemon) = harness(cx); + window + .update(cx, |view, _, cx| { + view.set_workspace(Some(PaneWorkspace { + workspace: WorkspaceId::new(), + target: RemoteTarget::direct("me", "build-box", 22), + spec: None, + label: Some("hummingbot".into()), + resize_echo: false, + })); + assert_eq!(view.title, "hummingbot", "an untitled tab shows the name"); + view.handle_event(AlacEvent::Exit, cx); + assert_eq!(view.title, "hummingbot — disconnected"); + }) + .unwrap(); + } + + /// What the link supervisor's relink sweep keys off: a workspace pane + /// whose link died asks to come back, until its machine refuses. + #[gpui::test] + fn only_a_dead_workspace_pane_wants_a_relink(cx: &mut TestAppContext) { + let (window, _daemon) = harness(cx); + window + .update(cx, |view, _, cx| { + bind_to_a_disconnected_remote_workspace(view, cx); + assert!(!view.wants_relink(), "a live pane asks for nothing"); + view.handle_event(AlacEvent::Exit, cx); + assert!(view.wants_relink()); + view.abandon_relink(); + assert!(!view.wants_relink(), "a refusal is final"); + }) + .unwrap(); + } + + /// The daemon keeps one subscriber per pane, so a second `Attach` for a + /// pane already being dialled for kicks the first off. A pane claimed for + /// an attempt therefore asks for nothing until that attempt reports back — + /// otherwise the pump, which sweeps every 250 ms, would join a dial that + /// can sit fifteen seconds waiting for the far end's verdict. + #[gpui::test] + fn a_pane_already_being_dialled_for_asks_for_nothing(cx: &mut TestAppContext) { + let (window, _daemon) = harness(cx); + window + .update(cx, |view, _, cx| { + bind_to_a_disconnected_remote_workspace(view, cx); + view.handle_event(AlacEvent::Exit, cx); + assert!(view.wants_relink()); + view.mark_relinking(); + assert!(!view.wants_relink(), "the attempt on the wire owns it"); + view.relink_settled(); + assert!( + view.wants_relink(), + "an attempt that came back wrong frees it" + ); + }) + .unwrap(); + } + #[gpui::test] fn an_exited_local_pane_still_swallows_every_key(cx: &mut TestAppContext) { let (window, _daemon) = harness(cx); @@ -11964,6 +12094,7 @@ mod gpui_tests { .update(cx, |view, _, cx| { view.adopt_relink( new_client, + Vec::new(), &crate::terminal::PaneRoute::Local, TermSize::new(100, 30), 8, diff --git a/src/ui/remote_workspace.rs b/src/ui/remote_workspace.rs index 8490202b..35580650 100644 --- a/src/ui/remote_workspace.rs +++ b/src/ui/remote_workspace.rs @@ -1106,10 +1106,20 @@ pub(crate) fn pane_workspace_for( host.host_id(), crate::daemon::protocol::FEATURE_RESIZE_ECHO, ); + // The workspace's own name where it has one, the machine's otherwise — + // what the pane's tab falls back to when nothing has titled it, and what + // a dead link's "— disconnected" suffix hangs off. + let label = WorkspaceStore::all(cx) + .get(workspace) + .and_then(|w| w.label.clone()) + .or_else(|| Some(remote_connect::route_label(cx, &host))) + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()); let pane = crate::terminal::PaneWorkspace { workspace, target: host.target, spec, + label, resize_echo, }; if let Ok(header) = pane.route_header() { @@ -1139,6 +1149,23 @@ struct MachineLink { attach_sent: std::collections::HashSet, } +/// The retry clock for one workspace's dead panes, kept while its machine's +/// control link is up. A pane whose stream dies alone — its exec channel +/// closed under it, siblings untouched — is invisible to the machine-level +/// reconnect, which only watches the control connection; this is the state +/// that keeps asking for such a pane back. Batched per workspace on purpose: +/// panes usually die together, and one clock per workspace means one dial per +/// try instead of a storm of them. +#[derive(Default)] +struct PaneRetry { + backoff: Backoff, + next_attempt: Option, + /// A batch of relinks is on the wire; the pump leaves the entry alone + /// until it reports back, or one slow attempt would be joined by a new + /// one every 250 ms tick. + inflight: bool, +} + #[derive(Clone, Debug, PartialEq, Eq)] enum LinkState { Connecting, @@ -1169,6 +1196,9 @@ pub(crate) enum MachineStatus { #[derive(Default)] pub(crate) struct RemoteLinks { machines: std::collections::HashMap, + /// Dead panes being asked for again, one clock per workspace — see + /// [`PaneRetry`]. + pane_retries: std::collections::HashMap, preempted: std::collections::HashMap, /// Workspaces whose takeover the user has asked to undo, each still /// carrying the name of the client that displaced it — an attach that @@ -1373,6 +1403,7 @@ fn pump_tick(cx: &mut gpui::App) -> bool { let links = cx.default_global::(); let forgotten = links.machines.len(); links.machines.clear(); + links.pane_retries.clear(); links.preempted.clear(); links.reclaiming.clear(); links.attaching.clear(); @@ -1415,6 +1446,10 @@ fn pump_tick(cx: &mut gpui::App) -> bool { // workspace wants to be claimed for this client before we start // rebuilding its tabs and their panes on the machine. pump_attachments(cx, host); + // A pane whose stream died while this link stayed up: the + // machine-level reconnect never fires for it, so the pump itself + // asks for it back. + pump_pane_relinks(cx, host); if became { changed = true; log::info!("link to {target} is attached"); @@ -1883,6 +1918,9 @@ fn finish_attempt( relink_panes(cx, id); crate::ui::tree_sync::hydrate_window_from_tree(cx, id); } + // Whatever a pane's retry clock said about the old link is + // stale on the new one; the pump re-collects survivors fresh. + cx.default_global::().pane_retries.remove(&id); refresh_window_shells(cx, id); } RemoteLinks::mark(cx, host, |link| { @@ -1940,7 +1978,14 @@ fn relink_panes(cx: &mut gpui::App, workspace: WorkspaceId) { } log::info!("relinking {} pane(s) of workspace {workspace}", panes.len()); for view in panes { - let (pane_id, size, cell_w, cell_h) = view.read(cx).relink_plan(); + // Claimed before the dial so the pump's sweep, which runs every + // 250 ms and sees these panes as dead until they adopt, does not fire + // a second `Attach` for the same pane and kick this one off the + // daemon's single subscriber slot. + let (pane_id, size, cell_w, cell_h) = view.update(cx, |view, _| { + view.mark_relinking(); + view.relink_plan() + }); let opening = route.clone(); let adopting = route.clone(); cx.spawn(async move |cx| { @@ -1954,22 +1999,185 @@ fn relink_panes(cx: &mut gpui::App, workspace: WorkspaceId) { }) .await; match opened { - Ok(stream) => { + Ok((stream, buffered)) => { view.update(cx, |view, cx| { if let Err(e) = - view.adopt_relink(stream, &adopting, size, cell_w, cell_h, cx) + view.adopt_relink(stream, buffered, &adopting, size, cell_w, cell_h, cx) { + view.relink_settled(); log::warn!("pane {pane_id} re-attached but could not be adopted: {e}"); } }); } - Err(e) => log::warn!("could not relink pane {pane_id}: {e}"), + Err(e) => { + view.update(cx, |view, _| view.relink_settled()); + log::warn!("could not relink pane {pane_id}: {e}"); + } } }) .detach(); } } +/// Finds panes whose links died while their machine's control link stayed up, +/// and asks for them back on a per-workspace backoff. Runs from the pump's +/// `live` branch, so a machine that is unreachable never gets here — its +/// panes come back through `relink_panes` when the machine-level reconnect +/// lands. +fn pump_pane_relinks(cx: &mut gpui::App, host: HostId) { + let now = Instant::now(); + for (id, _) in workspaces_on(cx, host) { + // Released panes are gone on purpose (preempted, or a Take Back on + // the wire); an attach still in flight will rebuild them itself. + let paused = { + let links = cx.default_global::(); + links.preempted.contains_key(&id) + || links.reclaiming.contains_key(&id) + || links.attaching.contains(&id) + }; + if paused { + continue; + } + let dead: Vec<_> = panes_of(cx, id) + .into_iter() + .filter(|view| view.read(cx).wants_relink()) + .collect(); + if dead.is_empty() { + // A batch on the wire owns the clock until it reports back — its + // panes read as claimed, not dead, and dropping the entry here + // would throw away the backoff the batch is about to advance. + let links = cx.default_global::(); + if !links.pane_retries.get(&id).is_some_and(|r| r.inflight) { + links.pane_retries.remove(&id); + } + continue; + } + let due = { + let retry = cx + .default_global::() + .pane_retries + .entry(id) + .or_default(); + match (retry.inflight, retry.next_attempt) { + (true, _) => false, + // First sighting: ask right away. The backoff only starts + // once an attempt has actually come back wrong. + (false, None) => true, + (false, Some(at)) => at <= now, + } + }; + if due { + relink_dead_panes(cx, id, dead); + } + } +} + +/// One batch of relinks for one workspace's dead panes. Every pane dials +/// concurrently; the batch reports back as a whole, and one transient failure +/// puts the whole workspace on the next backoff step. A refusal is final for +/// that pane — the machine said the pane is gone — so it is marked abandoned +/// instead of counted against the clock. +fn relink_dead_panes( + cx: &mut gpui::App, + workspace: WorkspaceId, + panes: Vec>, +) { + let route = pane_route_for(cx, workspace); + if route.header().is_none() { + // Local can't happen for a workspace with a host; Unroutable means + // the route stopped resolving, which is the machine supervisor's + // problem — count it as a miss and let the clock run. + note_pane_relink_outcome(cx, workspace, panes.len()); + return; + } + { + let retry = cx + .default_global::() + .pane_retries + .entry(workspace) + .or_default(); + retry.inflight = true; + log::info!( + "asking for {} dead pane(s) of workspace {workspace} back (attempt {})", + panes.len(), + retry.backoff.attempt() + 1 + ); + } + // Claimed pane by pane as well as workspace by workspace: the machine + // supervisor's own `relink_panes` reads the same flag, and two `Attach`es + // for one pane_id do not queue — the daemon's second one kicks the first. + let plans: Vec<_> = panes + .iter() + .map(|view| { + let plan = view.update(cx, |view, _| { + view.mark_relinking(); + view.relink_plan() + }); + (view.clone(), plan) + }) + .collect(); + cx.spawn(async move |cx| { + let attempts: Vec<_> = plans + .into_iter() + .map(|(view, (pane_id, size, cell_w, cell_h))| { + let opening = route.clone(); + let opened = cx.background_executor().spawn(async move { + crate::terminal::RemoteTerminal::open_relink( + &opening, pane_id, size, cell_w, cell_h, + ) + }); + (view, opened, pane_id, size, cell_w, cell_h) + }) + .collect(); + let mut misses = 0usize; + for (view, opened, pane_id, size, cell_w, cell_h) in attempts { + match opened.await { + Ok((stream, buffered)) => { + let adopted = view.update(cx, |view, cx| { + view.adopt_relink(stream, buffered, &route, size, cell_w, cell_h, cx) + }); + match adopted { + Ok(()) => log::info!("pane {pane_id} came back on its own"), + Err(e) => { + misses += 1; + view.update(cx, |view, _| view.relink_settled()); + log::warn!("pane {pane_id} re-attached but could not be adopted: {e}"); + } + } + } + Err(e) if crate::terminal::attach_refused(&e) => { + view.update(cx, |view, _| view.abandon_relink()); + log::warn!("pane {pane_id} is gone on its machine ({e}); not asking again"); + } + Err(e) => { + misses += 1; + view.update(cx, |view, _| view.relink_settled()); + log::warn!("could not relink pane {pane_id}: {e}"); + } + } + } + cx.update(|cx| note_pane_relink_outcome(cx, workspace, misses)); + }) + .detach(); +} + +/// Lands a batch's verdict on the workspace's retry clock: all found their +/// way back (or are past asking) and the entry retires; any miss advances the +/// backoff and books the next try. +fn note_pane_relink_outcome(cx: &mut gpui::App, workspace: WorkspaceId, misses: usize) { + let links = cx.default_global::(); + if misses == 0 { + links.pane_retries.remove(&workspace); + return; + } + let Some(retry) = links.pane_retries.get_mut(&workspace) else { + return; + }; + retry.inflight = false; + let delay = retry.backoff.advance(); + retry.next_attempt = Some(Instant::now() + delay); +} + fn server_restarted(cx: &mut gpui::App, host: HostId, peer: &RemoteHost) -> bool { let instance = peer.peer().instance.clone(); let seen = &mut cx.default_global::().instances; @@ -2089,6 +2297,38 @@ pub(crate) fn workspace_is_preempted(cx: &gpui::App, workspace: WorkspaceId) -> mod tests { use super::*; + /// The per-workspace relink clock: a miss puts the workspace on the next + /// backoff step, panes all found their way back retires the entry — so a + /// pane that cannot come back is asked for on a widening interval, and a + /// recovered workspace costs the pump nothing. + #[gpui::test] + fn a_pane_retry_backs_off_on_misses_and_retires_on_success(cx: &mut gpui::TestAppContext) { + cx.update(|cx| { + let ws = WorkspaceId::new(); + cx.default_global::() + .pane_retries + .entry(ws) + .or_default() + .inflight = true; + + note_pane_relink_outcome(cx, ws, 2); + let links = cx.default_global::(); + let retry = links.pane_retries.get(&ws).expect("a miss keeps the clock"); + assert!(!retry.inflight, "the batch reported back"); + assert_eq!(retry.backoff.attempt(), 1); + assert!(retry.next_attempt.is_some(), "the next try is booked"); + + note_pane_relink_outcome(cx, ws, 0); + assert!( + cx.default_global::() + .pane_retries + .get(&ws) + .is_none(), + "every pane back means no clock left to run" + ); + }); + } + #[gpui::test] fn taking_back_marks_the_workspace_for_a_whole_rebuild(cx: &mut gpui::TestAppContext) { cx.update(|cx| {