fix: clean up idle ssh bridges and back off flapping connections

This commit is contained in:
Ogulcan Celik
2026-09-14 03:32:01 +03:00
parent 21c1f8cf85
commit e3220da152
18 changed files with 645 additions and 24 deletions
@@ -73,10 +73,12 @@ Removing or disabling the machine you are viewing returns you to Local. If Local
## Connection problems
- **Reconnecting:** Herdr retries with bounded backoff after a network interruption, sleep, or SSH failure. SSH connections are checked for application-level activity and probed when quiet, so a broken connection does not stay Online indefinitely. Local detects native connection closure or failure instead of using remote health probes.
- **Reconnecting:** Herdr retries automatically after a network interruption, sleep, or SSH failure. Repeated failures increase the delay up to two minutes; brief successful connections do not reset it. A connection must remain healthy for a minute before the next interruption gets a fast retry. SSH connections are checked for application-level activity and probed when quiet, so a broken connection does not stay Online indefinitely. Local detects native connection closure or failure instead of using remote health probes.
- **Attention:** The target needs an action that cannot be completed in the background, such as host-key approval, authentication, or a compatible server. Other machines remain usable.
- **Saved-machine file error:** An unreadable or invalid catalog leaves current connections unchanged. Herdr shows a notice and automatically retries reading it.
When both installations support bridge idle cleanup, saved-machine connections to Linux and macOS also close their remote bridge after a minute without traffic in either direction. Sleep counts toward that deadline, which is checked when the host wakes. Quiet healthy connections exchange health checks; watching continuous output does not require typing. Cleanup leaves the remote Herdr server, sessions, and pane processes running. Older installations remain compatible without this optional cleanup.
Background connections never answer prompts or install, update, restart, or hand off a server. For Attention, run the standalone setup command shown by Herdr in an interactive terminal, for example:
```bash
+4
View File
@@ -19,6 +19,10 @@ maintenance-test:
test-one filter:
cargo nextest run --locked "{{filter}}" --status-level fail --final-status-level fail --failure-output final --success-output never
# Rootless Linux SSH teardown smoke, comparing an old and a candidate binary
smoke-ssh-bridge-liveness before after:
bun scripts/smoke_ssh_bridge_liveness.mjs "{{before}}" "{{after}}"
# Enforce deterministic UI hot-path architecture boundaries
ui-hot-path-architecture-test:
{{python}} -m unittest scripts.test_ui_hot_path_architecture
+160
View File
@@ -0,0 +1,160 @@
#!/usr/bin/env bun
// Linux-only, rootless network-namespace smoke. No host firewall or SSH configuration changes.
import { spawn, spawnSync } from 'node:child_process';
import { mkdirSync, mkdtempSync, readlinkSync, rmSync, writeFileSync } from 'node:fs';
import { resolve } from 'node:path';
import { userInfo } from 'node:os';
import { fileURLToPath } from 'node:url';
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
function run(program, args, env = process.env) {
const result = spawnSync(program, args, { encoding: 'utf8', env });
if (result.status !== 0) throw new Error(`${program}: ${result.stderr || result.error || result.status}`);
return result.stdout;
}
async function until(predicate, milliseconds, label) {
const deadline = Date.now() + milliseconds;
while (!predicate()) {
if (Date.now() >= deadline) throw new Error(`timed out: ${label}`);
await delay(50);
}
}
function string(value) {
const bytes = Buffer.from(value);
const length = bytes.length < 251 ? Buffer.from([bytes.length]) : Buffer.from([251, bytes.length & 255, bytes.length >> 8]);
return Buffer.concat([length, bytes]);
}
function control(kind, data) {
// Generation-1 EndpointControl tag and length framing, also exercised in tests/support/mod.rs.
const payload = Buffer.concat([Buffer.from([20]), string(kind), string(data)]);
const size = Buffer.alloc(4);
size.writeUInt32LE(payload.length);
return Buffer.concat([size, payload]);
}
const quote = (text) => `'${text.replaceAll("'", "'\\''")}'`;
function cleanEnvironment(root) {
const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith('HERDR_')));
Object.assign(env, {
XDG_CONFIG_HOME: `${root}/config`, XDG_STATE_HOME: `${root}/state`, XDG_RUNTIME_DIR: `${root}/run`,
HERDR_CONFIG_PATH: `${root}/config.toml`, HERDR_SOCKET_PATH: `${root}/api.sock`, HERDR_CLIENT_SOCKET_PATH: `${root}/client.sock`,
HERDR_DISABLE_SOUND: '1',
});
for (const path of [env.XDG_CONFIG_HOME, env.XDG_STATE_HOME, env.XDG_RUNTIME_DIR]) mkdirSync(path, { recursive: true });
writeFileSync(env.HERDR_CONFIG_PATH, '[experimental]\nallow_nested = true\n');
return env;
}
async function smoke(before, after) {
const root = mkdtempSync('/var/tmp/herdr-ssh-liveness-');
const children = [];
const bridges = new Set();
const launch = (program, args, options = {}) => {
const child = spawn(program, args, { stdio: ['ignore', 'inherit', 'inherit'], ...options });
children.push(child);
return child;
};
let success = false;
try {
run('ip', ['link', 'set', 'lo', 'up']);
run('ip', ['addr', 'add', '10.77.0.2/32', 'dev', 'lo']);
run('nft', ['add', 'table', 'inet', 'herdr_smoke']);
run('nft', ['add', 'chain', 'inet', 'herdr_smoke', 'output', '{ type filter hook output priority 0; policy accept; }']);
for (const key of ['host', 'client']) run('ssh-keygen', ['-q', '-t', 'ed25519', '-N', '', '-f', `${root}/${key}`]);
writeFileSync(`${root}/sshd_config`, `Port 22222\nListenAddress 10.77.0.2\nHostKey ${root}/host\nAuthorizedKeysFile ${root}/client.pub\nPidFile ${root}/sshd.pid\nStrictModes no\nPasswordAuthentication no\nKbdInteractiveAuthentication no\nUsePAM no\nClientAliveInterval 0\nTCPKeepAlive no\nLogLevel ERROR\n`);
const sshd = launch(Bun.which('sshd'), ['-D', '-e', '-f', `${root}/sshd_config`]);
await delay(400);
if (sshd.exitCode !== null) throw new Error('disposable sshd failed');
const sockets = () => run('ss', ['-Hnt', 'sport', '=', '22222']).split('\n').filter((line) => line.startsWith('ESTAB'));
const alive = (pid) => { try { process.kill(pid, 0); return true; } catch { return false; } };
for (const [label, binary, expectedCleanup] of [['before', before, false], ['after', after, true]]) {
const phase = `${root}/${label}`;
mkdirSync(phase);
const env = cleanEnvironment(phase);
const server = launch(binary, ['--session', 'ssh-smoke', 'server'], { env });
await until(() => {
try { return JSON.parse(run(binary, ['--session', 'ssh-smoke', 'status', 'server', '--json'], env)).running; }
catch { return false; }
}, 10000, 'isolated Herdr server');
const connections = [];
for (let index = 0; index < 3; index++) {
const remoteEnv = Object.entries(env).filter(([key]) => key.startsWith('HERDR_') || key.startsWith('XDG_')).map(([key, value]) => `${key}=${quote(value)}`).join(' ');
const command = `printf 'BRIDGE %s %s\\n' "$$" "$PPID"; exec env ${remoteEnv} ${quote(binary)} --session ssh-smoke remote-client-bridge --idle-timeout-v1`;
const ssh = launch('ssh', ['-F', '/dev/null', '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'BatchMode=yes', '-o', 'IdentitiesOnly=yes', '-o', 'IdentityAgent=none', '-i', `${root}/client`, '-p', '22222', `${userInfo().username}@10.77.0.2`, command], { stdio: ['pipe', 'pipe', 'inherit'] });
let output = Buffer.alloc(0);
let welcome = false;
let ids;
ssh.stdout.on('data', (chunk) => {
output = Buffer.concat([output, chunk]);
if (!ids) {
const end = output.indexOf(10);
if (end < 0) return;
const match = output.subarray(0, end).toString().match(/^BRIDGE (\d+) (\d+)$/);
if (!match) throw new Error('invalid bridge marker');
ids = match.slice(1).map(Number);
bridges.add(ids[0]);
output = output.subarray(end + 1);
}
while (output.length >= 4 && output.length >= 4 + output.readUInt32LE(0)) {
const size = output.readUInt32LE(0);
const frame = output.subarray(4, 4 + size);
if (frame.includes(Buffer.from('endpoint.welcome.v1'))) welcome = true;
output = output.subarray(4 + size);
}
});
await until(() => ids, 5000, 'bridge marker');
ssh.stdin.write(control('endpoint.hello.v1', JSON.stringify({ generation: 1, cell_width_px: 8, cell_height_px: 16, surface_size: { cols: 80, rows: 24 }, pixel_mouse: false, direct_graphics: false, endpoint_keybindings: false, mouse_capture: false, surface_active: false, snapshot_codecs: ['shell.snapshot.v1'], surface_codecs: ['shell.surface.v1'], input_codecs: ['shell.input.semantic.v1'], blob_codecs: ['shell.blob.v1'] })));
await until(() => welcome, 5000, 'real endpoint handshake');
connections.push({ ssh, ids });
}
await delay(1000);
console.log(`${label}: three real Herdr handshakes; suppressing disconnect packets`);
run('nft', ['add', 'rule', 'inet', 'herdr_smoke', 'output', 'tcp', 'dport', '22222', 'drop']);
for (const { ssh } of connections) ssh.kill('SIGKILL');
await delay(200);
run('ss', ['-K', 'dst', '10.77.0.2', 'dport', '=', '22222']);
if (run('ss', ['-Hnt', 'dst', '10.77.0.2', 'dport', '=', '22222']).trim()) throw new Error('desktop sockets remain');
if (sockets().length !== 3) throw new Error('failed to reproduce three abandoned remote SSH transports');
run('nft', ['flush', 'chain', 'inet', 'herdr_smoke', 'output']);
const started = Date.now();
if (expectedCleanup) {
await until(() => sockets().length === 0 && connections.every(({ ids }) => ids.every((pid) => !alive(pid))), 75000, 'all bridges AND SSH sessions reclaimed');
console.log(`PASS after: 3/3 bridges and SSH sessions reclaimed in ${((Date.now() - started) / 1000).toFixed(1)}s`);
} else {
await delay(65000);
if (sockets().length !== 3 || !connections.every(({ ids }) => ids.every(alive))) throw new Error('baseline did not retain the abandoned connections');
console.log('RED before: 3/3 bridges and SSH sessions still retained after 65s');
// Only these recorded disposable bridge PIDs are terminated, never sshd or a main server.
for (const { ids } of connections) process.kill(ids[0], 'SIGTERM');
await until(() => sockets().length === 0, 5000, 'baseline cleanup');
}
for (const { ids } of connections) bridges.delete(ids[0]);
if (server.exitCode !== null || sshd.exitCode !== null) throw new Error('server died');
run(binary, ['--session', 'ssh-smoke', 'api', 'snapshot'], env);
console.log(`${label}: isolated Herdr server still responds`);
server.kill('SIGTERM');
await until(() => server.exitCode !== null, 5000, 'isolated server shutdown');
}
success = true;
console.log('PASS: real SSH + Herdr, lost teardown, three concurrent connections, server survival');
} finally {
for (const pid of bridges) { try { process.kill(pid, 'SIGTERM'); } catch {} }
for (const child of children.reverse()) { if (child.exitCode === null && child.signalCode === null) child.kill('SIGTERM'); }
await delay(300);
for (const child of children) { if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); }
if (success) rmSync(root, { recursive: true, force: true });
else console.error(`Smoke evidence retained at ${root}`);
}
}
const args = process.argv.slice(2);
if (args[0] !== '--isolated') {
if (args.length !== 2 || process.platform !== 'linux') {
throw new Error('usage: bun scripts/smoke_ssh_bridge_liveness.mjs <before-binary> <after-binary> (Linux; requires unshare, nft, ss, sshd, ssh-keygen)');
}
const child = spawn('unshare', ['-Ucn', '--keep-caps', process.execPath, fileURLToPath(import.meta.url), '--isolated', ...args.map((path) => resolve(path))], { stdio: 'inherit' });
child.on('exit', (code) => process.exit(code ?? 1));
} else {
if (readlinkSync('/proc/self/ns/net') === readlinkSync('/proc/1/ns/net')) throw new Error('refusing host network namespace');
await smoke(args[1], args[2]);
}
+2
View File
@@ -253,6 +253,7 @@ struct ClientStatusJson {
endpoint_protocol_generation: u32,
endpoint_capabilities: Vec<&'static str>,
remote_host_bridge: bool,
remote_bridge_idle_timeout: bool,
binary: String,
session: Option<String>,
}
@@ -299,6 +300,7 @@ fn client_status_json() -> ClientStatusJson {
crate::protocol::endpoint::HEALTH_CHECK_CAPABILITY,
],
remote_host_bridge: true,
remote_bridge_idle_timeout: crate::platform::REMOTE_BRIDGE_IDLE_TIMEOUT_SUPPORTED,
binary: current_exe_label(),
session: crate::session::active_name(),
}
+53 -5
View File
@@ -9,7 +9,9 @@ use crate::protocol::{ClientSurfaceSize, RenderEncoding};
use interprocess::TryClone as _;
const INITIAL_RETRY_DELAY: Duration = Duration::from_millis(500);
const MAX_RETRY_DELAY: Duration = Duration::from_secs(30);
const MAX_RETRY_DELAY: Duration = Duration::from_secs(120);
const MAX_LOCAL_RETRY_DELAY: Duration = Duration::from_secs(30);
const STABLE_CONNECTION_PERIOD: Duration = Duration::from_secs(60);
#[derive(Clone, Copy)]
pub(crate) struct EndpointConnectOptions {
@@ -51,6 +53,7 @@ struct ReconnectState {
next_attempt: Option<Instant>,
in_flight: bool,
generation: Option<u64>,
online_since: Option<Instant>,
}
impl ReconnectState {
@@ -61,6 +64,7 @@ impl ReconnectState {
next_attempt: Some(now),
in_flight: false,
generation: None,
online_since: None,
}
}
}
@@ -200,15 +204,32 @@ impl EndpointSupervisors {
state.in_flight = false;
match status {
ClientEndpointStatus::Online => {
state.attempts = 0;
if endpoint_id.is_local() {
state.attempts = 0;
}
state.online_since.get_or_insert(now);
state.next_attempt = None;
}
ClientEndpointStatus::Attention | ClientEndpointStatus::Disabled => {
state.next_attempt = None
state.online_since = None;
state.next_attempt = None;
}
ClientEndpointStatus::Connecting | ClientEndpointStatus::Reconnecting => {
// A brief maintenance wake can complete a handshake without restoring the link.
if state.online_since.take().is_some_and(|connected| {
now.saturating_duration_since(connected) >= STABLE_CONNECTION_PERIOD
}) {
state.attempts = 0;
}
state.attempts = state.attempts.saturating_add(1);
state.next_attempt = Some(now + retry_delay(state.attempts));
let delay = retry_delay(state.attempts);
state.next_attempt = Some(
now + if endpoint_id.is_local() {
delay.min(MAX_LOCAL_RETRY_DELAY)
} else {
delay
},
);
}
}
true
@@ -334,7 +355,7 @@ fn retry_delay(attempt: u32) -> Duration {
INITIAL_RETRY_DELAY
.saturating_mul(
1_u32
.checked_shl(attempt.saturating_sub(1).min(6))
.checked_shl(attempt.saturating_sub(1).min(8))
.unwrap_or(u32::MAX),
)
.min(MAX_RETRY_DELAY)
@@ -425,6 +446,33 @@ mod tests {
assert_eq!(supervisors.endpoints[&other_id].generation, Some(3));
}
#[test]
fn brief_ssh_reconnections_do_not_reset_backoff() {
let now = Instant::now();
let profile = profile();
let id = ClientEndpointId::Ssh(profile.id.clone());
let mut supervisors = EndpointSupervisors::new(&[profile], now);
supervisors.endpoints.get_mut(&id).unwrap().generation = Some(2);
for attempt in 1..=5 {
let connected = now + Duration::from_secs(attempt * 20);
assert!(supervisors.record_status(&id, 2, ClientEndpointStatus::Online, connected));
let failed = connected + Duration::from_secs(15);
assert!(supervisors.disconnected(&id, 2, failed));
assert_eq!(
supervisors.endpoints[&id].next_attempt,
Some(failed + INITIAL_RETRY_DELAY * (1 << (attempt - 1)))
);
}
let connected = now + Duration::from_secs(200);
assert!(supervisors.record_status(&id, 2, ClientEndpointStatus::Online, connected));
let failed = connected + Duration::from_secs(60);
assert!(supervisors.disconnected(&id, 2, failed));
assert_eq!(
supervisors.endpoints[&id].next_attempt,
Some(failed + INITIAL_RETRY_DELAY)
);
}
#[test]
fn retry_backoff_is_bounded() {
assert_eq!(retry_delay(1), INITIAL_RETRY_DELAY);
+1 -1
View File
@@ -551,7 +551,7 @@ fn main() -> io::Result<()> {
// Subcommands and flags (no TUI, no logging needed)
if args.get(1).map(|s| s.as_str()) == Some("remote-client-bridge") {
return remote::run_remote_client_bridge();
return remote::run_remote_client_bridge(&args[2..]);
}
if args.get(1).map(|s| s.as_str()) == Some("server") {
+7 -1
View File
@@ -13,7 +13,13 @@ pub(crate) fn set_default_plugin_pane_pwd(
) {
}
pub(crate) fn forward_remote_bridge_stdio(stream: crate::ipc::LocalStream) -> std::io::Result<()> {
#[cfg(unix)]
pub(super) const REMOTE_BRIDGE_CLOCK: libc::clockid_t = libc::CLOCK_MONOTONIC;
pub(crate) fn forward_remote_bridge_stdio(
stream: crate::ipc::LocalStream,
_idle_timeout: bool,
) -> std::io::Result<()> {
use interprocess::TryClone as _;
let mut stdout = std::io::stdout().lock();
+2
View File
@@ -7,6 +7,8 @@ use std::{
sync::OnceLock,
};
pub(super) const REMOTE_BRIDGE_CLOCK: libc::clockid_t = libc::CLOCK_BOOTTIME;
use super::{
read_limited_reader, ClipboardCommand, ClipboardImage, ForegroundJob, ForegroundProcess,
LimitedRead, Signal,
+2
View File
@@ -7,6 +7,8 @@ use std::process::{Command, Stdio};
use std::ptr::NonNull;
use std::sync::OnceLock;
pub(super) const REMOTE_BRIDGE_CLOCK: libc::clockid_t = libc::CLOCK_MONOTONIC;
use super::{
read_limited_reader, ClipboardCommand, ClipboardImage, ForegroundJob, ForegroundProcess,
LimitedRead, Signal,
+7
View File
@@ -266,6 +266,13 @@ pub(crate) struct RemoteSshConfigPaths {
pub(crate) multiplexing: bool,
}
pub(crate) const REMOTE_BRIDGE_IDLE_TIMEOUT_SUPPORTED: bool =
cfg!(any(target_os = "linux", target_os = "macos"));
#[cfg(unix)]
mod remote_bridge;
#[cfg(all(test, unix))]
mod remote_bridge_tests;
#[cfg(unix)]
mod unix_common;
#[cfg(unix)]
+135
View File
@@ -0,0 +1,135 @@
use std::io::{self, Read, Write};
use std::sync::{
atomic::{AtomicU64, Ordering},
mpsc, Arc,
};
use std::time::Duration;
pub(crate) const IDLE_TIMEOUT: Duration = Duration::from_secs(60);
fn now() -> io::Result<u64> {
let mut time = libc::timespec {
tv_sec: 0,
tv_nsec: 0,
};
// This clock includes suspend, so short maintenance wakes can reap old bridges.
if unsafe { libc::clock_gettime(super::REMOTE_BRIDGE_CLOCK, &mut time) } != 0 {
return Err(io::Error::last_os_error());
}
Ok((time.tv_sec as u64) * 1_000_000_000 + time.tv_nsec as u64)
}
#[derive(Clone)]
pub(super) struct Activity {
last: Arc<AtomicU64>,
_stop: mpsc::Sender<()>,
}
impl Activity {
pub(super) fn start(timeout: Duration) -> io::Result<Self> {
let last = Arc::new(AtomicU64::new(now()?));
let watched = Arc::clone(&last);
let (stop, stopped) = mpsc::channel();
std::thread::Builder::new()
.name("ssh-bridge-liveness".into())
.spawn(move || {
loop {
match stopped.recv_timeout(timeout.min(Duration::from_secs(1))) {
Ok(()) | Err(mpsc::RecvTimeoutError::Disconnected) => return,
Err(mpsc::RecvTimeoutError::Timeout) => {}
}
let expired = now().map_or(true, |now| idle_expired(&watched, now, timeout));
if expired {
// Only the dedicated bridge process owns this watchdog. Returning from a
// blocked copy cannot guarantee shutdown, and joining it could hang forever.
std::process::exit(1);
}
}
})?;
Ok(Self { last, _stop: stop })
}
fn record(&self) -> io::Result<()> {
self.last.fetch_max(now()?, Ordering::Relaxed);
Ok(())
}
}
fn idle_expired(last: &AtomicU64, now: u64, timeout: Duration) -> bool {
Duration::from_nanos(now.saturating_sub(last.load(Ordering::Relaxed))) >= timeout
}
pub(super) struct TrackedIo<T> {
inner: T,
activity: Option<Activity>,
}
impl<T> TrackedIo<T> {
pub(super) fn new(inner: T, activity: Option<Activity>) -> Self {
Self { inner, activity }
}
fn progressed(&self, count: usize) -> io::Result<()> {
if count > 0 {
if let Some(activity) = &self.activity {
activity.record()?;
}
}
Ok(())
}
}
impl<T: Read> Read for TrackedIo<T> {
fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
let count = self.inner.read(buffer)?;
self.progressed(count)?;
Ok(count)
}
}
impl<T: Write> Write for TrackedIo<T> {
fn write(&mut self, buffer: &[u8]) -> io::Result<usize> {
let count = self.inner.write(buffer)?;
self.progressed(count)?;
Ok(count)
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn idle_deadline_counts_elapsed_sleep_and_only_positive_io_renews_it() {
let (stop, _stopped) = mpsc::channel();
let last = Arc::new(AtomicU64::new(0));
let activity = Activity {
last: Arc::clone(&last),
_stop: stop,
};
assert!(idle_expired(
&last,
IDLE_TIMEOUT.as_nanos() as u64,
IDLE_TIMEOUT
));
let mut reader = TrackedIo::new(&b"output"[..], Some(activity.clone()));
reader.read_exact(&mut [0; 6]).unwrap();
let after_read = last.load(Ordering::Relaxed);
assert!(after_read > 0);
assert!(!idle_expired(&last, after_read, IDLE_TIMEOUT));
assert_eq!(reader.read(&mut [0; 1]).unwrap(), 0);
assert_eq!(last.load(Ordering::Relaxed), after_read);
let mut writer = TrackedIo::new(Vec::new(), Some(activity));
writer.write_all(b"ping").unwrap();
assert!(last.load(Ordering::Relaxed) >= after_read);
assert!(idle_expired(
&last,
last.load(Ordering::Relaxed) + 120_000_000_000,
IDLE_TIMEOUT
));
}
}
+168
View File
@@ -0,0 +1,168 @@
use std::io::{Read, Write};
use std::os::unix::net::{UnixListener, UnixStream};
use std::path::PathBuf;
use std::process::{Child, Command, ExitStatus, Stdio};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
const TIMEOUT: Duration = Duration::from_millis(300);
#[test]
fn bridge_child() {
let Some(path) = std::env::var_os("HERDR_BRIDGE_TEST_SOCKET") else {
return;
};
let stream = crate::ipc::connect_local_stream(&PathBuf::from(path)).unwrap();
let timeout = (std::env::var_os("HERDR_BRIDGE_TEST_LEGACY").is_none()).then_some(TIMEOUT);
super::unix_common::forward_remote_bridge_stdio_with_timeout(stream, timeout).unwrap();
}
struct Bridge {
child: Child,
stream: UnixStream,
path: PathBuf,
}
impl Bridge {
fn start(legacy: bool) -> Self {
static NEXT: AtomicU64 = AtomicU64::new(0);
let path = std::env::temp_dir().join(format!(
"hbl-{}-{}.sock",
std::process::id(),
NEXT.fetch_add(1, Ordering::Relaxed)
));
let listener = UnixListener::bind(&path).unwrap();
listener.set_nonblocking(true).unwrap();
let mut command = Command::new(std::env::current_exe().unwrap());
command
.args([
"--exact",
"platform::remote_bridge_tests::bridge_child",
"--nocapture",
])
.env("HERDR_BRIDGE_TEST_SOCKET", &path)
.env_remove("HERDR_BRIDGE_TEST_LEGACY")
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::inherit());
if legacy {
command.env("HERDR_BRIDGE_TEST_LEGACY", "1");
}
let mut child = command.spawn().unwrap();
let deadline = Instant::now() + Duration::from_secs(3);
let stream = loop {
match listener.accept() {
Ok((stream, _)) => break stream,
Err(error)
if error.kind() == std::io::ErrorKind::WouldBlock
&& Instant::now() < deadline =>
{
std::thread::sleep(Duration::from_millis(5));
}
Err(error) => {
let _ = child.kill();
let _ = child.wait();
let _ = std::fs::remove_file(&path);
panic!("bridge did not connect: {error}");
}
}
};
stream.set_nonblocking(false).unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(3)))
.unwrap();
stream
.set_write_timeout(Some(Duration::from_millis(100)))
.unwrap();
Self {
child,
stream,
path,
}
}
fn wait(&mut self) -> ExitStatus {
let deadline = Instant::now() + Duration::from_secs(3);
loop {
if let Some(status) = self.child.try_wait().unwrap() {
return status;
}
assert!(Instant::now() < deadline, "bridge did not exit");
std::thread::sleep(Duration::from_millis(10));
}
}
fn finish(&mut self) -> String {
drop(self.child.stdin.take());
let mut input = Vec::new();
self.stream.read_to_end(&mut input).unwrap();
self.stream
.write_all(b"final-output-after-stdin-eof")
.unwrap();
self.stream.shutdown(std::net::Shutdown::Write).unwrap();
assert!(self.wait().success());
let mut output = String::new();
self.child
.stdout
.take()
.unwrap()
.read_to_string(&mut output)
.unwrap();
output
}
}
impl Drop for Bridge {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
let _ = std::fs::remove_file(&self.path);
}
}
#[test]
fn bridge_expires_when_silent_or_stdout_is_blocked() {
for blocked in [false, true] {
let mut bridge = Bridge::start(false);
if blocked {
let buffer = vec![b'x'; 64 * 1024];
let deadline = Instant::now() + Duration::from_secs(2);
while bridge.stream.write_all(&buffer).is_ok() {
assert!(Instant::now() < deadline, "failed to fill bridge stdout");
}
}
assert_eq!(bridge.wait().code(), Some(1));
}
}
#[test]
fn bridge_preserves_one_way_progress_and_drains_after_stdin_eof() {
for upload in [false, true] {
let mut bridge = Bridge::start(false);
for _ in 0..12 {
if upload {
bridge
.child
.stdin
.as_mut()
.unwrap()
.write_all(b"ping")
.unwrap();
bridge.stream.read_exact(&mut [0; 4]).unwrap();
} else {
bridge.stream.write_all(b"output").unwrap();
}
std::thread::sleep(Duration::from_millis(60));
assert!(bridge.child.try_wait().unwrap().is_none());
}
assert!(bridge.finish().contains("final-output-after-stdin-eof"));
}
}
#[test]
fn legacy_bridge_has_no_idle_deadline() {
let mut bridge = Bridge::start(true);
std::thread::sleep(TIMEOUT * 2);
assert!(bridge.child.try_wait().unwrap().is_none());
assert!(bridge.finish().contains("final-output-after-stdin-eof"));
}
+23 -5
View File
@@ -121,15 +121,33 @@ pub(crate) fn wait_client_stream_readable(stream: &crate::ipc::LocalStream) -> s
Ok(())
}
pub(crate) fn forward_remote_bridge_stdio(stream: crate::ipc::LocalStream) -> std::io::Result<()> {
pub(crate) fn forward_remote_bridge_stdio(
stream: crate::ipc::LocalStream,
idle_timeout: bool,
) -> std::io::Result<()> {
forward_remote_bridge_stdio_with_timeout(
stream,
idle_timeout.then_some(super::remote_bridge::IDLE_TIMEOUT),
)
}
pub(super) fn forward_remote_bridge_stdio_with_timeout(
stream: crate::ipc::LocalStream,
idle_timeout: Option<std::time::Duration>,
) -> std::io::Result<()> {
use super::remote_bridge::{Activity, TrackedIo};
use interprocess::TryClone as _;
let mut stdout = std::io::stdout().lock();
let mut socket_to_stdout = stream.try_clone()?;
let activity = idle_timeout.map(Activity::start).transpose()?;
let mut stdout = TrackedIo::new(std::io::stdout().lock(), activity.clone());
let mut socket_to_stdout = TrackedIo::new(stream.try_clone()?, activity.clone());
let mut stdin_to_socket = stream;
let _upload = std::thread::spawn(move || {
let mut stdin = std::io::stdin();
let _ = copy_flush(&mut stdin, &mut stdin_to_socket);
let mut stdin = TrackedIo::new(std::io::stdin(), activity.clone());
let _ = copy_flush(
&mut stdin,
&mut TrackedIo::new(&mut stdin_to_socket, activity),
);
let crate::ipc::LocalStream::UdSocket(stream) = stdin_to_socket;
let _ = stream.inner().shutdown(std::net::Shutdown::Write);
});
+4 -1
View File
@@ -52,7 +52,10 @@ pub(crate) fn wait_client_stream_readable(
Ok(())
}
pub(crate) fn forward_remote_bridge_stdio(stream: crate::ipc::LocalStream) -> std::io::Result<()> {
pub(crate) fn forward_remote_bridge_stdio(
stream: crate::ipc::LocalStream,
_idle_timeout: bool,
) -> std::io::Result<()> {
use interprocess::TryClone as _;
use std::sync::atomic::{AtomicBool, Ordering};
+1 -1
View File
@@ -23,7 +23,7 @@ pub(crate) fn run_remote_api_bridge(args: &[String]) -> std::io::Result<()> {
),
)
})?;
crate::platform::forward_remote_bridge_stdio(stream)
crate::platform::forward_remote_bridge_stdio(stream, false)
}
[flag] if flag == "--check" => {
println!("herdr-api-bridge-v1");
+47 -5
View File
@@ -258,7 +258,16 @@ impl RemoteExecutable {
}
fn bridge_command(&self, session_name: &str) -> String {
let args = Self::session_args(session_name, &["remote-client-bridge"]);
self.bridge_command_with_idle_timeout(session_name, false)
}
fn bridge_command_with_idle_timeout(&self, session_name: &str, idle_timeout: bool) -> String {
let command = if idle_timeout {
&["remote-client-bridge", "--idle-timeout-v1"][..]
} else {
&["remote-client-bridge"][..]
};
let args = Self::session_args(session_name, command);
match self {
Self::PosixShellPath(_) => {
posix_remote_output_command(&format!("exec {}", self.command(&args)))
@@ -310,6 +319,7 @@ pub(super) struct RemoteHerdr {
install_suffix: String,
executable: RemoteExecutable,
platform: RemotePlatform,
bridge_idle_timeout: bool,
}
impl RemoteHerdr {
@@ -328,6 +338,7 @@ impl RemoteHerdr {
install_suffix,
executable,
platform,
bridge_idle_timeout: false,
}
}
@@ -1150,9 +1161,12 @@ pub(super) fn find_installed_remote_herdr(ssh: &RemoteSsh) -> io::Result<RemoteH
let platform = detect_remote_platform(ssh)?;
let remote_herdr = RemoteHerdr::for_platform(platform);
let candidates = remote_binary_candidates(ssh, &remote_herdr)?;
for candidate in candidates {
if remote_binary_supports_endpoint_requirement(ssh, &candidate, true)? {
return Ok(candidate);
for mut candidate in candidates {
if let Some(status) = remote_client_status(ssh, &candidate)? {
if status.supports_endpoint_requirement(&candidate.platform, true) {
candidate.bridge_idle_timeout = status.remote_bridge_idle_timeout;
return Ok(candidate);
}
}
}
Err(io::Error::new(
@@ -1907,6 +1921,8 @@ struct RemoteClientStatusJson {
endpoint_capabilities: Vec<String>,
#[serde(default)]
remote_host_bridge: bool,
#[serde(default)]
remote_bridge_idle_timeout: bool,
}
impl RemoteClientStatusJson {
@@ -2438,7 +2454,13 @@ impl SshStdioBridge {
) -> io::Result<Self> {
Self::start_command(
target,
remote_herdr.executable.bridge_command(&session_name),
if noninteractive && remote_herdr.bridge_idle_timeout {
remote_herdr
.executable
.bridge_command_with_idle_timeout(&session_name, true)
} else {
remote_herdr.executable.bridge_command(&session_name)
},
local_socket,
ssh_options,
noninteractive,
@@ -3669,6 +3691,7 @@ mod tests {
crate::protocol::endpoint::HEALTH_CHECK_CAPABILITY.into(),
],
remote_host_bridge: false,
remote_bridge_idle_timeout: false,
};
assert!(status.supports_endpoint_requirement(&linux, true));
for index in 0..status.endpoint_capabilities.len() {
@@ -4300,6 +4323,25 @@ mod tests {
}
}
#[test]
fn remote_bridge_idle_timeout_requires_explicit_support_and_opt_in() {
let legacy = parse_client_status_json(r#"{"endpoint_protocol_generation":1}"#).unwrap();
assert!(!legacy.remote_bridge_idle_timeout);
let current = parse_client_status_json(
r#"{"endpoint_protocol_generation":1,"remote_bridge_idle_timeout":true}"#,
)
.unwrap();
assert!(current.remote_bridge_idle_timeout);
let remote = RemoteHerdr::for_platform(RemotePlatform {
os: "linux",
arch: "x86_64",
});
assert!(remote
.executable
.bridge_command_with_idle_timeout("agents", true)
.ends_with(" --session agents remote-client-bridge --idle-timeout-v1"));
}
#[test]
fn remote_bridge_command_uses_installed_binary() {
let remote_herdr = RemoteHerdr::for_platform(RemotePlatform {
+17 -2
View File
@@ -3,7 +3,22 @@
use std::io;
use std::time::Duration;
pub(crate) fn run_remote_client_bridge() -> io::Result<()> {
pub(crate) fn run_remote_client_bridge(args: &[String]) -> io::Result<()> {
let idle_timeout = match args {
[] => false,
[option]
if option == "--idle-timeout-v1"
&& crate::platform::REMOTE_BRIDGE_IDLE_TIMEOUT_SUPPORTED =>
{
true
}
_ => {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"unsupported remote client bridge option",
))
}
};
ensure_remote_server_running()?;
let socket_path = crate::server::socket_paths::client_socket_path();
@@ -17,7 +32,7 @@ pub(crate) fn run_remote_client_bridge() -> io::Result<()> {
)
})?;
crate::platform::forward_remote_bridge_stdio(stream)
crate::platform::forward_remote_bridge_stdio(stream, idle_timeout)
}
fn ensure_remote_server_running() -> io::Result<()> {
+9 -2
View File
@@ -889,9 +889,10 @@ fn federated_client_starts_without_local_and_survives_its_restart() {
std::os::unix::fs::symlink(env!("CARGO_BIN_EXE_herdr"), bin.join("herdr")).unwrap();
let quote =
|path: &std::path::Path| format!("'{}'", path.display().to_string().replace('\'', "'\\''"));
let ssh_commands = base.join("ssh-commands");
fs::write(bin.join("ssh"), format!(
"#!/bin/sh\nexport HOME={} XDG_CONFIG_HOME={} XDG_RUNTIME_DIR={} HERDR_SOCKET_PATH={}\nunset HERDR_CLIENT_SOCKET_PATH HERDR_SESSION\nfor arg do last=\"$arg\"; done\nexec /bin/sh -c \"$last\"\n",
quote(&base.join("home")), quote(&remote_config), quote(&remote_runtime), quote(&remote_api),
"#!/bin/sh\nexport HOME={} XDG_CONFIG_HOME={} XDG_RUNTIME_DIR={} HERDR_SOCKET_PATH={}\nunset HERDR_CLIENT_SOCKET_PATH HERDR_SESSION\nfor arg do last=\"$arg\"; done\nprintf '%s\\n' \"$last\" >> {}\nexec /bin/sh -c \"$last\"\n",
quote(&base.join("home")), quote(&remote_config), quote(&remote_runtime), quote(&remote_api), quote(&ssh_commands),
)).unwrap();
fs::set_permissions(bin.join("ssh"), fs::Permissions::from_mode(0o700)).unwrap();
let path = format!(
@@ -914,6 +915,12 @@ fn federated_client_starts_without_local_and_survives_its_restart() {
"remote must be usable before Local exists: {}",
read_output(&output)
);
assert!(
fs::read_to_string(&ssh_commands)
.unwrap()
.contains("remote-client-bridge --idle-timeout-v1"),
"saved machine discovery must opt into the advertised bridge idle timeout"
);
let mut local = spawn_server(&config_home, &runtime_dir, &api_socket, &client_socket);
wait_for_socket(&api_socket, Duration::from_secs(10));