From 67afcc98055dd15588a253c19382bc4f407aba69 Mon Sep 17 00:00:00 2001 From: l0ng-ai <24760907+l0ng-ai@users.noreply.github.com> Date: Thu, 12 Nov 2026 23:19:58 +0800 Subject: [PATCH] perf(remote): list the remote tree with one find before a run (#2986) Comparing the project with the server (sync before a run, Sync with Deployed) listed the remote tree folder by folder over SFTP: thousands of round trips on a big project, even when nothing had changed. Over SSH it now runs one GNU `find -printf` for the whole tree, pruning folders the deployment excludes by a plain name pattern, and keeps the same entries the walk would (excluded names anywhere in the path, excluded remote paths). Hosts without GNU find (BusyBox), an unreadable folder, or FTP list folder by folder as before. Refs #2873 --- crates/ide-remote/src/deploy.rs | 14 ++++++ crates/ide-remote/src/mapping.rs | 13 +++++- crates/ide-remote/src/service.rs | 4 +- crates/ide-remote/tests/remote.rs | 73 ++++++++++++++++++++++++++++++- crates/ide-ssh/src/filesystem.rs | 16 +++++++ crates/ide-ssh/src/sftp.rs | 71 +++++++++++++++++++++++++++++- 6 files changed, 186 insertions(+), 5 deletions(-) diff --git a/crates/ide-remote/src/deploy.rs b/crates/ide-remote/src/deploy.rs index ad9975ecf..04151c14e 100644 --- a/crates/ide-remote/src/deploy.rs +++ b/crates/ide-remote/src/deploy.rs @@ -207,6 +207,20 @@ impl Deployment<'_> { async fn remote_files(&self, dir: &str, out: &mut Vec) -> IdeResult<()> { use futures::stream::{FuturesUnordered, StreamExt}; let start = out.len(); + // One `find` over SSH instead of a listing per folder (thousands of round trips on a + // big project); the same entries the walk below would keep. + if let Some(entries) = self.fs.tree(dir, self.exclude.prunable_names()).await { + if self.cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } + let base = dir.trim_end_matches('/'); + out.extend(entries.into_iter().filter(|e| { + let rel = e.path.strip_prefix(base).unwrap_or(&e.path); + !self.exclude.remote(&e.path) && !rel.split('/').any(|name| !name.is_empty() && self.exclude.name(name)) + })); + out[start..].sort_by(|a, b| a.path.cmp(&b.path)); + return Ok(()); + } let mut pending = vec![dir.to_owned()]; let mut listing = FuturesUnordered::new(); loop { diff --git a/crates/ide-remote/src/mapping.rs b/crates/ide-remote/src/mapping.rs index b72807c81..ddabe704d 100644 --- a/crates/ide-remote/src/mapping.rs +++ b/crates/ide-remote/src/mapping.rs @@ -175,6 +175,8 @@ impl Mappings { #[derive(Debug, Clone)] pub struct Excludes { names: GlobSet, + /// The name patterns `find -name` matches the same way (no `/`, braces, classes or `**`). + plain: Vec, /// Remote paths (absolute) never touched. remote: Vec, } @@ -182,10 +184,14 @@ pub struct Excludes { impl Excludes { pub fn new(settings: &DeploymentSettings, server: &DeploymentServer, remote_root: &str) -> Self { let mut b = GlobSetBuilder::new(); + let mut plain = Vec::new(); for pattern in settings.exclude.iter().flat_map(|p| p.split(';')).map(str::trim).filter(|p| !p.is_empty()) { match Glob::new(pattern) { Ok(g) => { b.add(g); + if !pattern.contains(['/', '\\', '{', '}', '[', ']']) && !pattern.contains("**") { + plain.push(pattern.to_owned()); + } } Err(e) => tracing::warn!("deployment exclude {pattern:?}: {e}"), } @@ -197,7 +203,12 @@ impl Excludes { .filter(|p| !p.is_empty()) .map(|p| join(remote_root, p.trim_start_matches('/')).trim_end_matches('/').to_owned()) .collect(); - Self { names: b.build().unwrap_or_else(|_| GlobSet::empty()), remote } + Self { names: b.build().unwrap_or_else(|_| GlobSet::empty()), plain, remote } + } + + /// Name patterns a remote `find` can skip whole folders by; the rest is filtered afterwards. + pub fn prunable_names(&self) -> &[String] { + &self.plain } pub fn name(&self, name: &str) -> bool { diff --git a/crates/ide-remote/src/service.rs b/crates/ide-remote/src/service.rs index bffc04071..20d659106 100644 --- a/crates/ide-remote/src/service.rs +++ b/crates/ide-remote/src/service.rs @@ -87,7 +87,7 @@ impl RemoteService { return Err(IdeError::invalid("the deployment server has no SSH configuration")); } let session = self.ssh.session(ssh).await?; - Ok(RemoteFs::new(session.sftp().await?)) + Ok(RemoteFs::new(session.sftp().await?).with_shell(session)) } FsConnection::Ftp { server, secure, ftp } => { let password = self.ssh.password(&ide_ssh::ftp::secret_key(server)); @@ -418,7 +418,7 @@ impl RemoteService { return Err(IdeError::internal("container run targets start as local runs of the docker CLI")); } let session = self.ssh.session(&target.ssh_config).await?; - let fs = RemoteFs::new(session.sftp().await?); + let fs = RemoteFs::new(session.sftp().await?).with_shell(session.clone()); let folder = if target.project_root.trim().is_empty() { format!( "~/{}", diff --git a/crates/ide-remote/tests/remote.rs b/crates/ide-remote/tests/remote.rs index ee7d052a3..62534979c 100644 --- a/crates/ide-remote/tests/remote.rs +++ b/crates/ide-remote/tests/remote.rs @@ -16,6 +16,11 @@ struct Env { } async fn env() -> Env { + env_with(|_| Arc::new(|cmd| (format!("ran: {cmd}"), 0))).await +} + +/// `exec`: the server's command handler, given the folder that backs `/`. +async fn env_with(exec: impl FnOnce(std::path::PathBuf) -> ide_ssh::testing::ExecFn) -> Env { let tmp = tempfile::tempdir().unwrap(); let project = tmp.path().join("project"); let server_root = tmp.path().join("server"); @@ -25,7 +30,7 @@ async fn env() -> Env { std::fs::write(project.join("src/main.py"), "print('hi')\n").unwrap(); std::fs::write(project.join("README.md"), "# demo\n").unwrap(); std::fs::write(project.join(".git/HEAD"), "ref: refs/heads/main\n").unwrap(); - let port = ide_ssh::testing::sshd(server_root.clone(), Arc::new(|cmd| (format!("ran: {cmd}"), 0))).await; + let port = ide_ssh::testing::sshd(server_root.clone(), exec(server_root.clone())).await; let files = SshFiles { known_hosts: tmp.path().join("known_hosts"), config: None }; let target = SshTarget { host: "127.0.0.1".into(), @@ -184,6 +189,72 @@ async fn uploads_compares_syncs_and_downloads() { assert_eq!(d.push_changes(&env.project, &mut no_progress()).await.unwrap().files, 0); } +/// GNU `find -printf '%y\t%s\t%T@\t%p\0'` over the test server's files (no pruning: the +/// deployment filters what it excludes itself). +fn find_printf(root: std::path::PathBuf, calls: Arc) -> ide_ssh::testing::ExecFn { + Arc::new(move |cmd| { + let Some(dir) = cmd.strip_prefix("LC_ALL=C find ").and_then(|c| c.split(" -mindepth").next()) else { + return (String::new(), 127); + }; + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + fn walk(local: &std::path::Path, remote: &str, out: &mut String) { + for e in std::fs::read_dir(local).unwrap().flatten() { + let (path, meta) = (format!("{remote}/{}", e.file_name().to_string_lossy()), e.metadata().unwrap()); + if meta.is_dir() { + walk(&e.path(), &path, out); + } else { + let mtime = meta.modified().unwrap().duration_since(std::time::UNIX_EPOCH).unwrap(); + out.push_str(&format!( + "f\t{}\t{}.{:09}0\t{path}\0", + meta.len(), + mtime.as_secs(), + mtime.subsec_nanos() + )); + } + } + } + let mut out = String::new(); + let local = root.join(dir.trim_start_matches('/')); + if local.is_dir() { + walk(&local, dir, &mut out); + } + (out, 0) + }) +} + +#[tokio::test] +async fn compares_with_one_find_over_ssh_and_falls_back_to_listing_folders() { + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let counted = calls.clone(); + let env = env_with(move |root| find_printf(root, counted)).await; + let server = server(); + let settings = DeploymentSettings::default(); + let map = Mappings::new(&server, &env.project, "/srv"); + let exclude = Excludes::new(&settings, &server, "/srv"); + // An excluded folder on the server is not listed, whatever `find` returns. + std::fs::create_dir_all(env.server_root.join("srv/app/.git")).unwrap(); + std::fs::write(env.server_root.join("srv/app/.git/HEAD"), "x\n").unwrap(); + std::fs::write(env.server_root.join("srv/app/remote.txt"), "server side\n").unwrap(); + for shell in [true, false] { + let fs = RemoteFs::new(env.session.sftp().await.unwrap()); + let fs = if shell { fs.with_shell(env.session.clone()) } else { fs }; + let d = + Deployment { fs: &fs, map: &map, exclude: &exclude, preserve_time: true, cancel: CancellationToken::new() }; + let before = calls.load(std::sync::atomic::Ordering::SeqCst); + if shell { + assert_eq!(d.push_changes(&env.project, &mut no_progress()).await.unwrap().files, 2); + } + let entries = d.compare(&env.project).await.unwrap(); + let status = |p: &str| entries.iter().find(|e| e.path == p).map(|e| e.status); + assert_eq!(status("src/main.py"), Some(SyncStatus::Same), "{entries:?}"); + assert_eq!(status("README.md"), Some(SyncStatus::Same)); + assert_eq!(status("remote.txt"), Some(SyncStatus::RemoteOnly)); + assert!(entries.iter().all(|e| !e.path.starts_with(".git")), "{entries:?}"); + let used = calls.load(std::sync::atomic::Ordering::SeqCst) - before; + assert_eq!(used > 0, shell, "find is used only with a shell"); + } +} + struct Sink(Mutex>); impl ide_core::EventSink for Sink { diff --git a/crates/ide-ssh/src/filesystem.rs b/crates/ide-ssh/src/filesystem.rs index 5e8661b8f..e7f7325cf 100644 --- a/crates/ide-ssh/src/filesystem.rs +++ b/crates/ide-ssh/src/filesystem.rs @@ -35,6 +35,22 @@ impl RemoteFs { Self::Sftp(SftpFs::new(sftp)) } + /// Lets [`RemoteFs::tree`] list a whole folder tree with one command over `shell`. + pub fn with_shell(self, shell: std::sync::Arc) -> Self { + match self { + Self::Sftp(fs) => Self::Sftp(fs.with_shell(shell)), + other => other, + } + } + + /// See [`SftpFs::tree`]; `None` means list folder by folder. + pub async fn tree(&self, dir: &str, prune: &[String]) -> Option> { + match self { + Self::Sftp(fs) => fs.tree(dir, prune).await, + Self::Ftp(_) | Self::Local => None, + } + } + pub async fn home(&self) -> IdeResult { match self { Self::Sftp(fs) => fs.home().await, diff --git a/crates/ide-ssh/src/sftp.rs b/crates/ide-ssh/src/sftp.rs index c5522023a..14c66b9e3 100644 --- a/crates/ide-ssh/src/sftp.rs +++ b/crates/ide-ssh/src/sftp.rs @@ -80,11 +80,29 @@ fn err(path: &str, e: SftpError) -> IdeError { /// Higher-level operations on an [`SftpSession`]. pub struct SftpFs { sftp: SftpSession, + /// The SSH session behind it, for commands that list a whole tree at once ([`SftpFs::tree`]). + shell: Option>, } impl SftpFs { pub fn new(sftp: SftpSession) -> Self { - Self { sftp } + Self { sftp, shell: None } + } + + pub fn with_shell(self, shell: std::sync::Arc) -> Self { + Self { shell: Some(shell), ..self } + } + + /// Every file and symlink below `dir` (not into folders named by `prune`) in one `find` + /// command instead of a listing per folder. `None` when there is no shell or the host's + /// `find` can't do it (no GNU `-printf`, an unreadable folder): list folder by folder then. + pub async fn tree(&self, dir: &str, prune: &[String]) -> Option> { + let shell = self.shell.as_ref()?; + let out = shell.exec(&tree_command(dir, prune), None).await.ok()?; + if !out.success() { + return None; + } + parse_tree(&out.stdout) } /// The login directory. @@ -269,10 +287,61 @@ pub fn mtime_of(path: &Path) -> Option { u32::try_from(t.duration_since(std::time::UNIX_EPOCH).ok()?.as_secs()).ok() } +/// `find` listing the files and symlinks below `dir` as `type\tsize\tmtime\tpath\0` records. +fn tree_command(dir: &str, prune: &[String]) -> String { + let mut cmd = format!("LC_ALL=C find {} -mindepth 1 ", crate::shell::quote(dir)); + if !prune.is_empty() { + let names: Vec = prune.iter().map(|p| format!("-name {}", crate::shell::quote(p))).collect(); + cmd.push_str(&format!("\\( {} \\) -prune -o ", names.join(" -o "))); + } + cmd.push_str("\\( -type f -o -type l \\) -printf '%y\\t%s\\t%T@\\t%p\\0'"); + cmd +} + +fn parse_tree(out: &[u8]) -> Option> { + let mut entries = Vec::new(); + for record in out.split(|&b| b == 0).filter(|r| !r.is_empty()) { + let record = std::str::from_utf8(record).ok()?; + let mut fields = record.splitn(4, '\t'); + let (kind, size, mtime, path) = (fields.next()?, fields.next()?, fields.next()?, fields.next()?); + let mtime = mtime.split('.').next()?.parse::().ok()?; + entries.push(RemoteEntry { + name: path.rsplit('/').next().unwrap_or(path).to_owned(), + path: path.to_owned(), + is_dir: false, + is_symlink: kind == "l", + size: size.parse().ok()?, + mtime: u32::try_from(mtime).ok(), + permissions: String::new(), + }); + } + Some(entries) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn lists_a_tree_with_one_find() { + assert_eq!( + tree_command("/srv/my app", &[".git".into(), "*.pyc".into()]), + "LC_ALL=C find '/srv/my app' -mindepth 1 \\( -name .git -o -name '*.pyc' \\) -prune -o \ + \\( -type f -o -type l \\) -printf '%y\\t%s\\t%T@\\t%p\\0'" + ); + let out = b"f\t12\t1700000000.5000000000\t/srv/a b\ttab.txt\0l\t7\t1700000001.0\t/srv/link\0"; + let entries = parse_tree(out).unwrap(); + assert_eq!(entries.len(), 2); + assert_eq!( + (entries[0].path.as_str(), entries[0].name.as_str(), entries[0].size, entries[0].mtime), + ("/srv/a b\ttab.txt", "a b\ttab.txt", 12, Some(1_700_000_000)) + ); + assert!(entries[1].is_symlink && !entries[1].is_dir); + assert!(parse_tree(b"").unwrap().is_empty()); + // Not `find -printf` output (BusyBox, an error message): list folder by folder instead. + assert!(parse_tree(b"find: unrecognized: -printf\n").is_none()); + } + #[test] fn joins_and_splits_paths() { assert_eq!(join("/srv/app/", "a.txt"), "/srv/app/a.txt");