diff --git a/crates/ide-remote/src/deploy.rs b/crates/ide-remote/src/deploy.rs index ad9975ecf..04151c14e 100644 --- a/crates/ide-remote/src/deploy.rs +++ b/crates/ide-remote/src/deploy.rs @@ -207,6 +207,20 @@ impl Deployment<'_> { async fn remote_files(&self, dir: &str, out: &mut Vec) -> IdeResult<()> { use futures::stream::{FuturesUnordered, StreamExt}; let start = out.len(); + // One `find` over SSH instead of a listing per folder (thousands of round trips on a + // big project); the same entries the walk below would keep. + if let Some(entries) = self.fs.tree(dir, self.exclude.prunable_names()).await { + if self.cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } + let base = dir.trim_end_matches('/'); + out.extend(entries.into_iter().filter(|e| { + let rel = e.path.strip_prefix(base).unwrap_or(&e.path); + !self.exclude.remote(&e.path) && !rel.split('/').any(|name| !name.is_empty() && self.exclude.name(name)) + })); + out[start..].sort_by(|a, b| a.path.cmp(&b.path)); + return Ok(()); + } let mut pending = vec![dir.to_owned()]; let mut listing = FuturesUnordered::new(); loop { diff --git a/crates/ide-remote/src/mapping.rs b/crates/ide-remote/src/mapping.rs index b72807c81..ddabe704d 100644 --- a/crates/ide-remote/src/mapping.rs +++ b/crates/ide-remote/src/mapping.rs @@ -175,6 +175,8 @@ impl Mappings { #[derive(Debug, Clone)] pub struct Excludes { names: GlobSet, + /// The name patterns `find -name` matches the same way (no `/`, braces, classes or `**`). + plain: Vec, /// Remote paths (absolute) never touched. remote: Vec, } @@ -182,10 +184,14 @@ pub struct Excludes { impl Excludes { pub fn new(settings: &DeploymentSettings, server: &DeploymentServer, remote_root: &str) -> Self { let mut b = GlobSetBuilder::new(); + let mut plain = Vec::new(); for pattern in settings.exclude.iter().flat_map(|p| p.split(';')).map(str::trim).filter(|p| !p.is_empty()) { match Glob::new(pattern) { Ok(g) => { b.add(g); + if !pattern.contains(['/', '\\', '{', '}', '[', ']']) && !pattern.contains("**") { + plain.push(pattern.to_owned()); + } } Err(e) => tracing::warn!("deployment exclude {pattern:?}: {e}"), } @@ -197,7 +203,12 @@ impl Excludes { .filter(|p| !p.is_empty()) .map(|p| join(remote_root, p.trim_start_matches('/')).trim_end_matches('/').to_owned()) .collect(); - Self { names: b.build().unwrap_or_else(|_| GlobSet::empty()), remote } + Self { names: b.build().unwrap_or_else(|_| GlobSet::empty()), plain, remote } + } + + /// Name patterns a remote `find` can skip whole folders by; the rest is filtered afterwards. + pub fn prunable_names(&self) -> &[String] { + &self.plain } pub fn name(&self, name: &str) -> bool { diff --git a/crates/ide-remote/src/service.rs b/crates/ide-remote/src/service.rs index bffc04071..20d659106 100644 --- a/crates/ide-remote/src/service.rs +++ b/crates/ide-remote/src/service.rs @@ -87,7 +87,7 @@ impl RemoteService { return Err(IdeError::invalid("the deployment server has no SSH configuration")); } let session = self.ssh.session(ssh).await?; - Ok(RemoteFs::new(session.sftp().await?)) + Ok(RemoteFs::new(session.sftp().await?).with_shell(session)) } FsConnection::Ftp { server, secure, ftp } => { let password = self.ssh.password(&ide_ssh::ftp::secret_key(server)); @@ -418,7 +418,7 @@ impl RemoteService { return Err(IdeError::internal("container run targets start as local runs of the docker CLI")); } let session = self.ssh.session(&target.ssh_config).await?; - let fs = RemoteFs::new(session.sftp().await?); + let fs = RemoteFs::new(session.sftp().await?).with_shell(session.clone()); let folder = if target.project_root.trim().is_empty() { format!( "~/{}", diff --git a/crates/ide-remote/tests/remote.rs b/crates/ide-remote/tests/remote.rs index ee7d052a3..62534979c 100644 --- a/crates/ide-remote/tests/remote.rs +++ b/crates/ide-remote/tests/remote.rs @@ -16,6 +16,11 @@ struct Env { } async fn env() -> Env { + env_with(|_| Arc::new(|cmd| (format!("ran: {cmd}"), 0))).await +} + +/// `exec`: the server's command handler, given the folder that backs `/`. +async fn env_with(exec: impl FnOnce(std::path::PathBuf) -> ide_ssh::testing::ExecFn) -> Env { let tmp = tempfile::tempdir().unwrap(); let project = tmp.path().join("project"); let server_root = tmp.path().join("server"); @@ -25,7 +30,7 @@ async fn env() -> Env { std::fs::write(project.join("src/main.py"), "print('hi')\n").unwrap(); std::fs::write(project.join("README.md"), "# demo\n").unwrap(); std::fs::write(project.join(".git/HEAD"), "ref: refs/heads/main\n").unwrap(); - let port = ide_ssh::testing::sshd(server_root.clone(), Arc::new(|cmd| (format!("ran: {cmd}"), 0))).await; + let port = ide_ssh::testing::sshd(server_root.clone(), exec(server_root.clone())).await; let files = SshFiles { known_hosts: tmp.path().join("known_hosts"), config: None }; let target = SshTarget { host: "127.0.0.1".into(), @@ -184,6 +189,72 @@ async fn uploads_compares_syncs_and_downloads() { assert_eq!(d.push_changes(&env.project, &mut no_progress()).await.unwrap().files, 0); } +/// GNU `find -printf '%y\t%s\t%T@\t%p\0'` over the test server's files (no pruning: the +/// deployment filters what it excludes itself). +fn find_printf(root: std::path::PathBuf, calls: Arc) -> ide_ssh::testing::ExecFn { + Arc::new(move |cmd| { + let Some(dir) = cmd.strip_prefix("LC_ALL=C find ").and_then(|c| c.split(" -mindepth").next()) else { + return (String::new(), 127); + }; + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + fn walk(local: &std::path::Path, remote: &str, out: &mut String) { + for e in std::fs::read_dir(local).unwrap().flatten() { + let (path, meta) = (format!("{remote}/{}", e.file_name().to_string_lossy()), e.metadata().unwrap()); + if meta.is_dir() { + walk(&e.path(), &path, out); + } else { + let mtime = meta.modified().unwrap().duration_since(std::time::UNIX_EPOCH).unwrap(); + out.push_str(&format!( + "f\t{}\t{}.{:09}0\t{path}\0", + meta.len(), + mtime.as_secs(), + mtime.subsec_nanos() + )); + } + } + } + let mut out = String::new(); + let local = root.join(dir.trim_start_matches('/')); + if local.is_dir() { + walk(&local, dir, &mut out); + } + (out, 0) + }) +} + +#[tokio::test] +async fn compares_with_one_find_over_ssh_and_falls_back_to_listing_folders() { + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let counted = calls.clone(); + let env = env_with(move |root| find_printf(root, counted)).await; + let server = server(); + let settings = DeploymentSettings::default(); + let map = Mappings::new(&server, &env.project, "/srv"); + let exclude = Excludes::new(&settings, &server, "/srv"); + // An excluded folder on the server is not listed, whatever `find` returns. + std::fs::create_dir_all(env.server_root.join("srv/app/.git")).unwrap(); + std::fs::write(env.server_root.join("srv/app/.git/HEAD"), "x\n").unwrap(); + std::fs::write(env.server_root.join("srv/app/remote.txt"), "server side\n").unwrap(); + for shell in [true, false] { + let fs = RemoteFs::new(env.session.sftp().await.unwrap()); + let fs = if shell { fs.with_shell(env.session.clone()) } else { fs }; + let d = + Deployment { fs: &fs, map: &map, exclude: &exclude, preserve_time: true, cancel: CancellationToken::new() }; + let before = calls.load(std::sync::atomic::Ordering::SeqCst); + if shell { + assert_eq!(d.push_changes(&env.project, &mut no_progress()).await.unwrap().files, 2); + } + let entries = d.compare(&env.project).await.unwrap(); + let status = |p: &str| entries.iter().find(|e| e.path == p).map(|e| e.status); + assert_eq!(status("src/main.py"), Some(SyncStatus::Same), "{entries:?}"); + assert_eq!(status("README.md"), Some(SyncStatus::Same)); + assert_eq!(status("remote.txt"), Some(SyncStatus::RemoteOnly)); + assert!(entries.iter().all(|e| !e.path.starts_with(".git")), "{entries:?}"); + let used = calls.load(std::sync::atomic::Ordering::SeqCst) - before; + assert_eq!(used > 0, shell, "find is used only with a shell"); + } +} + struct Sink(Mutex>); impl ide_core::EventSink for Sink { diff --git a/crates/ide-ssh/src/filesystem.rs b/crates/ide-ssh/src/filesystem.rs index 5e8661b8f..e7f7325cf 100644 --- a/crates/ide-ssh/src/filesystem.rs +++ b/crates/ide-ssh/src/filesystem.rs @@ -35,6 +35,22 @@ impl RemoteFs { Self::Sftp(SftpFs::new(sftp)) } + /// Lets [`RemoteFs::tree`] list a whole folder tree with one command over `shell`. + pub fn with_shell(self, shell: std::sync::Arc) -> Self { + match self { + Self::Sftp(fs) => Self::Sftp(fs.with_shell(shell)), + other => other, + } + } + + /// See [`SftpFs::tree`]; `None` means list folder by folder. + pub async fn tree(&self, dir: &str, prune: &[String]) -> Option> { + match self { + Self::Sftp(fs) => fs.tree(dir, prune).await, + Self::Ftp(_) | Self::Local => None, + } + } + pub async fn home(&self) -> IdeResult { match self { Self::Sftp(fs) => fs.home().await, diff --git a/crates/ide-ssh/src/sftp.rs b/crates/ide-ssh/src/sftp.rs index c5522023a..14c66b9e3 100644 --- a/crates/ide-ssh/src/sftp.rs +++ b/crates/ide-ssh/src/sftp.rs @@ -80,11 +80,29 @@ fn err(path: &str, e: SftpError) -> IdeError { /// Higher-level operations on an [`SftpSession`]. pub struct SftpFs { sftp: SftpSession, + /// The SSH session behind it, for commands that list a whole tree at once ([`SftpFs::tree`]). + shell: Option>, } impl SftpFs { pub fn new(sftp: SftpSession) -> Self { - Self { sftp } + Self { sftp, shell: None } + } + + pub fn with_shell(self, shell: std::sync::Arc) -> Self { + Self { shell: Some(shell), ..self } + } + + /// Every file and symlink below `dir` (not into folders named by `prune`) in one `find` + /// command instead of a listing per folder. `None` when there is no shell or the host's + /// `find` can't do it (no GNU `-printf`, an unreadable folder): list folder by folder then. + pub async fn tree(&self, dir: &str, prune: &[String]) -> Option> { + let shell = self.shell.as_ref()?; + let out = shell.exec(&tree_command(dir, prune), None).await.ok()?; + if !out.success() { + return None; + } + parse_tree(&out.stdout) } /// The login directory. @@ -269,10 +287,61 @@ pub fn mtime_of(path: &Path) -> Option { u32::try_from(t.duration_since(std::time::UNIX_EPOCH).ok()?.as_secs()).ok() } +/// `find` listing the files and symlinks below `dir` as `type\tsize\tmtime\tpath\0` records. +fn tree_command(dir: &str, prune: &[String]) -> String { + let mut cmd = format!("LC_ALL=C find {} -mindepth 1 ", crate::shell::quote(dir)); + if !prune.is_empty() { + let names: Vec = prune.iter().map(|p| format!("-name {}", crate::shell::quote(p))).collect(); + cmd.push_str(&format!("\\( {} \\) -prune -o ", names.join(" -o "))); + } + cmd.push_str("\\( -type f -o -type l \\) -printf '%y\\t%s\\t%T@\\t%p\\0'"); + cmd +} + +fn parse_tree(out: &[u8]) -> Option> { + let mut entries = Vec::new(); + for record in out.split(|&b| b == 0).filter(|r| !r.is_empty()) { + let record = std::str::from_utf8(record).ok()?; + let mut fields = record.splitn(4, '\t'); + let (kind, size, mtime, path) = (fields.next()?, fields.next()?, fields.next()?, fields.next()?); + let mtime = mtime.split('.').next()?.parse::().ok()?; + entries.push(RemoteEntry { + name: path.rsplit('/').next().unwrap_or(path).to_owned(), + path: path.to_owned(), + is_dir: false, + is_symlink: kind == "l", + size: size.parse().ok()?, + mtime: u32::try_from(mtime).ok(), + permissions: String::new(), + }); + } + Some(entries) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn lists_a_tree_with_one_find() { + assert_eq!( + tree_command("/srv/my app", &[".git".into(), "*.pyc".into()]), + "LC_ALL=C find '/srv/my app' -mindepth 1 \\( -name .git -o -name '*.pyc' \\) -prune -o \ + \\( -type f -o -type l \\) -printf '%y\\t%s\\t%T@\\t%p\\0'" + ); + let out = b"f\t12\t1700000000.5000000000\t/srv/a b\ttab.txt\0l\t7\t1700000001.0\t/srv/link\0"; + let entries = parse_tree(out).unwrap(); + assert_eq!(entries.len(), 2); + assert_eq!( + (entries[0].path.as_str(), entries[0].name.as_str(), entries[0].size, entries[0].mtime), + ("/srv/a b\ttab.txt", "a b\ttab.txt", 12, Some(1_700_000_000)) + ); + assert!(entries[1].is_symlink && !entries[1].is_dir); + assert!(parse_tree(b"").unwrap().is_empty()); + // Not `find -printf` output (BusyBox, an error message): list folder by folder instead. + assert!(parse_tree(b"find: unrecognized: -printf\n").is_none()); + } + #[test] fn joins_and_splits_paths() { assert_eq!(join("/srv/app/", "a.txt"), "/srv/app/a.txt");