diff --git a/crates/ide-host/src/project.rs b/crates/ide-host/src/project.rs index c0ed6514..59860e91 100644 --- a/crates/ide-host/src/project.rs +++ b/crates/ide-host/src/project.rs @@ -844,7 +844,8 @@ impl Project { /// Project files of `q.languages` under `q.root` containing `q.word` as a whole /// word (unsaved buffers included; excluded roots, ignored names and - /// `.gitignore`d files skipped): the files a references search opens on its server. + /// `.gitignore`d folders skipped, while `.gitignore`d files elsewhere count, as + /// in IDEA's project scope): the files a references search opens on its server. pub fn reference_candidates( &self, q: &ide_lsp::CandidateQuery, diff --git a/crates/ide-search/src/engine.rs b/crates/ide-search/src/engine.rs index ccb3e356..638c1016 100644 --- a/crates/ide-search/src/engine.rs +++ b/crates/ide-search/src/engine.rs @@ -142,42 +142,16 @@ pub(crate) fn run(req: Request<'_>) -> IdeResult { } } - if let Some((first, rest)) = on_disk.split_first() - && !ctx.ctl.should_stop() - { - let mut walk = WalkBuilder::new(first); - for path in rest { - walk.add(path); - } - walk.hidden(true) - .ignore(true) - .git_ignore(true) - .git_global(true) - .git_exclude(true) - .parents(true) - .require_git(ide_core::ignored::inside_git_repo(first)) - .follow_links(false) - .threads(std::thread::available_parallelism().map_or(4, |n| n.get()).min(12)) - .filter_entry(|e| e.file_name() != ".git"); - walk.build_parallel().run(|| { + if !ctx.ctl.should_stop() { + walk_files(&on_disk, || { let mut searcher = searcher_builder.build(); let matcher = matcher.clone(); let ctx = &ctx; - Box::new(move |entry| { + Box::new(move |path| { if ctx.ctl.should_stop() { return WalkState::Quit; } - let entry = match entry { - Ok(entry) => entry, - Err(err) => { - tracing::debug!("search walk: {err}"); - return WalkState::Continue; - } - }; - // Archives (jar, zip) are browsed, not searched (IDEA skips them too). - if entry.file_type().is_some_and(|t| t.is_file()) && !ide_vfs::is_archive_file(entry.path()) { - ctx.search_file(entry.path(), &matcher, &mut searcher); - } + ctx.search_file(path, &matcher, &mut searcher); if ctx.ctl.stopped() { WalkState::Quit } else { WalkState::Continue } }) }); @@ -193,6 +167,67 @@ pub(crate) fn run(req: Request<'_>) -> IdeResult { }) } +/// A per-thread file visitor of [`walk_files`]. +pub(crate) type VisitFile<'s> = Box WalkState + Send + 's>; + +/// Visits the files below `roots` (and the roots that are files) in parallel, +/// the way IDEA's project scope sees them: directories ignored by +/// `.gitignore` / `.ignore` / `info/exclude` are skipped with everything in +/// them, while ignored files elsewhere (`.env`) and hidden files and folders +/// (`.github`, `.eslintrc`) are visited. `.git`, symlinks and archives (jar, +/// zip: browsed, not searched) are never visited. +pub(crate) fn walk_files<'s>(roots: &[PathBuf], mut visitor: impl FnMut() -> VisitFile<'s> + Send) { + let Some((first, rest)) = roots.split_first() else { return }; + let mut walk = WalkBuilder::new(first); + for path in rest { + walk.add(path); + } + walk.hidden(false) + .ignore(true) + .git_ignore(true) + .git_global(true) + .git_exclude(true) + .parents(true) + .require_git(ide_core::ignored::inside_git_repo(first)) + .follow_links(false) + .threads(std::thread::available_parallelism().map_or(4, |n| n.get()).min(12)) + .filter_entry(|e| e.file_name() != ".git"); + walk.build_parallel().run(|| { + let mut visit = visitor(); + Box::new(move |entry| { + let entry = match entry { + Ok(entry) => entry, + Err(err) => { + tracing::debug!("search walk: {err}"); + return WalkState::Continue; + } + }; + let Some(file_type) = entry.file_type() else { return WalkState::Continue }; + if file_type.is_dir() { + // The walker drops ignored files unseen, so the files of every walked + // directory come from listing it; the walker only picks the directories. + let Ok(children) = std::fs::read_dir(entry.path()) else { return WalkState::Continue }; + for child in children.flatten() { + let path = child.path(); + if child.file_name() != ".git" + && child.file_type().is_ok_and(|t| t.is_file()) + && !ide_vfs::is_archive_file(&path) + && matches!(visit(&path), WalkState::Quit) + { + return WalkState::Quit; + } + } + WalkState::Continue + } else if file_type.is_file() && entry.depth() == 0 && !ide_vfs::is_archive_file(entry.path()) { + // A root that is a file (an explicit file list); files below roots come from listings. + visit(entry.path()) + } else { + WalkState::Continue + } + }) + }); +} + /// Walk roots for a scope; relative paths are resolved against `root`. pub(crate) fn roots(root: &Path, scope: &SearchScope) -> IdeResult> { let dir = |path: &Path| -> IdeResult { diff --git a/crates/ide-search/src/files.rs b/crates/ide-search/src/files.rs index bada8830..97b3805d 100644 --- a/crates/ide-search/src/files.rs +++ b/crates/ide-search/src/files.rs @@ -1,10 +1,10 @@ //! The files of a search scope, for analyses that read whole files //! (Inspect Code, Locate Duplicates, Structural Search). -use crate::engine::{rel_path, roots}; +use crate::engine::{rel_path, roots, walk_files}; use crate::{FileMask, PathFilter, SearchScope}; use ide_core::{CancellationToken, IdeResult}; -use ignore::{WalkBuilder, WalkState}; +use ignore::WalkState; use std::path::PathBuf; use std::sync::Mutex; @@ -16,7 +16,7 @@ pub struct ScopeFile { } /// Lists the files of `scope` under `root` the way Find in Files walks them -/// (`.gitignore` honoured, hidden files and `.git` skipped), filtered by +/// (ignored folders and `.git` skipped; ignored and hidden files kept), filtered by /// `mask` and `filter`, sorted by relative path. Explicitly listed files are /// kept even when they do not exist on disk (unsaved buffers). Returns what was /// found so far when cancelled. @@ -43,42 +43,23 @@ pub fn scope_files( on_disk.push(path); } } - if let Some((first, rest)) = on_disk.split_first() { - let mut walk = WalkBuilder::new(first); - for path in rest { - walk.add(path); - } - walk.hidden(true) - .ignore(true) - .git_ignore(true) - .git_global(true) - .git_exclude(true) - .parents(true) - .require_git(ide_core::ignored::inside_git_repo(first)) - .follow_links(false) - .threads(std::thread::available_parallelism().map_or(4, |n| n.get()).min(12)) - .filter_entry(|e| e.file_name() != ".git"); - walk.build_parallel().run(|| { - let found = &found; - let keep = &keep; - Box::new(move |entry| { - if cancel.is_cancelled() { - return WalkState::Quit; - } - let Ok(entry) = entry else { return WalkState::Continue }; - if entry.file_type().is_some_and(|t| t.is_file()) && !ide_vfs::is_archive_file(entry.path()) { - let rel = rel_path(root, entry.path()); - if keep(&rel) { - found - .lock() - .unwrap_or_else(|e| e.into_inner()) - .push(ScopeFile { path: entry.into_path(), rel_path: rel }); - } - } - WalkState::Continue - }) - }); - } + walk_files(&on_disk, || { + let found = &found; + let keep = &keep; + Box::new(move |path| { + if cancel.is_cancelled() { + return WalkState::Quit; + } + let rel = rel_path(root, path); + if keep(&rel) { + found + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(ScopeFile { path: path.to_path_buf(), rel_path: rel }); + } + WalkState::Continue + }) + }); let mut files = found.into_inner().unwrap_or_else(|e| e.into_inner()); files.sort_by(|a, b| a.rel_path.cmp(&b.rel_path)); files.dedup_by(|a, b| a.path == b.path); @@ -107,12 +88,12 @@ mod tests { } #[test] - fn project_scope_honours_gitignore_and_masks() { + fn project_scope_skips_ignored_folders_and_honours_masks() { let dir = tree(); let none = FileMask::new(&[], &[]).unwrap(); let cancel = CancellationToken::new(); let all = scope_files(dir.path(), &SearchScope::Project, &none, None, &cancel).unwrap(); - assert_eq!(rels(&all), vec!["src/a.rs", "src/c.txt", "src/sub/b.rs"]); + assert_eq!(rels(&all), vec![".gitignore", "src/a.rs", "src/c.txt", "src/sub/b.rs"]); let rs = FileMask::new(&["*.rs".into()], &[]).unwrap(); let only_rs = scope_files(dir.path(), &SearchScope::Project, &rs, None, &cancel).unwrap(); @@ -120,7 +101,7 @@ mod tests { let not_sub: PathFilter<'_> = &|rel: &str| !rel.starts_with("src/sub/"); let filtered = scope_files(dir.path(), &SearchScope::Project, &none, Some(not_sub), &cancel).unwrap(); - assert_eq!(rels(&filtered), vec!["src/a.rs", "src/c.txt"]); + assert_eq!(rels(&filtered), vec![".gitignore", "src/a.rs", "src/c.txt"]); } #[test] diff --git a/crates/ide-search/src/lib.rs b/crates/ide-search/src/lib.rs index 97515510..eb220fbb 100644 --- a/crates/ide-search/src/lib.rs +++ b/crates/ide-search/src/lib.rs @@ -1,7 +1,8 @@ //! Find / Replace in Files (⌘⇧F / ⌘⇧R). //! -//! Built on ripgrep's engine crates: a parallel `ignore` walk (honouring -//! `.gitignore` / `.ignore`, skipping hidden files and `.git`) feeding +//! Built on ripgrep's engine crates: a parallel `ignore` walk (skipping the +//! folders `.gitignore` / `.ignore` exclude and `.git`, but like IDEA keeping +//! ignored files elsewhere and hidden files) feeding //! `grep-searcher` with a `grep-regex` matcher. Binary files are skipped, //! unsaved editor buffers (the overlay) take precedence over disk content, //! and all columns are UTF-16. diff --git a/crates/ide-search/src/words.rs b/crates/ide-search/src/words.rs index ceb94a25..722ebe63 100644 --- a/crates/ide-search/src/words.rs +++ b/crates/ide-search/src/words.rs @@ -25,8 +25,8 @@ pub struct WordFiles { } /// Files under `query.dir` containing `query.word` as a whole word, in walk -/// order (unsaved buffers through `overlay`, `.gitignore`d and hidden files -/// skipped like Find in Files). Stops at `query.limit` files or when cancelled. +/// order (unsaved buffers through `overlay`; ignored folders skipped, ignored +/// and hidden files searched, like Find in Files). Stops at `query.limit` files or when cancelled. pub fn files_with_word( root: &std::path::Path, query: &WordQuery, @@ -74,7 +74,9 @@ mod tests { let root = dir.path(); fs::create_dir_all(root.join("pkg/ignored")).unwrap(); fs::create_dir_all(root.join("build")).unwrap(); - fs::write(root.join(".gitignore"), "ignored/\n").unwrap(); + fs::write(root.join(".gitignore"), "ignored/\nlocal.py\n").unwrap(); + // An ignored file outside ignored folders is still a candidate, as in IDEA. + fs::write(root.join("pkg/local.py"), "get_value\n").unwrap(); fs::write(root.join("pkg/use.py"), "x.get_value(1)\n").unwrap(); fs::write(root.join("pkg/longer.py"), "get_values()\n").unwrap(); fs::write(root.join("pkg/case.py"), "Get_Value()\n").unwrap(); @@ -90,7 +92,10 @@ mod tests { found.files.sort(); assert_eq!( found, - WordFiles { files: vec![root.join("pkg/open.py"), root.join("pkg/use.py")], limit_hit: false } + WordFiles { + files: vec![root.join("pkg/local.py"), root.join("pkg/open.py"), root.join("pkg/use.py")], + limit_hit: false + } ); } diff --git a/crates/ide-search/tests/search.rs b/crates/ide-search/tests/search.rs index b63c9722..f4207d05 100644 --- a/crates/ide-search/tests/search.rs +++ b/crates/ide-search/tests/search.rs @@ -187,24 +187,59 @@ fn file_masks() { assert_eq!(with(&[], &["*.kt,*.ts"]), ["notes.md", "src/Util.java"]); } +/// The files of `scope` under `root` (relative), sorted. +fn scope_rels(root: &Path, scope: &SearchScope) -> Vec { + let none = ide_search::FileMask::new(&[], &[]).unwrap(); + let files = ide_search::scope_files(root, scope, &none, None, &CancellationToken::new()).unwrap(); + let mut rels: Vec<_> = files.into_iter().map(|f| f.rel_path).collect(); + rels.sort(); + rels +} + #[test] -fn honours_ignore_files_hidden_and_git() { - let p = project(&[ - (".gitignore", b"target/\n*.log\n"), - (".ignore", b"generated.rs\n"), - ("src/a.rs", b"needle\n"), - ("src/generated.rs", b"needle\n"), - ("target/debug/out.rs", b"needle\n"), - ("build.log", b"needle\n"), - (".hidden/secret.rs", b"needle\n"), - (".env", b"needle\n"), - (".git/config", b"needle\n"), - ("sub/.gitignore", b"local.txt\n"), - ("sub/local.txt", b"needle\n"), - ("sub/kept.txt", b"needle\n"), - ]); +fn searches_ignored_and_hidden_files_but_not_ignored_folders() { + // #1317: like IDEA, only ignored folders are left out; ignored files and dotfiles are searched. + // (path, content, walked): every file mentions `needle`. + let files: &[(&str, &[u8], bool)] = &[ + (".gitignore", b"# needle\ntarget/\n*.log\n.env\nsrc/local/\n", true), + (".ignore", b"# needle\ngenerated.rs\nout/\n", true), + ("src/a.rs", b"needle\n", true), + // Ignored files outside ignored folders. + ("src/generated.rs", b"needle\n", true), + ("build.log", b"needle\n", true), + (".env", b"needle\n", true), + ("sub/.gitignore", b"# needle\nlocal.txt\ncache/\n", true), + ("sub/local.txt", b"needle\n", true), + ("sub/kept.txt", b"needle\n", true), + // Hidden files and folders. + (".eslintrc", b"needle\n", true), + (".github/workflows/ci.yml", b"needle\n", true), + (".hidden/secret.rs", b"needle\n", true), + // Ignored folders, whatever they hold. + ("target/debug/out.rs", b"needle\n", false), + ("target/.env", b"needle\n", false), + ("src/local/notes.txt", b"needle\n", false), + ("out/gen.rs", b"needle\n", false), + ("sub/cache/blob.txt", b"needle\n", false), + // Never: Git's own folder. + (".git/config", b"needle\n", false), + ]; + let p = project(&files.iter().map(|(path, text, _)| (*path, *text)).collect::>()); + let mut expected: Vec<&str> = files.iter().filter(|(_, _, walked)| *walked).map(|(path, _, _)| *path).collect(); + expected.sort(); + let (views, _) = run(p.path(), &query("needle")); - assert_eq!(paths(&views), ["src/a.rs", "sub/kept.txt"]); + assert_eq!(paths(&views), expected, "Find in Files"); + + assert_eq!(scope_rels(p.path(), &SearchScope::Project), expected, "scope files"); + + // A directory scope and an explicit ignored file are searched the same way. + let dir = SearchScope::Directory { path: ide_core::IpcPath(p.path().join("sub")) }; + let (views, _) = run(p.path(), &SearchQuery { scope: dir, ..query("needle") }); + assert_eq!(paths(&views), ["sub/.gitignore", "sub/kept.txt", "sub/local.txt"]); + let one = SearchScope::Files { paths: vec![ide_core::IpcPath(p.path().join("build.log"))] }; + let (views, _) = run(p.path(), &SearchQuery { scope: one, ..query("needle") }); + assert_eq!(paths(&views), ["build.log"]); } #[test] @@ -214,25 +249,15 @@ fn nested_repository_ignores_outer_repository_rules() { (".git/HEAD", b"ref: refs/heads/main\n"), ("projects/.gitignore", b"*\n!.gitignore\n"), ("projects/nested/.git/HEAD", b"ref: refs/heads/main\n"), - ("projects/nested/.gitignore", b"*.log\n"), + ("projects/nested/.gitignore", b"logs/\n"), ("projects/nested/search-test.txt", b"somesss\n"), - ("projects/nested/debug.log", b"somesss\n"), + ("projects/nested/src/main.txt", b"somesss\n"), + ("projects/nested/logs/debug.txt", b"somesss\n"), ]); let nested = p.path().join("projects/nested"); let (views, _) = run(&nested, &query("somesss")); - assert_eq!(paths(&views), ["search-test.txt"]); - - let files = ide_search::scope_files( - &nested, - &SearchScope::Project, - &ide_search::FileMask::new(&[], &[]).unwrap(), - None, - &CancellationToken::new(), - ) - .unwrap(); - let mut rels: Vec<_> = files.iter().map(|f| f.rel_path.as_str()).collect(); - rels.sort(); - assert_eq!(rels, ["search-test.txt"]); + assert_eq!(paths(&views), ["search-test.txt", "src/main.txt"]); + assert_eq!(scope_rels(&nested, &SearchScope::Project), [".gitignore", "search-test.txt", "src/main.txt"]); } #[test] @@ -678,13 +703,18 @@ fn a_nested_repository_is_not_hidden_by_the_outer_repositorys_gitignore() { let p = project(&[ ("projects/.gitignore", b"*\n!.gitignore\n"), ("projects/nested/search-test.txt", b"somesss\n"), - ("projects/nested/.gitignore", b"*.log\n"), - ("projects/nested/debug.log", b"somesss\n"), + ("projects/nested/src/main.txt", b"somesss\n"), + ("projects/nested/.gitignore", b"logs/\n"), + ("projects/nested/logs/debug.txt", b"somesss\n"), ]); std::fs::create_dir(p.path().join(".git")).unwrap(); std::fs::create_dir(p.path().join("projects/nested/.git")).unwrap(); let (views, _) = run(&p.path().join("projects/nested"), &query("somesss")); - assert_eq!(paths(&views), ["search-test.txt"], "the nested repository's own .gitignore still applies"); + assert_eq!( + paths(&views), + ["search-test.txt", "src/main.txt"], + "the nested repository's own .gitignore still applies" + ); } #[test]