fix(search): search hidden and git-ignored files outside ignored folders (#1318)

Find in Files skipped dotfiles (.eslintrc, .github/workflows/*.yml) and git-ignored files (.env, *.log) because the ide-search walkers dropped every hidden or ignored entry, files as well as folders. A shared walker now lets the ignore rules pick only the folders to enter and lists each walked folder for its files, so ignored folders (target/, logs/) are still skipped while ignored files elsewhere and hidden files are searched, as Go to File does since #1313. Find in Files, Inspect Code / Locate Duplicates (scope_files) and LSP reference candidates (files_with_word) all use it; .git, symlinks, archives, files.exclude and excluded roots still apply.

Fixes #1317

Co-authored-by: jaw52 <53564781+jaw52@users.noreply.github.com>

crates/ide-host/src/project.rs

@@ -844,7 +844,8 @@ impl Project {
 
     /// Project files of `q.languages` under `q.root` containing `q.word` as a whole
     /// word (unsaved buffers included; excluded roots, ignored names and
-    /// `.gitignore`d files skipped): the files a references search opens on its server.
+    /// `.gitignore`d folders skipped, while `.gitignore`d files elsewhere count, as
+    /// in IDEA's project scope): the files a references search opens on its server.
     pub fn reference_candidates(
         &self,
         q: &ide_lsp::CandidateQuery,

crates/ide-search/src/engine.rs

@@ -142,42 +142,16 @@ pub(crate) fn run(req: Request<'_>) -> IdeResult<SearchSummary> {
         }
     }
 
-    if let Some((first, rest)) = on_disk.split_first()
-        && !ctx.ctl.should_stop()
-    {
-        let mut walk = WalkBuilder::new(first);
-        for path in rest {
-            walk.add(path);
-        }
-        walk.hidden(true)
-            .ignore(true)
-            .git_ignore(true)
-            .git_global(true)
-            .git_exclude(true)
-            .parents(true)
-            .require_git(ide_core::ignored::inside_git_repo(first))
-            .follow_links(false)
-            .threads(std::thread::available_parallelism().map_or(4, |n| n.get()).min(12))
-            .filter_entry(|e| e.file_name() != ".git");
-        walk.build_parallel().run(|| {
+    if !ctx.ctl.should_stop() {
+        walk_files(&on_disk, || {
             let mut searcher = searcher_builder.build();
             let matcher = matcher.clone();
             let ctx = &ctx;
-            Box::new(move |entry| {
+            Box::new(move |path| {
                 if ctx.ctl.should_stop() {
                     return WalkState::Quit;
                 }
-                let entry = match entry {
-                    Ok(entry) => entry,
-                    Err(err) => {
-                        tracing::debug!("search walk: {err}");
-                        return WalkState::Continue;
-                    }
-                };
-                // Archives (jar, zip) are browsed, not searched (IDEA skips them too).
-                if entry.file_type().is_some_and(|t| t.is_file()) && !ide_vfs::is_archive_file(entry.path()) {
-                    ctx.search_file(entry.path(), &matcher, &mut searcher);
-                }
+                ctx.search_file(path, &matcher, &mut searcher);
                 if ctx.ctl.stopped() { WalkState::Quit } else { WalkState::Continue }
             })
         });
@@ -193,6 +167,67 @@ pub(crate) fn run(req: Request<'_>) -> IdeResult<SearchSummary> {
     })
 }
 
+/// A per-thread file visitor of [`walk_files`].
+pub(crate) type VisitFile<'s> = Box<dyn FnMut(&Path) -> WalkState + Send + 's>;
+
+/// Visits the files below `roots` (and the roots that are files) in parallel,
+/// the way IDEA's project scope sees them: directories ignored by
+/// `.gitignore` / `.ignore` / `info/exclude` are skipped with everything in
+/// them, while ignored files elsewhere (`.env`) and hidden files and folders
+/// (`.github`, `.eslintrc`) are visited. `.git`, symlinks and archives (jar,
+/// zip: browsed, not searched) are never visited.
+pub(crate) fn walk_files<'s>(roots: &[PathBuf], mut visitor: impl FnMut() -> VisitFile<'s> + Send) {
+    let Some((first, rest)) = roots.split_first() else { return };
+    let mut walk = WalkBuilder::new(first);
+    for path in rest {
+        walk.add(path);
+    }
+    walk.hidden(false)
+        .ignore(true)
+        .git_ignore(true)
+        .git_global(true)
+        .git_exclude(true)
+        .parents(true)
+        .require_git(ide_core::ignored::inside_git_repo(first))
+        .follow_links(false)
+        .threads(std::thread::available_parallelism().map_or(4, |n| n.get()).min(12))
+        .filter_entry(|e| e.file_name() != ".git");
+    walk.build_parallel().run(|| {
+        let mut visit = visitor();
+        Box::new(move |entry| {
+            let entry = match entry {
+                Ok(entry) => entry,
+                Err(err) => {
+                    tracing::debug!("search walk: {err}");
+                    return WalkState::Continue;
+                }
+            };
+            let Some(file_type) = entry.file_type() else { return WalkState::Continue };
+            if file_type.is_dir() {
+                // The walker drops ignored files unseen, so the files of every walked
+                // directory come from listing it; the walker only picks the directories.
+                let Ok(children) = std::fs::read_dir(entry.path()) else { return WalkState::Continue };
+                for child in children.flatten() {
+                    let path = child.path();
+                    if child.file_name() != ".git"
+                        && child.file_type().is_ok_and(|t| t.is_file())
+                        && !ide_vfs::is_archive_file(&path)
+                        && matches!(visit(&path), WalkState::Quit)
+                    {
+                        return WalkState::Quit;
+                    }
+                }
+                WalkState::Continue
+            } else if file_type.is_file() && entry.depth() == 0 && !ide_vfs::is_archive_file(entry.path()) {
+                // A root that is a file (an explicit file list); files below roots come from listings.
+                visit(entry.path())
+            } else {
+                WalkState::Continue
+            }
+        })
+    });
+}
+
 /// Walk roots for a scope; relative paths are resolved against `root`.
 pub(crate) fn roots(root: &Path, scope: &SearchScope) -> IdeResult<Vec<PathBuf>> {
     let dir = |path: &Path| -> IdeResult<PathBuf> {

crates/ide-search/src/files.rs

@@ -1,10 +1,10 @@
 //! The files of a search scope, for analyses that read whole files
 //! (Inspect Code, Locate Duplicates, Structural Search).
 
-use crate::engine::{rel_path, roots};
+use crate::engine::{rel_path, roots, walk_files};
 use crate::{FileMask, PathFilter, SearchScope};
 use ide_core::{CancellationToken, IdeResult};
-use ignore::{WalkBuilder, WalkState};
+use ignore::WalkState;
 use std::path::PathBuf;
 use std::sync::Mutex;
 
@@ -16,7 +16,7 @@ pub struct ScopeFile {
 }
 
 /// Lists the files of `scope` under `root` the way Find in Files walks them
-/// (`.gitignore` honoured, hidden files and `.git` skipped), filtered by
+/// (ignored folders and `.git` skipped; ignored and hidden files kept), filtered by
 /// `mask` and `filter`, sorted by relative path. Explicitly listed files are
 /// kept even when they do not exist on disk (unsaved buffers). Returns what was
 /// found so far when cancelled.
@@ -43,42 +43,23 @@ pub fn scope_files(
             on_disk.push(path);
         }
     }
-    if let Some((first, rest)) = on_disk.split_first() {
-        let mut walk = WalkBuilder::new(first);
-        for path in rest {
-            walk.add(path);
-        }
-        walk.hidden(true)
-            .ignore(true)
-            .git_ignore(true)
-            .git_global(true)
-            .git_exclude(true)
-            .parents(true)
-            .require_git(ide_core::ignored::inside_git_repo(first))
-            .follow_links(false)
-            .threads(std::thread::available_parallelism().map_or(4, |n| n.get()).min(12))
-            .filter_entry(|e| e.file_name() != ".git");
-        walk.build_parallel().run(|| {
-            let found = &found;
-            let keep = &keep;
-            Box::new(move |entry| {
-                if cancel.is_cancelled() {
-                    return WalkState::Quit;
-                }
-                let Ok(entry) = entry else { return WalkState::Continue };
-                if entry.file_type().is_some_and(|t| t.is_file()) && !ide_vfs::is_archive_file(entry.path()) {
-                    let rel = rel_path(root, entry.path());
-                    if keep(&rel) {
-                        found
-                            .lock()
-                            .unwrap_or_else(|e| e.into_inner())
-                            .push(ScopeFile { path: entry.into_path(), rel_path: rel });
-                    }
-                }
-                WalkState::Continue
-            })
-        });
-    }
+    walk_files(&on_disk, || {
+        let found = &found;
+        let keep = &keep;
+        Box::new(move |path| {
+            if cancel.is_cancelled() {
+                return WalkState::Quit;
+            }
+            let rel = rel_path(root, path);
+            if keep(&rel) {
+                found
+                    .lock()
+                    .unwrap_or_else(|e| e.into_inner())
+                    .push(ScopeFile { path: path.to_path_buf(), rel_path: rel });
+            }
+            WalkState::Continue
+        })
+    });
     let mut files = found.into_inner().unwrap_or_else(|e| e.into_inner());
     files.sort_by(|a, b| a.rel_path.cmp(&b.rel_path));
     files.dedup_by(|a, b| a.path == b.path);
@@ -107,12 +88,12 @@ mod tests {
     }
 
     #[test]
-    fn project_scope_honours_gitignore_and_masks() {
+    fn project_scope_skips_ignored_folders_and_honours_masks() {
         let dir = tree();
         let none = FileMask::new(&[], &[]).unwrap();
         let cancel = CancellationToken::new();
         let all = scope_files(dir.path(), &SearchScope::Project, &none, None, &cancel).unwrap();
-        assert_eq!(rels(&all), vec!["src/a.rs", "src/c.txt", "src/sub/b.rs"]);
+        assert_eq!(rels(&all), vec![".gitignore", "src/a.rs", "src/c.txt", "src/sub/b.rs"]);
 
         let rs = FileMask::new(&["*.rs".into()], &[]).unwrap();
         let only_rs = scope_files(dir.path(), &SearchScope::Project, &rs, None, &cancel).unwrap();
@@ -120,7 +101,7 @@ mod tests {
 
         let not_sub: PathFilter<'_> = &|rel: &str| !rel.starts_with("src/sub/");
         let filtered = scope_files(dir.path(), &SearchScope::Project, &none, Some(not_sub), &cancel).unwrap();
-        assert_eq!(rels(&filtered), vec!["src/a.rs", "src/c.txt"]);
+        assert_eq!(rels(&filtered), vec![".gitignore", "src/a.rs", "src/c.txt"]);
     }
 
     #[test]

crates/ide-search/src/lib.rs

@@ -1,7 +1,8 @@
 //! Find / Replace in Files (⌘⇧F / ⌘⇧R).
 //!
-//! Built on ripgrep's engine crates: a parallel `ignore` walk (honouring
-//! `.gitignore` / `.ignore`, skipping hidden files and `.git`) feeding
+//! Built on ripgrep's engine crates: a parallel `ignore` walk (skipping the
+//! folders `.gitignore` / `.ignore` exclude and `.git`, but like IDEA keeping
+//! ignored files elsewhere and hidden files) feeding
 //! `grep-searcher` with a `grep-regex` matcher. Binary files are skipped,
 //! unsaved editor buffers (the overlay) take precedence over disk content,
 //! and all columns are UTF-16.

crates/ide-search/src/words.rs

@@ -25,8 +25,8 @@ pub struct WordFiles {
 }
 
 /// Files under `query.dir` containing `query.word` as a whole word, in walk
-/// order (unsaved buffers through `overlay`, `.gitignore`d and hidden files
-/// skipped like Find in Files). Stops at `query.limit` files or when cancelled.
+/// order (unsaved buffers through `overlay`; ignored folders skipped, ignored
+/// and hidden files searched, like Find in Files). Stops at `query.limit` files or when cancelled.
 pub fn files_with_word(
     root: &std::path::Path,
     query: &WordQuery,
@@ -74,7 +74,9 @@ mod tests {
         let root = dir.path();
         fs::create_dir_all(root.join("pkg/ignored")).unwrap();
         fs::create_dir_all(root.join("build")).unwrap();
-        fs::write(root.join(".gitignore"), "ignored/\n").unwrap();
+        fs::write(root.join(".gitignore"), "ignored/\nlocal.py\n").unwrap();
+        // An ignored file outside ignored folders is still a candidate, as in IDEA.
+        fs::write(root.join("pkg/local.py"), "get_value\n").unwrap();
         fs::write(root.join("pkg/use.py"), "x.get_value(1)\n").unwrap();
         fs::write(root.join("pkg/longer.py"), "get_values()\n").unwrap();
         fs::write(root.join("pkg/case.py"), "Get_Value()\n").unwrap();
@@ -90,7 +92,10 @@ mod tests {
         found.files.sort();
         assert_eq!(
             found,
-            WordFiles { files: vec![root.join("pkg/open.py"), root.join("pkg/use.py")], limit_hit: false }
+            WordFiles {
+                files: vec![root.join("pkg/local.py"), root.join("pkg/open.py"), root.join("pkg/use.py")],
+                limit_hit: false
+            }
         );
     }
 

crates/ide-search/tests/search.rs

@@ -187,24 +187,59 @@ fn file_masks() {
     assert_eq!(with(&[], &["*.kt,*.ts"]), ["notes.md", "src/Util.java"]);
 }
 
+/// The files of `scope` under `root` (relative), sorted.
+fn scope_rels(root: &Path, scope: &SearchScope) -> Vec<String> {
+    let none = ide_search::FileMask::new(&[], &[]).unwrap();
+    let files = ide_search::scope_files(root, scope, &none, None, &CancellationToken::new()).unwrap();
+    let mut rels: Vec<_> = files.into_iter().map(|f| f.rel_path).collect();
+    rels.sort();
+    rels
+}
+
 #[test]
-fn honours_ignore_files_hidden_and_git() {
-    let p = project(&[
-        (".gitignore", b"target/\n*.log\n"),
-        (".ignore", b"generated.rs\n"),
-        ("src/a.rs", b"needle\n"),
-        ("src/generated.rs", b"needle\n"),
-        ("target/debug/out.rs", b"needle\n"),
-        ("build.log", b"needle\n"),
-        (".hidden/secret.rs", b"needle\n"),
-        (".env", b"needle\n"),
-        (".git/config", b"needle\n"),
-        ("sub/.gitignore", b"local.txt\n"),
-        ("sub/local.txt", b"needle\n"),
-        ("sub/kept.txt", b"needle\n"),
-    ]);
+fn searches_ignored_and_hidden_files_but_not_ignored_folders() {
+    // #1317: like IDEA, only ignored folders are left out; ignored files and dotfiles are searched.
+    // (path, content, walked): every file mentions `needle`.
+    let files: &[(&str, &[u8], bool)] = &[
+        (".gitignore", b"# needle\ntarget/\n*.log\n.env\nsrc/local/\n", true),
+        (".ignore", b"# needle\ngenerated.rs\nout/\n", true),
+        ("src/a.rs", b"needle\n", true),
+        // Ignored files outside ignored folders.
+        ("src/generated.rs", b"needle\n", true),
+        ("build.log", b"needle\n", true),
+        (".env", b"needle\n", true),
+        ("sub/.gitignore", b"# needle\nlocal.txt\ncache/\n", true),
+        ("sub/local.txt", b"needle\n", true),
+        ("sub/kept.txt", b"needle\n", true),
+        // Hidden files and folders.
+        (".eslintrc", b"needle\n", true),
+        (".github/workflows/ci.yml", b"needle\n", true),
+        (".hidden/secret.rs", b"needle\n", true),
+        // Ignored folders, whatever they hold.
+        ("target/debug/out.rs", b"needle\n", false),
+        ("target/.env", b"needle\n", false),
+        ("src/local/notes.txt", b"needle\n", false),
+        ("out/gen.rs", b"needle\n", false),
+        ("sub/cache/blob.txt", b"needle\n", false),
+        // Never: Git's own folder.
+        (".git/config", b"needle\n", false),
+    ];
+    let p = project(&files.iter().map(|(path, text, _)| (*path, *text)).collect::<Vec<_>>());
+    let mut expected: Vec<&str> = files.iter().filter(|(_, _, walked)| *walked).map(|(path, _, _)| *path).collect();
+    expected.sort();
+
     let (views, _) = run(p.path(), &query("needle"));
-    assert_eq!(paths(&views), ["src/a.rs", "sub/kept.txt"]);
+    assert_eq!(paths(&views), expected, "Find in Files");
+
+    assert_eq!(scope_rels(p.path(), &SearchScope::Project), expected, "scope files");
+
+    // A directory scope and an explicit ignored file are searched the same way.
+    let dir = SearchScope::Directory { path: ide_core::IpcPath(p.path().join("sub")) };
+    let (views, _) = run(p.path(), &SearchQuery { scope: dir, ..query("needle") });
+    assert_eq!(paths(&views), ["sub/.gitignore", "sub/kept.txt", "sub/local.txt"]);
+    let one = SearchScope::Files { paths: vec![ide_core::IpcPath(p.path().join("build.log"))] };
+    let (views, _) = run(p.path(), &SearchQuery { scope: one, ..query("needle") });
+    assert_eq!(paths(&views), ["build.log"]);
 }
 
 #[test]
@@ -214,25 +249,15 @@ fn nested_repository_ignores_outer_repository_rules() {
         (".git/HEAD", b"ref: refs/heads/main\n"),
         ("projects/.gitignore", b"*\n!.gitignore\n"),
         ("projects/nested/.git/HEAD", b"ref: refs/heads/main\n"),
-        ("projects/nested/.gitignore", b"*.log\n"),
+        ("projects/nested/.gitignore", b"logs/\n"),
         ("projects/nested/search-test.txt", b"somesss\n"),
-        ("projects/nested/debug.log", b"somesss\n"),
+        ("projects/nested/src/main.txt", b"somesss\n"),
+        ("projects/nested/logs/debug.txt", b"somesss\n"),
     ]);
     let nested = p.path().join("projects/nested");
     let (views, _) = run(&nested, &query("somesss"));
-    assert_eq!(paths(&views), ["search-test.txt"]);
-
-    let files = ide_search::scope_files(
-        &nested,
-        &SearchScope::Project,
-        &ide_search::FileMask::new(&[], &[]).unwrap(),
-        None,
-        &CancellationToken::new(),
-    )
-    .unwrap();
-    let mut rels: Vec<_> = files.iter().map(|f| f.rel_path.as_str()).collect();
-    rels.sort();
-    assert_eq!(rels, ["search-test.txt"]);
+    assert_eq!(paths(&views), ["search-test.txt", "src/main.txt"]);
+    assert_eq!(scope_rels(&nested, &SearchScope::Project), [".gitignore", "search-test.txt", "src/main.txt"]);
 }
 
 #[test]
@@ -678,13 +703,18 @@ fn a_nested_repository_is_not_hidden_by_the_outer_repositorys_gitignore() {
     let p = project(&[
         ("projects/.gitignore", b"*\n!.gitignore\n"),
         ("projects/nested/search-test.txt", b"somesss\n"),
-        ("projects/nested/.gitignore", b"*.log\n"),
-        ("projects/nested/debug.log", b"somesss\n"),
+        ("projects/nested/src/main.txt", b"somesss\n"),
+        ("projects/nested/.gitignore", b"logs/\n"),
+        ("projects/nested/logs/debug.txt", b"somesss\n"),
     ]);
     std::fs::create_dir(p.path().join(".git")).unwrap();
     std::fs::create_dir(p.path().join("projects/nested/.git")).unwrap();
     let (views, _) = run(&p.path().join("projects/nested"), &query("somesss"));
-    assert_eq!(paths(&views), ["search-test.txt"], "the nested repository's own .gitignore still applies");
+    assert_eq!(
+        paths(&views),
+        ["search-test.txt", "src/main.txt"],
+        "the nested repository's own .gitignore still applies"
+    );
 }
 
 #[test]