From 96e80f2028772ff32ff20ee340724388ce4df594 Mon Sep 17 00:00:00 2001 From: Muu Date: Tue, 17 Nov 2026 01:27:54 +0800 Subject: [PATCH] perf(index): answer Python inheritance from the project index (#3759) The inheritance gutter graph parsed every Python file of the project on each request. The index already parses each file once for symbols, so class base names are now extracted in that pass and stored with the symbol (interned in SymTable); ProjectIndex::inheritance() answers every class from the flat tables and nav_inheritance reads it, so no whole-project parse is left on the request path. Open (possibly unsaved) buffers are still parsed live and replace the index's view of their file. SCHEMA 3 -> 4 rebuilds existing caches once. Co-authored-by: muutot <92625353+muutot@users.noreply.github.com> --- crates/ide-host/src/ops/nav.rs | 62 +++++++++++-------- crates/ide-host/tests/nav_inheritance.rs | 6 +- crates/ide-host/tests/plugin_indexing.rs | 1 + crates/ide-host/tests/plugin_refactoring.rs | 1 + crates/ide-index/src/data.rs | 45 +++++++++++++- crates/ide-index/src/lib.rs | 35 +++++++++++ crates/ide-index/src/store.rs | 16 ++++- crates/ide-index/src/tests.rs | 23 +++++++ crates/ide-index/src/worker.rs | 8 ++- crates/ide-language/src/lib.rs | 4 ++ crates/ide-language/src/syntax/inheritance.rs | 21 ++++++- crates/ide-language/src/syntax/outline.rs | 10 +++ crates/ide-language/src/syntax/web.rs | 2 +- 13 files changed, 198 insertions(+), 36 deletions(-) diff --git a/crates/ide-host/src/ops/nav.rs b/crates/ide-host/src/ops/nav.rs index cf49c2a0c..e023f9ba4 100644 --- a/crates/ide-host/src/ops/nav.rs +++ b/crates/ide-host/src/ops/nav.rs @@ -208,41 +208,52 @@ pub async fn nav_index_definitions( } /// Every Python class in the project (attached dependency roots included) and its base -/// names. The index covers content roots, so this is the whole-project answer the UI uses -/// for inheritance gutter markers when no type-hierarchy provider exists. +/// names, answered from the project index: its symbols already carry the bases, and it +/// covers content roots, so this is the whole-project answer the UI uses for inheritance +/// gutter markers when no type-hierarchy provider exists. No file is parsed here. pub async fn nav_inheritance(host: &Host, window: &str) -> IdeResult> { let project = host.project(window)?; let syntax = host.syntax(); blocking(move || { let python = LanguageId::new("python"); - let files = project.index.files_with_extensions(&["py".into(), "pyi".into()], 20_000); let mut cache = CLASS_BASES.lock(); - let mut out = Vec::new(); - for path in files { - // The open (possibly unsaved) buffer is keyed by its text, a file on disk by its - // size and modification time, so only changed files are parsed again. + // Disk-backed classes come from the index: it already parsed every file once for + // symbols, so no second parse of the project is needed here. + let mut out: Vec = project + .index + .inheritance() + .into_iter() + // The index holds classes of every language; the graph is Python's. + .filter(|c| matches!(c.path.extension().and_then(std::ffi::OsStr::to_str), Some("py" | "pyi"))) + .map(|c| ClassInheritance { + path: IpcPath::from(c.path.clone()), + line: c.range.start.line, + col: c.range.start.col, + name: c.name.clone(), + bases: c.bases.clone(), + }) + .collect(); + // Open buffers may be unsaved, so their classes are parsed live (and cached per + // text/stamp) and replace the index's view of that file. This keeps #3359's + // "open buffer by its text" behaviour while the disk path is index-backed. + let open_files: Vec = project + .index + .files_with_extensions(&["py".into(), "pyi".into()], 20_000) + .into_iter() + .filter(|path| project.documents.text_of(path).is_some()) + .collect(); + for path in open_files { let open = project.documents.text_of(&path); let stamp = match &open { - Some(text) => Stamp::Text(text_hash(text)), - None => match std::fs::metadata(&path) { - Ok(m) => Stamp::Disk(m.len(), m.modified().ok()), - Err(_) => continue, - }, + Some(text) => Stamp(text_hash(text)), + None => continue, }; + let text = open.unwrap(); let classes = match cache.get(&path) { Some((cached, classes)) if *cached == stamp => classes, - _ => { - let text = match open { - Some(text) => text, - None => match project.text_of(&path) { - Ok(text) => text, - Err(_) => continue, - }, - }; - let classes = syntax.class_bases(&python, &text); - &cache.entry(path.clone()).insert_entry((stamp, classes)).into_mut().1 - } + _ => &cache.entry(path.clone()).insert_entry((stamp, syntax.class_bases(&python, &text))).into_mut().1, }; + out.retain(|c| c.path.0 != *path); out.extend(classes.iter().map(|class| ClassInheritance { path: IpcPath::from(path.clone()), line: class.line, @@ -257,10 +268,7 @@ pub async fn nav_inheritance(host: &Host, window: &str) -> IdeResult), -} +struct Stamp(u64); fn text_hash(text: &str) -> u64 { use std::hash::{Hash, Hasher}; diff --git a/crates/ide-host/tests/nav_inheritance.rs b/crates/ide-host/tests/nav_inheritance.rs index d9adc6f14..4503fefb4 100644 --- a/crates/ide-host/tests/nav_inheritance.rs +++ b/crates/ide-host/tests/nav_inheritance.rs @@ -131,9 +131,13 @@ async fn returns_all_python_classes_and_bases() { // Every returned entry is a Python file. assert!(all.iter().all(|c| { matches!(c.path.0.extension().and_then(|e| e.to_str()), Some("py" | "pyi")) })); - // A file changed on disk is parsed again; the others come from the cache. + // A file changed on disk is picked up once the index re-reads it. The index is the + // single source of truth for on-disk classes now; edits go through its update path + // (the IDE's file watcher calls this in production). let extra = p.root.join("src/extra.pyi"); std::fs::write(&extra, "class Stub(Child): ...\nclass Added(Base): ...\n").unwrap(); + p.host.project(WINDOW).unwrap().index.update(std::slice::from_ref(&extra), &[]); + assert!(p.host.project(WINDOW).unwrap().index.wait_idle(Duration::from_secs(5))); let again = ops::nav::nav_inheritance(&p.host, WINDOW).await.unwrap(); assert_eq!(find(&again, "Stub").bases, vec!["Child".to_string()]); assert_eq!(find(&again, "Added").bases, vec!["Base".to_string()]); diff --git a/crates/ide-host/tests/plugin_indexing.rs b/crates/ide-host/tests/plugin_indexing.rs index db938890f..373ce3750 100644 --- a/crates/ide-host/tests/plugin_indexing.rs +++ b/crates/ide-host/tests/plugin_indexing.rs @@ -54,6 +54,7 @@ fn method() -> SymbolTag { kind: SymbolKind::Method, container: None, range: Range::new(Pos::new(0, 4), Pos::new(0, 8)), + bases: Vec::new(), } } const SOURCE: &str = "def source:item\nref mapping:item\n"; diff --git a/crates/ide-host/tests/plugin_refactoring.rs b/crates/ide-host/tests/plugin_refactoring.rs index 1556c1485..bee68badb 100644 --- a/crates/ide-host/tests/plugin_refactoring.rs +++ b/crates/ide-host/tests/plugin_refactoring.rs @@ -225,6 +225,7 @@ async fn indexed_usages_rename_preview_staleness_and_grouped_undo() { kind: ide_core::SymbolKind::Method, container: None, range: ide_core::Range::new(Pos::new(0, 4), Pos::new(0, 5)), + bases: Vec::new(), }; plugins .index_navigation( diff --git a/crates/ide-index/src/data.rs b/crates/ide-index/src/data.rs index 46d2872bc..0ecf5dfc8 100644 --- a/crates/ide-index/src/data.rs +++ b/crates/ide-index/src/data.rs @@ -106,6 +106,9 @@ pub(crate) struct StoredSymbol { pub kind: SymbolKind, pub container: Option, pub range: Range, + /// Base names of a class definition; empty for every other symbol. + #[serde(default)] + pub bases: Vec, } /// A symbol resolved out of [`SymTable`]. @@ -114,6 +117,8 @@ pub(crate) struct SymRef<'a> { pub kind: SymbolKind, pub container: Option<&'a str>, pub range: Range, + /// Base names joined with `'\0'`; empty when the symbol has none. + pub bases: &'a str, } pub(crate) struct FileEntry { @@ -402,6 +407,8 @@ pub(crate) struct SymTable { container: Vec, kind: Vec, range: Vec, + /// Base names joined with `'\0'`, one interned span per symbol (`NONE` for none). + bases: Vec, text: String, dead_rows: usize, } @@ -423,6 +430,18 @@ impl SymTable { (span.start != u32::MAX).then(|| span.get(&self.text)) } + /// The symbol's base names joined with `'\0'`; empty when it has none. + #[inline] + pub(crate) fn bases(&self, i: u32) -> &str { + let span = self.bases[i as usize]; + if span.start == u32::MAX { "" } else { span.get(&self.text) } + } + + /// The base names of row `i`, split out of the interned string. + pub(crate) fn base_list(&self, i: u32) -> Vec<&str> { + self.bases(i).split('\0').filter(|b| !b.is_empty()).collect() + } + /// The mask + name rows of a file's contiguous symbol range. #[inline] pub(crate) fn heads(&self, start: u32, len: u32) -> &[SymHead] { @@ -459,10 +478,12 @@ impl SymTable { Some(c) => self.intern(c), None => Span::NONE, }; + let bases = if s.bases.is_empty() { Span::NONE } else { self.intern(&s.bases.join("\0")) }; self.head.push(SymHead { mask, name }); self.container.push(container); self.kind.push(s.kind); self.range.push(s.range); + self.bases.push(bases); } (start, agg_mask) } @@ -476,6 +497,7 @@ impl SymTable { && self.container(i) == s.container.as_deref() && self.kind(i) == s.kind && self.range(i) == s.range + && self.base_list(i) == s.bases.iter().map(String::as_str).collect::>() }) } @@ -492,6 +514,7 @@ impl SymTable { let mut container = Vec::with_capacity(head.capacity()); let mut kind = Vec::with_capacity(head.capacity()); let mut range = Vec::with_capacity(head.capacity()); + let mut bases = Vec::with_capacity(head.capacity()); let mut live = 0u32; for (slot, row) in rows.iter_mut().enumerate() { if !row.live { @@ -518,6 +541,14 @@ impl SymTable { } kind.push(self.kind[i as usize]); range.push(self.range[i as usize]); + let bases_span = self.bases[i as usize]; + if bases_span.start == u32::MAX { + bases.push(Span::NONE); + } else { + let b = bases_span.get(&self.text); + bases.push(Span::of(&text, b)); + text.push_str(b); + } live += 1; } } @@ -525,6 +556,7 @@ impl SymTable { self.container = container; self.kind = kind; self.range = range; + self.bases = bases; self.text = text; self.dead_rows = 0; } @@ -534,6 +566,7 @@ impl SymTable { self.container.shrink_to_fit(); self.kind.shrink_to_fit(); self.range.shrink_to_fit(); + self.bases.shrink_to_fit(); self.text.shrink_to_fit(); } } @@ -577,6 +610,7 @@ impl IndexData { self.syms.container.reserve(symbols); self.syms.kind.reserve(symbols); self.syms.range.reserve(symbols); + self.syms.bases.reserve(symbols); } /// Hydrates a validated warm-start snapshot in one pass, without per-file @@ -640,6 +674,7 @@ impl IndexData { kind: s.kind, container: s.container.map(str::to_owned), range: s.range, + bases: s.bases.split('\0').filter(|b| !b.is_empty()).map(str::to_owned).collect(), }) .collect(), ignored: e.ignored, @@ -666,6 +701,7 @@ impl IndexData { kind: syms.kind(i), container: syms.container(i), range: syms.range(i), + bases: syms.bases(i), }) } @@ -823,6 +859,7 @@ mod tests { kind: SymbolKind::Function, container: None, range: Range::default(), + bases: Vec::new(), }) .collect(), ..Default::default() @@ -957,7 +994,13 @@ mod tests { } fn symbol(n: &str) -> StoredSymbol { - StoredSymbol { name: n.to_owned(), kind: SymbolKind::Function, container: None, range: Range::default() } + StoredSymbol { + name: n.to_owned(), + kind: SymbolKind::Function, + container: None, + range: Range::default(), + bases: Vec::new(), + } } #[test] diff --git a/crates/ide-index/src/lib.rs b/crates/ide-index/src/lib.rs index c37ebad12..c60531436 100644 --- a/crates/ide-index/src/lib.rs +++ b/crates/ide-index/src/lib.rs @@ -50,6 +50,17 @@ pub struct SymbolMatch { pub indices: MatchIndices, } +/// A class declaration of the index and the base names written in its source. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ClassDecl { + pub path: PathBuf, + pub name: String, + /// Selection range: the class name's identifier. + pub range: Range, + /// Bare base names, as written (`class X(pkg.mod.Base)` keeps `Base`). + pub bases: Vec, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Type)] #[serde(rename_all = "camelCase")] pub enum SymbolFilter { @@ -206,11 +217,34 @@ impl ProjectIndex { kind: s.kind, container: s.container.map(ToString::to_string), range: s.range, + bases: s.bases.split('\0').filter(|b| !b.is_empty()).map(str::to_owned).collect(), }) .collect(), ) } + /// Every class declaration of the indexed project (content roots included) with the + /// base names written in its source, in path order. The UI builds an inheritance + /// graph from it, so it needs no language server's type hierarchy. + pub fn inheritance(&self) -> Vec { + let data = self.shared.data.read(); + let mut out = Vec::new(); + for entry in data.files.iter().flatten() { + for s in data.symbols_of(entry) { + if s.kind != SymbolKind::Class { + continue; + } + out.push(ClassDecl { + path: entry.path.clone(), + name: s.name.to_owned(), + range: s.range, + bases: s.bases.split('\0').filter(|b| !b.is_empty()).map(str::to_owned).collect(), + }); + } + } + out + } + /// Bounded file inventory for background plugin indexers; honors the same /// project/exclude scope as search and never walks the filesystem. pub fn files_with_extensions(&self, extensions: &[String], limit: usize) -> Vec { @@ -517,6 +551,7 @@ pub mod bench_support { kind: if k % 3 == 0 { SymbolKind::Class } else { SymbolKind::Method }, container: (k % 3 != 0).then(|| format!("{w}{s}")), range: Range::default(), + bases: Vec::new(), }) .collect(); let record = FileRecord { symbols, ..Default::default() }; diff --git a/crates/ide-index/src/store.rs b/crates/ide-index/src/store.rs index c34c00ff0..3c6882148 100644 --- a/crates/ide-index/src/store.rs +++ b/crates/ide-index/src/store.rs @@ -11,8 +11,8 @@ use std::time::{SystemTime, UNIX_EPOCH}; const FILES: TableDefinition<&str, &[u8]> = TableDefinition::new("files"); const META: TableDefinition<&str, &[u8]> = TableDefinition::new("meta"); -/// Bump when [`FileRecord`] or its encoding changes. -const SCHEMA: &str = "openidea-index-3"; +/// Bump when [`FileRecord`] or its encoding changes. 4: a symbol's base names. +const SCHEMA: &str = "openidea-index-4"; /// The real Git directory for a checkout/worktree (`root/.git` may be a file /// containing `gitdir: ...`). @@ -113,6 +113,9 @@ struct FlatSymbol { name_len: u32, container_start: u32, container_len: u32, + /// Base names joined with `'\0'` (`bases_len` 0 when the symbol has none). + bases_start: u32, + bases_len: u32, kind: SymbolKind, range: Range, } @@ -324,11 +327,14 @@ impl Snapshot { let start = s.container_start as usize; &self.text[start..start + s.container_len as usize] }); + let bases_start = s.bases_start as usize; + let bases = &self.text[bases_start..bases_start + s.bases_len as usize]; StoredSymbol { name: name.to_owned(), kind: s.kind, container: container.map(str::to_owned), range: s.range, + bases: bases.split('\0').filter(|b| !b.is_empty()).map(str::to_owned).collect(), } }) .collect(); @@ -498,11 +504,15 @@ impl Store { } None => (u32::MAX, 0), }; + let bases_start = fit(snap.text.len())?; + snap.text.push_str(s.bases); snap.symbols.push(FlatSymbol { name_start, name_len: fit(s.name.len())?, container_start, container_len, + bases_start, + bases_len: fit(s.bases.len())?, kind: s.kind, range: s.range, }); @@ -717,6 +727,7 @@ mod tests { kind: SymbolKind::Class, container: Some("Order".into()), range: Range::default(), + bases: vec!["Base".into()], }], ..Default::default() }; @@ -740,6 +751,7 @@ mod tests { assert_eq!(symbols.len(), 1); assert_eq!(symbols[0].name, "OrderController"); assert_eq!(symbols[0].container.as_deref(), Some("Order")); + assert_eq!(symbols[0].bases, vec!["Base".to_string()], "bases survive the snapshot"); store.apply(&[], &[], false).unwrap(); assert!(store.load_snapshot().is_none(), "a file change invalidates the snapshot"); diff --git a/crates/ide-index/src/tests.rs b/crates/ide-index/src/tests.rs index e661faf35..abd658439 100644 --- a/crates/ide-index/src/tests.rs +++ b/crates/ide-index/src/tests.rs @@ -567,6 +567,29 @@ fn symbols_named_any_answers_every_name_inside_the_root_in_one_walk() { assert_eq!(found, expected.map(|(name, path)| (name.to_string(), path.to_string()))); } +#[test] +fn inheritance_reports_python_classes_and_their_bases() { + let f = Fixture::new(&[( + "app.py", + "class Base:\n pass\n\nclass Child(Base, other.Mixin):\n pass\n\nclass Plain:\n pass\n", + )]); + let idx = f.open(); + let mut found: Vec<(String, u32, Vec)> = + idx.inheritance().into_iter().map(|c| (c.name, c.range.start.line, c.bases)).collect(); + found.sort(); + assert_eq!( + found, + vec![ + ("Base".to_string(), 0, Vec::new()), + ("Child".to_string(), 3, vec!["Base".to_string(), "Mixin".to_string()]), + ("Plain".to_string(), 6, Vec::new()), + ] + ); + // Every class of the project is reported, even without a superclass: the UI needs + // them to resolve the base names of other files. + assert_eq!(idx.inheritance().len(), 3); +} + #[test] fn content_roots_and_excludes_change_without_reopening() { let fx = Fixture::new(&[("src/main.rs", "fn alpha() {}\n"), ("src/gen/out.rs", "fn generated() {}\n")]); diff --git a/crates/ide-index/src/worker.rs b/crates/ide-index/src/worker.rs index be91430af..fdd810df2 100644 --- a/crates/ide-index/src/worker.rs +++ b/crates/ide-index/src/worker.rs @@ -752,7 +752,13 @@ impl Shared { .syntax .tags(&lang, &text) .into_iter() - .map(|t| StoredSymbol { name: t.name, kind: t.kind, container: t.container, range: t.range }) + .map(|t| StoredSymbol { + name: t.name, + kind: t.kind, + container: t.container, + range: t.range, + bases: t.bases, + }) .collect(); } Extracted::New(rec) diff --git a/crates/ide-language/src/lib.rs b/crates/ide-language/src/lib.rs index 28be814e4..f14230b21 100644 --- a/crates/ide-language/src/lib.rs +++ b/crates/ide-language/src/lib.rs @@ -347,6 +347,10 @@ pub struct SymbolTag { /// Enclosing symbol, e.g. class name for a method. pub container: Option, pub range: Range, + /// Bare base names written in a class definition (`class X(A, B)`); empty for + /// every other symbol, and for languages whose classes carry none. + #[serde(default)] + pub bases: Vec, } /// One inline TS/JS `