perf(index): answer Python inheritance from the project index (#3759)

The inheritance gutter graph parsed every Python file of the project on each request. The index already parses each file once for symbols, so class base names are now extracted in that pass and stored with the symbol (interned in SymTable); ProjectIndex::inheritance() answers every class from the flat tables and nav_inheritance reads it, so no whole-project parse is left on the request path. Open (possibly unsaved) buffers are still parsed live and replace the index's view of their file. SCHEMA 3 -> 4 rebuilds existing caches once.

Co-authored-by: muutot <92625353+muutot@users.noreply.github.com>
This commit is contained in:
Muu
2026-11-17 01:27:54 +08:00
committed by GitHub
co-authored by muutot
parent e4ac29486c
commit 96e80f2028
13 changed files with 198 additions and 36 deletions
+35 -27
View File
@@ -208,41 +208,52 @@ pub async fn nav_index_definitions(
}
/// Every Python class in the project (attached dependency roots included) and its base
/// names. The index covers content roots, so this is the whole-project answer the UI uses
/// for inheritance gutter markers when no type-hierarchy provider exists.
/// names, answered from the project index: its symbols already carry the bases, and it
/// covers content roots, so this is the whole-project answer the UI uses for inheritance
/// gutter markers when no type-hierarchy provider exists. No file is parsed here.
pub async fn nav_inheritance(host: &Host, window: &str) -> IdeResult<Vec<ClassInheritance>> {
let project = host.project(window)?;
let syntax = host.syntax();
blocking(move || {
let python = LanguageId::new("python");
let files = project.index.files_with_extensions(&["py".into(), "pyi".into()], 20_000);
let mut cache = CLASS_BASES.lock();
let mut out = Vec::new();
for path in files {
// The open (possibly unsaved) buffer is keyed by its text, a file on disk by its
// size and modification time, so only changed files are parsed again.
// Disk-backed classes come from the index: it already parsed every file once for
// symbols, so no second parse of the project is needed here.
let mut out: Vec<ClassInheritance> = project
.index
.inheritance()
.into_iter()
// The index holds classes of every language; the graph is Python's.
.filter(|c| matches!(c.path.extension().and_then(std::ffi::OsStr::to_str), Some("py" | "pyi")))
.map(|c| ClassInheritance {
path: IpcPath::from(c.path.clone()),
line: c.range.start.line,
col: c.range.start.col,
name: c.name.clone(),
bases: c.bases.clone(),
})
.collect();
// Open buffers may be unsaved, so their classes are parsed live (and cached per
// text/stamp) and replace the index's view of that file. This keeps #3359's
// "open buffer by its text" behaviour while the disk path is index-backed.
let open_files: Vec<PathBuf> = project
.index
.files_with_extensions(&["py".into(), "pyi".into()], 20_000)
.into_iter()
.filter(|path| project.documents.text_of(path).is_some())
.collect();
for path in open_files {
let open = project.documents.text_of(&path);
let stamp = match &open {
Some(text) => Stamp::Text(text_hash(text)),
None => match std::fs::metadata(&path) {
Ok(m) => Stamp::Disk(m.len(), m.modified().ok()),
Err(_) => continue,
},
Some(text) => Stamp(text_hash(text)),
None => continue,
};
let text = open.unwrap();
let classes = match cache.get(&path) {
Some((cached, classes)) if *cached == stamp => classes,
_ => {
let text = match open {
Some(text) => text,
None => match project.text_of(&path) {
Ok(text) => text,
Err(_) => continue,
},
};
let classes = syntax.class_bases(&python, &text);
&cache.entry(path.clone()).insert_entry((stamp, classes)).into_mut().1
}
_ => &cache.entry(path.clone()).insert_entry((stamp, syntax.class_bases(&python, &text))).into_mut().1,
};
out.retain(|c| c.path.0 != *path);
out.extend(classes.iter().map(|class| ClassInheritance {
path: IpcPath::from(path.clone()),
line: class.line,
@@ -257,10 +268,7 @@ pub async fn nav_inheritance(host: &Host, window: &str) -> IdeResult<Vec<ClassIn
}
#[derive(Clone, PartialEq, Eq)]
enum Stamp {
Text(u64),
Disk(u64, Option<std::time::SystemTime>),
}
struct Stamp(u64);
fn text_hash(text: &str) -> u64 {
use std::hash::{Hash, Hasher};
+5 -1
View File
@@ -131,9 +131,13 @@ async fn returns_all_python_classes_and_bases() {
// Every returned entry is a Python file.
assert!(all.iter().all(|c| { matches!(c.path.0.extension().and_then(|e| e.to_str()), Some("py" | "pyi")) }));
// A file changed on disk is parsed again; the others come from the cache.
// A file changed on disk is picked up once the index re-reads it. The index is the
// single source of truth for on-disk classes now; edits go through its update path
// (the IDE's file watcher calls this in production).
let extra = p.root.join("src/extra.pyi");
std::fs::write(&extra, "class Stub(Child): ...\nclass Added(Base): ...\n").unwrap();
p.host.project(WINDOW).unwrap().index.update(std::slice::from_ref(&extra), &[]);
assert!(p.host.project(WINDOW).unwrap().index.wait_idle(Duration::from_secs(5)));
let again = ops::nav::nav_inheritance(&p.host, WINDOW).await.unwrap();
assert_eq!(find(&again, "Stub").bases, vec!["Child".to_string()]);
assert_eq!(find(&again, "Added").bases, vec!["Base".to_string()]);
+1
View File
@@ -54,6 +54,7 @@ fn method() -> SymbolTag {
kind: SymbolKind::Method,
container: None,
range: Range::new(Pos::new(0, 4), Pos::new(0, 8)),
bases: Vec::new(),
}
}
const SOURCE: &str = "def source:item\nref mapping:item\n";
@@ -225,6 +225,7 @@ async fn indexed_usages_rename_preview_staleness_and_grouped_undo() {
kind: ide_core::SymbolKind::Method,
container: None,
range: ide_core::Range::new(Pos::new(0, 4), Pos::new(0, 5)),
bases: Vec::new(),
};
plugins
.index_navigation(
+44 -1
View File
@@ -106,6 +106,9 @@ pub(crate) struct StoredSymbol {
pub kind: SymbolKind,
pub container: Option<String>,
pub range: Range,
/// Base names of a class definition; empty for every other symbol.
#[serde(default)]
pub bases: Vec<String>,
}
/// A symbol resolved out of [`SymTable`].
@@ -114,6 +117,8 @@ pub(crate) struct SymRef<'a> {
pub kind: SymbolKind,
pub container: Option<&'a str>,
pub range: Range,
/// Base names joined with `'\0'`; empty when the symbol has none.
pub bases: &'a str,
}
pub(crate) struct FileEntry {
@@ -402,6 +407,8 @@ pub(crate) struct SymTable {
container: Vec<Span>,
kind: Vec<SymbolKind>,
range: Vec<Range>,
/// Base names joined with `'\0'`, one interned span per symbol (`NONE` for none).
bases: Vec<Span>,
text: String,
dead_rows: usize,
}
@@ -423,6 +430,18 @@ impl SymTable {
(span.start != u32::MAX).then(|| span.get(&self.text))
}
/// The symbol's base names joined with `'\0'`; empty when it has none.
#[inline]
pub(crate) fn bases(&self, i: u32) -> &str {
let span = self.bases[i as usize];
if span.start == u32::MAX { "" } else { span.get(&self.text) }
}
/// The base names of row `i`, split out of the interned string.
pub(crate) fn base_list(&self, i: u32) -> Vec<&str> {
self.bases(i).split('\0').filter(|b| !b.is_empty()).collect()
}
/// The mask + name rows of a file's contiguous symbol range.
#[inline]
pub(crate) fn heads(&self, start: u32, len: u32) -> &[SymHead] {
@@ -459,10 +478,12 @@ impl SymTable {
Some(c) => self.intern(c),
None => Span::NONE,
};
let bases = if s.bases.is_empty() { Span::NONE } else { self.intern(&s.bases.join("\0")) };
self.head.push(SymHead { mask, name });
self.container.push(container);
self.kind.push(s.kind);
self.range.push(s.range);
self.bases.push(bases);
}
(start, agg_mask)
}
@@ -476,6 +497,7 @@ impl SymTable {
&& self.container(i) == s.container.as_deref()
&& self.kind(i) == s.kind
&& self.range(i) == s.range
&& self.base_list(i) == s.bases.iter().map(String::as_str).collect::<Vec<_>>()
})
}
@@ -492,6 +514,7 @@ impl SymTable {
let mut container = Vec::with_capacity(head.capacity());
let mut kind = Vec::with_capacity(head.capacity());
let mut range = Vec::with_capacity(head.capacity());
let mut bases = Vec::with_capacity(head.capacity());
let mut live = 0u32;
for (slot, row) in rows.iter_mut().enumerate() {
if !row.live {
@@ -518,6 +541,14 @@ impl SymTable {
}
kind.push(self.kind[i as usize]);
range.push(self.range[i as usize]);
let bases_span = self.bases[i as usize];
if bases_span.start == u32::MAX {
bases.push(Span::NONE);
} else {
let b = bases_span.get(&self.text);
bases.push(Span::of(&text, b));
text.push_str(b);
}
live += 1;
}
}
@@ -525,6 +556,7 @@ impl SymTable {
self.container = container;
self.kind = kind;
self.range = range;
self.bases = bases;
self.text = text;
self.dead_rows = 0;
}
@@ -534,6 +566,7 @@ impl SymTable {
self.container.shrink_to_fit();
self.kind.shrink_to_fit();
self.range.shrink_to_fit();
self.bases.shrink_to_fit();
self.text.shrink_to_fit();
}
}
@@ -577,6 +610,7 @@ impl IndexData {
self.syms.container.reserve(symbols);
self.syms.kind.reserve(symbols);
self.syms.range.reserve(symbols);
self.syms.bases.reserve(symbols);
}
/// Hydrates a validated warm-start snapshot in one pass, without per-file
@@ -640,6 +674,7 @@ impl IndexData {
kind: s.kind,
container: s.container.map(str::to_owned),
range: s.range,
bases: s.bases.split('\0').filter(|b| !b.is_empty()).map(str::to_owned).collect(),
})
.collect(),
ignored: e.ignored,
@@ -666,6 +701,7 @@ impl IndexData {
kind: syms.kind(i),
container: syms.container(i),
range: syms.range(i),
bases: syms.bases(i),
})
}
@@ -823,6 +859,7 @@ mod tests {
kind: SymbolKind::Function,
container: None,
range: Range::default(),
bases: Vec::new(),
})
.collect(),
..Default::default()
@@ -957,7 +994,13 @@ mod tests {
}
fn symbol(n: &str) -> StoredSymbol {
StoredSymbol { name: n.to_owned(), kind: SymbolKind::Function, container: None, range: Range::default() }
StoredSymbol {
name: n.to_owned(),
kind: SymbolKind::Function,
container: None,
range: Range::default(),
bases: Vec::new(),
}
}
#[test]
+35
View File
@@ -50,6 +50,17 @@ pub struct SymbolMatch {
pub indices: MatchIndices,
}
/// A class declaration of the index and the base names written in its source.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ClassDecl {
pub path: PathBuf,
pub name: String,
/// Selection range: the class name's identifier.
pub range: Range,
/// Bare base names, as written (`class X(pkg.mod.Base)` keeps `Base`).
pub bases: Vec<String>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Type)]
#[serde(rename_all = "camelCase")]
pub enum SymbolFilter {
@@ -206,11 +217,34 @@ impl ProjectIndex {
kind: s.kind,
container: s.container.map(ToString::to_string),
range: s.range,
bases: s.bases.split('\0').filter(|b| !b.is_empty()).map(str::to_owned).collect(),
})
.collect(),
)
}
/// Every class declaration of the indexed project (content roots included) with the
/// base names written in its source, in path order. The UI builds an inheritance
/// graph from it, so it needs no language server's type hierarchy.
pub fn inheritance(&self) -> Vec<ClassDecl> {
let data = self.shared.data.read();
let mut out = Vec::new();
for entry in data.files.iter().flatten() {
for s in data.symbols_of(entry) {
if s.kind != SymbolKind::Class {
continue;
}
out.push(ClassDecl {
path: entry.path.clone(),
name: s.name.to_owned(),
range: s.range,
bases: s.bases.split('\0').filter(|b| !b.is_empty()).map(str::to_owned).collect(),
});
}
}
out
}
/// Bounded file inventory for background plugin indexers; honors the same
/// project/exclude scope as search and never walks the filesystem.
pub fn files_with_extensions(&self, extensions: &[String], limit: usize) -> Vec<PathBuf> {
@@ -517,6 +551,7 @@ pub mod bench_support {
kind: if k % 3 == 0 { SymbolKind::Class } else { SymbolKind::Method },
container: (k % 3 != 0).then(|| format!("{w}{s}")),
range: Range::default(),
bases: Vec::new(),
})
.collect();
let record = FileRecord { symbols, ..Default::default() };
+14 -2
View File
@@ -11,8 +11,8 @@ use std::time::{SystemTime, UNIX_EPOCH};
const FILES: TableDefinition<&str, &[u8]> = TableDefinition::new("files");
const META: TableDefinition<&str, &[u8]> = TableDefinition::new("meta");
/// Bump when [`FileRecord`] or its encoding changes.
const SCHEMA: &str = "openidea-index-3";
/// Bump when [`FileRecord`] or its encoding changes. 4: a symbol's base names.
const SCHEMA: &str = "openidea-index-4";
/// The real Git directory for a checkout/worktree (`root/.git` may be a file
/// containing `gitdir: ...`).
@@ -113,6 +113,9 @@ struct FlatSymbol {
name_len: u32,
container_start: u32,
container_len: u32,
/// Base names joined with `'\0'` (`bases_len` 0 when the symbol has none).
bases_start: u32,
bases_len: u32,
kind: SymbolKind,
range: Range,
}
@@ -324,11 +327,14 @@ impl Snapshot {
let start = s.container_start as usize;
&self.text[start..start + s.container_len as usize]
});
let bases_start = s.bases_start as usize;
let bases = &self.text[bases_start..bases_start + s.bases_len as usize];
StoredSymbol {
name: name.to_owned(),
kind: s.kind,
container: container.map(str::to_owned),
range: s.range,
bases: bases.split('\0').filter(|b| !b.is_empty()).map(str::to_owned).collect(),
}
})
.collect();
@@ -498,11 +504,15 @@ impl Store {
}
None => (u32::MAX, 0),
};
let bases_start = fit(snap.text.len())?;
snap.text.push_str(s.bases);
snap.symbols.push(FlatSymbol {
name_start,
name_len: fit(s.name.len())?,
container_start,
container_len,
bases_start,
bases_len: fit(s.bases.len())?,
kind: s.kind,
range: s.range,
});
@@ -717,6 +727,7 @@ mod tests {
kind: SymbolKind::Class,
container: Some("Order".into()),
range: Range::default(),
bases: vec!["Base".into()],
}],
..Default::default()
};
@@ -740,6 +751,7 @@ mod tests {
assert_eq!(symbols.len(), 1);
assert_eq!(symbols[0].name, "OrderController");
assert_eq!(symbols[0].container.as_deref(), Some("Order"));
assert_eq!(symbols[0].bases, vec!["Base".to_string()], "bases survive the snapshot");
store.apply(&[], &[], false).unwrap();
assert!(store.load_snapshot().is_none(), "a file change invalidates the snapshot");
+23
View File
@@ -567,6 +567,29 @@ fn symbols_named_any_answers_every_name_inside_the_root_in_one_walk() {
assert_eq!(found, expected.map(|(name, path)| (name.to_string(), path.to_string())));
}
#[test]
fn inheritance_reports_python_classes_and_their_bases() {
let f = Fixture::new(&[(
"app.py",
"class Base:\n pass\n\nclass Child(Base, other.Mixin):\n pass\n\nclass Plain:\n pass\n",
)]);
let idx = f.open();
let mut found: Vec<(String, u32, Vec<String>)> =
idx.inheritance().into_iter().map(|c| (c.name, c.range.start.line, c.bases)).collect();
found.sort();
assert_eq!(
found,
vec![
("Base".to_string(), 0, Vec::new()),
("Child".to_string(), 3, vec!["Base".to_string(), "Mixin".to_string()]),
("Plain".to_string(), 6, Vec::new()),
]
);
// Every class of the project is reported, even without a superclass: the UI needs
// them to resolve the base names of other files.
assert_eq!(idx.inheritance().len(), 3);
}
#[test]
fn content_roots_and_excludes_change_without_reopening() {
let fx = Fixture::new(&[("src/main.rs", "fn alpha() {}\n"), ("src/gen/out.rs", "fn generated() {}\n")]);
+7 -1
View File
@@ -752,7 +752,13 @@ impl Shared {
.syntax
.tags(&lang, &text)
.into_iter()
.map(|t| StoredSymbol { name: t.name, kind: t.kind, container: t.container, range: t.range })
.map(|t| StoredSymbol {
name: t.name,
kind: t.kind,
container: t.container,
range: t.range,
bases: t.bases,
})
.collect();
}
Extracted::New(rec)
+4
View File
@@ -347,6 +347,10 @@ pub struct SymbolTag {
/// Enclosing symbol, e.g. class name for a method.
pub container: Option<String>,
pub range: Range,
/// Bare base names written in a class definition (`class X(A, B)`); empty for
/// every other symbol, and for languages whose classes carry none.
#[serde(default)]
pub bases: Vec<String>,
}
/// One inline TS/JS `<script>` block of a Vue SFC / HTML document.
+18 -3
View File
@@ -67,9 +67,10 @@ fn visit(cursor: &mut tree_sitter::TreeCursor, f: &mut impl FnMut(Node)) {
}
}
/// The bare base names a `class_definition` node writes (keyword arguments left out).
pub(super) fn class_base_names(node: Node, text: &str) -> Vec<String> {
node.child_by_field_name("superclasses").map(|args| base_names(args, text)).unwrap_or_default()
/// Bare base names of a `class_definition` node (`[]` without a superclass list).
/// Shared with the outline query, whose tags carry the bases of a class definition.
pub(super) fn class_base_names(class: Node, text: &str) -> Vec<String> {
class.child_by_field_name("superclasses").map(|args| base_names(args, text)).unwrap_or_default()
}
fn base_names(args: Node, text: &str) -> Vec<String> {
@@ -121,6 +122,20 @@ mod tests {
assert_eq!(shape("class A: pass"), vec![("A".to_owned(), 0, 6, vec![])]);
}
/// The project index stores bases on the symbol tags, so the outline query must
/// carry them too (one parse for both, not a second pass over every file).
#[test]
fn tags_carry_class_bases() {
static S: std::sync::OnceLock<Syntax> = std::sync::OnceLock::new();
let s = S.get_or_init(Syntax::new);
let tags = s.tags(&LanguageId::new("python"), "class A: pass\nclass B(A, pkg.C):\n def m(self): pass\n");
let bases_of = |name: &str| tags.iter().find(|t| t.name == name).map(|t| t.bases.clone()).unwrap_or_default();
assert!(bases_of("A").is_empty());
assert_eq!(bases_of("B"), vec!["A".to_owned(), "C".to_owned()]);
// Only classes carry bases; the method's tag stays empty.
assert!(bases_of("m").is_empty());
}
#[test]
fn multiple_bases_skips_metaclass() {
assert_eq!(
+10
View File
@@ -143,6 +143,8 @@ struct Raw {
visibility: Option<Visibility>,
/// Rust `impl Trait for Type`: members take the trait's visibility.
trait_impl: bool,
/// Base names of a class definition (empty where the grammar has none).
bases: Vec<String>,
}
struct Built {
@@ -221,6 +223,13 @@ fn collect(g: &Grammar, q: &OutlineQuery, tree: &Tree, text: &str) -> Vec<Built>
detail: detail.map(|d| clean(node_text(&d, text), 80)).filter(|d| !d.is_empty()),
visibility: explicit_visibility(&node, text),
trait_impl,
// The index needs every class's bases; Python is the grammar that has them.
bases: match kind {
ItemKind::Symbol(SymbolKind::Class) if g.spec.name == "python" => {
super::inheritance::class_base_names(node, text)
}
_ => Vec::new(),
},
};
match by_node.get(&raw.node_id) {
Some(existing) if existing.pattern <= raw.pattern => {}
@@ -369,6 +378,7 @@ pub(crate) fn tags(g: &Grammar, q: &OutlineQuery, tree: &Tree, text: &str) -> Ve
kind: b.kind,
container: own_container.map(str::to_owned),
range: b.raw.sel,
bases: b.raw.bases.clone(),
});
}
let next = if b.raw.tag || matches!(b.raw.kind, ItemKind::Impl | ItemKind::Extension) {
+1 -1
View File
@@ -129,7 +129,7 @@ pub(crate) fn definitions(g: &Grammar, css: Option<&Grammar>, tree: &Tree, text:
};
if let Some((name, kind)) = def {
let range = idx.range(base + n.start_byte(), base + n.end_byte());
out.push(SymbolTag { name, kind, container: None, range });
out.push(SymbolTag { name, kind, container: None, range, bases: Vec::new() });
}
true
});