From dd4a527727e3409b2a329225d802be45ca48ee86 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 22 Oct 2026 16:13:27 +0800 Subject: [PATCH] fix(index): preserve registered scalar plugin type names (#4627) Native `list_indices()` drops registered scalar plugin types absent from LanceDB's fixed enum, and `index_stats()` reports them as `Unknown`. Preserve their registered display names through `IndexType::Custom(String)`. Resolve these names through Lance's shared `IndexDetails` registry entry point. Registered types such as MinHashLSH become visible; system and unregistered types remain excluded. Existing builtin aliases retain their canonical names. Rust consumers with exhaustive `IndexType` matches need to handle the added variant. Manual validation with [Lance #10224](https://github.com/lance-format/lance/pull/10224) at `fcc070f88a`: a downstream statistics-only plugin appears as `Custom("Sketch")` in native listing and statistics. Its `load_index` deliberately returns an error, and repeated calls do not construct additional plugin instances, verifying metadata-only lookup through the shared registry. --- rust/lancedb/src/index.rs | 30 +++++++++ rust/lancedb/src/table.rs | 10 +-- .../lancedb/tests/registered_scalar_plugin.rs | 66 +++++++++++++++++++ 3 files changed, 98 insertions(+), 8 deletions(-) create mode 100644 rust/lancedb/tests/registered_scalar_plugin.rs diff --git a/rust/lancedb/src/index.rs b/rust/lancedb/src/index.rs index 82f8ad845..712a15fa0 100644 --- a/rust/lancedb/src/index.rs +++ b/rust/lancedb/src/index.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright The LanceDB Authors use chrono::{DateTime, Utc}; +use lance::index::scalar::IndexDetails; use scalar::FtsIndexBuilder; use serde::Deserialize; use serde_with::skip_serializing_none; @@ -382,10 +383,38 @@ pub enum IndexType { // FTS #[serde(alias = "INVERTED", alias = "Inverted")] FTS, + /// A registered scalar index plugin's display name, preserved verbatim. + /// + /// For example, `Custom("Sketch".into()).to_string()` returns `"Sketch"`. + Custom(String), /// Catch-all for index types not recognized by this version of LanceDB. Unknown, } +impl IndexType { + pub(crate) fn from_description( + description: &dyn lance_index::traits::IndexDescription, + ) -> Self { + let index_type = description.index_type().parse().unwrap_or(Self::Unknown); + if index_type != Self::Unknown { + return index_type; + } + let Some(metadata) = description.segments().first() else { + return Self::Unknown; + }; + if lance_index::is_system_index(metadata) { + return Self::Unknown; + } + let Some(details) = metadata.index_details.as_ref() else { + return Self::Unknown; + }; + IndexDetails(details.clone()) + .get_plugin() + .map(|plugin| Self::Custom(plugin.name().to_string())) + .unwrap_or(Self::Unknown) + } +} + impl std::fmt::Display for IndexType { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { @@ -405,6 +434,7 @@ impl std::fmt::Display for IndexType { Self::BloomFilter => write!(f, "BLOOM_FILTER"), Self::RTree => write!(f, "RTREE"), Self::FTS => write!(f, "FTS"), + Self::Custom(name) => f.write_str(name), Self::Unknown => write!(f, "UNKNOWN"), } } diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 61a710dcd..39e6f1911 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -3821,10 +3821,7 @@ impl BaseTable for NativeTable { let mut indices: Vec = descriptions .iter() .filter_map(|idx_desc| { - let index_type: crate::index::IndexType = idx_desc - .index_type() - .parse() - .unwrap_or(crate::index::IndexType::Unknown); + let index_type = crate::index::IndexType::from_description(idx_desc.as_ref()); if index_type == crate::index::IndexType::Unknown { // Internal or future index types that this version doesn't recognize // (e.g. Lance's internal FragReuseIndex) are silently excluded from @@ -3960,10 +3957,7 @@ impl BaseTable for NativeTable { return Ok(None); }; - let index_type: crate::index::IndexType = description - .index_type() - .parse() - .unwrap_or(crate::index::IndexType::Unknown); + let index_type = crate::index::IndexType::from_description(description.as_ref()); let is_vector = matches!( index_type, diff --git a/rust/lancedb/tests/registered_scalar_plugin.rs b/rust/lancedb/tests/registered_scalar_plugin.rs new file mode 100644 index 000000000..2f5620cc8 --- /dev/null +++ b/rust/lancedb/tests/registered_scalar_plugin.rs @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +use arrow_array::{RecordBatchIterator, record_batch}; +use lance::{Dataset, index::DatasetIndexExt}; +use lance_index::scalar::ScalarIndexParams; +use lancedb::index::IndexType; + +// MinHashLSH is a registered scalar plugin without a LanceDB IndexType variant. +// It exercises the same metadata-only fallback as a downstream plugin. +#[tokio::test] +async fn list_registered_scalar_plugin_with_its_type_name() { + let directory = tempfile::tempdir().unwrap(); + let batch = record_batch!(( + "value", + Utf8, + ["alpha beta", "beta gamma", "gamma delta", "delta alpha"] + )) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new([Ok(batch.clone())], batch.schema()), + directory.path().join("test.lance").to_str().unwrap(), + None, + ) + .await + .unwrap(); + dataset + .create_index_builder( + &["value"], + lance_index::IndexType::Scalar, + &ScalarIndexParams::new("minhashlsh".into()), + ) + .name("profile".into()) + .train(false) + .await + .unwrap(); + + let connection = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let table = connection.open_table("test").execute().await.unwrap(); + let indices = table.list_indices().await.unwrap(); + assert_eq!(indices.len(), 1); + assert_eq!(indices[0].name, "profile"); + assert_eq!( + indices[0].index_type, + IndexType::Custom("MinHashLsh".into()) + ); + assert_eq!(indices[0].index_type.to_string(), "MinHashLsh"); + assert_eq!(indices[0].columns, ["value"]); + assert!( + indices[0] + .type_url + .as_deref() + .unwrap() + .ends_with("MinHashLshIndexDetails") + ); + let statistics = table.index_stats("profile").await.unwrap().unwrap(); + assert_eq!( + statistics.index_type, + IndexType::Custom("MinHashLsh".into()) + ); + assert_eq!(statistics.num_indexed_rows, 0); + assert_eq!(statistics.num_unindexed_rows, 4); +}