perf: skip Dataset::index_statistics() for all index types (#3346)

`Dataset::index_statistics()` loads index files and does meaningful CPU work to serialize low-level info. Most fields `NativeTable::index_stats()` needs are available from manifest metadata via `Dataset::describe_indices()`, which is much cheaper. `NativeTable::index_stats()` now: - Calls `describe_indices()` filtered by name; returns `Ok(None)` if no match. - Parses `distance_type` from `description.details()` JSON (the `VectorIndexDetails` proto stored in the manifest by recent Lance versions). - Falls back to `index_statistics()` only for vector indices where `details()` returns no `distance_type` — this handles older Lance datasets that didn't write `VectorIndexDetails`. - `Unknown` index types (e.g. Lance's internal `FragReuseIndex`) are explicitly filtered out of `list_indices` rather than erroring. ## Test plan - [x] `test_create_scalar_index` — asserts `index_type`, `distance_type`, and `num_unindexed_rows > 0` after adding rows post-index - [x] `test_create_fm_index`, `test_create_bitmap_index`, `test_create_label_list_index` — added `index_stats` assertions - [x] IvfPq, IvfHnswPq, IvfHnswSq, IvfHnswFlat tests assert `distance_type == Some(L2)` - [x] `test_list_indices_skip_frag_reuse` — FragReuseIndex is filtered by the Unknown guard in `list_indices` --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-08-18 03:58:26 +00:00 · 2026-07-14 09:45:50 -07:00
parent afc0e5f497
commit 711e05619b
3 changed files with 128 additions and 50 deletions
@@ -317,6 +317,8 @@ pub enum IndexType {
    // FTS
    #[serde(alias = "INVERTED", alias = "Inverted")]
    FTS,
+    /// Catch-all for index types not recognized by this version of LanceDB.
+    Unknown,
 }

 impl std::fmt::Display for IndexType {
@@ -334,6 +336,7 @@ impl std::fmt::Display for IndexType {
            Self::LabelList => write!(f, "LABEL_LIST"),
            Self::Fm => write!(f, "FM"),
            Self::FTS => write!(f, "FTS"),
+            Self::Unknown => write!(f, "UNKNOWN"),
        }
    }
 }
@@ -355,9 +358,7 @@ impl std::str::FromStr for IndexType {
            "IVF_HNSW_PQ" => Ok(Self::IvfHnswPq),
            "IVF_HNSW_SQ" => Ok(Self::IvfHnswSq),
            "IVF_HNSW_FLAT" => Ok(Self::IvfHnswFlat),
-            _ => Err(Error::InvalidInput {
-                message: format!("the input value {} is not a valid IndexType", value),
-            }),
+            _ => Ok(Self::Unknown),
        }
    }
 }
@@ -425,20 +426,15 @@ pub struct IndexConfig {
 #[derive(Debug, Deserialize)]
 pub(crate) struct IndexMetadata {
    pub metric_type: Option<DistanceType>,
-    // Sometimes the index type is provided at this level.
-    pub index_type: Option<IndexType>,
 }

-// This struct is used to deserialize the JSON data returned from the Lance API
-// Dataset::index_statistics().
+// Deserializes the JSON returned by Dataset::index_statistics().
 #[skip_serializing_none]
 #[derive(Debug, Deserialize)]
 pub(crate) struct IndexStatisticsImpl {
    pub num_indexed_rows: usize,
    pub num_unindexed_rows: usize,
    pub indices: Vec<IndexMetadata>,
-    // Sometimes, the index type is provided at this level.
-    pub index_type: Option<IndexType>,
    pub num_indices: Option<u32>,
 }

@@ -23,6 +23,7 @@ use lance::dataset::{InsertBuilder, WriteParams};
 use lance::index::DatasetIndexExt;
 use lance::io::{ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::utils::StreamingWriteSource;
+use lance_index::IndexCriteria;
 use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, StorageOptionsAccessor};
 pub use query::AnyQuery;

@@ -42,6 +43,7 @@ use std::sync::Arc;

 use crate::connection::NamespaceClientPushdownOperation;

+use crate::DistanceType;
 use crate::data::scannable::{PeekedScannable, Scannable, estimate_write_partitions};
 use crate::database::Database;
 use crate::database::read_freshness::TableFreshness;
@@ -2967,17 +2969,21 @@ impl BaseTable for NativeTable {
            .await?
            .into_iter()
            .filter_map(|idx_desc| {
-                let index_type: crate::index::IndexType = match idx_desc.index_type().parse() {
-                    Ok(index_type) => index_type,
-                    Err(e) => {
-                        log::warn!(
-                            "Failed to parse index type for index {}: {}",
-                            idx_desc.name(),
-                            e
-                        );
-                        return None;
-                    }
-                };
+                let index_type: crate::index::IndexType = idx_desc
+                    .index_type()
+                    .parse()
+                    .unwrap_or(crate::index::IndexType::Unknown);
+                if index_type == crate::index::IndexType::Unknown {
+                    // Internal or future index types that this version doesn't recognize
+                    // (e.g. Lance's internal FragReuseIndex) are silently excluded from
+                    // the user-visible index listing.
+                    log::debug!(
+                        "Skipping unrecognized index '{}' (type '{}') in list_indices",
+                        idx_desc.name(),
+                        idx_desc.index_type(),
+                    );
+                    return None;
+                }

                let field_ids = idx_desc.field_ids();
                let mut columns = Vec::with_capacity(field_ids.len());
@@ -3044,40 +3050,83 @@ impl BaseTable for NativeTable {
    }

    async fn index_stats(&self, index_name: &str) -> Result<Option<IndexStatistics>> {
-        let stats = match self
-            .dataset
-            .get()
-            .await?
-            .index_statistics(index_name.as_ref())
-            .await
-        {
-            Ok(stats) => stats,
-            Err(lance_core::Error::IndexNotFound { .. }) => return Ok(None),
-            Err(e) => return Err(Error::from(e)),
+        // describe_indices() reads only manifest-level metadata (no index file I/O).
+        // VectorIndexDetails in the manifest carries distance_type for indices written
+        // by recent Lance versions. For older datasets that didn't write those details
+        // we fall back to index_statistics() for vector index types.
+        let dataset = self.dataset.get().await?;
+
+        let mut descriptions = dataset
+            .describe_indices(Some(IndexCriteria::default().with_name(index_name)))
+            .await?;
+        let Some(description) = descriptions.pop() else {
+            return Ok(None);
        };

-        let mut stats: IndexStatisticsImpl =
-            serde_json::from_str(&stats).map_err(|e| Error::InvalidInput {
-                message: format!("error deserializing index statistics: {}", e),
-            })?;
+        let index_type: crate::index::IndexType = description
+            .index_type()
+            .parse()
+            .unwrap_or(crate::index::IndexType::Unknown);

-        let first_index = stats.indices.pop().ok_or_else(|| Error::InvalidInput {
-            message: "index statistics is empty".to_string(),
-        })?;
-        // Index type should be present at one of the levels.
-        let index_type =
-            stats
-                .index_type
-                .or(first_index.index_type)
-                .ok_or_else(|| Error::InvalidInput {
-                    message: "index statistics was missing index type".to_string(),
-                })?;
-        Ok(Some(IndexStatistics {
-            num_indexed_rows: stats.num_indexed_rows,
-            num_unindexed_rows: stats.num_unindexed_rows,
+        let is_vector = matches!(
            index_type,
-            distance_type: first_index.metric_type,
-            num_indices: stats.num_indices,
+            crate::index::IndexType::IvfFlat
+                | crate::index::IndexType::IvfSq
+                | crate::index::IndexType::IvfPq
+                | crate::index::IndexType::IvfRq
+                | crate::index::IndexType::IvfHnswPq
+                | crate::index::IndexType::IvfHnswSq
+                | crate::index::IndexType::IvfHnswFlat
+        );
+
+        // details() serializes VectorIndexDetails to JSON with an uppercase "metric_type"
+        // field (e.g. "L2", "COSINE"). Parse it with a case-insensitive match.
+        let distance_type = description.details().ok().and_then(|json| {
+            #[derive(serde::Deserialize)]
+            struct Details {
+                metric_type: Option<String>,
+            }
+            serde_json::from_str::<Details>(&json)
+                .ok()
+                .and_then(|d| d.metric_type)
+                .and_then(|m| match m.to_uppercase().as_str() {
+                    "L2" => Some(DistanceType::L2),
+                    "COSINE" => Some(DistanceType::Cosine),
+                    "DOT" => Some(DistanceType::Dot),
+                    "HAMMING" => Some(DistanceType::Hamming),
+                    _ => None,
+                })
+        });
+
+        // Older Lance datasets didn't write VectorIndexDetails, so distance_type won't
+        // be in the manifest. Fall back to index_statistics() only in that case.
+        if is_vector && distance_type.is_none() {
+            let stats = dataset.index_statistics(index_name).await?;
+            let mut stats: IndexStatisticsImpl =
+                serde_json::from_str(&stats).map_err(|e| Error::InvalidInput {
+                    message: format!("error deserializing index statistics: {}", e),
+                })?;
+            let first_index = stats.indices.pop().ok_or_else(|| Error::InvalidInput {
+                message: "index statistics is empty".to_string(),
+            })?;
+            return Ok(Some(IndexStatistics {
+                num_indexed_rows: stats.num_indexed_rows,
+                num_unindexed_rows: stats.num_unindexed_rows,
+                index_type,
+                distance_type: first_index.metric_type,
+                num_indices: stats.num_indices,
+            }));
+        }
+
+        let num_indexed_rows = description.rows_indexed() as usize;
+        let total_rows = dataset.count_rows(None).await?;
+        let num_unindexed_rows = total_rows.saturating_sub(num_indexed_rows);
+        Ok(Some(IndexStatistics {
+            num_indexed_rows,
+            num_unindexed_rows,
+            index_type,
+            distance_type,
+            num_indices: Some(description.metadata().len() as u32),
        }))
    }

@@ -589,6 +589,7 @@ mod tests {
        let stats = table.index_stats(index_name).await.unwrap().unwrap();
        assert_eq!(stats.num_indexed_rows, 512);
        assert_eq!(stats.num_unindexed_rows, 0);
+        assert_eq!(stats.distance_type, Some(crate::DistanceType::L2));
    }

    #[tokio::test]
@@ -646,6 +647,7 @@ mod tests {
        let stats = table.index_stats(index_name).await.unwrap().unwrap();
        assert_eq!(stats.num_indexed_rows, 512);
        assert_eq!(stats.num_unindexed_rows, 0);
+        assert_eq!(stats.distance_type, Some(crate::DistanceType::L2));
    }

    #[tokio::test]
@@ -690,6 +692,10 @@ mod tests {
        assert_eq!(index.index_type, crate::index::IndexType::IvfHnswFlat);
        assert_eq!(index.columns, vec!["embeddings".to_string()]);
        assert_eq!(table.count_rows(None).await.unwrap(), 512);
+        let stats = table.index_stats(&index.name).await.unwrap().unwrap();
+        assert_eq!(stats.num_indexed_rows, 512);
+        assert_eq!(stats.num_unindexed_rows, 0);
+        assert_eq!(stats.distance_type, Some(crate::DistanceType::L2));
    }

    #[tokio::test]
@@ -747,6 +753,15 @@ mod tests {
        let stats = table.index_stats(index_name).await.unwrap().unwrap();
        assert_eq!(stats.num_indexed_rows, 1);
        assert_eq!(stats.num_unindexed_rows, 0);
+        assert_eq!(stats.index_type, crate::index::IndexType::BTree);
+        assert_eq!(stats.distance_type, None);
+
+        // Rows added after the index was built appear as unindexed.
+        let new_batch = record_batch!(("i", Int32, [2])).unwrap();
+        table.add(new_batch).execute().await.unwrap();
+        let stats = table.index_stats(index_name).await.unwrap().unwrap();
+        assert_eq!(stats.num_indexed_rows, 1);
+        assert_eq!(stats.num_unindexed_rows, 1);
    }

    #[tokio::test]
@@ -795,6 +810,12 @@ mod tests {
            .map(|b| b.num_rows())
            .sum::<usize>();
        assert_eq!(count, 1);
+
+        let stats = table.index_stats("text_idx").await.unwrap().unwrap();
+        assert_eq!(stats.num_indexed_rows, 1);
+        assert_eq!(stats.num_unindexed_rows, 0);
+        assert_eq!(stats.index_type, crate::index::IndexType::Fm);
+        assert_eq!(stats.distance_type, None);
    }

    #[tokio::test]
@@ -1188,6 +1209,12 @@ mod tests {
        let index = configs_iter.next().unwrap();
        assert_eq!(index.index_type, crate::index::IndexType::Bitmap);
        assert_eq!(index.columns, vec!["large_data".to_string()]);
+
+        let stats = table.index_stats("category_idx").await.unwrap().unwrap();
+        assert_eq!(stats.num_indexed_rows, 100);
+        assert_eq!(stats.num_unindexed_rows, 0);
+        assert_eq!(stats.index_type, crate::index::IndexType::Bitmap);
+        assert_eq!(stats.distance_type, None);
    }

    #[tokio::test]
@@ -1256,6 +1283,12 @@ mod tests {
        let index = index_configs.into_iter().next().unwrap();
        assert_eq!(index.index_type, crate::index::IndexType::LabelList);
        assert_eq!(index.columns, vec!["tags".to_string()]);
+
+        let stats = table.index_stats("tags_idx").await.unwrap().unwrap();
+        assert_eq!(stats.num_indexed_rows, 40);
+        assert_eq!(stats.num_unindexed_rows, 0);
+        assert_eq!(stats.index_type, crate::index::IndexType::LabelList);
+        assert_eq!(stats.distance_type, None);
    }

    #[tokio::test]