rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-07-04 03:20:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: expand IndexConfig with rich per-index metadata

Browse Source 
IndexConfig (returned by Table::list_indices) previously exposed only
name, index_type, and columns. Lance's describe_indices provides richer
per-index info cheaply, so this surfaces it as Option<T> fields populated
in NativeTable::list_indices from the IndexDescription:

- index_uuid, type_url, created_at, num_indexed_rows, num_unindexed_rows,
  size_bytes, num_segments, index_version, index_details

The field set mirrors the lance-namespace IndexContent contract so client
and server agree on the same shape. RemoteTable leaves the new fields None
until a follow-up wires them through the server response.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Will Jones
2026-06-10 10:22:05 -07:00
parent 3347c3afcf
commit 92e4e57f61
3 changed files with 261 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
rust/lancedb/src/index.rs
View File

@@ -1,6 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use chrono::{DateTime, Utc};
use scalar::FtsIndexBuilder;
use serde::Deserialize;
use serde_with::skip_serializing_none;
@@ -364,6 +365,51 @@ pub struct IndexConfig {
/// Currently this is always a Vec of size 1. In the future there may
/// be more columns to represent composite indices.
pub columns: Vec<String>,
/// The UUID of the first segment of the index.
///
/// An index may be made up of multiple segments, each with their own UUID.
/// This is the UUID of the first segment. `None` if it could not be
/// determined (e.g. for remote tables, which do not yet surface this).
pub index_uuid: Option<String>,
/// The protobuf type URL, a precise type identifier for the index.
///
/// `None` if unavailable (e.g. for remote tables).
pub type_url: Option<String>,
/// When the index was created, taken as the minimum creation time across
/// all segments.
///
/// `None` if unavailable, such as for indices created before creation
/// timestamps were tracked, or for remote tables.
pub created_at: Option<DateTime<Utc>>,
/// The number of rows indexed, across all segments.
///
/// This is approximate and may include rows that have since been deleted.
/// `None` if unavailable (e.g. for remote tables).
pub num_indexed_rows: Option<u64>,
/// The number of rows in the table that are not yet covered by this index.
///
/// Computed as the table's total row count minus [`Self::num_indexed_rows`].
/// Optimizing the index will fold these rows into it. `None` if unavailable
/// (e.g. for remote tables).
pub num_unindexed_rows: Option<u64>,
/// The total size in bytes of all index files across all segments.
///
/// `None` if size information is unavailable, such as for indices created
/// before file sizes were tracked, or for remote tables.
pub size_bytes: Option<u64>,
/// The number of segments that make up the index.
///
/// `None` if unavailable (e.g. for remote tables).
pub num_segments: Option<u32>,
/// The on-disk index format version, taken from the first segment.
///
/// `None` if unavailable (e.g. for remote tables).
pub index_version: Option<i32>,
/// Index-type-specific details, serialized as JSON.
///
/// The shape of this JSON varies by index type. `None` if the details
/// could not be produced (e.g. no plugin available) or for remote tables.
pub index_details: Option<String>,
}
#[skip_serializing_none]

179
rust/lancedb/src/remote/table.rs
View File

@@ -2119,6 +2119,17 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
name: index.index_name,
index_type: stats.index_type,
columns,
// These are left None until the server response wires
// them through. See https://github.com/lancedb/lancedb/issues/3494
index_uuid: None,
type_url: None,
created_at: None,
num_indexed_rows: None,
num_unindexed_rows: None,
size_bytes: None,
num_segments: None,
index_version: None,
index_details: None,
})),
Ok(None) => Ok(None), // The index must have been deleted since we listed it.
Err(e) => Err(e),
@@ -4037,16 +4048,184 @@ mod tests {
name: "vector_idx".into(),
index_type: IndexType::IvfPq,
columns: vec!["vector".into()],
index_uuid: None,
type_url: None,
created_at: None,
num_indexed_rows: None,
num_unindexed_rows: None,
size_bytes: None,
num_segments: None,
index_version: None,
index_details: None,
},
IndexConfig {
name: "my_idx".into(),
index_type: IndexType::LabelList,
columns: vec!["metadata.`my.column`".into()],
index_uuid: None,
type_url: None,
created_at: None,
num_indexed_rows: None,
num_unindexed_rows: None,
size_bytes: None,
num_segments: None,
index_version: None,
index_details: None,
},
];
assert_eq!(indices, expected);
}
#[tokio::test]
async fn test_list_indices_nested_field_paths() {
let schema = nested_index_schema();
let table = Table::new_with_handler("my_table", move |request| {
assert_eq!(request.method(), "POST");
let response_body = match request.url().path() {
"/v1/table/my_table/describe/" => {
return http::Response::builder()
.status(200)
.body(describe_response(&schema))
.unwrap();
}
"/v1/table/my_table/index/list/" => {
serde_json::json!({
"indexes": [
{
"index_name": "row_id_idx",
"index_uuid": "00000000-0000-0000-0000-000000000001",
"columns": ["rowId"],
"index_status": "done",
},
{
"index_name": "row_dash_id_idx",
"index_uuid": "00000000-0000-0000-0000-000000000002",
"columns": ["`ROW-ID`"],
"index_status": "done",
},
{
"index_name": "user_id_idx",
"index_uuid": "00000000-0000-0000-0000-000000000003",
"columns": ["userId"],
"index_status": "done",
},
{
"index_name": "mixed_case_metadata_user_id_idx",
"index_uuid": "00000000-0000-0000-0000-000000000004",
"columns": ["MetaData.userId"],
"index_status": "done",
},
{
"index_name": "metadata_user_id_idx",
"index_uuid": "00000000-0000-0000-0000-000000000005",
"columns": ["Metadata.USER_ID"],
"index_status": "done",
},
{
"index_name": "image_embedding_idx",
"index_uuid": "00000000-0000-0000-0000-000000000006",
"columns": ["Image.Embedding"],
"index_status": "done",
},
{
"index_name": "payload_text_idx",
"index_uuid": "00000000-0000-0000-0000-000000000007",
"columns": ["Payload.Text"],
"index_status": "done",
},
{
"index_name": "meta_data_user_id_idx",
"index_uuid": "00000000-0000-0000-0000-000000000008",
"columns": ["`META-DATA`.`USER-ID`"],
"index_status": "done",
},
{
"index_name": "literal_dot_idx",
"index_uuid": "00000000-0000-0000-0000-000000000009",
"columns": ["literal.`A.B`"],
"index_status": "done",
},
]
})
}
"/v1/table/my_table/index/row_id_idx/stats/"
| "/v1/table/my_table/index/row_dash_id_idx/stats/"
| "/v1/table/my_table/index/user_id_idx/stats/"
| "/v1/table/my_table/index/mixed_case_metadata_user_id_idx/stats/"
| "/v1/table/my_table/index/metadata_user_id_idx/stats/"
| "/v1/table/my_table/index/meta_data_user_id_idx/stats/"
| "/v1/table/my_table/index/literal_dot_idx/stats/" => {
serde_json::json!({
"num_indexed_rows": 100000,
"num_unindexed_rows": 0,
"index_type": "BTREE"
})
}
"/v1/table/my_table/index/image_embedding_idx/stats/" => {
serde_json::json!({
"num_indexed_rows": 100000,
"num_unindexed_rows": 0,
"index_type": "IVF_PQ",
"distance_type": "l2"
})
}
"/v1/table/my_table/index/payload_text_idx/stats/" => {
serde_json::json!({
"num_indexed_rows": 100000,
"num_unindexed_rows": 0,
"index_type": "FTS"
})
}
path => panic!("Unexpected path: {}", path),
};
http::Response::builder()
.status(200)
.body(serde_json::to_string(&response_body).unwrap())
.unwrap()
});
let indices = table.list_indices().await.unwrap();
// The remote path leaves the rich metadata fields None until the server
// wires them through. See https://github.com/lancedb/lancedb/issues/3494
let expected: Vec<IndexConfig> = [
("row_id_idx", IndexType::BTree, "rowId"),
("row_dash_id_idx", IndexType::BTree, "`row-id`"),
("user_id_idx", IndexType::BTree, "userId"),
(
"mixed_case_metadata_user_id_idx",
IndexType::BTree,
"MetaData.userId",
),
("metadata_user_id_idx", IndexType::BTree, "metadata.user_id"),
("image_embedding_idx", IndexType::IvfPq, "image.embedding"),
("payload_text_idx", IndexType::FTS, "payload.text"),
(
"meta_data_user_id_idx",
IndexType::BTree,
"`meta-data`.`user-id`",
),
("literal_dot_idx", IndexType::BTree, "literal.`a.b`"),
]
.into_iter()
.map(|(name, index_type, column)| IndexConfig {
name: name.into(),
index_type,
columns: vec![column.into()],
index_uuid: None,
type_url: None,
created_at: None,
num_indexed_rows: None,
num_unindexed_rows: None,
size_bytes: None,
num_segments: None,
index_version: None,
index_details: None,
})
.collect();
assert_eq!(indices, expected);
}
#[tokio::test]
async fn test_list_versions() {
let table = Table::new_with_handler("my_table", |request| {

36
rust/lancedb/src/table.rs
View File

@@ -2924,6 +2924,7 @@ impl BaseTable for NativeTable {
async fn list_indices(&self) -> Result<Vec<IndexConfig>> {
let dataset = self.dataset.get().await?;
let total_rows = dataset.count_rows(None).await? as u64;
let indices = dataset
.describe_indices(None)
.await?
@@ -2959,10 +2960,25 @@ impl BaseTable for NativeTable {
columns.push(field_path);
}
let segments = idx_desc.segments();
let index_uuid = segments.first().map(|seg| seg.uuid.to_string());
let created_at = segments.iter().filter_map(|seg| seg.created_at).min();
let index_version = segments.first().map(|seg| seg.index_version);
let num_indexed_rows = idx_desc.rows_indexed();
Some(IndexConfig {
name: idx_desc.name().to_string(),
index_type,
columns,
index_uuid,
type_url: Some(idx_desc.type_url().to_string()),
created_at,
num_indexed_rows: Some(num_indexed_rows),
num_unindexed_rows: Some(total_rows.saturating_sub(num_indexed_rows)),
size_bytes: idx_desc.total_size_bytes(),
num_segments: Some(segments.len() as u32),
index_version,
index_details: idx_desc.details().ok(),
})
})
.collect();
@@ -3417,6 +3433,15 @@ mod tests {
let index = index_configs.into_iter().next().unwrap();
assert_eq!(index.index_type, crate::index::IndexType::IvfPq);
assert_eq!(index.columns, vec!["embeddings".to_string()]);
assert!(index.index_uuid.is_some());
assert!(index.type_url.is_some());
assert_eq!(index.num_segments, Some(1));
assert_eq!(index.num_indexed_rows, Some(512));
assert_eq!(index.num_unindexed_rows, Some(0));
assert!(index.created_at.is_some());
assert!(index.size_bytes.is_some());
assert!(index.index_version.is_some());
assert!(index.index_details.is_some());
assert_eq!(table.count_rows(None).await.unwrap(), 512);
assert_eq!(table.name(), "test");
@@ -3766,6 +3791,17 @@ mod tests {
assert_eq!(index.index_type, crate::index::IndexType::BTree);
assert_eq!(index.columns, vec!["i".to_string()]);
// The richer metadata surfaced from describe_indices should be populated.
assert!(index.index_uuid.is_some());
assert!(index.type_url.is_some());
assert_eq!(index.num_segments, Some(1));
assert_eq!(index.num_indexed_rows, Some(1));
assert_eq!(index.num_unindexed_rows, Some(0));
assert!(index.created_at.is_some());
assert!(index.size_bytes.is_some());
assert!(index.index_version.is_some());
assert!(index.index_details.is_some());
let indices = table.as_native().unwrap().load_indices().await.unwrap();
let index_name = &indices[0].index_name;
let stats = table.index_stats(index_name).await.unwrap().unwrap();