rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-09-21 20:45:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(rust): add zonemap index builder (#4199)

Browse Source 
Lance supports ZoneMap scalar indexes, but the LanceDB Rust API did not
expose a first-class way to request one through `Table::create_index`.
Users had builders for the other scalar index families, while ZoneMap
was missing from the public `Index` model and remote create-index
serialization. This PR adds ZoneMap as a supported scalar index option
in LanceDB.

This was accomplished with the following changes:
- Added `ZoneMapIndexBuilder` in `rust/lancedb/src/index/scalar.rs`.
- Added `Index::ZoneMap` and `IndexType::ZoneMap`, including
display/from-string aliases for `ZONEMAP` and `ZONE_MAP`.
- Mapped local index creation to
`ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap)` and Lance
`IndexType::ZoneMap` in `rust/lancedb/src/table/create_index.rs`.
- Serialized remote create-index requests as `index_type: "ZONEMAP"` in
`rust/lancedb/src/remote/table.rs`.
- Added coverage for both local ZoneMap index creation and remote
request serialization.

Example:

```rust
table
    .create_index(&["my_column"], Index::ZoneMap(Default::default()))
    .execute()
    .await?;
```

### Testing

Added `test_create_zonemap_index` for local index creation and extended
the remote request body test matrix for `ZONEMAP`.
This commit is contained in:
Bruno Ramirez
2026-09-16 11:54:22 -07:00
committed by GitHub
parent 6a07f88980
commit a4f66afc69
4 changed files with 94 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/lancedb/src/index.rs
+14 -1
View File
@@ -13,7 +13,10 @@ use crate::index::vector::IvfRqIndexBuilder;
use crate::{DistanceType, Error, Result, job::Job, table::BaseTable};
use self::{
scalar::{BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder, LabelListIndexBuilder},
scalar::{
BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder, LabelListIndexBuilder,
ZoneMapIndexBuilder,
},
vector::{
IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder,
IvfSqIndexBuilder,
@@ -54,6 +57,12 @@ pub enum Index {
/// substrings of the raw bytes, unlike the tokenized [`Index::FTS`] index.
Fm(FmIndexBuilder),
/// A `ZoneMap` index stores min/max summaries for ranges of rows.
///
/// It can accelerate range filters by skipping zones whose min/max values
/// prove they cannot match the predicate.
ZoneMap(ZoneMapIndexBuilder),
/// Full text search index using BM25.
///
/// The posting block size defaults to 128. Supported values are 128 and 256;
@@ -341,6 +350,8 @@ pub enum IndexType {
LabelList,
#[serde(alias = "FM", alias = "FMINDEX", alias = "FMIndex")]
Fm,
#[serde(alias = "ZONEMAP", alias = "ZONE_MAP")]
ZoneMap,
// FTS
#[serde(alias = "INVERTED", alias = "Inverted")]
FTS,
@@ -362,6 +373,7 @@ impl std::fmt::Display for IndexType {
Self::Bitmap => write!(f, "BITMAP"),
Self::LabelList => write!(f, "LABEL_LIST"),
Self::Fm => write!(f, "FM"),
Self::ZoneMap => write!(f, "ZONEMAP"),
Self::FTS => write!(f, "FTS"),
Self::Unknown => write!(f, "UNKNOWN"),
}
@@ -377,6 +389,7 @@ impl std::str::FromStr for IndexType {
"BITMAP" => Ok(Self::Bitmap),
"LABEL_LIST" | "LABELLIST" => Ok(Self::LabelList),
"FM" | "FMINDEX" => Ok(Self::Fm),
"ZONEMAP" | "ZONE_MAP" => Ok(Self::ZoneMap),
"FTS" | "INVERTED" => Ok(Self::FTS),
"IVF_FLAT" => Ok(Self::IvfFlat),
"IVF_SQ" => Ok(Self::IvfSq),
rust/lancedb/src/index/scalar.rs
+22
View File
@@ -60,6 +60,28 @@ pub struct LabelListIndexBuilder {}
#[derive(Debug, Clone, Default, serde::Serialize)]
pub struct FmIndexBuilder {}
/// Builder for a ZoneMap index.
///
/// A ZoneMap index stores min/max summaries for ranges of rows and can
/// accelerate range predicates by pruning zones that cannot match.
///
/// ```
/// use lancedb::{
/// index::{scalar::ZoneMapIndexBuilder, Index},
/// Table,
/// };
///
/// # async fn create_zonemap_index(table: &Table) -> lancedb::Result<()> {
/// table
/// .create_index(&["timestamp"], Index::ZoneMap(ZoneMapIndexBuilder::default()))
/// .execute()
/// .await?;
/// # Ok(())
/// # }
/// ```
#[derive(Debug, Clone, Default, serde::Serialize)]
pub struct ZoneMapIndexBuilder {}
pub use lance_index::scalar::FullTextSearchQuery;
pub use lance_index::scalar::InvertedIndexParams as FtsIndexBuilder;
pub use lance_index::scalar::InvertedIndexParams;
rust/lancedb/src/remote/table.rs
+2
View File
@@ -587,6 +587,7 @@ impl<S: HttpSend> RemoteTable<S> {
Index::Bitmap(p) => ("BITMAP", Some(to_json(p)?)),
Index::LabelList(p) => ("LABEL_LIST", Some(to_json(p)?)),
Index::Fm(p) => ("FM", Some(to_json(p)?)),
Index::ZoneMap(p) => ("ZONEMAP", Some(to_json(p)?)),
Index::FTS(p) => {
let mut params = to_json(p)?;
if p.get_document_granularity().is_list_element() {
@@ -6376,6 +6377,7 @@ mod tests {
// HNSW_PQ isn't yet supported on SaaS
("BTREE", json!({}), Index::BTree(Default::default())),
("BITMAP", json!({}), Index::Bitmap(Default::default())),
("ZONEMAP", json!({}), Index::ZoneMap(Default::default())),
(
"LABEL_LIST",
json!({}),
rust/lancedb/src/table/create_index.rs
+56 -1
View File
@@ -259,6 +259,12 @@ impl NativeTable {
BuiltinIndexType::Fm,
)))
}
Index::ZoneMap(_) => {
Self::validate_index_type(field, "ZoneMap", supported_btree_data_type)?;
Ok(Box::new(ScalarIndexParams::for_builtin(
BuiltinIndexType::ZoneMap,
)))
}
Index::FTS(fts_opts) => {
Self::validate_index_type(field, "FTS", supported_fts_data_type)?;
Ok(Box::new(fts_opts))
@@ -418,6 +424,7 @@ impl NativeTable {
Index::Bitmap(_) => IndexType::Bitmap,
Index::LabelList(_) => IndexType::LabelList,
Index::Fm(_) => IndexType::Fm,
Index::ZoneMap(_) => IndexType::ZoneMap,
Index::FTS(_) => IndexType::Inverted,
Index::IvfFlat(_)
| Index::IvfSq(_)
@@ -451,7 +458,8 @@ mod tests {
use crate::connection::ConnectBuilder;
use crate::index::Index;
use crate::index::scalar::{
BTreeIndexBuilder, BitmapIndexBuilder, DocumentGranularity, FmIndexBuilder, FtsIndexBuilder,
BTreeIndexBuilder, BitmapIndexBuilder, DocumentGranularity, FmIndexBuilder,
FtsIndexBuilder, ZoneMapIndexBuilder,
};
use crate::index::vector::{
IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder,
@@ -1197,6 +1205,53 @@ mod tests {
assert_eq!(stats.distance_type, None);
}
#[tokio::test]
async fn test_create_zonemap_index() {
let conn = connect("memory://").execute().await.unwrap();
let batch = record_batch!(("i", Int32, [1, 2, 3, 4, 5])).unwrap();
let table = conn
.create_table("zonemap_table", batch)
.execute()
.await
.unwrap();
table
.create_index(&["i"], Index::ZoneMap(ZoneMapIndexBuilder::default()))
.execute()
.await
.unwrap();
table
.wait_for_index(&["i_idx"], Duration::from_millis(10))
.await
.unwrap();
let index_configs = table.list_indices().await.unwrap();
assert_eq!(index_configs.len(), 1);
let index = index_configs.into_iter().next().unwrap();
assert_eq!(index.index_type, crate::index::IndexType::ZoneMap);
assert_eq!(index.columns, vec!["i".to_string()]);
let count = table
.query()
.only_if("i >= 2 AND i < 5")
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap()
.iter()
.map(|b| b.num_rows())
.sum::<usize>();
assert_eq!(count, 3);
let stats = table.index_stats("i_idx").await.unwrap().unwrap();
assert_eq!(stats.num_indexed_rows, 5);
assert_eq!(stats.num_unindexed_rows, 0);
assert_eq!(stats.index_type, crate::index::IndexType::ZoneMap);
assert_eq!(stats.distance_type, None);
}
#[tokio::test]
async fn test_create_index_nested_field_paths() {
let tmp_dir = tempdir().unwrap();