diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 801035391..aec0f7fb1 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -25,6 +25,7 @@ from .index import ( LabelList, Fm, NGram, + ZoneMap, HnswPq, HnswSq, HnswFlat, @@ -553,6 +554,7 @@ class Table: LabelList, Fm, NGram, + ZoneMap, FTS, ], replace: Optional[bool], @@ -576,6 +578,7 @@ class Table: LabelList, Fm, NGram, + ZoneMap, FTS, ], replace: Optional[bool], diff --git a/python/python/lancedb/index.py b/python/python/lancedb/index.py index 9f53f203b..f1626c540 100644 --- a/python/python/lancedb/index.py +++ b/python/python/lancedb/index.py @@ -130,6 +130,23 @@ class NGram: pass +@dataclass +class ZoneMap: + """Describe a ZoneMap index configuration. + + A `ZoneMap` index stores compact min and max summaries for contiguous + ranges (zones) of rows. During a scan it skips any zone whose summary + cannot satisfy the predicate, so it accelerates range and equality + filters such as `col > 10`, `col BETWEEN a AND b`, and `col = x`, + without changing the results. + + It suits large numeric or temporal columns whose values are correlated + with row order (for example `timestamp`, `price`, `id`). + """ + + pass + + @dataclass class FTS: """Describe a FTS index configuration. diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 06bace5b8..bf13cc7bd 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -75,6 +75,7 @@ from .index import ( LabelList, Fm, NGram, + ZoneMap, HnswPq, HnswSq, HnswFlat, @@ -300,6 +301,7 @@ IndexConfigType = Union[ LabelList, Fm, NGram, + ZoneMap, FTS, ] @@ -1907,7 +1909,7 @@ class Table(ABC): config : IndexConfigType, optional The index configuration object. If provided, uses the new unified API. Can be one of: IvfFlat, IvfPq, IvfSq, IvfRq, HnswPq, HnswSq, - BTree, Bitmap, LabelList, Fm, NGram, FTS. + BTree, Bitmap, LabelList, Fm, NGram, ZoneMap, FTS. replace : bool, default True Whether to replace an existing index on this column. wait_timeout : timedelta, optional @@ -4285,7 +4287,7 @@ class LanceTable(Table): config : IndexConfigType, optional The index configuration object. If provided, uses the new unified API. Can be one of: IvfFlat, IvfPq, IvfSq, IvfRq, HnswPq, HnswSq, - BTree, Bitmap, LabelList, Fm, NGram, FTS. + BTree, Bitmap, LabelList, Fm, NGram, ZoneMap, FTS. replace : bool, default True Whether to replace an existing index on this column. wait_timeout : timedelta, optional @@ -6662,6 +6664,7 @@ class AsyncTable: LabelList, Fm, NGram, + ZoneMap, FTS, ] ] = None, @@ -6738,12 +6741,13 @@ class AsyncTable: LabelList, Fm, NGram, + ZoneMap, FTS, ), ): raise TypeError( "config must be an instance of IvfSq, IvfPq, IvfRq, HnswPq, HnswSq," - " BTree, Bitmap, LabelList, Fm, NGram, or FTS, but got " + " BTree, Bitmap, LabelList, Fm, NGram, ZoneMap, or FTS, but got " + str(type(config)) ) try: @@ -6782,6 +6786,7 @@ class AsyncTable: LabelList, Fm, NGram, + ZoneMap, FTS, ] ] = None, @@ -8741,6 +8746,7 @@ class IndexStatistics: "BITMAP", "LABEL_LIST", "NGRAM", + "ZONEMAP", ] distance_type: Optional[Literal["l2", "cosine", "dot"]] = None num_indices: Optional[int] = None diff --git a/python/python/tests/test_index.py b/python/python/tests/test_index.py index 1b252f993..738a791ad 100644 --- a/python/python/tests/test_index.py +++ b/python/python/tests/test_index.py @@ -22,6 +22,7 @@ from lancedb.index import ( LabelList, Fm, NGram, + ZoneMap, HnswPq, HnswSq, HnswFlat, @@ -279,6 +280,28 @@ async def test_create_ngram_index(db_async): assert "ScalarIndexQuery" in plan +@pytest.mark.asyncio +async def test_create_zonemap_index(db_async): + # ZoneMap indexes accelerate range and equality filters on ordered columns. + data = pa.table( + { + "id": list(range(100)), + "value": list(range(100)), + } + ) + table = await db_async.create_table("zonemap_table", data) + await table.create_index("value", config=ZoneMap()) + + indices = await table.list_indices() + assert len(indices) == 1 + assert indices[0].index_type == "ZoneMap" + assert indices[0].columns == ["value"] + + # The index narrows a range scan without changing its results. + result = await table.query().where("value >= 90").to_arrow() + assert sorted(result["id"].to_pylist()) == list(range(90, 100)) + + @pytest.mark.asyncio async def test_create_bitmap_index(some_table: AsyncTable): await some_table.create_index("id", config=Bitmap()) @@ -591,6 +614,7 @@ def test_index_statistics_index_type_lists_all_supported_values(): "BITMAP", "LABEL_LIST", "NGRAM", + "ZONEMAP", } assert ( diff --git a/python/src/index.rs b/python/src/index.rs index 3ec03b6fd..039440211 100644 --- a/python/src/index.rs +++ b/python/src/index.rs @@ -10,6 +10,7 @@ use lancedb::index::{ Index as LanceDbIndex, scalar::{ BTreeIndexBuilder, DocumentGranularity, FmIndexBuilder, FtsIndexBuilder, NGramIndexBuilder, + ZoneMapIndexBuilder, }, }; use pyo3::IntoPyObject; @@ -117,6 +118,7 @@ pub fn extract_index_params(source: &Option>) -> PyResult Ok(LanceDbIndex::LabelList(Default::default())), "Fm" => Ok(LanceDbIndex::Fm(FmIndexBuilder::default())), "NGram" => Ok(LanceDbIndex::NGram(NGramIndexBuilder::default())), + "ZoneMap" => Ok(LanceDbIndex::ZoneMap(ZoneMapIndexBuilder::default())), "FTS" => { let params = source.extract::()?; let code_options = CodeAnalyzerOptions { @@ -291,7 +293,7 @@ pub fn extract_index_params(source: &Option>) -> PyResult Err(PyValueError::new_err(format!( - "Invalid index type '{}'. Must be one of BTree, Bitmap, LabelList, Fm, NGram, FTS, IvfPq, IvfSq, IvfHnswPq, IvfHnswSq, or IvfHnswFlat", + "Invalid index type '{}'. Must be one of BTree, Bitmap, LabelList, Fm, NGram, ZoneMap, FTS, IvfPq, IvfSq, IvfHnswPq, IvfHnswSq, or IvfHnswFlat", not_supported ))), }