From a9146d536a86f5c80c61ba30257b5d717a3f2cb3 Mon Sep 17 00:00:00 2001 From: BHAWESHBHASKAR <90614484+BHAWESHBHASKAR@users.noreply.github.com> Date: Sun, 1 Nov 2026 09:24:27 +0530 Subject: [PATCH] feat(python): expose the NGram scalar index ## What kind of change does this PR introduce? Feature. Exposes the NGram scalar index in the Python SDK. ## What is the current behavior? The NGram scalar index already exists in the Rust core (`Index::NGram`) and is accepted over the remote protocol (the `"NGRAM"` type), but the Python SDK has no way to create it. `lancedb.index` offers `BTree`, `Bitmap`, `LabelList`, `Fm`, and the vector indexes, but not `NGram`, so it cannot be built through `create_index(config=...)`. ## What is the new behavior? Adds an `NGram` config to `lancedb.index` and wires it through the pyo3 binding: ```python from lancedb.index import NGram await table.create_index("text", config=NGram()) ``` The NGram index accelerates substring filters such as `contains(col, 'needle')`, along with some `LIKE` and regular-expression predicates, on UTF-8 string columns. It is parameter free and uses Lance's default trigram parameters. Changes: - `lancedb.index`: new `NGram` dataclass, added to `__all__`. - pyo3 binding (`python/src/index.rs`): a match arm mapping `NGram` to `LanceDbIndex::NGram`, plus the import and the error message. - Type hints: `NGram` added to the `create_index` config unions and the isinstance check in `table.py`, the `IndexConfigType` union, and the `_lancedb.pyi` stubs; `"NGRAM"` added to the `IndexType` and `IndexStatistics.index_type` literals. - Test: `test_create_ngram_index` covers creation, `list_indices`, `index_stats`, and that a substring filter uses the index. - Docs: an NGram section in the scalar index guide. This mirrors the approach used for the RTree index in https://github.com/lancedb/lancedb/pull/4555 (that PR keeps the RTree scope; this one adds the remaining parameter free string index). --- docs/web/indexing/index.mdx | 1 + docs/web/indexing/scalar-index.mdx | 43 +++++++++++++++++++++++++++++- python/python/lancedb/_lancedb.pyi | 3 +++ python/python/lancedb/index.py | 23 ++++++++++++++++ python/python/lancedb/table.py | 12 ++++++--- python/python/lancedb/types.py | 1 + python/python/tests/test_index.py | 32 ++++++++++++++++++++++ python/src/index.rs | 7 +++-- 8 files changed, 116 insertions(+), 6 deletions(-) diff --git a/docs/web/indexing/index.mdx b/docs/web/indexing/index.mdx index df8109f5b..4b91f261d 100644 --- a/docs/web/indexing/index.mdx +++ b/docs/web/indexing/index.mdx @@ -26,6 +26,7 @@ patterns. LanceDB supports three main index families: | `Bitmap` (Scalar) | Low-cardinality columns with few thousand or fewer distinct values. Accelerates equality and range filters. | Stores a bitmap for each distinct value in the column, with one bit per row indicating presence. Memory-efficient for low-cardinality data. | | `LabelList` (Scalar) | List columns (e.g., tags, categories, keywords) requiring `array_has_all` or `array_has_any` filters. | Scalar index for `List` and `LargeList` columns of primitive values, using an underlying bitmap index structure to enable fast array membership lookups. | | `FM` (Scalar) | String or binary columns that need raw substring search. | FM-Index over `Utf8`, `LargeUtf8`, `Binary`, or `LargeBinary` data for filters such as `contains(path, 'needle')`. Use FTS instead for tokenized word search and BM25 ranking. | +| `NGram` (Scalar) | String columns that need substring search using character trigrams. | Trigram index over `Utf8` and `LargeUtf8` columns that narrows `contains(col, 'needle')` and some `LIKE` and regular expression predicates to rows sharing the query's trigrams, then rechecks the predicate. A query shorter than the trigram length cannot use it; use `FM` for arbitrary length substrings or `FTS` for tokenized word search. | A deployment builds the vector indexes `IVF_FLAT`, `IVF_PQ`, `IVF_RQ` and `IVF_HNSW_SQ`. `IVF_SQ`, `IVF_HNSW_FLAT` and `IVF_HNSW_PQ` are built only by the diff --git a/docs/web/indexing/scalar-index.mdx b/docs/web/indexing/scalar-index.mdx index 5c483111b..b7d3ef38a 100644 --- a/docs/web/indexing/scalar-index.mdx +++ b/docs/web/indexing/scalar-index.mdx @@ -26,12 +26,13 @@ Scalar indexes accelerate filters on values such as numbers, categories and strings. Use them for selective table queries or metadata filters combined with vector and full-text search. -LanceDB supports four types of scalar indexes: +LanceDB supports five types of scalar indexes: - `BTREE`: Stores column data in sorted order for binary search. Best for columns with many unique values. - `BITMAP`: Uses bitmaps to track value presence. Ideal for columns with few unique values (e.g., categories, tags). - `LABEL_LIST`: Special index for `List` and `LargeList` columns of primitive values supporting `array_has_all` and `array_has_any` queries. - `FM`: FM-Index over string or binary columns that accelerates substring search via `contains(col, 'needle')`. +- `NGRAM`: Trigram index over UTF-8 string columns that accelerates substring search via `contains(col, 'needle')`, along with some `LIKE` and regular-expression filters. Use scalar indexes with: @@ -50,6 +51,7 @@ Use scalar indexes with: | Boolean, numbers or strings with fewer than 1,000 unique values | `<`, `=`, `>`, `in`, `between`, `is null` | `BITMAP` | | List of low cardinality of numbers or strings | `array_has_any`, `array_has_all` | `LABEL_LIST` | | String or binary (`Utf8`, `LargeUtf8`, `Binary`, `LargeBinary`) | `contains(col, 'needle')` | `FM` | +| String (`Utf8`, `LargeUtf8`) | `contains(col, 'needle')`, `LIKE`, regex | `NGRAM` | ## Scalar Index Operations @@ -167,6 +169,45 @@ table.search().where("contains(text, 'needle')").limit(10).to_pandas() `list_indices()` reports the index type as `"Fm"`. +## NGram index for substring search + +The `NGram` index is a scalar index over UTF-8 string columns. It tokenizes +values into overlapping character n-grams (using Lance's default trigram +parameters) and uses them to accelerate `contains(col, 'needle')`, along with +some `LIKE` and regular-expression filters. Unlike the `FM` index, which matches +arbitrary substrings of the raw bytes, the `NGram` index is trigram based, so a +query shorter than the n-gram length cannot be served by it. + +### Create an NGram index + +Build an NGram index with the async `create_index` API by passing the `NGram` +config in Python. In Rust, use `Index::NGram(NGramIndexBuilder::default())`. + + +```python Python icon="python" +from lancedb.index import NGram + +await tbl.create_index("text", config=NGram()) +``` + +```rust Rust icon="rust" +use lancedb::index::Index; +use lancedb::index::scalar::NGramIndexBuilder; + +tbl.create_index(&["text"], Index::NGram(NGramIndexBuilder::default())) + .execute() + .await?; +``` + + +After the index is built, substring filters use it automatically: + +```python +table.search().where("contains(text, 'needle')").limit(10).to_pandas() +``` + +`list_indices()` reports the index type as `"NGram"`. + ## Index UUID Columns LanceDB supports scalar indexes on UUID columns (stored as `FixedSizeBinary(16)`), enabling efficient lookups and filtering on UUID-based primary keys. diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 9dfed229d..36cf36ddf 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -13,6 +13,7 @@ from .index import ( Bitmap, LabelList, Fm, + NGram, HnswPq, HnswSq, HnswFlat, @@ -497,6 +498,7 @@ class Table: Bitmap, LabelList, Fm, + NGram, FTS, ], replace: Optional[bool], @@ -519,6 +521,7 @@ class Table: Bitmap, LabelList, Fm, + NGram, FTS, ], replace: Optional[bool], diff --git a/python/python/lancedb/index.py b/python/python/lancedb/index.py index 01c8ea3f5..d4c7f8c8c 100644 --- a/python/python/lancedb/index.py +++ b/python/python/lancedb/index.py @@ -108,6 +108,28 @@ class Fm: pass +@dataclass +class NGram: + """Describe an NGram index configuration. + + An `NGram` index is a scalar index on a UTF-8 string column that + accelerates substring filters such as `contains(col, 'needle')`, along + with some `LIKE` and regular-expression predicates. It tokenizes values + into overlapping character n-grams (using Lance's default trigram + parameters) and narrows a scan to the rows that share the query's n-grams; + the exact predicate is then evaluated on those candidates, so the results + are the same with or without the index. + + Unlike the `Fm` index, which matches arbitrary substrings of the raw + bytes, `NGram` is trigram based, so a query shorter than the n-gram length + cannot be served by the index. + + For example, it works with `text`, `description`, `comment`, etc. + """ + + pass + + @dataclass class FTS: """Describe a FTS index configuration. @@ -930,4 +952,5 @@ __all__ = [ "Bitmap", "LabelList", "Fm", + "NGram", ] diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 1a10f43b3..7d82409ef 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -71,6 +71,7 @@ from .index import ( IvfRq, LabelList, Fm, + NGram, HnswPq, HnswSq, HnswFlat, @@ -284,6 +285,7 @@ IndexConfigType = Union[ Bitmap, LabelList, Fm, + NGram, FTS, ] @@ -1883,7 +1885,7 @@ class Table(ABC): config : IndexConfigType, optional The index configuration object. If provided, uses the new unified API. Can be one of: IvfFlat, IvfPq, IvfSq, IvfRq, HnswPq, HnswSq, - BTree, Bitmap, LabelList, Fm, FTS. + BTree, Bitmap, LabelList, Fm, NGram, FTS. replace : bool, default True Whether to replace an existing index on this column. wait_timeout : timedelta, optional @@ -4022,7 +4024,7 @@ class LanceTable(Table): config : IndexConfigType, optional The index configuration object. If provided, uses the new unified API. Can be one of: IvfFlat, IvfPq, IvfSq, IvfRq, HnswPq, HnswSq, - BTree, Bitmap, LabelList, Fm, FTS. + BTree, Bitmap, LabelList, Fm, NGram, FTS. replace : bool, default True Whether to replace an existing index on this column. wait_timeout : timedelta, optional @@ -6320,6 +6322,7 @@ class AsyncTable: Bitmap, LabelList, Fm, + NGram, FTS, ] ] = None, @@ -6373,12 +6376,13 @@ class AsyncTable: Bitmap, LabelList, Fm, + NGram, FTS, ), ): raise TypeError( "config must be an instance of IvfSq, IvfPq, IvfRq, HnswPq, HnswSq," - " BTree, Bitmap, LabelList, Fm, or FTS, but got " + " BTree, Bitmap, LabelList, Fm, NGram, or FTS, but got " + str(type(config)) ) try: @@ -6416,6 +6420,7 @@ class AsyncTable: Bitmap, LabelList, Fm, + NGram, FTS, ] ] = None, @@ -8084,6 +8089,7 @@ class IndexStatistics: "BTREE", "BITMAP", "LABEL_LIST", + "NGRAM", ] distance_type: Optional[Literal["l2", "cosine", "dot"]] = None num_indices: Optional[int] = None diff --git a/python/python/lancedb/types.py b/python/python/lancedb/types.py index 567eecb4e..3d7168dcf 100644 --- a/python/python/lancedb/types.py +++ b/python/python/lancedb/types.py @@ -51,6 +51,7 @@ IndexType = Literal[ "BTREE", "BITMAP", "LABEL_LIST", + "NGRAM", "IVF_RQ", ] diff --git a/python/python/tests/test_index.py b/python/python/tests/test_index.py index a1ec1e04b..1b252f993 100644 --- a/python/python/tests/test_index.py +++ b/python/python/tests/test_index.py @@ -21,6 +21,7 @@ from lancedb.index import ( Bitmap, LabelList, Fm, + NGram, HnswPq, HnswSq, HnswFlat, @@ -248,6 +249,36 @@ async def test_create_fm_index(some_table: AsyncTable): assert indices[0].columns == ["data"] +@pytest.mark.asyncio +async def test_create_ngram_index(db_async): + # NGram indexes accelerate substring filters on UTF-8 string columns. + data = pa.table( + { + "id": list(range(5)), + "text": ["cat", "category", "concatenate", "dog", "cathedral"], + } + ) + table = await db_async.create_table("ngram_table", data) + await table.create_index("text", config=NGram()) + + indices = await table.list_indices() + assert len(indices) == 1 + assert indices[0].index_type == "NGram" + assert indices[0].columns == ["text"] + + stats = await table.index_stats(indices[0].name) + assert stats.index_type == "NGRAM" + assert stats.num_indexed_rows == await table.count_rows() + assert stats.num_unindexed_rows == 0 + + # The index accelerates substring predicates without changing their results. + result = await table.query().where("contains(text, 'cat')").to_arrow() + assert sorted(result["id"].to_pylist()) == [0, 1, 2, 4] + + plan = await table.query().where("contains(text, 'cat')").explain_plan() + assert "ScalarIndexQuery" in plan + + @pytest.mark.asyncio async def test_create_bitmap_index(some_table: AsyncTable): await some_table.create_index("id", config=Bitmap()) @@ -559,6 +590,7 @@ def test_index_statistics_index_type_lists_all_supported_values(): "BTREE", "BITMAP", "LABEL_LIST", + "NGRAM", } assert ( diff --git a/python/src/index.rs b/python/src/index.rs index c6c3fe17d..d80f8d282 100644 --- a/python/src/index.rs +++ b/python/src/index.rs @@ -8,7 +8,9 @@ use lancedb::index::vector::{ }; use lancedb::index::{ Index as LanceDbIndex, - scalar::{BTreeIndexBuilder, DocumentGranularity, FmIndexBuilder, FtsIndexBuilder}, + scalar::{ + BTreeIndexBuilder, DocumentGranularity, FmIndexBuilder, FtsIndexBuilder, NGramIndexBuilder, + }, }; use pyo3::IntoPyObject; use pyo3::types::PyStringMethods; @@ -40,6 +42,7 @@ pub fn extract_index_params(source: &Option>) -> PyResult Ok(LanceDbIndex::Bitmap(Default::default())), "LabelList" => Ok(LanceDbIndex::LabelList(Default::default())), "Fm" => Ok(LanceDbIndex::Fm(FmIndexBuilder::default())), + "NGram" => Ok(LanceDbIndex::NGram(NGramIndexBuilder::default())), "FTS" => { let params = source.extract::()?; let mut inner_opts = FtsIndexBuilder::default() @@ -205,7 +208,7 @@ pub fn extract_index_params(source: &Option>) -> PyResult Err(PyValueError::new_err(format!( - "Invalid index type '{}'. Must be one of BTree, Bitmap, LabelList, Fm, FTS, IvfPq, IvfSq, IvfHnswPq, IvfHnswSq, or IvfHnswFlat", + "Invalid index type '{}'. Must be one of BTree, Bitmap, LabelList, Fm, NGram, FTS, IvfPq, IvfSq, IvfHnswPq, IvfHnswSq, or IvfHnswFlat", not_supported ))), }