feat(python): expose the NGram scalar index

## What kind of change does this PR introduce?

Feature. Exposes the NGram scalar index in the Python SDK.

## What is the current behavior?

The NGram scalar index already exists in the Rust core (`Index::NGram`) and is accepted over the remote protocol (the `"NGRAM"` type), but the Python SDK has no way to create it. `lancedb.index` offers `BTree`, `Bitmap`, `LabelList`, `Fm`, and the vector indexes, but not `NGram`, so it cannot be built through `create_index(config=...)`.

## What is the new behavior?

Adds an `NGram` config to `lancedb.index` and wires it through the pyo3 binding:

```python
from lancedb.index import NGram

await table.create_index("text", config=NGram())
```

The NGram index accelerates substring filters such as `contains(col, 'needle')`, along with some `LIKE` and regular-expression predicates, on UTF-8 string columns. It is parameter free and uses Lance's default trigram parameters.

Changes:

- `lancedb.index`: new `NGram` dataclass, added to `__all__`.
- pyo3 binding (`python/src/index.rs`): a match arm mapping `NGram` to `LanceDbIndex::NGram`, plus the import and the error message.
- Type hints: `NGram` added to the `create_index` config unions and the isinstance check in `table.py`, the `IndexConfigType` union, and the `_lancedb.pyi` stubs; `"NGRAM"` added to the `IndexType` and `IndexStatistics.index_type` literals.
- Test: `test_create_ngram_index` covers creation, `list_indices`, `index_stats`, and that a substring filter uses the index.
- Docs: an NGram section in the scalar index guide.

This mirrors the approach used for the RTree index in https://github.com/lancedb/lancedb/pull/4555 (that PR keeps the RTree scope; this one adds the remaining parameter free string index).

python/python/lancedb/_lancedb.pyi

@@ -13,6 +13,7 @@ from .index import (
     Bitmap,
     LabelList,
     Fm,
+    NGram,
     HnswPq,
     HnswSq,
     HnswFlat,
@@ -497,6 +498,7 @@ class Table:
             Bitmap,
             LabelList,
             Fm,
+            NGram,
             FTS,
         ],
         replace: Optional[bool],
@@ -519,6 +521,7 @@ class Table:
             Bitmap,
             LabelList,
             Fm,
+            NGram,
             FTS,
         ],
         replace: Optional[bool],

python/python/lancedb/index.py

@@ -108,6 +108,28 @@ class Fm:
     pass
 
 
+@dataclass
+class NGram:
+    """Describe an NGram index configuration.
+
+    An `NGram` index is a scalar index on a UTF-8 string column that
+    accelerates substring filters such as `contains(col, 'needle')`, along
+    with some `LIKE` and regular-expression predicates. It tokenizes values
+    into overlapping character n-grams (using Lance's default trigram
+    parameters) and narrows a scan to the rows that share the query's n-grams;
+    the exact predicate is then evaluated on those candidates, so the results
+    are the same with or without the index.
+
+    Unlike the `Fm` index, which matches arbitrary substrings of the raw
+    bytes, `NGram` is trigram based, so a query shorter than the n-gram length
+    cannot be served by the index.
+
+    For example, it works with `text`, `description`, `comment`, etc.
+    """
+
+    pass
+
+
 @dataclass
 class FTS:
     """Describe a FTS index configuration.
@@ -930,4 +952,5 @@ __all__ = [
     "Bitmap",
     "LabelList",
     "Fm",
+    "NGram",
 ]

python/python/lancedb/table.py

@@ -71,6 +71,7 @@ from .index import (
     IvfRq,
     LabelList,
     Fm,
+    NGram,
     HnswPq,
     HnswSq,
     HnswFlat,
@@ -284,6 +285,7 @@ IndexConfigType = Union[
     Bitmap,
     LabelList,
     Fm,
+    NGram,
     FTS,
 ]
 
@@ -1883,7 +1885,7 @@ class Table(ABC):
         config : IndexConfigType, optional
             The index configuration object. If provided, uses the new unified API.
             Can be one of: IvfFlat, IvfPq, IvfSq, IvfRq, HnswPq, HnswSq,
-            BTree, Bitmap, LabelList, Fm, FTS.
+            BTree, Bitmap, LabelList, Fm, NGram, FTS.
         replace : bool, default True
             Whether to replace an existing index on this column.
         wait_timeout : timedelta, optional
@@ -4022,7 +4024,7 @@ class LanceTable(Table):
         config : IndexConfigType, optional
             The index configuration object. If provided, uses the new unified API.
             Can be one of: IvfFlat, IvfPq, IvfSq, IvfRq, HnswPq, HnswSq,
-            BTree, Bitmap, LabelList, Fm, FTS.
+            BTree, Bitmap, LabelList, Fm, NGram, FTS.
         replace : bool, default True
             Whether to replace an existing index on this column.
         wait_timeout : timedelta, optional
@@ -6320,6 +6322,7 @@ class AsyncTable:
                 Bitmap,
                 LabelList,
                 Fm,
+                NGram,
                 FTS,
             ]
         ] = None,
@@ -6373,12 +6376,13 @@ class AsyncTable:
                     Bitmap,
                     LabelList,
                     Fm,
+                    NGram,
                     FTS,
                 ),
             ):
                 raise TypeError(
                     "config must be an instance of IvfSq, IvfPq, IvfRq, HnswPq, HnswSq,"
-                    " BTree, Bitmap, LabelList, Fm, or FTS, but got "
+                    " BTree, Bitmap, LabelList, Fm, NGram, or FTS, but got "
                     + str(type(config))
                 )
         try:
@@ -6416,6 +6420,7 @@ class AsyncTable:
                 Bitmap,
                 LabelList,
                 Fm,
+                NGram,
                 FTS,
             ]
         ] = None,
@@ -8084,6 +8089,7 @@ class IndexStatistics:
         "BTREE",
         "BITMAP",
         "LABEL_LIST",
+        "NGRAM",
     ]
     distance_type: Optional[Literal["l2", "cosine", "dot"]] = None
     num_indices: Optional[int] = None

python/python/lancedb/types.py

@@ -51,6 +51,7 @@ IndexType = Literal[
     "BTREE",
     "BITMAP",
     "LABEL_LIST",
+    "NGRAM",
     "IVF_RQ",
 ]
 

python/python/tests/test_index.py

@@ -21,6 +21,7 @@ from lancedb.index import (
     Bitmap,
     LabelList,
     Fm,
+    NGram,
     HnswPq,
     HnswSq,
     HnswFlat,
@@ -248,6 +249,36 @@ async def test_create_fm_index(some_table: AsyncTable):
     assert indices[0].columns == ["data"]
 
 
+@pytest.mark.asyncio
+async def test_create_ngram_index(db_async):
+    # NGram indexes accelerate substring filters on UTF-8 string columns.
+    data = pa.table(
+        {
+            "id": list(range(5)),
+            "text": ["cat", "category", "concatenate", "dog", "cathedral"],
+        }
+    )
+    table = await db_async.create_table("ngram_table", data)
+    await table.create_index("text", config=NGram())
+
+    indices = await table.list_indices()
+    assert len(indices) == 1
+    assert indices[0].index_type == "NGram"
+    assert indices[0].columns == ["text"]
+
+    stats = await table.index_stats(indices[0].name)
+    assert stats.index_type == "NGRAM"
+    assert stats.num_indexed_rows == await table.count_rows()
+    assert stats.num_unindexed_rows == 0
+
+    # The index accelerates substring predicates without changing their results.
+    result = await table.query().where("contains(text, 'cat')").to_arrow()
+    assert sorted(result["id"].to_pylist()) == [0, 1, 2, 4]
+
+    plan = await table.query().where("contains(text, 'cat')").explain_plan()
+    assert "ScalarIndexQuery" in plan
+
+
 @pytest.mark.asyncio
 async def test_create_bitmap_index(some_table: AsyncTable):
     await some_table.create_index("id", config=Bitmap())
@@ -559,6 +590,7 @@ def test_index_statistics_index_type_lists_all_supported_values():
         "BTREE",
         "BITMAP",
         "LABEL_LIST",
+        "NGRAM",
     }
 
     assert (

python/src/index.rs

@@ -8,7 +8,9 @@ use lancedb::index::vector::{
 };
 use lancedb::index::{
     Index as LanceDbIndex,
-    scalar::{BTreeIndexBuilder, DocumentGranularity, FmIndexBuilder, FtsIndexBuilder},
+    scalar::{
+        BTreeIndexBuilder, DocumentGranularity, FmIndexBuilder, FtsIndexBuilder, NGramIndexBuilder,
+    },
 };
 use pyo3::IntoPyObject;
 use pyo3::types::PyStringMethods;
@@ -40,6 +42,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
             "Bitmap" => Ok(LanceDbIndex::Bitmap(Default::default())),
             "LabelList" => Ok(LanceDbIndex::LabelList(Default::default())),
             "Fm" => Ok(LanceDbIndex::Fm(FmIndexBuilder::default())),
+            "NGram" => Ok(LanceDbIndex::NGram(NGramIndexBuilder::default())),
             "FTS" => {
                 let params = source.extract::<FtsParams>()?;
                 let mut inner_opts = FtsIndexBuilder::default()
@@ -205,7 +208,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                 Ok(LanceDbIndex::IvfHnswFlat(hnsw_flat_builder))
             }
             not_supported => Err(PyValueError::new_err(format!(
-                "Invalid index type '{}'.  Must be one of BTree, Bitmap, LabelList, Fm, FTS, IvfPq, IvfSq, IvfHnswPq, IvfHnswSq, or IvfHnswFlat",
+                "Invalid index type '{}'.  Must be one of BTree, Bitmap, LabelList, Fm, NGram, FTS, IvfPq, IvfSq, IvfHnswPq, IvfHnswSq, or IvfHnswFlat",
                 not_supported
             ))),
         }