feat: add table FTS query tokenization (#3659)

## Summary
- add table-level FTS query tokenization returning token text and
position
- use the native index tokenizer for local tables and remote index
metadata for remote tables
- expose sync and async Python table wrappers with focused coverage

python/python/lancedb/__init__.py

@@ -6,11 +6,13 @@ import importlib.metadata
 import os
 from concurrent.futures import ThreadPoolExecutor
 from datetime import timedelta
-from typing import Dict, Optional, Union, Any, List
+from typing import Dict, Optional, Union, Any, List, Iterable
 
 __version__ = importlib.metadata.version("lancedb")
 
 from ._lancedb import connect as lancedb_connect
+from ._lancedb import FtsToken
+from ._lancedb import tokenize as _tokenize
 from .common import URI, sanitize_uri
 from urllib.parse import urlparse
 from .db import AsyncConnection, DBConnection, LanceDBConnection
@@ -19,6 +21,7 @@ from .remote.db import RemoteDBConnection
 from .expr import Expr, col, lit, func
 from .schema import blob, vector, BlobType
 from .table import AsyncTable, Table
+from .types import BaseTokenizerType
 from ._lancedb import Session
 from .namespace import (
     connect_namespace,
@@ -246,6 +249,40 @@ def connect(
     )
 
 
+def tokenize(
+    query: str,
+    *,
+    base_tokenizer: BaseTokenizerType = "simple",
+    language: str = "English",
+    max_token_length: Optional[int] = 40,
+    lower_case: bool = True,
+    stem: bool = True,
+    remove_stop_words: bool = True,
+    ascii_folding: bool = True,
+    ngram_min_length: int = 3,
+    ngram_max_length: int = 3,
+    prefix_only: bool = False,
+) -> Iterable[FtsToken]:
+    """Tokenize a full-text search query using an explicit tokenizer.
+
+    This does not require a table or FTS index. The tokenizer options match
+    :class:`lancedb.index.FTS`.
+    """
+    return _tokenize(
+        query,
+        base_tokenizer=base_tokenizer,
+        language=language,
+        max_token_length=max_token_length,
+        lower_case=lower_case,
+        stem=stem,
+        remove_stop_words=remove_stop_words,
+        ascii_folding=ascii_folding,
+        ngram_min_length=ngram_min_length,
+        ngram_max_length=ngram_max_length,
+        prefix_only=prefix_only,
+    )
+
+
 WORKER_PROPERTY_PREFIX = "_lancedb_worker_"
 
 
@@ -456,11 +493,13 @@ async def connect_async(
 __all__ = [
     "connect",
     "connect_async",
+    "tokenize",
     "connect_namespace",
     "connect_namespace_async",
     "AsyncConnection",
     "AsyncLanceNamespaceDBConnection",
     "AsyncTable",
+    "FtsToken",
     "col",
     "Expr",
     "func",

python/python/lancedb/_lancedb.pyi

@@ -25,6 +25,7 @@ from lance_namespace import (
     ListTablesResponse,
 )
 from .remote import ClientConfig
+from .types import BaseTokenizerType
 
 IvfHnswPq: type[HnswPq] = HnswPq
 IvfHnswSq: type[HnswSq] = HnswSq
@@ -48,6 +49,20 @@ class MetricDescription:
 def register_lancedb_metrics_recorder() -> bool: ...
 def lancedb_metrics_catalog() -> List[MetricDescription]: ...
 def snapshot_lancedb_metrics() -> List[MetricPoint]: ...
+def tokenize(
+    query: str,
+    *,
+    base_tokenizer: BaseTokenizerType = "simple",
+    language: str = "English",
+    max_token_length: Optional[int] = 40,
+    lower_case: bool = True,
+    stem: bool = True,
+    remove_stop_words: bool = True,
+    ascii_folding: bool = True,
+    ngram_min_length: int = 3,
+    ngram_max_length: int = 3,
+    prefix_only: bool = False,
+) -> List["FtsToken"]: ...
 
 class PyExpr:
     """A type-safe DataFusion expression node (Rust-side handle)."""
@@ -238,6 +253,13 @@ class Table:
     async def prewarm_index(self, index_name: str) -> None: ...
     async def prewarm_data(self, columns: Optional[List[str]] = None) -> None: ...
     async def list_indices(self) -> list[IndexConfig]: ...
+    async def tokenize(
+        self,
+        query: str,
+        *,
+        column: Optional[str] = None,
+        index_name: Optional[str] = None,
+    ) -> list[FtsToken]: ...
     async def delete(self, filter: Union[str, PyExpr]) -> DeleteResult: ...
     async def add_columns(self, columns: list[tuple[str, str]]) -> AddColumnsResult: ...
     async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ...
@@ -511,6 +533,10 @@ class MergeResult:
     num_attempts: int
     num_rows: int
 
+class FtsToken:
+    text: str
+    position: int
+
 class LsmWriteSpec:
     """Specification selecting Lance's MemWAL LSM-style write path for
     `merge_insert`."""

python/python/lancedb/index.py

@@ -127,6 +127,8 @@ class FTS:
         - "whitespace": Split text by whitespace, but not punctuation.
         - "raw": No tokenization. The entire text is treated as a single token.
         - "ngram": N-gram tokenizer for substring-style matching.
+        - "icu": ICU dictionary-based word segmentation.
+        - "icu/split": ICU segmentation with simple-style delimiter splitting.
         - "jieba/*": Jieba tokenizer loaded from Lance's language model home.
         - "lindera/*": Lindera tokenizer loaded from Lance's language model home.
     language : str, default "English"

python/python/lancedb/remote/table.py

@@ -28,6 +28,7 @@ from lancedb._lancedb import (
     UpdateFieldMetadataResult,
     DeleteResult,
     DropColumnsResult,
+    FtsToken,
     IndexConfig,
     LsmWriteSpec,
     MergeResult,
@@ -244,6 +245,23 @@ class RemoteTable(Table):
         """List all the indices on the table"""
         return LOOP.run(self._table.list_indices())
 
+    def tokenize(
+        self,
+        query: str,
+        *,
+        column: Optional[str] = None,
+        index_name: Optional[str] = None,
+    ) -> Iterable[FtsToken]:
+        """Tokenize a query using the tokenizer configured on an FTS index.
+
+        Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
+        rebuilt in the client process from index metadata, so the same tokenizer
+        model files must exist locally.
+        """
+        return LOOP.run(
+            self._table.tokenize(query, column=column, index_name=index_name)
+        )
+
     def index_stats(self, index_uuid: str) -> Optional[IndexStatistics]:
         """List all the stats of a specified index"""
         return LOOP.run(self._table.index_stats(index_uuid))

python/python/lancedb/table.py

@@ -173,6 +173,7 @@ if TYPE_CHECKING:
         UpdateFieldMetadataResult,
         DeleteResult,
         DropColumnsResult,
+        FtsToken,
         LsmWriteSpec,
         MergeResult,
         UpdateResult,
@@ -1147,6 +1148,8 @@ class Table(ABC):
             - "whitespace": Split text by whitespace, but not punctuation.
             - "raw": No tokenization. The entire text is treated as a single token.
             - "ngram": N-Gram tokenizer.
+            - "icu": ICU dictionary-based word segmentation.
+            - "icu/split": ICU segmentation with simple-style delimiter splitting.
             - "jieba/*": Jieba tokenizer loaded from Lance's language model home.
             - "lindera/*": Lindera tokenizer loaded from Lance's language model home.
         language : str, default "English"
@@ -1799,6 +1802,24 @@ class Table(ABC):
         [Table.create_index][lancedb.table.Table.create_index]
         """
 
+    @abstractmethod
+    def tokenize(
+        self,
+        query: str,
+        *,
+        column: Optional[str] = None,
+        index_name: Optional[str] = None,
+    ) -> Iterable[FtsToken]:
+        """
+        Tokenize a query using the tokenizer configured on an FTS index.
+
+        Specify exactly one of ``column`` or ``index_name``.
+
+        Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
+        rebuilt in the client process from index metadata. For remote tables,
+        this means the same tokenizer model files must also exist locally.
+        """
+
     @abstractmethod
     def index_stats(self, index_name: str) -> Optional[IndexStatistics]:
         """
@@ -3744,6 +3765,26 @@ class LanceTable(Table):
         """
         return LOOP.run(self._table.list_indices())
 
+    def tokenize(
+        self,
+        query: str,
+        *,
+        column: Optional[str] = None,
+        index_name: Optional[str] = None,
+    ) -> Iterable[FtsToken]:
+        """
+        Tokenize a query using the tokenizer configured on an FTS index.
+
+        Specify exactly one of ``column`` or ``index_name``.
+
+        Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
+        rebuilt in the client process from index metadata. For remote tables,
+        this means the same tokenizer model files must also exist locally.
+        """
+        return LOOP.run(
+            self._table.tokenize(query, column=column, index_name=index_name)
+        )
+
     def index_stats(self, index_name: str) -> Optional[IndexStatistics]:
         """
         Retrieve statistics about an index
@@ -5804,6 +5845,24 @@ class AsyncTable:
         """
         return await self._inner.list_indices()
 
+    async def tokenize(
+        self,
+        query: str,
+        *,
+        column: Optional[str] = None,
+        index_name: Optional[str] = None,
+    ) -> Iterable[FtsToken]:
+        """
+        Tokenize a query using the tokenizer configured on an FTS index.
+
+        Specify exactly one of ``column`` or ``index_name``.
+
+        Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
+        rebuilt in the client process from index metadata. For remote tables,
+        this means the same tokenizer model files must also exist locally.
+        """
+        return await self._inner.tokenize(query, column=column, index_name=index_name)
+
     async def index_stats(self, index_name: str) -> Optional[IndexStatistics]:
         """
         Retrieve statistics about an index

python/python/lancedb/types.py

@@ -55,5 +55,7 @@ IndexType = Literal[
 ]
 
 # Tokenizer literals
-BuiltinTokenizerType = Literal["simple", "raw", "whitespace", "ngram"]
+BuiltinTokenizerType = Literal[
+    "simple", "raw", "whitespace", "ngram", "icu", "icu/split"
+]
 BaseTokenizerType = BuiltinTokenizerType | str

python/python/tests/test_fts.py

@@ -786,6 +786,97 @@ def test_language(mem_db: DBConnection):
     assert len(results) == 0
 
 
+def test_tokenize_uses_simple_index_tokenizer(mem_db: DBConnection):
+    data = pa.table({"text": ["Running in cafés"], "other": ["Running in cafés"]})
+    table = mem_db.create_table("test_tokenize", data=data)
+    table.create_index("text", config=FTS(base_tokenizer="simple"))
+
+    tokens = table.tokenize("Running in cafés", column="text")
+
+    assert [(token.text, token.position) for token in tokens] == [
+        ("run", 0),
+        ("cafe", 2),
+    ]
+
+
+def test_tokenize_uses_icu_index_tokenizer_by_name(mem_db: DBConnection):
+    data = pa.table({"text": ["Hello, こんにちは世界!"]})
+    table = mem_db.create_table("test_tokenize_icu", data=data)
+    table.create_index(
+        "text",
+        config=FTS(
+            base_tokenizer="icu",
+            stem=False,
+            remove_stop_words=False,
+        ),
+        name="text_icu_idx",
+    )
+
+    tokens = table.tokenize("Hello, こんにちは世界!", index_name="text_icu_idx")
+
+    assert [(token.text, token.position) for token in tokens] == [
+        ("hello", 0),
+        ("こんにちは", 1),
+        ("世界", 2),
+    ]
+
+
+def test_tokenize_requires_one_selector(mem_db: DBConnection):
+    data = pa.table({"text": ["hello world"]})
+    table = mem_db.create_table("test_tokenize_selector", data=data)
+    table.create_index("text", config=FTS(), name="text_idx")
+
+    with pytest.raises(ValueError, match="Specify exactly one"):
+        table.tokenize("hello")
+
+    with pytest.raises(ValueError, match="Specify exactly one"):
+        table.tokenize("hello", column="text", index_name="text_idx")
+
+
+def test_tokenize_requires_fts_index(mem_db: DBConnection):
+    data = pa.table({"text": ["hello world"]})
+    table = mem_db.create_table("test_tokenize_no_index", data=data)
+
+    with pytest.raises(ValueError, match="does not have a full text search index"):
+        table.tokenize("hello", column="text")
+
+
+@pytest.mark.asyncio
+async def test_tokenize_async(async_table):
+    await async_table.create_index("text", config=FTS())
+
+    tokens = await async_table.tokenize("Running in cafés", column="text")
+
+    assert [(token.text, token.position) for token in tokens] == [
+        ("run", 0),
+        ("cafe", 2),
+    ]
+
+
+def test_tokenize_uses_explicit_simple_tokenizer():
+    tokens = ldb.tokenize("Running in cafés", base_tokenizer="simple")
+
+    assert [(token.text, token.position) for token in tokens] == [
+        ("run", 0),
+        ("cafe", 2),
+    ]
+
+
+def test_tokenize_uses_explicit_icu_tokenizer():
+    tokens = ldb.tokenize(
+        "Hello, こんにちは世界!",
+        base_tokenizer="icu",
+        stem=False,
+        remove_stop_words=False,
+    )
+
+    assert [(token.text, token.position) for token in tokens] == [
+        ("hello", 0),
+        ("こんにちは", 1),
+        ("世界", 2),
+    ]
+
+
 def test_fts_on_list(mem_db: DBConnection):
     data = pa.table(
         {

python/src/lib.rs

@@ -15,8 +15,8 @@ use pyo3::{
 use query::{FTSQuery, HybridQuery, Query, VectorQuery};
 use session::Session;
 use table::{
-    AddColumnsResult, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, LsmWriteSpec,
-    MergeResult, PyBlobFile, Table, UpdateFieldMetadataResult, UpdateResult,
+    AddColumnsResult, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, FtsToken,
+    LsmWriteSpec, MergeResult, PyBlobFile, Table, UpdateFieldMetadataResult, UpdateResult,
 };
 
 pub mod arrow;
@@ -60,6 +60,7 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<DeleteResult>()?;
     m.add_class::<DropColumnsResult>()?;
     m.add_class::<UpdateResult>()?;
+    m.add_class::<FtsToken>()?;
     m.add_class::<PyAsyncPermutationBuilder>()?;
     m.add_class::<PyPermutationReader>()?;
     m.add_class::<PyExpr>()?;
@@ -75,6 +76,7 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(connect, m)?)?;
     m.add_function(wrap_pyfunction!(connect_namespace, m)?)?;
     m.add_function(wrap_pyfunction!(connect_namespace_client, m)?)?;
+    m.add_function(wrap_pyfunction!(table::tokenize, m)?)?;
     m.add_function(wrap_pyfunction!(permutation::async_permutation_builder, m)?)?;
     m.add_function(wrap_pyfunction!(util::validate_table_name, m)?)?;
     m.add_function(wrap_pyfunction!(query::fts_query_to_json, m)?)?;

python/src/table.rs

@@ -18,14 +18,16 @@ use arrow::{
     pyarrow::{FromPyArrow, PyArrowType, ToPyArrow},
 };
 use lancedb::blob::BlobFile;
+use lancedb::index::scalar::FtsIndexBuilder;
 use lancedb::table::{
-    AddDataMode, ColumnAlteration, Duration, FieldMetadataUpdate, NewColumnTransform,
-    OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable,
+    AddDataMode, ColumnAlteration, Duration, FieldMetadataUpdate, FtsToken as LanceDbFtsToken,
+    NewColumnTransform, OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable,
 };
+use lancedb::tokenize as lancedb_tokenize;
 use pyo3::{
     Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
     exceptions::{PyRuntimeError, PyValueError},
-    pyclass, pymethods,
+    pyclass, pyfunction, pymethods,
     types::{IntoPyDict, PyAnyMethods, PyBytes, PyDict, PyDictMethods},
 };
 
@@ -486,6 +488,78 @@ impl PyBlobFile {
     }
 }
 
+#[pyclass(get_all, from_py_object)]
+#[derive(Clone, Debug)]
+pub struct FtsToken {
+    pub text: String,
+    pub position: u32,
+}
+
+#[pymethods]
+impl FtsToken {
+    pub fn __repr__(&self) -> String {
+        format!("FtsToken(text={:?}, position={})", self.text, self.position)
+    }
+}
+
+impl From<LanceDbFtsToken> for FtsToken {
+    fn from(token: LanceDbFtsToken) -> Self {
+        Self {
+            text: token.text,
+            position: token.position,
+        }
+    }
+}
+
+#[pyfunction(signature = (
+    query,
+    *,
+    base_tokenizer = "simple".to_string(),
+    language = "English".to_string(),
+    max_token_length = Some(40),
+    lower_case = true,
+    stem = true,
+    remove_stop_words = true,
+    ascii_folding = true,
+    ngram_min_length = 3,
+    ngram_max_length = 3,
+    prefix_only = false
+))]
+#[allow(clippy::too_many_arguments)]
+pub fn tokenize(
+    query: String,
+    base_tokenizer: String,
+    language: String,
+    max_token_length: Option<u32>,
+    lower_case: bool,
+    stem: bool,
+    remove_stop_words: bool,
+    ascii_folding: bool,
+    ngram_min_length: u32,
+    ngram_max_length: u32,
+    prefix_only: bool,
+) -> PyResult<Vec<FtsToken>> {
+    let params = FtsIndexBuilder::default()
+        .base_tokenizer(base_tokenizer)
+        .language(&language)
+        .map_err(|_| {
+            PyValueError::new_err(format!(
+                "LanceDB does not support the requested language: '{}'",
+                language
+            ))
+        })?
+        .max_token_length(max_token_length.map(|value| value as usize))
+        .lower_case(lower_case)
+        .stem(stem)
+        .remove_stop_words(remove_stop_words)
+        .ascii_folding(ascii_folding)
+        .ngram_min_length(ngram_min_length)
+        .ngram_max_length(ngram_max_length)
+        .ngram_prefix_only(prefix_only);
+    let tokens = lancedb_tokenize(&query, &params).infer_error()?;
+    Ok(tokens.into_iter().map(FtsToken::from).collect())
+}
+
 #[pyclass]
 pub struct Table {
     // We keep a copy of the name to use if the inner table is dropped
@@ -784,6 +858,29 @@ impl Table {
         })
     }
 
+    #[pyo3(signature = (query, *, column=None, index_name=None))]
+    pub fn tokenize(
+        self_: PyRef<'_, Self>,
+        query: String,
+        column: Option<String>,
+        index_name: Option<String>,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            let tokens = match (column.as_deref(), index_name.as_deref()) {
+                (Some(_), Some(_)) | (None, None) => {
+                    return Err(PyValueError::new_err(
+                        "Specify exactly one of 'column' or 'index_name'",
+                    ));
+                }
+                (Some(column), None) => inner.tokenize_with_column(&query, column).await,
+                (None, Some(index_name)) => inner.tokenize(&query, index_name).await,
+            }
+            .infer_error()?;
+            Ok(tokens.into_iter().map(FtsToken::from).collect::<Vec<_>>())
+        })
+    }
+
     pub fn index_stats(self_: PyRef<'_, Self>, index_name: String) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {