feat: add table FTS query tokenization (#3659)

## Summary
- add table-level FTS query tokenization returning token text and
position
- use the native index tokenizer for local tables and remote index
metadata for remote tables
- expose sync and async Python table wrappers with focused coverage

nodejs/src/index.rs

@@ -9,8 +9,11 @@ use lancedb::index::vector::{
     IvfFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder,
     IvfRqIndexBuilder,
 };
+use lancedb::tokenize as lancedb_tokenize;
 use napi_derive::napi;
 
+use crate::error::NapiErrorExt;
+use crate::table::FtsToken;
 use crate::util::parse_distance_type;
 
 #[napi]
@@ -30,6 +33,65 @@ impl Index {
     }
 }
 
+#[napi(catch_unwind)]
+#[allow(dead_code, clippy::too_many_arguments)]
+pub fn tokenize(
+    query: String,
+    base_tokenizer: Option<String>,
+    language: Option<String>,
+    max_token_length: Option<u32>,
+    lower_case: Option<bool>,
+    stem: Option<bool>,
+    remove_stop_words: Option<bool>,
+    ascii_folding: Option<bool>,
+    ngram_min_length: Option<u32>,
+    ngram_max_length: Option<u32>,
+    prefix_only: Option<bool>,
+) -> napi::Result<Vec<FtsToken>> {
+    let mut opts = FtsIndexBuilder::default();
+    if let Some(base_tokenizer) = base_tokenizer {
+        opts = opts.base_tokenizer(base_tokenizer);
+    }
+    if let Some(language) = language {
+        opts = opts.language(&language).map_err(|_| {
+            napi::Error::from_reason(format!(
+                "LanceDB does not support the requested language: '{}'",
+                language
+            ))
+        })?;
+    }
+    if let Some(max_token_length) = max_token_length {
+        opts = opts.max_token_length(Some(max_token_length as usize));
+    }
+    if let Some(lower_case) = lower_case {
+        opts = opts.lower_case(lower_case);
+    }
+    if let Some(stem) = stem {
+        opts = opts.stem(stem);
+    }
+    if let Some(remove_stop_words) = remove_stop_words {
+        opts = opts.remove_stop_words(remove_stop_words);
+    }
+    if let Some(ascii_folding) = ascii_folding {
+        opts = opts.ascii_folding(ascii_folding);
+    }
+    if let Some(ngram_min_length) = ngram_min_length {
+        opts = opts.ngram_min_length(ngram_min_length);
+    }
+    if let Some(ngram_max_length) = ngram_max_length {
+        opts = opts.ngram_max_length(ngram_max_length);
+    }
+    if let Some(prefix_only) = prefix_only {
+        opts = opts.ngram_prefix_only(prefix_only);
+    }
+
+    Ok(lancedb_tokenize(&query, &opts)
+        .default_error()?
+        .into_iter()
+        .map(FtsToken::from)
+        .collect())
+}
+
 #[napi]
 impl Index {
     #[napi(factory)]

nodejs/src/table.rs

@@ -8,8 +8,8 @@ use chrono::{DateTime, Utc};
 use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
 use lancedb::table::{
     AddDataMode, ColumnAlteration as LanceColumnAlteration, Duration,
-    FieldMetadataUpdate as LanceFieldMetadataUpdate, NewColumnTransform, OptimizeAction,
-    OptimizeOptions, Ref, Table as LanceDbTable,
+    FieldMetadataUpdate as LanceFieldMetadataUpdate, FtsToken as LanceDbFtsToken,
+    NewColumnTransform, OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable,
 };
 use napi::bindgen_prelude::*;
 use napi::threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode};
@@ -574,6 +574,27 @@ impl Table {
             .collect::<Vec<_>>())
     }
 
+    #[napi(catch_unwind)]
+    pub async fn tokenize(
+        &self,
+        query: String,
+        column: Option<String>,
+        index_name: Option<String>,
+    ) -> napi::Result<Vec<FtsToken>> {
+        let table = self.inner_ref()?;
+        let tokens = match (column.as_deref(), index_name.as_deref()) {
+            (Some(_), Some(_)) | (None, None) => {
+                return Err(napi::Error::from_reason(
+                    "Specify exactly one of 'column' or 'indexName'",
+                ));
+            }
+            (Some(column), None) => table.tokenize_with_column(&query, column).await,
+            (None, Some(index_name)) => table.tokenize(&query, index_name).await,
+        }
+        .default_error()?;
+        Ok(tokens.into_iter().map(FtsToken::from).collect())
+    }
+
     #[napi(catch_unwind)]
     pub async fn index_stats(&self, index_name: String) -> napi::Result<Option<IndexStatistics>> {
         let tbl = self.inner_ref()?;
@@ -681,6 +702,24 @@ impl From<lancedb::index::IndexConfig> for IndexConfig {
     }
 }
 
+#[napi(object)]
+/// A token produced by the tokenizer configured on a full-text search index.
+pub struct FtsToken {
+    /// The token text after the index tokenizer has applied its filters.
+    pub text: String,
+    /// The token position used by full-text query matching.
+    pub position: u32,
+}
+
+impl From<LanceDbFtsToken> for FtsToken {
+    fn from(token: LanceDbFtsToken) -> Self {
+        Self {
+            text: token.text,
+            position: token.position,
+        }
+    }
+}
+
 /// Specification selecting Lance's MemWAL LSM-style write path for
 /// `mergeInsert`.
 ///