feat: add table FTS query tokenization (#3659)

## Summary
- add table-level FTS query tokenization returning token text and
position
- use the native index tokenizer for local tables and remote index
metadata for remote tables
- expose sync and async Python table wrappers with focused coverage

nodejs/__test__/table.test.ts

@@ -16,6 +16,7 @@ import {
   PhraseQuery,
   Table,
   connect,
+  tokenize,
 } from "../lancedb";
 import {
   Table as ArrowTable,
@@ -2307,6 +2308,75 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       expect(results2[0].text).toBe(data[1].text);
     });
 
+    test("tokenizes FTS queries by column or index name", async () => {
+      const db = await connect(tmpDir.name);
+      const data = [
+        {
+          text: "Running in cafés",
+          japanese: "Hello, こんにちは世界!",
+          vector: [0.1, 0.2, 0.3],
+        },
+      ];
+      const table = await db.createTable("test", data);
+      await table.createIndex("text", {
+        config: Index.fts({ baseTokenizer: "simple" }),
+      });
+      await table.createIndex("japanese", {
+        config: Index.fts({
+          baseTokenizer: "icu",
+          stem: false,
+          removeStopWords: false,
+        }),
+        name: "japanese_icu_idx",
+      });
+
+      await expect(table.tokenize("hello", {} as never)).rejects.toThrow(
+        "Specify exactly one",
+      );
+      await expect(
+        table.tokenize("hello", {
+          column: "text",
+          indexName: "text_idx",
+        } as never),
+      ).rejects.toThrow("Specify exactly one");
+
+      const simpleTokens = await table.tokenize("Running in cafés", {
+        column: "text",
+      });
+      expect(simpleTokens).toEqual([
+        { text: "run", position: 0 },
+        { text: "cafe", position: 2 },
+      ]);
+
+      const icuTokens = await table.tokenize("Hello, こんにちは世界!", {
+        indexName: "japanese_icu_idx",
+      });
+      expect(icuTokens).toEqual([
+        { text: "hello", position: 0 },
+        { text: "こんにちは", position: 1 },
+        { text: "世界", position: 2 },
+      ]);
+
+      const directSimpleTokens = await tokenize("Running in cafés", {
+        baseTokenizer: "simple",
+      });
+      expect(directSimpleTokens).toEqual([
+        { text: "run", position: 0 },
+        { text: "cafe", position: 2 },
+      ]);
+
+      const directIcuTokens = await tokenize("Hello, こんにちは世界!", {
+        baseTokenizer: "icu",
+        stem: false,
+        removeStopWords: false,
+      });
+      expect(directIcuTokens).toEqual([
+        { text: "hello", position: 0 },
+        { text: "こんにちは", position: 1 },
+        { text: "世界", position: 2 },
+      ]);
+    });
+
     test("full text search fast search", async () => {
       const db = await connect(tmpDir.name);
       const data = [{ text: "hello world", vector: [0.1, 0.2, 0.3], id: 1 }];

nodejs/lancedb/index.ts

@@ -13,9 +13,12 @@ import {
   Connection as LanceDbConnection,
   JsHeaderProvider as NativeJsHeaderProvider,
   Session,
+  tokenize as nativeTokenize,
 } from "./native.js";
 
 import { HeaderProvider } from "./header";
+import type { BaseTokenizer } from "./indices";
+import type { FtsToken } from "./table";
 
 // Re-export native header provider for use with connectWithHeaderProvider
 export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
@@ -114,6 +117,7 @@ export {
   HnswPqOptions,
   HnswSqOptions,
   FtsOptions,
+  BaseTokenizer,
 } from "./indices";
 
 export {
@@ -124,6 +128,8 @@ export {
   OptimizeOptions,
   Version,
   WriteProgress,
+  FtsToken,
+  TokenizeTableOptions,
   LsmWriteSpec,
   ColumnAlteration,
   FieldMetadataUpdate,
@@ -155,6 +161,68 @@ export {
 } from "./arrow";
 export { IntoSql, packBits } from "./util";
 
+/**
+ * Options for tokenizing a full-text search query without a table index.
+ */
+export interface TokenizeOptions {
+  /**
+   * The tokenizer to use. The default is "simple".
+   */
+  baseTokenizer?: BaseTokenizer;
+
+  /** Language for stemming and stop words. */
+  language?: string;
+
+  /** Maximum token length; tokens longer than this are ignored. */
+  maxTokenLength?: number;
+
+  /** Whether to lowercase tokens. */
+  lowercase?: boolean;
+
+  /** Whether to stem tokens. */
+  stem?: boolean;
+
+  /** Whether to remove stop words. */
+  removeStopWords?: boolean;
+
+  /** Whether to fold ASCII characters. */
+  asciiFolding?: boolean;
+
+  /** N-gram minimum length. */
+  ngramMinLength?: number;
+
+  /** N-gram maximum length. */
+  ngramMaxLength?: number;
+
+  /** Whether to only emit token prefixes for the n-gram tokenizer. */
+  prefixOnly?: boolean;
+}
+
+/**
+ * Tokenize a full-text search query using an explicit tokenizer.
+ *
+ * This does not require a table or FTS index. The tokenizer options match
+ * {@link Index.fts}.
+ */
+export async function tokenize(
+  query: string,
+  options?: Partial<TokenizeOptions>,
+): Promise<FtsToken[]> {
+  return await nativeTokenize(
+    query,
+    options?.baseTokenizer,
+    options?.language,
+    options?.maxTokenLength,
+    options?.lowercase,
+    options?.stem,
+    options?.removeStopWords,
+    options?.asciiFolding,
+    options?.ngramMinLength,
+    options?.ngramMaxLength,
+    options?.prefixOnly,
+  );
+}
+
 /**
  * Connect to a LanceDB instance at the given URI.
  *

nodejs/lancedb/indices.ts

@@ -486,6 +486,16 @@ export interface IvfFlatOptions {
   sampleRate?: number;
 }
 
+export type BaseTokenizer =
+  | "simple"
+  | "whitespace"
+  | "raw"
+  | "ngram"
+  | "icu"
+  | "icu/split"
+  | `jieba/${string}`
+  | `lindera/${string}`;
+
 /**
  * Options to create a full text search index
  */
@@ -509,8 +519,12 @@ export interface FtsOptions {
    * "whitespace" - Whitespace tokenizer. This tokenizer splits the text into tokens using whitespace as a delimiter.
    *
    * "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
+   *
+   * "icu" - ICU dictionary-based word segmentation.
+   *
+   * "icu/split" - ICU segmentation with simple-style delimiter splitting.
    */
-  baseTokenizer?: "simple" | "whitespace" | "raw" | "ngram";
+  baseTokenizer?: BaseTokenizer;
 
   /**
    * language for stemming and stop words

nodejs/lancedb/table.ts

@@ -158,6 +158,26 @@ export interface Version {
   metadata: Record<string, string>;
 }
 
+/** Token produced by the tokenizer configured on a full-text search index. */
+export interface FtsToken {
+  /** Token text after tokenizer filters have been applied. */
+  text: string;
+  /** Token position used by full-text query matching. */
+  position: number;
+}
+
+export type TokenizeTableOptions =
+  | {
+      /** FTS-indexed column whose tokenizer should be used. */
+      column: string;
+      indexName?: never;
+    }
+  | {
+      /** Name of the FTS index whose tokenizer should be used. */
+      indexName: string;
+      column?: never;
+    };
+
 /**
  * Specification selecting Lance's MemWAL LSM-style write path for
  * `mergeInsert`.
@@ -716,6 +736,19 @@ export abstract class Table {
   abstract optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
   /** List all indices that have been created with {@link Table.createIndex} */
   abstract listIndices(): Promise<IndexConfig[]>;
+  /**
+   * Tokenize a full-text search query using the tokenizer configured on an FTS index.
+   *
+   * Specify exactly one of `column` or `indexName`.
+   *
+   * Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in
+   * the client process from index metadata. For remote tables, this means the
+   * same tokenizer model files must also exist locally.
+   */
+  abstract tokenize(
+    query: string,
+    options: TokenizeTableOptions,
+  ): Promise<FtsToken[]>;
   /** Return the table as an arrow table */
   abstract toArrow(): Promise<ArrowTable>;
 
@@ -1173,6 +1206,17 @@ export class LocalTable extends Table {
     return await this.inner.listIndices();
   }
 
+  async tokenize(
+    query: string,
+    options: TokenizeTableOptions,
+  ): Promise<FtsToken[]> {
+    return await this.inner.tokenize(
+      query,
+      options?.column,
+      options?.indexName,
+    );
+  }
+
   async toArrow(): Promise<ArrowTable> {
     return await this.query().toArrow();
   }

nodejs/src/index.rs

@@ -9,8 +9,11 @@ use lancedb::index::vector::{
     IvfFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder,
     IvfRqIndexBuilder,
 };
+use lancedb::tokenize as lancedb_tokenize;
 use napi_derive::napi;
 
+use crate::error::NapiErrorExt;
+use crate::table::FtsToken;
 use crate::util::parse_distance_type;
 
 #[napi]
@@ -30,6 +33,65 @@ impl Index {
     }
 }
 
+#[napi(catch_unwind)]
+#[allow(dead_code, clippy::too_many_arguments)]
+pub fn tokenize(
+    query: String,
+    base_tokenizer: Option<String>,
+    language: Option<String>,
+    max_token_length: Option<u32>,
+    lower_case: Option<bool>,
+    stem: Option<bool>,
+    remove_stop_words: Option<bool>,
+    ascii_folding: Option<bool>,
+    ngram_min_length: Option<u32>,
+    ngram_max_length: Option<u32>,
+    prefix_only: Option<bool>,
+) -> napi::Result<Vec<FtsToken>> {
+    let mut opts = FtsIndexBuilder::default();
+    if let Some(base_tokenizer) = base_tokenizer {
+        opts = opts.base_tokenizer(base_tokenizer);
+    }
+    if let Some(language) = language {
+        opts = opts.language(&language).map_err(|_| {
+            napi::Error::from_reason(format!(
+                "LanceDB does not support the requested language: '{}'",
+                language
+            ))
+        })?;
+    }
+    if let Some(max_token_length) = max_token_length {
+        opts = opts.max_token_length(Some(max_token_length as usize));
+    }
+    if let Some(lower_case) = lower_case {
+        opts = opts.lower_case(lower_case);
+    }
+    if let Some(stem) = stem {
+        opts = opts.stem(stem);
+    }
+    if let Some(remove_stop_words) = remove_stop_words {
+        opts = opts.remove_stop_words(remove_stop_words);
+    }
+    if let Some(ascii_folding) = ascii_folding {
+        opts = opts.ascii_folding(ascii_folding);
+    }
+    if let Some(ngram_min_length) = ngram_min_length {
+        opts = opts.ngram_min_length(ngram_min_length);
+    }
+    if let Some(ngram_max_length) = ngram_max_length {
+        opts = opts.ngram_max_length(ngram_max_length);
+    }
+    if let Some(prefix_only) = prefix_only {
+        opts = opts.ngram_prefix_only(prefix_only);
+    }
+
+    Ok(lancedb_tokenize(&query, &opts)
+        .default_error()?
+        .into_iter()
+        .map(FtsToken::from)
+        .collect())
+}
+
 #[napi]
 impl Index {
     #[napi(factory)]

nodejs/src/table.rs

@@ -8,8 +8,8 @@ use chrono::{DateTime, Utc};
 use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
 use lancedb::table::{
     AddDataMode, ColumnAlteration as LanceColumnAlteration, Duration,
-    FieldMetadataUpdate as LanceFieldMetadataUpdate, NewColumnTransform, OptimizeAction,
-    OptimizeOptions, Ref, Table as LanceDbTable,
+    FieldMetadataUpdate as LanceFieldMetadataUpdate, FtsToken as LanceDbFtsToken,
+    NewColumnTransform, OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable,
 };
 use napi::bindgen_prelude::*;
 use napi::threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode};
@@ -574,6 +574,27 @@ impl Table {
             .collect::<Vec<_>>())
     }
 
+    #[napi(catch_unwind)]
+    pub async fn tokenize(
+        &self,
+        query: String,
+        column: Option<String>,
+        index_name: Option<String>,
+    ) -> napi::Result<Vec<FtsToken>> {
+        let table = self.inner_ref()?;
+        let tokens = match (column.as_deref(), index_name.as_deref()) {
+            (Some(_), Some(_)) | (None, None) => {
+                return Err(napi::Error::from_reason(
+                    "Specify exactly one of 'column' or 'indexName'",
+                ));
+            }
+            (Some(column), None) => table.tokenize_with_column(&query, column).await,
+            (None, Some(index_name)) => table.tokenize(&query, index_name).await,
+        }
+        .default_error()?;
+        Ok(tokens.into_iter().map(FtsToken::from).collect())
+    }
+
     #[napi(catch_unwind)]
     pub async fn index_stats(&self, index_name: String) -> napi::Result<Option<IndexStatistics>> {
         let tbl = self.inner_ref()?;
@@ -681,6 +702,24 @@ impl From<lancedb::index::IndexConfig> for IndexConfig {
     }
 }
 
+#[napi(object)]
+/// A token produced by the tokenizer configured on a full-text search index.
+pub struct FtsToken {
+    /// The token text after the index tokenizer has applied its filters.
+    pub text: String,
+    /// The token position used by full-text query matching.
+    pub position: u32,
+}
+
+impl From<LanceDbFtsToken> for FtsToken {
+    fn from(token: LanceDbFtsToken) -> Self {
+        Self {
+            text: token.text,
+            position: token.position,
+        }
+    }
+}
+
 /// Specification selecting Lance's MemWAL LSM-style write path for
 /// `mergeInsert`.
 ///