From 9dc5ec03aa6cd666b2e08183e5dcfe99e2dd28de Mon Sep 17 00:00:00 2001 From: Yang Cen Date: Sat, 25 Jul 2026 06:02:38 +0800 Subject: [PATCH] feat(fts): add block size configuration (#3691) ## What changed - add `block_size` to Python FTS configuration and the deprecated local/remote helpers - add `blockSize` to the TypeScript FTS options and propagate it through the NAPI binding - serialize the value as `block_size` for remote index creation - document the existing Rust builder API and generate the TypeScript API reference - add local, remote, metadata, search, and invalid-value regression coverage ## Why Lance supports configuring the number of documents per compressed FTS posting block, but LanceDB's Python and TypeScript APIs did not expose the setting. This made the experimental FTS V3 layout unavailable through those clients and allowed the value to be dropped before index creation. ## How it works The default remains `128`. Supported values are `128` and `256`; selecting `256` uses the experimental FTS V3 format. Invalid values are rejected by the Lance builder and surfaced as Python or JavaScript errors. ## Validation - `cargo check --quiet --features remote --tests --examples` - `cargo +1.94.0 clippy --quiet --features remote --tests --examples -- -D warnings` - targeted Rust local and remote index tests - Rust doctests: 34 passed - Python Ruff checks, doctest, and targeted local/remote tests: 5 passed - TypeScript build, Biome lint, generated docs, and targeted Jest tests: 9 passed - `git diff --check` ## Limitations The Java client remains unchanged because its external remote REST model does not currently expose `block_size`. Co-authored-by: Yang Cen --- docs/src/js/interfaces/FtsOptions.md | 13 +++++++ nodejs/__test__/remote.test.ts | 49 ++++++++++++++++++++++++++ nodejs/__test__/table.test.ts | 29 +++++++++++++++ nodejs/lancedb/indices.ts | 9 +++++ nodejs/src/index.rs | 14 +++++--- python/python/lancedb/index.py | 11 ++++++ python/python/lancedb/remote/table.py | 2 ++ python/python/lancedb/table.py | 10 ++++-- python/python/tests/test_fts.py | 17 +++++++++ python/python/tests/test_remote_db.py | 9 +++-- python/src/index.rs | 4 +++ rust/lancedb/src/index.rs | 21 ++++++++++- rust/lancedb/src/remote/table.rs | 9 +++++ rust/lancedb/src/table/create_index.rs | 13 +++++-- 14 files changed, 198 insertions(+), 12 deletions(-) diff --git a/docs/src/js/interfaces/FtsOptions.md b/docs/src/js/interfaces/FtsOptions.md index 12ab8fc84..b8b264fed 100644 --- a/docs/src/js/interfaces/FtsOptions.md +++ b/docs/src/js/interfaces/FtsOptions.md @@ -43,6 +43,19 @@ The following tokenizers are available: *** +### blockSize? + +```ts +optional blockSize: 128 | 256; +``` + +Number of documents per compressed posting block. + +The default is 128. Supported values are 128 and 256. A value of 256 uses +the experimental FTS V3 format and may introduce breaking changes. + +*** + ### language? ```ts diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index a59dd4e71..22562b0bc 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -15,6 +15,7 @@ import { OAuthHeaderProvider, StaticHeaderProvider, } from "../lancedb/header"; +import { Index } from "../lancedb/indices"; // Test-only header providers class CustomProvider extends HeaderProvider { @@ -225,6 +226,54 @@ describe("remote connection", () => { ); }); + it("sends the FTS posting block size to remote tables", async () => { + let createIndexBody: Record | undefined; + + await withMockDatabase( + (req, res) => { + const path = req.url ?? ""; + if (path.endsWith("/describe/")) { + res.writeHead(200, { "Content-Type": "application/json" }).end( + JSON.stringify({ + name: "t", + version: 1, + schema: { + fields: [ + { name: "text", type: { type: "string" }, nullable: false }, + ], + }, + }), + ); + return; + } + + if (path.endsWith("/create_index/")) { + let raw = ""; + req.on("data", (chunk) => { + raw += chunk; + }); + req.on("end", () => { + createIndexBody = JSON.parse(raw); + res.writeHead(200).end(); + }); + return; + } + + res.writeHead(404).end(); + }, + async (db) => { + const table = await db.openTable("t"); + await table.createIndex("text", { + config: Index.fts({ blockSize: 256 }), + }); + }, + ); + + expect(createIndexBody?.["column"]).toBe("text"); + expect(createIndexBody?.["index_type"]).toBe("FTS"); + expect(createIndexBody?.["block_size"]).toBe(256); + }); + it("diffs and merges remote branches", async () => { const sampleDiff = { fromBranch: "exp", diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 39345996a..44edaa093 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -2527,6 +2527,35 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( expect(results3.length).toBe(1); }); + test("full text search with custom posting block size", async () => { + const db = await connect(tmpDir.name); + const data = [ + { text: "hello world", vector: [0.1, 0.2, 0.3] }, + { text: "goodbye world", vector: [0.4, 0.5, 0.6] }, + ]; + const table = await db.createTable("test", data); + await table.createIndex("text", { + config: Index.fts({ blockSize: 256 }), + }); + + const index = (await table.listIndices()).find( + (index) => index.indexType === "FTS", + ); + expect(index?.indexVersion).toBe(3); + expect( + (index?.indexDetails as Record)["block_size"], + ).toBe(256); + + const results = await table.search("hello").toArray(); + expect(results[0].text).toBe(data[0].text); + }); + + test("rejects invalid full text posting block size", () => { + expect(() => Index.fts({ blockSize: 129 as 128 | 256 })).toThrow( + "128 or 256", + ); + }); + test("full text search without lowercase", async () => { const db = await connect(tmpDir.name); const data = [ diff --git a/nodejs/lancedb/indices.ts b/nodejs/lancedb/indices.ts index e45556e5f..9b438ef0f 100644 --- a/nodejs/lancedb/indices.ts +++ b/nodejs/lancedb/indices.ts @@ -572,6 +572,14 @@ export interface FtsOptions { * whether to only index the prefix of the token for ngram tokenizer */ prefixOnly?: boolean; + + /** + * Number of documents per compressed posting block. + * + * The default is 128. Supported values are 128 and 256. A value of 256 uses + * the experimental FTS V3 format and may introduce breaking changes. + */ + blockSize?: 128 | 256; } export class Index { @@ -751,6 +759,7 @@ export class Index { options?.ngramMinLength, options?.ngramMaxLength, options?.prefixOnly, + options?.blockSize, ), ); } diff --git a/nodejs/src/index.rs b/nodejs/src/index.rs index db84710ba..66dee7913 100644 --- a/nodejs/src/index.rs +++ b/nodejs/src/index.rs @@ -226,7 +226,8 @@ impl Index { ngram_min_length: Option, ngram_max_length: Option, prefix_only: Option, - ) -> Self { + block_size: Option, + ) -> napi::Result { let mut opts = FtsIndexBuilder::default(); if let Some(with_position) = with_position { opts = opts.with_position(with_position); @@ -261,10 +262,15 @@ impl Index { if let Some(prefix_only) = prefix_only { opts = opts.ngram_prefix_only(prefix_only); } - - Self { - inner: Mutex::new(Some(LanceDbIndex::FTS(opts))), + if let Some(block_size) = block_size { + opts = opts + .block_size(block_size as usize) + .map_err(|err| napi::Error::from_reason(err.to_string()))?; } + + Ok(Self { + inner: Mutex::new(Some(LanceDbIndex::FTS(opts))), + }) } #[napi(factory)] diff --git a/python/python/lancedb/index.py b/python/python/lancedb/index.py index 76daaed11..c06bb9903 100644 --- a/python/python/lancedb/index.py +++ b/python/python/lancedb/index.py @@ -115,6 +115,12 @@ class FTS: For example, it works with `title`, `description`, `content`, etc. + Examples + -------- + Create an index configuration that uses 256-document posting blocks: + + >>> config = FTS(block_size=256) + Attributes ---------- with_position : bool, default False @@ -148,6 +154,10 @@ class FTS: ascii_folding : bool, default True Whether to fold ASCII characters. This converts accented characters to their ASCII equivalent. For example, "café" would be converted to "cafe". + block_size : int, default 128 + The number of documents per compressed posting block. Supported values + are 128 and 256. A value of 256 uses the experimental FTS V3 format + and may introduce breaking changes. Notes ----- @@ -168,6 +178,7 @@ class FTS: ngram_min_length: int = 3 ngram_max_length: int = 3 prefix_only: bool = False + block_size: int = 128 @dataclass diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 8c702aace..80c250bf5 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -344,6 +344,7 @@ class RemoteTable(Table): ngram_min_length: int = 3, ngram_max_length: int = 3, prefix_only: bool = False, + block_size: int = 128, name: Optional[str] = None, ): """Create a full-text search index on a column. @@ -364,6 +365,7 @@ class RemoteTable(Table): ngram_min_length=ngram_min_length, ngram_max_length=ngram_max_length, prefix_only=prefix_only, + block_size=block_size, ) LOOP.run( self._table.create_index( diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 0e181b457..5f89f6308 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -1106,6 +1106,7 @@ class Table(ABC): ngram_min_length: int = 3, ngram_max_length: int = 3, prefix_only: bool = False, + block_size: int = 128, wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, ): @@ -1177,6 +1178,10 @@ class Table(ABC): The maximum length of an n-gram. prefix_only: bool, default False Whether to only index the prefix of the token for ngram tokenizer. + block_size: int, default 128 + The number of documents per compressed posting block. Must be 128 + or 256. A value of 256 uses the experimental FTS V3 format and + may introduce breaking changes. wait_timeout: timedelta, optional The timeout to wait if indexing is asynchronous. name: str, optional @@ -3026,6 +3031,7 @@ class LanceTable(Table): ngram_min_length: int = 3, ngram_max_length: int = 3, prefix_only: bool = False, + block_size: int = 128, name: Optional[str] = None, ): """Create a full-text search index on a column. @@ -3075,9 +3081,7 @@ class LanceTable(Table): else: tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name) - config = FTS( - **tokenizer_configs, - ) + config = FTS(block_size=block_size, **tokenizer_configs) try: LOOP.run( diff --git a/python/python/tests/test_fts.py b/python/python/tests/test_fts.py index f822d1692..127510bc2 100644 --- a/python/python/tests/test_fts.py +++ b/python/python/tests/test_fts.py @@ -226,6 +226,23 @@ def test_create_inverted_index(table, with_position): assert any(i.name == "custom_fts_index" for i in fts_indices) +@pytest.mark.parametrize("block_size", [128, 256]) +def test_create_inverted_index_block_size(table, block_size): + table.create_index("text", config=FTS(block_size=block_size)) + + index = next(index for index in table.list_indices() if index.index_type == "FTS") + assert index.index_details["block_size"] == block_size + assert index.index_version == (2 if block_size == 128 else 3) + + results = table.search("puppy").limit(5).to_list() + assert len(results) == 5 + + +def test_create_inverted_index_rejects_invalid_block_size(table): + with pytest.raises(ValueError, match="128 or 256"): + table.create_index("text", config=FTS(block_size=129)) + + def test_search_fts(table): table.create_fts_index("text") results = table.search("puppy").select(["id", "text"]).limit(5).to_list() diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 477ee7496..648a52b7a 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -768,7 +768,10 @@ def test_table_create_indices(): # Test create_fts_index with custom name (legacy method) with pytest.warns(DeprecationWarning, match="create_fts_index"): table.create_fts_index( - "text", wait_timeout=timedelta(seconds=2), name="custom_fts_idx" + "text", + wait_timeout=timedelta(seconds=2), + block_size=256, + name="custom_fts_idx", ) # Test create_index with custom name (legacy form: vector_column_name kwarg) @@ -791,6 +794,7 @@ def test_table_create_indices(): fts_req = received_requests[1] assert "name" in fts_req assert fts_req["name"] == "custom_fts_idx" + assert fts_req["block_size"] == 256 # Check vector index request has custom name vector_req = received_requests[2] @@ -876,7 +880,7 @@ def test_remote_create_index_new_api(): _warnings.simplefilter("error", DeprecationWarning) table.create_index("vector", config=IvfPq(distance_type="l2")) table.create_index("category", config=BTree()) - table.create_index("text", config=FTS()) + table.create_index("text", config=FTS(block_size=256)) # IvfRq via new API table.create_index("vector", config=IvfRq(distance_type="l2")) @@ -896,6 +900,7 @@ def test_remote_create_index_new_api(): "vector", "vector", ] + assert received_requests[2]["block_size"] == 256 def test_table_wait_for_index_timeout(): diff --git a/python/src/index.rs b/python/src/index.rs index 121d3875d..a98b90cf9 100644 --- a/python/src/index.rs +++ b/python/src/index.rs @@ -60,6 +60,9 @@ pub fn extract_index_params(source: &Option>) -> PyResult { @@ -207,6 +210,7 @@ struct FtsParams { ngram_min_length: u32, ngram_max_length: u32, prefix_only: bool, + block_size: usize, } #[derive(FromPyObject)] diff --git a/rust/lancedb/src/index.rs b/rust/lancedb/src/index.rs index 08706ce54..308a64ca1 100644 --- a/rust/lancedb/src/index.rs +++ b/rust/lancedb/src/index.rs @@ -54,7 +54,26 @@ pub enum Index { /// substrings of the raw bytes, unlike the tokenized [`Index::FTS`] index. Fm(FmIndexBuilder), - /// Full text search index using bm25. + /// Full text search index using BM25. + /// + /// The posting block size defaults to 128. Supported values are 128 and 256; + /// a value of 256 uses the experimental FTS V3 format and may introduce + /// breaking changes. + /// + /// ``` + /// use lancedb::index::{Index, scalar::FtsIndexBuilder}; + /// + /// # async fn create_fts_index( + /// # table: &lancedb::Table, + /// # ) -> Result<(), Box> { + /// let params = FtsIndexBuilder::default().block_size(256)?; + /// table + /// .create_index(&["text"], Index::FTS(params)) + /// .execute() + /// .await?; + /// # Ok(()) + /// # } + /// ``` FTS(FtsIndexBuilder), /// IVF index diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 2518ed65c..92c4ad296 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -4496,6 +4496,15 @@ mod tests { serde_json::to_value(InvertedIndexParams::default()).unwrap(), Index::FTS(Default::default()), ), + ( + "FTS", + { + let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap(); + body["block_size"] = 256.into(); + body + }, + Index::FTS(InvertedIndexParams::default().block_size(256).unwrap()), + ), ]; for (index_type, expected_body, index) in cases { diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs index 3af661b1a..6be9de0e6 100644 --- a/rust/lancedb/src/table/create_index.rs +++ b/rust/lancedb/src/table/create_index.rs @@ -382,7 +382,9 @@ mod tests { use crate::connect; use crate::connection::ConnectBuilder; use crate::index::Index; - use crate::index::scalar::{BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder}; + use crate::index::scalar::{ + BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder, FtsIndexBuilder, + }; use crate::index::vector::{ IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, }; @@ -1362,7 +1364,10 @@ mod tests { .unwrap(); table - .create_index(&["text"], Index::FTS(Default::default())) + .create_index( + &["text"], + Index::FTS(FtsIndexBuilder::default().block_size(256).unwrap()), + ) .execute() .await .unwrap(); @@ -1372,6 +1377,10 @@ mod tests { assert_eq!(index.index_type, crate::index::IndexType::FTS); assert_eq!(index.columns, vec!["text".to_string()]); assert_eq!(index.name, "text_idx"); + assert_eq!(index.index_version, Some(3)); + let index_params: FtsIndexBuilder = + serde_json::from_str(index.index_details.as_deref().unwrap()).unwrap(); + assert_eq!(index_params.posting_block_size(), 256); let num_rows = 120; let stats = table.index_stats("text_idx").await.unwrap().unwrap();