diff --git a/docs/src/js/interfaces/FtsOptions.md b/docs/src/js/interfaces/FtsOptions.md index 12ab8fc84..b8b264fed 100644 --- a/docs/src/js/interfaces/FtsOptions.md +++ b/docs/src/js/interfaces/FtsOptions.md @@ -43,6 +43,19 @@ The following tokenizers are available: *** +### blockSize? + +```ts +optional blockSize: 128 | 256; +``` + +Number of documents per compressed posting block. + +The default is 128. Supported values are 128 and 256. A value of 256 uses +the experimental FTS V3 format and may introduce breaking changes. + +*** + ### language? ```ts diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index a59dd4e71..22562b0bc 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -15,6 +15,7 @@ import { OAuthHeaderProvider, StaticHeaderProvider, } from "../lancedb/header"; +import { Index } from "../lancedb/indices"; // Test-only header providers class CustomProvider extends HeaderProvider { @@ -225,6 +226,54 @@ describe("remote connection", () => { ); }); + it("sends the FTS posting block size to remote tables", async () => { + let createIndexBody: Record | undefined; + + await withMockDatabase( + (req, res) => { + const path = req.url ?? ""; + if (path.endsWith("/describe/")) { + res.writeHead(200, { "Content-Type": "application/json" }).end( + JSON.stringify({ + name: "t", + version: 1, + schema: { + fields: [ + { name: "text", type: { type: "string" }, nullable: false }, + ], + }, + }), + ); + return; + } + + if (path.endsWith("/create_index/")) { + let raw = ""; + req.on("data", (chunk) => { + raw += chunk; + }); + req.on("end", () => { + createIndexBody = JSON.parse(raw); + res.writeHead(200).end(); + }); + return; + } + + res.writeHead(404).end(); + }, + async (db) => { + const table = await db.openTable("t"); + await table.createIndex("text", { + config: Index.fts({ blockSize: 256 }), + }); + }, + ); + + expect(createIndexBody?.["column"]).toBe("text"); + expect(createIndexBody?.["index_type"]).toBe("FTS"); + expect(createIndexBody?.["block_size"]).toBe(256); + }); + it("diffs and merges remote branches", async () => { const sampleDiff = { fromBranch: "exp", diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 39345996a..44edaa093 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -2527,6 +2527,35 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( expect(results3.length).toBe(1); }); + test("full text search with custom posting block size", async () => { + const db = await connect(tmpDir.name); + const data = [ + { text: "hello world", vector: [0.1, 0.2, 0.3] }, + { text: "goodbye world", vector: [0.4, 0.5, 0.6] }, + ]; + const table = await db.createTable("test", data); + await table.createIndex("text", { + config: Index.fts({ blockSize: 256 }), + }); + + const index = (await table.listIndices()).find( + (index) => index.indexType === "FTS", + ); + expect(index?.indexVersion).toBe(3); + expect( + (index?.indexDetails as Record)["block_size"], + ).toBe(256); + + const results = await table.search("hello").toArray(); + expect(results[0].text).toBe(data[0].text); + }); + + test("rejects invalid full text posting block size", () => { + expect(() => Index.fts({ blockSize: 129 as 128 | 256 })).toThrow( + "128 or 256", + ); + }); + test("full text search without lowercase", async () => { const db = await connect(tmpDir.name); const data = [ diff --git a/nodejs/lancedb/indices.ts b/nodejs/lancedb/indices.ts index e45556e5f..9b438ef0f 100644 --- a/nodejs/lancedb/indices.ts +++ b/nodejs/lancedb/indices.ts @@ -572,6 +572,14 @@ export interface FtsOptions { * whether to only index the prefix of the token for ngram tokenizer */ prefixOnly?: boolean; + + /** + * Number of documents per compressed posting block. + * + * The default is 128. Supported values are 128 and 256. A value of 256 uses + * the experimental FTS V3 format and may introduce breaking changes. + */ + blockSize?: 128 | 256; } export class Index { @@ -751,6 +759,7 @@ export class Index { options?.ngramMinLength, options?.ngramMaxLength, options?.prefixOnly, + options?.blockSize, ), ); } diff --git a/nodejs/src/index.rs b/nodejs/src/index.rs index db84710ba..66dee7913 100644 --- a/nodejs/src/index.rs +++ b/nodejs/src/index.rs @@ -226,7 +226,8 @@ impl Index { ngram_min_length: Option, ngram_max_length: Option, prefix_only: Option, - ) -> Self { + block_size: Option, + ) -> napi::Result { let mut opts = FtsIndexBuilder::default(); if let Some(with_position) = with_position { opts = opts.with_position(with_position); @@ -261,10 +262,15 @@ impl Index { if let Some(prefix_only) = prefix_only { opts = opts.ngram_prefix_only(prefix_only); } - - Self { - inner: Mutex::new(Some(LanceDbIndex::FTS(opts))), + if let Some(block_size) = block_size { + opts = opts + .block_size(block_size as usize) + .map_err(|err| napi::Error::from_reason(err.to_string()))?; } + + Ok(Self { + inner: Mutex::new(Some(LanceDbIndex::FTS(opts))), + }) } #[napi(factory)] diff --git a/python/python/lancedb/index.py b/python/python/lancedb/index.py index 76daaed11..c06bb9903 100644 --- a/python/python/lancedb/index.py +++ b/python/python/lancedb/index.py @@ -115,6 +115,12 @@ class FTS: For example, it works with `title`, `description`, `content`, etc. + Examples + -------- + Create an index configuration that uses 256-document posting blocks: + + >>> config = FTS(block_size=256) + Attributes ---------- with_position : bool, default False @@ -148,6 +154,10 @@ class FTS: ascii_folding : bool, default True Whether to fold ASCII characters. This converts accented characters to their ASCII equivalent. For example, "café" would be converted to "cafe". + block_size : int, default 128 + The number of documents per compressed posting block. Supported values + are 128 and 256. A value of 256 uses the experimental FTS V3 format + and may introduce breaking changes. Notes ----- @@ -168,6 +178,7 @@ class FTS: ngram_min_length: int = 3 ngram_max_length: int = 3 prefix_only: bool = False + block_size: int = 128 @dataclass diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 8c702aace..80c250bf5 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -344,6 +344,7 @@ class RemoteTable(Table): ngram_min_length: int = 3, ngram_max_length: int = 3, prefix_only: bool = False, + block_size: int = 128, name: Optional[str] = None, ): """Create a full-text search index on a column. @@ -364,6 +365,7 @@ class RemoteTable(Table): ngram_min_length=ngram_min_length, ngram_max_length=ngram_max_length, prefix_only=prefix_only, + block_size=block_size, ) LOOP.run( self._table.create_index( diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 0e181b457..5f89f6308 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -1106,6 +1106,7 @@ class Table(ABC): ngram_min_length: int = 3, ngram_max_length: int = 3, prefix_only: bool = False, + block_size: int = 128, wait_timeout: Optional[timedelta] = None, name: Optional[str] = None, ): @@ -1177,6 +1178,10 @@ class Table(ABC): The maximum length of an n-gram. prefix_only: bool, default False Whether to only index the prefix of the token for ngram tokenizer. + block_size: int, default 128 + The number of documents per compressed posting block. Must be 128 + or 256. A value of 256 uses the experimental FTS V3 format and + may introduce breaking changes. wait_timeout: timedelta, optional The timeout to wait if indexing is asynchronous. name: str, optional @@ -3026,6 +3031,7 @@ class LanceTable(Table): ngram_min_length: int = 3, ngram_max_length: int = 3, prefix_only: bool = False, + block_size: int = 128, name: Optional[str] = None, ): """Create a full-text search index on a column. @@ -3075,9 +3081,7 @@ class LanceTable(Table): else: tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name) - config = FTS( - **tokenizer_configs, - ) + config = FTS(block_size=block_size, **tokenizer_configs) try: LOOP.run( diff --git a/python/python/tests/test_fts.py b/python/python/tests/test_fts.py index f822d1692..127510bc2 100644 --- a/python/python/tests/test_fts.py +++ b/python/python/tests/test_fts.py @@ -226,6 +226,23 @@ def test_create_inverted_index(table, with_position): assert any(i.name == "custom_fts_index" for i in fts_indices) +@pytest.mark.parametrize("block_size", [128, 256]) +def test_create_inverted_index_block_size(table, block_size): + table.create_index("text", config=FTS(block_size=block_size)) + + index = next(index for index in table.list_indices() if index.index_type == "FTS") + assert index.index_details["block_size"] == block_size + assert index.index_version == (2 if block_size == 128 else 3) + + results = table.search("puppy").limit(5).to_list() + assert len(results) == 5 + + +def test_create_inverted_index_rejects_invalid_block_size(table): + with pytest.raises(ValueError, match="128 or 256"): + table.create_index("text", config=FTS(block_size=129)) + + def test_search_fts(table): table.create_fts_index("text") results = table.search("puppy").select(["id", "text"]).limit(5).to_list() diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 477ee7496..648a52b7a 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -768,7 +768,10 @@ def test_table_create_indices(): # Test create_fts_index with custom name (legacy method) with pytest.warns(DeprecationWarning, match="create_fts_index"): table.create_fts_index( - "text", wait_timeout=timedelta(seconds=2), name="custom_fts_idx" + "text", + wait_timeout=timedelta(seconds=2), + block_size=256, + name="custom_fts_idx", ) # Test create_index with custom name (legacy form: vector_column_name kwarg) @@ -791,6 +794,7 @@ def test_table_create_indices(): fts_req = received_requests[1] assert "name" in fts_req assert fts_req["name"] == "custom_fts_idx" + assert fts_req["block_size"] == 256 # Check vector index request has custom name vector_req = received_requests[2] @@ -876,7 +880,7 @@ def test_remote_create_index_new_api(): _warnings.simplefilter("error", DeprecationWarning) table.create_index("vector", config=IvfPq(distance_type="l2")) table.create_index("category", config=BTree()) - table.create_index("text", config=FTS()) + table.create_index("text", config=FTS(block_size=256)) # IvfRq via new API table.create_index("vector", config=IvfRq(distance_type="l2")) @@ -896,6 +900,7 @@ def test_remote_create_index_new_api(): "vector", "vector", ] + assert received_requests[2]["block_size"] == 256 def test_table_wait_for_index_timeout(): diff --git a/python/src/index.rs b/python/src/index.rs index 121d3875d..a98b90cf9 100644 --- a/python/src/index.rs +++ b/python/src/index.rs @@ -60,6 +60,9 @@ pub fn extract_index_params(source: &Option>) -> PyResult { @@ -207,6 +210,7 @@ struct FtsParams { ngram_min_length: u32, ngram_max_length: u32, prefix_only: bool, + block_size: usize, } #[derive(FromPyObject)] diff --git a/rust/lancedb/src/index.rs b/rust/lancedb/src/index.rs index 08706ce54..308a64ca1 100644 --- a/rust/lancedb/src/index.rs +++ b/rust/lancedb/src/index.rs @@ -54,7 +54,26 @@ pub enum Index { /// substrings of the raw bytes, unlike the tokenized [`Index::FTS`] index. Fm(FmIndexBuilder), - /// Full text search index using bm25. + /// Full text search index using BM25. + /// + /// The posting block size defaults to 128. Supported values are 128 and 256; + /// a value of 256 uses the experimental FTS V3 format and may introduce + /// breaking changes. + /// + /// ``` + /// use lancedb::index::{Index, scalar::FtsIndexBuilder}; + /// + /// # async fn create_fts_index( + /// # table: &lancedb::Table, + /// # ) -> Result<(), Box> { + /// let params = FtsIndexBuilder::default().block_size(256)?; + /// table + /// .create_index(&["text"], Index::FTS(params)) + /// .execute() + /// .await?; + /// # Ok(()) + /// # } + /// ``` FTS(FtsIndexBuilder), /// IVF index diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 2518ed65c..92c4ad296 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -4496,6 +4496,15 @@ mod tests { serde_json::to_value(InvertedIndexParams::default()).unwrap(), Index::FTS(Default::default()), ), + ( + "FTS", + { + let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap(); + body["block_size"] = 256.into(); + body + }, + Index::FTS(InvertedIndexParams::default().block_size(256).unwrap()), + ), ]; for (index_type, expected_body, index) in cases { diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs index 3af661b1a..6be9de0e6 100644 --- a/rust/lancedb/src/table/create_index.rs +++ b/rust/lancedb/src/table/create_index.rs @@ -382,7 +382,9 @@ mod tests { use crate::connect; use crate::connection::ConnectBuilder; use crate::index::Index; - use crate::index::scalar::{BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder}; + use crate::index::scalar::{ + BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder, FtsIndexBuilder, + }; use crate::index::vector::{ IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, }; @@ -1362,7 +1364,10 @@ mod tests { .unwrap(); table - .create_index(&["text"], Index::FTS(Default::default())) + .create_index( + &["text"], + Index::FTS(FtsIndexBuilder::default().block_size(256).unwrap()), + ) .execute() .await .unwrap(); @@ -1372,6 +1377,10 @@ mod tests { assert_eq!(index.index_type, crate::index::IndexType::FTS); assert_eq!(index.columns, vec!["text".to_string()]); assert_eq!(index.name, "text_idx"); + assert_eq!(index.index_version, Some(3)); + let index_params: FtsIndexBuilder = + serde_json::from_str(index.index_details.as_deref().unwrap()).unwrap(); + assert_eq!(index_params.posting_block_size(), 256); let num_rows = 120; let stats = table.index_stats("text_idx").await.unwrap().unwrap();