feat(fts): add block size configuration (#3691)

## What changed

- add `block_size` to Python FTS configuration and the deprecated
local/remote helpers
- add `blockSize` to the TypeScript FTS options and propagate it through
the NAPI binding
- serialize the value as `block_size` for remote index creation
- document the existing Rust builder API and generate the TypeScript API
reference
- add local, remote, metadata, search, and invalid-value regression
coverage

## Why

Lance supports configuring the number of documents per compressed FTS
posting block, but LanceDB's Python and TypeScript APIs did not expose
the setting. This made the experimental FTS V3 layout unavailable
through those clients and allowed the value to be dropped before index
creation.

## How it works

The default remains `128`. Supported values are `128` and `256`;
selecting `256` uses the experimental FTS V3 format. Invalid values are
rejected by the Lance builder and surfaced as Python or JavaScript
errors.

## Validation

- `cargo check --quiet --features remote --tests --examples`
- `cargo +1.94.0 clippy --quiet --features remote --tests --examples --
-D warnings`
- targeted Rust local and remote index tests
- Rust doctests: 34 passed
- Python Ruff checks, doctest, and targeted local/remote tests: 5 passed
- TypeScript build, Biome lint, generated docs, and targeted Jest tests:
9 passed
- `git diff --check`

## Limitations

The Java client remains unchanged because its external remote REST model
does not currently expose `block_size`.

Co-authored-by: Yang Cen <yangcen@Yangs-Mac-mini.local>
This commit is contained in:
Yang Cen
2026-07-25 06:02:38 +08:00
committed by GitHub
parent 18760f74cd
commit 9dc5ec03aa
14 changed files with 198 additions and 12 deletions
+49
View File
@@ -15,6 +15,7 @@ import {
OAuthHeaderProvider,
StaticHeaderProvider,
} from "../lancedb/header";
import { Index } from "../lancedb/indices";
// Test-only header providers
class CustomProvider extends HeaderProvider {
@@ -225,6 +226,54 @@ describe("remote connection", () => {
);
});
it("sends the FTS posting block size to remote tables", async () => {
let createIndexBody: Record<string, unknown> | undefined;
await withMockDatabase(
(req, res) => {
const path = req.url ?? "";
if (path.endsWith("/describe/")) {
res.writeHead(200, { "Content-Type": "application/json" }).end(
JSON.stringify({
name: "t",
version: 1,
schema: {
fields: [
{ name: "text", type: { type: "string" }, nullable: false },
],
},
}),
);
return;
}
if (path.endsWith("/create_index/")) {
let raw = "";
req.on("data", (chunk) => {
raw += chunk;
});
req.on("end", () => {
createIndexBody = JSON.parse(raw);
res.writeHead(200).end();
});
return;
}
res.writeHead(404).end();
},
async (db) => {
const table = await db.openTable("t");
await table.createIndex("text", {
config: Index.fts({ blockSize: 256 }),
});
},
);
expect(createIndexBody?.["column"]).toBe("text");
expect(createIndexBody?.["index_type"]).toBe("FTS");
expect(createIndexBody?.["block_size"]).toBe(256);
});
it("diffs and merges remote branches", async () => {
const sampleDiff = {
fromBranch: "exp",
+29
View File
@@ -2527,6 +2527,35 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
expect(results3.length).toBe(1);
});
test("full text search with custom posting block size", async () => {
const db = await connect(tmpDir.name);
const data = [
{ text: "hello world", vector: [0.1, 0.2, 0.3] },
{ text: "goodbye world", vector: [0.4, 0.5, 0.6] },
];
const table = await db.createTable("test", data);
await table.createIndex("text", {
config: Index.fts({ blockSize: 256 }),
});
const index = (await table.listIndices()).find(
(index) => index.indexType === "FTS",
);
expect(index?.indexVersion).toBe(3);
expect(
(index?.indexDetails as Record<string, unknown>)["block_size"],
).toBe(256);
const results = await table.search("hello").toArray();
expect(results[0].text).toBe(data[0].text);
});
test("rejects invalid full text posting block size", () => {
expect(() => Index.fts({ blockSize: 129 as 128 | 256 })).toThrow(
"128 or 256",
);
});
test("full text search without lowercase", async () => {
const db = await connect(tmpDir.name);
const data = [