chore: update lance dependency to v10.0.0-beta.5

Bump version: 0.32.0-beta.2 → 0.32.0-beta.3
Bump version: 0.35.0-beta.2 → 0.35.0-beta.3
2026-08-26 16:08:43 +00:00 · 2026-07-25 04:17:17 +00:00 · 2026-07-24 22:04:02 +00:00 · 2026-07-24 22:03:30 +00:00 · 2026-07-24 15:02:38 -07:00 · 2026-07-24 15:02:23 -07:00
37 changed files with 316 additions and 97 deletions
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.32.0-beta.2"
+current_version = "0.32.0-beta.3"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
@@ -3421,8 +3421,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"

 [[package]]
 name = "fsst"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "rand 0.9.5",
@@ -4777,8 +4777,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"

 [[package]]
 name = "lance"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -4852,8 +4852,8 @@ dependencies = [

 [[package]]
 name = "lance-arrow"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4875,7 +4875,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4889,7 +4889,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -4898,8 +4898,8 @@ dependencies = [

 [[package]]
 name = "lance-bitpacking"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrayref",
 "crunchy",
@@ -4909,8 +4909,8 @@ dependencies = [

 [[package]]
 name = "lance-core"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4933,6 +4933,7 @@ dependencies = [
 "object_store",
 "pin-project",
 "prost",
+ "quick_cache",
 "rand 0.9.5",
 "roaring",
 "serde_json",
@@ -4948,8 +4949,8 @@ dependencies = [

 [[package]]
 name = "lance-datafusion"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4979,8 +4980,8 @@ dependencies = [

 [[package]]
 name = "lance-datagen"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4997,8 +4998,8 @@ dependencies = [

 [[package]]
 name = "lance-derive"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -5007,8 +5008,8 @@ dependencies = [

 [[package]]
 name = "lance-encoding"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5043,8 +5044,8 @@ dependencies = [

 [[package]]
 name = "lance-file"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5074,8 +5075,8 @@ dependencies = [

 [[package]]
 name = "lance-index"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -5142,8 +5143,8 @@ dependencies = [

 [[package]]
 name = "lance-index-core"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5165,8 +5166,8 @@ dependencies = [

 [[package]]
 name = "lance-io"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow",
 "arrow-arith",
@@ -5209,8 +5210,8 @@ dependencies = [

 [[package]]
 name = "lance-linalg"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5226,8 +5227,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow",
 "async-trait",
@@ -5239,8 +5240,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace-impls"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow",
 "arrow-ipc",
@@ -5294,8 +5295,8 @@ dependencies = [

 [[package]]
 name = "lance-select"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5310,8 +5311,8 @@ dependencies = [

 [[package]]
 name = "lance-table"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5350,8 +5351,8 @@ dependencies = [

 [[package]]
 name = "lance-testing"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5364,8 +5365,8 @@ dependencies = [

 [[package]]
 name = "lance-tokenizer"
-version = "9.1.0-beta.8"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
+version = "10.0.0-beta.5"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
 dependencies = [
 "icu_segmenter",
 "jieba-rs",
@@ -5378,7 +5379,7 @@ dependencies = [

 [[package]]
 name = "lancedb"
-version = "0.32.0-beta.2"
+version = "0.32.0-beta.3"
 dependencies = [
 "ahash",
 "anyhow",
@@ -5466,7 +5467,7 @@ dependencies = [

 [[package]]
 name = "lancedb-nodejs"
-version = "0.32.0-beta.2"
+version = "0.32.0-beta.3"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5491,7 +5492,7 @@ dependencies = [

 [[package]]
 name = "lancedb-python"
-version = "0.35.0-beta.2"
+version = "0.35.0-beta.3"
 dependencies = [
 "arrow",
 "async-trait",
@@ -7802,6 +7803,18 @@ dependencies = [
 "memchr",
 ]

+[[package]]
+name = "quick_cache"
+version = "0.6.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9"
+dependencies = [
+ "ahash",
+ "equivalent",
+ "hashbrown 0.16.1",
+ "parking_lot",
+]
+
 [[package]]
 name = "quinn"
 version = "0.11.9"
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"

 [workspace.dependencies]
-lance = { "version" = "=9.1.0-beta.8", default-features = false, "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=9.1.0-beta.8", default-features = false, "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=9.1.0-beta.8", default-features = false, "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false }
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-core</artifactId>
-    <version>0.32.0-beta.2</version>
+    <version>0.32.0-beta.3</version>
 </dependency>
 ```

@@ -43,6 +43,19 @@ The following tokenizers are available:

 ***

+### blockSize?
+
+```ts
+optional blockSize: 128 | 256;
+```
+
+Number of documents per compressed posting block.
+
+The default is 128. Supported values are 128 and 256. A value of 256 uses
+the experimental FTS V3 format and may introduce breaking changes.
+
+***
+
 ### language?

 ```ts
@@ -8,7 +8,7 @@
    <parent>
      <groupId>com.lancedb</groupId>
      <artifactId>lancedb-parent</artifactId>
-      <version>0.32.0-beta.2</version>
+      <version>0.32.0-beta.3</version>
      <relativePath>../pom.xml</relativePath>
    </parent>

@@ -6,7 +6,7 @@

    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-parent</artifactId>
-    <version>0.32.0-beta.2</version>
+    <version>0.32.0-beta.3</version>
    <packaging>pom</packaging>
    <name>${project.artifactId}</name>
    <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>9.1.0-beta.8</lance-core.version>
+        <lance-core.version>10.0.0-beta.5</lance-core.version>
        <spotless.skip>false</spotless.skip>
        <spotless.version>2.30.0</spotless.version>
        <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.32.0-beta.2"
+version = "0.32.0-beta.3"
 publish = false
 license.workspace = true
 description.workspace = true
@@ -15,6 +15,7 @@ import {
  OAuthHeaderProvider,
  StaticHeaderProvider,
 } from "../lancedb/header";
+import { Index } from "../lancedb/indices";

 // Test-only header providers
 class CustomProvider extends HeaderProvider {
@@ -225,6 +226,54 @@ describe("remote connection", () => {
    );
  });

+  it("sends the FTS posting block size to remote tables", async () => {
+    let createIndexBody: Record<string, unknown> | undefined;
+
+    await withMockDatabase(
+      (req, res) => {
+        const path = req.url ?? "";
+        if (path.endsWith("/describe/")) {
+          res.writeHead(200, { "Content-Type": "application/json" }).end(
+            JSON.stringify({
+              name: "t",
+              version: 1,
+              schema: {
+                fields: [
+                  { name: "text", type: { type: "string" }, nullable: false },
+                ],
+              },
+            }),
+          );
+          return;
+        }
+
+        if (path.endsWith("/create_index/")) {
+          let raw = "";
+          req.on("data", (chunk) => {
+            raw += chunk;
+          });
+          req.on("end", () => {
+            createIndexBody = JSON.parse(raw);
+            res.writeHead(200).end();
+          });
+          return;
+        }
+
+        res.writeHead(404).end();
+      },
+      async (db) => {
+        const table = await db.openTable("t");
+        await table.createIndex("text", {
+          config: Index.fts({ blockSize: 256 }),
+        });
+      },
+    );
+
+    expect(createIndexBody?.["column"]).toBe("text");
+    expect(createIndexBody?.["index_type"]).toBe("FTS");
+    expect(createIndexBody?.["block_size"]).toBe(256);
+  });
+
  it("diffs and merges remote branches", async () => {
    const sampleDiff = {
      fromBranch: "exp",
@@ -2527,6 +2527,35 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
      expect(results3.length).toBe(1);
    });

+    test("full text search with custom posting block size", async () => {
+      const db = await connect(tmpDir.name);
+      const data = [
+        { text: "hello world", vector: [0.1, 0.2, 0.3] },
+        { text: "goodbye world", vector: [0.4, 0.5, 0.6] },
+      ];
+      const table = await db.createTable("test", data);
+      await table.createIndex("text", {
+        config: Index.fts({ blockSize: 256 }),
+      });
+
+      const index = (await table.listIndices()).find(
+        (index) => index.indexType === "FTS",
+      );
+      expect(index?.indexVersion).toBe(3);
+      expect(
+        (index?.indexDetails as Record<string, unknown>)["block_size"],
+      ).toBe(256);
+
+      const results = await table.search("hello").toArray();
+      expect(results[0].text).toBe(data[0].text);
+    });
+
+    test("rejects invalid full text posting block size", () => {
+      expect(() => Index.fts({ blockSize: 129 as 128 | 256 })).toThrow(
+        "128 or 256",
+      );
+    });
+
    test("full text search without lowercase", async () => {
      const db = await connect(tmpDir.name);
      const data = [
@@ -572,6 +572,14 @@ export interface FtsOptions {
   * whether to only index the prefix of the token for ngram tokenizer
   */
  prefixOnly?: boolean;
+
+  /**
+   * Number of documents per compressed posting block.
+   *
+   * The default is 128. Supported values are 128 and 256. A value of 256 uses
+   * the experimental FTS V3 format and may introduce breaking changes.
+   */
+  blockSize?: 128 | 256;
 }

 export class Index {
@@ -751,6 +759,7 @@ export class Index {
        options?.ngramMinLength,
        options?.ngramMaxLength,
        options?.prefixOnly,
+        options?.blockSize,
      ),
    );
  }
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.32.0-beta.2",
+	"version": "0.32.0-beta.3",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.32.0-beta.2",
+	"version": "0.32.0-beta.3",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.32.0-beta.2",
+	"version": "0.32.0-beta.3",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.32.0-beta.2",
+	"version": "0.32.0-beta.3",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.32.0-beta.2",
+	"version": "0.32.0-beta.3",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",
@@ -1,6 +1,6 @@
 {
  "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.32.0-beta.2",
+  "version": "0.32.0-beta.3",
  "os": [
    "win32"
  ],
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.32.0-beta.2",
+	"version": "0.32.0-beta.3",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",
@@ -1,12 +1,12 @@
 {
  "name": "@lancedb/lancedb",
-  "version": "0.32.0-beta.2",
+  "version": "0.32.0-beta.3",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "@lancedb/lancedb",
-      "version": "0.32.0-beta.2",
+      "version": "0.32.0-beta.3",
      "cpu": [
        "x64",
        "arm64"
@@ -11,7 +11,7 @@
    "ann"
  ],
  "private": false,
-  "version": "0.32.0-beta.2",
+  "version": "0.32.0-beta.3",
  "main": "dist/index.js",
  "exports": {
    ".": "./dist/index.js",
@@ -226,7 +226,8 @@ impl Index {
        ngram_min_length: Option<u32>,
        ngram_max_length: Option<u32>,
        prefix_only: Option<bool>,
-    ) -> Self {
+        block_size: Option<u32>,
+    ) -> napi::Result<Self> {
        let mut opts = FtsIndexBuilder::default();
        if let Some(with_position) = with_position {
            opts = opts.with_position(with_position);
@@ -261,10 +262,15 @@ impl Index {
        if let Some(prefix_only) = prefix_only {
            opts = opts.ngram_prefix_only(prefix_only);
        }
-
-        Self {
-            inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),
+        if let Some(block_size) = block_size {
+            opts = opts
+                .block_size(block_size as usize)
+                .map_err(|err| napi::Error::from_reason(err.to_string()))?;
        }
+
+        Ok(Self {
+            inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),
+        })
    }

    #[napi(factory)]
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.35.0-beta.2"
+current_version = "0.35.0-beta.3"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.35.0-beta.2"
+version = "0.35.0-beta.3"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
@@ -115,6 +115,12 @@ class FTS:

    For example, it works with `title`, `description`, `content`, etc.

+    Examples
+    --------
+    Create an index configuration that uses 256-document posting blocks:
+
+    >>> config = FTS(block_size=256)
+
    Attributes
    ----------
    with_position : bool, default False
@@ -148,6 +154,10 @@ class FTS:
    ascii_folding : bool, default True
        Whether to fold ASCII characters. This converts accented characters to
        their ASCII equivalent. For example, "café" would be converted to "cafe".
+    block_size : int, default 128
+        The number of documents per compressed posting block. Supported values
+        are 128 and 256. A value of 256 uses the experimental FTS V3 format
+        and may introduce breaking changes.

    Notes
    -----
@@ -168,6 +178,7 @@ class FTS:
    ngram_min_length: int = 3
    ngram_max_length: int = 3
    prefix_only: bool = False
+    block_size: int = 128


@dataclass
@@ -344,6 +344,7 @@ class RemoteTable(Table):
        ngram_min_length: int = 3,
        ngram_max_length: int = 3,
        prefix_only: bool = False,
+        block_size: int = 128,
        name: Optional[str] = None,
    ):
        """Create a full-text search index on a column.
@@ -364,6 +365,7 @@ class RemoteTable(Table):
            ngram_min_length=ngram_min_length,
            ngram_max_length=ngram_max_length,
            prefix_only=prefix_only,
+            block_size=block_size,
        )
        LOOP.run(
            self._table.create_index(
@@ -23,7 +23,7 @@ class AnswerdotaiRerankers(Reranker):
    column : str, default "text"
        The name of the column to use as input to the cross encoder model.
    return_score : str, default "relevance"
-        options are "relevance" or "all". Only "relevance" is supported for now.
+        options are "relevance" or "all".
    **kwargs
        Additional keyword arguments to pass to the model. For example, 'device'.
        See AnswerDotAI/rerankers for more information.
@@ -77,12 +77,13 @@ class AnswerdotaiRerankers(Reranker):
        vector_results: pa.Table,
        fts_results: pa.Table,
    ):
-        combined_results = self.merge_results(vector_results, fts_results)
+        if self.score == "all":
+            combined_results = self._merge_and_keep_scores(vector_results, fts_results)
+        else:
+            combined_results = self.merge_results(vector_results, fts_results)
        combined_results = self._rerank(combined_results, query)
        if self.score == "relevance":
            combined_results = self._keep_relevance_score(combined_results)
-        elif self.score == "all":
-            combined_results = self._merge_and_keep_scores(vector_results, fts_results)
        combined_results = combined_results.sort_by(
            [("_relevance_score", "descending")]
        )
@@ -16,7 +16,7 @@ class ColbertReranker(AnswerdotaiRerankers):
    column : str, default "text"
        The name of the column to use as input to the cross encoder model.
    return_score : str, default "relevance"
-        options are "relevance" or "all". Only "relevance" is supported for now.
+        options are "relevance" or "all".
    **kwargs
        Additional keyword arguments to pass to the model, for example, 'device'.
        See AnswerDotAI/rerankers for more information.
@@ -1106,6 +1106,7 @@ class Table(ABC):
        ngram_min_length: int = 3,
        ngram_max_length: int = 3,
        prefix_only: bool = False,
+        block_size: int = 128,
        wait_timeout: Optional[timedelta] = None,
        name: Optional[str] = None,
    ):
@@ -1177,6 +1178,10 @@ class Table(ABC):
            The maximum length of an n-gram.
        prefix_only: bool, default False
            Whether to only index the prefix of the token for ngram tokenizer.
+        block_size: int, default 128
+            The number of documents per compressed posting block. Must be 128
+            or 256. A value of 256 uses the experimental FTS V3 format and
+            may introduce breaking changes.
        wait_timeout: timedelta, optional
            The timeout to wait if indexing is asynchronous.
        name: str, optional
@@ -3026,6 +3031,7 @@ class LanceTable(Table):
        ngram_min_length: int = 3,
        ngram_max_length: int = 3,
        prefix_only: bool = False,
+        block_size: int = 128,
        name: Optional[str] = None,
    ):
        """Create a full-text search index on a column.
@@ -3075,9 +3081,7 @@ class LanceTable(Table):
        else:
            tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)

-        config = FTS(
-            **tokenizer_configs,
-        )
+        config = FTS(block_size=block_size, **tokenizer_configs)

        try:
            LOOP.run(
@@ -226,6 +226,23 @@ def test_create_inverted_index(table, with_position):
    assert any(i.name == "custom_fts_index" for i in fts_indices)


+@pytest.mark.parametrize("block_size", [128, 256])
+def test_create_inverted_index_block_size(table, block_size):
+    table.create_index("text", config=FTS(block_size=block_size))
+
+    index = next(index for index in table.list_indices() if index.index_type == "FTS")
+    assert index.index_details["block_size"] == block_size
+    assert index.index_version == (2 if block_size == 128 else 3)
+
+    results = table.search("puppy").limit(5).to_list()
+    assert len(results) == 5
+
+
+def test_create_inverted_index_rejects_invalid_block_size(table):
+    with pytest.raises(ValueError, match="128 or 256"):
+        table.create_index("text", config=FTS(block_size=129))
+
+
 def test_search_fts(table):
    table.create_fts_index("text")
    results = table.search("puppy").select(["id", "text"]).limit(5).to_list()
@@ -768,7 +768,10 @@ def test_table_create_indices():
        # Test create_fts_index with custom name (legacy method)
        with pytest.warns(DeprecationWarning, match="create_fts_index"):
            table.create_fts_index(
-                "text", wait_timeout=timedelta(seconds=2), name="custom_fts_idx"
+                "text",
+                wait_timeout=timedelta(seconds=2),
+                block_size=256,
+                name="custom_fts_idx",
            )

        # Test create_index with custom name (legacy form: vector_column_name kwarg)
@@ -791,6 +794,7 @@ def test_table_create_indices():
        fts_req = received_requests[1]
        assert "name" in fts_req
        assert fts_req["name"] == "custom_fts_idx"
+        assert fts_req["block_size"] == 256

        # Check vector index request has custom name
        vector_req = received_requests[2]
@@ -876,7 +880,7 @@ def test_remote_create_index_new_api():
            _warnings.simplefilter("error", DeprecationWarning)
            table.create_index("vector", config=IvfPq(distance_type="l2"))
            table.create_index("category", config=BTree())
-            table.create_index("text", config=FTS())
+            table.create_index("text", config=FTS(block_size=256))
            # IvfRq via new API
            table.create_index("vector", config=IvfRq(distance_type="l2"))

@@ -896,6 +900,7 @@ def test_remote_create_index_new_api():
            "vector",
            "vector",
        ]
+        assert received_requests[2]["block_size"] == 256


 def test_table_wait_for_index_timeout():
@@ -644,6 +644,21 @@ def test_cross_encoder_reranker_return_all(tmp_path):
    assert "_distance" in result.column_names


+def test_answerdotai_reranker_return_all(tmp_path):
+    pytest.importorskip("rerankers")
+    reranker = AnswerdotaiRerankers(return_score="all")
+    table, schema = get_test_table(tmp_path)
+    query = "single player experience"
+    result = (
+        table.search(query, query_type="hybrid", vector_column_name="vector")
+        .rerank(reranker=reranker)
+        .to_arrow()
+    )
+    assert "_relevance_score" in result.column_names
+    assert "_score" in result.column_names
+    assert "_distance" in result.column_names
+
+
 # ---------------------------------------------------------------------------
 # Regression tests for LinearCombinationReranker scoring bugs (issue #3154)
 # ---------------------------------------------------------------------------
@@ -60,6 +60,9 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                    .ngram_min_length(params.ngram_min_length)
                    .ngram_max_length(params.ngram_max_length)
                    .ngram_prefix_only(params.prefix_only);
+                let inner_opts = inner_opts
+                    .block_size(params.block_size)
+                    .map_err(|err| PyValueError::new_err(err.to_string()))?;
                Ok(LanceDbIndex::FTS(inner_opts))
            }
            "IvfFlat" => {
@@ -207,6 +210,7 @@ struct FtsParams {
    ngram_min_length: u32,
    ngram_max_length: u32,
    prefix_only: bool,
+    block_size: usize,
 }

 #[derive(FromPyObject)]
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.32.0-beta.2"
+version = "0.32.0-beta.3"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -273,7 +273,11 @@ pub(crate) async fn take_blobs_aligned(
        if *is_null {
            builder.append_null();
        } else {
-            builder.append_value(payloads[payload_idx].data.as_ref());
+            if let Some(data) = &payloads[payload_idx].data {
+                builder.append_value(data);
+            } else {
+                builder.append_null();
+            }
            payload_idx += 1;
        }
    }
@@ -315,7 +319,7 @@ pub(crate) async fn take_blob_files_aligned(
            if *is_null {
                None
            } else {
-                Some(handles.next().unwrap())
+                handles.next().flatten()
            }
        })
        .collect())
@@ -54,7 +54,26 @@ pub enum Index {
    /// substrings of the raw bytes, unlike the tokenized [`Index::FTS`] index.
    Fm(FmIndexBuilder),

-    /// Full text search index using bm25.
+    /// Full text search index using BM25.
+    ///
+    /// The posting block size defaults to 128. Supported values are 128 and 256;
+    /// a value of 256 uses the experimental FTS V3 format and may introduce
+    /// breaking changes.
+    ///
+    /// ```
+    /// use lancedb::index::{Index, scalar::FtsIndexBuilder};
+    ///
+    /// # async fn create_fts_index(
+    /// #     table: &lancedb::Table,
+    /// # ) -> Result<(), Box<dyn std::error::Error>> {
+    /// let params = FtsIndexBuilder::default().block_size(256)?;
+    /// table
+    ///     .create_index(&["text"], Index::FTS(params))
+    ///     .execute()
+    ///     .await?;
+    /// # Ok(())
+    /// # }
+    /// ```
    FTS(FtsIndexBuilder),

    /// IVF index
@@ -4496,6 +4496,15 @@ mod tests {
                serde_json::to_value(InvertedIndexParams::default()).unwrap(),
                Index::FTS(Default::default()),
            ),
+            (
+                "FTS",
+                {
+                    let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap();
+                    body["block_size"] = 256.into();
+                    body
+                },
+                Index::FTS(InvertedIndexParams::default().block_size(256).unwrap()),
+            ),
        ];

        for (index_type, expected_body, index) in cases {
@@ -382,7 +382,9 @@ mod tests {
    use crate::connect;
    use crate::connection::ConnectBuilder;
    use crate::index::Index;
-    use crate::index::scalar::{BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder};
+    use crate::index::scalar::{
+        BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder, FtsIndexBuilder,
+    };
    use crate::index::vector::{
        IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder,
    };
@@ -1362,7 +1364,10 @@ mod tests {
            .unwrap();

        table
-            .create_index(&["text"], Index::FTS(Default::default()))
+            .create_index(
+                &["text"],
+                Index::FTS(FtsIndexBuilder::default().block_size(256).unwrap()),
+            )
            .execute()
            .await
            .unwrap();
@@ -1372,6 +1377,10 @@ mod tests {
        assert_eq!(index.index_type, crate::index::IndexType::FTS);
        assert_eq!(index.columns, vec!["text".to_string()]);
        assert_eq!(index.name, "text_idx");
+        assert_eq!(index.index_version, Some(3));
+        let index_params: FtsIndexBuilder =
+            serde_json::from_str(index.index_details.as_deref().unwrap()).unwrap();
+        assert_eq!(index_params.posting_block_size(), 256);

        let num_rows = 120;
        let stats = table.index_stats("text_idx").await.unwrap().unwrap();
Author	SHA1	Message	Date
lancedb automation	5eedeea120	chore: update lance dependency to v10.0.0-beta.5	2026-07-25 04:17:17 +00:00
Lance Release	a00edef0e6	Bump version: 0.32.0-beta.2 → 0.32.0-beta.3	2026-07-24 22:04:02 +00:00
Lance Release	1b2670443e	Bump version: 0.35.0-beta.2 → 0.35.0-beta.3	2026-07-24 22:03:30 +00:00
Yang Cen	9dc5ec03aa	feat(fts): add block size configuration (#3691 ) ## What changed - add `block_size` to Python FTS configuration and the deprecated local/remote helpers - add `blockSize` to the TypeScript FTS options and propagate it through the NAPI binding - serialize the value as `block_size` for remote index creation - document the existing Rust builder API and generate the TypeScript API reference - add local, remote, metadata, search, and invalid-value regression coverage ## Why Lance supports configuring the number of documents per compressed FTS posting block, but LanceDB's Python and TypeScript APIs did not expose the setting. This made the experimental FTS V3 layout unavailable through those clients and allowed the value to be dropped before index creation. ## How it works The default remains `128`. Supported values are `128` and `256`; selecting `256` uses the experimental FTS V3 format. Invalid values are rejected by the Lance builder and surfaced as Python or JavaScript errors. ## Validation - `cargo check --quiet --features remote --tests --examples` - `cargo +1.94.0 clippy --quiet --features remote --tests --examples -- -D warnings` - targeted Rust local and remote index tests - Rust doctests: 34 passed - Python Ruff checks, doctest, and targeted local/remote tests: 5 passed - TypeScript build, Biome lint, generated docs, and targeted Jest tests: 9 passed - `git diff --check` ## Limitations The Java client remains unchanged because its external remote REST model does not currently expose `block_size`. Co-authored-by: Yang Cen <yangcen@Yangs-Mac-mini.local>	2026-07-24 15:02:38 -07:00
Andrew Chen	18760f74cd	fix: crash in AnswerdotaiRerankers/ColbertReranker for return_score="all" (#3671 ) ## What `AnswerdotaiRerankers(return_score="all").rerank_hybrid(...)` (and `ColbertReranker`, which subclasses it without overriding `rerank_hybrid`) raises: ``` pyarrow.lib.ArrowInvalid: Invalid sort key column: No match for FieldRef.Name(_relevance_score) in _rowid: int64 ... ``` ## Why ```python combined_results = self.merge_results(vector_results, fts_results) combined_results = self._rerank(combined_results, query) if self.score == "relevance": combined_results = self._keep_relevance_score(combined_results) elif self.score == "all": combined_results = self._merge_and_keep_scores(vector_results, fts_results) ``` When `score == "all"`, `combined_results` is unconditionally overwritten by `_merge_and_keep_scores(vector_results, fts_results)` after `_rerank()` already computed and appended `_relevance_score` — discarding it. The following `sort_by("_relevance_score", ...)` then has nothing to sort on. Every sibling reranker that supports `return_score="all"` (`cross_encoder`, `openai`, `cohere`, `jinaai`, `voyageai`, `watsonx`) instead calls `_merge_and_keep_scores()` before `_rerank()`. This file is the one place the ordering got inverted when `"all"` support was added (#2509) — a copy/paste inconsistency across the six files that PR touched. Fix mirrors the pattern already used (and tested) by the other five rerankers. Also drops the now-stale `"Only 'relevance' is supported for now"` docstring line on both classes, left over from before `"all"` support existed. ## Testing Added `test_answerdotai_reranker_return_all`, mirroring the existing `test_cross_encoder_reranker_return_all`. Verified locally with the real built Rust extension: red (reproduces the exact `ArrowInvalid` above) → green, using the actual `rerank_hybrid`/`_rerank`/`base.py` code path with the model call mocked out — my local environment's `rerankers==0.10.0` fails to load the real ColBERT model against the available `transformers` version (`AttributeError: 'ColBERTModel' object has no attribute 'all_tied_weights_keys'`), which I confirmed also breaks the pre-existing, unmodified `test_colbert_reranker`/`test_answerdotai_reranker` baseline tests identically — an unrelated local dependency-version issue, not a regression from this change. `ruff check`/`ruff format` clean; full `test_rerankers.py` run: 9 passed / 8 skipped / 3 failed (the 3 failures are exactly those two pre-existing tests plus my new one, all failing at model-loading time for the same unrelated reason before reaching the changed code). --- Disclosure: this PR was drafted with AI assistance (Claude); I reviewed, tested, and take responsibility for the change. Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>	2026-07-24 15:02:23 -07:00
LanceDB Robot	c9d07ef6fc	chore: update lance dependency to v10.0.0-beta.3 (#3710 ) Updates the Rust workspace and Java lance-core dependencies to [Lance v10.0.0-beta.3](https://github.com/lance-format/lance/releases/tag/v10.0.0-beta.3). Includes compatibility updates for Lance’s nullable blob payload and handle APIs.	2026-07-24 15:01:30 -07:00