build: point lance dependencies at the sibling submodule checkout

Path deps replace the git+tag pins so the sophon build resolves lance
from src/lance directly, independent of the pinned tag's version.

Cargo.lock

@@ -3421,8 +3421,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow-array",
  "rand 0.9.5",
@@ -4777,8 +4777,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
 [[package]]
 name = "lance"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -4852,8 +4852,8 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4875,7 +4875,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4889,7 +4889,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -4898,8 +4898,8 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrayref",
  "crunchy",
@@ -4909,8 +4909,8 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4948,8 +4948,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4979,8 +4979,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4997,8 +4997,8 @@ dependencies = [
 
 [[package]]
 name = "lance-derive"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5007,8 +5007,8 @@ dependencies = [
 
 [[package]]
 name = "lance-encoding"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5043,8 +5043,8 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5074,8 +5074,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -5142,8 +5142,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index-core"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5165,8 +5165,8 @@ dependencies = [
 
 [[package]]
 name = "lance-io"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -5209,8 +5209,8 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5226,8 +5226,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow",
  "async-trait",
@@ -5239,8 +5239,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -5294,8 +5294,8 @@ dependencies = [
 
 [[package]]
 name = "lance-select"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5310,8 +5310,8 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5350,8 +5350,8 @@ dependencies = [
 
 [[package]]
 name = "lance-testing"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5364,8 +5364,8 @@ dependencies = [
 
 [[package]]
 name = "lance-tokenizer"
-version = "10.0.0-beta.3"
-source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.3#ed078e7c19c9560906e1eae54b0baf01dc509986"
+version = "10.0.0-beta.2"
+source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.2#bbc04776fff39355a7287e6feacb0041395a45e1"
 dependencies = [
  "icu_segmenter",
  "jieba-rs",

Cargo.toml

@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"
 
 [workspace.dependencies]
-lance = { "version" = "=10.0.0-beta.3", default-features = false, "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=10.0.0-beta.3", default-features = false, "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=10.0.0-beta.3", default-features = false, "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=10.0.0-beta.3", "tag" = "v10.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
+lance = { path = "../lance/rust/lance", default-features = false }
+lance-core = { path = "../lance/rust/lance-core" }
+lance-datagen = { path = "../lance/rust/lance-datagen" }
+lance-file = { path = "../lance/rust/lance-file" }
+lance-io = { path = "../lance/rust/lance-io", default-features = false }
+lance-index = { path = "../lance/rust/lance-index" }
+lance-linalg = { path = "../lance/rust/lance-linalg" }
+lance-namespace = { path = "../lance/rust/lance-namespace" }
+lance-namespace-impls = { path = "../lance/rust/lance-namespace-impls", default-features = false }
+lance-table = { path = "../lance/rust/lance-table" }
+lance-testing = { path = "../lance/rust/lance-testing" }
+lance-datafusion = { path = "../lance/rust/lance-datafusion" }
+lance-encoding = { path = "../lance/rust/lance-encoding" }
+lance-arrow = { path = "../lance/rust/lance-arrow" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false }

docs/src/js/interfaces/FtsOptions.md

@@ -43,19 +43,6 @@ The following tokenizers are available:
 
 ***
 
-### blockSize?
-
-```ts
-optional blockSize: 128 | 256;
-```
-
-Number of documents per compressed posting block.
-
-The default is 128. Supported values are 128 and 256. A value of 256 uses
-the experimental FTS V3 format and may introduce breaking changes.
-
-***
-
 ### language?
 
 ```ts

java/pom.xml

@@ -28,7 +28,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>10.0.0-beta.3</lance-core.version>
+        <lance-core.version>10.0.0-beta.2</lance-core.version>
         <spotless.skip>false</spotless.skip>
         <spotless.version>2.30.0</spotless.version>
         <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>

nodejs/__test__/remote.test.ts

@@ -15,7 +15,6 @@ import {
   OAuthHeaderProvider,
   StaticHeaderProvider,
 } from "../lancedb/header";
-import { Index } from "../lancedb/indices";
 
 // Test-only header providers
 class CustomProvider extends HeaderProvider {
@@ -226,54 +225,6 @@ describe("remote connection", () => {
     );
   });
 
-  it("sends the FTS posting block size to remote tables", async () => {
-    let createIndexBody: Record<string, unknown> | undefined;
-
-    await withMockDatabase(
-      (req, res) => {
-        const path = req.url ?? "";
-        if (path.endsWith("/describe/")) {
-          res.writeHead(200, { "Content-Type": "application/json" }).end(
-            JSON.stringify({
-              name: "t",
-              version: 1,
-              schema: {
-                fields: [
-                  { name: "text", type: { type: "string" }, nullable: false },
-                ],
-              },
-            }),
-          );
-          return;
-        }
-
-        if (path.endsWith("/create_index/")) {
-          let raw = "";
-          req.on("data", (chunk) => {
-            raw += chunk;
-          });
-          req.on("end", () => {
-            createIndexBody = JSON.parse(raw);
-            res.writeHead(200).end();
-          });
-          return;
-        }
-
-        res.writeHead(404).end();
-      },
-      async (db) => {
-        const table = await db.openTable("t");
-        await table.createIndex("text", {
-          config: Index.fts({ blockSize: 256 }),
-        });
-      },
-    );
-
-    expect(createIndexBody?.["column"]).toBe("text");
-    expect(createIndexBody?.["index_type"]).toBe("FTS");
-    expect(createIndexBody?.["block_size"]).toBe(256);
-  });
-
   it("diffs and merges remote branches", async () => {
     const sampleDiff = {
       fromBranch: "exp",

nodejs/__test__/table.test.ts

@@ -2527,35 +2527,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       expect(results3.length).toBe(1);
     });
 
-    test("full text search with custom posting block size", async () => {
-      const db = await connect(tmpDir.name);
-      const data = [
-        { text: "hello world", vector: [0.1, 0.2, 0.3] },
-        { text: "goodbye world", vector: [0.4, 0.5, 0.6] },
-      ];
-      const table = await db.createTable("test", data);
-      await table.createIndex("text", {
-        config: Index.fts({ blockSize: 256 }),
-      });
-
-      const index = (await table.listIndices()).find(
-        (index) => index.indexType === "FTS",
-      );
-      expect(index?.indexVersion).toBe(3);
-      expect(
-        (index?.indexDetails as Record<string, unknown>)["block_size"],
-      ).toBe(256);
-
-      const results = await table.search("hello").toArray();
-      expect(results[0].text).toBe(data[0].text);
-    });
-
-    test("rejects invalid full text posting block size", () => {
-      expect(() => Index.fts({ blockSize: 129 as 128 | 256 })).toThrow(
-        "128 or 256",
-      );
-    });
-
     test("full text search without lowercase", async () => {
       const db = await connect(tmpDir.name);
       const data = [

nodejs/lancedb/indices.ts

@@ -572,14 +572,6 @@ export interface FtsOptions {
    * whether to only index the prefix of the token for ngram tokenizer
    */
   prefixOnly?: boolean;
-
-  /**
-   * Number of documents per compressed posting block.
-   *
-   * The default is 128. Supported values are 128 and 256. A value of 256 uses
-   * the experimental FTS V3 format and may introduce breaking changes.
-   */
-  blockSize?: 128 | 256;
 }
 
 export class Index {
@@ -759,7 +751,6 @@ export class Index {
         options?.ngramMinLength,
         options?.ngramMaxLength,
         options?.prefixOnly,
-        options?.blockSize,
       ),
     );
   }

nodejs/src/index.rs

@@ -226,8 +226,7 @@ impl Index {
         ngram_min_length: Option<u32>,
         ngram_max_length: Option<u32>,
         prefix_only: Option<bool>,
-        block_size: Option<u32>,
-    ) -> napi::Result<Self> {
+    ) -> Self {
         let mut opts = FtsIndexBuilder::default();
         if let Some(with_position) = with_position {
             opts = opts.with_position(with_position);
@@ -262,15 +261,10 @@ impl Index {
         if let Some(prefix_only) = prefix_only {
             opts = opts.ngram_prefix_only(prefix_only);
         }
-        if let Some(block_size) = block_size {
-            opts = opts
-                .block_size(block_size as usize)
-                .map_err(|err| napi::Error::from_reason(err.to_string()))?;
-        }
 
-        Ok(Self {
+        Self {
             inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),
-        })
+        }
     }
 
     #[napi(factory)]

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.35.0-beta.3"
+current_version = "0.35.0-beta.2"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.35.0-beta.3"
+version = "0.35.0-beta.2"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"

python/python/lancedb/index.py

@@ -115,12 +115,6 @@ class FTS:
 
     For example, it works with `title`, `description`, `content`, etc.
 
-    Examples
-    --------
-    Create an index configuration that uses 256-document posting blocks:
-
-    >>> config = FTS(block_size=256)
-
     Attributes
     ----------
     with_position : bool, default False
@@ -154,10 +148,6 @@ class FTS:
     ascii_folding : bool, default True
         Whether to fold ASCII characters. This converts accented characters to
         their ASCII equivalent. For example, "café" would be converted to "cafe".
-    block_size : int, default 128
-        The number of documents per compressed posting block. Supported values
-        are 128 and 256. A value of 256 uses the experimental FTS V3 format
-        and may introduce breaking changes.
 
     Notes
     -----
@@ -178,7 +168,6 @@ class FTS:
     ngram_min_length: int = 3
     ngram_max_length: int = 3
     prefix_only: bool = False
-    block_size: int = 128
 
 
 @dataclass

python/python/lancedb/remote/table.py

@@ -344,7 +344,6 @@ class RemoteTable(Table):
         ngram_min_length: int = 3,
         ngram_max_length: int = 3,
         prefix_only: bool = False,
-        block_size: int = 128,
         name: Optional[str] = None,
     ):
         """Create a full-text search index on a column.
@@ -365,7 +364,6 @@ class RemoteTable(Table):
             ngram_min_length=ngram_min_length,
             ngram_max_length=ngram_max_length,
             prefix_only=prefix_only,
-            block_size=block_size,
         )
         LOOP.run(
             self._table.create_index(

python/python/lancedb/rerankers/answerdotai.py

@@ -23,7 +23,7 @@ class AnswerdotaiRerankers(Reranker):
     column : str, default "text"
         The name of the column to use as input to the cross encoder model.
     return_score : str, default "relevance"
-        options are "relevance" or "all".
+        options are "relevance" or "all". Only "relevance" is supported for now.
     **kwargs
         Additional keyword arguments to pass to the model. For example, 'device'.
         See AnswerDotAI/rerankers for more information.
@@ -77,13 +77,12 @@ class AnswerdotaiRerankers(Reranker):
         vector_results: pa.Table,
         fts_results: pa.Table,
     ):
-        if self.score == "all":
-            combined_results = self._merge_and_keep_scores(vector_results, fts_results)
-        else:
-            combined_results = self.merge_results(vector_results, fts_results)
+        combined_results = self.merge_results(vector_results, fts_results)
         combined_results = self._rerank(combined_results, query)
         if self.score == "relevance":
             combined_results = self._keep_relevance_score(combined_results)
+        elif self.score == "all":
+            combined_results = self._merge_and_keep_scores(vector_results, fts_results)
         combined_results = combined_results.sort_by(
             [("_relevance_score", "descending")]
         )

python/python/lancedb/rerankers/colbert.py

@@ -16,7 +16,7 @@ class ColbertReranker(AnswerdotaiRerankers):
     column : str, default "text"
         The name of the column to use as input to the cross encoder model.
     return_score : str, default "relevance"
-        options are "relevance" or "all".
+        options are "relevance" or "all". Only "relevance" is supported for now.
     **kwargs
         Additional keyword arguments to pass to the model, for example, 'device'.
         See AnswerDotAI/rerankers for more information.

python/python/lancedb/table.py

@@ -1106,7 +1106,6 @@ class Table(ABC):
         ngram_min_length: int = 3,
         ngram_max_length: int = 3,
         prefix_only: bool = False,
-        block_size: int = 128,
         wait_timeout: Optional[timedelta] = None,
         name: Optional[str] = None,
     ):
@@ -1178,10 +1177,6 @@ class Table(ABC):
             The maximum length of an n-gram.
         prefix_only: bool, default False
             Whether to only index the prefix of the token for ngram tokenizer.
-        block_size: int, default 128
-            The number of documents per compressed posting block. Must be 128
-            or 256. A value of 256 uses the experimental FTS V3 format and
-            may introduce breaking changes.
         wait_timeout: timedelta, optional
             The timeout to wait if indexing is asynchronous.
         name: str, optional
@@ -3031,7 +3026,6 @@ class LanceTable(Table):
         ngram_min_length: int = 3,
         ngram_max_length: int = 3,
         prefix_only: bool = False,
-        block_size: int = 128,
         name: Optional[str] = None,
     ):
         """Create a full-text search index on a column.
@@ -3081,7 +3075,9 @@ class LanceTable(Table):
         else:
             tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
 
-        config = FTS(block_size=block_size, **tokenizer_configs)
+        config = FTS(
+            **tokenizer_configs,
+        )
 
         try:
             LOOP.run(

python/python/tests/test_fts.py

@@ -226,23 +226,6 @@ def test_create_inverted_index(table, with_position):
     assert any(i.name == "custom_fts_index" for i in fts_indices)
 
 
-@pytest.mark.parametrize("block_size", [128, 256])
-def test_create_inverted_index_block_size(table, block_size):
-    table.create_index("text", config=FTS(block_size=block_size))
-
-    index = next(index for index in table.list_indices() if index.index_type == "FTS")
-    assert index.index_details["block_size"] == block_size
-    assert index.index_version == (2 if block_size == 128 else 3)
-
-    results = table.search("puppy").limit(5).to_list()
-    assert len(results) == 5
-
-
-def test_create_inverted_index_rejects_invalid_block_size(table):
-    with pytest.raises(ValueError, match="128 or 256"):
-        table.create_index("text", config=FTS(block_size=129))
-
-
 def test_search_fts(table):
     table.create_fts_index("text")
     results = table.search("puppy").select(["id", "text"]).limit(5).to_list()

python/python/tests/test_remote_db.py

@@ -768,10 +768,7 @@ def test_table_create_indices():
         # Test create_fts_index with custom name (legacy method)
         with pytest.warns(DeprecationWarning, match="create_fts_index"):
             table.create_fts_index(
-                "text",
-                wait_timeout=timedelta(seconds=2),
-                block_size=256,
-                name="custom_fts_idx",
+                "text", wait_timeout=timedelta(seconds=2), name="custom_fts_idx"
             )
 
         # Test create_index with custom name (legacy form: vector_column_name kwarg)
@@ -794,7 +791,6 @@ def test_table_create_indices():
         fts_req = received_requests[1]
         assert "name" in fts_req
         assert fts_req["name"] == "custom_fts_idx"
-        assert fts_req["block_size"] == 256
 
         # Check vector index request has custom name
         vector_req = received_requests[2]
@@ -880,7 +876,7 @@ def test_remote_create_index_new_api():
             _warnings.simplefilter("error", DeprecationWarning)
             table.create_index("vector", config=IvfPq(distance_type="l2"))
             table.create_index("category", config=BTree())
-            table.create_index("text", config=FTS(block_size=256))
+            table.create_index("text", config=FTS())
             # IvfRq via new API
             table.create_index("vector", config=IvfRq(distance_type="l2"))
 
@@ -900,7 +896,6 @@ def test_remote_create_index_new_api():
             "vector",
             "vector",
         ]
-        assert received_requests[2]["block_size"] == 256
 
 
 def test_table_wait_for_index_timeout():

python/python/tests/test_rerankers.py

@@ -644,21 +644,6 @@ def test_cross_encoder_reranker_return_all(tmp_path):
     assert "_distance" in result.column_names
 
 
-def test_answerdotai_reranker_return_all(tmp_path):
-    pytest.importorskip("rerankers")
-    reranker = AnswerdotaiRerankers(return_score="all")
-    table, schema = get_test_table(tmp_path)
-    query = "single player experience"
-    result = (
-        table.search(query, query_type="hybrid", vector_column_name="vector")
-        .rerank(reranker=reranker)
-        .to_arrow()
-    )
-    assert "_relevance_score" in result.column_names
-    assert "_score" in result.column_names
-    assert "_distance" in result.column_names
-
-
 # ---------------------------------------------------------------------------
 # Regression tests for LinearCombinationReranker scoring bugs (issue #3154)
 # ---------------------------------------------------------------------------

python/src/index.rs

@@ -60,9 +60,6 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     .ngram_min_length(params.ngram_min_length)
                     .ngram_max_length(params.ngram_max_length)
                     .ngram_prefix_only(params.prefix_only);
-                let inner_opts = inner_opts
-                    .block_size(params.block_size)
-                    .map_err(|err| PyValueError::new_err(err.to_string()))?;
                 Ok(LanceDbIndex::FTS(inner_opts))
             }
             "IvfFlat" => {
@@ -210,7 +207,6 @@ struct FtsParams {
     ngram_min_length: u32,
     ngram_max_length: u32,
     prefix_only: bool,
-    block_size: usize,
 }
 
 #[derive(FromPyObject)]

rust/lancedb/src/blob.rs

@@ -273,11 +273,15 @@ pub(crate) async fn take_blobs_aligned(
         if *is_null {
             builder.append_null();
         } else {
-            if let Some(data) = &payloads[payload_idx].data {
-                builder.append_value(data);
-            } else {
-                builder.append_null();
-            }
+            let data = payloads[payload_idx]
+                .data
+                .as_ref()
+                .ok_or_else(|| Error::Runtime {
+                    message: format!(
+                        "blob read for column '{column}' returned null payload for a non-null row"
+                    ),
+                })?;
+            builder.append_value(data);
             payload_idx += 1;
         }
     }
@@ -313,16 +317,20 @@ pub(crate) async fn take_blob_files_aligned(
     }
 
     let mut handles = handles.into_iter();
-    Ok(null_mask
-        .iter()
-        .map(|is_null| {
-            if *is_null {
-                None
-            } else {
-                handles.next().flatten()
-            }
-        })
-        .collect())
+    let mut aligned_handles = Vec::with_capacity(row_ids.len());
+    for is_null in null_mask {
+        if is_null {
+            aligned_handles.push(None);
+        } else {
+            let handle = handles.next().unwrap().ok_or_else(|| Error::Runtime {
+                message: format!(
+                    "blob take for column '{column}' returned a null handle for a non-null row"
+                ),
+            })?;
+            aligned_handles.push(Some(handle));
+        }
+    }
+    Ok(aligned_handles)
 }
 
 #[cfg(test)]

rust/lancedb/src/index.rs

@@ -54,26 +54,7 @@ pub enum Index {
     /// substrings of the raw bytes, unlike the tokenized [`Index::FTS`] index.
     Fm(FmIndexBuilder),
 
-    /// Full text search index using BM25.
-    ///
-    /// The posting block size defaults to 128. Supported values are 128 and 256;
-    /// a value of 256 uses the experimental FTS V3 format and may introduce
-    /// breaking changes.
-    ///
-    /// ```
-    /// use lancedb::index::{Index, scalar::FtsIndexBuilder};
-    ///
-    /// # async fn create_fts_index(
-    /// #     table: &lancedb::Table,
-    /// # ) -> Result<(), Box<dyn std::error::Error>> {
-    /// let params = FtsIndexBuilder::default().block_size(256)?;
-    /// table
-    ///     .create_index(&["text"], Index::FTS(params))
-    ///     .execute()
-    ///     .await?;
-    /// # Ok(())
-    /// # }
-    /// ```
+    /// Full text search index using bm25.
     FTS(FtsIndexBuilder),
 
     /// IVF index

rust/lancedb/src/remote/table.rs

@@ -4496,15 +4496,6 @@ mod tests {
                 serde_json::to_value(InvertedIndexParams::default()).unwrap(),
                 Index::FTS(Default::default()),
             ),
-            (
-                "FTS",
-                {
-                    let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap();
-                    body["block_size"] = 256.into();
-                    body
-                },
-                Index::FTS(InvertedIndexParams::default().block_size(256).unwrap()),
-            ),
         ];
 
         for (index_type, expected_body, index) in cases {

rust/lancedb/src/table/create_index.rs

@@ -382,9 +382,7 @@ mod tests {
     use crate::connect;
     use crate::connection::ConnectBuilder;
     use crate::index::Index;
-    use crate::index::scalar::{
-        BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder, FtsIndexBuilder,
-    };
+    use crate::index::scalar::{BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder};
     use crate::index::vector::{
         IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder,
     };
@@ -1364,10 +1362,7 @@ mod tests {
             .unwrap();
 
         table
-            .create_index(
-                &["text"],
-                Index::FTS(FtsIndexBuilder::default().block_size(256).unwrap()),
-            )
+            .create_index(&["text"], Index::FTS(Default::default()))
             .execute()
             .await
             .unwrap();
@@ -1377,10 +1372,6 @@ mod tests {
         assert_eq!(index.index_type, crate::index::IndexType::FTS);
         assert_eq!(index.columns, vec!["text".to_string()]);
         assert_eq!(index.name, "text_idx");
-        assert_eq!(index.index_version, Some(3));
-        let index_params: FtsIndexBuilder =
-            serde_json::from_str(index.index_details.as_deref().unwrap()).unwrap();
-        assert_eq!(index_params.posting_block_size(), 256);
 
         let num_rows = 120;
         let stats = table.index_stats("text_idx").await.unwrap().unwrap();