feat(search): expose search effort (#4520)

Lance supports `search_effort`, but LanceDB cannot pass it through its
query APIs. Expose the parameter in Rust, Python sync/async and
TypeScript, including remote requests and hybrid query conversion, using
the [Lance
v14.0.0-beta.13](https://github.com/lance-format/lance/releases/tag/v14.0.0-beta.13)
dependency already on main.

The default is `None` (Auto, internally 0.5). Any explicit effort,
including 0.5, overrides fixed `nprobes` in either setter order;
explicit minimum/maximum bounds remain effective. Invalid values are
rejected in the Rust core. Hybrid conversion now retains the vector
probe parameters and handles an inferred FTS column without panicking.

Namespace queries with explicit effort execute locally because the
namespace wire model cannot carry it. MemWAL LSM queries reject it
explicitly because Lance's LSM scanner does not yet expose this option.
Maturin is included in the dev extra so the documented build command
works in the project environment.

BREAKING CHANGE: The public Rust `VectorQueryRequest` struct now
includes `search_effort: Option<f64>`. Downstream code that initializes
every field explicitly must add `search_effort: None` or use
`..Default::default()`. Exhaustive destructuring patterns must include
the new field or `..`. Query-builder calls and struct literals that
already use `..Default::default()` require no source changes. The new
effort setters in Python and TypeScript are additive. This PR does not
change the persisted file format.

Using explicit effort with remote tables requires server support for
`search_effort` and its precedence over `nprobes`; the client wire tests
do not establish that a deployed server supports it. Server-side support
is tracked in lancedb/sophon#8607.

Validation on AWS after merging current main (Lance beta.13 / SDK
beta.5):

- 14 Rust regressions and the API doctest pass.
- 60 parameter regressions, the full 47-test namespace suite, and three
API doctests pass. The parameter selection includes some namespace
cases.
- 40 TypeScript query tests and 54 remote tests pass; build, lint and
generated docs pass.
- Rust Clippy, Python package lint/format, Pyright and lockfile checks
pass. Repository-wide Ruff reports 49 lint errors and 11 formatting
failures in unchanged `docs/web-tests` files; those files are
byte-identical to main and outside this change.

All 1,103 Git-tracked files match the AWS validation checkout. No
benchmark scripts or result artifacts are included, and no performance
improvement is claimed.

Uses lance-format/lance#9798. Server-side support is in
lancedb/sophon#8607.

nodejs/__test__/query.test.ts

@@ -527,3 +527,106 @@ describe("Query orderBy", () => {
     ]);
   });
 });
+
+describe("IVF search effort", () => {
+  let directory: tmp.DirResult;
+  let table: Table;
+  const vector = Array.from({ length: 16 }, (_, i) => (i === 0 ? 1 : 0));
+
+  beforeEach(async () => {
+    directory = tmp.dirSync({ unsafeCleanup: true });
+    const db = await connect(directory.name);
+    const rows = Array.from({ length: 512 }, (_, id) => ({
+      id,
+      vector: Array.from(
+        { length: 16 },
+        (_, j) =>
+          (Math.floor(id / 32) === j ? 1 : 0) + Math.sin(id * 17 + j) * 0.001,
+      ),
+    }));
+    table = await db.createTable("effort", rows.slice(0, 256));
+    await table.add(rows.slice(256));
+    await table.createIndex("vector", {
+      config: Index.ivfFlat({ numPartitions: 16 }),
+    });
+  });
+
+  afterEach(() => {
+    directory.removeCallback();
+  });
+
+  it.each([
+    [null, 6],
+    [0, 1],
+    [0.25, 3],
+    [0.5, 6],
+    [0.75, 10],
+    [1, 16],
+  ])(
+    "preserves effort %s through execution and probe precedence",
+    async (effort, expected) => {
+      const partitions = async (query: ReturnType<Table["vectorSearch"]>) => {
+        const plan = await query.limit(8).analyzePlan();
+        const matches = [...plan.matchAll(/partitions_searched=(\d+)/g)];
+        expect(matches.length).toBeGreaterThan(0);
+        return matches.reduce((total, match) => total + Number(match[1]), 0);
+      };
+      expect(
+        await partitions(table.vectorSearch(vector).searchEffort(effort)),
+      ).toBe(expected);
+      expect(
+        await partitions(
+          table.vectorSearch(vector).nprobes(1).searchEffort(effort),
+        ),
+      ).toBe(effort === null ? 1 : expected);
+      expect(
+        await partitions(
+          table.vectorSearch(vector).searchEffort(effort).nprobes(1),
+        ),
+      ).toBe(effort === null ? 1 : expected);
+      expect(
+        await partitions(
+          table
+            .vectorSearch(vector)
+            .searchEffort(effort)
+            .nprobes(1)
+            .searchEffort(null),
+        ),
+      ).toBe(1);
+      expect(
+        await partitions(
+          table.vectorSearch(vector).searchEffort(effort).maximumNprobes(2),
+        ),
+      ).toBeLessThanOrEqual(2);
+      const actual = await table
+        .vectorSearch(vector)
+        .searchEffort(effort)
+        .limit(8)
+        .toArray();
+      const exact = await table
+        .vectorSearch(vector)
+        .bypassVectorIndex()
+        .limit(8)
+        .toArray();
+      const ids = new Set(exact.map((row) => row.id));
+      expect(
+        actual.filter((row) => ids.has(row.id)).length / 8,
+      ).toBeGreaterThanOrEqual(0.5);
+    },
+  );
+
+  it.each([
+    -0.1,
+    1.1,
+    Number.NaN,
+    Number.POSITIVE_INFINITY,
+    Number.NEGATIVE_INFINITY,
+  ])("rejects invalid effort %s before execution", (effort) => {
+    expect(() => table.vectorSearch(vector).searchEffort(effort)).toThrow(
+      "search_effort must be finite and in [0, 1]",
+    );
+    expect(() =>
+      table.vectorSearch(vector).fullTextSearch("dog").searchEffort(effort),
+    ).toThrow("search_effort must be finite and in [0, 1]");
+  });
+});

nodejs/lancedb/query.ts

@@ -549,6 +549,27 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
     return this;
   }
 
+  /**
+   * Set the initial IVF search effort to a finite value in [0, 1].
+   *
+   * The default, null, uses Auto (internally 0.5). Zero starts at the caller
+   * minimum; one searches all partitions within the caller maximum. Later
+   * probing may continue when too few candidates are found.
+   * Any explicit effort, including 0.5, overrides nprobes in either call order.
+   * Explicit minimum and maximum bounds still apply. This is not a recall
+   * guarantee and does not change refinement or HNSW ef. The MemWAL LSM
+   * scanner does not yet support this option.
+   *
+   * @example
+   * ```ts
+   * table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10);
+   * ```
+   */
+  searchEffort(searchEffort: number | null): VectorQuery {
+    this.doVectorCall((inner) => inner.searchEffort(searchEffort));
+    return this;
+  }
+
   /**
    * Set the minimum number of probes used.
    *

nodejs/src/query.rs

@@ -298,6 +298,16 @@ impl VectorQuery {
         Ok(())
     }
 
+    #[napi]
+    pub fn search_effort(&mut self, search_effort: Option<f64>) -> napi::Result<()> {
+        self.inner = self
+            .inner
+            .clone()
+            .search_effort(search_effort)
+            .default_error()?;
+        Ok(())
+    }
+
     #[napi]
     pub fn minimum_nprobes(&mut self, minimum_nprobe: u32) -> napi::Result<()> {
         self.inner = self