diff --git a/Cargo.lock b/Cargo.lock
index 45b286d85..cdf4e699f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arc-swap",
"arrow",
@@ -5133,8 +5133,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5156,7 +5156,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5170,7 +5170,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5179,8 +5179,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrayref",
"crunchy",
@@ -5190,8 +5190,8 @@ dependencies = [
[[package]]
name = "lance-core"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5228,8 +5228,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrow",
"arrow-array",
@@ -5260,8 +5260,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrow",
"arrow-array",
@@ -5278,8 +5278,8 @@ dependencies = [
[[package]]
name = "lance-derive"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"proc-macro2",
"quote",
@@ -5288,8 +5288,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5322,8 +5322,8 @@ dependencies = [
[[package]]
name = "lance-file"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5355,8 +5355,8 @@ dependencies = [
[[package]]
name = "lance-geo"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"datafusion",
"geo-traits",
@@ -5370,8 +5370,8 @@ dependencies = [
[[package]]
name = "lance-index"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arc-swap",
"arrow",
@@ -5442,8 +5442,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5466,8 +5466,8 @@ dependencies = [
[[package]]
name = "lance-io"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrow",
"arrow-array",
@@ -5509,8 +5509,8 @@ dependencies = [
[[package]]
name = "lance-jsonb"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"byteorder",
"ethnum",
@@ -5527,8 +5527,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5542,8 +5542,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrow",
"async-trait",
@@ -5557,8 +5557,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arc-swap",
"arrow",
@@ -5612,8 +5612,8 @@ dependencies = [
[[package]]
name = "lance-select"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5627,8 +5627,8 @@ dependencies = [
[[package]]
name = "lance-table"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrow",
"arrow-array",
@@ -5668,8 +5668,8 @@ dependencies = [
[[package]]
name = "lance-testing"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5682,8 +5682,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
-version = "14.0.0-beta.11"
-source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
+version = "14.0.0-beta.12"
+source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.12#b3c8619886b9ba3a26f323241d16c7f36b28db7f"
dependencies = [
"frostem",
"icu_segmenter",
diff --git a/Cargo.toml b/Cargo.toml
index 49d1792de..9246f85fc 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.94.0"
[workspace.dependencies]
-lance = { "version" = "=14.0.0-beta.11", default-features = false, "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=14.0.0-beta.11", default-features = false, "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=14.0.0-beta.11", default-features = false, "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=14.0.0-beta.12", default-features = false, "tag" = "v14.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=14.0.0-beta.12", "tag" = "v14.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=14.0.0-beta.12", "tag" = "v14.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=14.0.0-beta.12", "tag" = "v14.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=14.0.0-beta.12", default-features = false, "tag" = "v14.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=14.0.0-beta.12", "tag" = "v14.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=14.0.0-beta.12", "tag" = "v14.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=14.0.0-beta.12", "tag" = "v14.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=14.0.0-beta.12", default-features = false, "tag" = "v14.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=14.0.0-beta.12", "tag" = "v14.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=14.0.0-beta.12", "tag" = "v14.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=14.0.0-beta.12", "tag" = "v14.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=14.0.0-beta.12", "tag" = "v14.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=14.0.0-beta.12", "tag" = "v14.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
diff --git a/nodejs/__test__/query.test.ts b/nodejs/__test__/query.test.ts
index 16a595002..786f04015 100644
--- a/nodejs/__test__/query.test.ts
+++ b/nodejs/__test__/query.test.ts
@@ -527,3 +527,106 @@ describe("Query orderBy", () => {
]);
});
});
+
+describe("IVF search effort", () => {
+ let directory: tmp.DirResult;
+ let table: Table;
+ const vector = Array.from({ length: 16 }, (_, i) => (i === 0 ? 1 : 0));
+
+ beforeEach(async () => {
+ directory = tmp.dirSync({ unsafeCleanup: true });
+ const db = await connect(directory.name);
+ const rows = Array.from({ length: 512 }, (_, id) => ({
+ id,
+ vector: Array.from(
+ { length: 16 },
+ (_, j) =>
+ (Math.floor(id / 32) === j ? 1 : 0) + Math.sin(id * 17 + j) * 0.001,
+ ),
+ }));
+ table = await db.createTable("effort", rows.slice(0, 256));
+ await table.add(rows.slice(256));
+ await table.createIndex("vector", {
+ config: Index.ivfFlat({ numPartitions: 16 }),
+ });
+ });
+
+ afterEach(() => {
+ directory.removeCallback();
+ });
+
+ it.each([
+ [null, 6],
+ [0, 1],
+ [0.25, 3],
+ [0.5, 6],
+ [0.75, 10],
+ [1, 16],
+ ])(
+ "preserves effort %s through execution and probe precedence",
+ async (effort, expected) => {
+ const partitions = async (query: ReturnType
) => {
+ const plan = await query.limit(8).analyzePlan();
+ const matches = [...plan.matchAll(/partitions_searched=(\d+)/g)];
+ expect(matches.length).toBeGreaterThan(0);
+ return matches.reduce((total, match) => total + Number(match[1]), 0);
+ };
+ expect(
+ await partitions(table.vectorSearch(vector).searchEffort(effort)),
+ ).toBe(expected);
+ expect(
+ await partitions(
+ table.vectorSearch(vector).nprobes(1).searchEffort(effort),
+ ),
+ ).toBe(effort === null ? 1 : expected);
+ expect(
+ await partitions(
+ table.vectorSearch(vector).searchEffort(effort).nprobes(1),
+ ),
+ ).toBe(effort === null ? 1 : expected);
+ expect(
+ await partitions(
+ table
+ .vectorSearch(vector)
+ .searchEffort(effort)
+ .nprobes(1)
+ .searchEffort(null),
+ ),
+ ).toBe(1);
+ expect(
+ await partitions(
+ table.vectorSearch(vector).searchEffort(effort).maximumNprobes(2),
+ ),
+ ).toBeLessThanOrEqual(2);
+ const actual = await table
+ .vectorSearch(vector)
+ .searchEffort(effort)
+ .limit(8)
+ .toArray();
+ const exact = await table
+ .vectorSearch(vector)
+ .bypassVectorIndex()
+ .limit(8)
+ .toArray();
+ const ids = new Set(exact.map((row) => row.id));
+ expect(
+ actual.filter((row) => ids.has(row.id)).length / 8,
+ ).toBeGreaterThanOrEqual(0.5);
+ },
+ );
+
+ it.each([
+ -0.1,
+ 1.1,
+ Number.NaN,
+ Number.POSITIVE_INFINITY,
+ Number.NEGATIVE_INFINITY,
+ ])("rejects invalid effort %s before execution", (effort) => {
+ expect(() => table.vectorSearch(vector).searchEffort(effort)).toThrow(
+ "search_effort must be finite and in [0, 1]",
+ );
+ expect(() =>
+ table.vectorSearch(vector).fullTextSearch("dog").searchEffort(effort),
+ ).toThrow("search_effort must be finite and in [0, 1]");
+ });
+});
diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts
index 0f0eaf610..cce505c97 100644
--- a/nodejs/lancedb/query.ts
+++ b/nodejs/lancedb/query.ts
@@ -549,6 +549,27 @@ export class VectorQuery extends StandardQueryBase {
return this;
}
+ /**
+ * Set the initial IVF search effort to a finite value in [0, 1].
+ *
+ * The default, null, uses Auto (internally 0.5). Zero starts at the caller
+ * minimum; one searches all partitions within the caller maximum. Later
+ * probing may continue when too few candidates are found.
+ * Any explicit effort, including 0.5, overrides nprobes in either call order.
+ * Explicit minimum and maximum bounds still apply. This is not a recall
+ * guarantee and does not change refinement or HNSW ef. The MemWAL LSM
+ * scanner does not yet support this option.
+ *
+ * @example
+ * ```ts
+ * table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10);
+ * ```
+ */
+ searchEffort(searchEffort: number | null): VectorQuery {
+ this.doVectorCall((inner) => inner.searchEffort(searchEffort));
+ return this;
+ }
+
/**
* Set the minimum number of probes used.
*
diff --git a/nodejs/src/query.rs b/nodejs/src/query.rs
index 712569b2f..480426411 100644
--- a/nodejs/src/query.rs
+++ b/nodejs/src/query.rs
@@ -298,6 +298,16 @@ impl VectorQuery {
Ok(())
}
+ #[napi]
+ pub fn search_effort(&mut self, search_effort: Option) -> napi::Result<()> {
+ self.inner = self
+ .inner
+ .clone()
+ .search_effort(search_effort)
+ .default_error()?;
+ Ok(())
+ }
+
#[napi]
pub fn minimum_nprobes(&mut self, minimum_nprobe: u32) -> napi::Result<()> {
self.inner = self
diff --git a/python/pyproject.toml b/python/pyproject.toml
index cde7247f3..ed54ae536 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -63,7 +63,7 @@ tests = [
"polars>=0.19, <=1.32.3",
"pyarrow<25",
"pyarrow-stubs>=16.0",
- "pylance==14.0.0b10",
+ "pylance==14.0.0b12",
"requests>=2.31.0",
"datafusion>=54,<55",
"opentelemetry-sdk>=1.30.0",
@@ -73,6 +73,7 @@ dev = [
"pre-commit>=3.5.0",
"pyright>=1.1.350",
'typing-extensions>=4.0.0; python_version < "3.11"',
+ "maturin>=1.10",
]
docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings-python"]
clip = ["torch", "pillow>=12.1.1", "open-clip-torch"]
diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi
index cfaa8c68a..8d0a86e4a 100644
--- a/python/python/lancedb/_lancedb.pyi
+++ b/python/python/lancedb/_lancedb.pyi
@@ -727,6 +727,7 @@ class VectorQuery:
def use_lsm(self, enable: bool): ...
def refine_factor(self, refine_factor: int): ...
def nprobes(self, nprobes: int): ...
+ def search_effort(self, search_effort: Optional[float]): ...
def minimum_nprobes(self, minimum_nprobes: int): ...
def maximum_nprobes(self, maximum_nprobes: int): ...
def bypass_vector_index(self): ...
@@ -752,6 +753,7 @@ class HybridQuery:
def distance_type(self, distance_type: str): ...
def refine_factor(self, refine_factor: int): ...
def nprobes(self, nprobes: int): ...
+ def search_effort(self, search_effort: Optional[float]): ...
def minimum_nprobes(self, minimum_nprobes: int): ...
def maximum_nprobes(self, maximum_nprobes: int): ...
def bypass_vector_index(self): ...
@@ -781,6 +783,7 @@ class PyQueryRequest:
nprobes: Optional[int]
minimum_nprobes: Optional[int]
maximum_nprobes: Optional[int]
+ search_effort: Optional[float]
lower_bound: Optional[float]
upper_bound: Optional[float]
ef: Optional[int]
diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py
index ed87e441f..26c62925f 100644
--- a/python/python/lancedb/namespace.py
+++ b/python/python/lancedb/namespace.py
@@ -105,6 +105,11 @@ def _query_to_namespace_request(
if not isinstance(query, Query):
raise TypeError(f"Expected Query, got {type(query)}")
+ if query.search_effort is not None:
+ raise ValueError(
+ "namespace QueryTable does not support search_effort; "
+ "execute the query locally"
+ )
vector_request = None
if query.vector is not None:
diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py
index c7664ab78..fc9aea9a4 100644
--- a/python/python/lancedb/query.py
+++ b/python/python/lancedb/query.py
@@ -475,8 +475,8 @@ class MatchQuery(FullTextQuery):
----------
query : str
The query string to match against.
- column : str
- The name of the column to match against.
+ column : str, optional
+ The column to match against. None leaves column selection to query planning.
boost : float, default 1.0
The boost factor for the query.
The score of each matching document is multiplied by this value.
@@ -505,7 +505,7 @@ class MatchQuery(FullTextQuery):
"""
query: str
- column: str
+ column: Optional[str] = None
boost: float = pydantic.Field(1.0, kw_only=True)
fuzziness: Optional[int] = pydantic.Field(0, kw_only=True)
max_expansions: int = pydantic.Field(50, kw_only=True)
@@ -793,6 +793,9 @@ class Query(pydantic.BaseModel):
# and all partitions could be searched if needed to satisfy the limit.
maximum_nprobes: Optional[int] = None
+ # Optional initial IVF effort; None preserves Auto and explicit nprobes.
+ search_effort: Optional[float] = None
+
# lower bound for distance search
lower_bound: Optional[float] = None
@@ -854,6 +857,7 @@ class Query(pydantic.BaseModel):
query.nprobes = req.nprobes
query.minimum_nprobes = req.minimum_nprobes
query.maximum_nprobes = req.maximum_nprobes
+ query.search_effort = req.search_effort
query.lower_bound = req.lower_bound
query.upper_bound = req.upper_bound
query.ef = req.ef
@@ -1635,6 +1639,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
self._nprobes = None
self._minimum_nprobes = None
self._maximum_nprobes = None
+ self._search_effort = None
self._lower_bound = None
self._upper_bound = None
self._refine_factor = None
@@ -1716,6 +1721,24 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
self._nprobes = nprobes
return self
+ def search_effort(self, search_effort: Optional[float]) -> LanceVectorQueryBuilder:
+ """Set the initial IVF search effort to a finite value in [0, 1].
+
+ None (the default) uses Auto, internally 0.5. Zero starts at the caller
+ minimum; one searches all partitions within the caller maximum. Later
+ probing may continue when too few candidates are found. Any explicit
+ effort, including 0.5, overrides nprobes in either call order. Explicit
+ minimum and maximum bounds still apply. This is not a recall guarantee
+ and does not change refinement or HNSW ef. MemWAL LSM queries do not
+ yet support this option.
+
+ Examples
+ --------
+ >>> query = table.search([0.1, 0.2]).search_effort(0.75)
+ """
+ self._search_effort = search_effort
+ return self
+
def minimum_nprobes(self, minimum_nprobes: int) -> LanceVectorQueryBuilder:
"""Set the minimum number of probes to use.
@@ -1858,6 +1881,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
nprobes=self._nprobes,
minimum_nprobes=self._minimum_nprobes,
maximum_nprobes=self._maximum_nprobes,
+ search_effort=self._search_effort,
lower_bound=self._lower_bound,
upper_bound=self._upper_bound,
refine_factor=self._refine_factor,
@@ -2217,6 +2241,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
self._nprobes = None
self._minimum_nprobes = None
self._maximum_nprobes = None
+ self._search_effort = None
self._refine_factor = None
self._distance_type = None
self._phrase_query = None
@@ -2524,6 +2549,26 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
self._nprobes = nprobes
return self
+ def search_effort(self, search_effort: Optional[float]) -> LanceHybridQueryBuilder:
+ """Set the initial IVF search effort to a finite value in [0, 1].
+
+ None (the default) uses Auto, internally 0.5. Zero starts at the caller
+ minimum; one searches all partitions within the caller maximum. Later
+ probing may continue when too few candidates are found. Any explicit
+ effort, including 0.5, overrides nprobes in either call order. Explicit
+ minimum and maximum bounds still apply. This is not a recall guarantee
+ and does not change refinement or HNSW ef. MemWAL LSM queries do not
+ yet support this option.
+
+ Examples
+ --------
+ >>> query = table.search([0.1, 0.2]).search_effort(0.75)
+ """
+ self._search_effort = search_effort
+ if self._vector_query is not None:
+ self._vector_query.search_effort(search_effort)
+ return self
+
def minimum_nprobes(self, minimum_nprobes: int) -> LanceHybridQueryBuilder:
"""Set the minimum number of probes to use.
@@ -2802,6 +2847,8 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
self._vector_query.metric(self._distance_type)
if self._nprobes is not None:
self._vector_query.nprobes(self._nprobes)
+ if self._search_effort is not None:
+ self._vector_query.search_effort(self._search_effort)
if self._minimum_nprobes is not None:
self._vector_query.minimum_nprobes(self._minimum_nprobes)
if self._maximum_nprobes is not None:
@@ -3730,6 +3777,24 @@ class AsyncVectorQueryBase:
self._inner.nprobes(nprobes)
return self
+ def search_effort(self, search_effort: Optional[float]) -> Self:
+ """Set the initial IVF search effort to a finite value in [0, 1].
+
+ None preserves Auto (internally 0.5) and fixed nprobes. Any explicit
+ effort, including 0.5, overrides nprobes in either call order; explicit
+ minimum and maximum bounds remain effective. Zero starts at the caller
+ minimum and one searches all partitions within the maximum. Later
+ probing may continue to obtain enough candidates. This is not a recall
+ guarantee and does not change refinement or HNSW ef. MemWAL LSM queries
+ do not yet support this option.
+
+ Examples
+ --------
+ >>> query = table.vector_search([0.1, 0.2]).search_effort(0.75)
+ """
+ self._inner.search_effort(search_effort)
+ return self
+
def minimum_nprobes(self, minimum_nprobes: int) -> Self:
"""Set the minimum number of probes to use.
diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py
index 21838ecef..81b376ebd 100644
--- a/python/python/lancedb/table.py
+++ b/python/python/lancedb/table.py
@@ -4440,6 +4440,7 @@ class LanceTable(Table):
and not self._route_pushdown_to_rust
and self.current_branch() is None
and query.take_offsets is None
+ and query.search_effort is None
):
from lancedb.namespace import _execute_server_side_query
@@ -6441,6 +6442,8 @@ class AsyncTable:
async_query = async_query.distance_type(query.distance_type)
if query.nprobes is not None:
async_query = async_query.nprobes(query.nprobes)
+ if query.search_effort is not None:
+ async_query = async_query.search_effort(query.search_effort)
if query.minimum_nprobes is not None and query.maximum_nprobes is not None:
async_query = async_query.minimum_nprobes(
query.minimum_nprobes
@@ -6481,6 +6484,7 @@ class AsyncTable:
)
and not self._route_pushdown_to_rust
and query.take_offsets is None
+ and query.search_effort is None
):
from lancedb.namespace import _execute_server_side_query
diff --git a/python/python/tests/test_namespace.py b/python/python/tests/test_namespace.py
index 76aa5fc55..520a6992d 100644
--- a/python/python/tests/test_namespace.py
+++ b/python/python/tests/test_namespace.py
@@ -1209,3 +1209,15 @@ def test_local_table_to_arrow_and_to_pandas_are_unchanged(tmp_path):
assert table.to_arrow().column("id").to_pylist() == [1, 2]
assert table.to_pandas()["id"].tolist() == [1, 2]
+
+
+@pytest.mark.parametrize("effort", [0.0, 0.5, 1.0])
+def test_namespace_request_rejects_search_effort(effort):
+ from lancedb.query import Query
+
+ with pytest.raises(
+ ValueError, match="namespace QueryTable does not support search_effort"
+ ):
+ _query_to_namespace_request(
+ ["test"], Query(vector=[1.0, 0.0], search_effort=effort)
+ )
diff --git a/python/python/tests/test_query.py b/python/python/tests/test_query.py
index daee0dd07..71c293628 100644
--- a/python/python/tests/test_query.py
+++ b/python/python/tests/test_query.py
@@ -2265,3 +2265,112 @@ def test_blob_v2_to_batches_row_id(tmp_db):
assert "_rowid" in hits.column_names
blobs = table.fetch_blobs("blob", hits)
assert [blobs[i].as_py() for i in range(len(blobs))] == [b"one", b"two"]
+
+
+@pytest.mark.parametrize("metric", ["l2", "cosine", "dot"])
+@pytest.mark.parametrize(
+ "effort,probes", [(None, 4), (0, 1), (0.25, 2), (0.5, 4), (0.75, 8), (1, 16)]
+)
+def test_search_effort_probe_budget(tmp_path, monkeypatch, metric, effort, probes):
+ import re
+ from lancedb.index import IvfFlat
+
+ monkeypatch.setenv("LANCE_AUTO_PROBE_MARGIN", "0")
+ monkeypatch.setenv("LANCE_AUTO_MIN_INITIAL_NPROBES", "4")
+ monkeypatch.setenv("LANCE_AUTO_MAX_INITIAL_NPROBES", "4")
+ rng = np.random.default_rng(42)
+ vectors = np.repeat(np.eye(16, dtype=np.float32), 32, axis=0)
+ vectors += rng.normal(0, 0.001, vectors.shape).astype(np.float32)
+ data = pa.table(
+ {
+ "id": np.arange(512),
+ "vector": pa.FixedSizeListArray.from_arrays(pa.array(vectors.ravel()), 16),
+ }
+ )
+ table = lancedb.connect(tmp_path).create_table("effort", data.slice(0, 256))
+ table.add(data.slice(256))
+ table.create_index(
+ "vector", config=IvfFlat(distance_type=metric, num_partitions=16)
+ )
+ vector = np.eye(16, dtype=np.float32)[0]
+
+ def search(value=effort):
+ return table.search(vector).distance_type(metric).search_effort(value).limit(8)
+
+ def partitions(query):
+ values = re.findall(r"partitions_searched=(\d+)", query.analyze_plan())
+ assert values
+ return sum(map(int, values))
+
+ assert partitions(search()) == probes
+ exact = set(
+ table.search(vector)
+ .distance_type(metric)
+ .bypass_vector_index()
+ .limit(8)
+ .to_arrow()["id"]
+ .to_pylist()
+ )
+ actual = set(search().to_arrow()["id"].to_pylist())
+ assert len(actual & exact) / 8 >= 0.5
+ expected = 1 if effort is None else probes
+ assert partitions(search().nprobes(1)) == expected
+ assert (
+ partitions(
+ table.search(vector)
+ .distance_type(metric)
+ .nprobes(1)
+ .search_effort(effort)
+ .limit(8)
+ )
+ == expected
+ )
+ assert partitions(search().nprobes(1).search_effort(None)) == 1
+ assert partitions(search().maximum_nprobes(2)) <= 2
+ assert partitions(search().minimum_nprobes(3).maximum_nprobes(3)) == 3
+ assert partitions(search(0).nprobes(16).maximum_nprobes(0)) == 1
+ if effort is None:
+ assert search().to_arrow() == search(0.5).to_arrow()
+ filtered = search(0).maximum_nprobes(2).where("id % 8 = 0", prefilter=True)
+ assert partitions(filtered) <= 2
+ assert all(row["id"] % 8 == 0 for row in filtered.to_list())
+
+
+@pytest.mark.parametrize(
+ "effort", [-0.1, 1.1, float("nan"), float("inf"), -float("inf")]
+)
+def test_search_effort_invalid_sync(table, effort):
+ with pytest.raises(ValueError, match="search_effort must be finite and in"):
+ table.search([0, 0]).search_effort(effort).to_list()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("hybrid", [False, True])
+@pytest.mark.parametrize("effort", [None, 0, 0.25, 0.5, 0.75, 1])
+async def test_search_effort_async_roundtrip(table_async, hybrid, effort):
+ query = table_async.vector_search([0, 0])
+ if hybrid:
+ query = query.nearest_to_text("dog")
+ query = query.nprobes(5).search_effort(effort).minimum_nprobes(2).maximum_nprobes(3)
+ request = query.to_query_object()
+ assert request.search_effort == effort
+ assert request.nprobes == 5
+ assert request.minimum_nprobes == 2
+ assert request.maximum_nprobes == 3
+ assert query.search_effort(None).to_query_object().search_effort is None
+ query.search_effort(effort)
+ if not hybrid:
+ assert len(await query.limit(1).to_list()) == 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("hybrid", [False, True])
+@pytest.mark.parametrize(
+ "effort", [-0.1, 1.1, float("nan"), float("inf"), -float("inf")]
+)
+async def test_search_effort_invalid_async(table_async, hybrid, effort):
+ query = table_async.vector_search([0, 0])
+ if hybrid:
+ query = query.nearest_to_text("dog")
+ with pytest.raises(ValueError, match="search_effort must be finite and in"):
+ query.search_effort(effort)
diff --git a/python/src/query.rs b/python/src/query.rs
index 88b7243d4..e5547881a 100644
--- a/python/src/query.rs
+++ b/python/src/query.rs
@@ -71,7 +71,7 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB {
match class_name(&ob)?.as_str() {
"MatchQuery" => {
let query = ob.getattr("query")?.extract()?;
- let column = ob.getattr("column")?.extract()?;
+ let column = ob.getattr("column")?.extract::