rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-13 23:32:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main and retain sliced Arrow validity fix

Browse Source
This commit is contained in:
Gatefixer
2026-10-13 17:08:34 +00:00
parent 0c9e235bff 1ea47a5055
commit 433bd29429
44 changed files with 1460 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.41.0-beta.5"
current_version = "0.41.0-beta.6"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
Cargo.lock
Generated
+48 -48
View File
@@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arc-swap",
"arrow",
@@ -5133,7 +5133,7 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "14.0.0-beta.13"
version = "14.0.0-beta.15"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5155,7 +5155,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5169,7 +5169,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5178,8 +5178,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arrayref",
"crunchy",
@@ -5189,8 +5189,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5227,8 +5227,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arrow",
"arrow-array",
@@ -5259,8 +5259,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arrow",
"arrow-array",
@@ -5277,8 +5277,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"proc-macro2",
"quote",
@@ -5287,8 +5287,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5321,8 +5321,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5354,8 +5354,8 @@ dependencies = [
[[package]]
name = "lance-geo"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"datafusion",
"geo-traits",
@@ -5369,8 +5369,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arc-swap",
"arrow",
@@ -5441,8 +5441,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5465,8 +5465,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arrow",
"arrow-array",
@@ -5508,8 +5508,8 @@ dependencies = [
[[package]]
name = "lance-jsonb"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"byteorder",
"ethnum",
@@ -5526,8 +5526,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5541,8 +5541,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arrow",
"async-trait",
@@ -5556,8 +5556,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arc-swap",
"arrow",
@@ -5611,8 +5611,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5626,8 +5626,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arrow",
"arrow-array",
@@ -5667,8 +5667,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5681,8 +5681,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
version = "14.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5695,7 +5695,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.41.0-beta.5"
version = "0.41.0-beta.6"
dependencies = [
"ahash",
"anyhow",
@@ -5792,7 +5792,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.41.0-beta.5"
version = "0.41.0-beta.6"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5817,7 +5817,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.41.0-beta.5"
version = "0.41.0-beta.6"
dependencies = [
"arc-swap",
"arrow",
Cargo.toml
+15 -15
View File
@@ -13,21 +13,21 @@ categories = ["database-implementations"]
rust-version = "1.94.0"
[workspace.dependencies]
lance = { "version" = "=14.0.0-beta.13", default-features = false, "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=14.0.0-beta.13", default-features = false, "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-jsonb = { version = "=14.0.0-beta.13", tag = "v14.0.0-beta.13", git = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=14.0.0-beta.13", default-features = false, "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { version = "=14.0.0-beta.13", path = "python/vendor/lance-arrow" }
lance = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-jsonb = { version = "=14.0.0-beta.15", tag = "v14.0.0-beta.15", git = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { version = "=14.0.0-beta.15", path = "python/vendor/lance-arrow" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
docs/README.md
+7 -4
View File
@@ -15,18 +15,21 @@ unreleased features.
### Preview the docs
Run these commands from the LanceDB repository root. The subshell changes into
`docs` to load `mkdocs.yml`, while leaving your shell in the repository root.
```shell
cd docs
mkdocs serve
(cd docs && PYTHONPATH=. mkdocs serve)
```
If you want to just generate the HTML files:
```shell
PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
(cd docs && PYTHONPATH=. mkdocs build)
```
If successful, you should see a `docs/site` directory that you can verify locally.
If successful, you should see a `docs/site` directory in the repository root that
you can verify locally.
## Adding examples
docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.41.0-beta.5</version>
<version>0.41.0-beta.6</version>
</dependency>
```
docs/src/js/classes/VectorQuery.md
+32
View File
@@ -646,6 +646,38 @@ rerank(reranker): VectorQuery
***
### searchEffort()
```ts
searchEffort(searchEffort): VectorQuery
```
Set the initial IVF search effort to a finite value in [0, 1].
The default, null, uses Auto (internally 0.5). Zero starts at the caller
minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found.
Any explicit effort, including 0.5, overrides nprobes in either call order.
Explicit minimum and maximum bounds still apply. This is not a recall
guarantee and does not change refinement or HNSW ef. The MemWAL LSM
scanner does not yet support this option.
#### Parameters
* **searchEffort**: `null` \| `number`
#### Returns
[`VectorQuery`](VectorQuery.md)
#### Example
```ts
table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10);
```
***
### select()
```ts
java/lancedb-core/pom.xml
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.41.0-beta.5</version>
<version>0.41.0-beta.6</version>
<relativePath>../pom.xml</relativePath>
</parent>
java/pom.xml
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.41.0-beta.5</version>
<version>0.41.0-beta.6</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>14.0.0-beta.13</lance-core.version>
<lance-core.version>14.0.0-beta.15</lance-core.version>
<jackson.version>2.18.11</jackson.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
nodejs/Cargo.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.41.0-beta.5"
version = "0.41.0-beta.6"
publish = false
license.workspace = true
description.workspace = true
nodejs/__test__/query.test.ts
+103
View File
@@ -527,3 +527,106 @@ describe("Query orderBy", () => {
]);
});
});
describe("IVF search effort", () => {
let directory: tmp.DirResult;
let table: Table;
const vector = Array.from({ length: 16 }, (_, i) => (i === 0 ? 1 : 0));
beforeEach(async () => {
directory = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(directory.name);
const rows = Array.from({ length: 512 }, (_, id) => ({
id,
vector: Array.from(
{ length: 16 },
(_, j) =>
(Math.floor(id / 32) === j ? 1 : 0) + Math.sin(id * 17 + j) * 0.001,
),
}));
table = await db.createTable("effort", rows.slice(0, 256));
await table.add(rows.slice(256));
await table.createIndex("vector", {
config: Index.ivfFlat({ numPartitions: 16 }),
});
});
afterEach(() => {
directory.removeCallback();
});
it.each([
[null, 6],
[0, 1],
[0.25, 3],
[0.5, 6],
[0.75, 10],
[1, 16],
])(
"preserves effort %s through execution and probe precedence",
async (effort, expected) => {
const partitions = async (query: ReturnType<Table["vectorSearch"]>) => {
const plan = await query.limit(8).analyzePlan();
const matches = [...plan.matchAll(/partitions_searched=(\d+)/g)];
expect(matches.length).toBeGreaterThan(0);
return matches.reduce((total, match) => total + Number(match[1]), 0);
};
expect(
await partitions(table.vectorSearch(vector).searchEffort(effort)),
).toBe(expected);
expect(
await partitions(
table.vectorSearch(vector).nprobes(1).searchEffort(effort),
),
).toBe(effort === null ? 1 : expected);
expect(
await partitions(
table.vectorSearch(vector).searchEffort(effort).nprobes(1),
),
).toBe(effort === null ? 1 : expected);
expect(
await partitions(
table
.vectorSearch(vector)
.searchEffort(effort)
.nprobes(1)
.searchEffort(null),
),
).toBe(1);
expect(
await partitions(
table.vectorSearch(vector).searchEffort(effort).maximumNprobes(2),
),
).toBeLessThanOrEqual(2);
const actual = await table
.vectorSearch(vector)
.searchEffort(effort)
.limit(8)
.toArray();
const exact = await table
.vectorSearch(vector)
.bypassVectorIndex()
.limit(8)
.toArray();
const ids = new Set(exact.map((row) => row.id));
expect(
actual.filter((row) => ids.has(row.id)).length / 8,
).toBeGreaterThanOrEqual(0.5);
},
);
it.each([
-0.1,
1.1,
Number.NaN,
Number.POSITIVE_INFINITY,
Number.NEGATIVE_INFINITY,
])("rejects invalid effort %s before execution", (effort) => {
expect(() => table.vectorSearch(vector).searchEffort(effort)).toThrow(
"search_effort must be finite and in [0, 1]",
);
expect(() =>
table.vectorSearch(vector).fullTextSearch("dog").searchEffort(effort),
).toThrow("search_effort must be finite and in [0, 1]");
});
});
nodejs/lancedb/query.ts
+21
View File
@@ -549,6 +549,27 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
return this;
}
/**
* Set the initial IVF search effort to a finite value in [0, 1].
*
* The default, null, uses Auto (internally 0.5). Zero starts at the caller
* minimum; one searches all partitions within the caller maximum. Later
* probing may continue when too few candidates are found.
* Any explicit effort, including 0.5, overrides nprobes in either call order.
* Explicit minimum and maximum bounds still apply. This is not a recall
* guarantee and does not change refinement or HNSW ef. The MemWAL LSM
* scanner does not yet support this option.
*
* @example
* ```ts
* table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10);
* ```
*/
searchEffort(searchEffort: number | null): VectorQuery {
this.doVectorCall((inner) => inner.searchEffort(searchEffort));
return this;
}
/**
* Set the minimum number of probes used.
*
nodejs/npm/darwin-arm64/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.41.0-beta.5",
"version": "0.41.0-beta.6",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
nodejs/npm/linux-arm64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.41.0-beta.5",
"version": "0.41.0-beta.6",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
nodejs/npm/linux-arm64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.41.0-beta.5",
"version": "0.41.0-beta.6",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
nodejs/npm/linux-x64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.41.0-beta.5",
"version": "0.41.0-beta.6",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
nodejs/npm/linux-x64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.41.0-beta.5",
"version": "0.41.0-beta.6",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
nodejs/npm/win32-arm64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.41.0-beta.5",
"version": "0.41.0-beta.6",
"os": [
"win32"
],
nodejs/npm/win32-x64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.41.0-beta.5",
"version": "0.41.0-beta.6",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
nodejs/package.json
+1 -1
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.41.0-beta.5",
"version": "0.41.0-beta.6",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
nodejs/src/query.rs
+10
View File
@@ -298,6 +298,16 @@ impl VectorQuery {
Ok(())
}
#[napi]
pub fn search_effort(&mut self, search_effort: Option<f64>) -> napi::Result<()> {
self.inner = self
.inner
.clone()
.search_effort(search_effort)
.default_error()?;
Ok(())
}
#[napi]
pub fn minimum_nprobes(&mut self, minimum_nprobe: u32) -> napi::Result<()> {
self.inner = self
python/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.41.0-beta.5"
version = "0.41.0-beta.6"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
python/pyproject.toml
+2 -1
View File
@@ -63,7 +63,7 @@ tests = [
"polars>=0.19, <=1.32.3",
"pyarrow<25",
"pyarrow-stubs>=16.0",
"pylance==14.0.0b10",
"pylance==14.0.0b12",
"requests>=2.31.0",
"datafusion>=54,<55",
"opentelemetry-sdk>=1.30.0",
@@ -73,6 +73,7 @@ dev = [
"pre-commit>=3.5.0",
"pyright>=1.1.350",
'typing-extensions>=4.0.0; python_version < "3.11"',
"maturin>=1.10",
]
docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings-python"]
clip = ["torch", "pillow>=12.1.1", "open-clip-torch"]
python/python/lancedb/_blob.py
+52 -11
View File
@@ -110,7 +110,9 @@ def projection_includes_blob_column(
if projection is None:
return True
for output, source in _iter_projection_pairs(projection):
if output in columns or source in columns:
if output in columns or any(
column == source or column.startswith(f"{source}.") for column in columns
):
return True
return False
@@ -122,14 +124,16 @@ def blob_v2_projection_sources(
blob_columns = row_addressable_blob_v2_paths(schema)
if not blob_columns:
return {}
columns = set(blob_columns)
if projection is None:
return {column: column for column in blob_columns}
return {
output: source
for output, source in _iter_projection_pairs(projection)
if source in columns
}
sources = {}
for output, source in _iter_projection_pairs(projection):
for column in blob_columns:
if column == source:
sources[output] = column
elif column.startswith(f"{source}."):
sources[f"{output}{column[len(source) :]}"] = column
return sources
def v2_projection_needs_row_id(
@@ -180,7 +184,7 @@ async def replace_v2_blob_columns_with_bytes(
fetch_blobs: FetchBlobsAsync,
) -> pa.Table:
for output_name, source_name in blob_sources.items():
if output_name not in tbl.column_names:
if not _has_column_path(tbl.schema, output_name):
continue
blobs = await fetch_blobs(source_name, tbl)
tbl = _set_blob_column(tbl, output_name, blobs)
@@ -193,7 +197,7 @@ def replace_v2_blob_columns_with_bytes_sync(
fetch_blobs: FetchBlobsSync,
) -> pa.Table:
for output_name, source_name in blob_sources.items():
if output_name not in tbl.column_names:
if not _has_column_path(tbl.schema, output_name):
continue
blobs = fetch_blobs(source_name, tbl)
tbl = _set_blob_column(tbl, output_name, blobs)
@@ -287,9 +291,46 @@ def _iter_projection_pairs(
yield name, source if source is not None else expr.to_sql()
def _set_blob_column(tbl: pa.Table, output_name: str, blobs: pa.Array) -> pa.Table:
def _has_column_path(schema: pa.Schema, path: str) -> bool:
if schema.get_field_index(path) >= 0:
return True
top_name, *rest = path.split(".")
index = schema.get_field_index(top_name)
if index < 0:
return False
field = schema.field(index)
for name in rest:
if not pa.types.is_struct(field.type):
return False
index = field.type.get_field_index(name)
if index < 0:
return False
field = field.type.field(index)
return True
def _set_blob_column(
tbl: pa.Table, output_name: str, blobs: pa.Array | pa.ChunkedArray
) -> pa.Table:
index = tbl.schema.get_field_index(output_name)
return tbl.set_column(index, pa.field(output_name, blobs.type), [blobs])
if index >= 0:
return tbl.set_column(index, pa.field(output_name, blobs.type), [blobs])
parent_path, leaf_name = output_name.rsplit(".", 1)
if isinstance(blobs, pa.ChunkedArray):
blobs = blobs.combine_chunks()
def replace_blob(children: list, child_fields: list) -> None:
for i, field in enumerate(child_fields):
if field.name == leaf_name:
children[i] = blobs
child_fields[i] = pa.field(
leaf_name, blobs.type, nullable=field.nullable
)
return
raise KeyError(f"Blob field '{leaf_name}' not found in struct '{parent_path}'")
return _transform_struct_column(tbl, parent_path, replace_blob)
def _embed_row_id_in_column(tbl: pa.Table, path: str, row_ids: pa.Array) -> pa.Table:
python/python/lancedb/_lancedb.pyi
+3
View File
@@ -727,6 +727,7 @@ class VectorQuery:
def use_lsm(self, enable: bool): ...
def refine_factor(self, refine_factor: int): ...
def nprobes(self, nprobes: int): ...
def search_effort(self, search_effort: Optional[float]): ...
def minimum_nprobes(self, minimum_nprobes: int): ...
def maximum_nprobes(self, maximum_nprobes: int): ...
def bypass_vector_index(self): ...
@@ -752,6 +753,7 @@ class HybridQuery:
def distance_type(self, distance_type: str): ...
def refine_factor(self, refine_factor: int): ...
def nprobes(self, nprobes: int): ...
def search_effort(self, search_effort: Optional[float]): ...
def minimum_nprobes(self, minimum_nprobes: int): ...
def maximum_nprobes(self, maximum_nprobes: int): ...
def bypass_vector_index(self): ...
@@ -781,6 +783,7 @@ class PyQueryRequest:
nprobes: Optional[int]
minimum_nprobes: Optional[int]
maximum_nprobes: Optional[int]
search_effort: Optional[float]
lower_bound: Optional[float]
upper_bound: Optional[float]
ef: Optional[int]
python/python/lancedb/namespace.py
+5
View File
@@ -105,6 +105,11 @@ def _query_to_namespace_request(
if not isinstance(query, Query):
raise TypeError(f"Expected Query, got {type(query)}")
if query.search_effort is not None:
raise ValueError(
"namespace QueryTable does not support search_effort; "
"execute the query locally"
)
vector_request = None
if query.vector is not None:
python/python/lancedb/query.py
+85 -3
View File
@@ -475,8 +475,8 @@ class MatchQuery(FullTextQuery):
----------
query : str
The query string to match against.
column : str
The name of the column to match against.
column : str, optional
The column to match against. None leaves column selection to query planning.
boost : float, default 1.0
The boost factor for the query.
The score of each matching document is multiplied by this value.
@@ -505,7 +505,7 @@ class MatchQuery(FullTextQuery):
"""
query: str
column: str
column: Optional[str] = None
boost: float = pydantic.Field(1.0, kw_only=True)
fuzziness: Optional[int] = pydantic.Field(0, kw_only=True)
max_expansions: int = pydantic.Field(50, kw_only=True)
@@ -793,6 +793,9 @@ class Query(pydantic.BaseModel):
# and all partitions could be searched if needed to satisfy the limit.
maximum_nprobes: Optional[int] = None
# Optional initial IVF effort; None preserves Auto and explicit nprobes.
search_effort: Optional[float] = None
# lower bound for distance search
lower_bound: Optional[float] = None
@@ -854,6 +857,7 @@ class Query(pydantic.BaseModel):
query.nprobes = req.nprobes
query.minimum_nprobes = req.minimum_nprobes
query.maximum_nprobes = req.maximum_nprobes
query.search_effort = req.search_effort
query.lower_bound = req.lower_bound
query.upper_bound = req.upper_bound
query.ef = req.ef
@@ -1635,6 +1639,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
self._nprobes = None
self._minimum_nprobes = None
self._maximum_nprobes = None
self._search_effort = None
self._lower_bound = None
self._upper_bound = None
self._refine_factor = None
@@ -1716,6 +1721,27 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
self._nprobes = nprobes
return self
def search_effort(self, search_effort: Optional[float]) -> LanceVectorQueryBuilder:
"""Set the initial IVF search effort to a finite value in [0, 1].
None (the default) uses Auto, internally 0.5. Zero starts at the caller
minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found. Any explicit
effort, including 0.5, overrides nprobes in either call order. Explicit
minimum and maximum bounds still apply. This is not a recall guarantee
and does not change refinement or HNSW ef. MemWAL LSM queries do not
yet support this option.
Examples
--------
>>> import lancedb
>>> db = lancedb.connect("memory://")
>>> table = db.create_table("effort_vectors", [{"vector": [0.1, 0.2]}])
>>> query = table.search([0.1, 0.2]).search_effort(0.75)
"""
self._search_effort = search_effort
return self
def minimum_nprobes(self, minimum_nprobes: int) -> LanceVectorQueryBuilder:
"""Set the minimum number of probes to use.
@@ -1858,6 +1884,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
nprobes=self._nprobes,
minimum_nprobes=self._minimum_nprobes,
maximum_nprobes=self._maximum_nprobes,
search_effort=self._search_effort,
lower_bound=self._lower_bound,
upper_bound=self._upper_bound,
refine_factor=self._refine_factor,
@@ -2217,6 +2244,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
self._nprobes = None
self._minimum_nprobes = None
self._maximum_nprobes = None
self._search_effort = None
self._refine_factor = None
self._distance_type = None
self._phrase_query = None
@@ -2524,6 +2552,32 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
self._nprobes = nprobes
return self
def search_effort(self, search_effort: Optional[float]) -> LanceHybridQueryBuilder:
"""Set the initial IVF search effort to a finite value in [0, 1].
None (the default) uses Auto, internally 0.5. Zero starts at the caller
minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found. Any explicit
effort, including 0.5, overrides nprobes in either call order. Explicit
minimum and maximum bounds still apply. This is not a recall guarantee
and does not change refinement or HNSW ef. MemWAL LSM queries do not
yet support this option.
Examples
--------
>>> import lancedb
>>> db = lancedb.connect("memory://")
>>> table = db.create_table(
... "effort_hybrid", [{"vector": [0.1, 0.2], "text": "dog"}]
... )
>>> query = (
... table.search(query_type="hybrid").vector([0.1, 0.2])
... .text("dog").search_effort(0.75)
... )
"""
self._search_effort = search_effort
return self
def minimum_nprobes(self, minimum_nprobes: int) -> LanceHybridQueryBuilder:
"""Set the minimum number of probes to use.
@@ -2802,6 +2856,8 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
self._vector_query.metric(self._distance_type)
if self._nprobes is not None:
self._vector_query.nprobes(self._nprobes)
if self._search_effort is not None:
self._vector_query.search_effort(self._search_effort)
if self._minimum_nprobes is not None:
self._vector_query.minimum_nprobes(self._minimum_nprobes)
if self._maximum_nprobes is not None:
@@ -3730,6 +3786,32 @@ class AsyncVectorQueryBase:
self._inner.nprobes(nprobes)
return self
def search_effort(self, search_effort: Optional[float]) -> Self:
"""Set the initial IVF search effort to a finite value in [0, 1].
None preserves Auto (internally 0.5) and fixed nprobes. Any explicit
effort, including 0.5, overrides nprobes in either call order; explicit
minimum and maximum bounds remain effective. Zero starts at the caller
minimum and one searches all partitions within the maximum. Later
probing may continue to obtain enough candidates. This is not a recall
guarantee and does not change refinement or HNSW ef. MemWAL LSM queries
do not yet support this option.
Examples
--------
>>> import asyncio
>>> import lancedb
>>> async def build_query():
... db = await lancedb.connect_async("memory://")
... table = await db.create_table(
... "effort_async", [{"vector": [0.1, 0.2]}]
... )
... return table.vector_search([0.1, 0.2]).search_effort(0.75)
>>> query = asyncio.run(build_query())
"""
self._inner.search_effort(search_effort)
return self
def minimum_nprobes(self, minimum_nprobes: int) -> Self:
"""Set the minimum number of probes to use.
python/python/lancedb/table.py
+4
View File
@@ -4440,6 +4440,7 @@ class LanceTable(Table):
and not self._route_pushdown_to_rust
and self.current_branch() is None
and query.take_offsets is None
and query.search_effort is None
):
from lancedb.namespace import _execute_server_side_query
@@ -6441,6 +6442,8 @@ class AsyncTable:
async_query = async_query.distance_type(query.distance_type)
if query.nprobes is not None:
async_query = async_query.nprobes(query.nprobes)
if query.search_effort is not None:
async_query = async_query.search_effort(query.search_effort)
if query.minimum_nprobes is not None and query.maximum_nprobes is not None:
async_query = async_query.minimum_nprobes(
query.minimum_nprobes
@@ -6481,6 +6484,7 @@ class AsyncTable:
)
and not self._route_pushdown_to_rust
and query.take_offsets is None
and query.search_effort is None
):
from lancedb.namespace import _execute_server_side_query
python/python/tests/test_blob.py
+116
View File
@@ -15,6 +15,7 @@ from lance.blob import Blob, BlobType as LanceBlobType
import lancedb
from lancedb._blob import (
_set_blob_column,
blob_v2_projection_sources,
read_row_ids_from_hits,
stash_auto_row_ids,
@@ -297,6 +298,121 @@ def test_blob_v2_projection_sources_use_typed_column_name():
}
def test_blob_v2_projection_sources_include_selected_struct():
schema = pa.schema(
[
pa.field(
"info", pa.struct([pa.field("name", pa.string()), lancedb.blob("blob")])
)
]
)
assert blob_v2_projection_sources(schema, ["info"]) == {"info.blob": "info.blob"}
assert blob_v2_projection_sources(schema, {"renamed": "info"}) == {
"renamed.blob": "info.blob"
}
@pytest.mark.parametrize("parent_path", ["info", "outer.info"])
def test_set_blob_column_missing_nested_field_raises(parent_path):
row = {"name": "first"}
for name in reversed(parent_path.split(".")):
row = {name: row}
table = pa.Table.from_pylist([row])
blobs = pa.array([b"hello"], type=pa.large_binary())
with pytest.raises(KeyError) as exc_info:
_set_blob_column(table, f"{parent_path}.blob", blobs)
assert parent_path in str(exc_info.value)
assert "'blob'" in str(exc_info.value)
def _nested_blob_table(db, name):
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field(
"info", pa.struct([pa.field("name", pa.string()), lancedb.blob("blob")])
),
]
)
table = db.create_table(name, schema=schema)
info = pa.StructArray.from_arrays(
[
pa.array(["first", "second", "third"]),
pa.array([b"hello", None, b"hidden"], type=pa.large_binary()),
],
names=["name", "blob"],
mask=pa.array([False, False, True]),
)
table.add(pa.table({"id": pa.array([0, 1, 2]), "info": info}))
return table
def test_nested_blob_to_pandas_bytes(tmp_path):
table = _nested_blob_table(lancedb.connect(tmp_path), "nested_pandas_bytes")
expected = [
{"name": "first", "blob": b"hello"},
{"name": "second", "blob": None},
None,
]
assert table.to_pandas(blob_mode="bytes")["info"].tolist() == expected
assert table.search().to_pandas(blob_mode="bytes")["info"].tolist() == expected
assert (
table.search().select(["info"]).to_pandas(blob_mode="bytes")["info"].tolist()
== expected
)
assert (
table.search()
.select({"renamed": "info"})
.to_pandas(blob_mode="bytes")["renamed"]
.tolist()
== expected
)
assert table.search().select({"b": "info.blob"}).to_pandas(blob_mode="bytes")[
"b"
].tolist() == [b"hello", None, None]
@pytest.mark.asyncio
async def test_nested_blob_to_pandas_bytes_async(tmp_path):
db = await lancedb.connect_async(tmp_path)
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field(
"info", pa.struct([pa.field("name", pa.string()), lancedb.blob("blob")])
),
]
)
table = await db.create_table("nested_pandas_bytes_async", schema=schema)
await table.add(
pa.table(
{
"id": pa.array([0, 1]),
"info": pa.StructArray.from_arrays(
[
pa.array(["first", "second"]),
pa.array([b"hello", None], type=pa.large_binary()),
],
names=["name", "blob"],
),
}
)
)
expected = [
{"name": "first", "blob": b"hello"},
{"name": "second", "blob": None},
]
assert (await table.to_pandas(blob_mode="bytes"))["info"].tolist() == expected
assert (await table.query().to_pandas(blob_mode="bytes"))[
"info"
].tolist() == expected
def _legacy_v1_table(name):
# Legacy v1 blob columns are only writable at file version <= 2.1.
db = lancedb.connect(
python/python/tests/test_namespace.py
+12
View File
@@ -1209,3 +1209,15 @@ def test_local_table_to_arrow_and_to_pandas_are_unchanged(tmp_path):
assert table.to_arrow().column("id").to_pylist() == [1, 2]
assert table.to_pandas()["id"].tolist() == [1, 2]
@pytest.mark.parametrize("effort", [0.0, 0.5, 1.0])
def test_namespace_request_rejects_search_effort(effort):
from lancedb.query import Query
with pytest.raises(
ValueError, match="namespace QueryTable does not support search_effort"
):
_query_to_namespace_request(
["test"], Query(vector=[1.0, 0.0], search_effort=effort)
)
python/python/tests/test_query.py
+124
View File
@@ -2265,3 +2265,127 @@ def test_blob_v2_to_batches_row_id(tmp_db):
assert "_rowid" in hits.column_names
blobs = table.fetch_blobs("blob", hits)
assert [blobs[i].as_py() for i in range(len(blobs))] == [b"one", b"two"]
@pytest.mark.parametrize("metric", ["l2", "cosine", "dot"])
@pytest.mark.parametrize(
"effort,probes", [(None, 4), (0, 1), (0.25, 2), (0.5, 4), (0.75, 8), (1, 16)]
)
def test_search_effort_probe_budget(tmp_path, monkeypatch, metric, effort, probes):
import re
from lancedb.index import IvfFlat
monkeypatch.setenv("LANCE_AUTO_PROBE_MARGIN", "0")
monkeypatch.setenv("LANCE_AUTO_MIN_INITIAL_NPROBES", "4")
monkeypatch.setenv("LANCE_AUTO_MAX_INITIAL_NPROBES", "4")
rng = np.random.default_rng(42)
vectors = np.repeat(np.eye(16, dtype=np.float32), 32, axis=0)
vectors += rng.normal(0, 0.001, vectors.shape).astype(np.float32)
data = pa.table(
{
"id": np.arange(512),
"text": ["dog"] * 512,
"vector": pa.FixedSizeListArray.from_arrays(pa.array(vectors.ravel()), 16),
}
)
table = lancedb.connect(tmp_path).create_table("effort", data.slice(0, 256))
table.add(data.slice(256))
table.create_index(
"vector", config=IvfFlat(distance_type=metric, num_partitions=16)
)
vector = np.eye(16, dtype=np.float32)[0]
def search(value=effort):
return table.search(vector).distance_type(metric).search_effort(value).limit(8)
def partitions(query):
vector_plan = query.analyze_plan().split("FTS Search Plan:", 1)[0]
values = re.findall(r"partitions_searched=(\d+)", vector_plan)
assert values
return sum(map(int, values))
assert partitions(search()) == probes
exact = set(
table.search(vector)
.distance_type(metric)
.bypass_vector_index()
.limit(8)
.to_arrow()["id"]
.to_pylist()
)
actual = set(search().to_arrow()["id"].to_pylist())
assert len(actual & exact) / 8 >= 0.5
expected = 1 if effort is None else probes
assert partitions(search().nprobes(1)) == expected
assert (
partitions(
table.search(vector)
.distance_type(metric)
.nprobes(1)
.search_effort(effort)
.limit(8)
)
== expected
)
assert partitions(search().nprobes(1).search_effort(None)) == 1
assert partitions(search().maximum_nprobes(2)) <= 2
assert partitions(search().minimum_nprobes(3).maximum_nprobes(3)) == 3
assert partitions(search(0).nprobes(16).maximum_nprobes(0)) == 1
if effort is None:
assert search().to_arrow() == search(0.5).to_arrow()
filtered = search(0).maximum_nprobes(2).where("id % 8 = 0", prefilter=True)
assert partitions(filtered) <= 2
assert all(row["id"] % 8 == 0 for row in filtered.to_list())
if metric == "l2" and effort == 0.5:
table.create_index("text", config=FTS())
hybrid = (
table.search(query_type="hybrid")
.vector(vector)
.text("dog")
.nprobes(1)
.search_effort(0.5)
.limit(8)
)
assert partitions(hybrid) == 4
assert partitions(hybrid.search_effort(None)) == 1
@pytest.mark.parametrize(
"effort", [-0.1, 1.1, float("nan"), float("inf"), -float("inf")]
)
def test_search_effort_invalid_sync(table, effort):
with pytest.raises(ValueError, match="search_effort must be finite and in"):
table.search([0, 0]).search_effort(effort).to_list()
@pytest.mark.asyncio
@pytest.mark.parametrize("hybrid", [False, True])
@pytest.mark.parametrize("effort", [None, 0, 0.25, 0.5, 0.75, 1])
async def test_search_effort_async_roundtrip(table_async, hybrid, effort):
query = table_async.vector_search([0, 0])
if hybrid:
query = query.nearest_to_text("dog")
query = query.nprobes(5).search_effort(effort).minimum_nprobes(2).maximum_nprobes(3)
request = query.to_query_object()
assert request.search_effort == effort
assert request.nprobes == 5
assert request.minimum_nprobes == 2
assert request.maximum_nprobes == 3
assert query.search_effort(None).to_query_object().search_effort is None
query.search_effort(effort)
if not hybrid:
assert len(await query.limit(1).to_list()) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("hybrid", [False, True])
@pytest.mark.parametrize(
"effort", [-0.1, 1.1, float("nan"), float("inf"), -float("inf")]
)
async def test_search_effort_invalid_async(table_async, hybrid, effort):
query = table_async.vector_search([0, 0])
if hybrid:
query = query.nearest_to_text("dog")
with pytest.raises(ValueError, match="search_effort must be finite and in"):
query.search_effort(effort)
python/src/query.rs
+31 -3
View File
@@ -71,7 +71,7 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
match class_name(&ob)?.as_str() {
"MatchQuery" => {
let query = ob.getattr("query")?.extract()?;
let column = ob.getattr("column")?.extract()?;
let column = ob.getattr("column")?.extract::<Option<String>>()?;
let boost = ob.getattr("boost")?.extract()?;
let fuzziness = ob.getattr("fuzziness")?.extract()?;
let max_expansions = ob.getattr("max_expansions")?.extract()?;
@@ -88,7 +88,7 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
let mut query =
MatchQuery::new(query)
.with_column(Some(column))
.with_column(column)
.with_boost(boost)
.with_fuzziness(fuzziness)
.with_max_expansions(max_expansions)
@@ -199,7 +199,7 @@ impl<'py> IntoPyObject<'py> for PyLanceDB<FtsQuery> {
}
namespace
.getattr(intern!(py, "MatchQuery"))?
.call((query.terms, query.column.unwrap()), Some(&kwargs))
.call((query.terms, query.column), Some(&kwargs))
}
FtsQuery::Phrase(query) => {
let kwargs = PyDict::new(py);
@@ -339,6 +339,7 @@ pub struct PyQueryRequest {
pub nprobes: Option<usize>,
pub minimum_nprobes: Option<usize>,
pub maximum_nprobes: Option<usize>,
pub search_effort: Option<f64>,
pub lower_bound: Option<f32>,
pub upper_bound: Option<f32>,
pub ef: Option<usize>,
@@ -371,6 +372,7 @@ impl From<AnyQuery> for PyQueryRequest {
nprobes: None,
minimum_nprobes: None,
maximum_nprobes: None,
search_effort: None,
lower_bound: None,
upper_bound: None,
ef: None,
@@ -399,6 +401,7 @@ impl From<AnyQuery> for PyQueryRequest {
nprobes: vector_query.nprobes,
minimum_nprobes: vector_query.minimum_nprobes,
maximum_nprobes: vector_query.maximum_nprobes,
search_effort: vector_query.search_effort,
lower_bound: vector_query.lower_bound,
upper_bound: vector_query.upper_bound,
ef: vector_query.ef,
@@ -998,6 +1001,15 @@ impl VectorQuery {
Ok(())
}
pub fn search_effort(&mut self, search_effort: Option<f64>) -> PyResult<()> {
self.inner = self
.inner
.clone()
.search_effort(search_effort)
.infer_error()?;
Ok(())
}
pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> {
self.inner = self
.inner
@@ -1191,6 +1203,18 @@ impl HybridQuery {
self.inner_vec.nprobes(nprobe)
}
pub fn search_effort(&mut self, search_effort: Option<f64>) -> PyResult<()> {
self.inner_vec.search_effort(search_effort)
}
pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> {
self.inner_vec.minimum_nprobes(minimum_nprobes)
}
pub fn maximum_nprobes(&mut self, maximum_nprobes: u32) -> PyResult<()> {
self.inner_vec.maximum_nprobes(maximum_nprobes)
}
pub fn ef(&mut self, ef: u32) {
self.inner_vec.ef(ef);
}
@@ -1235,6 +1259,10 @@ impl HybridQuery {
req.query_vector = vec_req.query_vector;
req.column = vec_req.column;
req.distance_type = vec_req.distance_type;
req.nprobes = vec_req.nprobes;
req.minimum_nprobes = vec_req.minimum_nprobes;
req.maximum_nprobes = vec_req.maximum_nprobes;
req.search_effort = vec_req.search_effort;
req.ef = vec_req.ef;
req.refine_factor = vec_req.refine_factor;
req.lower_bound = vec_req.lower_bound;
python/uv.lock
Generated
+34 -8
View File
@@ -1900,6 +1900,7 @@ clip = [
{ name = "torch" },
]
dev = [
{ name = "maturin" },
{ name = "pre-commit" },
{ name = "pyright" },
{ name = "ruff" },
@@ -1981,6 +1982,7 @@ requires-dist = [
{ name = "ibm-watsonx-ai", marker = "python_full_version >= '3.10' and extra == 'embeddings'", specifier = ">=1.1.2" },
{ name = "instructorembedding", marker = "extra == 'embeddings'", specifier = ">=1.0.1" },
{ name = "lance-namespace", specifier = ">=0.13.0" },
{ name = "maturin", marker = "extra == 'dev'", specifier = ">=1.10" },
{ name = "mkdocs", marker = "extra == 'docs'" },
{ name = "mkdocs-jupyter", marker = "extra == 'docs'" },
{ name = "mkdocs-material", marker = "extra == 'docs'" },
@@ -2005,7 +2007,7 @@ requires-dist = [
{ name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" },
{ name = "pydantic", specifier = ">=2.7.4,<3" },
{ name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b10" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b12" },
{ name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" },
{ name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" },
{ name = "pytest-asyncio", marker = "extra == 'tests'", specifier = ">=0.21" },
@@ -2156,6 +2158,30 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" },
]
[[package]]
name = "maturin"
version = "1.15.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "tomli", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b9/c8/22e5e21b2679c9bce6415ca578034ca2cc9316be0642ae21e051a2d5198c/maturin-1.15.0.tar.gz", hash = "sha256:94b26cc8e8aba61a5f2099715fe640e18c5f678e9a500408b38761263954228a", size = 385504, upload-time = "2026-08-24T12:11:22.665Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/14/69/5c01b461044eb1f45ddcce006706eb88110c793cdb11c7ae0b5e08492e94/maturin-1.15.0-py3-none-linux_armv6l.whl", hash = "sha256:6bf6dc62e22d4dcfd5a51244ff0d58975fa4979c48209fe84159617648956d82", size = 10206220, upload-time = "2026-08-24T12:10:53.327Z" },
{ url = "https://files.pythonhosted.org/packages/eb/1f/2b431554e11687cdb1077e0cdadcc118c53f611086b3af00c8545a67c6a5/maturin-1.15.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:cd35772633f489841132bc8e71d6fc7f842df30b9c05cd5cdf1ee1ddcb744cc7", size = 19416513, upload-time = "2026-08-24T12:10:56.126Z" },
{ url = "https://files.pythonhosted.org/packages/51/36/e23a21cb34a648b711036b9b2fe1d4f3f4ee24f8db54215d73f1a9a3a3ec/maturin-1.15.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c40b4eae7bf5ef1f4b1af8d623fe4105016f93578fb15b764e741d08ec3b92dd", size = 10014962, upload-time = "2026-08-24T12:10:58.486Z" },
{ url = "https://files.pythonhosted.org/packages/8c/80/33b15cb2d8f30f12c807955e8f2fd775027692904e30ec0784744ce8cd83/maturin-1.15.0-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:7eb066372f541f8eb4909c79c5d9bd0b9e8125980bdf1ec9e8aba23c6c8d6c55", size = 10196223, upload-time = "2026-08-24T12:11:00.696Z" },
{ url = "https://files.pythonhosted.org/packages/fe/91/b495e19e2f5c503b540452b2039115e7b2363867e8c5ad4179eb752fa92c/maturin-1.15.0-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:653020a63525bb224e5ab0adf02e17a2e08bc86dbea7fc1399c9a56d7529b99e", size = 10541186, upload-time = "2026-08-24T12:11:02.857Z" },
{ url = "https://files.pythonhosted.org/packages/5d/2b/2abff58037188d852b124871b1f0d720e1c2bfb3d4f1b03d87c52cd66488/maturin-1.15.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:0ebf9767892725083138e671c34482c660317a2f3d6a29fc0e0f34e9d8c99136", size = 10083468, upload-time = "2026-08-24T12:11:05.012Z" },
{ url = "https://files.pythonhosted.org/packages/3f/07/b7e9f8be99a6627849e81ac7b6694876bce8f50a92995fe17e3cf2610f0a/maturin-1.15.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:7ab7eebffd7b8debca2265985de4eaeb332141276d24b9560b5ad484d4b3add1", size = 10047786, upload-time = "2026-08-24T12:11:07.042Z" },
{ url = "https://files.pythonhosted.org/packages/ca/ab/167e3cb7accee11b507dbe53e0e87aeccb376d44ae66284c96ee4df3a9fd/maturin-1.15.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:126e12e618b4db42f68c779a56d41f82a390145ba36ac3f621d057eb34f5ad9d", size = 13315332, upload-time = "2026-08-24T12:11:09.433Z" },
{ url = "https://files.pythonhosted.org/packages/14/4d/801379f646cbc6b00998e5289b0630a886be3a4ee4c75b6bc9b87478a7f1/maturin-1.15.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4f9d33e6c3f9615c8caceecbbbd440f8eb25a3ddeb687077682cd5eca2e9ae15", size = 10807183, upload-time = "2026-08-24T12:11:11.73Z" },
{ url = "https://files.pythonhosted.org/packages/89/27/2e612e1cbd1580e9e94d4722c227b5180dca27b32b955a34b79918aa1292/maturin-1.15.0-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:bf29beddd0c6708f112db51d5275fc28b28b9e9c9c5faae387eaef662918b176", size = 10413274, upload-time = "2026-08-24T12:11:14.04Z" },
{ url = "https://files.pythonhosted.org/packages/70/d8/202a7b4d75a51f20f84ec9ce3b7345b12b822207072164e2b1c6ef665125/maturin-1.15.0-py3-none-win32.whl", hash = "sha256:da649988be98e87e009e51b1bf0d301b6a301bc0cecbdd60d40d8ba60748d1ca", size = 8928744, upload-time = "2026-08-24T12:11:16.269Z" },
{ url = "https://files.pythonhosted.org/packages/40/dc/4e90da594986ba78dd3bc8a5921ecdcdb11085b22b02a412caab3b225601/maturin-1.15.0-py3-none-win_amd64.whl", hash = "sha256:552c2be4afd43fe8d5c9f3ec8d4c4756d973b8dcbe94c14084390301f50243e1", size = 10335085, upload-time = "2026-08-24T12:11:18.326Z" },
{ url = "https://files.pythonhosted.org/packages/8b/10/15d4314edf130955edf2dc237aa393a8a7c10f2b9b57b89fa2f61f915659/maturin-1.15.0-py3-none-win_arm64.whl", hash = "sha256:c7dc0c66c78d3debdd9c5aa807e861fbcbf07f3505d34b125df74c03986b0f48", size = 9713795, upload-time = "2026-08-24T12:11:20.83Z" },
]
[[package]]
name = "mdit-py-plugins"
version = "0.6.1"
@@ -3912,7 +3938,7 @@ crypto = [
[[package]]
name = "pylance"
version = "14.0.0b10"
version = "14.0.0b12"
source = { registry = "https://pypi.fury.io/lance-format/" }
dependencies = [
{ name = "lance-namespace" },
@@ -3922,12 +3948,12 @@ dependencies = [
{ name = "pyarrow" },
]
wheels = [
{ url = "https://pypi.fury.io/lance-format/-/ver_4hvE/pylance-14.0.0b10-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:415b6b1b9fe735fbae7c755eb9774ae2fe0feddc90bf80bee24087cbfbfdf8c1" },
{ url = "https://pypi.fury.io/lance-format/-/ver_pM1km/pylance-14.0.0b10-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87317cad22f4d35fea11eb70aa5c0d71a0e1ea3b96de38f3deca933daa749fff" },
{ url = "https://pypi.fury.io/lance-format/-/ver_24jdyj/pylance-14.0.0b10-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6da554a2fc2197ba254676b12824d8120b48d65937697e0661a61302458a4740" },
{ url = "https://pypi.fury.io/lance-format/-/ver_XclaZ/pylance-14.0.0b10-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:36c80a801675bb9dcbd681fa99d0cdd5ca642dfb3ca08d71ac980278a2f7748b" },
{ url = "https://pypi.fury.io/lance-format/-/ver_L0yh/pylance-14.0.0b10-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:e50ad0b866faefb2fa3ae87328b93fa0d06b2bdd5bb5d0c1a6c74b907cc2fd93" },
{ url = "https://pypi.fury.io/lance-format/-/ver_2gbNr8/pylance-14.0.0b10-cp310-abi3-win_amd64.whl", hash = "sha256:3d86451fe00f71510005bb700737fb1a4f9361554541cbf802f3c5072554d014" },
{ url = "https://pypi.fury.io/lance-format/-/ver_1Ixasc/pylance-14.0.0b12-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:84350a3c5c1eea3398f4c4fad7cfe85b831d831a5e43b5ca2a783dbdb0e11969" },
{ url = "https://pypi.fury.io/lance-format/-/ver_sCEkG/pylance-14.0.0b12-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76b386c182a6a8e7b86fbef28f84d1dd39e4626ac8681a0cf36f40a45dd7cb87" },
{ url = "https://pypi.fury.io/lance-format/-/ver_1vdT1C/pylance-14.0.0b12-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6aa548c7937fcdfa8f1943a2d59041a42f8b500199880dbb509c7195fa253381" },
{ url = "https://pypi.fury.io/lance-format/-/ver_2fXu1V/pylance-14.0.0b12-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:79f82d6a6c50f37aab338470e3733436e6a27fbb5972d29fcf453ece56d3b3c6" },
{ url = "https://pypi.fury.io/lance-format/-/ver_24UavN/pylance-14.0.0b12-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:de0305999257002785f60169e6926896584d66f1da050bb5afd1d48c6e3d9bb9" },
{ url = "https://pypi.fury.io/lance-format/-/ver_XNynX/pylance-14.0.0b12-cp310-abi3-win_amd64.whl", hash = "sha256:4d0cdb689e898ce71e8e434d00a3b97639cea8fb97da21f1c921543be363268f" },
]
[[package]]
python/vendor/lance-arrow/Cargo.toml
Vendored
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lance-arrow"
version = "14.0.0-beta.13"
version = "14.0.0-beta.15"
edition = "2024"
authors = ["Lance Devs <dev@lance.org>"]
license = "Apache-2.0"
python/vendor/lance-arrow/README.lancedb.md
Vendored
+2 -2
View File
@@ -1,5 +1,5 @@
This directory contains `rust/lance-arrow` from Lance tag `v14.0.0-beta.13`
(commit `145fd4b933fb4a9a45831f0ae9185c88e908fe6a`), licensed under Apache-2.0.
This directory contains `rust/lance-arrow` from Lance tag `v14.0.0-beta.15`
(commit `765641dc0d7100f032fdebc00d73eef49711c4d4`), licensed under Apache-2.0.
LanceDB carries a local fix in `src/lib.rs` for sliced child validity bitmaps
until a Lance release includes the fix. The patch is selected in the root
rust/lancedb/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.41.0-beta.5"
version = "0.41.0-beta.6"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
rust/lancedb/src/query.rs
+66 -1
View File
@@ -1127,6 +1127,11 @@ pub struct VectorQueryRequest {
/// If unset, Lance's default is used. Zero represents an explicitly unbounded
/// maximum on request and wire representations.
pub maximum_nprobes: Option<usize>,
/// Optional initial IVF search effort in [0, 1].
///
/// None preserves Auto and fixed `nprobes`; any explicit value overrides
/// `nprobes` while retaining explicit minimum and maximum bounds.
pub search_effort: Option<f64>,
/// The lower bound (inclusive) of the distance to search for.
pub lower_bound: Option<f32>,
/// The upper bound (exclusive) of the distance to search for.
@@ -1153,6 +1158,7 @@ impl Default for VectorQueryRequest {
nprobes: None,
minimum_nprobes: None,
maximum_nprobes: None,
search_effort: None,
lower_bound: None,
upper_bound: None,
ef: None,
@@ -1165,6 +1171,17 @@ impl Default for VectorQueryRequest {
}
impl VectorQueryRequest {
pub(crate) fn validate_search_effort(&self) -> Result<()> {
if let Some(effort) = self.search_effort
&& (!effort.is_finite() || !(0.0..=1.0).contains(&effort))
{
return Err(Error::InvalidInput {
message: format!("search_effort must be finite and in [0, 1], got {effort}"),
});
}
Ok(())
}
fn add_query_vector(&mut self, vector: impl IntoQueryVector) -> Result<()> {
let vector = vector.to_query_vector(&DataType::Float32, "default")?;
let array =
@@ -1279,7 +1296,8 @@ impl VectorQuery {
/// you the desired recall.
///
/// LanceDB retains this as `nprobes` through local and remote request
/// construction. Lance sets both probe bounds to this value. For more
/// construction. Lance sets both probe bounds to this value only when
/// [`Self::search_effort`] is unset. For more
/// fine-grained control see [`VectorQuery::minimum_nprobes`] and
/// [`VectorQuery::maximum_nprobes`].
///
@@ -1294,6 +1312,31 @@ impl VectorQuery {
Ok(self)
}
/// Set the initial IVF search effort to a finite value in [0, 1].
///
/// None (the default) resolves to 0.5, preserving Auto probing. At 0 the
/// search starts at the caller minimum; at 1 it searches all partitions
/// within the caller maximum. Intermediate values interpolate around Auto.
/// Later probing may continue when too few candidates are found.
/// Any explicit value, including 0.5, overrides [`Self::nprobes`] regardless
/// of setter order. Explicit minimum and maximum bounds still apply.
/// This does not change refinement or HNSW ef and is not a recall guarantee.
/// Lance's MemWAL LSM scanner does not yet support this option.
///
/// ```
/// # use lancedb::{Table, query::{QueryBase, ExecutableQuery}};
/// # async fn example(table: &Table) -> lancedb::Result<()> {
/// let results = table.query().nearest_to([0.1, 0.2].as_slice())?
/// .search_effort(0.75)?.execute().await?;
/// # Ok(())
/// # }
/// ```
pub fn search_effort(mut self, search_effort: impl Into<Option<f64>>) -> Result<Self> {
self.request.search_effort = search_effort.into();
self.request.validate_search_effort()?;
Ok(self)
}
/// Set the minimum number of partitions to search
///
/// This argument is only used when the vector column has an IVF PQ index.
@@ -1333,6 +1376,8 @@ impl VectorQuery {
///
/// If this value is greater than minimum_nprobes then the excess partitions will
/// only be searched if the initial search does not return enough results.
/// The bound applies per index segment to initial and later probing without
/// disabling Auto probing.
///
/// This can be useful when there is a narrow filter to allow these queries to
/// spend more time searching and avoid potential false negatives.
@@ -2448,6 +2493,26 @@ mod tests {
assert_eq!(query.request.nprobes, None);
assert_eq!(query.request.minimum_nprobes, None);
assert_eq!(query.request.maximum_nprobes, None);
assert_eq!(query.request.search_effort, None);
let effort_query = query
.clone()
.nprobes(20)
.unwrap()
.search_effort(0.5)
.unwrap();
assert_eq!(effort_query.request.search_effort, Some(0.5));
assert_eq!(effort_query.request.nprobes, Some(20));
assert_eq!(effort_query.request.minimum_nprobes, None);
assert_eq!(effort_query.request.maximum_nprobes, None);
assert_eq!(
effort_query
.search_effort(None)
.unwrap()
.request
.search_effort,
None
);
assert_eq!(
*query
.request
rust/lancedb/src/remote/table.rs
+75
View File
@@ -1137,6 +1137,7 @@ impl<S: HttpSend> RemoteTable<S> {
mut body: serde_json::Value,
query: &VectorQueryRequest,
) -> Result<Vec<serde_json::Value>> {
query.validate_search_effort()?;
self.apply_query_params(&mut body, &query.base)?;
// Apply general parameters, before we dispatch based on number of query vectors.
@@ -1155,6 +1156,9 @@ impl<S: HttpSend> RemoteTable<S> {
if let Some(maximum_nprobes) = query.maximum_nprobes {
body["maximum_nprobes"] = maximum_nprobes.into();
}
if let Some(search_effort) = query.search_effort {
body["search_effort"] = search_effort.into();
}
body["lower_bound"] = query.lower_bound.into();
body["upper_bound"] = query.upper_bound.into();
body["ef"] = query.ef.into();
@@ -6269,6 +6273,77 @@ mod tests {
));
}
#[rstest]
#[case::unset(None)]
#[case::minimum(Some(0.0))]
#[case::low(Some(0.25))]
#[case::auto(Some(0.5))]
#[case::high(Some(0.75))]
#[case::maximum(Some(1.0))]
#[tokio::test]
async fn test_query_vector_search_effort_wire(#[case] effort: Option<f64>) {
let table = Table::new_with_handler("my_table", move |request| {
let body = request_body_json(&request);
assert_eq!(body["nprobes"], 7);
assert_eq!(body["minimum_nprobes"], 2);
assert_eq!(body["maximum_nprobes"], 8);
if let Some(effort) = effort {
assert_eq!(body["search_effort"], effort);
} else {
assert!(body.get("search_effort").is_none());
}
let batch = RecordBatch::try_new(
Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])),
vec![Arc::new(Int32Array::from(vec![1]))],
)
.unwrap();
http::Response::builder()
.status(200)
.header(CONTENT_TYPE, ARROW_FILE_CONTENT_TYPE)
.body(write_ipc_file(&batch))
.unwrap()
});
let batches = table
.query()
.nearest_to(vec![0.1, 0.2])
.unwrap()
.search_effort(effort)
.unwrap()
.nprobes(7)
.unwrap()
.minimum_nprobes(2)
.unwrap()
.maximum_nprobes(Some(8))
.unwrap()
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
assert_eq!(batches[0].num_rows(), 1);
}
#[rstest]
#[case::negative(-0.1)]
#[case::too_large(1.1)]
#[case::nan(f64::NAN)]
#[case::infinity(f64::INFINITY)]
#[case::negative_infinity(f64::NEG_INFINITY)]
#[tokio::test]
async fn test_query_vector_search_effort_invalid(#[case] effort: f64) {
let table = Table::new_with_handler::<&str>("my_table", |_| {
panic!("invalid search_effort must not send a request")
});
let result = table
.query()
.nearest_to(vec![0.1, 0.2])
.unwrap()
.search_effort(effort);
assert!(matches!(result, Err(Error::InvalidInput { message })
if message.contains("search_effort must be finite and in [0, 1]")));
}
#[tokio::test]
async fn test_query_vector_all_params() {
let table = Table::new_with_handler("my_table", |request| {
rust/lancedb/src/table.rs
+54 -1
View File
@@ -19,6 +19,9 @@ pub use lance::dataset::ReadParams;
pub use lance::dataset::Version;
use lance::dataset::WriteMode;
use lance::dataset::builder::DatasetBuilder;
pub use lance::dataset::mem_wal::MemIndexRegistry;
/// The traits a memtable index plugin implements; see [`MemIndexRegistry`].
pub use lance::dataset::mem_wal::index as mem_index;
use lance::dataset::{InsertBuilder, WriteParams};
use lance::index::DatasetIndexExt;
use lance::index::scalar::load_segment_params;
@@ -424,7 +427,8 @@ impl LsmWriteSpec {
/// changes, so an index created later is maintained too; one of a kind the
/// MemWAL cannot mirror is skipped rather than failing the table. A list is
/// verbatim: each name must already exist and be maintainable, and an empty
/// list maintains nothing.
/// list maintains nothing. The maintainable kinds are Lance's built-ins plus
/// any added with [`NativeTable::set_mem_index_registry`].
///
/// ```
/// # use lancedb::table::LsmWriteSpec;
@@ -2590,6 +2594,9 @@ pub struct NativeTable {
pub(crate) pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
// Read-freshness baseline; `Some` only for namespace-backed tables.
freshness: Option<TableFreshness>,
// Memtable index kinds this handle's LSM writes maintain, shared with its
// clones and branch handles.
mem_index_registry: Arc<std::sync::RwLock<MemIndexRegistry>>,
}
impl std::fmt::Debug for NativeTable {
@@ -2756,9 +2763,51 @@ impl NativeTable {
namespace_client,
pushdown_operations,
freshness: None,
mem_index_registry: Default::default(),
})
}
/// Maintain the memtable index kinds in `registry` when this table writes
/// through its [`LsmWriteSpec`], and accept them when
/// [`LsmWriteSpec::with_maintained_indexes`] names one.
///
/// The registry belongs to this handle, its clones and its branch handles,
/// and is not stored with the table: a handle opened separately starts from
/// Lance's built-ins. Set it before the handle's first LSM write; writers
/// already open keep the registry they were opened with. Local tables only,
/// reached through [`Table::as_native`].
///
/// ```
/// # use std::sync::Arc;
/// # use lancedb::Table;
/// # use lancedb::table::{LsmWriteSpec, MemIndexRegistry, mem_index::MemIndexPlugin};
/// # async fn example(
/// # table: &Table,
/// # plugin: Arc<dyn MemIndexPlugin>,
/// # ) -> Result<(), Box<dyn std::error::Error>> {
/// let registry = MemIndexRegistry::default().with_plugin(plugin)?;
/// if let Some(native) = table.as_native() {
/// native.set_mem_index_registry(registry);
/// }
/// let spec = LsmWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]);
/// table.set_lsm_write_spec(spec).await?;
/// # Ok(())
/// # }
/// ```
pub fn set_mem_index_registry(&self, registry: MemIndexRegistry) {
*self
.mem_index_registry
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = registry;
}
pub(crate) fn mem_index_registry(&self) -> MemIndexRegistry {
self.mem_index_registry
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
}
/// Set the namespace client for server-side query execution.
///
/// When set, queries will be executed on the namespace server instead of locally.
@@ -2786,6 +2835,7 @@ impl NativeTable {
namespace_client: self.namespace_client.clone(),
pushdown_operations: self.pushdown_operations.clone(),
freshness: self.freshness.clone(),
mem_index_registry: self.mem_index_registry.clone(),
}
}
@@ -2893,6 +2943,7 @@ impl NativeTable {
namespace_client: stored_namespace_client,
pushdown_operations,
freshness: None,
mem_index_registry: Default::default(),
})
}
@@ -2984,6 +3035,7 @@ impl NativeTable {
namespace_client,
pushdown_operations,
freshness: None,
mem_index_registry: Default::default(),
})
}
@@ -3117,6 +3169,7 @@ impl NativeTable {
namespace_client: stored_namespace_client,
pushdown_operations,
freshness: None,
mem_index_registry: Default::default(),
})
}
rust/lancedb/src/table/datafusion.rs
+10
View File
@@ -166,6 +166,7 @@ pub struct BaseTableAdapter {
table: Arc<dyn BaseTable>,
schema: Arc<ArrowSchema>,
fts_query: Option<FullTextSearchQuery>,
use_lsm: Option<bool>,
}
impl BaseTableAdapter {
@@ -181,9 +182,16 @@ impl BaseTableAdapter {
table,
schema: Arc::new(schema),
fts_query: None,
use_lsm: None,
})
}
/// Set the MemWAL read routing of every scan; see [`QueryRequest::use_lsm`].
pub fn with_use_lsm(mut self, use_lsm: Option<bool>) -> Self {
self.use_lsm = use_lsm;
self
}
/// Create a new adapter with an FTS query applied.
pub fn with_fts_query(&self, fts_query: FullTextSearchQuery) -> Self {
// Add _score column to the schema
@@ -196,6 +204,7 @@ impl BaseTableAdapter {
table: self.table.clone(),
schema,
fts_query: Some(fts_query),
use_lsm: self.use_lsm,
}
}
}
@@ -223,6 +232,7 @@ impl TableProvider for BaseTableAdapter {
let mut query = QueryRequest {
full_text_search: self.fts_query.clone(),
disable_scoring_autoprojection: disable_scoring,
use_lsm: self.use_lsm,
..Default::default()
};
rust/lancedb/src/table/merge.rs
+472 -21
View File
@@ -1349,6 +1349,41 @@ mod lsm_tests {
);
}
#[tokio::test]
async fn base_table_adapter_use_lsm_false_scans_the_base_only() {
use crate::table::datafusion::BaseTableAdapter;
use datafusion::prelude::SessionContext;
let dir = tempdir().unwrap();
let table = id_value_table(&dir).await; // base: ids 1,2,3
table
.set_lsm_write_spec(LsmWriteSpec::unsharded())
.await
.unwrap();
// Ids 4,5 land in the active memtable, not the base table.
lsm_upsert(&table, vec![4, 5]).await;
let adapter = BaseTableAdapter::try_new(table.base_table().clone())
.await
.unwrap()
.with_use_lsm(Some(false));
let ctx = SessionContext::new();
ctx.register_table("base_only", Arc::new(adapter)).unwrap();
let batches = ctx
.sql("SELECT id FROM base_only")
.await
.unwrap()
.collect()
.await
.unwrap();
let mut ids: Vec<i64> = batches
.iter()
.flat_map(|b| b.column(0).as_primitive::<Int64Type>().values().to_vec())
.collect();
ids.sort();
assert_eq!(ids, vec![1, 2, 3]);
}
#[tokio::test]
async fn query_snapshot_preserves_lsm_read_semantics() {
let dir = tempdir().unwrap();
@@ -1782,8 +1817,13 @@ mod lsm_tests {
assert_eq!(rows, 1, "LSM FTS must still see base rows");
}
#[tokio::test]
async fn lsm_read_vector_search() {
/// A table whose vector index the LSM maintains: base rows filled with
/// their id, and one memtable row filled with 1000, nearest to `[1000; 8]`.
/// `registry` replaces the table's memtable kinds.
async fn lsm_vector_table(
dir: &std::path::Path,
registry: Option<lance::dataset::mem_wal::MemIndexRegistry>,
) -> Table {
use crate::index::Index;
use crate::index::vector::IvfPqIndexBuilder;
use arrow::array::{FixedSizeListBuilder, Float32Builder};
@@ -1815,12 +1855,7 @@ mod lsm_tests {
.unwrap()
};
let dir = tempdir().unwrap();
let conn = connect(dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
// Base rows fill each vector with its own id (0..256); all far from 1000.
let conn = connect(dir.to_str().unwrap()).execute().await.unwrap();
let base = make_batch((0..N).map(|i| (i, i as f32)).collect());
let base_reader: Box<dyn RecordBatchReader + Send> =
Box::new(RecordBatchIterator::new(vec![Ok(base)], schema.clone()));
@@ -1838,13 +1873,15 @@ mod lsm_tests {
.execute()
.await
.unwrap();
if let Some(registry) = registry {
table.as_native().unwrap().set_mem_index_registry(registry);
}
let vec_index = table.list_indices().await.unwrap()[0].name.clone();
table
.set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![vec_index]))
.await
.unwrap();
// Insert a vector (filled with 1000) that is nearest to the query.
let mut builder = table.merge_insert(&[]);
builder
.when_matched_update_all(None)
@@ -1854,20 +1891,21 @@ mod lsm_tests {
schema.clone(),
));
builder.execute(insert_reader).await.unwrap();
table
}
// KNN near [1000; DIM]: the default (auto-routed) read surfaces the
// memtable row.
let stream = table
.query()
.nearest_to(&[1000.0_f32; 8])
.unwrap()
.limit(1)
.execute()
.await
.unwrap();
let ids = collect_ids(stream).await;
/// The ids of the nearest row to `[1000; 8]` through the default read.
async fn nearest_ids(table: &Table) -> Vec<i64> {
let query = table.query().nearest_to(&[1000.0_f32; 8]).unwrap().limit(1);
collect_ids(query.execute().await.unwrap()).await
}
#[tokio::test]
async fn lsm_read_vector_search() {
let dir = tempdir().unwrap();
let table = lsm_vector_table(dir.path(), None).await;
assert_eq!(
ids,
nearest_ids(&table).await,
vec![9999],
"LSM vector search must rank the memtable row first"
);
@@ -1893,4 +1931,417 @@ mod lsm_tests {
assert_eq!(ids, vec![255]);
}
}
/// The built-in vector plugin, except that its index answers no search.
#[derive(Debug)]
struct AnswersNothing;
#[derive(Debug)]
struct AnswersNothingIndex(Arc<dyn lance::dataset::mem_wal::index::MemIndex>);
fn vector_plugin() -> Arc<dyn lance::dataset::mem_wal::index::MemIndexPlugin> {
Arc::new(lance::dataset::mem_wal::index::HnswMemIndexPlugin)
}
#[async_trait::async_trait]
impl lance::dataset::mem_wal::index::MemIndexPlugin for AnswersNothing {
fn name(&self) -> &str {
"AnswersNothing"
}
fn details_message(&self) -> &str {
"VectorIndexDetails"
}
fn flush_index_type(&self) -> lance_index::IndexType {
vector_plugin().flush_index_type()
}
fn training_criteria(&self) -> lance_index::scalar::registry::TrainingCriteria {
vector_plugin().training_criteria()
}
async fn resolve(
&self,
ctx: &lance::dataset::mem_wal::index::ResolveContext<'_>,
) -> lance_core::Result<lance::dataset::mem_wal::index::ResolvedIndex> {
vector_plugin().resolve(ctx).await
}
fn validate(
&self,
ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>,
) -> lance_core::Result<()> {
vector_plugin().validate(ctx)
}
fn create(
&self,
ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>,
) -> lance_core::Result<Arc<dyn lance::dataset::mem_wal::index::MemIndex>> {
Ok(Arc::new(AnswersNothingIndex(vector_plugin().create(ctx)?)))
}
}
#[async_trait::async_trait]
impl lance::dataset::mem_wal::index::MemIndex for AnswersNothingIndex {
fn columns(&self) -> &[String] {
self.0.columns()
}
fn can_answer(&self, _query: &dyn lance::dataset::mem_wal::index::MemQuery) -> bool {
false
}
fn insert(&self, batch: &RecordBatch, row_offset: u64) -> lance_core::Result<()> {
self.0.insert(batch, row_offset)
}
fn insert_batches(
&self,
batches: &[lance::dataset::mem_wal::write::StoredBatch],
) -> lance_core::Result<()> {
self.0.insert_batches(batches)
}
fn resident_bytes(&self) -> usize {
self.0.resident_bytes()
}
fn search(
&self,
_query: &dyn lance::dataset::mem_wal::index::MemQuery,
_ctx: &lance::dataset::mem_wal::index::SearchContext,
) -> lance_core::Result<Option<lance::dataset::mem_wal::index::MemMatches>> {
Ok(None)
}
async fn flush(
&self,
ctx: &lance::dataset::mem_wal::index::FlushContext<'_>,
) -> lance_core::Result<lance::dataset::mem_wal::index::FlushOutcome> {
self.0.flush(ctx).await
}
}
/// A resident memtable whose vector index answers nothing still serves an
/// LSM vector search: Lance compares every row instead.
#[tokio::test]
async fn lsm_vector_search_compares_every_row_when_no_resident_index_answers() {
let dir = tempdir().unwrap();
let mut registry = lance::dataset::mem_wal::MemIndexRegistry::default();
registry.replace_plugin(Arc::new(AnswersNothing)).unwrap();
let table = lsm_vector_table(dir.path(), Some(registry)).await;
assert_eq!(nearest_ids(&table).await, vec![9999]);
// The second page starts after the memtable row.
let page = table
.query()
.nearest_to(&[1000.0_f32; 8])
.unwrap()
.limit(2)
.offset(1);
assert_eq!(
collect_ids(page.execute().await.unwrap()).await,
vec![255, 254]
);
}
/// Claims the base table's bitmap index and keeps it as a B-tree, whose
/// flush rows are in the shape the bitmap trainer takes.
#[derive(Debug)]
struct BitmapAsBTree;
fn btree_plugin() -> Arc<dyn lance::dataset::mem_wal::index::MemIndexPlugin> {
lance::dataset::mem_wal::MemIndexRegistry::default()
.plugin_for_details_url("/lance.table.BTreeIndexDetails")
.unwrap()
.clone()
}
#[async_trait::async_trait]
impl lance::dataset::mem_wal::index::MemIndexPlugin for BitmapAsBTree {
fn name(&self) -> &str {
"BitmapAsBTree"
}
fn details_message(&self) -> &str {
"BitmapIndexDetails"
}
fn flush_index_type(&self) -> lance_index::IndexType {
lance_index::IndexType::Bitmap
}
fn training_criteria(&self) -> lance_index::scalar::registry::TrainingCriteria {
btree_plugin().training_criteria()
}
fn validate(
&self,
ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>,
) -> lance_core::Result<()> {
btree_plugin().validate(ctx)
}
fn create(
&self,
ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>,
) -> lance_core::Result<Arc<dyn lance::dataset::mem_wal::index::MemIndex>> {
btree_plugin().create(ctx)
}
}
/// A kind Lance has no memtable index for is refused when named until the
/// table's registry maintains it; then it is maintained, flushed with each
/// generation, and answers searches on the flushed generations.
#[tokio::test]
async fn lsm_maintains_an_index_kind_from_the_tables_registry() {
let dir = tempdir().unwrap();
let conn = connect(dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
let table = conn
.create_table("t", id_region_reader(vec![(1, "us"), (2, "eu"), (3, "us")]))
.execute()
.await
.unwrap();
table.set_unenforced_primary_key(["id"]).await.unwrap();
table
.create_index(&["region"], crate::index::Index::Bitmap(Default::default()))
.name("region_bitmap".to_string())
.execute()
.await
.unwrap();
let spec =
|| LsmWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]);
let refused = table.set_lsm_write_spec(spec()).await.unwrap_err();
assert!(
refused.to_string().contains("region_bitmap")
&& refused
.to_string()
.contains("no registered plugin maintains"),
"{refused:?}"
);
let registry = lance::dataset::mem_wal::MemIndexRegistry::default()
.with_plugin(Arc::new(BitmapAsBTree))
.unwrap();
table.as_native().unwrap().set_mem_index_registry(registry);
table.set_lsm_write_spec(spec()).await.unwrap();
// The registry is not stored with the table.
let other = conn.open_table("t").execute().await.unwrap();
let mut builder = other.merge_insert(&["id"]);
builder
.when_matched_update_all(None)
.when_not_matched_insert_all();
let refused = builder
.execute(id_region_reader(vec![(20, "us")]))
.await
.unwrap_err();
assert!(
refused
.to_string()
.contains("no registered plugin maintains"),
"{refused:?}"
);
let mut builder = table.merge_insert(&["id"]);
builder
.when_matched_update_all(None)
.when_not_matched_insert_all();
builder
.execute(id_region_reader(vec![
(10, "fresh"),
(11, "fresh"),
(12, "eu"),
]))
.await
.unwrap();
let fresh = || async {
let query = table.query().only_if("region = 'fresh'");
let mut ids = collect_ids(query.execute().await.unwrap()).await;
ids.sort_unstable();
ids
};
assert_eq!(fresh().await, vec![10, 11]);
let (_, _, memtables) = table
.as_native()
.unwrap()
.dataset
.shard_writer()
.read_snapshot()
.await
.unwrap()
.expect("the merge opened a writer");
let active = memtables.expect("the writer keeps memtables").active;
assert!(
active
.index_store
.index_names()
.contains(&"region_bitmap".to_string()),
"the memtable maintains the plugin's index: {:?}",
active.index_store.index_names()
);
table.close_lsm_writers().await.unwrap();
assert_eq!(fresh().await, vec![10, 11]);
assert_generations_answer_through_region_bitmap(&table, vec![10, 11]).await;
}
/// The generations each shard's manifest publishes.
async fn published_generations(table: &Table) -> Vec<lance::Dataset> {
use lance::dataset::mem_wal::ShardManifestStore;
let uri = table.uri().await.unwrap();
let (store, base) = lance_io::object_store::ObjectStore::from_uri(&uri)
.await
.unwrap();
let mem_wal = std::path::Path::new(&uri).join("_mem_wal");
let mut generations = Vec::new();
for shard in std::fs::read_dir(&mem_wal).unwrap() {
let Ok(shard_id) = shard.unwrap().file_name().to_string_lossy().parse() else {
continue;
};
let manifest = ShardManifestStore::new(store.clone(), &base, shard_id, 2)
.latest()
.await
.unwrap()
.expect("a shard directory has a manifest");
for sstable in manifest.sstables {
let path = mem_wal.join(shard_id.to_string()).join(&sstable.path);
generations.push(lance::Dataset::open(path.to_str().unwrap()).await.unwrap());
}
}
generations
}
/// Every published generation carries `region_bitmap` as a bitmap index on
/// `region`, and a search for `region = 'fresh'` on it uses that index.
async fn assert_generations_answer_through_region_bitmap(table: &Table, fresh: Vec<i64>) {
use lance::index::DatasetIndexExt;
let generations = published_generations(table).await;
assert!(
!generations.is_empty(),
"closing the writer flushed a generation"
);
let mut found = Vec::new();
for generation in generations {
let region = generation.schema().field("region").unwrap().id;
let indices = generation.load_indices().await.unwrap();
let index = indices
.iter()
.find(|index| index.name == "region_bitmap")
.expect("the generation carries the index the plugin trained");
assert_eq!(index.fields, vec![region]);
assert!(
index
.index_details
.as_ref()
.is_some_and(|details| details.type_url.ends_with("BitmapIndexDetails")),
"{:?}",
index.index_details
);
let mut scan = generation.scan();
scan.filter("region = 'fresh'").unwrap();
let plan = scan.explain_plan(false).await.unwrap();
assert!(plan.contains("ScalarIndexQuery"), "{plan}");
let batch = scan.try_into_batch().await.unwrap();
found.extend(
batch["id"]
.as_primitive::<Int64Type>()
.values()
.iter()
.copied(),
);
}
found.sort_unstable();
assert_eq!(found, fresh);
}
/// Under the default set, a kind only the registry maintains is picked up
/// when its index is created after the writer opened, and the table's
/// clones share the registry.
#[tokio::test]
async fn lsm_default_set_maintains_a_registry_kind_created_while_writing() {
let dir = tempdir().unwrap();
let conn = connect(dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
let table = conn
.create_table("t", id_region_reader(vec![(1, "us"), (2, "eu")]))
.execute()
.await
.unwrap();
table.set_unenforced_primary_key(["id"]).await.unwrap();
let registry = lance::dataset::mem_wal::MemIndexRegistry::default()
.with_plugin(Arc::new(BitmapAsBTree))
.unwrap();
table.as_native().unwrap().set_mem_index_registry(registry);
let clone = table.clone();
assert!(
clone
.as_native()
.unwrap()
.mem_index_registry()
.plugin_for_details_url("/lance.table.BitmapIndexDetails")
.is_some(),
"a clone shares the handle's registry"
);
table
.set_lsm_write_spec(LsmWriteSpec::unsharded())
.await
.unwrap();
let upsert = |rows: Vec<(i64, &'static str)>| {
let table = table.clone();
async move {
let mut builder = table.merge_insert(&["id"]);
builder
.when_matched_update_all(None)
.when_not_matched_insert_all();
builder.execute(id_region_reader(rows)).await.unwrap();
}
};
// Opens the writer before the table has a bitmap index.
upsert(vec![(3, "us")]).await;
table
.create_index(&["region"], crate::index::Index::Bitmap(Default::default()))
.name("region_bitmap".to_string())
.execute()
.await
.unwrap();
upsert(vec![(10, "fresh"), (11, "fresh")]).await;
let (_, _, memtables) = table
.as_native()
.unwrap()
.dataset
.shard_writer()
.read_snapshot()
.await
.unwrap()
.expect("the merge opened a writer");
let active = memtables.expect("the writer keeps memtables").active;
assert!(
active
.index_store
.index_names()
.contains(&"region_bitmap".to_string()),
"the open writer picked up the new index: {:?}",
active.index_store.index_names()
);
table.close_lsm_writers().await.unwrap();
// Only the generation written after the index exists carries it.
let generations = published_generations(&table).await;
let mut carrying = 0;
for generation in &generations {
use lance::index::DatasetIndexExt;
if generation
.load_indices()
.await
.unwrap()
.iter()
.any(|index| index.name == "region_bitmap")
{
carrying += 1;
}
}
assert!(
carrying >= 1,
"{} generations, none carry the index",
generations.len()
);
}
}
rust/lancedb/src/table/merge/lsm.rs
+8 -4
View File
@@ -29,7 +29,7 @@ use arrow_schema::{DataType, Schema as ArrowSchema, SchemaRef};
use lance::Dataset;
use lance::dataset::mem_wal::{
DatasetMemWalExt, ShardWriter, ShardWriterConfig, evaluate_sharding_spec,
validate_maintained_indexes,
validate_maintained_indexes_with,
};
use lance::index::DatasetIndexExt;
use lance_core::datatypes::Schema as LanceSchema;
@@ -103,7 +103,8 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
}
}
let dataset = table.dataset.get().await?;
validate_maintained_indexes(&dataset, requested).await?;
validate_maintained_indexes_with(&dataset, requested, &table.mem_index_registry())
.await?;
Some(requested.to_vec())
}
None => None,
@@ -126,7 +127,9 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
.into(),
});
}
let mut builder = dataset.initialize_mem_wal();
let mut builder = dataset
.initialize_mem_wal()
.mem_index_registry(table.mem_index_registry());
let writer_config_defaults = match spec {
LsmWriteSpec::Bucket {
column,
@@ -725,7 +728,8 @@ pub(crate) async fn execute_lsm_merge_insert(
return Ok(lsm_merge_result(0));
};
let config = shard_writer_config_from_defaults(&plan.writer_config_defaults);
let mut config = shard_writer_config_from_defaults(&plan.writer_config_defaults);
config.mem_index_registry = table.mem_index_registry();
let writer = table
.dataset
.shard_writer()
rust/lancedb/src/table/query.rs
+27 -3
View File
@@ -106,7 +106,7 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R
}
fn requires_local_namespace_execution(query: &AnyQuery) -> bool {
// The namespace QueryTable request cannot represent approx_mode or use_lsm.
// The namespace QueryTable request cannot represent approx_mode, search_effort or use_lsm.
// For use_lsm that is worse than a tuning miss: MemWAL read routing lives only
// in `create_plan`, so a pushed-down query would return stale base-only data.
if query.base().use_lsm.is_some() || query.base().take_offsets.is_some() {
@@ -115,7 +115,7 @@ fn requires_local_namespace_execution(query: &AnyQuery) -> bool {
matches!(
query,
AnyQuery::VectorQuery(vector_query)
if vector_query.approx_mode.is_some()
if vector_query.approx_mode.is_some() || vector_query.search_effort.is_some()
)
}
@@ -163,6 +163,7 @@ pub async fn create_plan(
AnyQuery::Query(query) => VectorQueryRequest::from_plain_query(query),
};
query.base.check_filter()?;
query.validate_search_effort()?;
let ds_ref = table.dataset.get().await?;
@@ -293,7 +294,9 @@ pub async fn create_plan(
scanner.approx_mode(approx_mode.into());
}
if let Some(nprobes) = query.nprobes {
if query.search_effort.is_none()
&& let Some(nprobes) = query.nprobes
{
if query.maximum_nprobes == Some(0) {
scanner.minimum_nprobes(nprobes);
} else {
@@ -306,6 +309,9 @@ pub async fn create_plan(
if let Some(maximum_nprobes) = query.maximum_nprobes.filter(|nprobes| *nprobes != 0) {
scanner.maximum_nprobes(maximum_nprobes);
}
if let Some(search_effort) = query.search_effort {
scanner.search_effort(search_effort)?;
}
}
// For a batch query, `nearest` already applies k to each query vector.
@@ -604,6 +610,11 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result<NsQueryTableRequest> {
};
// Check for unsupported features
if vq.search_effort.is_some() {
return Err(Error::NotSupported {
message: "namespace QueryTable does not support search_effort; execute the query locally".to_string(),
});
}
if vq.base.reranker.is_some() {
return Err(Error::NotSupported {
message: "Reranker is not supported for server-side queries".to_string(),
@@ -962,6 +973,19 @@ mod tests {
let default_query = AnyQuery::VectorQuery(VectorQueryRequest::default());
assert!(!requires_local_namespace_execution(&default_query));
for effort in [0.0, 0.5, 1.0] {
let effort_query = AnyQuery::VectorQuery(VectorQueryRequest {
query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))],
search_effort: Some(effort),
..Default::default()
});
assert!(requires_local_namespace_execution(&effort_query));
assert!(matches!(
convert_to_namespace_query(&effort_query),
Err(Error::NotSupported { message }) if message.contains("search_effort")
));
}
let nprobes_query = AnyQuery::VectorQuery(VectorQueryRequest {
query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))],
nprobes: Some(20),
rust/lancedb/src/table/query/lsm.rs
+23 -32
View File
@@ -33,7 +33,6 @@ use datafusion_physical_plan::expressions::Column;
use datafusion_physical_plan::projection::ProjectionExec;
use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr};
use lance::Dataset;
use lance::dataset::mem_wal::index::{IndexStore, VectorMemQuery};
use lance::dataset::mem_wal::scanner::{InMemoryMemTableRef, InMemoryMemTables};
use lance::dataset::mem_wal::{
DatasetMemWalExt, LsmScanner, ShardManifestStore, ShardSnapshot, ShardWriterConfig,
@@ -174,6 +173,9 @@ fn reject_unsupported(query: &VectorQueryRequest) -> Result<()> {
// brute-force search the index-only base arm can't do. (ef / approx_mode are
// recall/speed knobs and are left to no-op.)
if !query.query_vector.is_empty() {
if query.search_effort.is_some() {
return unsupported("search_effort on vector search");
}
if query.lower_bound.is_some() || query.upper_bound.is_some() {
return unsupported("distance_range on vector search");
}
@@ -708,17 +710,14 @@ async fn vector_plan(
// The base arm relies on the column's vector index (`fast_search`). Unmaintained,
// its catch-up is untracked and exclusion falls back to the compaction watermark,
// dropping compacted SSTables the lagging base index has not re-indexed; and a
// resident MemTable built before the index joined the set carries none either.
// dropping compacted SSTables the lagging base index has not re-indexed.
// Reject rather than silently omit rows, mirroring the FTS arm.
if !index_maintained(dataset, &column, details, "VectorIndexDetails").await?
|| !resident_memtables_carry(&in_memory, |memtable| {
carries_vector_index(&memtable.index_store, &column)
})
{
// A resident MemTable needs no index of its own: one that cannot answer
// compares every row.
if !index_maintained(dataset, &column, details, "VectorIndexDetails").await? {
return Err(Error::NotSupported {
message: format!(
"the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only"
"the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only"
),
});
}
@@ -761,22 +760,6 @@ async fn vector_plan(
Ok(scanner.create_plan().await?)
}
/// Probe for a resident vector index without imposing a distance metric, matching
/// the presence check used before Lance introduced query-based index lookup.
fn carries_vector_index(store: &IndexStore, column: &str) -> bool {
let probe = VectorMemQuery {
vector: arrow_array::FixedSizeListArray::new_null(
Arc::new(arrow_schema::Field::new("item", DataType::Float32, true)),
1,
1,
),
k: 1,
ef: None,
distance_type: None,
};
store.index_answering(column, &probe).is_some()
}
/// Whether `column` stores binary (uint8) vectors, which the LSM vector planner
/// does not support.
fn is_binary_vector_column(schema: &ArrowSchema, column: &str) -> bool {
@@ -822,9 +805,22 @@ mod tests {
use lance_index::mem_wal::{CompactedSsTable, IndexCatchupProgress};
#[test]
fn resident_index_checks_distinguish_columns_and_query_types() {
fn search_effort_is_not_silently_ignored() {
let query = VectorQueryRequest {
query_vector: vec![Arc::new(arrow_array::Float32Array::from(vec![1.0, 0.0]))],
search_effort: Some(0.5),
..Default::default()
};
assert!(
matches!(reject_unsupported(&query), Err(Error::NotSupported { message })
if message.contains("search_effort") && message.contains("un-compacted MemWAL"))
);
}
#[test]
fn resident_fts_check_distinguishes_columns() {
use arrow_schema::Field;
use lance::dataset::mem_wal::index::MemIndexSpec;
use lance::dataset::mem_wal::index::{IndexStore, MemIndexSpec};
let schema = ArrowSchema::new(vec![
Field::new("id", DataType::Int32, false),
@@ -847,16 +843,11 @@ mod tests {
),
];
let store = IndexStore::from_specs(&specs, &schema, 1_000, 16).unwrap();
assert!(carries_vector_index(&store, "vector"));
for column in ["id", "text", "missing"] {
assert!(!carries_vector_index(&store, column));
}
assert!(!store.fts_granularities_on("text").is_empty());
for column in ["id", "vector", "missing"] {
assert!(store.fts_granularities_on(column).is_empty());
}
let empty = IndexStore::from_specs(&[], &schema, 1_000, 16).unwrap();
assert!(!carries_vector_index(&empty, "vector"));
assert!(empty.fts_granularities_on("text").is_empty());
}
rust/lancedb/src/table/refresh.rs
+2 -2
View File
@@ -1847,8 +1847,8 @@ mod tests {
assert_eq!(
layouts,
vec![
(1, Some(BlobKind::Managed)),
(2, Some(BlobKind::Managed)),
(1, Some(BlobKind::Dedicated)),
(2, Some(BlobKind::Packed)),
(3, Some(BlobKind::Inline)),
(4, None),
]