diff --git a/Cargo.lock b/Cargo.lock index eecf3d0d1..18cb1a98e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3588,8 +3588,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5095,26 +5095,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "jsonb" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb98fb29636087c40ad0d1274d9a30c0c1e83e03ae93f6e7e89247b37fcc6953" -dependencies = [ - "byteorder", - "ethnum", - "fast-float2", - "itoa", - "jiff", - "nom 8.0.0", - "num-traits", - "ordered-float 5.3.0", - "rand 0.9.5", - "serde", - "serde_json", - "zmij", -] - [[package]] name = "kanaria" version = "0.2.0" @@ -5143,8 +5123,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arc-swap", "arrow", @@ -5217,8 +5197,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-buffer", @@ -5232,7 +5212,7 @@ dependencies = [ "futures", "getrandom 0.2.17", "half", - "jsonb", + "lance-jsonb", "num-traits", "rand 0.9.5", ] @@ -5240,7 +5220,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-buffer", @@ -5254,7 +5234,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-schema", @@ -5263,8 +5243,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrayref", "crunchy", @@ -5274,8 +5254,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-buffer", @@ -5312,8 +5292,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow", "arrow-array", @@ -5330,10 +5310,10 @@ dependencies = [ "datafusion-physical-expr", "futures", "half", - "jsonb", "lance-arrow", "lance-core", "lance-geo", + "lance-jsonb", "log", "pin-project", "prost", @@ -5344,8 +5324,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow", "arrow-array", @@ -5362,8 +5342,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "proc-macro2", "quote", @@ -5372,8 +5352,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-arith", "arrow-array", @@ -5406,8 +5386,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-arith", "arrow-array", @@ -5439,8 +5419,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "datafusion", "geo-traits", @@ -5454,8 +5434,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arc-swap", "arrow", @@ -5486,7 +5466,6 @@ dependencies = [ "hex", "itertools 0.14.0", "jieba-rs", - "jsonb", "lance-arrow", "lance-arrow-stats", "lance-bitpacking", @@ -5497,6 +5476,7 @@ dependencies = [ "lance-geo", "lance-index-core", "lance-io", + "lance-jsonb", "lance-linalg", "lance-select", "lance-table", @@ -5526,8 +5506,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-schema", @@ -5550,8 +5530,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow", "arrow-array", @@ -5591,10 +5571,28 @@ dependencies = [ "uuid", ] +[[package]] +name = "lance-jsonb" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +dependencies = [ + "byteorder", + "ethnum", + "fast-float2", + "itoa", + "jiff", + "nom 8.0.0", + "num-traits", + "ordered-float 5.3.0", + "serde", + "serde_json", + "zmij", +] + [[package]] name = "lance-linalg" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-schema", @@ -5608,8 +5606,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow", "async-trait", @@ -5623,8 +5621,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arc-swap", "arrow", @@ -5678,8 +5676,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-buffer", @@ -5693,8 +5691,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow", "arrow-array", @@ -5734,8 +5732,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-schema", @@ -5748,8 +5746,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index ad7c00477..6c05f297a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.94.0" [workspace.dependencies] -lance = { "version" = "=14.0.0-beta.7", default-features = false, "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=14.0.0-beta.7", default-features = false, "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=14.0.0-beta.7", default-features = false, "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/docs/src/js/classes/VectorQuery.md b/docs/src/js/classes/VectorQuery.md index a0a77fd7c..3f56bb600 100644 --- a/docs/src/js/classes/VectorQuery.md +++ b/docs/src/js/classes/VectorQuery.md @@ -460,17 +460,16 @@ exhaustiely searched to find matches. This parameter controls how many partitions should be searched. Increasing this value will increase the recall of your query but will -also increase the latency of your query. The default value is 20. This -default is good for many cases but the best value to use will depend on -your data and the recall that you need to achieve. +also increase the latency of your query. If this method is not called, +Lance's adaptive probe defaults are used. For best results we recommend tuning this parameter with a benchmark against your actual data to find the smallest possible value that will still give you the desired recall. -For more fine grained control over behavior when you have a very narrow filter -you can use `minimumNprobes` and `maximumNprobes`. This method sets both -the minimum and maximum to the same value. +The value is retained as `nprobes` through client and server request +construction. Lance sets both probe bounds to this value. Explicit minimum +or maximum settings can override their respective bounds. #### Parameters diff --git a/docs/web/indexing/vector-index.mdx b/docs/web/indexing/vector-index.mdx index 3cd3a970a..b10da3f6f 100644 --- a/docs/web/indexing/vector-index.mdx +++ b/docs/web/indexing/vector-index.mdx @@ -139,9 +139,9 @@ Learn how to configure, build, and search LanceDB vector indexes, including buil | Parameter | Description | | :--- | :--- | | `limit` | Number of results to return (the `k` in `k-ANN`). | -| `nprobes` | Shorthand that sets both `minimum_nprobes` and `maximum_nprobes` to the same value. LanceDB defaults this to 20 which is a good default for many cases. At very high scales you will likely need to experiment to find your preferred recall / latency balance. | -| `minimum_nprobes` | Minimum number of partitions scanned. | -| `maximum_nprobes` | Maximum number of partitions scanned. Only scans more than `minimum_nprobes` if an initial pass does not return enough results — useful for narrow filters. Set to `0` to remove the cap. | +| `nprobes` | Sets both `minimum_nprobes` and `maximum_nprobes` to the same value in Lance. Leave unset to use Lance's adaptive defaults. At very high scales you will likely need to experiment to find your preferred recall / latency balance. | +| `minimum_nprobes` | Minimum number of partitions scanned. Leave unset to use Lance's default. | +| `maximum_nprobes` | Maximum number of partitions scanned. Only scans more than `minimum_nprobes` if an initial pass does not return enough results — useful for narrow filters. Leave unset, or set to `0`, to remove the cap. | | `ef` | HNSW search-time exploration factor. Start around `1.5 * k` and increase up to `10 * k` for higher recall. | | `refine_factor` | Reads and reranks additional candidates in memory to recover recall lost to quantization. Start between `5` and `50` (a factor of `25` reads and reranks the top `25 * k` candidates) and measure the latency cost, typically a few milliseconds, against the recall gained. | | `distance_range(lower_bound, upper_bound)` | Return only rows whose distance falls within `[lower_bound, upper_bound)`. Either bound is optional. Useful for near-duplicate detection or "close-enough" matching. | @@ -165,8 +165,8 @@ older client. | Index type | Guidance | | :--- | :--- | -| `IVF_HNSW_FLAT`, `IVF_HNSW_SQ` | Keep the default `nprobes`, then tune `ef` first. Expect higher latency variance under filtered search. | -| `IVF_RQ`, `IVF_PQ` | Keep the default `nprobes` and raise when recall is insufficient. If using a selective filter you may need to increase `maximum_nprobes` if you do not received enough results. | +| `IVF_HNSW_FLAT`, `IVF_HNSW_SQ` | Leave the probe settings unset, then tune `ef` first. Expect higher latency variance under filtered search. | +| `IVF_RQ`, `IVF_PQ` | Leave the probe settings unset and configure them only when recall is insufficient. If using a selective filter you may need to increase `maximum_nprobes` if you do not receive enough results. | diff --git a/java/pom.xml b/java/pom.xml index 59efeb627..964dfed0c 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 14.0.0-beta.7 + 14.0.0-beta.10 false 2.30.0 1.7 diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index eee40fb9c..d2244b48c 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -1231,7 +1231,9 @@ describe("When creating an index", () => { expect(() => tbl.query().nearestTo(queryVec).minimumNprobes(0)).toThrow( "Invalid input, minimum_nprobes must be greater than 0", ); - expect(() => tbl.query().nearestTo(queryVec).maximumNprobes(5)).toThrow( + expect(() => + tbl.query().nearestTo(queryVec).minimumNprobes(10).maximumNprobes(5), + ).toThrow( "Invalid input, maximum_nprobes must be greater than or equal to minimum_nprobes", ); diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts index f370e1b31..0f0eaf610 100644 --- a/nodejs/lancedb/query.ts +++ b/nodejs/lancedb/query.ts @@ -532,17 +532,16 @@ export class VectorQuery extends StandardQueryBase { * partitions should be searched. * * Increasing this value will increase the recall of your query but will - * also increase the latency of your query. The default value is 20. This - * default is good for many cases but the best value to use will depend on - * your data and the recall that you need to achieve. + * also increase the latency of your query. If this method is not called, + * Lance's adaptive probe defaults are used. * * For best results we recommend tuning this parameter with a benchmark against * your actual data to find the smallest possible value that will still give * you the desired recall. * - * For more fine grained control over behavior when you have a very narrow filter - * you can use `minimumNprobes` and `maximumNprobes`. This method sets both - * the minimum and maximum to the same value. + * The value is retained as `nprobes` through client and server request + * construction. Lance sets both probe bounds to this value. Explicit minimum + * or maximum settings can override their respective bounds. */ nprobes(nprobes: number): VectorQuery { this.doVectorCall((inner) => inner.nprobes(nprobes)); diff --git a/python/pyproject.toml b/python/pyproject.toml index 2d753c139..cde7247f3 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "pyarrow>=16", "pydantic>=2.7.4,<3", "tqdm>=4.27.0", - "lance-namespace>=0.3.2" + "lance-namespace>=0.13.0" ] description = "lancedb" authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }] @@ -63,7 +63,7 @@ tests = [ "polars>=0.19, <=1.32.3", "pyarrow<25", "pyarrow-stubs>=16.0", - "pylance==14.0.0b6", + "pylance==14.0.0b10", "requests>=2.31.0", "datafusion>=54,<55", "opentelemetry-sdk>=1.30.0", diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index a8f02e458..cfaa8c68a 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -778,6 +778,7 @@ class PyQueryRequest: use_lsm: Optional[bool] column: Optional[str] query_vector: Optional[List[pa.Array]] + nprobes: Optional[int] minimum_nprobes: Optional[int] maximum_nprobes: Optional[int] lower_bound: Optional[float] diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py index 195c59e04..ed87e441f 100644 --- a/python/python/lancedb/namespace.py +++ b/python/python/lancedb/namespace.py @@ -190,8 +190,12 @@ def _query_to_namespace_request( kwargs["with_row_id"] = query.with_row_id if query.ef is not None: kwargs["ef"] = query.ef + if query.nprobes is not None: + kwargs["nprobes"] = query.nprobes if query.minimum_nprobes is not None: - kwargs["nprobes"] = query.minimum_nprobes + kwargs["minimum_nprobes"] = query.minimum_nprobes + if query.maximum_nprobes is not None: + kwargs["maximum_nprobes"] = query.maximum_nprobes if query.refine_factor is not None: kwargs["refine_factor"] = query.refine_factor if query.lower_bound is not None: @@ -205,6 +209,13 @@ def _query_to_namespace_request( if query.distance_type is not None: kwargs["distance_type"] = query.distance_type + if kwargs.get("maximum_nprobes") == 0: + # The generated 0.13 model rejects LanceDB's unbounded sentinel; + # model_copy preserves direct wire pass-through to the Lance receiver. + maximum_nprobes = kwargs.pop("maximum_nprobes") + return QueryTableRequest(**kwargs).model_copy( + update={"maximum_nprobes": maximum_nprobes} + ) return QueryTableRequest(**kwargs) diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py index 6f68692a2..c7664ab78 100644 --- a/python/python/lancedb/query.py +++ b/python/python/lancedb/query.py @@ -693,8 +693,9 @@ class Query(pydantic.BaseModel): if True then apply the filter after vector / FTS search. This is ignored for plain SQL filtering. nprobes : Optional[int] - The number of IVF partitions to search. If this is None then a default - number of partitions will be used. + The legacy number of IVF partitions to search. Lance sets both probe + bounds to this value. If this is None then Lance's default probe settings + will be used. - A higher number makes search more accurate but also slower. @@ -773,20 +774,23 @@ class Query(pydantic.BaseModel): # distance type to use for vector search distance_type: Optional[str] = None + # legacy number of IVF partitions to search + # + # Lance sets both probe bounds to this value. Explicit bounds can override it. + nprobes: Optional[int] = None + # which columns to return in the results (dict values may be str or Expr) columns: QueryProjection = None # minimum number of IVF partitions to search # - # If None then a default value (20) will be used. + # If None then Lance's default will be used. minimum_nprobes: Optional[int] = None # maximum number of IVF partitions to search # - # If None then a default value (20) will be used. - # - # If 0 then no limit will be applied and all partitions could be searched - # if needed to satisfy the limit. + # If None then Lance's default will be used. If 0 then no limit will be applied + # and all partitions could be searched if needed to satisfy the limit. maximum_nprobes: Optional[int] = None # lower bound for distance search @@ -847,6 +851,7 @@ class Query(pydantic.BaseModel): query.vector_column = req.column query.vector = req.query_vector query.distance_type = req.distance_type + query.nprobes = req.nprobes query.minimum_nprobes = req.minimum_nprobes query.maximum_nprobes = req.maximum_nprobes query.lower_bound = req.lower_bound @@ -1627,6 +1632,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): super().__init__(table) self._query = query self._distance_type = None + self._nprobes = None self._minimum_nprobes = None self._maximum_nprobes = None self._lower_bound = None @@ -1691,9 +1697,9 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): See discussion in [Querying an ANN Index](https://lancedb.com/docs/indexing/) for tuning advice. - This method sets both the minimum and maximum number of probes to the same - value. See `minimum_nprobes` and `maximum_nprobes` for more fine-grained - control. + The value is retained as `nprobes` through client and server request + construction. Lance sets both probe bounds to this value. Explicit minimum + or maximum settings can override their respective bounds. Parameters ---------- @@ -1707,8 +1713,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): """ if nprobes <= 0: raise ValueError("Invalid input, nprobes must be greater than 0") - self._minimum_nprobes = nprobes - self._maximum_nprobes = nprobes + self._nprobes = nprobes return self def minimum_nprobes(self, minimum_nprobes: int) -> LanceVectorQueryBuilder: @@ -1850,6 +1855,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): limit=self._limit, distance_type=self._distance_type, columns=self._columns, + nprobes=self._nprobes, minimum_nprobes=self._minimum_nprobes, maximum_nprobes=self._maximum_nprobes, lower_bound=self._lower_bound, @@ -2208,6 +2214,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): self._fts_columns = fts_columns self._norm = None self._reranker = None + self._nprobes = None self._minimum_nprobes = None self._maximum_nprobes = None self._refine_factor = None @@ -2499,6 +2506,9 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): Higher values will yield better recall (more likely to find vectors if they exist) at the expense of latency. + The value is retained as `nprobes` until Lance sets both probe bounds to it. + Explicit minimum or maximum settings can override their respective bounds. + Parameters ---------- nprobes: int @@ -2511,8 +2521,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): """ if nprobes <= 0: raise ValueError("Invalid input, nprobes must be greater than 0") - self._minimum_nprobes = nprobes - self._maximum_nprobes = nprobes + self._nprobes = nprobes return self def minimum_nprobes(self, minimum_nprobes: int) -> LanceHybridQueryBuilder: @@ -2791,6 +2800,8 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): self._fts_query.phrase_query(True) if self._distance_type: self._vector_query.metric(self._distance_type) + if self._nprobes is not None: + self._vector_query.nprobes(self._nprobes) if self._minimum_nprobes is not None: self._vector_query.minimum_nprobes(self._minimum_nprobes) if self._maximum_nprobes is not None: @@ -3690,7 +3701,7 @@ class AsyncVectorQueryBase: def nprobes(self, nprobes: int) -> Self: """ - Set the number of partitions to search (probe) + Set the legacy IVF probe parameter The number of probes must be greater than 0. @@ -3702,16 +3713,19 @@ class AsyncVectorQueryBase: The partition whose centroids are closest to the query vector will be exhaustiely searched to find matches. This parameter controls how many - partitions should be searched. + partitions will be searched. Increasing this value will increase the recall of your query but will - also increase the latency of your query. The default value is 20. This - default is good for many cases but the best value to use will depend on - your data and the recall that you need to achieve. + also increase the latency of your query. If this method is not called, + Lance's adaptive probe defaults are used. For best results we recommend tuning this parameter with a benchmark against your actual data to find the smallest possible value that will still give you the desired recall. + + LanceDB retains this as `nprobes` through local and remote request + construction. Lance sets both probe bounds to this value. Explicit minimum + or maximum settings can override their respective bounds. """ self._inner.nprobes(nprobes) return self diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 2970cb902..21838ecef 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -6439,9 +6439,10 @@ class AsyncTable: ) if query.distance_type is not None: async_query = async_query.distance_type(query.distance_type) + if query.nprobes is not None: + async_query = async_query.nprobes(query.nprobes) if query.minimum_nprobes is not None and query.maximum_nprobes is not None: - # Set both to the minimum first to avoid min > max error. - async_query = async_query.nprobes( + async_query = async_query.minimum_nprobes( query.minimum_nprobes ).maximum_nprobes(query.maximum_nprobes) elif query.minimum_nprobes is not None: diff --git a/python/python/tests/test_hybrid_query.py b/python/python/tests/test_hybrid_query.py index c5f58308d..8d670a136 100644 --- a/python/python/tests/test_hybrid_query.py +++ b/python/python/tests/test_hybrid_query.py @@ -355,6 +355,20 @@ def test_hybrid_query_minimum_nprobes_zero_raises(sync_table: Table): ) +def test_hybrid_query_nprobes_remains_independent(sync_table: Table): + query = sync_table.search(query_type="hybrid").nprobes(20) + + assert query._nprobes == 20 + assert query._minimum_nprobes is None + assert query._maximum_nprobes is None + + query = query.minimum_nprobes(5).nprobes(20) + + assert query._nprobes == 20 + assert query._minimum_nprobes == 5 + assert query._maximum_nprobes is None + + def test_hybrid_query_distance_range(sync_table: Table): reranker = RRFReranker(return_score="all") result = ( diff --git a/python/python/tests/test_namespace.py b/python/python/tests/test_namespace.py index 2a537bbf9..76aa5fc55 100644 --- a/python/python/tests/test_namespace.py +++ b/python/python/tests/test_namespace.py @@ -15,7 +15,8 @@ import pyarrow as pa import lancedb from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError from lancedb.listing import AsyncListing, Listing -from lancedb.namespace import _MAX_QUERY_K +from lancedb.namespace import _MAX_QUERY_K, _query_to_namespace_request +from lancedb.query import Query from lancedb.table import AsyncTable, LanceTable @@ -1111,6 +1112,29 @@ class TestPushdownOperations: ] assert all(r.k <= 2**31 - 1 for r in namespace_client.requests) + def test_probe_fields_pass_through_independently(self): + request = _query_to_namespace_request( + ["geneva", "hist"], + Query( + vector=[1.0, 2.0], + nprobes=20, + minimum_nprobes=3, + maximum_nprobes=10, + ), + ) + + assert request.nprobes == 20 + assert request.minimum_nprobes == 3 + assert request.maximum_nprobes == 10 + + request = _query_to_namespace_request( + ["geneva", "hist"], + Query(vector=[1.0, 2.0], nprobes=5, maximum_nprobes=0), + ) + assert request.nprobes == 5 + assert request.maximum_nprobes == 0 + assert request.to_dict()["maximum_nprobes"] == 0 + @pytest.mark.asyncio class TestAsyncPushdownOperations: diff --git a/python/python/tests/test_query.py b/python/python/tests/test_query.py index d3fedbcc8..daee0dd07 100644 --- a/python/python/tests/test_query.py +++ b/python/python/tests/test_query.py @@ -995,12 +995,22 @@ def test_invalid_nprobes_sync(table): ValueError, match="maximum_nprobes must be greater than or equal to minimum_nprobes", ): - LanceVectorQueryBuilder(table, [0, 0], "vector").maximum_nprobes(5).to_list() + ( + LanceVectorQueryBuilder(table, [0, 0], "vector") + .minimum_nprobes(10) + .maximum_nprobes(5) + .to_list() + ) with pytest.raises( ValueError, - match="minimum_nprobes must be less than or equal to maximum_nprobes", + match="maximum_nprobes must be greater than or equal to minimum_nprobes", ): - LanceVectorQueryBuilder(table, [0, 0], "vector").minimum_nprobes(100).to_list() + ( + LanceVectorQueryBuilder(table, [0, 0], "vector") + .maximum_nprobes(5) + .minimum_nprobes(100) + .to_list() + ) def test_nprobes_works_sync(table): @@ -1039,12 +1049,22 @@ async def test_invalid_nprobes_async(table_async: AsyncTable): ValueError, match="maximum_nprobes must be greater than or equal to minimum_nprobes", ): - await table_async.vector_search([0, 0]).maximum_nprobes(5).to_list() + await ( + table_async.vector_search([0, 0]) + .minimum_nprobes(10) + .maximum_nprobes(5) + .to_list() + ) with pytest.raises( ValueError, match="minimum_nprobes must be less than or equal to maximum_nprobes", ): - await table_async.vector_search([0, 0]).minimum_nprobes(100).to_list() + await ( + table_async.vector_search([0, 0]) + .maximum_nprobes(5) + .minimum_nprobes(100) + .to_list() + ) def test_query_builder_with_prefilter(table): @@ -1635,8 +1655,9 @@ def test_query_serialization_sync(table: lancedb.table.Table): q, vector_column="vector", vector=[5.0, 6.0], - minimum_nprobes=10, - maximum_nprobes=10, + nprobes=10, + minimum_nprobes=None, + maximum_nprobes=None, refine_factor=5, ) @@ -1654,8 +1675,19 @@ def test_query_serialization_sync(table: lancedb.table.Table): q, vector_column="vector", vector=[5.0, 6.0], - minimum_nprobes=50, - maximum_nprobes=50, + nprobes=50, + minimum_nprobes=None, + maximum_nprobes=None, + ) + + q = table.search([5.0, 6.0]).minimum_nprobes(5).nprobes(50).to_query_object() + check_set_props( + q, + vector_column="vector", + vector=[5.0, 6.0], + nprobes=50, + minimum_nprobes=5, + maximum_nprobes=None, ) q = table.search([5.0, 6.0]).maximum_nprobes(10).to_query_object() @@ -1667,6 +1699,15 @@ def test_query_serialization_sync(table: lancedb.table.Table): minimum_nprobes=None, ) + q = table.search([5.0, 6.0]).nprobes(5).maximum_nprobes(0).to_query_object() + check_set_props( + q, + vector_column="vector", + vector=[5.0, 6.0], + nprobes=5, + maximum_nprobes=0, + ) + q = table.search([5.0, 6.0]).distance_range(0.0, 1.0).to_query_object() check_set_props( q, vector_column="vector", vector=[5.0, 6.0], lower_bound=0.0, upper_bound=1.0 @@ -1716,8 +1757,8 @@ async def test_query_serialization_async(table_async: AsyncTable): limit=10, vector=sample_vector, postfilter=False, - minimum_nprobes=20, - maximum_nprobes=20, + minimum_nprobes=None, + maximum_nprobes=None, with_row_id=False, bypass_vector_index=False, ) @@ -1727,8 +1768,8 @@ async def test_query_serialization_async(table_async: AsyncTable): q, vector=sample_vector, postfilter=False, - minimum_nprobes=20, - maximum_nprobes=20, + minimum_nprobes=None, + maximum_nprobes=None, with_row_id=False, bypass_vector_index=False, limit=10, @@ -1739,8 +1780,9 @@ async def test_query_serialization_async(table_async: AsyncTable): q, vector=sample_vector, postfilter=False, - minimum_nprobes=50, - maximum_nprobes=50, + nprobes=50, + minimum_nprobes=None, + maximum_nprobes=None, with_row_id=False, bypass_vector_index=False, limit=10, @@ -1759,8 +1801,8 @@ async def test_query_serialization_async(table_async: AsyncTable): filter="id = 1", postfilter=True, vector=sample_vector, - minimum_nprobes=20, - maximum_nprobes=20, + minimum_nprobes=None, + maximum_nprobes=None, with_row_id=False, bypass_vector_index=False, ) @@ -1774,8 +1816,9 @@ async def test_query_serialization_async(table_async: AsyncTable): check_set_props( q, vector=sample_vector, - minimum_nprobes=10, - maximum_nprobes=10, + nprobes=10, + minimum_nprobes=None, + maximum_nprobes=None, refine_factor=5, postfilter=False, with_row_id=False, @@ -1788,7 +1831,24 @@ async def test_query_serialization_async(table_async: AsyncTable): q, vector=sample_vector, minimum_nprobes=5, - maximum_nprobes=20, + maximum_nprobes=None, + postfilter=False, + with_row_id=False, + bypass_vector_index=False, + limit=10, + ) + + q = ( + (await table_async.search([5.0, 6.0])) + .nprobes(5) + .maximum_nprobes(0) + .to_query_object() + ) + check_set_props( + q, + vector=sample_vector, + nprobes=5, + maximum_nprobes=0, postfilter=False, with_row_id=False, bypass_vector_index=False, @@ -1806,8 +1866,8 @@ async def test_query_serialization_async(table_async: AsyncTable): lower_bound=0.0, upper_bound=1.0, postfilter=False, - minimum_nprobes=20, - maximum_nprobes=20, + minimum_nprobes=None, + maximum_nprobes=None, with_row_id=False, bypass_vector_index=False, limit=10, @@ -1819,8 +1879,8 @@ async def test_query_serialization_async(table_async: AsyncTable): distance_type="cosine", vector=sample_vector, postfilter=False, - minimum_nprobes=20, - maximum_nprobes=20, + minimum_nprobes=None, + maximum_nprobes=None, with_row_id=False, bypass_vector_index=False, limit=10, @@ -1832,8 +1892,8 @@ async def test_query_serialization_async(table_async: AsyncTable): ef=7, vector=sample_vector, postfilter=False, - minimum_nprobes=20, - maximum_nprobes=20, + minimum_nprobes=None, + maximum_nprobes=None, with_row_id=False, bypass_vector_index=False, limit=10, @@ -1845,8 +1905,8 @@ async def test_query_serialization_async(table_async: AsyncTable): bypass_vector_index=True, vector=sample_vector, postfilter=False, - minimum_nprobes=20, - maximum_nprobes=20, + minimum_nprobes=None, + maximum_nprobes=None, with_row_id=False, limit=10, ) diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 81fb3e10e..3e66ac0e0 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -2164,9 +2164,6 @@ def test_query_sync_minimal(): "ef": None, "vector": [1.0, 2.0, 3.0], "vector_column": "vector", - "nprobes": 20, - "minimum_nprobes": 20, - "maximum_nprobes": 20, "version": None, } @@ -2338,8 +2335,6 @@ def test_query_sync_maximal(): "refine_factor": 10, "vector": [1.0, 2.0, 3.0], "nprobes": 5, - "minimum_nprobes": 5, - "maximum_nprobes": 5, "lower_bound": None, "upper_bound": None, "ef": None, @@ -2435,7 +2430,7 @@ async def test_query_async_nprobes_zero(hybrid): assert requests == ["/v1/table/test/describe/"] -def test_query_sync_nprobes(): +def test_query_sync_probe_bounds(): def handler(body): assert body == { "k": 10, @@ -2447,7 +2442,6 @@ def test_query_sync_nprobes(): "upper_bound": None, "ef": None, "vector": [1.0, 2.0, 3.0], - "nprobes": 5, "minimum_nprobes": 5, "maximum_nprobes": 15, "version": None, @@ -2464,7 +2458,28 @@ def test_query_sync_nprobes(): ) -def test_query_sync_no_max_nprobes(): +def test_query_sync_nprobes_passthrough(): + def handler(body): + assert body["nprobes"] == 20 + assert "minimum_nprobes" not in body + assert "maximum_nprobes" not in body + return pa.table({"id": [1]}) + + with query_test_table(handler) as table: + table.search([1, 2, 3]).nprobes(20).to_list() + + +def test_query_sync_nprobes_with_unbounded_maximum(): + def handler(body): + assert body["nprobes"] == 20 + assert body["maximum_nprobes"] == 0 + return pa.table({"id": [1]}) + + with query_test_table(handler) as table: + table.search([1, 2, 3]).nprobes(20).maximum_nprobes(0).to_list() + + +def test_query_sync_minimum_nprobes_only(): def handler(body): assert body == { "k": 10, @@ -2476,9 +2491,7 @@ def test_query_sync_no_max_nprobes(): "upper_bound": None, "ef": None, "vector": [1.0, 2.0, 3.0], - "nprobes": 5, "minimum_nprobes": 5, - "maximum_nprobes": 0, "version": None, } @@ -2488,7 +2501,6 @@ def test_query_sync_no_max_nprobes(): ( table.search([1, 2, 3], vector_column_name="vector2", fast_search=True) .minimum_nprobes(5) - .maximum_nprobes(0) .to_list() ) @@ -2643,9 +2655,6 @@ def test_query_sync_hybrid(): "refine_factor": None, "vector": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "vector_column": "vector", - "nprobes": 20, - "minimum_nprobes": 20, - "maximum_nprobes": 20, "lower_bound": None, "upper_bound": None, "ef": None, diff --git a/python/src/query.rs b/python/src/query.rs index 1079fb314..88b7243d4 100644 --- a/python/src/query.rs +++ b/python/src/query.rs @@ -336,9 +336,8 @@ pub struct PyQueryRequest { pub use_lsm: Option, pub column: Option, pub query_vector: Option, + pub nprobes: Option, pub minimum_nprobes: Option, - // None means user did not set it and default should be used (currently 20) - // Some(0) means user set it to None and there is no limit pub maximum_nprobes: Option, pub lower_bound: Option, pub upper_bound: Option, @@ -369,6 +368,7 @@ impl From for PyQueryRequest { use_lsm: query_request.use_lsm, column: None, query_vector: None, + nprobes: None, minimum_nprobes: None, maximum_nprobes: None, lower_bound: None, @@ -396,11 +396,9 @@ impl From for PyQueryRequest { use_lsm: vector_query.base.use_lsm, column: vector_query.column, query_vector: Some(PyQueryVectors(vector_query.query_vector)), - minimum_nprobes: Some(vector_query.minimum_nprobes), - maximum_nprobes: match vector_query.maximum_nprobes { - None => Some(0), - Some(value) => Some(value), - }, + nprobes: vector_query.nprobes, + minimum_nprobes: vector_query.minimum_nprobes, + maximum_nprobes: vector_query.maximum_nprobes, lower_bound: vector_query.lower_bound, upper_bound: vector_query.upper_bound, ef: vector_query.ef, diff --git a/python/uv.lock b/python/uv.lock index f80541711..b013e5982 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -1849,19 +1849,19 @@ wheels = [ [[package]] name = "lance-namespace" -version = "0.11.1" +version = "0.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lance-namespace-urllib3-client" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bf/93/da5f7fcac690db9b282a3439ed9e34960c147619a0d6e1f4eb8cd240e7a5/lance_namespace-0.11.1.tar.gz", hash = "sha256:f67cfbbe0647b7cb42f23b673e7edf8a75b7d8a047265a916492f8d247ee1bc2", size = 11631, upload-time = "2026-08-18T17:40:06.294Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/e7/d5d46594678ee479c0eda830c47b2f5c46133bd100e84e9aa6e01306eca7/lance_namespace-0.13.0.tar.gz", hash = "sha256:24554a0997bdb39595c6e4cb3ac6722069f6cd3bd1a74e7acbba7bfaf774a40d", size = 11636, upload-time = "2026-09-14T02:47:03.871523Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/bc/601f2b3cc4cfa0070d858a33223bc823fffdd7981a25c45984a5216ca952/lance_namespace-0.11.1-py3-none-any.whl", hash = "sha256:07643fce9a42ad4d58cc8bf91e3f592bc7f4cbd8d0ad5233223506debf67551c", size = 13507, upload-time = "2026-08-18T17:40:03.561Z" }, + { url = "https://files.pythonhosted.org/packages/e7/47/cfa33cca1ba7c749fd2918cfc5c8ded788f378cc6424d23e4fead5a14125/lance_namespace-0.13.0-py3-none-any.whl", hash = "sha256:438c7b17aef421c21c138196e715f2510d62f07e865372047f87c1e75e618c7a", size = 13507, upload-time = "2026-09-14T02:47:01.765655Z" }, ] [[package]] name = "lance-namespace-urllib3-client" -version = "0.11.1" +version = "0.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, @@ -1869,9 +1869,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e4/c5/2bdd0ff98b469894c8a73be809d26ffdad5402517b0e5f9e758026cba29e/lance_namespace_urllib3_client-0.11.1.tar.gz", hash = "sha256:145a9e9424d7597487249b5b95ee274423bf2910e1a9160b6a07b676b61ea46a", size = 237345, upload-time = "2026-08-18T17:40:07.308Z" } +sdist = { url = "https://files.pythonhosted.org/packages/69/25/9aaa4a5e8999693fb0f227c2c0c4b97bd8f0539066408cdff0b89d41b2d5/lance_namespace_urllib3_client-0.13.0.tar.gz", hash = "sha256:1e8a79c6e4e6277033597fd76aa0e2f33d909ca1436c935936e9e569221f43ef", size = 238348, upload-time = "2026-09-14T02:47:04.521878Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/2a/eaaefd55d1190291207049fedc6b3eb22b506e57d6de91bae46bbaaa9c60/lance_namespace_urllib3_client-0.11.1-py3-none-any.whl", hash = "sha256:36537f529294da6d884ba0fe783704483f0a75463497c7705fd083a4d0257990", size = 406311, upload-time = "2026-08-18T17:40:04.842Z" }, + { url = "https://files.pythonhosted.org/packages/60/4c/7b8f0712a7fe1b8655b711342bc4989696635a79e7027ee56ba3cae23e99/lance_namespace_urllib3_client-0.13.0-py3-none-any.whl", hash = "sha256:fb361eb4f6c7f2d1f9e92809609657b73e38241393e9c4516e12d6b6ee643a0a", size = 406848, upload-time = "2026-09-14T02:47:02.715954Z" }, ] [[package]] @@ -1980,7 +1980,7 @@ requires-dist = [ { name = "huggingface-hub", marker = "extra == 'embeddings'", specifier = ">=0.19.0" }, { name = "ibm-watsonx-ai", marker = "python_full_version >= '3.10' and extra == 'embeddings'", specifier = ">=1.1.2" }, { name = "instructorembedding", marker = "extra == 'embeddings'", specifier = ">=1.0.1" }, - { name = "lance-namespace", specifier = ">=0.3.2" }, + { name = "lance-namespace", specifier = ">=0.13.0" }, { name = "mkdocs", marker = "extra == 'docs'" }, { name = "mkdocs-jupyter", marker = "extra == 'docs'" }, { name = "mkdocs-material", marker = "extra == 'docs'" }, @@ -2005,7 +2005,7 @@ requires-dist = [ { name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" }, { name = "pydantic", specifier = ">=2.7.4,<3" }, { name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" }, - { name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b6" }, + { name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b10" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" }, { name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" }, { name = "pytest-asyncio", marker = "extra == 'tests'", specifier = ">=0.21" }, @@ -3912,7 +3912,7 @@ crypto = [ [[package]] name = "pylance" -version = "14.0.0b6" +version = "14.0.0b10" source = { registry = "https://pypi.fury.io/lance-format/" } dependencies = [ { name = "lance-namespace" }, @@ -3922,12 +3922,12 @@ dependencies = [ { name = "pyarrow" }, ] wheels = [ - { url = "https://pypi.fury.io/lance-format/-/ver_DuQEn/pylance-14.0.0b6-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:3eade86fa30d9ff3248ee7ffb0b27ce29495394d12327da4d22b5c54bc2b59fe" }, - { url = "https://pypi.fury.io/lance-format/-/ver_K5sqj/pylance-14.0.0b6-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10e291974ee738a1d5d7e9dad17a634394a91cb60080b5ceac209077fbaec43c" }, - { url = "https://pypi.fury.io/lance-format/-/ver_EnbkT/pylance-14.0.0b6-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61bac10f5081c1210edd017f81c8c784d800b89446bd0ed05d4a24e7001965bc" }, - { url = "https://pypi.fury.io/lance-format/-/ver_19ffYg/pylance-14.0.0b6-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3c627ad4a9870611becf84409d8e5ecddc6be64e1dd53bee4bd5fb10ab561e5c" }, - { url = "https://pypi.fury.io/lance-format/-/ver_2eLLY5/pylance-14.0.0b6-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:bc603a7dd690233f1fe000600171e1c092f153bc9060cdf198825c4bbe1418aa" }, - { url = "https://pypi.fury.io/lance-format/-/ver_t8ggc/pylance-14.0.0b6-cp310-abi3-win_amd64.whl", hash = "sha256:1d32df32a3d157611450def07eae8ace3a7401286eac68cad9a24e0d5deed2db" }, + { url = "https://pypi.fury.io/lance-format/-/ver_4hvE/pylance-14.0.0b10-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:415b6b1b9fe735fbae7c755eb9774ae2fe0feddc90bf80bee24087cbfbfdf8c1" }, + { url = "https://pypi.fury.io/lance-format/-/ver_pM1km/pylance-14.0.0b10-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87317cad22f4d35fea11eb70aa5c0d71a0e1ea3b96de38f3deca933daa749fff" }, + { url = "https://pypi.fury.io/lance-format/-/ver_24jdyj/pylance-14.0.0b10-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6da554a2fc2197ba254676b12824d8120b48d65937697e0661a61302458a4740" }, + { url = "https://pypi.fury.io/lance-format/-/ver_XclaZ/pylance-14.0.0b10-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:36c80a801675bb9dcbd681fa99d0cdd5ca642dfb3ca08d71ac980278a2f7748b" }, + { url = "https://pypi.fury.io/lance-format/-/ver_L0yh/pylance-14.0.0b10-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:e50ad0b866faefb2fa3ae87328b93fa0d06b2bdd5bb5d0c1a6c74b907cc2fd93" }, + { url = "https://pypi.fury.io/lance-format/-/ver_2gbNr8/pylance-14.0.0b10-cp310-abi3-win_amd64.whl", hash = "sha256:3d86451fe00f71510005bb700737fb1a4f9361554541cbf802f3c5072554d014" }, ] [[package]] diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs index 6b20fb123..8fb56ceca 100644 --- a/rust/lancedb/src/query.rs +++ b/rust/lancedb/src/query.rs @@ -1114,9 +1114,18 @@ pub struct VectorQueryRequest { pub column: Option, /// The vector(s) to search for pub query_vector: Vec>, - /// The minimum number of partitions to search - pub minimum_nprobes: usize, - /// The maximum number of partitions to search + /// The legacy number of partitions to search. + /// + /// Lance sets both probe bounds to this value. Explicit bounds can override it. + pub nprobes: Option, + /// The minimum number of partitions to search. + /// + /// If unset, Lance's default is used. + pub minimum_nprobes: Option, + /// The maximum number of partitions to search. + /// + /// If unset, Lance's default is used. Zero represents an explicitly unbounded + /// maximum on request and wire representations. pub maximum_nprobes: Option, /// The lower bound (inclusive) of the distance to search for. pub lower_bound: Option, @@ -1141,8 +1150,9 @@ impl Default for VectorQueryRequest { base: QueryRequest::default(), column: None, query_vector: Vec::new(), - minimum_nprobes: 20, - maximum_nprobes: Some(20), + nprobes: None, + minimum_nprobes: None, + maximum_nprobes: None, lower_bound: None, upper_bound: None, ef: None, @@ -1248,7 +1258,7 @@ impl VectorQuery { Ok(self) } - /// Set the number of partitions to search (probe) + /// Set the legacy IVF probe parameter /// /// This argument is only used when the vector column has an IVF PQ index. /// If there is no index then this value is ignored. @@ -1258,19 +1268,19 @@ impl VectorQuery { /// /// The partition whose centroids are closest to the query vector will be /// exhaustiely searched to find matches. This parameter controls how many - /// partitions should be searched. + /// partitions may be searched. /// /// Increasing this value will increase the recall of your query but will - /// also increase the latency of your query. The default value is 20. This - /// default is good for many cases but the best value to use will depend on - /// your data and the recall that you need to achieve. + /// also increase the latency of your query. If this method is not called, + /// Lance's adaptive probe defaults are used. /// /// For best results we recommend tuning this parameter with a benchmark against /// your actual data to find the smallest possible value that will still give /// you the desired recall. /// - /// This method sets both the minimum and maximum number of partitions to search. - /// For more fine-grained control see [`VectorQuery::minimum_nprobes`] and + /// LanceDB retains this as `nprobes` through local and remote request + /// construction. Lance sets both probe bounds to this value. For more + /// fine-grained control see [`VectorQuery::minimum_nprobes`] and /// [`VectorQuery::maximum_nprobes`]. /// /// Returns an error if `nprobes` is not greater than 0. @@ -1280,8 +1290,7 @@ impl VectorQuery { message: "nprobes must be greater than 0".to_string(), }); } - self.request.minimum_nprobes = nprobes; - self.request.maximum_nprobes = Some(nprobes); + self.request.nprobes = Some(nprobes); Ok(self) } @@ -1303,6 +1312,7 @@ impl VectorQuery { }); } if let Some(maximum_nprobes) = self.request.maximum_nprobes + && maximum_nprobes != 0 && minimum_nprobes > maximum_nprobes { return Err(Error::InvalidInput { @@ -1310,7 +1320,7 @@ impl VectorQuery { .to_string(), }); } - self.request.minimum_nprobes = minimum_nprobes; + self.request.minimum_nprobes = Some(minimum_nprobes); Ok(self) } @@ -1327,7 +1337,7 @@ impl VectorQuery { /// This can be useful when there is a narrow filter to allow these queries to /// spend more time searching and avoid potential false negatives. /// - /// Set to None to search all partitions, if needed, to satisfy the limit + /// Set to None to use Lance's default maximum probe behavior. pub fn maximum_nprobes(mut self, maximum_nprobes: Option) -> Result { if let Some(maximum_nprobes) = maximum_nprobes { if maximum_nprobes == 0 { @@ -1335,14 +1345,18 @@ impl VectorQuery { message: "maximum_nprobes must be greater than 0".to_string(), }); } - if maximum_nprobes < self.request.minimum_nprobes { + if let Some(minimum_nprobes) = self.request.minimum_nprobes + && maximum_nprobes < minimum_nprobes + { return Err(Error::InvalidInput { message: "maximum_nprobes must be greater than or equal to minimum_nprobes" .to_string(), }); } } - self.request.maximum_nprobes = maximum_nprobes; + // Preserve an explicit reset separately from an untouched maximum. The + // request protocols already use zero to represent an unbounded maximum. + self.request.maximum_nprobes = Some(maximum_nprobes.unwrap_or(0)); Ok(self) } @@ -2431,6 +2445,9 @@ mod tests { let vector = Float32Array::from_iter_values([0.1, 0.2]); let query = table.query().nearest_to(&[0.1, 0.2]).unwrap(); + assert_eq!(query.request.nprobes, None); + assert_eq!(query.request.minimum_nprobes, None); + assert_eq!(query.request.maximum_nprobes, None); assert_eq!( *query .request @@ -2442,6 +2459,37 @@ mod tests { vector ); + let minimum_only_query = query.clone().minimum_nprobes(20).unwrap(); + assert_eq!(minimum_only_query.request.minimum_nprobes, Some(20)); + assert_eq!(minimum_only_query.request.maximum_nprobes, None); + + let maximum_only_query = query.maximum_nprobes(Some(20)).unwrap(); + assert_eq!(maximum_only_query.request.minimum_nprobes, None); + assert_eq!(maximum_only_query.request.maximum_nprobes, Some(20)); + + let explicitly_unbounded_query = table + .query() + .nearest_to(&[0.1, 0.2]) + .unwrap() + .nprobes(20) + .unwrap() + .maximum_nprobes(None) + .unwrap(); + assert_eq!(explicitly_unbounded_query.request.nprobes, Some(20)); + assert_eq!(explicitly_unbounded_query.request.maximum_nprobes, Some(0)); + + let bounded_query = table + .query() + .nearest_to(&[0.1, 0.2]) + .unwrap() + .minimum_nprobes(5) + .unwrap() + .nprobes(20) + .unwrap(); + assert_eq!(bounded_query.request.nprobes, Some(20)); + assert_eq!(bounded_query.request.minimum_nprobes, Some(5)); + assert_eq!(bounded_query.request.maximum_nprobes, None); + let new_vector = Float32Array::from_iter_values([9.8, 8.7]); let query = table @@ -2450,7 +2498,7 @@ mod tests { .offset(1) .nearest_to(&[9.8, 8.7]) .unwrap() - .nprobes(1000) + .nprobes(20) .unwrap() .postfilter() .distance_type(DistanceType::Cosine) @@ -2469,8 +2517,9 @@ mod tests { ); assert_eq!(query.request.base.limit.unwrap(), 100); assert_eq!(query.request.base.offset.unwrap(), 1); - assert_eq!(query.request.minimum_nprobes, 1000); - assert_eq!(query.request.maximum_nprobes, Some(1000)); + assert_eq!(query.request.nprobes, Some(20)); + assert_eq!(query.request.minimum_nprobes, None); + assert_eq!(query.request.maximum_nprobes, None); assert!(query.request.use_index); assert_eq!(query.request.distance_type, Some(DistanceType::Cosine)); assert_eq!(query.request.approx_mode, Some(ApproxMode::Accurate)); @@ -2492,13 +2541,13 @@ mod tests { Err(Error::InvalidInput { message }) if message == "nprobes must be greater than 0" )); - // Fixed probe counts replace both bounds, even when moving below or - // above the previous range. + // Legacy probe counts remain independent until Lance resolves them. let mut query = query; for nprobes in [30, 1, 50] { query = query.nprobes(nprobes).unwrap(); - assert_eq!(query.request.minimum_nprobes, nprobes); - assert_eq!(query.request.maximum_nprobes, Some(nprobes)); + assert_eq!(query.request.nprobes, Some(nprobes)); + assert_eq!(query.request.minimum_nprobes, None); + assert_eq!(query.request.maximum_nprobes, None); } } diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index f0d1e554d..119577dc5 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -77,7 +77,6 @@ use lance::dataset::{ColumnAlteration, NewColumnTransform, Version}; use lance_datafusion::exec::{OneShotExec, execute_plan}; use reqwest::{RequestBuilder, Response}; use serde::{Deserialize, Serialize}; -use serde_json::Number; use std::collections::{HashMap, HashSet}; use std::io::Cursor; use std::pin::Pin; @@ -1147,17 +1146,14 @@ impl RemoteTable { if let Some(approx_mode) = query.approx_mode { body["approx_mode"] = serde_json::json!(approx_mode); } - // In 0.23.1 we migrated from `nprobes` to `minimum_nprobes` and `maximum_nprobes`. - // Old client / new server: since minimum_nprobes is missing, fallback to nprobes - // New client / old server: old server will only see nprobes, make sure to set both - // nprobes and minimum_nprobes - // New client / new server: since minimum_nprobes is present, server can ignore nprobes - body["nprobes"] = query.minimum_nprobes.into(); - body["minimum_nprobes"] = query.minimum_nprobes.into(); + if let Some(nprobes) = query.nprobes { + body["nprobes"] = nprobes.into(); + } + if let Some(minimum_nprobes) = query.minimum_nprobes { + body["minimum_nprobes"] = minimum_nprobes.into(); + } if let Some(maximum_nprobes) = query.maximum_nprobes { body["maximum_nprobes"] = maximum_nprobes.into(); - } else { - body["maximum_nprobes"] = serde_json::Value::Number(Number::from_u128(0).unwrap()) } body["lower_bound"] = query.lower_bound.into(); body["upper_bound"] = query.upper_bound.into(); @@ -6090,9 +6086,6 @@ mod tests { let body: serde_json::Value = serde_json::from_slice(body).unwrap(); let mut expected_body = serde_json::json!({ "prefilter": true, - "nprobes": 20, - "minimum_nprobes": 20, - "maximum_nprobes": 20, "lower_bound": Option::::None, "upper_bound": Option::::None, "k": 10, @@ -6124,7 +6117,7 @@ mod tests { } #[tokio::test] - async fn test_query_vector_approx_mode_sent_when_set() { + async fn test_query_vector_minimum_only_and_approx_mode() { let expected_data = RecordBatch::try_new( Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])), vec![Arc::new(Int32Array::from(vec![1, 2, 3]))], @@ -6144,9 +6137,7 @@ mod tests { let body: serde_json::Value = serde_json::from_slice(body).unwrap(); let mut expected_body = serde_json::json!({ "prefilter": true, - "nprobes": 20, - "minimum_nprobes": 20, - "maximum_nprobes": 20, + "minimum_nprobes": 5, "approx_mode": "accurate", "lower_bound": Option::::None, "upper_bound": Option::::None, @@ -6170,6 +6161,8 @@ mod tests { .query() .nearest_to(vec![0.1, 0.2, 0.3]) .unwrap() + .minimum_nprobes(5) + .unwrap() .approx_mode(crate::ApproxMode::Accurate) .execute() .await; @@ -6178,6 +6171,39 @@ mod tests { assert_eq!(data[0].as_ref().unwrap(), &expected_data); } + #[tokio::test] + async fn test_query_vector_explicitly_unbounded_maximum() { + let table = Table::new_with_handler("my_table", |request| { + let body = request.body().unwrap().as_bytes().unwrap(); + let body: serde_json::Value = serde_json::from_slice(body).unwrap(); + assert_eq!(body["nprobes"], 5); + assert_eq!(body["maximum_nprobes"], 0); + + let data = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])), + vec![Arc::new(Int32Array::from(vec![1]))], + ) + .unwrap(); + http::Response::builder() + .status(200) + .header(CONTENT_TYPE, ARROW_FILE_CONTENT_TYPE) + .body(write_ipc_file(&data)) + .unwrap() + }); + + table + .query() + .nearest_to(vec![0.1, 0.2, 0.3]) + .unwrap() + .nprobes(5) + .unwrap() + .maximum_nprobes(None) + .unwrap() + .execute() + .await + .unwrap(); + } + #[tokio::test] async fn test_query_fts_default_values() { let expected_data = RecordBatch::try_new( @@ -6276,8 +6302,8 @@ mod tests { } ], "nprobes": 12, - "minimum_nprobes": 12, - "maximum_nprobes": 12, + "minimum_nprobes": 3, + "maximum_nprobes": 10, "lower_bound": Option::::None, "upper_bound": Option::::None, "ef": Option::::None, @@ -6317,6 +6343,10 @@ mod tests { .distance_type(crate::DistanceType::Cosine) .nprobes(12) .unwrap() + .minimum_nprobes(3) + .unwrap() + .maximum_nprobes(Some(10)) + .unwrap() .refine_factor(2) .bypass_vector_index() .execute() @@ -6347,9 +6377,6 @@ mod tests { "vector_column": "image.embedding", "prefilter": true, "k": 10, - "nprobes": 20, - "minimum_nprobes": 20, - "maximum_nprobes": 20, "lower_bound": Option::::None, "upper_bound": Option::::None, "ef": Option::::None, diff --git a/rust/lancedb/src/table/datafusion/udtf/vector_query.rs b/rust/lancedb/src/table/datafusion/udtf/vector_query.rs index 000addbe1..543a59589 100644 --- a/rust/lancedb/src/table/datafusion/udtf/vector_query.rs +++ b/rust/lancedb/src/table/datafusion/udtf/vector_query.rs @@ -411,9 +411,7 @@ impl ExecutionPlan for VectorDedupExec { .map_err(|e| datafusion_common::DataFusionError::External(Box::new(e)))?; let mut scanner = dataset.scan(); scanner - .with_row_addr_prefilter(RowAddrMask::from_block(RowAddrTreeMap::from_iter( - removed, - ))) + .with_row_id_prefilter(RowAddrMask::from_block(RowAddrTreeMap::from_iter(removed))) .blob_handling(BlobHandling::AllBinary) .batch_size(MATERIALIZE_BATCH_ROWS) .strict_batch_size(true); diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index 354eef5a5..1ad084e22 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -1296,6 +1296,21 @@ mod lsm_tests { rows } + async fn collect_ids(stream: SendableRecordBatchStream) -> Vec { + let batches: Vec<_> = stream.try_collect().await.unwrap(); + batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_primitive::() + .values() + .to_vec() + }) + .collect() + } + /// Upsert `ids` (value = 0..n) through the LSM `merge_insert` path. async fn lsm_upsert(table: &Table, ids: Vec) { let mut builder = table.merge_insert(&[]); @@ -1772,7 +1787,6 @@ mod lsm_tests { use crate::index::Index; use crate::index::vector::IvfPqIndexBuilder; use arrow::array::{FixedSizeListBuilder, Float32Builder}; - use arrow::datatypes::Int64Type; const DIM: i32 = 8; const N: i64 = 256; @@ -1817,7 +1831,7 @@ mod lsm_tests { &["vec"], Index::IvfPq( IvfPqIndexBuilder::default() - .num_partitions(1) + .num_partitions(8) .num_sub_vectors(2), ), ) @@ -1851,21 +1865,32 @@ mod lsm_tests { .execute() .await .unwrap(); - let batches: Vec<_> = stream.try_collect().await.unwrap(); - let ids: Vec = batches - .iter() - .flat_map(|b| { - b.column_by_name("id") - .unwrap() - .as_primitive::() - .values() - .to_vec() - }) - .collect(); + let ids = collect_ids(stream).await; assert_eq!( ids, vec![9999], "LSM vector search must rank the memtable row first" ); + + let base_query = table + .query() + .nearest_to(&[0.0_f32; 8]) + .unwrap() + .only_if("id = 255") + .limit(1); + for query in [ + base_query.clone(), + base_query.clone().nprobes(8).unwrap(), + base_query.clone().minimum_nprobes(8).unwrap(), + base_query.clone().maximum_nprobes(Some(8)).unwrap(), + base_query + .nprobes(1) + .unwrap() + .maximum_nprobes(None) + .unwrap(), + ] { + let ids = collect_ids(query.execute().await.unwrap()).await; + assert_eq!(ids, vec![255]); + } } } diff --git a/rust/lancedb/src/table/query.rs b/rust/lancedb/src/table/query.rs index d611fe8a5..77c4ca1a5 100644 --- a/rust/lancedb/src/table/query.rs +++ b/rust/lancedb/src/table/query.rs @@ -106,19 +106,16 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R } fn requires_local_namespace_execution(query: &AnyQuery) -> bool { - // The namespace QueryTable request has no approx_mode or use_lsm field yet, so - // pushing these down would silently ignore the user's setting. For use_lsm that - // is worse than a tuning miss: MemWAL read routing lives only in `create_plan`, - // so a pushed-down query would return stale base-only data with no error. + // The namespace QueryTable request cannot represent approx_mode or use_lsm. + // For use_lsm that is worse than a tuning miss: MemWAL read routing lives only + // in `create_plan`, so a pushed-down query would return stale base-only data. if query.base().use_lsm.is_some() || query.base().take_offsets.is_some() { return true; } matches!( query, - AnyQuery::VectorQuery(VectorQueryRequest { - approx_mode: Some(_), - .. - }) + AnyQuery::VectorQuery(vector_query) + if vector_query.approx_mode.is_some() ) } @@ -296,8 +293,17 @@ pub async fn create_plan( scanner.approx_mode(approx_mode.into()); } - scanner.minimum_nprobes(query.minimum_nprobes); - if let Some(maximum_nprobes) = query.maximum_nprobes { + if let Some(nprobes) = query.nprobes { + if query.maximum_nprobes == Some(0) { + scanner.minimum_nprobes(nprobes); + } else { + scanner.nprobes(nprobes); + } + } + if let Some(minimum_nprobes) = query.minimum_nprobes { + scanner.minimum_nprobes(minimum_nprobes); + } + if let Some(maximum_nprobes) = query.maximum_nprobes.filter(|nprobes| *nprobes != 0) { scanner.maximum_nprobes(maximum_nprobes); } } @@ -630,7 +636,9 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result { columns, offset: vq.base.offset.map(|o| o as i32), distance_type: vq.distance_type.map(|dt| dt.to_string()), - nprobes: Some(vq.minimum_nprobes as i32), + nprobes: vq.nprobes.map(|nprobes| nprobes as i32), + minimum_nprobes: vq.minimum_nprobes.map(|nprobes| nprobes as i32), + maximum_nprobes: vq.maximum_nprobes.map(|nprobes| nprobes as i32), ef: vq.ef.map(|e| e as i32), refine_factor: vq.refine_factor.map(|r| r as i32), lower_bound: vq.lower_bound, @@ -919,7 +927,6 @@ mod tests { column: Some("vector".to_string()), // We cast here to satisfy the struct definition query_vector: vec![query_vector as Arc], - minimum_nprobes: 20, distance_type: Some(crate::DistanceType::L2), ..Default::default() }; @@ -940,6 +947,7 @@ mod tests { ); assert_eq!(ns_request.vector_column, Some("vector".to_string())); assert_eq!(ns_request.distance_type, Some("l2".to_string())); + assert_eq!(ns_request.nprobes, None); // Verify the vector data was extracted correctly assert!(ns_request.vector.single_vector.is_some()); @@ -949,6 +957,66 @@ mod tests { ); } + #[test] + fn test_namespace_query_probe_pushdown_compatibility() { + let default_query = AnyQuery::VectorQuery(VectorQueryRequest::default()); + assert!(!requires_local_namespace_execution(&default_query)); + + let nprobes_query = AnyQuery::VectorQuery(VectorQueryRequest { + query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))], + nprobes: Some(20), + ..Default::default() + }); + assert!(!requires_local_namespace_execution(&nprobes_query)); + assert_eq!( + convert_to_namespace_query(&nprobes_query).unwrap().nprobes, + Some(20) + ); + + let minimum_query = AnyQuery::VectorQuery(VectorQueryRequest { + query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))], + minimum_nprobes: Some(5), + ..Default::default() + }); + assert!(!requires_local_namespace_execution(&minimum_query)); + let request = convert_to_namespace_query(&minimum_query).unwrap(); + assert_eq!(request.minimum_nprobes, Some(5)); + assert_eq!(request.maximum_nprobes, None); + + let maximum_query = AnyQuery::VectorQuery(VectorQueryRequest { + query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))], + maximum_nprobes: Some(10), + ..Default::default() + }); + assert!(!requires_local_namespace_execution(&maximum_query)); + let request = convert_to_namespace_query(&maximum_query).unwrap(); + assert_eq!(request.minimum_nprobes, None); + assert_eq!(request.maximum_nprobes, Some(10)); + + let exact_query = AnyQuery::VectorQuery(VectorQueryRequest { + query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))], + nprobes: Some(8), + minimum_nprobes: Some(20), + maximum_nprobes: Some(20), + ..Default::default() + }); + assert!(!requires_local_namespace_execution(&exact_query)); + let request = convert_to_namespace_query(&exact_query).unwrap(); + assert_eq!(request.nprobes, Some(8)); + assert_eq!(request.minimum_nprobes, Some(20)); + assert_eq!(request.maximum_nprobes, Some(20)); + + let explicitly_unbounded_query = AnyQuery::VectorQuery(VectorQueryRequest { + query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))], + nprobes: Some(8), + maximum_nprobes: Some(0), + ..Default::default() + }); + let request = convert_to_namespace_query(&explicitly_unbounded_query).unwrap(); + assert_eq!(request.nprobes, Some(8)); + assert_eq!(request.maximum_nprobes, Some(0)); + } + #[test] fn test_convert_to_namespace_query_plain_query() { let q = QueryRequest { diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index f3e40fc7b..5ec053483 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -170,9 +170,8 @@ fn reject_unsupported(query: &VectorQueryRequest) -> Result<()> { // Vector-only knobs the LSM scanner cannot honor. Both change results rather // than just recall, so error instead of silently ignoring them: distance_range // would return rows outside the bound, and use_index(false) asks for a - // brute-force search the index-only base arm can't do. (ef / approx_mode / - // maximum_nprobes are recall/speed knobs and are left to no-op — and - // maximum_nprobes defaults to Some, so it cannot be rejected on presence.) + // brute-force search the index-only base arm can't do. (ef / approx_mode are + // recall/speed knobs and are left to no-op.) if !query.query_vector.is_empty() { if query.lower_bound.is_some() || query.upper_bound.is_some() { return unsupported("distance_range on vector search"); @@ -740,8 +739,20 @@ async fn vector_plan( let mut scanner = base_scanner(dataset, query, pk_columns, snapshots, in_memory)? .with_overfetch_factor(LSM_OVERFETCH_FACTOR) .nearest(&column, query_vector.as_ref(), k)? - .nprobes(query.minimum_nprobes) .distance_metric(distance_type.into()); + if let Some(nprobes) = query.nprobes { + scanner = if query.maximum_nprobes == Some(0) { + scanner.minimum_nprobes(nprobes) + } else { + scanner.nprobes(nprobes) + }; + } + if let Some(minimum_nprobes) = query.minimum_nprobes { + scanner = scanner.minimum_nprobes(minimum_nprobes); + } + if let Some(maximum_nprobes) = query.maximum_nprobes.filter(|nprobes| *nprobes != 0) { + scanner = scanner.maximum_nprobes(maximum_nprobes); + } if let Some(refine_factor) = query.refine_factor { scanner = scanner.refine(refine_factor); }