rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-12 23:02:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main into gatekeeper/fix-4273-1

Browse Source 
Retain the upstream nested-blob missing-field checks and regression tests while preserving non-scan blob pandas conversion and remote snapshot safeguards.
This commit is contained in:
Gatefixer
2026-10-12 20:40:44 +00:00
parent febb9b8731 792ffd1082
commit 10e876db6a
64 changed files with 2857 additions and 892 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.41.0-beta.1"
current_version = "0.41.0-beta.6"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
Cargo.lock
Generated
+143 -248
View File
@@ -986,40 +986,36 @@ dependencies = [
[[package]]
name = "aws-smithy-http-client"
version = "1.1.12"
version = "1.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a2f165a7feee6f263028b899d0a181987f4fa7179a6411a32a439fba7c5f769"
checksum = "5c3ef8931ad1c98aa6a55b4256f847f3116090819844e0dd41ea682cac5dd2d3"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api",
"aws-smithy-types",
"h2 0.3.27",
"h2 0.4.16",
"http 0.2.12",
"h2",
"http 1.5.0",
"http-body 0.4.6",
"hyper 0.14.32",
"hyper 1.9.0",
"hyper-rustls 0.24.2",
"hyper-rustls 0.27.9",
"hyper",
"hyper-rustls",
"hyper-util",
"pin-project-lite",
"rustls 0.21.12",
"rustls 0.23.45",
"rustls",
"rustls-native-certs",
"rustls-pki-types",
"tokio",
"tokio-rustls 0.26.4",
"tokio-rustls",
"tower",
"tracing",
]
[[package]]
name = "aws-smithy-json"
version = "0.62.5"
version = "0.62.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9648b0bb82a2eedd844052c6ad2a1a822d1f8e3adee5fbf668366717e428856a"
checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-schema",
"aws-smithy-types",
]
@@ -1070,9 +1066,9 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime-api"
version = "1.12.1"
version = "1.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc117c179ecf39a62a0a3f49f600e9ac26a7ad7dd172177999f83933af776c32"
checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api-macros",
@@ -1170,7 +1166,7 @@ dependencies = [
"http 1.5.0",
"http-body 1.1.0",
"http-body-util",
"hyper 1.9.0",
"hyper",
"hyper-util",
"itoa",
"matchit 0.7.3",
@@ -1819,7 +1815,7 @@ version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -3315,7 +3311,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -3588,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4000,11 +3996,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
dependencies = [
"cfg-if 1.0.4",
"js-sys",
"libc",
"r-efi 5.3.0",
"wasip2",
"wasm-bindgen",
]
[[package]]
@@ -4110,25 +4104,6 @@ dependencies = [
"subtle",
]
[[package]]
name = "h2"
version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d"
dependencies = [
"bytes",
"fnv",
"futures-core",
"futures-sink",
"futures-util",
"http 0.2.12",
"indexmap 2.14.0",
"slab",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "h2"
version = "0.4.16"
@@ -4430,30 +4405,6 @@ dependencies = [
"typenum",
]
[[package]]
name = "hyper"
version = "0.14.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7"
dependencies = [
"bytes",
"futures-channel",
"futures-core",
"futures-util",
"h2 0.3.27",
"http 0.2.12",
"http-body 0.4.6",
"httparse",
"httpdate",
"itoa",
"pin-project-lite",
"socket2 0.5.10",
"tokio",
"tower-service",
"tracing",
"want",
]
[[package]]
name = "hyper"
version = "1.9.0"
@@ -4464,7 +4415,7 @@ dependencies = [
"bytes",
"futures-channel",
"futures-core",
"h2 0.4.16",
"h2",
"http 1.5.0",
"http-body 1.1.0",
"httparse",
@@ -4476,21 +4427,6 @@ dependencies = [
"want",
]
[[package]]
name = "hyper-rustls"
version = "0.24.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
dependencies = [
"futures-util",
"http 0.2.12",
"hyper 0.14.32",
"log",
"rustls 0.21.12",
"tokio",
"tokio-rustls 0.24.1",
]
[[package]]
name = "hyper-rustls"
version = "0.27.9"
@@ -4498,12 +4434,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
dependencies = [
"http 1.5.0",
"hyper 1.9.0",
"hyper",
"hyper-util",
"rustls 0.23.45",
"rustls",
"rustls-native-certs",
"tokio",
"tokio-rustls 0.26.4",
"tokio-rustls",
"tower-service",
"webpki-roots 1.0.7",
]
@@ -4514,7 +4450,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
dependencies = [
"hyper 1.9.0",
"hyper",
"hyper-util",
"pin-project-lite",
"tokio",
@@ -4533,12 +4469,12 @@ dependencies = [
"futures-util",
"http 1.5.0",
"http-body 1.1.0",
"hyper 1.9.0",
"hyper",
"ipnet",
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.6.3",
"socket2",
"system-configuration",
"tokio",
"tower-service",
@@ -4880,7 +4816,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -5095,26 +5031,6 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "jsonb"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb98fb29636087c40ad0d1274d9a30c0c1e83e03ae93f6e7e89247b37fcc6953"
dependencies = [
"byteorder",
"ethnum",
"fast-float2",
"itoa",
"jiff",
"nom 8.0.0",
"num-traits",
"ordered-float 5.3.0",
"rand 0.9.5",
"serde",
"serde_json",
"zmij",
]
[[package]]
name = "kanaria"
version = "0.2.0"
@@ -5143,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arc-swap",
"arrow",
@@ -5217,8 +5133,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5232,7 +5148,7 @@ dependencies = [
"futures",
"getrandom 0.2.17",
"half",
"jsonb",
"lance-jsonb",
"num-traits",
"rand 0.9.5",
]
@@ -5240,7 +5156,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5254,7 +5170,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5263,8 +5179,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrayref",
"crunchy",
@@ -5274,8 +5190,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5312,8 +5228,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrow",
"arrow-array",
@@ -5330,10 +5246,10 @@ dependencies = [
"datafusion-physical-expr",
"futures",
"half",
"jsonb",
"lance-arrow",
"lance-core",
"lance-geo",
"lance-jsonb",
"log",
"pin-project",
"prost",
@@ -5344,8 +5260,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrow",
"arrow-array",
@@ -5362,8 +5278,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"proc-macro2",
"quote",
@@ -5372,8 +5288,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5406,8 +5322,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5439,8 +5355,8 @@ dependencies = [
[[package]]
name = "lance-geo"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"datafusion",
"geo-traits",
@@ -5454,8 +5370,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arc-swap",
"arrow",
@@ -5486,7 +5402,6 @@ dependencies = [
"hex",
"itertools 0.14.0",
"jieba-rs",
"jsonb",
"lance-arrow",
"lance-arrow-stats",
"lance-bitpacking",
@@ -5497,6 +5412,7 @@ dependencies = [
"lance-geo",
"lance-index-core",
"lance-io",
"lance-jsonb",
"lance-linalg",
"lance-select",
"lance-table",
@@ -5526,8 +5442,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5550,8 +5466,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrow",
"arrow-array",
@@ -5591,10 +5507,28 @@ dependencies = [
"uuid",
]
[[package]]
name = "lance-jsonb"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"byteorder",
"ethnum",
"fast-float2",
"itoa",
"jiff",
"nom 8.0.0",
"num-traits",
"ordered-float 5.3.0",
"serde",
"serde_json",
"zmij",
]
[[package]]
name = "lance-linalg"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5608,8 +5542,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrow",
"async-trait",
@@ -5623,8 +5557,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arc-swap",
"arrow",
@@ -5678,8 +5612,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5693,8 +5627,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrow",
"arrow-array",
@@ -5734,8 +5668,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5748,8 +5682,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "14.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21"
version = "14.0.0-beta.13"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.13#145fd4b933fb4a9a45831f0ae9185c88e908fe6a"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5762,7 +5696,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.41.0-beta.1"
version = "0.41.0-beta.6"
dependencies = [
"ahash",
"anyhow",
@@ -5835,7 +5769,7 @@ dependencies = [
"rand 0.9.5",
"random_word",
"regex",
"reqwest 0.12.28",
"reqwest 0.13.4",
"roaring",
"rstest",
"semver",
@@ -5859,7 +5793,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.41.0-beta.1"
version = "0.41.0-beta.6"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5884,7 +5818,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.41.0-beta.1"
version = "0.41.0-beta.6"
dependencies = [
"arc-swap",
"arrow",
@@ -6901,7 +6835,7 @@ dependencies = [
"http-body-util",
"httparse",
"humantime",
"hyper 1.9.0",
"hyper",
"itertools 0.15.0",
"md-5",
"nix 0.31.3",
@@ -8381,8 +8315,8 @@ dependencies = [
"quinn-proto",
"quinn-udp",
"rustc-hash",
"rustls 0.23.45",
"socket2 0.6.3",
"rustls",
"socket2",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -8391,18 +8325,19 @@ dependencies = [
[[package]]
name = "quinn-proto"
version = "0.11.14"
version = "0.11.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe"
dependencies = [
"aws-lc-rs",
"bytes",
"getrandom 0.3.4",
"getrandom 0.4.2",
"lru-slab",
"rand 0.9.5",
"rand 0.10.1",
"rand_pcg",
"ring",
"rustc-hash",
"rustls 0.23.45",
"rustls",
"rustls-pki-types",
"slab",
"thiserror 2.0.18",
@@ -8420,9 +8355,9 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.6.3",
"socket2",
"tracing",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -8567,6 +8502,15 @@ dependencies = [
"rand 0.9.5",
]
[[package]]
name = "rand_pcg"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a"
dependencies = [
"rand_core 0.10.1",
]
[[package]]
name = "rand_xoshiro"
version = "0.7.0"
@@ -8931,12 +8875,12 @@ dependencies = [
"encoding_rs",
"futures-core",
"futures-util",
"h2 0.4.16",
"h2",
"http 1.5.0",
"http-body 1.1.0",
"http-body-util",
"hyper 1.9.0",
"hyper-rustls 0.27.9",
"hyper",
"hyper-rustls",
"hyper-util",
"js-sys",
"log",
@@ -8945,7 +8889,7 @@ dependencies = [
"percent-encoding",
"pin-project-lite",
"quinn",
"rustls 0.23.45",
"rustls",
"rustls-native-certs",
"rustls-pki-types",
"serde",
@@ -8953,7 +8897,7 @@ dependencies = [
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tokio-rustls 0.26.4",
"tokio-rustls",
"tokio-util",
"tower",
"tower-http 0.6.10",
@@ -8974,28 +8918,31 @@ checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
dependencies = [
"base64 0.22.1",
"bytes",
"encoding_rs",
"futures-core",
"futures-util",
"h2 0.4.16",
"h2",
"http 1.5.0",
"http-body 1.1.0",
"http-body-util",
"hyper 1.9.0",
"hyper-rustls 0.27.9",
"hyper",
"hyper-rustls",
"hyper-util",
"js-sys",
"log",
"mime",
"percent-encoding",
"pin-project-lite",
"quinn",
"rustls 0.23.45",
"rustls",
"rustls-pki-types",
"rustls-platform-verifier",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tokio-rustls 0.26.4",
"tokio-rustls",
"tokio-util",
"tower",
"tower-http 0.6.10",
@@ -9220,19 +9167,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.59.0",
]
[[package]]
name = "rustls"
version = "0.21.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e"
dependencies = [
"log",
"ring",
"rustls-webpki 0.101.7",
"sct",
"windows-sys 0.52.0",
]
[[package]]
@@ -9246,7 +9181,7 @@ dependencies = [
"once_cell",
"ring",
"rustls-pki-types",
"rustls-webpki 0.103.15",
"rustls-webpki",
"subtle",
"zeroize",
]
@@ -9284,14 +9219,14 @@ dependencies = [
"jni",
"log",
"once_cell",
"rustls 0.23.45",
"rustls",
"rustls-native-certs",
"rustls-platform-verifier-android",
"rustls-webpki 0.103.15",
"rustls-webpki",
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -9300,16 +9235,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
[[package]]
name = "rustls-webpki"
version = "0.101.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
dependencies = [
"ring",
"untrusted 0.9.0",
]
[[package]]
name = "rustls-webpki"
version = "0.103.15"
@@ -9425,16 +9350,6 @@ dependencies = [
"sha2 0.10.9",
]
[[package]]
name = "sct"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
dependencies = [
"ring",
"untrusted 0.9.0",
]
[[package]]
name = "sec1"
version = "0.3.0"
@@ -9856,16 +9771,6 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "socket2"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
dependencies = [
"libc",
"windows-sys 0.52.0",
]
[[package]]
name = "socket2"
version = "0.6.3"
@@ -10284,7 +10189,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -10497,7 +10402,7 @@ dependencies = [
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2 0.6.3",
"socket2",
"tokio-macros",
"windows-sys 0.61.2",
]
@@ -10524,23 +10429,13 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-rustls"
version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
dependencies = [
"rustls 0.21.12",
"tokio",
]
[[package]]
name = "tokio-rustls"
version = "0.26.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
dependencies = [
"rustls 0.23.45",
"rustls",
"tokio",
]
@@ -10633,20 +10528,20 @@ dependencies = [
"axum 0.8.9",
"base64 0.22.1",
"bytes",
"h2 0.4.16",
"h2",
"http 1.5.0",
"http-body 1.1.0",
"http-body-util",
"hyper 1.9.0",
"hyper",
"hyper-timeout",
"hyper-util",
"percent-encoding",
"pin-project",
"rustls-native-certs",
"socket2 0.6.3",
"socket2",
"sync_wrapper",
"tokio",
"tokio-rustls 0.26.4",
"tokio-rustls",
"tokio-stream",
"tower",
"tower-layer",
@@ -10954,7 +10849,7 @@ dependencies = [
"flate2",
"log",
"once_cell",
"rustls 0.23.45",
"rustls",
"rustls-pki-types",
"serde",
"serde_json",
@@ -11305,7 +11200,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -11775,7 +11670,7 @@ dependencies = [
"crc32fast",
"futures",
"http 1.5.0",
"hyper 1.9.0",
"hyper",
"more-asserts",
"rand 0.10.1",
"redb",
@@ -11911,9 +11806,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
[[package]]
name = "xxhash-rust"
version = "0.8.15"
version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7"
[[package]]
name = "yoke"
Cargo.toml
+15 -15
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.94.0"
[workspace.dependencies]
lance = { "version" = "=14.0.0-beta.7", default-features = false, "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=14.0.0-beta.7", default-features = false, "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=14.0.0-beta.7", default-features = false, "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=14.0.0-beta.13", default-features = false, "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=14.0.0-beta.13", default-features = false, "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=14.0.0-beta.13", default-features = false, "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=14.0.0-beta.13", "tag" = "v14.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
@@ -75,7 +75,7 @@ serde = "1"
serde_json = "1"
tempfile = "3.5.0"
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
tonic = { version = "0.14", features = ["tls-native-roots", "tls-ring"] }
tonic = { version = "0.14", features = ["tls-native-roots", "tls-aws-lc"] }
uuid = { version = "1.7.0", features = ["v4", "v7"] }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.41.0-beta.1</version>
<version>0.41.0-beta.6</version>
</dependency>
```
docs/src/js/classes/VectorQuery.md
+37 -6
View File
@@ -460,17 +460,16 @@ exhaustiely searched to find matches. This parameter controls how many
partitions should be searched.
Increasing this value will increase the recall of your query but will
also increase the latency of your query. The default value is 20. This
default is good for many cases but the best value to use will depend on
your data and the recall that you need to achieve.
also increase the latency of your query. If this method is not called,
Lance's adaptive probe defaults are used.
For best results we recommend tuning this parameter with a benchmark against
your actual data to find the smallest possible value that will still give
you the desired recall.
For more fine grained control over behavior when you have a very narrow filter
you can use `minimumNprobes` and `maximumNprobes`. This method sets both
the minimum and maximum to the same value.
The value is retained as `nprobes` through client and server request
construction. Lance sets both probe bounds to this value. Explicit minimum
or maximum settings can override their respective bounds.
#### Parameters
@@ -647,6 +646,38 @@ rerank(reranker): VectorQuery
***
### searchEffort()
```ts
searchEffort(searchEffort): VectorQuery
```
Set the initial IVF search effort to a finite value in [0, 1].
The default, null, uses Auto (internally 0.5). Zero starts at the caller
minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found.
Any explicit effort, including 0.5, overrides nprobes in either call order.
Explicit minimum and maximum bounds still apply. This is not a recall
guarantee and does not change refinement or HNSW ef. The MemWAL LSM
scanner does not yet support this option.
#### Parameters
* **searchEffort**: `null` \| `number`
#### Returns
[`VectorQuery`](VectorQuery.md)
#### Example
```ts
table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10);
```
***
### select()
```ts
docs/web-tests/rs/Cargo.lock
Generated
+64 -85
View File
@@ -556,6 +556,15 @@ dependencies = [
"generic-array",
]
[[package]]
name = "bs58"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4"
dependencies = [
"tinyvec",
]
[[package]]
name = "bumpalo"
version = "3.19.0"
@@ -1065,12 +1074,12 @@ dependencies = [
[[package]]
name = "darling"
version = "0.21.3"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0"
checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
dependencies = [
"darling_core 0.21.3",
"darling_macro 0.21.3",
"darling_core 0.23.0",
"darling_macro 0.23.0",
]
[[package]]
@@ -1089,11 +1098,10 @@ dependencies = [
[[package]]
name = "darling_core"
version = "0.21.3"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4"
checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
@@ -1114,11 +1122,11 @@ dependencies = [
[[package]]
name = "darling_macro"
version = "0.21.3"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81"
checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
dependencies = [
"darling_core 0.21.3",
"darling_core 0.23.0",
"quote",
"syn 2.0.117",
]
@@ -1177,7 +1185,7 @@ dependencies = [
"log",
"object_store",
"parking_lot",
"rand 0.9.2",
"rand 0.9.5",
"regex",
"sqlparser 0.61.0",
"tempfile",
@@ -1292,7 +1300,7 @@ dependencies = [
"itertools 0.14.0",
"log",
"object_store",
"rand 0.9.2",
"rand 0.9.5",
"tokio",
"url",
]
@@ -1392,7 +1400,7 @@ dependencies = [
"log",
"object_store",
"parking_lot",
"rand 0.9.2",
"rand 0.9.5",
"tempfile",
"url",
]
@@ -1457,7 +1465,7 @@ dependencies = [
"md-5",
"memchr",
"num-traits",
"rand 0.9.2",
"rand 0.9.5",
"regex",
"sha2",
"unicode-segmentation",
@@ -2078,7 +2086,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7af6e24ed12cf382082d5a7f365df2b8e3d6b1518f615d54c85165e9b9718250"
dependencies = [
"arrow-array",
"rand 0.9.2",
"rand 0.9.5",
]
[[package]]
@@ -2498,7 +2506,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
"windows-core 0.62.2",
"windows-core 0.61.2",
]
[[package]]
@@ -2877,7 +2885,7 @@ dependencies = [
"nom",
"num-traits",
"ordered-float",
"rand 0.9.2",
"rand 0.9.5",
"ryu",
"serde",
"serde_json",
@@ -2950,7 +2958,7 @@ dependencies = [
"prost",
"prost-build",
"prost-types",
"rand 0.9.2",
"rand 0.9.5",
"rayon",
"roaring",
"rustc-hash",
@@ -2985,7 +2993,7 @@ dependencies = [
"half",
"jsonb",
"num-traits",
"rand 0.9.2",
"rand 0.9.5",
]
[[package]]
@@ -3007,7 +3015,7 @@ dependencies = [
"half",
"jsonb",
"num-traits",
"rand 0.9.2",
"rand 0.9.5",
]
[[package]]
@@ -3071,7 +3079,7 @@ dependencies = [
"object_store",
"pin-project",
"prost",
"rand 0.9.2",
"rand 0.9.5",
"roaring",
"serde_json",
"snafu 0.9.1",
@@ -3110,7 +3118,7 @@ dependencies = [
"object_store",
"pin-project",
"prost",
"rand 0.9.2",
"rand 0.9.5",
"roaring",
"serde_json",
"snafu 0.9.1",
@@ -3169,7 +3177,7 @@ dependencies = [
"futures",
"half",
"hex",
"rand 0.9.2",
"rand 0.9.5",
"rand_distr 0.5.1",
"rand_xoshiro",
]
@@ -3214,7 +3222,7 @@ dependencies = [
"num-traits",
"prost",
"prost-build",
"rand 0.9.2",
"rand 0.9.5",
"strum 0.26.3",
"tokio",
"tracing",
@@ -3306,7 +3314,7 @@ dependencies = [
"prost",
"prost-build",
"prost-types",
"rand 0.9.2",
"rand 0.9.5",
"rand_distr 0.5.1",
"rangemap",
"rayon",
@@ -3352,7 +3360,7 @@ dependencies = [
"path_abs",
"pin-project",
"prost",
"rand 0.9.2",
"rand 0.9.5",
"serde",
"tempfile",
"tokio",
@@ -3374,7 +3382,7 @@ dependencies = [
"lance-arrow 8.0.0",
"lance-core 8.0.0",
"num-traits",
"rand 0.9.2",
"rand 0.9.5",
]
[[package]]
@@ -3428,7 +3436,7 @@ dependencies = [
"lance-table",
"log",
"object_store",
"rand 0.9.2",
"rand 0.9.5",
"roaring",
"serde_json",
"time",
@@ -3508,7 +3516,7 @@ dependencies = [
"prost",
"prost-build",
"prost-types",
"rand 0.9.2",
"rand 0.9.5",
"rangemap",
"roaring",
"semver",
@@ -3533,7 +3541,7 @@ dependencies = [
"lance-arrow 8.0.0",
"num-traits",
"pprof",
"rand 0.9.2",
"rand 0.9.5",
]
[[package]]
@@ -3603,7 +3611,7 @@ dependencies = [
"pin-project",
"polars",
"polars-arrow",
"rand 0.9.2",
"rand 0.9.5",
"regex",
"semver",
"serde",
@@ -4366,7 +4374,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
dependencies = [
"phf_shared 0.11.3",
"rand 0.8.5",
"rand 0.8.8",
]
[[package]]
@@ -4584,7 +4592,7 @@ dependencies = [
"polars-error",
"polars-row",
"polars-utils",
"rand 0.8.5",
"rand 0.8.8",
"rand_distr 0.4.3",
"rayon",
"regex",
@@ -4783,7 +4791,7 @@ dependencies = [
"polars-error",
"polars-lazy",
"polars-plan",
"rand 0.8.5",
"rand 0.8.8",
"serde",
"serde_json",
"sqlparser 0.39.0",
@@ -5096,9 +5104,9 @@ dependencies = [
[[package]]
name = "rand"
version = "0.8.5"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c"
dependencies = [
"libc",
"rand_chacha 0.3.1",
@@ -5107,9 +5115,9 @@ dependencies = [
[[package]]
name = "rand"
version = "0.9.2"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
dependencies = [
"rand_chacha 0.9.0",
"rand_core 0.9.3",
@@ -5177,7 +5185,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31"
dependencies = [
"num-traits",
"rand 0.8.5",
"rand 0.8.8",
]
[[package]]
@@ -5187,7 +5195,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"
dependencies = [
"num-traits",
"rand 0.9.2",
"rand 0.9.5",
]
[[package]]
@@ -5533,9 +5541,9 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.23.35"
version = "0.23.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f"
checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
dependencies = [
"once_cell",
"ring",
@@ -5569,9 +5577,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
version = "0.103.13"
version = "0.103.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
dependencies = [
"ring",
"rustls-pki-types",
@@ -5747,15 +5755,17 @@ dependencies = [
[[package]]
name = "serde_with"
version = "3.16.1"
version = "3.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7"
checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a"
dependencies = [
"base64 0.22.1",
"bs58",
"chrono",
"hex",
"indexmap 1.9.3",
"indexmap 2.14.0",
"jiff",
"schemars 0.9.0",
"schemars 1.1.0",
"serde_core",
@@ -5766,11 +5776,11 @@ dependencies = [
[[package]]
name = "serde_with_macros"
version = "3.16.1"
version = "3.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c"
checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46"
dependencies = [
"darling 0.21.3",
"darling 0.23.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -6544,7 +6554,7 @@ version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
dependencies = [
"rand 0.9.2",
"rand 0.9.5",
]
[[package]]
@@ -6917,21 +6927,8 @@ dependencies = [
"windows-implement",
"windows-interface",
"windows-link 0.1.3",
"windows-result 0.3.4",
"windows-strings 0.4.2",
]
[[package]]
name = "windows-core"
version = "0.62.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
dependencies = [
"windows-implement",
"windows-interface",
"windows-link 0.2.1",
"windows-result 0.4.1",
"windows-strings 0.5.1",
"windows-result",
"windows-strings",
]
[[package]]
@@ -6998,15 +6995,6 @@ dependencies = [
"windows-link 0.1.3",
]
[[package]]
name = "windows-result"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
dependencies = [
"windows-link 0.2.1",
]
[[package]]
name = "windows-strings"
version = "0.4.2"
@@ -7016,15 +7004,6 @@ dependencies = [
"windows-link 0.1.3",
]
[[package]]
name = "windows-strings"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
dependencies = [
"windows-link 0.2.1",
]
[[package]]
name = "windows-sys"
version = "0.52.0"
@@ -7310,9 +7289,9 @@ dependencies = [
[[package]]
name = "xxhash-rust"
version = "0.8.15"
version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7"
[[package]]
name = "yoke"
docs/web/indexing/vector-index.mdx
+5 -5
View File
@@ -139,9 +139,9 @@ Learn how to configure, build, and search LanceDB vector indexes, including buil
| Parameter | Description |
| :--- | :--- |
| `limit` | Number of results to return (the `k` in `k-ANN`). |
| `nprobes` | Shorthand that sets both `minimum_nprobes` and `maximum_nprobes` to the same value. LanceDB defaults this to 20 which is a good default for many cases. At very high scales you will likely need to experiment to find your preferred recall / latency balance. |
| `minimum_nprobes` | Minimum number of partitions scanned. |
| `maximum_nprobes` | Maximum number of partitions scanned. Only scans more than `minimum_nprobes` if an initial pass does not return enough results — useful for narrow filters. Set to `0` to remove the cap. |
| `nprobes` | Sets both `minimum_nprobes` and `maximum_nprobes` to the same value in Lance. Leave unset to use Lance's adaptive defaults. At very high scales you will likely need to experiment to find your preferred recall / latency balance. |
| `minimum_nprobes` | Minimum number of partitions scanned. Leave unset to use Lance's default. |
| `maximum_nprobes` | Maximum number of partitions scanned. Only scans more than `minimum_nprobes` if an initial pass does not return enough results — useful for narrow filters. Leave unset, or set to `0`, to remove the cap. |
| `ef` | HNSW search-time exploration factor. Start around `1.5 * k` and increase up to `10 * k` for higher recall. |
| `refine_factor` | Reads and reranks additional candidates in memory to recover recall lost to quantization. Start between `5` and `50` (a factor of `25` reads and reranks the top `25 * k` candidates) and measure the latency cost, typically a few milliseconds, against the recall gained. |
| `distance_range(lower_bound, upper_bound)` | Return only rows whose distance falls within `[lower_bound, upper_bound)`. Either bound is optional. Useful for near-duplicate detection or "close-enough" matching. |
@@ -165,8 +165,8 @@ older client.
| Index type | Guidance |
| :--- | :--- |
| `IVF_HNSW_FLAT`, `IVF_HNSW_SQ` | Keep the default `nprobes`, then tune `ef` first. Expect higher latency variance under filtered search. |
| `IVF_RQ`, `IVF_PQ` | Keep the default `nprobes` and raise when recall is insufficient. If using a selective filter you may need to increase `maximum_nprobes` if you do not received enough results. |
| `IVF_HNSW_FLAT`, `IVF_HNSW_SQ` | Leave the probe settings unset, then tune `ef` first. Expect higher latency variance under filtered search. |
| `IVF_RQ`, `IVF_PQ` | Leave the probe settings unset and configure them only when recall is insufficient. If using a selective filter you may need to increase `maximum_nprobes` if you do not receive enough results. |
<Note>
java/lancedb-core/pom.xml
+25 -2
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.41.0-beta.1</version>
<version>0.41.0-beta.6</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -44,7 +44,30 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.17.1</version>
</dependency>
<!-- Declared directly so downstream consumers resolve the patched versions:
the parent's jackson-bom only manages our own build, and otherwise Maven's
nearest-wins mediation picks older versions via arrow-vector and
lance-namespace-apache-client. -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-json-provider</artifactId>
</dependency>
<dependency>
java/pom.xml
+12 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.41.0-beta.1</version>
<version>0.41.0-beta.6</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,8 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>14.0.0-beta.7</lance-core.version>
<lance-core.version>14.0.0-beta.13</lance-core.version>
<jackson.version>2.18.11</jackson.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
@@ -62,6 +63,15 @@
<dependencyManagement>
<dependencies>
<!-- Pin every Jackson artifact, including ones pulled in transitively
(e.g. jackson-core), to a single patched version. -->
<dependency>
<groupId>com.fasterxml.jackson</groupId>
<artifactId>jackson-bom</artifactId>
<version>${jackson.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.lance</groupId>
<artifactId>lance-core</artifactId>
nodejs/Cargo.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.41.0-beta.1"
version = "0.41.0-beta.6"
publish = false
license.workspace = true
description.workspace = true
nodejs/__test__/query.test.ts
+103
View File
@@ -527,3 +527,106 @@ describe("Query orderBy", () => {
]);
});
});
describe("IVF search effort", () => {
let directory: tmp.DirResult;
let table: Table;
const vector = Array.from({ length: 16 }, (_, i) => (i === 0 ? 1 : 0));
beforeEach(async () => {
directory = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(directory.name);
const rows = Array.from({ length: 512 }, (_, id) => ({
id,
vector: Array.from(
{ length: 16 },
(_, j) =>
(Math.floor(id / 32) === j ? 1 : 0) + Math.sin(id * 17 + j) * 0.001,
),
}));
table = await db.createTable("effort", rows.slice(0, 256));
await table.add(rows.slice(256));
await table.createIndex("vector", {
config: Index.ivfFlat({ numPartitions: 16 }),
});
});
afterEach(() => {
directory.removeCallback();
});
it.each([
[null, 6],
[0, 1],
[0.25, 3],
[0.5, 6],
[0.75, 10],
[1, 16],
])(
"preserves effort %s through execution and probe precedence",
async (effort, expected) => {
const partitions = async (query: ReturnType<Table["vectorSearch"]>) => {
const plan = await query.limit(8).analyzePlan();
const matches = [...plan.matchAll(/partitions_searched=(\d+)/g)];
expect(matches.length).toBeGreaterThan(0);
return matches.reduce((total, match) => total + Number(match[1]), 0);
};
expect(
await partitions(table.vectorSearch(vector).searchEffort(effort)),
).toBe(expected);
expect(
await partitions(
table.vectorSearch(vector).nprobes(1).searchEffort(effort),
),
).toBe(effort === null ? 1 : expected);
expect(
await partitions(
table.vectorSearch(vector).searchEffort(effort).nprobes(1),
),
).toBe(effort === null ? 1 : expected);
expect(
await partitions(
table
.vectorSearch(vector)
.searchEffort(effort)
.nprobes(1)
.searchEffort(null),
),
).toBe(1);
expect(
await partitions(
table.vectorSearch(vector).searchEffort(effort).maximumNprobes(2),
),
).toBeLessThanOrEqual(2);
const actual = await table
.vectorSearch(vector)
.searchEffort(effort)
.limit(8)
.toArray();
const exact = await table
.vectorSearch(vector)
.bypassVectorIndex()
.limit(8)
.toArray();
const ids = new Set(exact.map((row) => row.id));
expect(
actual.filter((row) => ids.has(row.id)).length / 8,
).toBeGreaterThanOrEqual(0.5);
},
);
it.each([
-0.1,
1.1,
Number.NaN,
Number.POSITIVE_INFINITY,
Number.NEGATIVE_INFINITY,
])("rejects invalid effort %s before execution", (effort) => {
expect(() => table.vectorSearch(vector).searchEffort(effort)).toThrow(
"search_effort must be finite and in [0, 1]",
);
expect(() =>
table.vectorSearch(vector).fullTextSearch("dog").searchEffort(effort),
).toThrow("search_effort must be finite and in [0, 1]");
});
});
nodejs/__test__/table.test.ts
+3 -1
View File
@@ -1231,7 +1231,9 @@ describe("When creating an index", () => {
expect(() => tbl.query().nearestTo(queryVec).minimumNprobes(0)).toThrow(
"Invalid input, minimum_nprobes must be greater than 0",
);
expect(() => tbl.query().nearestTo(queryVec).maximumNprobes(5)).toThrow(
expect(() =>
tbl.query().nearestTo(queryVec).minimumNprobes(10).maximumNprobes(5),
).toThrow(
"Invalid input, maximum_nprobes must be greater than or equal to minimum_nprobes",
);
nodejs/lancedb/query.ts
+26 -6
View File
@@ -532,17 +532,16 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* partitions should be searched.
*
* Increasing this value will increase the recall of your query but will
* also increase the latency of your query. The default value is 20. This
* default is good for many cases but the best value to use will depend on
* your data and the recall that you need to achieve.
* also increase the latency of your query. If this method is not called,
* Lance's adaptive probe defaults are used.
*
* For best results we recommend tuning this parameter with a benchmark against
* your actual data to find the smallest possible value that will still give
* you the desired recall.
*
* For more fine grained control over behavior when you have a very narrow filter
* you can use `minimumNprobes` and `maximumNprobes`. This method sets both
* the minimum and maximum to the same value.
* The value is retained as `nprobes` through client and server request
* construction. Lance sets both probe bounds to this value. Explicit minimum
* or maximum settings can override their respective bounds.
*/
nprobes(nprobes: number): VectorQuery {
this.doVectorCall((inner) => inner.nprobes(nprobes));
@@ -550,6 +549,27 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
return this;
}
/**
* Set the initial IVF search effort to a finite value in [0, 1].
*
* The default, null, uses Auto (internally 0.5). Zero starts at the caller
* minimum; one searches all partitions within the caller maximum. Later
* probing may continue when too few candidates are found.
* Any explicit effort, including 0.5, overrides nprobes in either call order.
* Explicit minimum and maximum bounds still apply. This is not a recall
* guarantee and does not change refinement or HNSW ef. The MemWAL LSM
* scanner does not yet support this option.
*
* @example
* ```ts
* table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10);
* ```
*/
searchEffort(searchEffort: number | null): VectorQuery {
this.doVectorCall((inner) => inner.searchEffort(searchEffort));
return this;
}
/**
* Set the minimum number of probes used.
*
nodejs/npm/darwin-arm64/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.41.0-beta.1",
"version": "0.41.0-beta.6",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
nodejs/npm/linux-arm64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.41.0-beta.1",
"version": "0.41.0-beta.6",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
nodejs/npm/linux-arm64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.41.0-beta.1",
"version": "0.41.0-beta.6",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
nodejs/npm/linux-x64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.41.0-beta.1",
"version": "0.41.0-beta.6",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
nodejs/npm/linux-x64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.41.0-beta.1",
"version": "0.41.0-beta.6",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
nodejs/npm/win32-arm64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.41.0-beta.1",
"version": "0.41.0-beta.6",
"os": [
"win32"
],
nodejs/npm/win32-x64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.41.0-beta.1",
"version": "0.41.0-beta.6",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
nodejs/package.json
+1 -1
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.41.0-beta.1",
"version": "0.41.0-beta.6",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
nodejs/src/query.rs
+10
View File
@@ -298,6 +298,16 @@ impl VectorQuery {
Ok(())
}
#[napi]
pub fn search_effort(&mut self, search_effort: Option<f64>) -> napi::Result<()> {
self.inner = self
.inner
.clone()
.search_effort(search_effort)
.default_error()?;
Ok(())
}
#[napi]
pub fn minimum_nprobes(&mut self, minimum_nprobe: u32) -> napi::Result<()> {
self.inner = self
python/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.41.0-beta.1"
version = "0.41.0-beta.6"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
python/pyproject.toml
+3 -2
View File
@@ -10,7 +10,7 @@ dependencies = [
"pyarrow>=16",
"pydantic>=2.7.4,<3",
"tqdm>=4.27.0",
"lance-namespace>=0.3.2"
"lance-namespace>=0.13.0"
]
description = "lancedb"
authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]
@@ -63,7 +63,7 @@ tests = [
"polars>=0.19, <=1.32.3",
"pyarrow<25",
"pyarrow-stubs>=16.0",
"pylance==14.0.0b6",
"pylance==14.0.0b12",
"requests>=2.31.0",
"datafusion>=54,<55",
"opentelemetry-sdk>=1.30.0",
@@ -73,6 +73,7 @@ dev = [
"pre-commit>=3.5.0",
"pyright>=1.1.350",
'typing-extensions>=4.0.0; python_version < "3.11"',
"maturin>=1.10",
]
docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings-python"]
clip = ["torch", "pillow>=12.1.1", "open-clip-torch"]
python/python/lancedb/_blob.py
+1
View File
@@ -328,6 +328,7 @@ def _set_blob_column(
leaf_name, blobs.type, nullable=field.nullable
)
return
raise KeyError(f"Blob field '{leaf_name}' not found in struct '{parent_path}'")
return _transform_struct_column(tbl, parent_path, replace_blob)
python/python/lancedb/_lancedb.pyi
+4
View File
@@ -727,6 +727,7 @@ class VectorQuery:
def use_lsm(self, enable: bool): ...
def refine_factor(self, refine_factor: int): ...
def nprobes(self, nprobes: int): ...
def search_effort(self, search_effort: Optional[float]): ...
def minimum_nprobes(self, minimum_nprobes: int): ...
def maximum_nprobes(self, maximum_nprobes: int): ...
def bypass_vector_index(self): ...
@@ -752,6 +753,7 @@ class HybridQuery:
def distance_type(self, distance_type: str): ...
def refine_factor(self, refine_factor: int): ...
def nprobes(self, nprobes: int): ...
def search_effort(self, search_effort: Optional[float]): ...
def minimum_nprobes(self, minimum_nprobes: int): ...
def maximum_nprobes(self, maximum_nprobes: int): ...
def bypass_vector_index(self): ...
@@ -778,8 +780,10 @@ class PyQueryRequest:
use_lsm: Optional[bool]
column: Optional[str]
query_vector: Optional[List[pa.Array]]
nprobes: Optional[int]
minimum_nprobes: Optional[int]
maximum_nprobes: Optional[int]
search_effort: Optional[float]
lower_bound: Optional[float]
upper_bound: Optional[float]
ef: Optional[int]
python/python/lancedb/functions.py
+1 -1
View File
@@ -372,7 +372,7 @@ class FunctionVersion(_RemoteValue):
image: FunctionImage
signature: FunctionSignature
secret_bindings: tuple[SecretBinding, ...] = ()
created_at: str
created_at_millis: int
metadata: Mapping[str, str]
disabled: bool
python/python/lancedb/index.py
+35 -2
View File
@@ -112,9 +112,42 @@ class Fm:
class FTS:
"""Describe a FTS index configuration.
`FTS` is a full-text search index that can be used on `String` columns
`FTS` supports string columns, lists of strings, and JSON columns created
from PyArrow ``pa.json_()``. JSON columns use Lance's JSON-aware tokenizer
and are stored as ``LargeBinary`` with the ``lance.json`` logical type.
Arbitrary binary columns are not supported.
For example, it works with `title`, `description`, `content`, etc.
JSON queries use ``path,str,value`` triples. Paths are relative to the JSON
column and use dots for nested object keys. For example,
``meta.tag,str,nature`` matches the string value at ``$.meta.tag``;
it does not search other paths or JSON key names. The value is processed
with the configured text tokenizer. Use ``str``, not ``string``.
Create and query a JSON index through the modern API::
import pyarrow as pa
from lancedb.index import FTS
table = db.create_table(
"documents",
pa.table({"doc": pa.array(
['{"meta":{"tag":"nature"}}'], type=pa.json_()
)}),
)
table.create_index("doc", config=FTS())
table.search(
"meta.tag,str,nature", query_type="fts", fts_columns="doc"
).to_list()
Create the index on the JSON column itself, not a JSON subpath such as
``doc.meta.tag``. JSON-path scalar indexes do not provide FTS path scoping.
Plain text such as ``nature`` is not a valid JSON FTS query. The current
Lance tokenizer can panic on malformed triples; see
https://github.com/lance-format/lance/issues/8812.
Remote tables use the same ``FTS()`` configuration and require a server
that supports JSON FTS. Wait for ``table.wait_for_index(["doc_idx"])``
before searching a newly created remote index with its default name.
Examples
--------
python/python/lancedb/namespace.py
+17 -1
View File
@@ -105,6 +105,11 @@ def _query_to_namespace_request(
if not isinstance(query, Query):
raise TypeError(f"Expected Query, got {type(query)}")
if query.search_effort is not None:
raise ValueError(
"namespace QueryTable does not support search_effort; "
"execute the query locally"
)
vector_request = None
if query.vector is not None:
@@ -190,8 +195,12 @@ def _query_to_namespace_request(
kwargs["with_row_id"] = query.with_row_id
if query.ef is not None:
kwargs["ef"] = query.ef
if query.nprobes is not None:
kwargs["nprobes"] = query.nprobes
if query.minimum_nprobes is not None:
kwargs["nprobes"] = query.minimum_nprobes
kwargs["minimum_nprobes"] = query.minimum_nprobes
if query.maximum_nprobes is not None:
kwargs["maximum_nprobes"] = query.maximum_nprobes
if query.refine_factor is not None:
kwargs["refine_factor"] = query.refine_factor
if query.lower_bound is not None:
@@ -205,6 +214,13 @@ def _query_to_namespace_request(
if query.distance_type is not None:
kwargs["distance_type"] = query.distance_type
if kwargs.get("maximum_nprobes") == 0:
# The generated 0.13 model rejects LanceDB's unbounded sentinel;
# model_copy preserves direct wire pass-through to the Lance receiver.
maximum_nprobes = kwargs.pop("maximum_nprobes")
return QueryTableRequest(**kwargs).model_copy(
update={"maximum_nprobes": maximum_nprobes}
)
return QueryTableRequest(**kwargs)
python/python/lancedb/query.py
+118 -22
View File
@@ -551,8 +551,8 @@ class MatchQuery(FullTextQuery):
----------
query : str
The query string to match against.
column : str
The name of the column to match against.
column : str, optional
The column to match against. None leaves column selection to query planning.
boost : float, default 1.0
The boost factor for the query.
The score of each matching document is multiplied by this value.
@@ -581,7 +581,7 @@ class MatchQuery(FullTextQuery):
"""
query: str
column: str
column: Optional[str] = None
boost: float = pydantic.Field(1.0, kw_only=True)
fuzziness: Optional[int] = pydantic.Field(0, kw_only=True)
max_expansions: int = pydantic.Field(50, kw_only=True)
@@ -769,8 +769,9 @@ class Query(pydantic.BaseModel):
if True then apply the filter after vector / FTS search. This is ignored for
plain SQL filtering.
nprobes : Optional[int]
The number of IVF partitions to search. If this is None then a default
number of partitions will be used.
The legacy number of IVF partitions to search. Lance sets both probe
bounds to this value. If this is None then Lance's default probe settings
will be used.
- A higher number makes search more accurate but also slower.
@@ -849,22 +850,28 @@ class Query(pydantic.BaseModel):
# distance type to use for vector search
distance_type: Optional[str] = None
# legacy number of IVF partitions to search
#
# Lance sets both probe bounds to this value. Explicit bounds can override it.
nprobes: Optional[int] = None
# which columns to return in the results (dict values may be str or Expr)
columns: QueryProjection = None
# minimum number of IVF partitions to search
#
# If None then a default value (20) will be used.
# If None then Lance's default will be used.
minimum_nprobes: Optional[int] = None
# maximum number of IVF partitions to search
#
# If None then a default value (20) will be used.
#
# If 0 then no limit will be applied and all partitions could be searched
# if needed to satisfy the limit.
# If None then Lance's default will be used. If 0 then no limit will be applied
# and all partitions could be searched if needed to satisfy the limit.
maximum_nprobes: Optional[int] = None
# Optional initial IVF effort; None preserves Auto and explicit nprobes.
search_effort: Optional[float] = None
# lower bound for distance search
lower_bound: Optional[float] = None
@@ -923,8 +930,10 @@ class Query(pydantic.BaseModel):
query.vector_column = req.column
query.vector = req.query_vector
query.distance_type = req.distance_type
query.nprobes = req.nprobes
query.minimum_nprobes = req.minimum_nprobes
query.maximum_nprobes = req.maximum_nprobes
query.search_effort = req.search_effort
query.lower_bound = req.lower_bound
query.upper_bound = req.upper_bound
query.ef = req.ef
@@ -1715,8 +1724,10 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
super().__init__(table)
self._query = query
self._distance_type = None
self._nprobes = None
self._minimum_nprobes = None
self._maximum_nprobes = None
self._search_effort = None
self._lower_bound = None
self._upper_bound = None
self._refine_factor = None
@@ -1779,9 +1790,9 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
See discussion in [Querying an ANN Index](https://lancedb.com/docs/indexing/)
for tuning advice.
This method sets both the minimum and maximum number of probes to the same
value. See `minimum_nprobes` and `maximum_nprobes` for more fine-grained
control.
The value is retained as `nprobes` through client and server request
construction. Lance sets both probe bounds to this value. Explicit minimum
or maximum settings can override their respective bounds.
Parameters
----------
@@ -1795,8 +1806,28 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
"""
if nprobes <= 0:
raise ValueError("Invalid input, nprobes must be greater than 0")
self._minimum_nprobes = nprobes
self._maximum_nprobes = nprobes
self._nprobes = nprobes
return self
def search_effort(self, search_effort: Optional[float]) -> LanceVectorQueryBuilder:
"""Set the initial IVF search effort to a finite value in [0, 1].
None (the default) uses Auto, internally 0.5. Zero starts at the caller
minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found. Any explicit
effort, including 0.5, overrides nprobes in either call order. Explicit
minimum and maximum bounds still apply. This is not a recall guarantee
and does not change refinement or HNSW ef. MemWAL LSM queries do not
yet support this option.
Examples
--------
>>> import lancedb
>>> db = lancedb.connect("memory://")
>>> table = db.create_table("effort_vectors", [{"vector": [0.1, 0.2]}])
>>> query = table.search([0.1, 0.2]).search_effort(0.75)
"""
self._search_effort = search_effort
return self
def minimum_nprobes(self, minimum_nprobes: int) -> LanceVectorQueryBuilder:
@@ -1938,8 +1969,10 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
limit=self._limit,
distance_type=self._distance_type,
columns=self._columns,
nprobes=self._nprobes,
minimum_nprobes=self._minimum_nprobes,
maximum_nprobes=self._maximum_nprobes,
search_effort=self._search_effort,
lower_bound=self._lower_bound,
upper_bound=self._upper_bound,
refine_factor=self._refine_factor,
@@ -2296,8 +2329,10 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
self._fts_columns = fts_columns
self._norm = None
self._reranker = None
self._nprobes = None
self._minimum_nprobes = None
self._maximum_nprobes = None
self._search_effort = None
self._refine_factor = None
self._distance_type = None
self._phrase_query = None
@@ -2587,6 +2622,9 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
Higher values will yield better recall (more likely to find vectors if
they exist) at the expense of latency.
The value is retained as `nprobes` until Lance sets both probe bounds to it.
Explicit minimum or maximum settings can override their respective bounds.
Parameters
----------
nprobes: int
@@ -2599,8 +2637,33 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
"""
if nprobes <= 0:
raise ValueError("Invalid input, nprobes must be greater than 0")
self._minimum_nprobes = nprobes
self._maximum_nprobes = nprobes
self._nprobes = nprobes
return self
def search_effort(self, search_effort: Optional[float]) -> LanceHybridQueryBuilder:
"""Set the initial IVF search effort to a finite value in [0, 1].
None (the default) uses Auto, internally 0.5. Zero starts at the caller
minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found. Any explicit
effort, including 0.5, overrides nprobes in either call order. Explicit
minimum and maximum bounds still apply. This is not a recall guarantee
and does not change refinement or HNSW ef. MemWAL LSM queries do not
yet support this option.
Examples
--------
>>> import lancedb
>>> db = lancedb.connect("memory://")
>>> table = db.create_table(
... "effort_hybrid", [{"vector": [0.1, 0.2], "text": "dog"}]
... )
>>> query = (
... table.search(query_type="hybrid").vector([0.1, 0.2])
... .text("dog").search_effort(0.75)
... )
"""
self._search_effort = search_effort
return self
def minimum_nprobes(self, minimum_nprobes: int) -> LanceHybridQueryBuilder:
@@ -2879,6 +2942,10 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
self._fts_query.phrase_query(True)
if self._distance_type:
self._vector_query.metric(self._distance_type)
if self._nprobes is not None:
self._vector_query.nprobes(self._nprobes)
if self._search_effort is not None:
self._vector_query.search_effort(self._search_effort)
if self._minimum_nprobes is not None:
self._vector_query.minimum_nprobes(self._minimum_nprobes)
if self._maximum_nprobes is not None:
@@ -3782,7 +3849,7 @@ class AsyncVectorQueryBase:
def nprobes(self, nprobes: int) -> Self:
"""
Set the number of partitions to search (probe)
Set the legacy IVF probe parameter
The number of probes must be greater than 0.
@@ -3794,20 +3861,49 @@ class AsyncVectorQueryBase:
The partition whose centroids are closest to the query vector will be
exhaustiely searched to find matches. This parameter controls how many
partitions should be searched.
partitions will be searched.
Increasing this value will increase the recall of your query but will
also increase the latency of your query. The default value is 20. This
default is good for many cases but the best value to use will depend on
your data and the recall that you need to achieve.
also increase the latency of your query. If this method is not called,
Lance's adaptive probe defaults are used.
For best results we recommend tuning this parameter with a benchmark against
your actual data to find the smallest possible value that will still give
you the desired recall.
LanceDB retains this as `nprobes` through local and remote request
construction. Lance sets both probe bounds to this value. Explicit minimum
or maximum settings can override their respective bounds.
"""
self._inner.nprobes(nprobes)
return self
def search_effort(self, search_effort: Optional[float]) -> Self:
"""Set the initial IVF search effort to a finite value in [0, 1].
None preserves Auto (internally 0.5) and fixed nprobes. Any explicit
effort, including 0.5, overrides nprobes in either call order; explicit
minimum and maximum bounds remain effective. Zero starts at the caller
minimum and one searches all partitions within the maximum. Later
probing may continue to obtain enough candidates. This is not a recall
guarantee and does not change refinement or HNSW ef. MemWAL LSM queries
do not yet support this option.
Examples
--------
>>> import asyncio
>>> import lancedb
>>> async def build_query():
... db = await lancedb.connect_async("memory://")
... table = await db.create_table(
... "effort_async", [{"vector": [0.1, 0.2]}]
... )
... return table.vector_search([0.1, 0.2]).search_effort(0.75)
>>> query = asyncio.run(build_query())
"""
self._inner.search_effort(search_effort)
return self
def minimum_nprobes(self, minimum_nprobes: int) -> Self:
"""Set the minimum number of probes to use.
python/python/lancedb/table.py
+7 -2
View File
@@ -4440,6 +4440,7 @@ class LanceTable(Table):
and not self._route_pushdown_to_rust
and self.current_branch() is None
and query.take_offsets is None
and query.search_effort is None
):
from lancedb.namespace import _execute_server_side_query
@@ -6439,9 +6440,12 @@ class AsyncTable:
)
if query.distance_type is not None:
async_query = async_query.distance_type(query.distance_type)
if query.nprobes is not None:
async_query = async_query.nprobes(query.nprobes)
if query.search_effort is not None:
async_query = async_query.search_effort(query.search_effort)
if query.minimum_nprobes is not None and query.maximum_nprobes is not None:
# Set both to the minimum first to avoid min > max error.
async_query = async_query.nprobes(
async_query = async_query.minimum_nprobes(
query.minimum_nprobes
).maximum_nprobes(query.maximum_nprobes)
elif query.minimum_nprobes is not None:
@@ -6480,6 +6484,7 @@ class AsyncTable:
)
and not self._route_pushdown_to_rust
and query.take_offsets is None
and query.search_effort is None
):
from lancedb.namespace import _execute_server_side_query
python/python/lancedb/util.py
+9
View File
@@ -10,6 +10,7 @@ import os
import pathlib
import warnings
from datetime import date, datetime
from decimal import Decimal
from functools import singledispatch
from typing import Tuple, Union, Optional, Any, List
from urllib.parse import urlparse
@@ -373,6 +374,14 @@ def _(value: float):
return str(value)
@value_to_sql.register(Decimal)
def _(value: Decimal):
if not value.is_finite():
raise ValueError("Non-finite Decimal values cannot be converted to SQL")
# Quote exact decimal text so the target-column cast avoids float rounding.
return value_to_sql(format(value, "f"))
@value_to_sql.register(bool)
def _(value: bool):
return str(value).upper()
python/python/tests/test_blob.py
+16
View File
@@ -15,6 +15,7 @@ from lance.blob import Blob, BlobType as LanceBlobType
import lancedb
from lancedb._blob import (
_set_blob_column,
blob_v2_projection_sources,
read_row_ids_from_hits,
stash_auto_row_ids,
@@ -312,6 +313,21 @@ def test_blob_v2_projection_sources_include_selected_struct():
}
@pytest.mark.parametrize("parent_path", ["info", "outer.info"])
def test_set_blob_column_missing_nested_field_raises(parent_path):
row = {"name": "first"}
for name in reversed(parent_path.split(".")):
row = {name: row}
table = pa.Table.from_pylist([row])
blobs = pa.array([b"hello"], type=pa.large_binary())
with pytest.raises(KeyError) as exc_info:
_set_blob_column(table, f"{parent_path}.blob", blobs)
assert parent_path in str(exc_info.value)
assert "'blob'" in str(exc_info.value)
def _nested_blob_table(db, name):
schema = pa.schema(
[
python/python/tests/test_e2e_remote_db.py
+65
View File
@@ -2,13 +2,78 @@
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import os
from datetime import timedelta
from uuid import uuid4
import lancedb
import numpy as np
import pyarrow as pa
import pytest
from lancedb import LanceDBConnection
from lancedb.index import FTS
# TODO: setup integ test mark and script
@pytest.mark.skipif(
not os.environ.get("LANCEDB_URI"), reason="Set LANCEDB_URI to a JSON FTS server"
)
def test_json_fts_against_server():
"""Requires a writable database and capacity to run the distributed indexer.
Configure LANCEDB_URI, LANCEDB_API_KEY, and optionally LANCEDB_REGION and
LANCEDB_HOST_OVERRIDE before running this test.
"""
uri = os.environ["LANCEDB_URI"]
assert uri.startswith("db://"), "LANCEDB_URI must identify a remote database"
conn = lancedb.connect(
uri,
api_key=os.environ.get("LANCEDB_API_KEY"),
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
host_override=os.environ.get("LANCEDB_HOST_OVERRIDE"),
)
name = f"json_fts_{uuid4().hex}"
table = conn.create_table(
name,
pa.table(
{
"id": [0, 1, 2],
"doc": pa.array(
[
'{"meta":{"tag":"nature"},"author":"alice"}',
'{"meta":{"tag":"city"},"author":"nature"}',
'{"meta":{"tag":"nature"},"author":"bob"}',
],
type=pa.json_(),
),
"raw": pa.array([b"nature"] * 3, type=pa.large_binary()),
}
),
)
try:
table.create_index("doc", config=FTS(), name="doc_fts")
table.wait_for_index(["doc_fts"], timeout=timedelta(minutes=5))
stats = table.index_stats("doc_fts")
assert stats.num_indexed_rows == 3
assert stats.num_unindexed_rows == 0
assert stats.index_type == "FTS"
for query, expected in [
("meta.tag,str,nature", [0, 2]),
("meta.tag,str,alice", []),
("meta.tag,str,tag", []),
]:
search = table.search(query, query_type="fts", fts_columns="doc")
assert sorted(row["id"] for row in search.to_list()) == expected
plan = search.explain_plan(True)
assert "MatchQuery" in plan, plan
assert "FlatMatchQuery" not in plan, plan
with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"):
table.create_index("raw", config=FTS())
finally:
conn.drop_table(name)
@pytest.mark.skip(reason="Need to set up a local server")
def test_against_local_server():
conn = LanceDBConnection("lancedb+http://localhost:10024")
python/python/tests/test_first_class_function_slice2.py
+1 -1
View File
@@ -1555,7 +1555,7 @@ def _mock_remote_function_catalog():
)["image"],
"signature": body["signature"],
"secret_bindings": body.get("secret_bindings", []),
"created_at": "2026-08-21T00:00:00Z",
"created_at_millis": 1787270400000, # 2026-08-21T00:00:00Z
}
response = {"job_id": "job-register"}
status = 202
python/python/tests/test_fts.py
+69
View File
@@ -45,6 +45,75 @@ from utils import exception_output
TEST_LANGUAGE_MODEL_HOME = Path(__file__).parent / "models"
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
@pytest.mark.parametrize("block_size", [128, 256])
def test_json_fts(tmp_path, block_size):
db = ldb.connect(tmp_path)
data = pa.table(
{
"id": [0, 1, 2, 3],
"doc": pa.array(
[
'{"title":"brown fox","meta":{"tag":"nature"},"author":"alice"}',
'{"title":"blue car","meta":{"tag":"city"}}',
'{"title":"brown bear","meta":{"tag":"nature"}}',
'{"title":"red fox","author":"alice"}',
],
type=pa.json_(),
),
}
)
table = db.create_table("json_fts", data)
table.create_index("doc", config=FTS(block_size=block_size), name="doc_fts")
table = db.open_table("json_fts")
index = table.list_indices()[0]
assert index.name == "doc_fts"
assert index.index_type == "FTS"
assert index.columns == ["doc"]
stats = table.index_stats("doc_fts")
assert stats.num_indexed_rows == 4
assert stats.num_unindexed_rows == 0
for query, expected in [
("title,str,brown", [0, 2]),
("meta.tag,str,nature", [0, 2]),
("author,str,alice", [0, 3]),
("title,str,alice", []),
("title,str,title", []),
]:
search = table.search(query, query_type="fts", fts_columns="doc")
assert sorted(row["id"] for row in search.to_list()) == expected
plan = search.explain_plan(True)
assert "MatchQuery" in plan, plan
assert "FlatMatchQuery" not in plan, plan
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
@pytest.mark.asyncio
async def test_json_fts_async(tmp_path):
db = await ldb.connect_async(tmp_path)
table = await db.create_table(
"json_fts",
pa.table({"doc": pa.array(['{"meta":{"tag":"nature"}}'], type=pa.json_())}),
)
await table.create_index("doc", config=FTS())
search = table.query().nearest_to_text(MatchQuery("meta.tag,str,nature", "doc"))
assert len(await search.to_list()) == 1
plan = await search.explain_plan(True)
assert "MatchQuery" in plan, plan
assert "FlatMatchQuery" not in plan, plan
@pytest.mark.parametrize("dtype", [pa.binary(), pa.large_binary()])
def test_fts_rejects_raw_binary(tmp_path, dtype):
table = ldb.connect(tmp_path).create_table(
"binary_fts", pa.table({"doc": pa.array([b'{"title":"brown"}'], type=dtype)})
)
with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"):
table.create_index("doc", config=FTS())
assert table.list_indices() == []
@pytest.fixture
def table(tmp_path) -> ldb.table.LanceTable:
# Use local random state to avoid affecting other tests
python/python/tests/test_hybrid_query.py
+14
View File
@@ -355,6 +355,20 @@ def test_hybrid_query_minimum_nprobes_zero_raises(sync_table: Table):
)
def test_hybrid_query_nprobes_remains_independent(sync_table: Table):
query = sync_table.search(query_type="hybrid").nprobes(20)
assert query._nprobes == 20
assert query._minimum_nprobes is None
assert query._maximum_nprobes is None
query = query.minimum_nprobes(5).nprobes(20)
assert query._nprobes == 20
assert query._minimum_nprobes == 5
assert query._maximum_nprobes is None
def test_hybrid_query_distance_range(sync_table: Table):
reranker = RRFReranker(return_score="all")
result = (
python/python/tests/test_namespace.py
+37 -1
View File
@@ -15,7 +15,8 @@ import pyarrow as pa
import lancedb
from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
from lancedb.listing import AsyncListing, Listing
from lancedb.namespace import _MAX_QUERY_K
from lancedb.namespace import _MAX_QUERY_K, _query_to_namespace_request
from lancedb.query import Query
from lancedb.table import AsyncTable, LanceTable
@@ -1111,6 +1112,29 @@ class TestPushdownOperations:
]
assert all(r.k <= 2**31 - 1 for r in namespace_client.requests)
def test_probe_fields_pass_through_independently(self):
request = _query_to_namespace_request(
["geneva", "hist"],
Query(
vector=[1.0, 2.0],
nprobes=20,
minimum_nprobes=3,
maximum_nprobes=10,
),
)
assert request.nprobes == 20
assert request.minimum_nprobes == 3
assert request.maximum_nprobes == 10
request = _query_to_namespace_request(
["geneva", "hist"],
Query(vector=[1.0, 2.0], nprobes=5, maximum_nprobes=0),
)
assert request.nprobes == 5
assert request.maximum_nprobes == 0
assert request.to_dict()["maximum_nprobes"] == 0
@pytest.mark.asyncio
class TestAsyncPushdownOperations:
@@ -1185,3 +1209,15 @@ def test_local_table_to_arrow_and_to_pandas_are_unchanged(tmp_path):
assert table.to_arrow().column("id").to_pylist() == [1, 2]
assert table.to_pandas()["id"].tolist() == [1, 2]
@pytest.mark.parametrize("effort", [0.0, 0.5, 1.0])
def test_namespace_request_rejects_search_effort(effort):
from lancedb.query import Query
with pytest.raises(
ValueError, match="namespace QueryTable does not support search_effort"
):
_query_to_namespace_request(
["test"], Query(vector=[1.0, 0.0], search_effort=effort)
)
python/python/tests/test_query.py
+212 -28
View File
@@ -1174,12 +1174,22 @@ def test_invalid_nprobes_sync(table):
ValueError,
match="maximum_nprobes must be greater than or equal to minimum_nprobes",
):
LanceVectorQueryBuilder(table, [0, 0], "vector").maximum_nprobes(5).to_list()
(
LanceVectorQueryBuilder(table, [0, 0], "vector")
.minimum_nprobes(10)
.maximum_nprobes(5)
.to_list()
)
with pytest.raises(
ValueError,
match="minimum_nprobes must be less than or equal to maximum_nprobes",
match="maximum_nprobes must be greater than or equal to minimum_nprobes",
):
LanceVectorQueryBuilder(table, [0, 0], "vector").minimum_nprobes(100).to_list()
(
LanceVectorQueryBuilder(table, [0, 0], "vector")
.maximum_nprobes(5)
.minimum_nprobes(100)
.to_list()
)
def test_nprobes_works_sync(table):
@@ -1218,12 +1228,22 @@ async def test_invalid_nprobes_async(table_async: AsyncTable):
ValueError,
match="maximum_nprobes must be greater than or equal to minimum_nprobes",
):
await table_async.vector_search([0, 0]).maximum_nprobes(5).to_list()
await (
table_async.vector_search([0, 0])
.minimum_nprobes(10)
.maximum_nprobes(5)
.to_list()
)
with pytest.raises(
ValueError,
match="minimum_nprobes must be less than or equal to maximum_nprobes",
):
await table_async.vector_search([0, 0]).minimum_nprobes(100).to_list()
await (
table_async.vector_search([0, 0])
.maximum_nprobes(5)
.minimum_nprobes(100)
.to_list()
)
def test_query_builder_with_prefilter(table):
@@ -1814,8 +1834,9 @@ def test_query_serialization_sync(table: lancedb.table.Table):
q,
vector_column="vector",
vector=[5.0, 6.0],
minimum_nprobes=10,
maximum_nprobes=10,
nprobes=10,
minimum_nprobes=None,
maximum_nprobes=None,
refine_factor=5,
)
@@ -1833,8 +1854,19 @@ def test_query_serialization_sync(table: lancedb.table.Table):
q,
vector_column="vector",
vector=[5.0, 6.0],
minimum_nprobes=50,
maximum_nprobes=50,
nprobes=50,
minimum_nprobes=None,
maximum_nprobes=None,
)
q = table.search([5.0, 6.0]).minimum_nprobes(5).nprobes(50).to_query_object()
check_set_props(
q,
vector_column="vector",
vector=[5.0, 6.0],
nprobes=50,
minimum_nprobes=5,
maximum_nprobes=None,
)
q = table.search([5.0, 6.0]).maximum_nprobes(10).to_query_object()
@@ -1846,6 +1878,15 @@ def test_query_serialization_sync(table: lancedb.table.Table):
minimum_nprobes=None,
)
q = table.search([5.0, 6.0]).nprobes(5).maximum_nprobes(0).to_query_object()
check_set_props(
q,
vector_column="vector",
vector=[5.0, 6.0],
nprobes=5,
maximum_nprobes=0,
)
q = table.search([5.0, 6.0]).distance_range(0.0, 1.0).to_query_object()
check_set_props(
q, vector_column="vector", vector=[5.0, 6.0], lower_bound=0.0, upper_bound=1.0
@@ -1895,8 +1936,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
limit=10,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
)
@@ -1906,8 +1947,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
q,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1918,8 +1959,9 @@ async def test_query_serialization_async(table_async: AsyncTable):
q,
vector=sample_vector,
postfilter=False,
minimum_nprobes=50,
maximum_nprobes=50,
nprobes=50,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1938,8 +1980,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
filter="id = 1",
postfilter=True,
vector=sample_vector,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
)
@@ -1953,8 +1995,9 @@ async def test_query_serialization_async(table_async: AsyncTable):
check_set_props(
q,
vector=sample_vector,
minimum_nprobes=10,
maximum_nprobes=10,
nprobes=10,
minimum_nprobes=None,
maximum_nprobes=None,
refine_factor=5,
postfilter=False,
with_row_id=False,
@@ -1967,7 +2010,24 @@ async def test_query_serialization_async(table_async: AsyncTable):
q,
vector=sample_vector,
minimum_nprobes=5,
maximum_nprobes=20,
maximum_nprobes=None,
postfilter=False,
with_row_id=False,
bypass_vector_index=False,
limit=10,
)
q = (
(await table_async.search([5.0, 6.0]))
.nprobes(5)
.maximum_nprobes(0)
.to_query_object()
)
check_set_props(
q,
vector=sample_vector,
nprobes=5,
maximum_nprobes=0,
postfilter=False,
with_row_id=False,
bypass_vector_index=False,
@@ -1985,8 +2045,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
lower_bound=0.0,
upper_bound=1.0,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1998,8 +2058,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
distance_type="cosine",
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -2011,8 +2071,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
ef=7,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -2024,8 +2084,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
bypass_vector_index=True,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
limit=10,
)
@@ -2384,3 +2444,127 @@ def test_blob_v2_to_batches_row_id(tmp_db):
assert "_rowid" in hits.column_names
blobs = table.fetch_blobs("blob", hits)
assert [blobs[i].as_py() for i in range(len(blobs))] == [b"one", b"two"]
@pytest.mark.parametrize("metric", ["l2", "cosine", "dot"])
@pytest.mark.parametrize(
"effort,probes", [(None, 4), (0, 1), (0.25, 2), (0.5, 4), (0.75, 8), (1, 16)]
)
def test_search_effort_probe_budget(tmp_path, monkeypatch, metric, effort, probes):
import re
from lancedb.index import IvfFlat
monkeypatch.setenv("LANCE_AUTO_PROBE_MARGIN", "0")
monkeypatch.setenv("LANCE_AUTO_MIN_INITIAL_NPROBES", "4")
monkeypatch.setenv("LANCE_AUTO_MAX_INITIAL_NPROBES", "4")
rng = np.random.default_rng(42)
vectors = np.repeat(np.eye(16, dtype=np.float32), 32, axis=0)
vectors += rng.normal(0, 0.001, vectors.shape).astype(np.float32)
data = pa.table(
{
"id": np.arange(512),
"text": ["dog"] * 512,
"vector": pa.FixedSizeListArray.from_arrays(pa.array(vectors.ravel()), 16),
}
)
table = lancedb.connect(tmp_path).create_table("effort", data.slice(0, 256))
table.add(data.slice(256))
table.create_index(
"vector", config=IvfFlat(distance_type=metric, num_partitions=16)
)
vector = np.eye(16, dtype=np.float32)[0]
def search(value=effort):
return table.search(vector).distance_type(metric).search_effort(value).limit(8)
def partitions(query):
vector_plan = query.analyze_plan().split("FTS Search Plan:", 1)[0]
values = re.findall(r"partitions_searched=(\d+)", vector_plan)
assert values
return sum(map(int, values))
assert partitions(search()) == probes
exact = set(
table.search(vector)
.distance_type(metric)
.bypass_vector_index()
.limit(8)
.to_arrow()["id"]
.to_pylist()
)
actual = set(search().to_arrow()["id"].to_pylist())
assert len(actual & exact) / 8 >= 0.5
expected = 1 if effort is None else probes
assert partitions(search().nprobes(1)) == expected
assert (
partitions(
table.search(vector)
.distance_type(metric)
.nprobes(1)
.search_effort(effort)
.limit(8)
)
== expected
)
assert partitions(search().nprobes(1).search_effort(None)) == 1
assert partitions(search().maximum_nprobes(2)) <= 2
assert partitions(search().minimum_nprobes(3).maximum_nprobes(3)) == 3
assert partitions(search(0).nprobes(16).maximum_nprobes(0)) == 1
if effort is None:
assert search().to_arrow() == search(0.5).to_arrow()
filtered = search(0).maximum_nprobes(2).where("id % 8 = 0", prefilter=True)
assert partitions(filtered) <= 2
assert all(row["id"] % 8 == 0 for row in filtered.to_list())
if metric == "l2" and effort == 0.5:
table.create_index("text", config=FTS())
hybrid = (
table.search(query_type="hybrid")
.vector(vector)
.text("dog")
.nprobes(1)
.search_effort(0.5)
.limit(8)
)
assert partitions(hybrid) == 4
assert partitions(hybrid.search_effort(None)) == 1
@pytest.mark.parametrize(
"effort", [-0.1, 1.1, float("nan"), float("inf"), -float("inf")]
)
def test_search_effort_invalid_sync(table, effort):
with pytest.raises(ValueError, match="search_effort must be finite and in"):
table.search([0, 0]).search_effort(effort).to_list()
@pytest.mark.asyncio
@pytest.mark.parametrize("hybrid", [False, True])
@pytest.mark.parametrize("effort", [None, 0, 0.25, 0.5, 0.75, 1])
async def test_search_effort_async_roundtrip(table_async, hybrid, effort):
query = table_async.vector_search([0, 0])
if hybrid:
query = query.nearest_to_text("dog")
query = query.nprobes(5).search_effort(effort).minimum_nprobes(2).maximum_nprobes(3)
request = query.to_query_object()
assert request.search_effort == effort
assert request.nprobes == 5
assert request.minimum_nprobes == 2
assert request.maximum_nprobes == 3
assert query.search_effort(None).to_query_object().search_effort is None
query.search_effort(effort)
if not hybrid:
assert len(await query.limit(1).to_list()) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("hybrid", [False, True])
@pytest.mark.parametrize(
"effort", [-0.1, 1.1, float("nan"), float("inf"), -float("inf")]
)
async def test_search_effort_invalid_async(table_async, hybrid, effort):
query = table_async.vector_search([0, 0])
if hybrid:
query = query.nearest_to_text("dog")
with pytest.raises(ValueError, match="search_effort must be finite and in"):
query.search_effort(effort)
python/python/tests/test_remote_db.py
+89 -14
View File
@@ -1410,6 +1410,72 @@ def test_remote_job_wait_raises_on_failure():
job.wait()
@pytest.mark.parametrize("server_version", ["0.2.0", "0.6.0"])
def test_remote_json_fts_config(server_version):
from lancedb.index import FTS
received_requests = []
def handler(request):
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.send_header("phalanx-version", server_version)
request.end_headers()
if request.path == "/v1/table/test/describe/":
fields = [
{
"name": "doc",
"type": {"type": "large_binary"},
"nullable": True,
"metadata": {"ARROW:extension:name": "lance.json"},
},
{
"name": "raw",
"type": {"type": "large_binary"},
"nullable": True,
},
]
request.wfile.write(
json.dumps({"version": 1, "schema": {"fields": fields}}).encode()
)
elif request.path == "/v1/table/test/create_index/":
received_requests.append(read_json_body(request))
request.wfile.write(b"{}")
else:
raise AssertionError(f"Unexpected request: {request.path}")
with mock_lancedb_connection(handler) as db:
table = db.open_table("test")
table.create_index(
"doc",
config=FTS(
with_position=True,
base_tokenizer="whitespace",
stem=False,
remove_stop_words=False,
lower_case=False,
block_size=256,
),
name="doc_fts",
replace=False,
)
with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"):
table.create_index("raw", config=FTS())
assert len(received_requests) == 1
request = received_requests[0]
assert request["column"] == "doc"
assert request["index_type"] == "FTS"
assert request["name"] == "doc_fts"
assert request["replace"] is False
assert request["with_position"] is True
assert request["base_tokenizer"] == "whitespace"
assert request["stem"] is False
assert request["remove_stop_words"] is False
assert request["lower_case"] is False
assert request["block_size"] == 256
def test_remote_create_index_new_api(caplog):
received_requests = []
@@ -2098,9 +2164,6 @@ def test_query_sync_minimal():
"ef": None,
"vector": [1.0, 2.0, 3.0],
"vector_column": "vector",
"nprobes": 20,
"minimum_nprobes": 20,
"maximum_nprobes": 20,
"version": None,
}
@@ -2272,8 +2335,6 @@ def test_query_sync_maximal():
"refine_factor": 10,
"vector": [1.0, 2.0, 3.0],
"nprobes": 5,
"minimum_nprobes": 5,
"maximum_nprobes": 5,
"lower_bound": None,
"upper_bound": None,
"ef": None,
@@ -2369,7 +2430,7 @@ async def test_query_async_nprobes_zero(hybrid):
assert requests == ["/v1/table/test/describe/"]
def test_query_sync_nprobes():
def test_query_sync_probe_bounds():
def handler(body):
assert body == {
"k": 10,
@@ -2381,7 +2442,6 @@ def test_query_sync_nprobes():
"upper_bound": None,
"ef": None,
"vector": [1.0, 2.0, 3.0],
"nprobes": 5,
"minimum_nprobes": 5,
"maximum_nprobes": 15,
"version": None,
@@ -2398,7 +2458,28 @@ def test_query_sync_nprobes():
)
def test_query_sync_no_max_nprobes():
def test_query_sync_nprobes_passthrough():
def handler(body):
assert body["nprobes"] == 20
assert "minimum_nprobes" not in body
assert "maximum_nprobes" not in body
return pa.table({"id": [1]})
with query_test_table(handler) as table:
table.search([1, 2, 3]).nprobes(20).to_list()
def test_query_sync_nprobes_with_unbounded_maximum():
def handler(body):
assert body["nprobes"] == 20
assert body["maximum_nprobes"] == 0
return pa.table({"id": [1]})
with query_test_table(handler) as table:
table.search([1, 2, 3]).nprobes(20).maximum_nprobes(0).to_list()
def test_query_sync_minimum_nprobes_only():
def handler(body):
assert body == {
"k": 10,
@@ -2410,9 +2491,7 @@ def test_query_sync_no_max_nprobes():
"upper_bound": None,
"ef": None,
"vector": [1.0, 2.0, 3.0],
"nprobes": 5,
"minimum_nprobes": 5,
"maximum_nprobes": 0,
"version": None,
}
@@ -2422,7 +2501,6 @@ def test_query_sync_no_max_nprobes():
(
table.search([1, 2, 3], vector_column_name="vector2", fast_search=True)
.minimum_nprobes(5)
.maximum_nprobes(0)
.to_list()
)
@@ -2577,9 +2655,6 @@ def test_query_sync_hybrid():
"refine_factor": None,
"vector": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
"vector_column": "vector",
"nprobes": 20,
"minimum_nprobes": 20,
"maximum_nprobes": 20,
"lower_bound": None,
"upper_bound": None,
"ef": None,
python/python/tests/test_table.py
+46 -1
View File
@@ -12,7 +12,7 @@ import warnings
import weakref
from concurrent.futures import ThreadPoolExecutor
from datetime import date, datetime, timedelta
from decimal import Decimal
from decimal import Decimal, localcontext
from time import sleep
from typing import List
from unittest.mock import patch
@@ -2612,6 +2612,51 @@ def test_update_with_arrow_scalar(mem_db: DBConnection):
assert table.to_arrow()["vector"].to_pylist() == [[1.0, 2.0, 3.0, 4.0]]
@pytest.mark.parametrize("use_arrow_scalar", [False, True])
@pytest.mark.parametrize(
"decimal_type, value",
[
(pa.decimal128(10, 2), "9.99"),
(pa.decimal128(10, 2), "-9.99"),
(pa.decimal128(10, 2), "0.00"),
(pa.decimal128(10, 2), "1.23E+5"),
(pa.decimal128(38, 18), "1E-18"),
(pa.decimal128(38, 18), "12345678901234567.123456789012345678"),
(pa.decimal128(38, 18), "12345678901234567890.123456789012345678"),
(pa.decimal128(38, 18), "-12345678901234567890.123456789012345678"),
(
pa.decimal256(76, 38),
"12345678901234567890123456789012345678."
"12345678901234567890123456789012345678",
),
(
pa.decimal256(76, 38),
"-12345678901234567890123456789012345678."
"12345678901234567890123456789012345678",
),
],
)
def test_update_decimal(mem_db: DBConnection, decimal_type, value, use_arrow_scalar):
expected = Decimal(value)
table = mem_db.create_table(
"my_table",
pa.table(
{
"id": [1, 2],
"price": pa.array([Decimal("1"), expected], type=decimal_type),
}
),
)
update = table.to_arrow()["price"][1] if use_arrow_scalar else expected
with localcontext() as context:
context.prec = 6
result = table.update(where="id = 1", values={"price": update})
assert result.rows_updated == 1
actual = table.to_arrow()
assert actual.schema.field("price").type == decimal_type
assert actual["price"].to_pylist() == [expected, expected]
def test_update_types(mem_db: DBConnection):
table = mem_db.create_table(
"my_table",
python/python/tests/test_util.py
+36
View File
@@ -4,6 +4,7 @@
import os
import pathlib
from decimal import Decimal, localcontext
from typing import Optional
import lance
@@ -157,6 +158,41 @@ def test_value_to_sql_string(tmp_path):
assert table.to_pandas().query("search == @value")["replace"].item() == value
@pytest.mark.parametrize(
"value, expected",
[
("9.99", "'9.99'"),
("-9.99", "'-9.99'"),
("0", "'0'"),
("-0.00", "'-0.00'"),
("1.23E+5", "'123000'"),
("1E-18", "'0.000000000000000001'"),
(
"12345678901234567890.123456789012345678",
"'12345678901234567890.123456789012345678'",
),
(
"12345678901234567890123456789012345678."
"12345678901234567890123456789012345678",
"'12345678901234567890123456789012345678."
"12345678901234567890123456789012345678'",
),
],
)
def test_value_to_sql_decimal(value, expected):
with localcontext() as context:
context.prec = 6
assert value_to_sql(Decimal(value)) == expected
@pytest.mark.parametrize("value", ["NaN", "sNaN", "NaN123456", "Infinity", "-Infinity"])
def test_value_to_sql_decimal_non_finite(value):
with pytest.raises(
ValueError, match="^Non-finite Decimal values cannot be converted to SQL$"
):
value_to_sql(Decimal(value))
def test_value_to_sql_dict():
# Simple flat struct
assert value_to_sql({"a": 1, "b": "hello"}) == "named_struct('a', 1, 'b', 'hello')"
python/src/query.rs
+36 -10
View File
@@ -71,7 +71,7 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
match class_name(&ob)?.as_str() {
"MatchQuery" => {
let query = ob.getattr("query")?.extract()?;
let column = ob.getattr("column")?.extract()?;
let column = ob.getattr("column")?.extract::<Option<String>>()?;
let boost = ob.getattr("boost")?.extract()?;
let fuzziness = ob.getattr("fuzziness")?.extract()?;
let max_expansions = ob.getattr("max_expansions")?.extract()?;
@@ -88,7 +88,7 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
let mut query =
MatchQuery::new(query)
.with_column(Some(column))
.with_column(column)
.with_boost(boost)
.with_fuzziness(fuzziness)
.with_max_expansions(max_expansions)
@@ -199,7 +199,7 @@ impl<'py> IntoPyObject<'py> for PyLanceDB<FtsQuery> {
}
namespace
.getattr(intern!(py, "MatchQuery"))?
.call((query.terms, query.column.unwrap()), Some(&kwargs))
.call((query.terms, query.column), Some(&kwargs))
}
FtsQuery::Phrase(query) => {
let kwargs = PyDict::new(py);
@@ -336,10 +336,10 @@ pub struct PyQueryRequest {
pub use_lsm: Option<bool>,
pub column: Option<String>,
pub query_vector: Option<PyQueryVectors>,
pub nprobes: Option<usize>,
pub minimum_nprobes: Option<usize>,
// None means user did not set it and default should be used (currently 20)
// Some(0) means user set it to None and there is no limit
pub maximum_nprobes: Option<usize>,
pub search_effort: Option<f64>,
pub lower_bound: Option<f32>,
pub upper_bound: Option<f32>,
pub ef: Option<usize>,
@@ -369,8 +369,10 @@ impl From<AnyQuery> for PyQueryRequest {
use_lsm: query_request.use_lsm,
column: None,
query_vector: None,
nprobes: None,
minimum_nprobes: None,
maximum_nprobes: None,
search_effort: None,
lower_bound: None,
upper_bound: None,
ef: None,
@@ -396,11 +398,10 @@ impl From<AnyQuery> for PyQueryRequest {
use_lsm: vector_query.base.use_lsm,
column: vector_query.column,
query_vector: Some(PyQueryVectors(vector_query.query_vector)),
minimum_nprobes: Some(vector_query.minimum_nprobes),
maximum_nprobes: match vector_query.maximum_nprobes {
None => Some(0),
Some(value) => Some(value),
},
nprobes: vector_query.nprobes,
minimum_nprobes: vector_query.minimum_nprobes,
maximum_nprobes: vector_query.maximum_nprobes,
search_effort: vector_query.search_effort,
lower_bound: vector_query.lower_bound,
upper_bound: vector_query.upper_bound,
ef: vector_query.ef,
@@ -1000,6 +1001,15 @@ impl VectorQuery {
Ok(())
}
pub fn search_effort(&mut self, search_effort: Option<f64>) -> PyResult<()> {
self.inner = self
.inner
.clone()
.search_effort(search_effort)
.infer_error()?;
Ok(())
}
pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> {
self.inner = self
.inner
@@ -1193,6 +1203,18 @@ impl HybridQuery {
self.inner_vec.nprobes(nprobe)
}
pub fn search_effort(&mut self, search_effort: Option<f64>) -> PyResult<()> {
self.inner_vec.search_effort(search_effort)
}
pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> {
self.inner_vec.minimum_nprobes(minimum_nprobes)
}
pub fn maximum_nprobes(&mut self, maximum_nprobes: u32) -> PyResult<()> {
self.inner_vec.maximum_nprobes(maximum_nprobes)
}
pub fn ef(&mut self, ef: u32) {
self.inner_vec.ef(ef);
}
@@ -1237,6 +1259,10 @@ impl HybridQuery {
req.query_vector = vec_req.query_vector;
req.column = vec_req.column;
req.distance_type = vec_req.distance_type;
req.nprobes = vec_req.nprobes;
req.minimum_nprobes = vec_req.minimum_nprobes;
req.maximum_nprobes = vec_req.maximum_nprobes;
req.search_effort = vec_req.search_effort;
req.ef = vec_req.ef;
req.refine_factor = vec_req.refine_factor;
req.lower_bound = vec_req.lower_bound;
python/uv.lock
Generated
+41 -15
View File
@@ -1849,19 +1849,19 @@ wheels = [
[[package]]
name = "lance-namespace"
version = "0.11.1"
version = "0.13.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "lance-namespace-urllib3-client" },
]
sdist = { url = "https://files.pythonhosted.org/packages/bf/93/da5f7fcac690db9b282a3439ed9e34960c147619a0d6e1f4eb8cd240e7a5/lance_namespace-0.11.1.tar.gz", hash = "sha256:f67cfbbe0647b7cb42f23b673e7edf8a75b7d8a047265a916492f8d247ee1bc2", size = 11631, upload-time = "2026-08-18T17:40:06.294Z" }
sdist = { url = "https://files.pythonhosted.org/packages/c0/e7/d5d46594678ee479c0eda830c47b2f5c46133bd100e84e9aa6e01306eca7/lance_namespace-0.13.0.tar.gz", hash = "sha256:24554a0997bdb39595c6e4cb3ac6722069f6cd3bd1a74e7acbba7bfaf774a40d", size = 11636, upload-time = "2026-09-14T02:47:03.871523Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fa/bc/601f2b3cc4cfa0070d858a33223bc823fffdd7981a25c45984a5216ca952/lance_namespace-0.11.1-py3-none-any.whl", hash = "sha256:07643fce9a42ad4d58cc8bf91e3f592bc7f4cbd8d0ad5233223506debf67551c", size = 13507, upload-time = "2026-08-18T17:40:03.561Z" },
{ url = "https://files.pythonhosted.org/packages/e7/47/cfa33cca1ba7c749fd2918cfc5c8ded788f378cc6424d23e4fead5a14125/lance_namespace-0.13.0-py3-none-any.whl", hash = "sha256:438c7b17aef421c21c138196e715f2510d62f07e865372047f87c1e75e618c7a", size = 13507, upload-time = "2026-09-14T02:47:01.765655Z" },
]
[[package]]
name = "lance-namespace-urllib3-client"
version = "0.11.1"
version = "0.13.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pydantic" },
@@ -1869,9 +1869,9 @@ dependencies = [
{ name = "typing-extensions" },
{ name = "urllib3" },
]
sdist = { url = "https://files.pythonhosted.org/packages/e4/c5/2bdd0ff98b469894c8a73be809d26ffdad5402517b0e5f9e758026cba29e/lance_namespace_urllib3_client-0.11.1.tar.gz", hash = "sha256:145a9e9424d7597487249b5b95ee274423bf2910e1a9160b6a07b676b61ea46a", size = 237345, upload-time = "2026-08-18T17:40:07.308Z" }
sdist = { url = "https://files.pythonhosted.org/packages/69/25/9aaa4a5e8999693fb0f227c2c0c4b97bd8f0539066408cdff0b89d41b2d5/lance_namespace_urllib3_client-0.13.0.tar.gz", hash = "sha256:1e8a79c6e4e6277033597fd76aa0e2f33d909ca1436c935936e9e569221f43ef", size = 238348, upload-time = "2026-09-14T02:47:04.521878Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/2a/2a/eaaefd55d1190291207049fedc6b3eb22b506e57d6de91bae46bbaaa9c60/lance_namespace_urllib3_client-0.11.1-py3-none-any.whl", hash = "sha256:36537f529294da6d884ba0fe783704483f0a75463497c7705fd083a4d0257990", size = 406311, upload-time = "2026-08-18T17:40:04.842Z" },
{ url = "https://files.pythonhosted.org/packages/60/4c/7b8f0712a7fe1b8655b711342bc4989696635a79e7027ee56ba3cae23e99/lance_namespace_urllib3_client-0.13.0-py3-none-any.whl", hash = "sha256:fb361eb4f6c7f2d1f9e92809609657b73e38241393e9c4516e12d6b6ee643a0a", size = 406848, upload-time = "2026-09-14T02:47:02.715954Z" },
]
[[package]]
@@ -1900,6 +1900,7 @@ clip = [
{ name = "torch" },
]
dev = [
{ name = "maturin" },
{ name = "pre-commit" },
{ name = "pyright" },
{ name = "ruff" },
@@ -1980,7 +1981,8 @@ requires-dist = [
{ name = "huggingface-hub", marker = "extra == 'embeddings'", specifier = ">=0.19.0" },
{ name = "ibm-watsonx-ai", marker = "python_full_version >= '3.10' and extra == 'embeddings'", specifier = ">=1.1.2" },
{ name = "instructorembedding", marker = "extra == 'embeddings'", specifier = ">=1.0.1" },
{ name = "lance-namespace", specifier = ">=0.3.2" },
{ name = "lance-namespace", specifier = ">=0.13.0" },
{ name = "maturin", marker = "extra == 'dev'", specifier = ">=1.10" },
{ name = "mkdocs", marker = "extra == 'docs'" },
{ name = "mkdocs-jupyter", marker = "extra == 'docs'" },
{ name = "mkdocs-material", marker = "extra == 'docs'" },
@@ -2005,7 +2007,7 @@ requires-dist = [
{ name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" },
{ name = "pydantic", specifier = ">=2.7.4,<3" },
{ name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b6" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b12" },
{ name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" },
{ name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" },
{ name = "pytest-asyncio", marker = "extra == 'tests'", specifier = ">=0.21" },
@@ -2156,6 +2158,30 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" },
]
[[package]]
name = "maturin"
version = "1.15.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "tomli", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b9/c8/22e5e21b2679c9bce6415ca578034ca2cc9316be0642ae21e051a2d5198c/maturin-1.15.0.tar.gz", hash = "sha256:94b26cc8e8aba61a5f2099715fe640e18c5f678e9a500408b38761263954228a", size = 385504, upload-time = "2026-08-24T12:11:22.665Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/14/69/5c01b461044eb1f45ddcce006706eb88110c793cdb11c7ae0b5e08492e94/maturin-1.15.0-py3-none-linux_armv6l.whl", hash = "sha256:6bf6dc62e22d4dcfd5a51244ff0d58975fa4979c48209fe84159617648956d82", size = 10206220, upload-time = "2026-08-24T12:10:53.327Z" },
{ url = "https://files.pythonhosted.org/packages/eb/1f/2b431554e11687cdb1077e0cdadcc118c53f611086b3af00c8545a67c6a5/maturin-1.15.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:cd35772633f489841132bc8e71d6fc7f842df30b9c05cd5cdf1ee1ddcb744cc7", size = 19416513, upload-time = "2026-08-24T12:10:56.126Z" },
{ url = "https://files.pythonhosted.org/packages/51/36/e23a21cb34a648b711036b9b2fe1d4f3f4ee24f8db54215d73f1a9a3a3ec/maturin-1.15.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c40b4eae7bf5ef1f4b1af8d623fe4105016f93578fb15b764e741d08ec3b92dd", size = 10014962, upload-time = "2026-08-24T12:10:58.486Z" },
{ url = "https://files.pythonhosted.org/packages/8c/80/33b15cb2d8f30f12c807955e8f2fd775027692904e30ec0784744ce8cd83/maturin-1.15.0-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:7eb066372f541f8eb4909c79c5d9bd0b9e8125980bdf1ec9e8aba23c6c8d6c55", size = 10196223, upload-time = "2026-08-24T12:11:00.696Z" },
{ url = "https://files.pythonhosted.org/packages/fe/91/b495e19e2f5c503b540452b2039115e7b2363867e8c5ad4179eb752fa92c/maturin-1.15.0-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:653020a63525bb224e5ab0adf02e17a2e08bc86dbea7fc1399c9a56d7529b99e", size = 10541186, upload-time = "2026-08-24T12:11:02.857Z" },
{ url = "https://files.pythonhosted.org/packages/5d/2b/2abff58037188d852b124871b1f0d720e1c2bfb3d4f1b03d87c52cd66488/maturin-1.15.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:0ebf9767892725083138e671c34482c660317a2f3d6a29fc0e0f34e9d8c99136", size = 10083468, upload-time = "2026-08-24T12:11:05.012Z" },
{ url = "https://files.pythonhosted.org/packages/3f/07/b7e9f8be99a6627849e81ac7b6694876bce8f50a92995fe17e3cf2610f0a/maturin-1.15.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:7ab7eebffd7b8debca2265985de4eaeb332141276d24b9560b5ad484d4b3add1", size = 10047786, upload-time = "2026-08-24T12:11:07.042Z" },
{ url = "https://files.pythonhosted.org/packages/ca/ab/167e3cb7accee11b507dbe53e0e87aeccb376d44ae66284c96ee4df3a9fd/maturin-1.15.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:126e12e618b4db42f68c779a56d41f82a390145ba36ac3f621d057eb34f5ad9d", size = 13315332, upload-time = "2026-08-24T12:11:09.433Z" },
{ url = "https://files.pythonhosted.org/packages/14/4d/801379f646cbc6b00998e5289b0630a886be3a4ee4c75b6bc9b87478a7f1/maturin-1.15.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4f9d33e6c3f9615c8caceecbbbd440f8eb25a3ddeb687077682cd5eca2e9ae15", size = 10807183, upload-time = "2026-08-24T12:11:11.73Z" },
{ url = "https://files.pythonhosted.org/packages/89/27/2e612e1cbd1580e9e94d4722c227b5180dca27b32b955a34b79918aa1292/maturin-1.15.0-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:bf29beddd0c6708f112db51d5275fc28b28b9e9c9c5faae387eaef662918b176", size = 10413274, upload-time = "2026-08-24T12:11:14.04Z" },
{ url = "https://files.pythonhosted.org/packages/70/d8/202a7b4d75a51f20f84ec9ce3b7345b12b822207072164e2b1c6ef665125/maturin-1.15.0-py3-none-win32.whl", hash = "sha256:da649988be98e87e009e51b1bf0d301b6a301bc0cecbdd60d40d8ba60748d1ca", size = 8928744, upload-time = "2026-08-24T12:11:16.269Z" },
{ url = "https://files.pythonhosted.org/packages/40/dc/4e90da594986ba78dd3bc8a5921ecdcdb11085b22b02a412caab3b225601/maturin-1.15.0-py3-none-win_amd64.whl", hash = "sha256:552c2be4afd43fe8d5c9f3ec8d4c4756d973b8dcbe94c14084390301f50243e1", size = 10335085, upload-time = "2026-08-24T12:11:18.326Z" },
{ url = "https://files.pythonhosted.org/packages/8b/10/15d4314edf130955edf2dc237aa393a8a7c10f2b9b57b89fa2f61f915659/maturin-1.15.0-py3-none-win_arm64.whl", hash = "sha256:c7dc0c66c78d3debdd9c5aa807e861fbcbf07f3505d34b125df74c03986b0f48", size = 9713795, upload-time = "2026-08-24T12:11:20.83Z" },
]
[[package]]
name = "mdit-py-plugins"
version = "0.6.1"
@@ -3912,7 +3938,7 @@ crypto = [
[[package]]
name = "pylance"
version = "14.0.0b6"
version = "14.0.0b12"
source = { registry = "https://pypi.fury.io/lance-format/" }
dependencies = [
{ name = "lance-namespace" },
@@ -3922,12 +3948,12 @@ dependencies = [
{ name = "pyarrow" },
]
wheels = [
{ url = "https://pypi.fury.io/lance-format/-/ver_DuQEn/pylance-14.0.0b6-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:3eade86fa30d9ff3248ee7ffb0b27ce29495394d12327da4d22b5c54bc2b59fe" },
{ url = "https://pypi.fury.io/lance-format/-/ver_K5sqj/pylance-14.0.0b6-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10e291974ee738a1d5d7e9dad17a634394a91cb60080b5ceac209077fbaec43c" },
{ url = "https://pypi.fury.io/lance-format/-/ver_EnbkT/pylance-14.0.0b6-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61bac10f5081c1210edd017f81c8c784d800b89446bd0ed05d4a24e7001965bc" },
{ url = "https://pypi.fury.io/lance-format/-/ver_19ffYg/pylance-14.0.0b6-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3c627ad4a9870611becf84409d8e5ecddc6be64e1dd53bee4bd5fb10ab561e5c" },
{ url = "https://pypi.fury.io/lance-format/-/ver_2eLLY5/pylance-14.0.0b6-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:bc603a7dd690233f1fe000600171e1c092f153bc9060cdf198825c4bbe1418aa" },
{ url = "https://pypi.fury.io/lance-format/-/ver_t8ggc/pylance-14.0.0b6-cp310-abi3-win_amd64.whl", hash = "sha256:1d32df32a3d157611450def07eae8ace3a7401286eac68cad9a24e0d5deed2db" },
{ url = "https://pypi.fury.io/lance-format/-/ver_1Ixasc/pylance-14.0.0b12-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:84350a3c5c1eea3398f4c4fad7cfe85b831d831a5e43b5ca2a783dbdb0e11969" },
{ url = "https://pypi.fury.io/lance-format/-/ver_sCEkG/pylance-14.0.0b12-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76b386c182a6a8e7b86fbef28f84d1dd39e4626ac8681a0cf36f40a45dd7cb87" },
{ url = "https://pypi.fury.io/lance-format/-/ver_1vdT1C/pylance-14.0.0b12-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6aa548c7937fcdfa8f1943a2d59041a42f8b500199880dbb509c7195fa253381" },
{ url = "https://pypi.fury.io/lance-format/-/ver_2fXu1V/pylance-14.0.0b12-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:79f82d6a6c50f37aab338470e3733436e6a27fbb5972d29fcf453ece56d3b3c6" },
{ url = "https://pypi.fury.io/lance-format/-/ver_24UavN/pylance-14.0.0b12-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:de0305999257002785f60169e6926896584d66f1da050bb5afd1d48c6e3d9bb9" },
{ url = "https://pypi.fury.io/lance-format/-/ver_XNynX/pylance-14.0.0b12-cp310-abi3-win_amd64.whl", hash = "sha256:4d0cdb689e898ce71e8e434d00a3b97639cea8fb97da21f1c921543be363268f" },
]
[[package]]
rust/lancedb/Cargo.toml
+12 -8
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.41.0-beta.1"
version = "0.41.0-beta.6"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
@@ -68,17 +68,20 @@ async-openai = { version = "0.20.0", optional = true }
serde_with = { version = "3.8.1" }
tempfile = { workspace = true }
aws-smithy-types = { workspace = true, optional = true }
aws-sdk-bedrockruntime = { version = "1.27.0", optional = true }
# Default features include the legacy `rustls` feature, which pulls in rustls 0.21.
aws-sdk-bedrockruntime = { version = "1.77.0", optional = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
# For remote feature
reqwest = { version = "0.12.0", default-features = false, features = [
reqwest = { version = "0.13.0", default-features = false, features = [
"charset",
"gzip",
"http2",
"json",
"macos-system-configuration",
"query",
# Avoid linking OpenSSL into Python wheels, which breaks on FIPS hosts.
"rustls-tls-native-roots",
# This uses the aws-lc-rs crypto provider.
"rustls",
"stream",
"system-proxy",
], optional = true }
tonic = { workspace = true, optional = true }
http = { version = "1", optional = true } # Matching what is in reqwest
@@ -114,9 +117,10 @@ random_word = { version = "0.4.3", features = ["en"] }
tokio = { workspace = true, features = ["io-util", "macros", "net", "test-util"] }
uuid = { workspace = true }
walkdir = "2"
aws-sdk-dynamodb = { version = "1.55.0" }
aws-sdk-s3 = { version = "1.55.0" }
aws-sdk-kms = { version = "1.48.0" }
# Default features include the legacy `rustls` feature, which pulls in rustls 0.21.
aws-sdk-dynamodb = { version = "1.68.0", default-features = false, features = ["default-https-client", "rt-tokio"] }
aws-sdk-s3 = { version = "1.79.0", default-features = false, features = ["default-https-client", "rt-tokio", "sigv4a", "http-1x"] }
aws-sdk-kms = { version = "1.63.0", default-features = false, features = ["default-https-client", "rt-tokio"] }
aws-config = { version = "1.5.10" }
aws-smithy-runtime = { version = "1.9.1" }
aws-smithy-types.workspace = true
rust/lancedb/src/connection.rs
+2 -7
View File
@@ -38,10 +38,7 @@ use crate::remote::{
},
};
use crate::secrets::SecretInfo;
use crate::utils::{
validate_namespace, validate_secret_component, validate_secret_reference,
validate_view_reference,
};
use crate::utils::{validate_namespace, validate_secret_reference, validate_view_reference};
use crate::view::ViewDescription;
use lance::io::ObjectStoreParams;
pub use lance_file::version::LanceFileVersion;
@@ -811,9 +808,7 @@ impl Connection {
let database = database.clone();
let namespace_path = namespace_path.clone();
async move {
for segment in &namespace_path {
validate_secret_component("Secret namespace path segment", segment)?;
}
validate_namespace(&namespace_path)?;
database.list_secrets(&namespace_path, options).await
}
})
rust/lancedb/src/function.rs
+5 -3
View File
@@ -453,7 +453,8 @@ pub struct FunctionVersion {
signature: FunctionSignature,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
secret_bindings: Vec<SecretBinding>,
created_at: String,
/// When this version was created, in milliseconds since the epoch.
created_at_millis: i64,
metadata: BTreeMap<String, String>,
disabled: bool,
}
@@ -507,8 +508,9 @@ impl FunctionVersion {
&self.secret_bindings
}
pub fn created_at(&self) -> &str {
&self.created_at
/// When this version was created, in milliseconds since the epoch.
pub fn created_at_millis(&self) -> i64 {
self.created_at_millis
}
}
rust/lancedb/src/index.rs
+11
View File
@@ -75,6 +75,17 @@ pub enum Index {
/// Full text search index using BM25.
///
/// Supports strings, lists of strings, and `lance.json` logical fields
/// stored as `LargeBinary`. Raw binary fields are not supported. Lance
/// selects its JSON-aware tokenizer from the field metadata.
///
/// JSON queries use `path,str,value` triples, where the dotted path is
/// relative to the indexed column. For example, `meta.tag,str,nature`
/// searches the string value at `$.meta.tag`, not other paths or key names.
/// Create the index on the JSON column, not on a JSON subpath. Plain text
/// queries are invalid for this tokenizer; malformed triples can panic in
/// the current Lance dependency (see [lance#8812](https://github.com/lance-format/lance/issues/8812)).
///
/// The posting block size defaults to 128. Supported values are 128 and 256;
/// a value of 256 uses the experimental FTS V3 format and may introduce
/// breaking changes.
rust/lancedb/src/query.rs
+139 -25
View File
@@ -1114,10 +1114,24 @@ pub struct VectorQueryRequest {
pub column: Option<String>,
/// The vector(s) to search for
pub query_vector: Vec<Arc<dyn Array>>,
/// The minimum number of partitions to search
pub minimum_nprobes: usize,
/// The maximum number of partitions to search
/// The legacy number of partitions to search.
///
/// Lance sets both probe bounds to this value. Explicit bounds can override it.
pub nprobes: Option<usize>,
/// The minimum number of partitions to search.
///
/// If unset, Lance's default is used.
pub minimum_nprobes: Option<usize>,
/// The maximum number of partitions to search.
///
/// If unset, Lance's default is used. Zero represents an explicitly unbounded
/// maximum on request and wire representations.
pub maximum_nprobes: Option<usize>,
/// Optional initial IVF search effort in [0, 1].
///
/// None preserves Auto and fixed `nprobes`; any explicit value overrides
/// `nprobes` while retaining explicit minimum and maximum bounds.
pub search_effort: Option<f64>,
/// The lower bound (inclusive) of the distance to search for.
pub lower_bound: Option<f32>,
/// The upper bound (exclusive) of the distance to search for.
@@ -1141,8 +1155,10 @@ impl Default for VectorQueryRequest {
base: QueryRequest::default(),
column: None,
query_vector: Vec::new(),
minimum_nprobes: 20,
maximum_nprobes: Some(20),
nprobes: None,
minimum_nprobes: None,
maximum_nprobes: None,
search_effort: None,
lower_bound: None,
upper_bound: None,
ef: None,
@@ -1155,6 +1171,17 @@ impl Default for VectorQueryRequest {
}
impl VectorQueryRequest {
pub(crate) fn validate_search_effort(&self) -> Result<()> {
if let Some(effort) = self.search_effort
&& (!effort.is_finite() || !(0.0..=1.0).contains(&effort))
{
return Err(Error::InvalidInput {
message: format!("search_effort must be finite and in [0, 1], got {effort}"),
});
}
Ok(())
}
fn add_query_vector(&mut self, vector: impl IntoQueryVector) -> Result<()> {
let vector = vector.to_query_vector(&DataType::Float32, "default")?;
let array =
@@ -1248,7 +1275,7 @@ impl VectorQuery {
Ok(self)
}
/// Set the number of partitions to search (probe)
/// Set the legacy IVF probe parameter
///
/// This argument is only used when the vector column has an IVF PQ index.
/// If there is no index then this value is ignored.
@@ -1258,19 +1285,20 @@ impl VectorQuery {
///
/// The partition whose centroids are closest to the query vector will be
/// exhaustiely searched to find matches. This parameter controls how many
/// partitions should be searched.
/// partitions may be searched.
///
/// Increasing this value will increase the recall of your query but will
/// also increase the latency of your query. The default value is 20. This
/// default is good for many cases but the best value to use will depend on
/// your data and the recall that you need to achieve.
/// also increase the latency of your query. If this method is not called,
/// Lance's adaptive probe defaults are used.
///
/// For best results we recommend tuning this parameter with a benchmark against
/// your actual data to find the smallest possible value that will still give
/// you the desired recall.
///
/// This method sets both the minimum and maximum number of partitions to search.
/// For more fine-grained control see [`VectorQuery::minimum_nprobes`] and
/// LanceDB retains this as `nprobes` through local and remote request
/// construction. Lance sets both probe bounds to this value only when
/// [`Self::search_effort`] is unset. For more
/// fine-grained control see [`VectorQuery::minimum_nprobes`] and
/// [`VectorQuery::maximum_nprobes`].
///
/// Returns an error if `nprobes` is not greater than 0.
@@ -1280,8 +1308,32 @@ impl VectorQuery {
message: "nprobes must be greater than 0".to_string(),
});
}
self.request.minimum_nprobes = nprobes;
self.request.maximum_nprobes = Some(nprobes);
self.request.nprobes = Some(nprobes);
Ok(self)
}
/// Set the initial IVF search effort to a finite value in [0, 1].
///
/// None (the default) resolves to 0.5, preserving Auto probing. At 0 the
/// search starts at the caller minimum; at 1 it searches all partitions
/// within the caller maximum. Intermediate values interpolate around Auto.
/// Later probing may continue when too few candidates are found.
/// Any explicit value, including 0.5, overrides [`Self::nprobes`] regardless
/// of setter order. Explicit minimum and maximum bounds still apply.
/// This does not change refinement or HNSW ef and is not a recall guarantee.
/// Lance's MemWAL LSM scanner does not yet support this option.
///
/// ```
/// # use lancedb::{Table, query::{QueryBase, ExecutableQuery}};
/// # async fn example(table: &Table) -> lancedb::Result<()> {
/// let results = table.query().nearest_to([0.1, 0.2].as_slice())?
/// .search_effort(0.75)?.execute().await?;
/// # Ok(())
/// # }
/// ```
pub fn search_effort(mut self, search_effort: impl Into<Option<f64>>) -> Result<Self> {
self.request.search_effort = search_effort.into();
self.request.validate_search_effort()?;
Ok(self)
}
@@ -1303,6 +1355,7 @@ impl VectorQuery {
});
}
if let Some(maximum_nprobes) = self.request.maximum_nprobes
&& maximum_nprobes != 0
&& minimum_nprobes > maximum_nprobes
{
return Err(Error::InvalidInput {
@@ -1310,7 +1363,7 @@ impl VectorQuery {
.to_string(),
});
}
self.request.minimum_nprobes = minimum_nprobes;
self.request.minimum_nprobes = Some(minimum_nprobes);
Ok(self)
}
@@ -1323,11 +1376,13 @@ impl VectorQuery {
///
/// If this value is greater than minimum_nprobes then the excess partitions will
/// only be searched if the initial search does not return enough results.
/// The bound applies per index segment to initial and later probing without
/// disabling Auto probing.
///
/// This can be useful when there is a narrow filter to allow these queries to
/// spend more time searching and avoid potential false negatives.
///
/// Set to None to search all partitions, if needed, to satisfy the limit
/// Set to None to use Lance's default maximum probe behavior.
pub fn maximum_nprobes(mut self, maximum_nprobes: Option<usize>) -> Result<Self> {
if let Some(maximum_nprobes) = maximum_nprobes {
if maximum_nprobes == 0 {
@@ -1335,14 +1390,18 @@ impl VectorQuery {
message: "maximum_nprobes must be greater than 0".to_string(),
});
}
if maximum_nprobes < self.request.minimum_nprobes {
if let Some(minimum_nprobes) = self.request.minimum_nprobes
&& maximum_nprobes < minimum_nprobes
{
return Err(Error::InvalidInput {
message: "maximum_nprobes must be greater than or equal to minimum_nprobes"
.to_string(),
});
}
}
self.request.maximum_nprobes = maximum_nprobes;
// Preserve an explicit reset separately from an untouched maximum. The
// request protocols already use zero to represent an unbounded maximum.
self.request.maximum_nprobes = Some(maximum_nprobes.unwrap_or(0));
Ok(self)
}
@@ -2431,6 +2490,29 @@ mod tests {
let vector = Float32Array::from_iter_values([0.1, 0.2]);
let query = table.query().nearest_to(&[0.1, 0.2]).unwrap();
assert_eq!(query.request.nprobes, None);
assert_eq!(query.request.minimum_nprobes, None);
assert_eq!(query.request.maximum_nprobes, None);
assert_eq!(query.request.search_effort, None);
let effort_query = query
.clone()
.nprobes(20)
.unwrap()
.search_effort(0.5)
.unwrap();
assert_eq!(effort_query.request.search_effort, Some(0.5));
assert_eq!(effort_query.request.nprobes, Some(20));
assert_eq!(effort_query.request.minimum_nprobes, None);
assert_eq!(effort_query.request.maximum_nprobes, None);
assert_eq!(
effort_query
.search_effort(None)
.unwrap()
.request
.search_effort,
None
);
assert_eq!(
*query
.request
@@ -2442,6 +2524,37 @@ mod tests {
vector
);
let minimum_only_query = query.clone().minimum_nprobes(20).unwrap();
assert_eq!(minimum_only_query.request.minimum_nprobes, Some(20));
assert_eq!(minimum_only_query.request.maximum_nprobes, None);
let maximum_only_query = query.maximum_nprobes(Some(20)).unwrap();
assert_eq!(maximum_only_query.request.minimum_nprobes, None);
assert_eq!(maximum_only_query.request.maximum_nprobes, Some(20));
let explicitly_unbounded_query = table
.query()
.nearest_to(&[0.1, 0.2])
.unwrap()
.nprobes(20)
.unwrap()
.maximum_nprobes(None)
.unwrap();
assert_eq!(explicitly_unbounded_query.request.nprobes, Some(20));
assert_eq!(explicitly_unbounded_query.request.maximum_nprobes, Some(0));
let bounded_query = table
.query()
.nearest_to(&[0.1, 0.2])
.unwrap()
.minimum_nprobes(5)
.unwrap()
.nprobes(20)
.unwrap();
assert_eq!(bounded_query.request.nprobes, Some(20));
assert_eq!(bounded_query.request.minimum_nprobes, Some(5));
assert_eq!(bounded_query.request.maximum_nprobes, None);
let new_vector = Float32Array::from_iter_values([9.8, 8.7]);
let query = table
@@ -2450,7 +2563,7 @@ mod tests {
.offset(1)
.nearest_to(&[9.8, 8.7])
.unwrap()
.nprobes(1000)
.nprobes(20)
.unwrap()
.postfilter()
.distance_type(DistanceType::Cosine)
@@ -2469,8 +2582,9 @@ mod tests {
);
assert_eq!(query.request.base.limit.unwrap(), 100);
assert_eq!(query.request.base.offset.unwrap(), 1);
assert_eq!(query.request.minimum_nprobes, 1000);
assert_eq!(query.request.maximum_nprobes, Some(1000));
assert_eq!(query.request.nprobes, Some(20));
assert_eq!(query.request.minimum_nprobes, None);
assert_eq!(query.request.maximum_nprobes, None);
assert!(query.request.use_index);
assert_eq!(query.request.distance_type, Some(DistanceType::Cosine));
assert_eq!(query.request.approx_mode, Some(ApproxMode::Accurate));
@@ -2492,13 +2606,13 @@ mod tests {
Err(Error::InvalidInput { message }) if message == "nprobes must be greater than 0"
));
// Fixed probe counts replace both bounds, even when moving below or
// above the previous range.
// Legacy probe counts remain independent until Lance resolves them.
let mut query = query;
for nprobes in [30, 1, 50] {
query = query.nprobes(nprobes).unwrap();
assert_eq!(query.request.minimum_nprobes, nprobes);
assert_eq!(query.request.maximum_nprobes, Some(nprobes));
assert_eq!(query.request.nprobes, Some(nprobes));
assert_eq!(query.request.minimum_nprobes, None);
assert_eq!(query.request.maximum_nprobes, None);
}
}
rust/lancedb/src/remote/catalog.rs
+22 -27
View File
@@ -19,6 +19,7 @@ use crate::catalog::{
ListDatabasesResponse,
};
use crate::database::Database;
use crate::utils::validate_database_name;
use crate::{Error, Result};
/// Authentication and client settings shared by a catalog and its databases.
@@ -154,29 +155,6 @@ impl RemoteCatalog {
})
}
fn validate_name(&self, name: &str) -> Result<()> {
let delimiter = self
.options
.client_config
.id_delimiter
.as_deref()
.unwrap_or("$");
if name.is_empty()
|| name.trim() != name
|| !name.is_ascii()
|| name.chars().any(char::is_control)
|| name.contains(delimiter)
|| matches!(name, "." | "..")
{
return Err(Error::InvalidInput {
message: format!(
"Invalid database name '{name}': expected a nonempty ASCII name without surrounding whitespace, control characters, or namespace delimiter '{delimiter}'"
),
});
}
Ok(())
}
fn database(&self, name: &str) -> Result<Arc<dyn Database>> {
Ok(Arc::new(RemoteDatabase::for_catalog(
&self.endpoint,
@@ -203,7 +181,7 @@ impl Catalog for RemoteCatalog {
}
async fn create_database(&self, request: CreateDatabaseRequest) -> Result<Arc<dyn Database>> {
self.validate_name(&request.name)?;
validate_database_name(&request.name)?;
self.root
.create_namespace(CreateNamespaceRequest {
id: Some(vec![request.name.clone()]),
@@ -231,7 +209,7 @@ impl Catalog for RemoteCatalog {
}
async fn drop_database(&self, request: DropDatabaseRequest) -> Result<()> {
self.validate_name(&request.name)?;
validate_database_name(&request.name)?;
let result = self
.root
.drop_namespace(DropNamespaceRequest {
@@ -286,7 +264,7 @@ impl Catalog for RemoteCatalog {
}
async fn connect_database(&self, name: &str) -> Result<Arc<dyn Database>> {
self.validate_name(name)?;
validate_database_name(name)?;
self.root
.describe_namespace(DescribeNamespaceRequest {
id: Some(vec![name.into()]),
@@ -676,7 +654,24 @@ mod tests {
}
let catalog =
RemoteCatalog::try_new("http://127.0.0.1:1", RemoteCatalogOptions::default()).unwrap();
for name in ["", "a$b", "\r\ninjected", "..", "café", " padded "] {
for name in [
"",
"/",
"//",
"a$b",
"\r\ninjected",
"..",
"café",
" padded ",
"a b",
"a?b",
"/db",
"db/",
"a//b",
"a/../b",
"a/./b",
&"a".repeat(65),
] {
assert!(matches!(
catalog.create_database(name.into()).await,
Err(Error::InvalidInput { .. })
rust/lancedb/src/remote/client.rs
+25
View File
@@ -12,6 +12,7 @@ use std::{collections::HashMap, future::Future, str::FromStr, sync::Arc, time::D
use crate::error::{Error, Result};
use crate::remote::db::RemoteOptions;
use crate::remote::retry::{ResolvedRetryConfig, RetryCounter};
use crate::utils::validate_database_name;
const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id");
@@ -383,6 +384,7 @@ pub fn parse_db_url(db_url: &str) -> Result<ParsedDbUrl> {
message: format!("Invalid encoded database name: {err}"),
})?
.into_owned();
validate_database_name(&db_name)?;
let db_prefix = {
let prefix = parsed_url.path().trim_start_matches('/');
if prefix.is_empty() {
@@ -1204,6 +1206,29 @@ mod tests {
assert_eq!(parsed.db_prefix.as_deref(), Some("prefix"));
}
#[test]
fn test_parse_database_uri_validates_decoded_name() {
for name in [
"a b",
"a$b",
"a?b",
"café",
"/db",
"db/",
"a//b",
"a/../b",
"a/./b",
&"a".repeat(65),
] {
let uri = format!("db://{}", urlencoding::encode(name));
assert!(
matches!(parse_db_url(&uri), Err(Error::InvalidInput { .. })),
"accepted {name:?}"
);
}
assert!(parse_db_url(&format!("db://{}", "a".repeat(64))).is_ok());
}
#[test]
fn test_timeout_config_default() {
let config = TimeoutConfig::default();
rust/lancedb/src/remote/db.rs
+147 -118
View File
@@ -36,7 +36,10 @@ use crate::remote::util::stream_as_body;
use crate::secrets::SecretBinding;
use crate::secrets::SecretInfo;
use crate::table::BaseTable;
use crate::utils::{reject_relative_segment, validate_table_name};
use crate::utils::{
validate_database_name, validate_function_name, validate_namespace, validate_secret_name,
validate_table_name, validate_view_name,
};
use crate::view::ViewDescription;
use super::client::{
@@ -230,6 +233,7 @@ pub struct RemoteDatabase<S: HttpSend = Sender> {
/// TLS configuration for mTLS support
tls_config: Option<super::client::TlsConfig>,
sql_client: Option<SqlClient>,
is_catalog_root: bool,
}
#[derive(Clone)]
@@ -320,6 +324,9 @@ impl RemoteDatabase {
name: Option<&str>,
options: &super::catalog::RemoteCatalogOptions,
) -> Result<Self> {
if let Some(name) = name {
validate_database_name(name)?;
}
let scope = super::catalog::ScopedHeaderProvider {
provider: options.client_config.header_provider.clone(),
database: name.map(str::to_string),
@@ -348,6 +355,7 @@ impl RemoteDatabase {
)?;
if name.is_none() {
db.sql_client = None;
db.is_catalog_root = true;
}
Ok(db)
}
@@ -422,11 +430,38 @@ impl RemoteDatabase {
namespace_context_provider,
tls_config: client_config.tls_config,
sql_client: Some(sql_client),
is_catalog_root: false,
})
}
}
impl<S: HttpSend> RemoteDatabase<S> {
// A catalog's root namespaces are database names, which may contain slashes.
fn namespace_identifier(&self, namespace: &[String]) -> Result<String> {
match namespace.first() {
None => Ok(ID_DELIMITER.to_string()),
Some(first) => {
if self.is_catalog_root {
// When dealing with the catalog root, the first component is a
// database name. Those need to be validated differently because
// they can contain slashes. Also, we need to use percent encoding
// for the slashes.
validate_database_name(first)?;
validate_namespace(&namespace[1..])?;
Ok(namespace
.iter()
.map(|component| urlencoding::encode(component).into_owned())
.collect::<Vec<_>>()
.join(ID_DELIMITER))
} else {
// If we're not dealing with the catalog root, build the namespace identifier in the
// usual way.
build_namespace_identifier(namespace)
}
}
}
}
/// Post a request whose body carries a credential.
///
/// Shared by the create and alter verbs, which declare their own request
@@ -537,6 +572,7 @@ mod test_utils {
namespace_context_provider: None,
tls_config: None,
sql_client: None,
is_catalog_root: false,
}
}
@@ -560,6 +596,7 @@ mod test_utils {
namespace_context_provider,
tls_config: config.tls_config.clone(),
sql_client: None,
is_catalog_root: false,
}
}
}
@@ -581,93 +618,60 @@ impl From<&CreateTableMode> for &'static str {
}
}
/// The path segment addressing one object: its namespace path and its name.
///
/// One builder for tables, Secrets, Functions and materialized views: the
/// identifier grammar belongs to the namespace spec, not to an object type. An
/// empty path addresses an object with no namespace.
///
/// Components are checked for addressability, not a character set. The name's
/// grammar is the caller's, so a table reports [`Error::InvalidTableName`], a
/// Function admits names a table may not, and a catalog database carries the
/// `/` that [`RemoteCatalog`] allows.
///
/// [`RemoteCatalog`]: super::catalog::RemoteCatalog
fn build_object_identifier(what: &str, name: &str, namespace: &[String]) -> Result<String> {
for segment in namespace {
reject_unaddressable_component("namespace segment", segment)?;
fn build_compound_identifier(name: &str, namespace: &[String]) -> Result<String> {
validate_namespace(namespace)?;
let mut identifier = namespace.join("$");
if !namespace.is_empty() {
identifier.push('$');
}
reject_unaddressable_component(what, name)?;
Ok(join_identifier(
namespace.iter().map(String::as_str).chain([name]),
))
identifier.push_str(name);
Ok(identifier)
}
/// What a component may not be if the join is to survive being split back
/// apart: empty, a segment URL parsing resolves away, or the delimiter itself.
///
/// Each erases a boundary no encoding of the joined form recovers. `["prod",
/// ""]` joins to `prod$`, which reads back as `["prod"]`, so a drop reaches the
/// parent of the namespace the caller named.
///
/// Not a character set: per-component percent-encoding makes the wider set
/// safe, since a `/` in a name reaches the service as `%2F`, still one
/// segment.
fn reject_unaddressable_component(what: &str, value: &str) -> Result<()> {
if value.is_empty() {
return Err(Error::InvalidInput {
message: format!(
"{what} must not be empty: the identifier would carry two delimiters in a row, \
and splitting it back apart would name a different object"
),
});
}
reject_relative_segment(what, value)?;
if value.contains(ID_DELIMITER) {
return Err(Error::InvalidInput {
message: format!(
"{what} '{value}' contains the identifier delimiter '{ID_DELIMITER}', so the \
namespace path and the name it joins could not be told apart"
),
});
}
Ok(())
}
/// The path segment addressing one table. A wrapper for the error type:
/// callers match on [`Error::InvalidTableName`].
/// Build a full path to a table. Note that we do not need to percent-encode
/// this when using it in a URL because the validation functions ensure that
/// only URL-safe characters are present.
fn build_table_identifier(name: &str, namespace: &[String]) -> Result<String> {
validate_table_name(name)?;
build_object_identifier("table name", name, namespace)
build_compound_identifier(name, namespace)
}
/// Join components into the `{id}` a route addresses: each percent-encoded,
/// then joined by the delimiter.
///
/// Per component rather than over the joined string, so the delimiter stays a
/// delimiter and nothing inside a component can end the path segment.
///
/// A second line, not the first: a component from the object charset is all
/// unreserved and encodes to itself, so the route reads as the caller wrote it.
/// It does not cover `.` and `..`, which are unreserved too and resolve away
/// after decoding -- [`build_object_identifier`] refuses those.
fn join_identifier<'a>(components: impl Iterator<Item = &'a str>) -> String {
components
.map(|component| urlencoding::encode(component).into_owned())
.collect::<Vec<_>>()
.join(ID_DELIMITER)
}
/// The path segment addressing one namespace.
/// Translate an array of namespace components into a namespace string.
/// Note that we do not need to percent-encode this when using it in a URL
/// because validate_namespace ensures that only URL-safe characters are
/// present.
fn build_namespace_identifier(namespace: &[String]) -> Result<String> {
for segment in namespace {
reject_unaddressable_component("namespace segment", segment)?;
}
validate_namespace(namespace)?;
if namespace.is_empty() {
// According to the namespace spec, use delimiter to represent root namespace
return Ok(ID_DELIMITER.to_string());
Ok(ID_DELIMITER.to_string())
} else {
Ok(namespace.join(ID_DELIMITER))
}
Ok(join_identifier(namespace.iter().map(String::as_str)))
}
/// Build a full path to a function. Note that we do not need to percent-encode
/// this when using it in a URL because the validation functions ensure that
/// only URL-safe characters are present.
fn build_function_identifier(name: &str, namespace: &[String]) -> Result<String> {
validate_function_name(name)?;
build_compound_identifier(name, namespace)
}
/// Build a full path to a secret. Note that we do not need to percent-encode
/// this when using it in a URL because the validation functions ensure that
/// only URL-safe characters are present.
fn build_secret_identifier(name: &str, namespace: &[String]) -> Result<String> {
validate_secret_name(name)?;
build_compound_identifier(name, namespace)
}
/// Build a full path to a view. Note that we do not need to percent-encode
/// this when using it in a URL because the validation functions ensure that
/// only URL-safe characters are present.
fn build_view_identifier(name: &str, namespace: &[String]) -> Result<String> {
validate_view_name(name)?;
build_compound_identifier(name, namespace)
}
/// Build a secure cache key using length prefixes.
@@ -945,7 +949,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
request: FunctionRegistrationRequest,
namespace_path: &[String],
) -> Result<Job<FunctionVersion>> {
let function_id = build_object_identifier("Function name", &request.name, namespace_path)?;
let function_id = build_function_identifier(&request.name, namespace_path)?;
let req = self
.client
.post(&format!("/v1/function/{function_id}/create"))
@@ -976,7 +980,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
version: &str,
namespace_path: &[String],
) -> Result<FunctionVersion> {
let function_id = build_object_identifier("Function name", name, namespace_path)?;
let function_id = build_function_identifier(name, namespace_path)?;
let req = self
.client
.post(&format!("/v1/function/{function_id}/describe"))
@@ -1036,7 +1040,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
version: &str,
namespace_path: &[String],
) -> Result<(bool, Job)> {
let function_id = build_object_identifier("Function name", name, namespace_path)?;
let function_id = build_function_identifier(name, namespace_path)?;
let req = self
.client
.post(&format!("/v1/function/{function_id}/drop"))
@@ -1081,7 +1085,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
value: &str,
namespace_path: &[String],
) -> Result<()> {
let secret_id = build_object_identifier("Secret name", name, namespace_path)?;
let secret_id = build_secret_identifier(name, namespace_path)?;
self.post_secret_write(
&format!("/v1/secret/{secret_id}/create"),
&RemoteCreateSecretRequest { value },
@@ -1090,7 +1094,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
}
async fn alter_secret(&self, name: &str, value: &str, namespace_path: &[String]) -> Result<()> {
let secret_id = build_object_identifier("Secret name", name, namespace_path)?;
let secret_id = build_secret_identifier(name, namespace_path)?;
self.post_secret_write(
&format!("/v1/secret/{secret_id}/alter"),
&RemoteAlterSecretRequest { value },
@@ -1127,7 +1131,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
}
async fn drop_secret(&self, name: &str, namespace_path: &[String]) -> Result<()> {
let secret_id = build_object_identifier("Secret name", name, namespace_path)?;
let secret_id = build_secret_identifier(name, namespace_path)?;
let req = self.client.post(&format!("/v1/secret/{secret_id}/drop"));
let (request_id, response) = self.client.send(req).await?;
self.client.check_response(&request_id, response).await?;
@@ -1135,7 +1139,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
}
async fn describe_secret(&self, name: &str, namespace_path: &[String]) -> Result<SecretInfo> {
let secret_id = build_object_identifier("Secret name", name, namespace_path)?;
let secret_id = build_secret_identifier(name, namespace_path)?;
let req = self
.client
.post(&format!("/v1/secret/{secret_id}/describe"));
@@ -1150,7 +1154,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
query: &str,
namespace_path: &[String],
) -> Result<ViewDescription> {
let view_id = build_object_identifier("View name", name, namespace_path)?;
let view_id = build_view_identifier(name, namespace_path)?;
let req = self
.client
.post(&format!("/v1/view/{view_id}/create"))
@@ -1167,7 +1171,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
name: &str,
namespace_path: &[String],
) -> Result<ViewDescription> {
let view_id = build_object_identifier("View name", name, namespace_path)?;
let view_id = build_view_identifier(name, namespace_path)?;
let req = self.client.post(&format!("/v1/view/{view_id}/describe"));
let (request_id, response) = self.client.send(req).await?;
let response = self.client.check_response(&request_id, response).await?;
@@ -1184,7 +1188,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
}
async fn drop_view_async(&self, name: &str, namespace_path: &[String]) -> Result<Job> {
let view_id = build_object_identifier("View name", name, namespace_path)?;
let view_id = build_view_identifier(name, namespace_path)?;
let req = self.client.post(&format!("/v1/view/{view_id}/drop"));
let (request_id, response) = self.client.send(req).await?;
let response = self.client.check_response(&request_id, response).await?;
@@ -1407,7 +1411,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
let namespace_parts = request.id.as_deref().unwrap_or(&[]);
let namespace_id = build_namespace_identifier(namespace_parts)?;
let namespace_id = self.namespace_identifier(namespace_parts)?;
let mut req = self
.client
.get(&format!("/v1/namespace/{}/table/list", namespace_id));
@@ -1674,7 +1678,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
request: ListNamespacesRequest,
) -> Result<ListNamespacesResponse> {
let namespace_parts = request.id.as_deref().unwrap_or(&[]);
let namespace_id = build_namespace_identifier(namespace_parts)?;
let namespace_id = self.namespace_identifier(namespace_parts)?;
let mut req = self
.client
.get(&format!("/v1/namespace/{}/list", namespace_id));
@@ -1696,7 +1700,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
request: CreateNamespaceRequest,
) -> Result<CreateNamespaceResponse> {
let namespace_parts = request.id.as_deref().unwrap_or(&[]);
let namespace_id = build_namespace_identifier(namespace_parts)?;
let namespace_id = self.namespace_identifier(namespace_parts)?;
let mut req = self
.client
.post(&format!("/v1/namespace/{}/create", namespace_id));
@@ -1727,7 +1731,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result<DropNamespaceResponse> {
let namespace_parts = request.id.as_deref().unwrap_or(&[]);
let namespace_id = build_namespace_identifier(namespace_parts)?;
let namespace_id = self.namespace_identifier(namespace_parts)?;
let mut req = self
.client
.post(&format!("/v1/namespace/{}/drop", namespace_id));
@@ -1761,7 +1765,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
request: DescribeNamespaceRequest,
) -> Result<DescribeNamespaceResponse> {
let namespace_parts = request.id.as_deref().unwrap_or(&[]);
let namespace_id = build_namespace_identifier(namespace_parts)?;
let namespace_id = self.namespace_identifier(namespace_parts)?;
let req = self
.client
.post(&format!("/v1/namespace/{}/describe", namespace_id))
@@ -1887,6 +1891,40 @@ mod tests {
remote::{ARROW_STREAM_CONTENT_TYPE, ClientConfig, HeaderProvider, JSON_CONTENT_TYPE},
};
#[tokio::test]
async fn test_catalog_namespace_identifier_validates_all_namespace_components() {
let mut db = super::RemoteDatabase::new_mock(|_| -> http::Response<String> {
panic!("identifier validation must not send a request")
});
db.is_catalog_root = true;
for (path, expected) in [
(vec![], "$"),
(vec!["team/search"], "team%2Fsearch"),
(vec!["team/search", "ns"], "team%2Fsearch$ns"),
(vec!["team/search", "ns", "child"], "team%2Fsearch$ns$child"),
] {
let path = path.into_iter().map(String::from).collect::<Vec<_>>();
assert_eq!(db.namespace_identifier(&path).unwrap(), expected);
}
for path in [
vec!["../search"],
vec!["team/search", "bad/name"],
vec!["team/search", "ns", "bad/name"],
vec!["team/search", "public", "child"],
] {
let path = path.into_iter().map(String::from).collect::<Vec<_>>();
assert!(
matches!(
db.namespace_identifier(&path),
Err(Error::InvalidInput { .. })
),
"accepted {path:?}"
);
}
}
#[test]
fn test_cache_key_security() {
// Test that cache keys are unique regardless of delimiter manipulation
@@ -4045,7 +4083,7 @@ mod tests {
.expect_err("an illegal component must be refused");
assert!(!*reached.lock().unwrap(), "{name:?} reached the transport");
assert!(
error.to_string().contains("Secret name"),
error.to_string().contains("secret name"),
"{name:?}: {error}"
);
}
@@ -4076,7 +4114,7 @@ mod tests {
.await
.expect_err("a dot-only name must be refused");
assert!(
by_name.to_string().contains("relative path segments"),
by_name.to_string().contains("Illegal secret name"),
"{by_name}"
);
@@ -4086,7 +4124,9 @@ mod tests {
.await
.expect_err("a dot-only namespace segment must be refused");
assert!(
by_segment.to_string().contains("relative path segments"),
by_segment
.to_string()
.contains("Illegal namespace component"),
"{by_segment}"
);
assert!(
@@ -4165,28 +4205,20 @@ mod tests {
}
}
/// A segment outside the table charset still addresses one segment: the
/// service decides whether it may exist, and percent-encoding is what keeps
/// the question reaching the right route. A catalog database is named this
/// way.
/// Slashes belong to catalog database names, not table namespace segments.
#[tokio::test]
async fn test_a_namespace_segment_outside_the_charset_is_encoded_not_refused() {
use std::sync::{Arc, Mutex};
let seen = Arc::new(Mutex::new(String::new()));
let path = seen.clone();
let conn = Connection::new_with_handler(move |request| {
*path.lock().unwrap() = request.url().path().to_string();
http::Response::builder().status(200).body("{}").unwrap()
async fn test_a_namespace_segment_outside_the_charset_is_refused() {
let conn = Connection::new_with_handler(|_| -> http::Response<String> {
panic!("invalid namespace reached the transport")
});
conn.drop_table("t", &["team/search".to_string()])
.await
.unwrap();
assert_eq!(*seen.lock().unwrap(), "/v1/table/team%2Fsearch$t/drop/");
assert!(matches!(
conn.drop_table("t", &["team/search".to_string()]).await,
Err(Error::InvalidInput { .. })
));
}
/// A Function name is percent-encoded, which covers everything but the
/// relative segment: `..` is unreserved, so it survives encoding and is
/// then resolved away, posting a registration body to `/v1/create`.
/// Reject relative segments and their encoded forms before sending a request,
/// so a function name cannot change the route through URL normalization.
#[tokio::test]
async fn test_a_relative_segment_function_name_is_refused() {
use std::sync::{Arc, Mutex};
@@ -4201,10 +4233,7 @@ mod tests {
.drop_function(name, "fv_1", &[])
.await
.expect_err("a dot-only Function name must be refused");
assert!(
error.to_string().contains("relative path segments"),
"{error}"
);
assert!(matches!(error, Error::InvalidInput { .. }), "{error}");
assert!(!*reached.lock().unwrap(), "{name:?} reached the transport");
}
}
@@ -4585,7 +4614,7 @@ mod tests {
};
// A view is not a table, so the refusal says so.
assert!(
error.contains("view name") || error.contains("view namespace path segment"),
error.contains("view name") || error.contains("namespace component"),
"{error}"
);
}
rust/lancedb/src/remote/table.rs
+138 -29
View File
@@ -47,6 +47,7 @@ use crate::utils::background_cache::BackgroundCache;
use crate::utils::{
MaxBatchLengthStream, TimeoutStream, public_fts_field_path_by_id, resolve_arrow_field_path,
resolve_arrow_fts_field_path, supported_btree_data_type, supported_vector_data_type,
validate_fts_field,
};
use crate::{DistanceType, Error};
use crate::{
@@ -76,7 +77,6 @@ use lance::dataset::{ColumnAlteration, NewColumnTransform, Version};
use lance_datafusion::exec::{OneShotExec, execute_plan};
use reqwest::{RequestBuilder, Response};
use serde::{Deserialize, Serialize};
use serde_json::Number;
use std::collections::{HashMap, HashSet};
use std::io::Cursor;
use std::pin::Pin;
@@ -597,6 +597,7 @@ impl<S: HttpSend> RemoteTable<S> {
Index::BloomFilter(p) => ("BLOOM_FILTER", Some(to_json(p)?)),
Index::RTree(p) => ("RTREE", Some(to_json(p)?)),
Index::FTS(p) => {
validate_fts_field(&field)?;
let mut params = to_json(p)?;
if p.get_document_granularity().is_list_element() {
params["document_granularity"] = "list_element".into();
@@ -1136,6 +1137,7 @@ impl<S: HttpSend> RemoteTable<S> {
mut body: serde_json::Value,
query: &VectorQueryRequest,
) -> Result<Vec<serde_json::Value>> {
query.validate_search_effort()?;
self.apply_query_params(&mut body, &query.base)?;
// Apply general parameters, before we dispatch based on number of query vectors.
@@ -1145,17 +1147,17 @@ impl<S: HttpSend> RemoteTable<S> {
if let Some(approx_mode) = query.approx_mode {
body["approx_mode"] = serde_json::json!(approx_mode);
}
// In 0.23.1 we migrated from `nprobes` to `minimum_nprobes` and `maximum_nprobes`.
// Old client / new server: since minimum_nprobes is missing, fallback to nprobes
// New client / old server: old server will only see nprobes, make sure to set both
// nprobes and minimum_nprobes
// New client / new server: since minimum_nprobes is present, server can ignore nprobes
body["nprobes"] = query.minimum_nprobes.into();
body["minimum_nprobes"] = query.minimum_nprobes.into();
if let Some(nprobes) = query.nprobes {
body["nprobes"] = nprobes.into();
}
if let Some(minimum_nprobes) = query.minimum_nprobes {
body["minimum_nprobes"] = minimum_nprobes.into();
}
if let Some(maximum_nprobes) = query.maximum_nprobes {
body["maximum_nprobes"] = maximum_nprobes.into();
} else {
body["maximum_nprobes"] = serde_json::Value::Number(Number::from_u128(0).unwrap())
}
if let Some(search_effort) = query.search_effort {
body["search_effort"] = search_effort.into();
}
body["lower_bound"] = query.lower_bound.into();
body["upper_bound"] = query.upper_bound.into();
@@ -6088,9 +6090,6 @@ mod tests {
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
let mut expected_body = serde_json::json!({
"prefilter": true,
"nprobes": 20,
"minimum_nprobes": 20,
"maximum_nprobes": 20,
"lower_bound": Option::<f32>::None,
"upper_bound": Option::<f32>::None,
"k": 10,
@@ -6122,7 +6121,7 @@ mod tests {
}
#[tokio::test]
async fn test_query_vector_approx_mode_sent_when_set() {
async fn test_query_vector_minimum_only_and_approx_mode() {
let expected_data = RecordBatch::try_new(
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
@@ -6142,9 +6141,7 @@ mod tests {
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
let mut expected_body = serde_json::json!({
"prefilter": true,
"nprobes": 20,
"minimum_nprobes": 20,
"maximum_nprobes": 20,
"minimum_nprobes": 5,
"approx_mode": "accurate",
"lower_bound": Option::<f32>::None,
"upper_bound": Option::<f32>::None,
@@ -6168,6 +6165,8 @@ mod tests {
.query()
.nearest_to(vec![0.1, 0.2, 0.3])
.unwrap()
.minimum_nprobes(5)
.unwrap()
.approx_mode(crate::ApproxMode::Accurate)
.execute()
.await;
@@ -6176,6 +6175,39 @@ mod tests {
assert_eq!(data[0].as_ref().unwrap(), &expected_data);
}
#[tokio::test]
async fn test_query_vector_explicitly_unbounded_maximum() {
let table = Table::new_with_handler("my_table", |request| {
let body = request.body().unwrap().as_bytes().unwrap();
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(body["nprobes"], 5);
assert_eq!(body["maximum_nprobes"], 0);
let data = RecordBatch::try_new(
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
vec![Arc::new(Int32Array::from(vec![1]))],
)
.unwrap();
http::Response::builder()
.status(200)
.header(CONTENT_TYPE, ARROW_FILE_CONTENT_TYPE)
.body(write_ipc_file(&data))
.unwrap()
});
table
.query()
.nearest_to(vec![0.1, 0.2, 0.3])
.unwrap()
.nprobes(5)
.unwrap()
.maximum_nprobes(None)
.unwrap()
.execute()
.await
.unwrap();
}
#[tokio::test]
async fn test_query_fts_default_values() {
let expected_data = RecordBatch::try_new(
@@ -6241,6 +6273,77 @@ mod tests {
));
}
#[rstest]
#[case::unset(None)]
#[case::minimum(Some(0.0))]
#[case::low(Some(0.25))]
#[case::auto(Some(0.5))]
#[case::high(Some(0.75))]
#[case::maximum(Some(1.0))]
#[tokio::test]
async fn test_query_vector_search_effort_wire(#[case] effort: Option<f64>) {
let table = Table::new_with_handler("my_table", move |request| {
let body = request_body_json(&request);
assert_eq!(body["nprobes"], 7);
assert_eq!(body["minimum_nprobes"], 2);
assert_eq!(body["maximum_nprobes"], 8);
if let Some(effort) = effort {
assert_eq!(body["search_effort"], effort);
} else {
assert!(body.get("search_effort").is_none());
}
let batch = RecordBatch::try_new(
Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])),
vec![Arc::new(Int32Array::from(vec![1]))],
)
.unwrap();
http::Response::builder()
.status(200)
.header(CONTENT_TYPE, ARROW_FILE_CONTENT_TYPE)
.body(write_ipc_file(&batch))
.unwrap()
});
let batches = table
.query()
.nearest_to(vec![0.1, 0.2])
.unwrap()
.search_effort(effort)
.unwrap()
.nprobes(7)
.unwrap()
.minimum_nprobes(2)
.unwrap()
.maximum_nprobes(Some(8))
.unwrap()
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
assert_eq!(batches[0].num_rows(), 1);
}
#[rstest]
#[case::negative(-0.1)]
#[case::too_large(1.1)]
#[case::nan(f64::NAN)]
#[case::infinity(f64::INFINITY)]
#[case::negative_infinity(f64::NEG_INFINITY)]
#[tokio::test]
async fn test_query_vector_search_effort_invalid(#[case] effort: f64) {
let table = Table::new_with_handler::<&str>("my_table", |_| {
panic!("invalid search_effort must not send a request")
});
let result = table
.query()
.nearest_to(vec![0.1, 0.2])
.unwrap()
.search_effort(effort);
assert!(matches!(result, Err(Error::InvalidInput { message })
if message.contains("search_effort must be finite and in [0, 1]")));
}
#[tokio::test]
async fn test_query_vector_all_params() {
let table = Table::new_with_handler("my_table", |request| {
@@ -6274,8 +6377,8 @@ mod tests {
}
],
"nprobes": 12,
"minimum_nprobes": 12,
"maximum_nprobes": 12,
"minimum_nprobes": 3,
"maximum_nprobes": 10,
"lower_bound": Option::<f32>::None,
"upper_bound": Option::<f32>::None,
"ef": Option::<usize>::None,
@@ -6315,6 +6418,10 @@ mod tests {
.distance_type(crate::DistanceType::Cosine)
.nprobes(12)
.unwrap()
.minimum_nprobes(3)
.unwrap()
.maximum_nprobes(Some(10))
.unwrap()
.refine_factor(2)
.bypass_vector_index()
.execute()
@@ -6345,9 +6452,6 @@ mod tests {
"vector_column": "image.embedding",
"prefilter": true,
"k": 10,
"nprobes": 20,
"minimum_nprobes": 20,
"maximum_nprobes": 20,
"lower_bound": Option::<f32>::None,
"upper_bound": Option::<f32>::None,
"ef": Option::<usize>::None,
@@ -7083,19 +7187,24 @@ mod tests {
];
for (index_type, expected_body, index) in cases {
let data_type = match &index {
Index::FTS(params) if params.get_document_granularity().is_list_element() => {
DataType::List(Arc::new(Field::new("item", DataType::Utf8, true)))
}
Index::FTS(_) => DataType::Utf8,
_ => DataType::Int32,
};
let schema = Schema::new(vec![Field::new("a", data_type, false)]);
let table = Table::new_with_handler_version(
"my_table",
semver::Version::new(0, 6, 0),
move |request| {
assert_eq!(request.method(), "POST");
match request.url().path() {
"/v1/table/my_table/describe/" => {
let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
http::Response::builder()
.status(200)
.body(describe_response(&schema))
.unwrap()
}
"/v1/table/my_table/describe/" => http::Response::builder()
.status(200)
.body(describe_response(&schema))
.unwrap(),
"/v1/table/my_table/create_index/" => {
assert_eq!(
request.headers().get("Content-Type").unwrap(),
rust/lancedb/src/table.rs
+54 -1
View File
@@ -19,6 +19,9 @@ pub use lance::dataset::ReadParams;
pub use lance::dataset::Version;
use lance::dataset::WriteMode;
use lance::dataset::builder::DatasetBuilder;
pub use lance::dataset::mem_wal::MemIndexRegistry;
/// The traits a memtable index plugin implements; see [`MemIndexRegistry`].
pub use lance::dataset::mem_wal::index as mem_index;
use lance::dataset::{InsertBuilder, WriteParams};
use lance::index::DatasetIndexExt;
use lance::index::scalar::load_segment_params;
@@ -424,7 +427,8 @@ impl LsmWriteSpec {
/// changes, so an index created later is maintained too; one of a kind the
/// MemWAL cannot mirror is skipped rather than failing the table. A list is
/// verbatim: each name must already exist and be maintainable, and an empty
/// list maintains nothing.
/// list maintains nothing. The maintainable kinds are Lance's built-ins plus
/// any added with [`NativeTable::set_mem_index_registry`].
///
/// ```
/// # use lancedb::table::LsmWriteSpec;
@@ -2590,6 +2594,9 @@ pub struct NativeTable {
pub(crate) pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
// Read-freshness baseline; `Some` only for namespace-backed tables.
freshness: Option<TableFreshness>,
// Memtable index kinds this handle's LSM writes maintain, shared with its
// clones and branch handles.
mem_index_registry: Arc<std::sync::RwLock<MemIndexRegistry>>,
}
impl std::fmt::Debug for NativeTable {
@@ -2756,9 +2763,51 @@ impl NativeTable {
namespace_client,
pushdown_operations,
freshness: None,
mem_index_registry: Default::default(),
})
}
/// Maintain the memtable index kinds in `registry` when this table writes
/// through its [`LsmWriteSpec`], and accept them when
/// [`LsmWriteSpec::with_maintained_indexes`] names one.
///
/// The registry belongs to this handle, its clones and its branch handles,
/// and is not stored with the table: a handle opened separately starts from
/// Lance's built-ins. Set it before the handle's first LSM write; writers
/// already open keep the registry they were opened with. Local tables only,
/// reached through [`Table::as_native`].
///
/// ```
/// # use std::sync::Arc;
/// # use lancedb::Table;
/// # use lancedb::table::{LsmWriteSpec, MemIndexRegistry, mem_index::MemIndexPlugin};
/// # async fn example(
/// # table: &Table,
/// # plugin: Arc<dyn MemIndexPlugin>,
/// # ) -> Result<(), Box<dyn std::error::Error>> {
/// let registry = MemIndexRegistry::default().with_plugin(plugin)?;
/// if let Some(native) = table.as_native() {
/// native.set_mem_index_registry(registry);
/// }
/// let spec = LsmWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]);
/// table.set_lsm_write_spec(spec).await?;
/// # Ok(())
/// # }
/// ```
pub fn set_mem_index_registry(&self, registry: MemIndexRegistry) {
*self
.mem_index_registry
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = registry;
}
pub(crate) fn mem_index_registry(&self) -> MemIndexRegistry {
self.mem_index_registry
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
}
/// Set the namespace client for server-side query execution.
///
/// When set, queries will be executed on the namespace server instead of locally.
@@ -2786,6 +2835,7 @@ impl NativeTable {
namespace_client: self.namespace_client.clone(),
pushdown_operations: self.pushdown_operations.clone(),
freshness: self.freshness.clone(),
mem_index_registry: self.mem_index_registry.clone(),
}
}
@@ -2893,6 +2943,7 @@ impl NativeTable {
namespace_client: stored_namespace_client,
pushdown_operations,
freshness: None,
mem_index_registry: Default::default(),
})
}
@@ -2984,6 +3035,7 @@ impl NativeTable {
namespace_client,
pushdown_operations,
freshness: None,
mem_index_registry: Default::default(),
})
}
@@ -3117,6 +3169,7 @@ impl NativeTable {
namespace_client: stored_namespace_client,
pushdown_operations,
freshness: None,
mem_index_registry: Default::default(),
})
}
rust/lancedb/src/table/create_index.rs
+3 -3
View File
@@ -29,8 +29,8 @@ use crate::index::Index;
use crate::index::vector::{VectorIndex, suggested_num_sub_vectors};
use crate::utils::{
resolve_lance_fts_field_path, supported_bitmap_data_type, supported_btree_data_type,
supported_fm_data_type, supported_fts_data_type, supported_label_list_data_type,
supported_vector_data_type, supported_zonemap_data_type,
supported_fm_data_type, supported_label_list_data_type, supported_vector_data_type,
supported_zonemap_data_type, validate_fts_field,
};
use super::NativeTable;
@@ -292,7 +292,7 @@ impl NativeTable {
))
}
Index::FTS(fts_opts) => {
Self::validate_index_type(field, "FTS", supported_fts_data_type)?;
validate_fts_field(field)?;
Ok(Box::new(fts_opts))
}
Index::IvfFlat(index) => {
rust/lancedb/src/table/datafusion.rs
+38
View File
@@ -699,6 +699,44 @@ pub mod tests {
);
}
#[tokio::test]
async fn test_topk_late_materialization_over_table_adapter() {
use lance::io::exec::topk_late_materialization::TopKLateMaterialization;
let fixture = TestFixture::new().await;
let run = |rule: Option<TopKLateMaterialization>| {
let adapter = fixture.adapter2.clone();
async move {
let mut state =
datafusion::execution::SessionStateBuilder::new().with_default_features();
if let Some(rule) = rule {
state = state.with_physical_optimizer_rule(Arc::new(rule));
}
let ctx = SessionContext::new_with_state(state.build());
ctx.register_table("tbl2", adapter).unwrap();
let df = ctx
.sql("SELECT ints, strings, jsons FROM tbl2 WHERE floats > 10 ORDER BY ints DESC LIMIT 5")
.await
.unwrap();
let plan = df.clone().create_physical_plan().await.unwrap();
let plan = datafusion_physical_plan::displayable(plan.as_ref())
.indent(true)
.to_string();
(plan, df.collect().await.unwrap())
}
};
let (plan, expected) = run(None).await;
assert!(!plan.contains("Take"), "{plan}");
// The sort lands below `MetadataEraserExec`, so the rule needs no
// certification for it.
let (plan, batches) = run(Some(TopKLateMaterialization::new())).await;
assert!(plan.contains("Take"), "{plan}");
assert_eq!(batches, expected);
assert!(batches[0].schema().metadata().is_empty());
}
#[tokio::test]
async fn test_metadata_eraser_with_new_children_takes_child_schema() {
let fixture = TestFixture::new().await;
rust/lancedb/src/table/datafusion/udtf/vector_query.rs
+1 -3
View File
@@ -411,9 +411,7 @@ impl ExecutionPlan for VectorDedupExec {
.map_err(|e| datafusion_common::DataFusionError::External(Box::new(e)))?;
let mut scanner = dataset.scan();
scanner
.with_row_addr_prefilter(RowAddrMask::from_block(RowAddrTreeMap::from_iter(
removed,
)))
.with_row_id_prefilter(RowAddrMask::from_block(RowAddrTreeMap::from_iter(removed)))
.blob_handling(BlobHandling::AllBinary)
.batch_size(MATERIALIZE_BATCH_ROWS)
.strict_batch_size(true);
rust/lancedb/src/table/merge.rs
+474 -33
View File
@@ -1296,6 +1296,21 @@ mod lsm_tests {
rows
}
async fn collect_ids(stream: SendableRecordBatchStream) -> Vec<i64> {
let batches: Vec<_> = stream.try_collect().await.unwrap();
batches
.iter()
.flat_map(|batch| {
batch
.column_by_name("id")
.unwrap()
.as_primitive::<Int64Type>()
.values()
.to_vec()
})
.collect()
}
/// Upsert `ids` (value = 0..n) through the LSM `merge_insert` path.
async fn lsm_upsert(table: &Table, ids: Vec<i64>) {
let mut builder = table.merge_insert(&[]);
@@ -1767,12 +1782,16 @@ mod lsm_tests {
assert_eq!(rows, 1, "LSM FTS must still see base rows");
}
#[tokio::test]
async fn lsm_read_vector_search() {
/// A table whose vector index the LSM maintains: base rows filled with
/// their id, and one memtable row filled with 1000, nearest to `[1000; 8]`.
/// `registry` replaces the table's memtable kinds.
async fn lsm_vector_table(
dir: &std::path::Path,
registry: Option<lance::dataset::mem_wal::MemIndexRegistry>,
) -> Table {
use crate::index::Index;
use crate::index::vector::IvfPqIndexBuilder;
use arrow::array::{FixedSizeListBuilder, Float32Builder};
use arrow::datatypes::Int64Type;
const DIM: i32 = 8;
const N: i64 = 256;
@@ -1801,12 +1820,7 @@ mod lsm_tests {
.unwrap()
};
let dir = tempdir().unwrap();
let conn = connect(dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
// Base rows fill each vector with its own id (0..256); all far from 1000.
let conn = connect(dir.to_str().unwrap()).execute().await.unwrap();
let base = make_batch((0..N).map(|i| (i, i as f32)).collect());
let base_reader: Box<dyn RecordBatchReader + Send> =
Box::new(RecordBatchIterator::new(vec![Ok(base)], schema.clone()));
@@ -1817,20 +1831,22 @@ mod lsm_tests {
&["vec"],
Index::IvfPq(
IvfPqIndexBuilder::default()
.num_partitions(1)
.num_partitions(8)
.num_sub_vectors(2),
),
)
.execute()
.await
.unwrap();
if let Some(registry) = registry {
table.as_native().unwrap().set_mem_index_registry(registry);
}
let vec_index = table.list_indices().await.unwrap()[0].name.clone();
table
.set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![vec_index]))
.await
.unwrap();
// Insert a vector (filled with 1000) that is nearest to the query.
let mut builder = table.merge_insert(&[]);
builder
.when_matched_update_all(None)
@@ -1840,32 +1856,457 @@ mod lsm_tests {
schema.clone(),
));
builder.execute(insert_reader).await.unwrap();
table
}
// KNN near [1000; DIM]: the default (auto-routed) read surfaces the
// memtable row.
let stream = table
.query()
.nearest_to(&[1000.0_f32; 8])
.unwrap()
.limit(1)
.execute()
.await
.unwrap();
let batches: Vec<_> = stream.try_collect().await.unwrap();
let ids: Vec<i64> = batches
.iter()
.flat_map(|b| {
b.column_by_name("id")
.unwrap()
.as_primitive::<Int64Type>()
.values()
.to_vec()
})
.collect();
/// The ids of the nearest row to `[1000; 8]` through the default read.
async fn nearest_ids(table: &Table) -> Vec<i64> {
let query = table.query().nearest_to(&[1000.0_f32; 8]).unwrap().limit(1);
collect_ids(query.execute().await.unwrap()).await
}
#[tokio::test]
async fn lsm_read_vector_search() {
let dir = tempdir().unwrap();
let table = lsm_vector_table(dir.path(), None).await;
assert_eq!(
ids,
nearest_ids(&table).await,
vec![9999],
"LSM vector search must rank the memtable row first"
);
let base_query = table
.query()
.nearest_to(&[0.0_f32; 8])
.unwrap()
.only_if("id = 255")
.limit(1);
for query in [
base_query.clone(),
base_query.clone().nprobes(8).unwrap(),
base_query.clone().minimum_nprobes(8).unwrap(),
base_query.clone().maximum_nprobes(Some(8)).unwrap(),
base_query
.nprobes(1)
.unwrap()
.maximum_nprobes(None)
.unwrap(),
] {
let ids = collect_ids(query.execute().await.unwrap()).await;
assert_eq!(ids, vec![255]);
}
}
/// The built-in vector plugin, except that its index answers no search.
#[derive(Debug)]
struct AnswersNothing;
#[derive(Debug)]
struct AnswersNothingIndex(Arc<dyn lance::dataset::mem_wal::index::MemIndex>);
fn vector_plugin() -> Arc<dyn lance::dataset::mem_wal::index::MemIndexPlugin> {
Arc::new(lance::dataset::mem_wal::index::HnswMemIndexPlugin)
}
#[async_trait::async_trait]
impl lance::dataset::mem_wal::index::MemIndexPlugin for AnswersNothing {
fn name(&self) -> &str {
"AnswersNothing"
}
fn details_message(&self) -> &str {
"VectorIndexDetails"
}
fn flush_index_type(&self) -> lance_index::IndexType {
vector_plugin().flush_index_type()
}
fn training_criteria(&self) -> lance_index::scalar::registry::TrainingCriteria {
vector_plugin().training_criteria()
}
async fn resolve(
&self,
ctx: &lance::dataset::mem_wal::index::ResolveContext<'_>,
) -> lance_core::Result<lance::dataset::mem_wal::index::ResolvedIndex> {
vector_plugin().resolve(ctx).await
}
fn validate(
&self,
ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>,
) -> lance_core::Result<()> {
vector_plugin().validate(ctx)
}
fn create(
&self,
ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>,
) -> lance_core::Result<Arc<dyn lance::dataset::mem_wal::index::MemIndex>> {
Ok(Arc::new(AnswersNothingIndex(vector_plugin().create(ctx)?)))
}
}
#[async_trait::async_trait]
impl lance::dataset::mem_wal::index::MemIndex for AnswersNothingIndex {
fn columns(&self) -> &[String] {
self.0.columns()
}
fn can_answer(&self, _query: &dyn lance::dataset::mem_wal::index::MemQuery) -> bool {
false
}
fn insert(&self, batch: &RecordBatch, row_offset: u64) -> lance_core::Result<()> {
self.0.insert(batch, row_offset)
}
fn insert_batches(
&self,
batches: &[lance::dataset::mem_wal::write::StoredBatch],
) -> lance_core::Result<()> {
self.0.insert_batches(batches)
}
fn resident_bytes(&self) -> usize {
self.0.resident_bytes()
}
fn search(
&self,
_query: &dyn lance::dataset::mem_wal::index::MemQuery,
_ctx: &lance::dataset::mem_wal::index::SearchContext,
) -> lance_core::Result<Option<lance::dataset::mem_wal::index::MemMatches>> {
Ok(None)
}
async fn flush(
&self,
ctx: &lance::dataset::mem_wal::index::FlushContext<'_>,
) -> lance_core::Result<lance::dataset::mem_wal::index::FlushOutcome> {
self.0.flush(ctx).await
}
}
/// A resident memtable whose vector index answers nothing still serves an
/// LSM vector search: Lance compares every row instead.
#[tokio::test]
async fn lsm_vector_search_compares_every_row_when_no_resident_index_answers() {
let dir = tempdir().unwrap();
let mut registry = lance::dataset::mem_wal::MemIndexRegistry::default();
registry.replace_plugin(Arc::new(AnswersNothing)).unwrap();
let table = lsm_vector_table(dir.path(), Some(registry)).await;
assert_eq!(nearest_ids(&table).await, vec![9999]);
// The second page starts after the memtable row.
let page = table
.query()
.nearest_to(&[1000.0_f32; 8])
.unwrap()
.limit(2)
.offset(1);
assert_eq!(
collect_ids(page.execute().await.unwrap()).await,
vec![255, 254]
);
}
/// Claims the base table's bitmap index and keeps it as a B-tree, whose
/// flush rows are in the shape the bitmap trainer takes.
#[derive(Debug)]
struct BitmapAsBTree;
fn btree_plugin() -> Arc<dyn lance::dataset::mem_wal::index::MemIndexPlugin> {
lance::dataset::mem_wal::MemIndexRegistry::default()
.plugin_for_details_url("/lance.table.BTreeIndexDetails")
.unwrap()
.clone()
}
#[async_trait::async_trait]
impl lance::dataset::mem_wal::index::MemIndexPlugin for BitmapAsBTree {
fn name(&self) -> &str {
"BitmapAsBTree"
}
fn details_message(&self) -> &str {
"BitmapIndexDetails"
}
fn flush_index_type(&self) -> lance_index::IndexType {
lance_index::IndexType::Bitmap
}
fn training_criteria(&self) -> lance_index::scalar::registry::TrainingCriteria {
btree_plugin().training_criteria()
}
fn validate(
&self,
ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>,
) -> lance_core::Result<()> {
btree_plugin().validate(ctx)
}
fn create(
&self,
ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>,
) -> lance_core::Result<Arc<dyn lance::dataset::mem_wal::index::MemIndex>> {
btree_plugin().create(ctx)
}
}
/// A kind Lance has no memtable index for is refused when named until the
/// table's registry maintains it; then it is maintained, flushed with each
/// generation, and answers searches on the flushed generations.
#[tokio::test]
async fn lsm_maintains_an_index_kind_from_the_tables_registry() {
let dir = tempdir().unwrap();
let conn = connect(dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
let table = conn
.create_table("t", id_region_reader(vec![(1, "us"), (2, "eu"), (3, "us")]))
.execute()
.await
.unwrap();
table.set_unenforced_primary_key(["id"]).await.unwrap();
table
.create_index(&["region"], crate::index::Index::Bitmap(Default::default()))
.name("region_bitmap".to_string())
.execute()
.await
.unwrap();
let spec =
|| LsmWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]);
let refused = table.set_lsm_write_spec(spec()).await.unwrap_err();
assert!(
refused.to_string().contains("region_bitmap")
&& refused
.to_string()
.contains("no registered plugin maintains"),
"{refused:?}"
);
let registry = lance::dataset::mem_wal::MemIndexRegistry::default()
.with_plugin(Arc::new(BitmapAsBTree))
.unwrap();
table.as_native().unwrap().set_mem_index_registry(registry);
table.set_lsm_write_spec(spec()).await.unwrap();
// The registry is not stored with the table.
let other = conn.open_table("t").execute().await.unwrap();
let mut builder = other.merge_insert(&["id"]);
builder
.when_matched_update_all(None)
.when_not_matched_insert_all();
let refused = builder
.execute(id_region_reader(vec![(20, "us")]))
.await
.unwrap_err();
assert!(
refused
.to_string()
.contains("no registered plugin maintains"),
"{refused:?}"
);
let mut builder = table.merge_insert(&["id"]);
builder
.when_matched_update_all(None)
.when_not_matched_insert_all();
builder
.execute(id_region_reader(vec![
(10, "fresh"),
(11, "fresh"),
(12, "eu"),
]))
.await
.unwrap();
let fresh = || async {
let query = table.query().only_if("region = 'fresh'");
let mut ids = collect_ids(query.execute().await.unwrap()).await;
ids.sort_unstable();
ids
};
assert_eq!(fresh().await, vec![10, 11]);
let (_, _, memtables) = table
.as_native()
.unwrap()
.dataset
.shard_writer()
.read_snapshot()
.await
.unwrap()
.expect("the merge opened a writer");
let active = memtables.expect("the writer keeps memtables").active;
assert!(
active
.index_store
.index_names()
.contains(&"region_bitmap".to_string()),
"the memtable maintains the plugin's index: {:?}",
active.index_store.index_names()
);
table.close_lsm_writers().await.unwrap();
assert_eq!(fresh().await, vec![10, 11]);
assert_generations_answer_through_region_bitmap(&table, vec![10, 11]).await;
}
/// The generations each shard's manifest publishes.
async fn published_generations(table: &Table) -> Vec<lance::Dataset> {
use lance::dataset::mem_wal::ShardManifestStore;
let uri = table.uri().await.unwrap();
let (store, base) = lance_io::object_store::ObjectStore::from_uri(&uri)
.await
.unwrap();
let mem_wal = std::path::Path::new(&uri).join("_mem_wal");
let mut generations = Vec::new();
for shard in std::fs::read_dir(&mem_wal).unwrap() {
let Ok(shard_id) = shard.unwrap().file_name().to_string_lossy().parse() else {
continue;
};
let manifest = ShardManifestStore::new(store.clone(), &base, shard_id, 2)
.latest()
.await
.unwrap()
.expect("a shard directory has a manifest");
for sstable in manifest.sstables {
let path = mem_wal.join(shard_id.to_string()).join(&sstable.path);
generations.push(lance::Dataset::open(path.to_str().unwrap()).await.unwrap());
}
}
generations
}
/// Every published generation carries `region_bitmap` as a bitmap index on
/// `region`, and a search for `region = 'fresh'` on it uses that index.
async fn assert_generations_answer_through_region_bitmap(table: &Table, fresh: Vec<i64>) {
use lance::index::DatasetIndexExt;
let generations = published_generations(table).await;
assert!(
!generations.is_empty(),
"closing the writer flushed a generation"
);
let mut found = Vec::new();
for generation in generations {
let region = generation.schema().field("region").unwrap().id;
let indices = generation.load_indices().await.unwrap();
let index = indices
.iter()
.find(|index| index.name == "region_bitmap")
.expect("the generation carries the index the plugin trained");
assert_eq!(index.fields, vec![region]);
assert!(
index
.index_details
.as_ref()
.is_some_and(|details| details.type_url.ends_with("BitmapIndexDetails")),
"{:?}",
index.index_details
);
let mut scan = generation.scan();
scan.filter("region = 'fresh'").unwrap();
let plan = scan.explain_plan(false).await.unwrap();
assert!(plan.contains("ScalarIndexQuery"), "{plan}");
let batch = scan.try_into_batch().await.unwrap();
found.extend(
batch["id"]
.as_primitive::<Int64Type>()
.values()
.iter()
.copied(),
);
}
found.sort_unstable();
assert_eq!(found, fresh);
}
/// Under the default set, a kind only the registry maintains is picked up
/// when its index is created after the writer opened, and the table's
/// clones share the registry.
#[tokio::test]
async fn lsm_default_set_maintains_a_registry_kind_created_while_writing() {
let dir = tempdir().unwrap();
let conn = connect(dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
let table = conn
.create_table("t", id_region_reader(vec![(1, "us"), (2, "eu")]))
.execute()
.await
.unwrap();
table.set_unenforced_primary_key(["id"]).await.unwrap();
let registry = lance::dataset::mem_wal::MemIndexRegistry::default()
.with_plugin(Arc::new(BitmapAsBTree))
.unwrap();
table.as_native().unwrap().set_mem_index_registry(registry);
let clone = table.clone();
assert!(
clone
.as_native()
.unwrap()
.mem_index_registry()
.plugin_for_details_url("/lance.table.BitmapIndexDetails")
.is_some(),
"a clone shares the handle's registry"
);
table
.set_lsm_write_spec(LsmWriteSpec::unsharded())
.await
.unwrap();
let upsert = |rows: Vec<(i64, &'static str)>| {
let table = table.clone();
async move {
let mut builder = table.merge_insert(&["id"]);
builder
.when_matched_update_all(None)
.when_not_matched_insert_all();
builder.execute(id_region_reader(rows)).await.unwrap();
}
};
// Opens the writer before the table has a bitmap index.
upsert(vec![(3, "us")]).await;
table
.create_index(&["region"], crate::index::Index::Bitmap(Default::default()))
.name("region_bitmap".to_string())
.execute()
.await
.unwrap();
upsert(vec![(10, "fresh"), (11, "fresh")]).await;
let (_, _, memtables) = table
.as_native()
.unwrap()
.dataset
.shard_writer()
.read_snapshot()
.await
.unwrap()
.expect("the merge opened a writer");
let active = memtables.expect("the writer keeps memtables").active;
assert!(
active
.index_store
.index_names()
.contains(&"region_bitmap".to_string()),
"the open writer picked up the new index: {:?}",
active.index_store.index_names()
);
table.close_lsm_writers().await.unwrap();
// Only the generation written after the index exists carries it.
let generations = published_generations(&table).await;
let mut carrying = 0;
for generation in &generations {
use lance::index::DatasetIndexExt;
if generation
.load_indices()
.await
.unwrap()
.iter()
.any(|index| index.name == "region_bitmap")
{
carrying += 1;
}
}
assert!(
carrying >= 1,
"{} generations, none carry the index",
generations.len()
);
}
}
rust/lancedb/src/table/merge/lsm.rs
+8 -4
View File
@@ -29,7 +29,7 @@ use arrow_schema::{DataType, Schema as ArrowSchema, SchemaRef};
use lance::Dataset;
use lance::dataset::mem_wal::{
DatasetMemWalExt, ShardWriter, ShardWriterConfig, evaluate_sharding_spec,
validate_maintained_indexes,
validate_maintained_indexes_with,
};
use lance::index::DatasetIndexExt;
use lance_core::datatypes::Schema as LanceSchema;
@@ -103,7 +103,8 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
}
}
let dataset = table.dataset.get().await?;
validate_maintained_indexes(&dataset, requested).await?;
validate_maintained_indexes_with(&dataset, requested, &table.mem_index_registry())
.await?;
Some(requested.to_vec())
}
None => None,
@@ -126,7 +127,9 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
.into(),
});
}
let mut builder = dataset.initialize_mem_wal();
let mut builder = dataset
.initialize_mem_wal()
.mem_index_registry(table.mem_index_registry());
let writer_config_defaults = match spec {
LsmWriteSpec::Bucket {
column,
@@ -725,7 +728,8 @@ pub(crate) async fn execute_lsm_merge_insert(
return Ok(lsm_merge_result(0));
};
let config = shard_writer_config_from_defaults(&plan.writer_config_defaults);
let mut config = shard_writer_config_from_defaults(&plan.writer_config_defaults);
config.mem_index_registry = table.mem_index_registry();
let writer = table
.dataset
.shard_writer()
rust/lancedb/src/table/query.rs
+104 -12
View File
@@ -106,19 +106,16 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R
}
fn requires_local_namespace_execution(query: &AnyQuery) -> bool {
// The namespace QueryTable request has no approx_mode or use_lsm field yet, so
// pushing these down would silently ignore the user's setting. For use_lsm that
// is worse than a tuning miss: MemWAL read routing lives only in `create_plan`,
// so a pushed-down query would return stale base-only data with no error.
// The namespace QueryTable request cannot represent approx_mode, search_effort or use_lsm.
// For use_lsm that is worse than a tuning miss: MemWAL read routing lives only
// in `create_plan`, so a pushed-down query would return stale base-only data.
if query.base().use_lsm.is_some() || query.base().take_offsets.is_some() {
return true;
}
matches!(
query,
AnyQuery::VectorQuery(VectorQueryRequest {
approx_mode: Some(_),
..
})
AnyQuery::VectorQuery(vector_query)
if vector_query.approx_mode.is_some() || vector_query.search_effort.is_some()
)
}
@@ -166,6 +163,7 @@ pub async fn create_plan(
AnyQuery::Query(query) => VectorQueryRequest::from_plain_query(query),
};
query.base.check_filter()?;
query.validate_search_effort()?;
let ds_ref = table.dataset.get().await?;
@@ -296,10 +294,24 @@ pub async fn create_plan(
scanner.approx_mode(approx_mode.into());
}
scanner.minimum_nprobes(query.minimum_nprobes);
if let Some(maximum_nprobes) = query.maximum_nprobes {
if query.search_effort.is_none()
&& let Some(nprobes) = query.nprobes
{
if query.maximum_nprobes == Some(0) {
scanner.minimum_nprobes(nprobes);
} else {
scanner.nprobes(nprobes);
}
}
if let Some(minimum_nprobes) = query.minimum_nprobes {
scanner.minimum_nprobes(minimum_nprobes);
}
if let Some(maximum_nprobes) = query.maximum_nprobes.filter(|nprobes| *nprobes != 0) {
scanner.maximum_nprobes(maximum_nprobes);
}
if let Some(search_effort) = query.search_effort {
scanner.search_effort(search_effort)?;
}
}
// For a batch query, `nearest` already applies k to each query vector.
@@ -598,6 +610,11 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result<NsQueryTableRequest> {
};
// Check for unsupported features
if vq.search_effort.is_some() {
return Err(Error::NotSupported {
message: "namespace QueryTable does not support search_effort; execute the query locally".to_string(),
});
}
if vq.base.reranker.is_some() {
return Err(Error::NotSupported {
message: "Reranker is not supported for server-side queries".to_string(),
@@ -630,7 +647,9 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result<NsQueryTableRequest> {
columns,
offset: vq.base.offset.map(|o| o as i32),
distance_type: vq.distance_type.map(|dt| dt.to_string()),
nprobes: Some(vq.minimum_nprobes as i32),
nprobes: vq.nprobes.map(|nprobes| nprobes as i32),
minimum_nprobes: vq.minimum_nprobes.map(|nprobes| nprobes as i32),
maximum_nprobes: vq.maximum_nprobes.map(|nprobes| nprobes as i32),
ef: vq.ef.map(|e| e as i32),
refine_factor: vq.refine_factor.map(|r| r as i32),
lower_bound: vq.lower_bound,
@@ -919,7 +938,6 @@ mod tests {
column: Some("vector".to_string()),
// We cast here to satisfy the struct definition
query_vector: vec![query_vector as Arc<dyn Array>],
minimum_nprobes: 20,
distance_type: Some(crate::DistanceType::L2),
..Default::default()
};
@@ -940,6 +958,7 @@ mod tests {
);
assert_eq!(ns_request.vector_column, Some("vector".to_string()));
assert_eq!(ns_request.distance_type, Some("l2".to_string()));
assert_eq!(ns_request.nprobes, None);
// Verify the vector data was extracted correctly
assert!(ns_request.vector.single_vector.is_some());
@@ -949,6 +968,79 @@ mod tests {
);
}
#[test]
fn test_namespace_query_probe_pushdown_compatibility() {
let default_query = AnyQuery::VectorQuery(VectorQueryRequest::default());
assert!(!requires_local_namespace_execution(&default_query));
for effort in [0.0, 0.5, 1.0] {
let effort_query = AnyQuery::VectorQuery(VectorQueryRequest {
query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))],
search_effort: Some(effort),
..Default::default()
});
assert!(requires_local_namespace_execution(&effort_query));
assert!(matches!(
convert_to_namespace_query(&effort_query),
Err(Error::NotSupported { message }) if message.contains("search_effort")
));
}
let nprobes_query = AnyQuery::VectorQuery(VectorQueryRequest {
query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))],
nprobes: Some(20),
..Default::default()
});
assert!(!requires_local_namespace_execution(&nprobes_query));
assert_eq!(
convert_to_namespace_query(&nprobes_query).unwrap().nprobes,
Some(20)
);
let minimum_query = AnyQuery::VectorQuery(VectorQueryRequest {
query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))],
minimum_nprobes: Some(5),
..Default::default()
});
assert!(!requires_local_namespace_execution(&minimum_query));
let request = convert_to_namespace_query(&minimum_query).unwrap();
assert_eq!(request.minimum_nprobes, Some(5));
assert_eq!(request.maximum_nprobes, None);
let maximum_query = AnyQuery::VectorQuery(VectorQueryRequest {
query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))],
maximum_nprobes: Some(10),
..Default::default()
});
assert!(!requires_local_namespace_execution(&maximum_query));
let request = convert_to_namespace_query(&maximum_query).unwrap();
assert_eq!(request.minimum_nprobes, None);
assert_eq!(request.maximum_nprobes, Some(10));
let exact_query = AnyQuery::VectorQuery(VectorQueryRequest {
query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))],
nprobes: Some(8),
minimum_nprobes: Some(20),
maximum_nprobes: Some(20),
..Default::default()
});
assert!(!requires_local_namespace_execution(&exact_query));
let request = convert_to_namespace_query(&exact_query).unwrap();
assert_eq!(request.nprobes, Some(8));
assert_eq!(request.minimum_nprobes, Some(20));
assert_eq!(request.maximum_nprobes, Some(20));
let explicitly_unbounded_query = AnyQuery::VectorQuery(VectorQueryRequest {
query_vector: vec![Arc::new(Float32Array::from(vec![1.0, 2.0]))],
nprobes: Some(8),
maximum_nprobes: Some(0),
..Default::default()
});
let request = convert_to_namespace_query(&explicitly_unbounded_query).unwrap();
assert_eq!(request.nprobes, Some(8));
assert_eq!(request.maximum_nprobes, Some(0));
}
#[test]
fn test_convert_to_namespace_query_plain_query() {
let q = QueryRequest {
rust/lancedb/src/table/query/lsm.rs
+71 -13
View File
@@ -170,10 +170,12 @@ fn reject_unsupported(query: &VectorQueryRequest) -> Result<()> {
// Vector-only knobs the LSM scanner cannot honor. Both change results rather
// than just recall, so error instead of silently ignoring them: distance_range
// would return rows outside the bound, and use_index(false) asks for a
// brute-force search the index-only base arm can't do. (ef / approx_mode /
// maximum_nprobes are recall/speed knobs and are left to no-op — and
// maximum_nprobes defaults to Some, so it cannot be rejected on presence.)
// brute-force search the index-only base arm can't do. (ef / approx_mode are
// recall/speed knobs and are left to no-op.)
if !query.query_vector.is_empty() {
if query.search_effort.is_some() {
return unsupported("search_effort on vector search");
}
if query.lower_bound.is_some() || query.upper_bound.is_some() {
return unsupported("distance_range on vector search");
}
@@ -475,7 +477,7 @@ async fn fts_plan(
// omits un-compacted documents. Reject rather than mislead.
if !index_maintained(dataset, column, details, "InvertedIndexDetails").await?
|| !resident_memtables_carry(&in_memory, |memtable| {
memtable.index_store.get_fts_by_column(column).is_some()
!memtable.index_store.fts_granularities_on(column).is_empty()
})
{
return Err(Error::NotSupported {
@@ -708,17 +710,14 @@ async fn vector_plan(
// The base arm relies on the column's vector index (`fast_search`). Unmaintained,
// its catch-up is untracked and exclusion falls back to the compaction watermark,
// dropping compacted SSTables the lagging base index has not re-indexed; and a
// resident MemTable built before the index joined the set carries none either.
// dropping compacted SSTables the lagging base index has not re-indexed.
// Reject rather than silently omit rows, mirroring the FTS arm.
if !index_maintained(dataset, &column, details, "VectorIndexDetails").await?
|| !resident_memtables_carry(&in_memory, |memtable| {
memtable.index_store.get_hnsw_by_column(&column).is_some()
})
{
// A resident MemTable needs no index of its own: one that cannot answer
// compares every row.
if !index_maintained(dataset, &column, details, "VectorIndexDetails").await? {
return Err(Error::NotSupported {
message: format!(
"the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only"
"the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only"
),
});
}
@@ -740,8 +739,20 @@ async fn vector_plan(
let mut scanner = base_scanner(dataset, query, pk_columns, snapshots, in_memory)?
.with_overfetch_factor(LSM_OVERFETCH_FACTOR)
.nearest(&column, query_vector.as_ref(), k)?
.nprobes(query.minimum_nprobes)
.distance_metric(distance_type.into());
if let Some(nprobes) = query.nprobes {
scanner = if query.maximum_nprobes == Some(0) {
scanner.minimum_nprobes(nprobes)
} else {
scanner.nprobes(nprobes)
};
}
if let Some(minimum_nprobes) = query.minimum_nprobes {
scanner = scanner.minimum_nprobes(minimum_nprobes);
}
if let Some(maximum_nprobes) = query.maximum_nprobes.filter(|nprobes| *nprobes != 0) {
scanner = scanner.maximum_nprobes(maximum_nprobes);
}
if let Some(refine_factor) = query.refine_factor {
scanner = scanner.refine(refine_factor);
}
@@ -793,6 +804,53 @@ mod tests {
use super::*;
use lance_index::mem_wal::{CompactedSsTable, IndexCatchupProgress};
#[test]
fn search_effort_is_not_silently_ignored() {
let query = VectorQueryRequest {
query_vector: vec![Arc::new(arrow_array::Float32Array::from(vec![1.0, 0.0]))],
search_effort: Some(0.5),
..Default::default()
};
assert!(
matches!(reject_unsupported(&query), Err(Error::NotSupported { message })
if message.contains("search_effort") && message.contains("un-compacted MemWAL"))
);
}
#[test]
fn resident_fts_check_distinguishes_columns() {
use arrow_schema::Field;
use lance::dataset::mem_wal::index::{IndexStore, MemIndexSpec};
let schema = ArrowSchema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("text", DataType::Utf8, true),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
true,
),
]);
let schema = lance_core::datatypes::Schema::try_from(&schema).unwrap();
let specs = vec![
MemIndexSpec::btree("id_idx", 0, "id"),
MemIndexSpec::fts("text_idx", 1, "text"),
MemIndexSpec::hnsw(
"vector_idx",
2,
"vector",
lance_linalg::distance::DistanceType::Cosine,
),
];
let store = IndexStore::from_specs(&specs, &schema, 1_000, 16).unwrap();
assert!(!store.fts_granularities_on("text").is_empty());
for column in ["id", "vector", "missing"] {
assert!(store.fts_granularities_on(column).is_empty());
}
let empty = IndexStore::from_specs(&[], &schema, 1_000, 16).unwrap();
assert!(empty.fts_granularities_on("text").is_empty());
}
#[test]
fn exclusion_watermark_gates_on_lagging_index_catchup() {
let shard = Uuid::from_u128(1);
rust/lancedb/src/utils/mod.rs
+289 -122
View File
@@ -19,15 +19,27 @@ use std::pin::Pin;
use crate::error::{Error, Result};
use datafusion_physical_plan::SendableRecordBatchStream;
/// The characters any object name may contain: a table, a namespace segment, a
/// Secret, a materialized view.
///
/// No positional rule on top of it -- a name may begin with `_`, `-` or `.`,
/// as LanceDB namespaces already do. `.` and `..` are excluded separately, by
/// [`reject_relative_segment`]: that is a property of where a name sits in a
/// URL, not of the name. Length is the service's to bound.
static OBJECT_NAME_REGEX: std::sync::LazyLock<regex::Regex> =
std::sync::LazyLock::new(|| regex::Regex::new(r"^[A-Za-z0-9_.\-]+$").unwrap());
/// The maximum length of a table name. Table names are used in URLs and in the keys used with the
/// object store.
const MAX_TABLE_NAME_CHARS: usize = 128;
/// The maximum length of a component of a namespace.
const MAX_NAMESPACE_NAME_CHARS: usize = 64;
/// The maximum number of namespace components that are allowed.
const MAX_NAMESPACE_COMPONENTS: usize = 10;
/// The maximum length of a view name.
const MAX_VIEW_NAME_CHARS: usize = 128;
/// The maximum length of a secret name.
const MAX_SECRET_NAME_CHARS: usize = 128;
/// The maximum length of a function name.
const MAX_FUNCTION_NAME_CHARS: usize = 128;
/// The maximum length of a database name.
const MAX_DB_NAME_CHARS: usize = 64;
pub trait PatchStoreParam {
fn patch_with_store_wrapper(
@@ -86,129 +98,190 @@ impl PatchReadParam for ReadParams {
}
}
/// The reason `.` and `..` are refused wherever a name becomes a path segment.
const RELATIVE_SEGMENT_REASON: &str =
"'.' and '..' are read as relative path segments and cannot address an object";
/// Whether URL parsing would resolve this component away rather than keep it.
///
/// Exactly `.` and `..`, and their percent-encoded spellings -- resolution
/// happens after decoding, so `%2E%2E` collapses as surely as `..` does, and
/// `drop_table("..")` would reach `/v1/drop/`. No wider than that: `...` is an
/// ordinary segment that addresses fine.
fn is_relative_segment(value: &str) -> bool {
let decoded = value.replace("%2e", ".").replace("%2E", ".");
decoded == "." || decoded == ".."
/// Translate a string into something that can be safely logged. This involves removing control
/// characters that could be used in an exploit like CVE-2003-0063.
fn sanitize(input: &str) -> String {
input.chars().filter(|c| !c.is_control()).collect()
}
/// Refuse a path component that URL parsing resolves as a relative segment.
///
/// Reachable on its own for an identifier with no other validator: a Function
/// name has no client-side grammar, so this is the only rule that applies.
pub(crate) fn reject_relative_segment(what: &str, value: &str) -> Result<()> {
if is_relative_segment(value) {
return Err(Error::InvalidInput {
message: format!("invalid {what} '{value}': {RELATIVE_SEGMENT_REASON}"),
pub fn validate_table_name(name: &str) -> Result<()> {
if name.is_empty() || name == "." || name == ".." {
return Err(Error::InvalidTableName {
name: name.to_string(),
reason: "Illegal table name.".to_string(),
});
}
if name.len() > MAX_TABLE_NAME_CHARS {
return Err(Error::InvalidTableName {
name: sanitize(name),
reason: format!(
"Table name cannot be more than {MAX_TABLE_NAME_CHARS} characters long."
),
});
}
if !name
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.')
{
return Err(Error::InvalidTableName {
name: sanitize(name),
reason: "Only alphanumeric characters, dashes, underscores, and\
periods are allowed in table names."
.to_string(),
});
}
Ok(())
}
/// Every rule an object name obeys: non-empty, inside [`OBJECT_NAME_REGEX`],
/// and addressable as a path segment.
///
/// Returns the reason rather than an [`Error`], because the error type is each
/// API's own -- a table reports [`Error::InvalidTableName`], the rest
/// [`Error::InvalidInput`]. Sharing the rules but not the error keeps a table,
/// a namespace segment and a Secret from drifting apart.
fn check_object_name(name: &str) -> std::result::Result<(), &'static str> {
if name.is_empty() {
return Err("it must not be empty");
/// Validate that an input string contains alphanumerics, or dashes, underscores, and periods.
fn validate_view_or_namespace_or_secret(input: &str, kind: &str, max_len: usize) -> Result<()> {
if input.is_empty() || input == "." || input == ".." {
return Err(Error::InvalidInput {
message: format!("Illegal {} '{}'.", kind, input),
});
}
if !OBJECT_NAME_REGEX.is_match(name) {
return Err(
"it may contain only alphanumeric characters, underscores, hyphens and periods",
);
if input.len() > max_len {
return Err(Error::InvalidInput {
message: format!(
"Invalid {} '{}' is more than {} characters long.",
kind,
sanitize(input),
max_len
),
});
}
if is_relative_segment(name) {
return Err(RELATIVE_SEGMENT_REASON);
if !input
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.')
{
return Err(Error::InvalidInput {
message: format!(
"Invalid {} '{}': only alphanumeric characters, \
dashes, underscores, and periods are allowed.",
kind,
sanitize(input)
),
});
}
Ok(())
}
/// Validate a table name.
pub fn validate_table_name(name: &str) -> Result<()> {
check_object_name(name).map_err(|reason| Error::InvalidTableName {
name: name.to_string(),
reason: reason.to_string(),
})
}
/// Validate one component of a namespace path -- a single segment, not the
/// whole path. [`validate_namespace`] covers a path.
pub fn validate_namespace_name(name: &str) -> Result<()> {
check_object_name(name).map_err(|reason| Error::InvalidInput {
message: format!("invalid namespace name '{name}': {reason}"),
})
pub fn validate_namespace_component(name: &str) -> Result<()> {
validate_view_or_namespace_or_secret(name, "namespace component", MAX_NAMESPACE_NAME_CHARS)
}
/// Validate one component of a Secret identifier: a Secret name, or one segment
/// of the namespace path holding it.
///
/// The join decides identity, and the service only sees what the split
/// produced. `"a$b"` is not a name the service accepts, but joined and split it
/// reads as the namespace `a` and the name `b` -- a different Secret that may
/// already exist. This is not a second opinion on the name; it is what lets the
/// service have one.
pub fn validate_secret_component(what: &str, value: &str) -> Result<()> {
check_object_name(value).map_err(|reason| Error::InvalidInput {
message: format!("invalid {what} '{value}': {reason}"),
})
/// Validate all components of a namespace
pub fn validate_namespace(namespace: &[String]) -> Result<()> {
if namespace.len() > 1 && namespace.first().map(String::as_str) == Some("public") {
return Err(Error::InvalidInput {
message: "Namespaces that begin with public cannot have multiple namespace components."
.to_string(),
});
}
if namespace.len() > MAX_NAMESPACE_COMPONENTS {
return Err(Error::InvalidInput {
message: format!(
"Cannot have a namespace with more than {MAX_NAMESPACE_COMPONENTS} components."
),
});
}
for component in namespace {
validate_namespace_component(component)?;
}
Ok(())
}
/// Validate a secret name.
pub fn validate_secret_name(name: &str) -> Result<()> {
validate_view_or_namespace_or_secret(name, "secret name", MAX_SECRET_NAME_CHARS)
}
/// Validate a Secret name and every segment of the namespace path holding it.
pub fn validate_secret_reference(name: &str, namespace_path: &[String]) -> Result<()> {
for segment in namespace_path {
validate_secret_component("Secret namespace path segment", segment)?;
}
validate_secret_component("Secret name", name)
validate_namespace(namespace_path)?;
validate_secret_name(name)
}
pub fn validate_view_name(name: &str) -> Result<()> {
validate_view_or_namespace_or_secret(name, "view name", MAX_VIEW_NAME_CHARS)
}
/// Validate a view name and every segment of the namespace path holding it.
///
/// Worded for a view rather than deferring to [`validate_table_name`]: a view
/// is not a table, and a caller who mistypes one should not be told their
/// table name is invalid. The rule is the same one every object name follows,
/// which is what lets the `$`-joined identifier split back apart.
pub fn validate_view_reference(name: &str, namespace_path: &[String]) -> Result<()> {
for segment in namespace_path {
validate_view_component("view namespace path segment", segment)?;
validate_namespace(namespace_path)?;
validate_view_name(name)
}
/// Validate a function name. These do not allow dashes or periods.
pub fn validate_function_name(name: &str) -> Result<()> {
if name.is_empty() {
return Err(Error::InvalidInput {
message: format!("Invalid function name '{}'.", name),
});
}
validate_view_component("view name", name)
if name.len() > MAX_FUNCTION_NAME_CHARS {
return Err(Error::InvalidInput {
message: format!(
"Invalid function name '{}' is more than {} characters long.",
sanitize(name),
MAX_FUNCTION_NAME_CHARS
),
});
}
if !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') {
return Err(Error::InvalidInput {
message: format!(
"Invalid function name '{}': only alphanumeric \
characters and underscores are allowed.",
sanitize(name)
),
});
}
Ok(())
}
/// Validate one component of a view identifier: a view name, or one segment of
/// the namespace path holding it.
pub fn validate_view_component(what: &str, value: &str) -> Result<()> {
check_object_name(value).map_err(|reason| Error::InvalidInput {
message: format!("invalid {what} '{value}': {reason}"),
})
}
/// Validate all components of a namespace
///
/// Iterates through all namespace components and validates each one.
/// Returns an error if any component is invalid.
///
/// # Arguments
/// * `namespace` - The namespace components to validate
///
/// # Returns
/// * `Ok(())` if all namespace components are valid
/// * `Err(Error)` if any component is invalid
pub fn validate_namespace(namespace: &[String]) -> Result<()> {
for component in namespace {
validate_namespace_name(component)?;
/// Validate a database name, including slash-separated components.
pub fn validate_database_name(name: &str) -> Result<()> {
if name.len() > MAX_DB_NAME_CHARS {
return Err(Error::InvalidInput {
message: format!(
"Invalid database name '{}' is more than {} characters long.",
sanitize(name),
MAX_DB_NAME_CHARS
),
});
}
if !name
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' || c == '/')
{
return Err(Error::InvalidInput {
message: format!(
"Invalid database name '{}': only alphanumeric characters, dashes, \
underscores, periods, and slashes are allowed.",
sanitize(name)
),
});
}
for component in name.split('/') {
if component.is_empty() {
return Err(Error::InvalidInput {
message: format!(
"Invalid database name '{}': invalid empty path component.",
sanitize(name)
),
});
}
if component == "." || component == ".." {
return Err(Error::InvalidInput {
message: format!(
"Invalid database name '{}': illegal path component.",
sanitize(name)
),
});
}
}
Ok(())
}
@@ -494,6 +567,26 @@ pub fn supported_fts_data_type(dtype: &DataType) -> bool {
supported_fts_data_type_impl(dtype, false)
}
/// Validate FTS input without losing the logical type in field metadata.
pub(crate) fn validate_fts_field(field: &Field) -> Result<()> {
if lance_arrow::json::is_json_field(field) || supported_fts_data_type(field.data_type()) {
return Ok(());
}
let logical_type = field
.metadata()
.get("ARROW:extension:name")
.map(|name| format!(" (logical type {name})"))
.unwrap_or_default();
Err(Error::Schema {
message: format!(
"A FTS index cannot be created on the field `{}` which has data type {}{}. FTS supports strings, lists of strings, and lance.json fields stored as LargeBinary; raw binary is not supported",
field.name(),
field.data_type(),
logical_type,
),
})
}
fn supported_fts_data_type_impl(dtype: &DataType, in_list: bool) -> bool {
match (dtype, in_list) {
(DataType::Utf8 | DataType::LargeUtf8, _) => true,
@@ -707,6 +800,43 @@ mod tests {
use super::*;
#[test]
fn test_fts_field_logical_type_validation() {
let json = lance_arrow::json::json_field("doc", true);
assert!(validate_fts_field(&json).is_ok());
for dtype in [DataType::Binary, DataType::LargeBinary] {
let raw = Field::new("doc", dtype.clone(), true);
let err = validate_fts_field(&raw).unwrap_err().to_string();
assert!(err.contains("raw binary is not supported"), "{err}");
assert!(err.contains("lance.json"), "{err}");
let other_extension = raw.with_metadata(std::collections::HashMap::from([(
"ARROW:extension:name".into(),
"other.json".into(),
)]));
let err = validate_fts_field(&other_extension)
.unwrap_err()
.to_string();
assert!(err.contains("logical type other.json"), "{err}");
}
// A JSON marker on the wrong binary storage type is insufficient.
let wrong_storage =
Field::new("doc", DataType::Binary, true).with_metadata(json.metadata().clone());
assert!(validate_fts_field(&wrong_storage).is_err());
for dtype in [DataType::Utf8, DataType::LargeUtf8] {
let text = Field::new("text", dtype, true);
assert!(validate_fts_field(&text).is_ok());
for list_type in [
DataType::List(Arc::new(text.clone())),
DataType::LargeList(Arc::new(text.clone())),
] {
assert!(validate_fts_field(&Field::new("texts", list_type, true)).is_ok());
}
}
}
#[test]
fn test_public_fts_field_path_prefers_exact_case() {
let text_list = || {
@@ -870,6 +1000,37 @@ mod tests {
assert!(err.contains("text.embedding"));
}
#[test]
fn test_validate_database_name() {
for name in [
"db",
"team/search",
"Team_1/db-2.3",
&"a".repeat(MAX_DB_NAME_CHARS),
] {
assert!(validate_database_name(name).is_ok(), "rejected {name:?}");
}
for name in [
"",
"/",
"//",
".",
"..",
"/db",
"db/",
"a//b",
"a/./b",
"a/../b",
"a$b",
"a b",
"a?b",
"café",
&"a".repeat(MAX_DB_NAME_CHARS + 1),
] {
assert!(validate_database_name(name).is_err(), "accepted {name:?}");
}
}
#[test]
fn test_validate_table_name() {
assert!(validate_table_name("my_table").is_ok());
@@ -890,27 +1051,27 @@ mod tests {
}
#[test]
fn test_validate_namespace_name() {
fn test_validate_namespace_component() {
// Valid namespace names
assert!(validate_namespace_name("ns1").is_ok());
assert!(validate_namespace_name("namespace_123").is_ok());
assert!(validate_namespace_name("my-namespace").is_ok());
assert!(validate_namespace_name("my.namespace").is_ok());
assert!(validate_namespace_name("NS_1.2.3").is_ok());
assert!(validate_namespace_name("a").is_ok());
assert!(validate_namespace_name("123").is_ok());
assert!(validate_namespace_name("_underscore").is_ok());
assert!(validate_namespace_name("-hyphen").is_ok());
assert!(validate_namespace_name(".period").is_ok());
assert!(validate_namespace_component("ns1").is_ok());
assert!(validate_namespace_component("namespace_123").is_ok());
assert!(validate_namespace_component("my-namespace").is_ok());
assert!(validate_namespace_component("my.namespace").is_ok());
assert!(validate_namespace_component("NS_1.2.3").is_ok());
assert!(validate_namespace_component("a").is_ok());
assert!(validate_namespace_component("123").is_ok());
assert!(validate_namespace_component("_underscore").is_ok());
assert!(validate_namespace_component("-hyphen").is_ok());
assert!(validate_namespace_component(".period").is_ok());
// Invalid namespace names
assert!(validate_namespace_name("").is_err());
assert!(validate_namespace_name("namespace with spaces").is_err());
assert!(validate_namespace_name("namespace/with/slashes").is_err());
assert!(validate_namespace_name("namespace\\with\\backslashes").is_err());
assert!(validate_namespace_name("namespace$with$delimiter").is_err());
assert!(validate_namespace_name("namespace@special").is_err());
assert!(validate_namespace_name("namespace#hash").is_err());
assert!(validate_namespace_component("").is_err());
assert!(validate_namespace_component("namespace with spaces").is_err());
assert!(validate_namespace_component("namespace/with/slashes").is_err());
assert!(validate_namespace_component("namespace\\with\\backslashes").is_err());
assert!(validate_namespace_component("namespace$with$delimiter").is_err());
assert!(validate_namespace_component("namespace@special").is_err());
assert!(validate_namespace_component("namespace#hash").is_err());
}
#[test]
@@ -926,6 +1087,9 @@ mod tests {
// Empty namespace (root) is valid
assert!(validate_namespace(&[]).is_ok());
// Valid public namespace (which is equivalent to the root namespace.)
assert!(validate_namespace(&["public".to_string()]).is_ok());
// Invalid: contains empty component
assert!(validate_namespace(&["ns1".to_string(), "".to_string()]).is_err());
@@ -937,6 +1101,9 @@ mod tests {
assert!(validate_namespace(&["ns1".to_string(), "ns/2".to_string()]).is_err());
assert!(validate_namespace(&["ns1".to_string(), "ns$2".to_string()]).is_err());
// Invalid: starts with public but contains other components.
assert!(validate_namespace(&["public".to_string(), "foo".to_string()]).is_err());
// Valid: underscores, hyphens, and periods are allowed
assert!(
validate_namespace(&["ns_1".to_string(), "ns-2".to_string(), "ns.3".to_string()])
rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json
Vendored
+1 -1
View File
@@ -23,7 +23,7 @@
"nullable": false
}
},
"created_at": "2026-08-21T00:00:00Z",
"created_at_millis": 1787270400000,
"image": {
"manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6",
"descriptor": {
rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json
Vendored
+1 -1
View File
@@ -1 +1 @@
{"created_at":"2026-08-21T00:00:00Z","disabled":false,"image":{"descriptor":{"behavior":{"result_stability":"input_and_context","side_effects":"non_idempotent"},"entrypoint":"app.function:create","format_version":1,"interface":{"type":"lance.scalar","version":1},"python":{"abi_tag":"cp312","executable":"/usr/local/bin/python3","implementation":"cpython","import_paths":["/opt/function/code"],"version":"3.12.14"},"python_api":1,"requires":{"capabilities":[],"kernel_min":"4.18.0"},"schemas":{"initialization":"/opt/function/schemas/initialization.arrow","input":"/opt/function/schemas/input.arrow","output":"/opt/function/schemas/output.arrow"}},"manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","source":false},"location":"memory:///fixture","metadata":{},"name":"embed","object_id":"fixture","secret_bindings":[{"kind":"env","secret_ref":{"name":"hf-prod"},"variable":"HF_TOKEN"}],"signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}},"version":"1"}
{"created_at_millis":1787270400000,"disabled":false,"image":{"descriptor":{"behavior":{"result_stability":"input_and_context","side_effects":"non_idempotent"},"entrypoint":"app.function:create","format_version":1,"interface":{"type":"lance.scalar","version":1},"python":{"abi_tag":"cp312","executable":"/usr/local/bin/python3","implementation":"cpython","import_paths":["/opt/function/code"],"version":"3.12.14"},"python_api":1,"requires":{"capabilities":[],"kernel_min":"4.18.0"},"schemas":{"initialization":"/opt/function/schemas/initialization.arrow","input":"/opt/function/schemas/input.arrow","output":"/opt/function/schemas/output.arrow"}},"manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","source":false},"location":"memory:///fixture","metadata":{},"name":"embed","object_id":"fixture","secret_bindings":[{"kind":"env","secret_ref":{"name":"hf-prod"},"variable":"HF_TOKEN"}],"signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}},"version":"1"}