mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-04 12:38:38 +00:00
Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Gatefixer
|
3d562a7c28 | ||
|
lancedb-gatefixer[bot]
|
7357d63e87 | ||
|
lancedb-gatefixer[bot]
|
624a75edf7 | ||
 Justin Miller
|
c7ea91f3ea | ||
 Wyatt Alt
|
8e24dd3828 | ||
 Adityaj0
|
f79dc017c4 | ||
|
Adityaj0
|
e6ae93f52a | ||
 Drew Gallardo
|
3dd9c598e9 | ||
 dependabot[bot]
|
9e26bf3fba | ||
 Will Jones
|
93354baf34 | ||
 LuQQiu
|
05602ec7d5 |
Generated
+113
-118
@@ -601,7 +601,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"hex",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"sha1 0.10.6",
|
||||
"time",
|
||||
"tokio",
|
||||
@@ -664,7 +664,7 @@ dependencies = [
|
||||
"bytes-utils",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 0.4.6",
|
||||
"http-body 1.1.0",
|
||||
"percent-encoding",
|
||||
@@ -694,7 +694,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body-util",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
@@ -719,7 +719,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
@@ -743,7 +743,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
@@ -773,7 +773,7 @@ dependencies = [
|
||||
"hex",
|
||||
"hmac 0.13.0",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"lru",
|
||||
"percent-encoding",
|
||||
@@ -802,7 +802,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
@@ -826,7 +826,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
@@ -851,7 +851,7 @@ dependencies = [
|
||||
"aws-types",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
@@ -873,7 +873,7 @@ dependencies = [
|
||||
"hex",
|
||||
"hmac 0.13.0",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"p256",
|
||||
"percent-encoding",
|
||||
"ring",
|
||||
@@ -906,7 +906,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"crc-fast",
|
||||
"hex",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"md-5 0.11.0",
|
||||
@@ -940,7 +940,7 @@ dependencies = [
|
||||
"bytes-utils",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"percent-encoding",
|
||||
@@ -961,7 +961,7 @@ dependencies = [
|
||||
"h2 0.3.27",
|
||||
"h2 0.4.14",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 0.4.6",
|
||||
"hyper 0.14.32",
|
||||
"hyper 1.9.0",
|
||||
@@ -1023,7 +1023,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 0.4.6",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
@@ -1044,7 +1044,7 @@ dependencies = [
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -1070,7 +1070,7 @@ checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1084,7 +1084,7 @@ dependencies = [
|
||||
"bytes-utils",
|
||||
"futures-core",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 0.4.6",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
@@ -1132,7 +1132,7 @@ dependencies = [
|
||||
"axum-core",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"hyper 1.9.0",
|
||||
@@ -1165,7 +1165,7 @@ dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"mime",
|
||||
@@ -3421,8 +3421,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -3842,7 +3842,7 @@ dependencies = [
|
||||
"fnv",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"indexmap 2.14.0",
|
||||
"slab",
|
||||
"tokio",
|
||||
@@ -3953,7 +3953,7 @@ checksum = "629d8f3bbeda9d148036d6b0de0a3ab947abd08ce90626327fc3547a49d59d97"
|
||||
dependencies = [
|
||||
"dirs",
|
||||
"futures",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"indicatif",
|
||||
"libc",
|
||||
"log",
|
||||
@@ -3976,7 +3976,7 @@ checksum = "430b33fa84f92796d4d263070b6c0d3ca219df7b9a0e1853ee431029b1612bcd"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"more-asserts",
|
||||
"serde",
|
||||
"thiserror 2.0.18",
|
||||
@@ -4041,9 +4041,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "1.4.2"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
|
||||
checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"itoa",
|
||||
@@ -4067,7 +4067,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4078,7 +4078,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"pin-project-lite",
|
||||
]
|
||||
@@ -4145,7 +4145,7 @@ dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"h2 0.4.14",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"httparse",
|
||||
"httpdate",
|
||||
@@ -4177,7 +4177,7 @@ version = "0.27.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
|
||||
dependencies = [
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"hyper 1.9.0",
|
||||
"hyper-util",
|
||||
"rustls 0.23.40",
|
||||
@@ -4211,7 +4211,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"hyper 1.9.0",
|
||||
"ipnet",
|
||||
@@ -4777,8 +4777,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4852,8 +4852,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4875,7 +4875,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4889,7 +4889,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -4898,8 +4898,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -4909,8 +4909,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4950,8 +4950,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4981,8 +4981,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4999,8 +4999,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5009,8 +5009,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5045,12 +5045,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
"arrow-cast",
|
||||
"arrow-data",
|
||||
"arrow-schema",
|
||||
"arrow-select",
|
||||
@@ -5076,8 +5077,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5144,8 +5145,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5167,18 +5168,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
"arrow-cast",
|
||||
"arrow-data",
|
||||
"arrow-schema",
|
||||
"arrow-select",
|
||||
"async-recursion",
|
||||
"async-trait",
|
||||
"aws-config",
|
||||
"aws-credential-types",
|
||||
@@ -5187,7 +5182,7 @@ dependencies = [
|
||||
"chrono",
|
||||
"futures",
|
||||
"goosefs-sdk",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"io-uring",
|
||||
"lance-arrow",
|
||||
"lance-core",
|
||||
@@ -5211,8 +5206,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5228,8 +5223,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5241,8 +5236,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5296,8 +5291,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5312,8 +5307,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5352,8 +5347,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5366,8 +5361,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "10.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.7#c164cdf7cedbcb04ab1c44a6101722a33cfa9232"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"icu_segmenter",
|
||||
"jieba-rs",
|
||||
@@ -5419,7 +5414,7 @@ dependencies = [
|
||||
"goosefs-sdk",
|
||||
"half",
|
||||
"hf-hub",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"lance",
|
||||
"lance-arrow",
|
||||
@@ -6085,15 +6080,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "napi-build"
|
||||
version = "2.3.2"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9c366d2c8c60b86fa632df75f745509b52f9128f91a6bad4c796e44abb505e1"
|
||||
checksum = "5282704fbe8d49b0cf8b08e3f33233416a528658f205c7e5ace63b582de0b11c"
|
||||
|
||||
[[package]]
|
||||
name = "napi-derive"
|
||||
version = "3.6.0"
|
||||
version = "3.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a49c513341a61a16a10af6efcce46b30d0822ba2d4fb197d24d33dfc199c78d5"
|
||||
checksum = "4d5c9c02556ea6dc99dffd36c1ce60141411657438501a125b675776d011ce92"
|
||||
dependencies = [
|
||||
"convert_case",
|
||||
"ctor 1.0.5",
|
||||
@@ -6105,9 +6100,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "napi-derive-backend"
|
||||
version = "6.0.0"
|
||||
version = "6.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4747005fa3e2c9989ac45a723a514c5db2411238b72981a3cda4c701a9dfea17"
|
||||
checksum = "d60b5d773ad46c698c8cc2cd9fde0b283d39cbb7f71c04bee633c7bdba4423bd"
|
||||
dependencies = [
|
||||
"convert_case",
|
||||
"proc-macro2",
|
||||
@@ -6361,7 +6356,7 @@ dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body-util",
|
||||
"httparse",
|
||||
"humantime",
|
||||
@@ -6482,7 +6477,7 @@ dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"jiff",
|
||||
"log",
|
||||
@@ -6507,7 +6502,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"mea",
|
||||
"opendal-core",
|
||||
]
|
||||
@@ -6551,7 +6546,7 @@ checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"log",
|
||||
"opendal-core",
|
||||
"opendal-service-azure-common",
|
||||
@@ -6572,7 +6567,7 @@ checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"log",
|
||||
"opendal-core",
|
||||
"opendal-service-azure-common",
|
||||
@@ -6590,7 +6585,7 @@ version = "0.57.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051"
|
||||
dependencies = [
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"opendal-core",
|
||||
]
|
||||
|
||||
@@ -6601,7 +6596,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"log",
|
||||
"opendal-core",
|
||||
"quick-xml 0.39.4",
|
||||
@@ -6619,7 +6614,7 @@ checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"log",
|
||||
"opendal-core",
|
||||
"percent-encoding",
|
||||
@@ -6654,7 +6649,7 @@ checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"hf-xet",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"log",
|
||||
"opendal-core",
|
||||
"percent-encoding",
|
||||
@@ -6670,7 +6665,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"log",
|
||||
"opendal-core",
|
||||
"quick-xml 0.39.4",
|
||||
@@ -6689,7 +6684,7 @@ dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"crc32c",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"log",
|
||||
"md-5 0.11.0",
|
||||
"opendal-core",
|
||||
@@ -7588,7 +7583,7 @@ version = "0.14.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"heck 0.4.1",
|
||||
"itertools 0.14.0",
|
||||
"log",
|
||||
"multimap",
|
||||
@@ -8231,7 +8226,7 @@ checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"form_urlencoded",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"reqsign-core",
|
||||
@@ -8249,7 +8244,7 @@ dependencies = [
|
||||
"anyhow",
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"quick-xml 0.39.4",
|
||||
@@ -8271,7 +8266,7 @@ dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"jsonwebtoken",
|
||||
"log",
|
||||
"pem",
|
||||
@@ -8296,7 +8291,7 @@ dependencies = [
|
||||
"futures",
|
||||
"hex",
|
||||
"hmac 0.12.1",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"jiff",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
@@ -8323,7 +8318,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e"
|
||||
dependencies = [
|
||||
"form_urlencoded",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"jsonwebtoken",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
@@ -8343,7 +8338,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e128f19525861dbded59e1e7c17653a8ed63d573ca04aed708d552dbef5bb32a"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"reqsign-core",
|
||||
@@ -8363,7 +8358,7 @@ dependencies = [
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2 0.4.14",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"hyper 1.9.0",
|
||||
@@ -8407,7 +8402,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"hyper 1.9.0",
|
||||
@@ -8461,7 +8456,7 @@ checksum = "199dda04a536b532d0cc04d7979e39b1c763ea749bf91507017069c00b96056f"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"reqwest 0.13.3",
|
||||
"thiserror 2.0.18",
|
||||
"tower-service",
|
||||
@@ -8503,9 +8498,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rkyv"
|
||||
version = "0.8.16"
|
||||
version = "0.8.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3"
|
||||
checksum = "815cc8a37159a463064825246cadb07961e25cd9885908606f6d08a98d8f8874"
|
||||
dependencies = [
|
||||
"bytecheck",
|
||||
"bytes",
|
||||
@@ -8522,9 +8517,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rkyv_derive"
|
||||
version = "0.8.16"
|
||||
version = "0.8.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6"
|
||||
checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -9282,7 +9277,7 @@ version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"heck 0.4.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
@@ -9294,7 +9289,7 @@ version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"heck 0.4.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
@@ -9735,7 +9730,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.2",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -10063,7 +10058,7 @@ dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"h2 0.4.14",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"hyper 1.9.0",
|
||||
@@ -10119,7 +10114,7 @@ checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
|
||||
dependencies = [
|
||||
"bitflags 2.11.1",
|
||||
"bytes",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"pin-project-lite",
|
||||
@@ -10139,7 +10134,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"pin-project-lite",
|
||||
@@ -11160,7 +11155,7 @@ dependencies = [
|
||||
"clap",
|
||||
"crc32fast",
|
||||
"futures",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"hyper 1.9.0",
|
||||
"lazy_static",
|
||||
"more-asserts",
|
||||
@@ -11234,7 +11229,7 @@ dependencies = [
|
||||
"chrono",
|
||||
"clap",
|
||||
"gearhash",
|
||||
"http 1.4.2",
|
||||
"http 1.5.0",
|
||||
"itertools 0.14.0",
|
||||
"lazy_static",
|
||||
"more-asserts",
|
||||
|
||||
+14
-14
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=10.0.0-beta.7", default-features = false, "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=10.0.0-beta.7", default-features = false, "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=10.0.0-beta.7", default-features = false, "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=10.0.0-beta.7", "tag" = "v10.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
arrow = { version = "58.0.0", optional = false }
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>10.0.0-beta.7</lance-core.version>
|
||||
<lance-core.version>10.1.0-beta.1</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
+6
-2
@@ -339,7 +339,9 @@ impl Table {
|
||||
let transforms = NewColumnTransform::SqlExpressions(transforms);
|
||||
let res = self
|
||||
.inner_ref()?
|
||||
.add_columns(transforms, None)
|
||||
.add_columns()
|
||||
.transform(transforms)
|
||||
.execute()
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(res.into())
|
||||
@@ -356,7 +358,9 @@ impl Table {
|
||||
let transforms = NewColumnTransform::AllNulls(schema);
|
||||
let res = self
|
||||
.inner_ref()?
|
||||
.add_columns(transforms, None)
|
||||
.add_columns()
|
||||
.transform(transforms)
|
||||
.execute()
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(res.into())
|
||||
|
||||
@@ -707,6 +707,9 @@ class LanceDBConnection(DBConnection):
|
||||
self._namespace_client_properties = namespace_client_properties
|
||||
if _inner is not None:
|
||||
self._conn = _inner
|
||||
# Native-derived wrappers resolve this in their async reconstruction
|
||||
# path so construction never synchronously re-enters LOOP.
|
||||
self._read_consistency_interval = read_consistency_interval
|
||||
self._cached_namespace_client = None
|
||||
return
|
||||
|
||||
@@ -756,11 +759,14 @@ class LanceDBConnection(DBConnection):
|
||||
# storage_options. Also, this class really shouldn't be holding any state
|
||||
# beyond _conn.
|
||||
self._conn = AsyncConnection(LOOP.run(do_connect()))
|
||||
# Keep property access synchronous so debugger introspection cannot wait on
|
||||
# the background loop while that thread is suspended at a breakpoint.
|
||||
self._read_consistency_interval = read_consistency_interval
|
||||
self._cached_namespace_client: Optional[LanceNamespace] = None
|
||||
|
||||
@property
|
||||
def read_consistency_interval(self) -> Optional[timedelta]:
|
||||
return LOOP.run(self._conn.get_read_consistency_interval())
|
||||
return self._read_consistency_interval
|
||||
|
||||
@property
|
||||
def session(self) -> Optional[Session]:
|
||||
@@ -771,8 +777,16 @@ class LanceDBConnection(DBConnection):
|
||||
return self._conn.uri
|
||||
|
||||
@classmethod
|
||||
def from_inner(cls, inner: LanceDbConnection):
|
||||
return cls(None, _inner=inner)
|
||||
def from_inner(
|
||||
cls,
|
||||
inner: LanceDbConnection,
|
||||
read_consistency_interval: Optional[timedelta],
|
||||
):
|
||||
return cls(
|
||||
None,
|
||||
read_consistency_interval=read_consistency_interval,
|
||||
_inner=inner,
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{self.__class__.__name__}(uri={self._conn.uri!r})"
|
||||
|
||||
@@ -92,8 +92,10 @@ class LanceMergeInsertBuilder(object):
|
||||
self._when_not_matched_by_source_delete = True
|
||||
if isinstance(condition, Expr):
|
||||
self._when_not_matched_by_source_condition_expr = condition._inner
|
||||
elif condition is not None:
|
||||
self._when_not_matched_by_source_condition = None
|
||||
else:
|
||||
self._when_not_matched_by_source_condition = condition
|
||||
self._when_not_matched_by_source_condition_expr = None
|
||||
return self
|
||||
|
||||
def use_index(self, use_index: bool) -> LanceMergeInsertBuilder:
|
||||
|
||||
@@ -226,7 +226,7 @@ class PermutationBuilder:
|
||||
|
||||
async def do_execute():
|
||||
inner_tbl = await self._async.execute()
|
||||
return LanceTable.from_inner(inner_tbl)
|
||||
return await LanceTable.from_inner(inner_tbl)
|
||||
|
||||
return LOOP.run(do_execute())
|
||||
|
||||
|
||||
@@ -2697,7 +2697,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
|
||||
self._fts_query.phrase_query(True)
|
||||
if self._distance_type:
|
||||
self._vector_query.metric(self._distance_type)
|
||||
if self._minimum_nprobes:
|
||||
if self._minimum_nprobes is not None:
|
||||
self._vector_query.minimum_nprobes(self._minimum_nprobes)
|
||||
if self._maximum_nprobes is not None:
|
||||
self._vector_query.maximum_nprobes(self._maximum_nprobes)
|
||||
|
||||
+136
-39
@@ -1595,8 +1595,8 @@ class Table(ABC):
|
||||
Prefer this over :meth:`fetch_blobs` for large payloads. ``row_ids`` is
|
||||
a ``list[int]`` or a query ``pyarrow.Table`` carrying row identity via
|
||||
``_rowid`` or a ``_lance_row_id`` field on the blob descriptor. Null
|
||||
rows are ``None``. Unsupported on LanceDB Cloud, where
|
||||
:meth:`fetch_blobs` returns full bytes instead.
|
||||
rows are ``None``. Remote tables require LanceDB Cloud server 0.5.0 or
|
||||
newer.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
@@ -2182,11 +2182,15 @@ class LanceTable(Table):
|
||||
return self.name
|
||||
|
||||
@classmethod
|
||||
def from_inner(cls, tbl: LanceDBTable):
|
||||
from .db import LanceDBConnection
|
||||
async def from_inner(cls, tbl: LanceDBTable):
|
||||
from .db import AsyncConnection, LanceDBConnection
|
||||
|
||||
async_tbl = AsyncTable(tbl)
|
||||
conn = LanceDBConnection.from_inner(tbl.database())
|
||||
inner_conn = tbl.database()
|
||||
read_consistency_interval = await AsyncConnection(
|
||||
inner_conn
|
||||
).get_read_consistency_interval()
|
||||
conn = LanceDBConnection.from_inner(inner_conn, read_consistency_interval)
|
||||
return cls(
|
||||
conn,
|
||||
async_tbl.name,
|
||||
@@ -4031,7 +4035,10 @@ def _handle_bad_vectors(
|
||||
for vector_column in vector_columns:
|
||||
dim = vector_column["expected_dim"]
|
||||
if target_schema is not None and dim is None:
|
||||
dim = _infer_vector_dim(batch[vector_column["name"]])
|
||||
dim = _infer_vector_column_dim(
|
||||
batch[vector_column["name"]],
|
||||
vector_column["is_multivector"],
|
||||
)
|
||||
pending_dims.append(vector_column)
|
||||
batch = _handle_bad_vector_column(
|
||||
batch,
|
||||
@@ -4040,11 +4047,13 @@ def _handle_bad_vectors(
|
||||
fill_value=fill_value,
|
||||
expected_dim=dim,
|
||||
expected_value_type=vector_column["expected_value_type"],
|
||||
is_multivector=vector_column["is_multivector"],
|
||||
)
|
||||
for vector_column in pending_dims:
|
||||
if vector_column["expected_dim"] is None:
|
||||
vector_column["expected_dim"] = _infer_vector_dim(
|
||||
batch[vector_column["name"]]
|
||||
vector_column["expected_dim"] = _infer_vector_column_dim(
|
||||
batch[vector_column["name"]],
|
||||
vector_column["is_multivector"],
|
||||
)
|
||||
if batch.schema.equals(output_schema, check_metadata=True):
|
||||
yield batch
|
||||
@@ -4070,22 +4079,30 @@ def _find_vector_columns(
|
||||
if target_schema is None:
|
||||
vector_columns = []
|
||||
for field in reader_schema:
|
||||
named_vector_col = (
|
||||
_is_list_like(field.type)
|
||||
and pa.types.is_floating(field.type.value_type)
|
||||
and field.name == VECTOR_COLUMN_NAME
|
||||
is_multivector = _is_multivector_type(field.type)
|
||||
is_fixed_multivector = is_multivector and pa.types.is_fixed_size_list(
|
||||
field.type.value_type
|
||||
)
|
||||
named_vector_col = (
|
||||
_is_float_vector_type(field.type) or is_multivector
|
||||
) and field.name == VECTOR_COLUMN_NAME
|
||||
likely_vector_col = (
|
||||
pa.types.is_fixed_size_list(field.type)
|
||||
and pa.types.is_floating(field.type.value_type)
|
||||
and (field.type.list_size >= 10)
|
||||
)
|
||||
if named_vector_col or likely_vector_col:
|
||||
if named_vector_col or likely_vector_col or is_fixed_multivector:
|
||||
vector_type = field.type.value_type if is_multivector else field.type
|
||||
vector_columns.append(
|
||||
{
|
||||
"name": field.name,
|
||||
"expected_dim": None,
|
||||
"expected_value_type": None,
|
||||
"expected_dim": (
|
||||
vector_type.list_size
|
||||
if pa.types.is_fixed_size_list(vector_type)
|
||||
else None
|
||||
),
|
||||
"expected_value_type": vector_type.value_type,
|
||||
"is_multivector": is_multivector,
|
||||
}
|
||||
)
|
||||
return vector_columns
|
||||
@@ -4099,9 +4116,8 @@ def _find_vector_columns(
|
||||
for field in target_schema:
|
||||
if field.name not in reader_column_names:
|
||||
continue
|
||||
if not _is_list_like(field.type) or not pa.types.is_floating(
|
||||
field.type.value_type
|
||||
):
|
||||
is_multivector = _is_multivector_type(field.type)
|
||||
if not _is_float_vector_type(field.type) and not is_multivector:
|
||||
continue
|
||||
|
||||
reader_field = reader_schema.field(field.name)
|
||||
@@ -4116,16 +4132,18 @@ def _find_vector_columns(
|
||||
and reader_field.type.list_size >= 10
|
||||
)
|
||||
|
||||
if named_vector_col or typed_fixed_vector_col:
|
||||
if named_vector_col or typed_fixed_vector_col or is_multivector:
|
||||
vector_type = field.type.value_type if is_multivector else field.type
|
||||
vector_columns.append(
|
||||
{
|
||||
"name": field.name,
|
||||
"expected_dim": (
|
||||
field.type.list_size
|
||||
if pa.types.is_fixed_size_list(field.type)
|
||||
vector_type.list_size
|
||||
if pa.types.is_fixed_size_list(vector_type)
|
||||
else None
|
||||
),
|
||||
"expected_value_type": field.type.value_type,
|
||||
"expected_value_type": vector_type.value_type,
|
||||
"is_multivector": is_multivector,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -4176,6 +4194,7 @@ def _handle_bad_vector_column(
|
||||
fill_value: float = 0.0,
|
||||
expected_dim: Optional[int] = None,
|
||||
expected_value_type: Optional[pa.DataType] = None,
|
||||
is_multivector: bool = False,
|
||||
) -> pa.RecordBatch:
|
||||
"""
|
||||
Ensure that the vector column exists and has type fixed_size_list(float)
|
||||
@@ -4196,6 +4215,8 @@ def _handle_bad_vector_column(
|
||||
vec_arr = data[vector_column_name]
|
||||
if not _is_list_like(vec_arr.type):
|
||||
return data
|
||||
if is_multivector and not _is_multivector_type(vec_arr.type):
|
||||
return data
|
||||
|
||||
if (
|
||||
expected_dim is not None
|
||||
@@ -4212,12 +4233,21 @@ def _handle_bad_vector_column(
|
||||
vec_arr = pa.array(vec_arr.to_pylist(), type=pa.list_(expected_value_type))
|
||||
data = data.set_column(position, vector_column_name, vec_arr)
|
||||
|
||||
if pa.types.is_floating(vec_arr.type.value_type):
|
||||
if is_multivector or pa.types.is_floating(vec_arr.type.value_type):
|
||||
has_nan = has_nan_values(vec_arr)
|
||||
else:
|
||||
has_nan = pa.array([False] * len(vec_arr))
|
||||
|
||||
if expected_dim is not None:
|
||||
if is_multivector:
|
||||
dim = (
|
||||
expected_dim
|
||||
if expected_dim is not None
|
||||
else _infer_vector_column_dim(vec_arr, True)
|
||||
)
|
||||
if dim is None:
|
||||
return data
|
||||
has_wrong_dim = _multivector_has_wrong_dim(vec_arr, dim)
|
||||
elif expected_dim is not None:
|
||||
dim = expected_dim
|
||||
elif pa.types.is_fixed_size_list(vec_arr.type):
|
||||
dim = vec_arr.type.list_size
|
||||
@@ -4226,15 +4256,16 @@ def _handle_bad_vector_column(
|
||||
if dim is None:
|
||||
return data
|
||||
|
||||
is_null = pc.is_null(vec_arr)
|
||||
# pc.list_value_length returns null for null list entries, so
|
||||
# pc.not_equal(null, dim) also returns null. Use or_kleene so that
|
||||
# True OR null = True (Kleene three-valued logic), ensuring null vectors
|
||||
# are counted as wrong-dim.
|
||||
has_wrong_dim = pc.or_kleene(
|
||||
is_null,
|
||||
pc.not_equal(pc.list_value_length(vec_arr), dim),
|
||||
)
|
||||
if not is_multivector:
|
||||
is_null = pc.is_null(vec_arr)
|
||||
# pc.list_value_length returns null for null list entries, so
|
||||
# pc.not_equal(null, dim) also returns null. Use or_kleene so that
|
||||
# True OR null = True (Kleene three-valued logic), ensuring null vectors
|
||||
# are counted as wrong-dim.
|
||||
has_wrong_dim = pc.or_kleene(
|
||||
is_null,
|
||||
pc.not_equal(pc.list_value_length(vec_arr), dim),
|
||||
)
|
||||
|
||||
has_bad_vectors = pc.any(has_nan).as_py() or pc.any(has_wrong_dim).as_py()
|
||||
|
||||
@@ -4269,7 +4300,10 @@ def _handle_bad_vector_column(
|
||||
raise ValueError(
|
||||
"`fill_value` must not be None if `on_bad_vectors` is 'fill'"
|
||||
)
|
||||
vec_arr = _fill_bad_vector_values(vec_arr, dim, fill_value)
|
||||
if is_multivector:
|
||||
vec_arr = _fill_bad_multivector_values(vec_arr, dim, fill_value)
|
||||
else:
|
||||
vec_arr = _fill_bad_vector_values(vec_arr, dim, fill_value)
|
||||
else:
|
||||
raise ValueError(f"Invalid value for on_bad_vectors: {on_bad_vectors}")
|
||||
|
||||
@@ -4321,17 +4355,60 @@ def _fill_bad_vector_values(
|
||||
return filled.cast(arr.type)
|
||||
|
||||
|
||||
def _fill_bad_multivector_values(
|
||||
arr: Union[pa.Array, pa.ChunkedArray], dim: int, fill_value: float
|
||||
) -> pa.Array:
|
||||
if not isinstance(arr, pa.ChunkedArray):
|
||||
arr = pa.chunked_array([arr])
|
||||
arr = arr.combine_chunks()
|
||||
|
||||
filled_vectors = _fill_bad_vector_values(arr.values, dim, fill_value)
|
||||
parent_nulls = pc.is_null(arr)
|
||||
if pa.types.is_large_list(arr.type):
|
||||
filled = pa.LargeListArray.from_arrays(
|
||||
arr.offsets, filled_vectors, mask=parent_nulls
|
||||
)
|
||||
else:
|
||||
filled = pa.ListArray.from_arrays(
|
||||
arr.offsets, filled_vectors, mask=parent_nulls
|
||||
)
|
||||
return filled.cast(arr.type)
|
||||
|
||||
|
||||
def _multivector_has_wrong_dim(
|
||||
arr: Union[pa.Array, pa.ChunkedArray], dim: int
|
||||
) -> pa.BooleanArray:
|
||||
if isinstance(arr, pa.ChunkedArray):
|
||||
results = [_multivector_has_wrong_dim(chunk, dim) for chunk in arr.chunks]
|
||||
return pa.concat_arrays(results) if results else pa.array([], type=pa.bool_())
|
||||
|
||||
vectors = arr.flatten()
|
||||
vector_is_wrong = pc.or_kleene(
|
||||
pc.is_null(vectors),
|
||||
pc.not_equal(pc.list_value_length(vectors), dim),
|
||||
)
|
||||
parent_indices = pc.list_parent_indices(arr)
|
||||
wrong_parent_indices = pc.unique(pc.filter(parent_indices, vector_is_wrong))
|
||||
indices = pa.array(range(len(arr)), type=pa.uint32())
|
||||
return pc.or_(pc.is_null(arr), pc.is_in(indices, wrong_parent_indices))
|
||||
|
||||
|
||||
def has_nan_values(arr: Union[pa.ListArray, pa.ChunkedArray]) -> pa.BooleanArray:
|
||||
if isinstance(arr, pa.ChunkedArray):
|
||||
values = pa.chunked_array([chunk.flatten() for chunk in arr.chunks])
|
||||
else:
|
||||
values = arr.flatten()
|
||||
if pa.types.is_float16(values.type):
|
||||
results = [has_nan_values(chunk) for chunk in arr.chunks]
|
||||
return pa.concat_arrays(results) if results else pa.array([], type=pa.bool_())
|
||||
|
||||
values = arr.flatten()
|
||||
if _is_list_like(values.type):
|
||||
values_has_nan = has_nan_values(values)
|
||||
elif pa.types.is_float16(values.type):
|
||||
# is_nan isn't yet implemented for f16, so we cast to f32
|
||||
# https://github.com/apache/arrow/issues/45083
|
||||
values_has_nan = pc.is_nan(values.cast(pa.float32()))
|
||||
else:
|
||||
elif pa.types.is_floating(values.type):
|
||||
values_has_nan = pc.is_nan(values)
|
||||
else:
|
||||
return pa.array([False] * len(arr))
|
||||
values_indices = pc.list_parent_indices(arr)
|
||||
has_nan_indices = pc.unique(pc.filter(values_indices, values_has_nan))
|
||||
indices = pa.array(range(len(arr)), type=pa.uint32())
|
||||
@@ -4346,6 +4423,16 @@ def _is_list_like(data_type: pa.DataType) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _is_float_vector_type(data_type: pa.DataType) -> bool:
|
||||
return _is_list_like(data_type) and pa.types.is_floating(data_type.value_type)
|
||||
|
||||
|
||||
def _is_multivector_type(data_type: pa.DataType) -> bool:
|
||||
return (
|
||||
pa.types.is_list(data_type) or pa.types.is_large_list(data_type)
|
||||
) and _is_float_vector_type(data_type.value_type)
|
||||
|
||||
|
||||
def _merge_metadata(*metadata_dicts: Optional[dict]) -> dict:
|
||||
merged = {}
|
||||
for metadata in metadata_dicts:
|
||||
@@ -4437,6 +4524,16 @@ def _infer_vector_dim(arr: Union[pa.Array, pa.ChunkedArray]) -> Optional[int]:
|
||||
return pc.mode(lengths)[0].as_py()["mode"]
|
||||
|
||||
|
||||
def _infer_vector_column_dim(
|
||||
arr: Union[pa.Array, pa.ChunkedArray], is_multivector: bool
|
||||
) -> Optional[int]:
|
||||
if not is_multivector:
|
||||
return _infer_vector_dim(arr)
|
||||
if isinstance(arr, pa.ChunkedArray):
|
||||
arr = arr.combine_chunks()
|
||||
return _infer_vector_dim(arr.flatten())
|
||||
|
||||
|
||||
def _validate_schema(schema: pa.Schema):
|
||||
"""
|
||||
Make sure the metadata is valid utf8
|
||||
|
||||
@@ -77,6 +77,23 @@ def test_sync_repr_does_not_use_background_loop(tmp_path, monkeypatch):
|
||||
assert repr(table) == f"LanceTable(name='test', _conn={db!r})"
|
||||
|
||||
|
||||
def test_read_consistency_interval_does_not_use_background_loop(tmp_path, monkeypatch):
|
||||
from lancedb.background_loop import LOOP
|
||||
from lancedb.db import LanceDBConnection
|
||||
|
||||
consistency_interval = timedelta(seconds=5)
|
||||
db = lancedb.connect(tmp_path, read_consistency_interval=consistency_interval)
|
||||
db_from_inner = LanceDBConnection.from_inner(db._inner, consistency_interval)
|
||||
|
||||
def fail_run(*args, **kwargs):
|
||||
raise AssertionError("properties should not use the Python background loop")
|
||||
|
||||
monkeypatch.setattr(LOOP, "run", fail_run)
|
||||
|
||||
assert db.read_consistency_interval == consistency_interval
|
||||
assert db_from_inner.read_consistency_interval == consistency_interval
|
||||
|
||||
|
||||
def test_ingest_pd(tmp_path):
|
||||
db = lancedb.connect(tmp_path)
|
||||
|
||||
|
||||
@@ -123,6 +123,19 @@ async def test_async_hybrid_query_default_limit(table: AsyncTable):
|
||||
assert texts.count("a") == 1
|
||||
|
||||
|
||||
def test_hybrid_query_minimum_nprobes_zero_raises(sync_table: Table):
|
||||
# minimum_nprobes(0) must raise the same validation error a plain vector
|
||||
# query raises, not silently no-op because 0 is falsy.
|
||||
with pytest.raises(ValueError, match="minimum_nprobes must be greater than 0"):
|
||||
(
|
||||
sync_table.search(query_type="hybrid")
|
||||
.vector([0.0, 0.4])
|
||||
.text("dog")
|
||||
.minimum_nprobes(0)
|
||||
.to_arrow()
|
||||
)
|
||||
|
||||
|
||||
def test_hybrid_query_distance_range(sync_table: Table):
|
||||
reranker = RRFReranker(return_score="all")
|
||||
result = (
|
||||
|
||||
@@ -6,6 +6,7 @@ import math
|
||||
import pytest
|
||||
|
||||
from lancedb import DBConnection, Table, connect
|
||||
from lancedb.background_loop import LOOP
|
||||
from lancedb.permutation import Permutation, Permutations, permutation_builder
|
||||
|
||||
|
||||
@@ -31,6 +32,25 @@ def test_split_random_ratios(mem_db):
|
||||
assert 65 <= split_1_count <= 75 # ~70% ± tolerance
|
||||
|
||||
|
||||
def test_execute_does_not_reenter_background_loop(tmp_path, monkeypatch):
|
||||
import threading
|
||||
|
||||
db = connect(tmp_path)
|
||||
tbl = db.create_table("test_table", pa.table({"x": range(10)}))
|
||||
original_run = LOOP.run
|
||||
|
||||
def fail_on_reentry(future):
|
||||
assert threading.current_thread() is not LOOP.thread
|
||||
return original_run(future)
|
||||
|
||||
monkeypatch.setattr(LOOP, "run", fail_on_reentry)
|
||||
|
||||
permutation_tbl = permutation_builder(tbl).execute()
|
||||
|
||||
assert permutation_tbl.count_rows() == 10
|
||||
assert permutation_tbl._conn.read_consistency_interval is None
|
||||
|
||||
|
||||
def test_split_random_counts(mem_db):
|
||||
"""Test random splitting with absolute counts."""
|
||||
tbl = mem_db.create_table(
|
||||
|
||||
@@ -2055,6 +2055,24 @@ def blob_remote_table(*, server_version=Version("0.5.0")):
|
||||
request.send_header("phalanx-version", str(server_version))
|
||||
request.end_headers()
|
||||
request.wfile.write(json.dumps(BLOB_DESCRIBE_RESPONSE).encode())
|
||||
elif request.path.startswith("/v1/table/test/blob/image/"):
|
||||
path = request.path.partition("?")[0]
|
||||
row_id = int(path.split("/")[-2])
|
||||
payload = {10: b"alpha", 20: None, 30: b"gamma"}[row_id]
|
||||
if payload is None:
|
||||
request.send_response(204)
|
||||
request.end_headers()
|
||||
return
|
||||
byte_range = request.headers["Range"].removeprefix("bytes=")
|
||||
start_text, end_text = byte_range.split("-", maxsplit=1)
|
||||
start = int(start_text)
|
||||
end = int(end_text) if end_text else len(payload) - 1
|
||||
chunk = payload[start : end + 1]
|
||||
request.send_response(206)
|
||||
request.send_header("Content-Range", f"bytes {start}-{end}/{len(payload)}")
|
||||
request.send_header("Content-Length", str(len(chunk)))
|
||||
request.end_headers()
|
||||
request.wfile.write(chunk)
|
||||
elif request.path == "/v1/table/test/query/":
|
||||
content_len = int(request.headers.get("Content-Length", 0))
|
||||
body = json.loads(request.rfile.read(content_len))
|
||||
@@ -2092,8 +2110,21 @@ def test_remote_blob_columns_and_fetch():
|
||||
assert table.blob_columns() == ["image"]
|
||||
blobs = table.fetch_blobs("image", [10, 20, 30])
|
||||
assert blobs.to_pylist() == [b"alpha", None, b"gamma"]
|
||||
with pytest.raises(NotImplementedError, match="Use fetch_blobs for full bytes"):
|
||||
table.fetch_blob_files("image", [10, 20, 30])
|
||||
|
||||
|
||||
def test_remote_blob_files_are_lazy_seekable_handles():
|
||||
with blob_remote_table() as table:
|
||||
files = table.fetch_blob_files("image", [10, 20, 30])
|
||||
|
||||
assert len(files) == 3
|
||||
alpha, null_row, gamma = files
|
||||
assert null_row is None
|
||||
assert alpha is not None
|
||||
assert gamma is not None
|
||||
assert alpha.size() == 5
|
||||
assert alpha.read_range(1, 3) == b"lph"
|
||||
gamma.seek(2)
|
||||
assert gamma.read() == b"mma"
|
||||
|
||||
|
||||
def test_remote_blob_fetch_accepts_query_table():
|
||||
|
||||
@@ -6,6 +6,7 @@ import os
|
||||
import sys
|
||||
import threading
|
||||
import warnings
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import date, datetime, timedelta
|
||||
from time import sleep
|
||||
from typing import List
|
||||
@@ -1709,6 +1710,19 @@ def test_create_with_nans(mem_db: DBConnection):
|
||||
assert np.allclose(filled_vectors[22.0], np.array([5.0, 0.0]))
|
||||
|
||||
|
||||
def test_create_with_nans_in_multivectors(mem_db: DBConnection):
|
||||
multivector_type = pa.list_(pa.list_(pa.float32(), 128))
|
||||
schema = pa.schema(
|
||||
[pa.field("filename", pa.string()), pa.field("vector", multivector_type)]
|
||||
)
|
||||
vector = [0.1] * 128
|
||||
vector[-1] = np.nan
|
||||
data = [{"filename": "img1.jpg", "vector": [vector]}]
|
||||
|
||||
with pytest.raises(RuntimeError, match="Vector column 'vector' has NaNs"):
|
||||
mem_db.create_table("nan_multivector", data=data, schema=schema)
|
||||
|
||||
|
||||
def test_add_with_nans(mem_db: DBConnection):
|
||||
schema = pa.schema(
|
||||
[
|
||||
@@ -2124,6 +2138,27 @@ def test_delete(mem_db: DBConnection):
|
||||
assert table.to_arrow()["id"].to_pylist() == [1]
|
||||
|
||||
|
||||
def test_concurrent_deletes_are_thread_safe(mem_db: DBConnection):
|
||||
num_workers = 8
|
||||
table = mem_db.create_table(
|
||||
"my_table", data=[{"id": row_id} for row_id in range(num_workers)]
|
||||
)
|
||||
barrier = threading.Barrier(num_workers)
|
||||
|
||||
def delete(row_id: int):
|
||||
barrier.wait()
|
||||
return table.delete(f"id = {row_id}")
|
||||
|
||||
with ThreadPoolExecutor(max_workers=num_workers) as pool:
|
||||
results = list(pool.map(delete, range(num_workers)))
|
||||
|
||||
assert all(result.num_deleted_rows == 1 for result in results)
|
||||
assert sorted(result.version for result in results) == list(
|
||||
range(2, num_workers + 2)
|
||||
)
|
||||
assert table.count_rows() == 0
|
||||
|
||||
|
||||
def test_delete_expr(mem_db: DBConnection):
|
||||
table = mem_db.create_table(
|
||||
"my_table",
|
||||
@@ -2364,6 +2399,29 @@ def test_merge_insert_by_source_delete_expr(mem_db: DBConnection):
|
||||
assert table.to_arrow().sort_by("a") == expected
|
||||
|
||||
|
||||
def test_merge_insert_by_source_delete_reconfigure(mem_db: DBConnection):
|
||||
# Calling when_not_matched_by_source_delete() again with no condition must
|
||||
# widen the delete to unconditional, not keep the earlier condition around.
|
||||
table = mem_db.create_table(
|
||||
"my_table",
|
||||
data=pa.table({"a": [1, 2, 3], "b": ["a", "b", "c"]}),
|
||||
)
|
||||
new_data = pa.table({"a": [2, 4], "b": ["x", "z"]})
|
||||
|
||||
merge_insert_res = (
|
||||
table.merge_insert("a")
|
||||
.when_matched_update_all()
|
||||
.when_not_matched_insert_all()
|
||||
.when_not_matched_by_source_delete("a > 2")
|
||||
.when_not_matched_by_source_delete()
|
||||
.execute(new_data)
|
||||
)
|
||||
assert merge_insert_res.num_deleted_rows == 2
|
||||
|
||||
expected = pa.table({"a": [2, 4], "b": ["x", "z"]})
|
||||
assert table.to_arrow().sort_by("a") == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_merge_insert_by_source_delete_expr_async(
|
||||
mem_db_async: AsyncConnection,
|
||||
|
||||
@@ -400,6 +400,83 @@ def test_handle_bad_vectors_nan(on_bad_vectors):
|
||||
assert output["vector"].combine_chunks() == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("on_bad_vectors", ["error", "drop", "fill", "null"])
|
||||
def test_handle_bad_multivectors_nan(on_bad_vectors):
|
||||
multivector_type = pa.list_(pa.list_(pa.float32(), 2))
|
||||
vectors = pa.array(
|
||||
[
|
||||
[[1.0, float("nan")], [2.0, 3.0]],
|
||||
[[4.0, 5.0]],
|
||||
],
|
||||
type=multivector_type,
|
||||
)
|
||||
data = pa.table({"vector": vectors})
|
||||
|
||||
if on_bad_vectors == "error":
|
||||
with pytest.raises(ValueError, match="Vector column 'vector' has NaNs"):
|
||||
_handle_bad_vectors(data.to_reader()).read_all()
|
||||
return
|
||||
|
||||
output = _handle_bad_vectors(
|
||||
data.to_reader(),
|
||||
on_bad_vectors=on_bad_vectors,
|
||||
fill_value=42.0,
|
||||
).read_all()
|
||||
|
||||
if on_bad_vectors == "drop":
|
||||
expected = pa.array([[[4.0, 5.0]]], type=multivector_type)
|
||||
elif on_bad_vectors == "fill":
|
||||
expected = pa.array(
|
||||
[[[1.0, 42.0], [2.0, 3.0]], [[4.0, 5.0]]],
|
||||
type=multivector_type,
|
||||
)
|
||||
else:
|
||||
expected = pa.array([None, [[4.0, 5.0]]], type=multivector_type)
|
||||
|
||||
assert output["vector"].combine_chunks() == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("on_bad_vectors", ["error", "drop", "fill", "null"])
|
||||
def test_handle_bad_variable_multivectors(on_bad_vectors):
|
||||
target_type = pa.list_(pa.list_(pa.float32(), 2))
|
||||
vectors = pa.array(
|
||||
[
|
||||
[[1.0, float("nan")], [2.0, 3.0]],
|
||||
[[4.0]],
|
||||
[[5.0, 6.0]],
|
||||
]
|
||||
)
|
||||
data = pa.table({"vector": vectors})
|
||||
|
||||
if on_bad_vectors == "error":
|
||||
with pytest.raises(ValueError, match="variable length vectors"):
|
||||
_handle_bad_vectors(
|
||||
data.to_reader(),
|
||||
target_schema=pa.schema({"vector": target_type}),
|
||||
).read_all()
|
||||
return
|
||||
|
||||
output = _handle_bad_vectors(
|
||||
data.to_reader(),
|
||||
on_bad_vectors=on_bad_vectors,
|
||||
fill_value=42.0,
|
||||
target_schema=pa.schema({"vector": target_type}),
|
||||
).read_all()
|
||||
|
||||
if on_bad_vectors == "drop":
|
||||
expected = [[[5.0, 6.0]]]
|
||||
elif on_bad_vectors == "fill":
|
||||
expected = [
|
||||
[[1.0, 42.0], [2.0, 3.0]],
|
||||
[[4.0, 42.0]],
|
||||
[[5.0, 6.0]],
|
||||
]
|
||||
else:
|
||||
expected = [None, None, [[5.0, 6.0]]]
|
||||
|
||||
assert output["vector"].combine_chunks().to_pylist() == expected
|
||||
|
||||
|
||||
def test_handle_bad_vectors_noop():
|
||||
# ChunkedArray should be preserved as-is
|
||||
vector = pa.chunked_array(
|
||||
|
||||
+40
-13
@@ -426,9 +426,11 @@ pub struct PyBlobFile {
|
||||
impl PyBlobFile {
|
||||
fn read_bytes(self_: PyRef<'_, Self>) -> PyResult<Py<PyBytes>> {
|
||||
let inner = self_.inner.clone();
|
||||
let bytes = block_on(async move { inner.read().await })
|
||||
let py = self_.py();
|
||||
let bytes = py
|
||||
.detach(move || block_on(async move { inner.read().await }))
|
||||
.map_err(|e| PyRuntimeError::new_err(format!("blob read failed: {e}")))?;
|
||||
Ok(PyBytes::new(self_.py(), bytes.as_ref()).unbind())
|
||||
Ok(PyBytes::new(py, bytes.as_ref()).unbind())
|
||||
}
|
||||
|
||||
pub fn read(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
@@ -444,24 +446,32 @@ impl PyBlobFile {
|
||||
|
||||
fn close(self_: PyRef<'_, Self>) -> PyResult<()> {
|
||||
let inner = self_.inner.clone();
|
||||
block_on(async move { inner.close().await })
|
||||
self_
|
||||
.py()
|
||||
.detach(move || block_on(async move { inner.close().await }))
|
||||
.map_err(|e| PyRuntimeError::new_err(format!("blob close failed: {e}")))
|
||||
}
|
||||
|
||||
fn is_closed(self_: PyRef<'_, Self>) -> bool {
|
||||
let inner = self_.inner.clone();
|
||||
block_on(async move { inner.is_closed().await })
|
||||
self_
|
||||
.py()
|
||||
.detach(move || block_on(async move { inner.is_closed().await }))
|
||||
}
|
||||
|
||||
fn seek(self_: PyRef<'_, Self>, position: u64) -> PyResult<()> {
|
||||
let inner = self_.inner.clone();
|
||||
block_on(async move { inner.seek(position).await })
|
||||
self_
|
||||
.py()
|
||||
.detach(move || block_on(async move { inner.seek(position).await }))
|
||||
.map_err(|e| PyRuntimeError::new_err(format!("blob seek failed: {e}")))
|
||||
}
|
||||
|
||||
fn tell(self_: PyRef<'_, Self>) -> PyResult<u64> {
|
||||
let inner = self_.inner.clone();
|
||||
block_on(async move { inner.tell().await })
|
||||
self_
|
||||
.py()
|
||||
.detach(move || block_on(async move { inner.tell().await }))
|
||||
.map_err(|e| PyRuntimeError::new_err(format!("blob tell failed: {e}")))
|
||||
}
|
||||
|
||||
@@ -475,16 +485,20 @@ impl PyBlobFile {
|
||||
.checked_add(length as u64)
|
||||
.ok_or_else(|| PyValueError::new_err("offset + length overflowed"))?;
|
||||
let inner = self_.inner.clone();
|
||||
let bytes = block_on(async move { inner.read_range(offset..end).await })
|
||||
let py = self_.py();
|
||||
let bytes = py
|
||||
.detach(move || block_on(async move { inner.read_range(offset..end).await }))
|
||||
.map_err(|e| PyRuntimeError::new_err(format!("blob read_range failed: {e}")))?;
|
||||
Ok(PyBytes::new(self_.py(), bytes.as_ref()).unbind())
|
||||
Ok(PyBytes::new(py, bytes.as_ref()).unbind())
|
||||
}
|
||||
|
||||
fn read_up_to(self_: PyRef<'_, Self>, length: usize) -> PyResult<Py<PyBytes>> {
|
||||
let inner = self_.inner.clone();
|
||||
let bytes = block_on(async move { inner.read_up_to(length).await })
|
||||
.map_err(|e| PyRuntimeError::new_err(format!("blob read failed: {e}")))?;
|
||||
Ok(PyBytes::new(self_.py(), bytes.as_ref()).unbind())
|
||||
let py = self_.py();
|
||||
let bytes = py
|
||||
.detach(move || block_on(async move { inner.read_up_to(length).await }))
|
||||
.map_err(|e| PyRuntimeError::new_err(format!("blob read_up_to failed: {e}")))?;
|
||||
Ok(PyBytes::new(py, bytes.as_ref()).unbind())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -731,6 +745,9 @@ impl Table {
|
||||
|
||||
#[allow(private_interfaces)]
|
||||
pub fn delete(self_: PyRef<'_, Self>, condition: PredicateArg) -> PyResult<Bound<'_, PyAny>> {
|
||||
// Do not hold the Python borrow across the await. The cloned Rust table
|
||||
// handle is thread-safe and allows deletes on the same Python table to
|
||||
// run concurrently without PyO3 reporting "Already borrowed".
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let result = match &condition {
|
||||
@@ -1361,7 +1378,12 @@ impl Table {
|
||||
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let result = inner.add_columns(definitions, None).await.infer_error()?;
|
||||
let result = inner
|
||||
.add_columns()
|
||||
.transform(definitions)
|
||||
.execute()
|
||||
.await
|
||||
.infer_error()?;
|
||||
Ok(AddColumnsResult::from(result))
|
||||
})
|
||||
}
|
||||
@@ -1375,7 +1397,12 @@ impl Table {
|
||||
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let result = inner.add_columns(transform, None).await.infer_error()?;
|
||||
let result = inner
|
||||
.add_columns()
|
||||
.transform(transform)
|
||||
.execute()
|
||||
.await
|
||||
.infer_error()?;
|
||||
Ok(AddColumnsResult::from(result))
|
||||
})
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
[toolchain]
|
||||
channel = "1.95.0"
|
||||
channel = "1.97.0"
|
||||
|
||||
+198
-2
@@ -9,6 +9,7 @@
|
||||
//!
|
||||
//! Blob tables require Lance file format >= 2.2 and stable row ids at create.
|
||||
|
||||
use std::ops::Range;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::LargeBinaryArray;
|
||||
@@ -17,10 +18,202 @@ use arrow_schema::{DataType, Field, Schema};
|
||||
use lance::dataset::{BlobRangeRequest as LanceBlobRangeRequest, Dataset, WriteParams};
|
||||
use lance_arrow::FieldExt;
|
||||
use lance_encoding::version::LanceFileVersion;
|
||||
use lance_io::object_store::ObjectStore;
|
||||
use object_store::path::Path;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
pub use lance::dataset::BlobFile;
|
||||
/// Seekable handle for one blob value, backed by local storage or a remote
|
||||
/// HTTP byte-range endpoint.
|
||||
#[derive(Debug)]
|
||||
pub struct BlobFile {
|
||||
inner: BlobFileInner,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum BlobFileInner {
|
||||
Native(lance::dataset::BlobFile),
|
||||
#[cfg(feature = "remote")]
|
||||
Remote(Box<crate::remote::table::blobs::RemoteBlobFile>),
|
||||
}
|
||||
|
||||
impl From<lance::dataset::BlobFile> for BlobFile {
|
||||
fn from(value: lance::dataset::BlobFile) -> Self {
|
||||
Self {
|
||||
inner: BlobFileInner::Native(value),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "remote")]
|
||||
impl From<crate::remote::table::blobs::RemoteBlobFile> for BlobFile {
|
||||
fn from(value: crate::remote::table::blobs::RemoteBlobFile) -> Self {
|
||||
Self {
|
||||
inner: BlobFileInner::Remote(Box::new(value)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BlobFile {
|
||||
/// Inline reader over a data-file slice.
|
||||
pub fn new_inline(
|
||||
object_store: Arc<ObjectStore>,
|
||||
path: Path,
|
||||
position: u64,
|
||||
size: u64,
|
||||
) -> Self {
|
||||
lance::dataset::BlobFile::new_inline(object_store, path, position, size).into()
|
||||
}
|
||||
|
||||
/// Dedicated sidecar-file reader.
|
||||
pub fn new_dedicated(object_store: Arc<ObjectStore>, path: Path, size: u64) -> Self {
|
||||
lance::dataset::BlobFile::new_dedicated(object_store, path, size).into()
|
||||
}
|
||||
|
||||
/// Packed reader for a slice in a shared sidecar.
|
||||
pub fn new_packed(
|
||||
object_store: Arc<ObjectStore>,
|
||||
path: Path,
|
||||
position: u64,
|
||||
size: u64,
|
||||
) -> Self {
|
||||
lance::dataset::BlobFile::new_packed(object_store, path, position, size).into()
|
||||
}
|
||||
|
||||
/// External reader at a resolved object location.
|
||||
pub fn new_external(
|
||||
object_store: Arc<ObjectStore>,
|
||||
path: Path,
|
||||
uri: String,
|
||||
position: u64,
|
||||
size: u64,
|
||||
) -> Self {
|
||||
lance::dataset::BlobFile::new_external(object_store, path, uri, position, size).into()
|
||||
}
|
||||
|
||||
/// Close the handle.
|
||||
pub async fn close(&self) -> lance_core::Result<()> {
|
||||
match &self.inner {
|
||||
BlobFileInner::Native(file) => file.close().await,
|
||||
#[cfg(feature = "remote")]
|
||||
BlobFileInner::Remote(file) => file.close().await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the handle is closed.
|
||||
pub async fn is_closed(&self) -> bool {
|
||||
match &self.inner {
|
||||
BlobFileInner::Native(file) => file.is_closed().await,
|
||||
#[cfg(feature = "remote")]
|
||||
BlobFileInner::Remote(file) => file.is_closed(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a range without moving the cursor.
|
||||
pub async fn read_range(&self, range: Range<u64>) -> lance_core::Result<bytes::Bytes> {
|
||||
match &self.inner {
|
||||
BlobFileInner::Native(file) => file.read_range(range).await,
|
||||
#[cfg(feature = "remote")]
|
||||
BlobFileInner::Remote(file) => file.read_range(range).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Read ranges without moving the cursor.
|
||||
pub async fn read_ranges(
|
||||
&self,
|
||||
ranges: &[Range<u64>],
|
||||
) -> lance_core::Result<Vec<bytes::Bytes>> {
|
||||
match &self.inner {
|
||||
BlobFileInner::Native(file) => file.read_ranges(ranges).await,
|
||||
#[cfg(feature = "remote")]
|
||||
BlobFileInner::Remote(file) => file.read_ranges(ranges).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Read from the cursor to the end.
|
||||
pub async fn read(&self) -> lance_core::Result<bytes::Bytes> {
|
||||
match &self.inner {
|
||||
BlobFileInner::Native(file) => file.read().await,
|
||||
#[cfg(feature = "remote")]
|
||||
BlobFileInner::Remote(file) => file.read().await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Read up to `len` bytes and advance the cursor.
|
||||
pub async fn read_up_to(&self, len: usize) -> lance_core::Result<bytes::Bytes> {
|
||||
match &self.inner {
|
||||
BlobFileInner::Native(file) => file.read_up_to(len).await,
|
||||
#[cfg(feature = "remote")]
|
||||
BlobFileInner::Remote(file) => file.read_up_to(len).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Move the cursor to `new_cursor`.
|
||||
pub async fn seek(&self, new_cursor: u64) -> lance_core::Result<()> {
|
||||
match &self.inner {
|
||||
BlobFileInner::Native(file) => file.seek(new_cursor).await,
|
||||
#[cfg(feature = "remote")]
|
||||
BlobFileInner::Remote(file) => file.seek(new_cursor).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Current cursor position.
|
||||
pub async fn tell(&self) -> lance_core::Result<u64> {
|
||||
match &self.inner {
|
||||
BlobFileInner::Native(file) => file.tell().await,
|
||||
#[cfg(feature = "remote")]
|
||||
BlobFileInner::Remote(file) => file.tell().await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Blob length in bytes.
|
||||
pub fn size(&self) -> u64 {
|
||||
match &self.inner {
|
||||
BlobFileInner::Native(file) => file.size(),
|
||||
#[cfg(feature = "remote")]
|
||||
BlobFileInner::Remote(file) => file.size(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Physical byte offset in the data file. `None` on remote handles. The
|
||||
/// Cloud byte-range route does not expose storage layout.
|
||||
pub fn position(&self) -> Option<u64> {
|
||||
match &self.inner {
|
||||
BlobFileInner::Native(file) => Some(file.position()),
|
||||
#[cfg(feature = "remote")]
|
||||
BlobFileInner::Remote(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Path of the data file holding the blob. `None` on remote handles. The
|
||||
/// Cloud byte-range route does not expose storage layout.
|
||||
pub fn data_path(&self) -> Option<&Path> {
|
||||
match &self.inner {
|
||||
BlobFileInner::Native(file) => Some(file.data_path()),
|
||||
#[cfg(feature = "remote")]
|
||||
BlobFileInner::Remote(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Native storage layout. `None` on remote handles. The Cloud byte-range
|
||||
/// route does not expose layout.
|
||||
pub fn kind(&self) -> Option<lance_core::datatypes::BlobKind> {
|
||||
match &self.inner {
|
||||
BlobFileInner::Native(file) => Some(file.kind()),
|
||||
#[cfg(feature = "remote")]
|
||||
BlobFileInner::Remote(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// External URI for native handles. Remote handles do not expose storage URIs.
|
||||
pub fn uri(&self) -> Option<&str> {
|
||||
match &self.inner {
|
||||
BlobFileInner::Native(file) => file.uri(),
|
||||
#[cfg(feature = "remote")]
|
||||
BlobFileInner::Remote(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One row-specific blob range read request.
|
||||
///
|
||||
@@ -264,7 +457,10 @@ pub(crate) async fn take_blob_files_aligned(
|
||||
|
||||
let handles = dataset.take_blobs(row_ids, column).await?;
|
||||
ensure_all_row_ids_resolved(column, row_ids.len(), handles.len())?;
|
||||
Ok(handles)
|
||||
Ok(handles
|
||||
.into_iter()
|
||||
.map(|handle| handle.map(Into::into))
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -11,7 +11,9 @@ use lance_core::{cache::LanceCache, utils::futures::FinallyStreamExt};
|
||||
use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
|
||||
use lance_file::{
|
||||
reader::{FileReader, FileReaderOptions},
|
||||
writer::{FileWriter, FileWriterOptions},
|
||||
version::ConcreteFileVersion,
|
||||
versions,
|
||||
writer::FileWriterOptions,
|
||||
};
|
||||
use lance_io::{
|
||||
ReadBatchParams,
|
||||
@@ -152,8 +154,12 @@ impl Shuffler {
|
||||
source: None,
|
||||
})?;
|
||||
let object_writer = object_store.create(&path).await?;
|
||||
let writer =
|
||||
FileWriter::try_new(object_writer, schema.clone(), FileWriterOptions::default())?;
|
||||
let writer = versions::create_writer(
|
||||
ConcreteFileVersion::V2_1,
|
||||
object_writer,
|
||||
schema.clone(),
|
||||
FileWriterOptions::default(),
|
||||
)?;
|
||||
file_writers.push(writer);
|
||||
}
|
||||
|
||||
|
||||
@@ -264,7 +264,7 @@ pub fn compute_output_schema(
|
||||
let field_name = ed
|
||||
.dest_column
|
||||
.clone()
|
||||
.unwrap_or_else(|| format!("{}_embedding", &ed.source_column));
|
||||
.unwrap_or_else(|| format!("{}_embedding", ed.source_column));
|
||||
|
||||
sb.push(Field::new(
|
||||
field_name,
|
||||
@@ -291,7 +291,7 @@ pub fn compute_embeddings_for_batch(
|
||||
let dst_field_name = fld
|
||||
.dest_column
|
||||
.clone()
|
||||
.unwrap_or_else(|| format!("{}_embedding", &fld.source_column));
|
||||
.unwrap_or_else(|| format!("{}_embedding", fld.source_column));
|
||||
|
||||
let dst_field = Field::new(
|
||||
dst_field_name,
|
||||
@@ -315,7 +315,7 @@ impl<R: RecordBatchReader> WithEmbeddings<R> {
|
||||
let field_name = ed
|
||||
.dest_column
|
||||
.clone()
|
||||
.unwrap_or_else(|| format!("{}_embedding", &ed.source_column));
|
||||
.unwrap_or_else(|| format!("{}_embedding", ed.source_column));
|
||||
Ok(Field::new(
|
||||
field_name,
|
||||
func.dest_type()?.into_owned(),
|
||||
|
||||
@@ -706,7 +706,7 @@ impl<S: HttpSend> RestfulLanceDbClient<S> {
|
||||
.err_to_http(request_id.clone())?;
|
||||
debug!(
|
||||
"Received response for request_id={}: {:?}",
|
||||
request_id, &response
|
||||
request_id, response
|
||||
);
|
||||
Ok((request_id, response))
|
||||
}
|
||||
@@ -768,7 +768,7 @@ impl<S: HttpSend> RestfulLanceDbClient<S> {
|
||||
Ok((status, response)) if status.is_success() => {
|
||||
debug!(
|
||||
"Received response for request_id={}: {:?}",
|
||||
retry_counter.request_id, &response
|
||||
retry_counter.request_id, response
|
||||
);
|
||||
return Ok((retry_counter.request_id, response));
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
mod blobs;
|
||||
pub mod blobs;
|
||||
pub mod insert;
|
||||
|
||||
use self::insert::{RemoteWriteExec, WriteOp};
|
||||
@@ -3089,10 +3089,12 @@ mod tests {
|
||||
Box::pin(table.delete("false").map_ok(|_| ())),
|
||||
Box::pin(
|
||||
table
|
||||
.add_columns(
|
||||
NewColumnTransform::SqlExpressions(vec![("x".into(), "y".into())]),
|
||||
None,
|
||||
)
|
||||
.add_columns()
|
||||
.transform(NewColumnTransform::SqlExpressions(vec![(
|
||||
"x".into(),
|
||||
"y".into(),
|
||||
)]))
|
||||
.execute()
|
||||
.map_ok(|_| ()),
|
||||
),
|
||||
Box::pin(async {
|
||||
@@ -4300,32 +4302,9 @@ mod tests {
|
||||
"fetch_blobs",
|
||||
);
|
||||
|
||||
let message = table
|
||||
.fetch_blob_files("image", &[1])
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(
|
||||
message.contains("fetch_blob_files is not supported on LanceDB Cloud"),
|
||||
"got: {message}"
|
||||
);
|
||||
assert!(
|
||||
!message.contains("Use fetch_blobs"),
|
||||
"old server must not be told to use fetch_blobs, got: {message}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_blob_files_point_at_fetch_blobs_on_a_blob_capable_server() {
|
||||
let table = Table::new_with_handler_version(
|
||||
"my_table",
|
||||
semver::Version::new(0, 5, 0),
|
||||
|_| -> http::Response<String> { panic!("fetch_blob_files must not reach the server") },
|
||||
);
|
||||
|
||||
assert_not_supported_error(
|
||||
table.fetch_blob_files("image", &[1]).await.unwrap_err(),
|
||||
"Use fetch_blobs for full bytes",
|
||||
"requires LanceDB Cloud server 0.5.0 or newer",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -6411,13 +6390,12 @@ mod tests {
|
||||
});
|
||||
|
||||
let result = table
|
||||
.add_columns(
|
||||
NewColumnTransform::SqlExpressions(vec![
|
||||
("b".into(), "a + 1".into()),
|
||||
("x".into(), "cast(NULL as int32)".into()),
|
||||
]),
|
||||
None,
|
||||
)
|
||||
.add_columns()
|
||||
.transform(NewColumnTransform::SqlExpressions(vec![
|
||||
("b".into(), "a + 1".into()),
|
||||
("x".into(), "cast(NULL as int32)".into()),
|
||||
]))
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -7142,10 +7120,12 @@ mod tests {
|
||||
}
|
||||
"add_columns" => {
|
||||
let _ = table
|
||||
.add_columns(
|
||||
NewColumnTransform::SqlExpressions(vec![("c".into(), "a + 1".into())]),
|
||||
None,
|
||||
)
|
||||
.add_columns()
|
||||
.transform(NewColumnTransform::SqlExpressions(vec![(
|
||||
"c".into(),
|
||||
"a + 1".into(),
|
||||
)]))
|
||||
.execute()
|
||||
.await;
|
||||
}
|
||||
"drop_columns" => {
|
||||
@@ -9903,10 +9883,12 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
branch
|
||||
.add_columns(
|
||||
NewColumnTransform::SqlExpressions(vec![("b".into(), "a + 1".into())]),
|
||||
None,
|
||||
)
|
||||
.add_columns()
|
||||
.transform(NewColumnTransform::SqlExpressions(vec![(
|
||||
"b".into(),
|
||||
"a + 1".into(),
|
||||
)]))
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
branch
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,6 +3,7 @@
|
||||
|
||||
//! LanceDB Table APIs
|
||||
|
||||
use crate::blob::BlobFile;
|
||||
use arrow_array::{LargeBinaryArray, RecordBatch, RecordBatchReader};
|
||||
use arrow_schema::{Schema, SchemaRef};
|
||||
use async_trait::async_trait;
|
||||
@@ -12,7 +13,6 @@ use datafusion_physical_plan::ExecutionPlan;
|
||||
use datafusion_physical_plan::display::DisplayableExecutionPlan;
|
||||
use futures::StreamExt;
|
||||
use futures::stream::FuturesUnordered;
|
||||
use lance::dataset::BlobFile;
|
||||
pub use lance::dataset::ColumnAlteration;
|
||||
pub use lance::dataset::NewColumnTransform;
|
||||
pub use lance::dataset::ReadParams;
|
||||
@@ -65,6 +65,7 @@ use crate::utils::{PatchReadParam, PatchWriteParam, resolve_arrow_field_path};
|
||||
use self::dataset::DatasetConsistencyWrapper;
|
||||
use self::merge::MergeInsertBuilder;
|
||||
|
||||
pub mod add_columns;
|
||||
mod add_data;
|
||||
pub mod branch_merge;
|
||||
mod create_index;
|
||||
@@ -79,6 +80,7 @@ pub mod schema_evolution;
|
||||
pub mod update;
|
||||
pub mod write_progress;
|
||||
use crate::index::waiter::wait_for_index;
|
||||
pub use add_columns::AddColumnsBuilder;
|
||||
#[cfg(feature = "remote")]
|
||||
pub(crate) use add_data::PreprocessingOutput;
|
||||
pub use add_data::{AddDataBuilder, AddDataMode, AddResult, NaNVectorBehavior};
|
||||
@@ -1620,12 +1622,8 @@ impl Table {
|
||||
}
|
||||
|
||||
/// Add new columns to the table, providing values to fill in.
|
||||
pub async fn add_columns(
|
||||
&self,
|
||||
transforms: NewColumnTransform,
|
||||
read_columns: Option<Vec<String>>,
|
||||
) -> Result<AddColumnsResult> {
|
||||
self.inner.add_columns(transforms, read_columns).await
|
||||
pub fn add_columns(&self) -> AddColumnsBuilder {
|
||||
AddColumnsBuilder::new(self.inner.clone())
|
||||
}
|
||||
|
||||
/// Change a column's name or nullability.
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
//! Builder for adding columns to a table.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use lance::dataset::NewColumnTransform;
|
||||
|
||||
use super::BaseTable;
|
||||
use super::schema_evolution::AddColumnsResult;
|
||||
use crate::{Error, Result};
|
||||
|
||||
/// Adds columns to a table. See [`Table::add_columns`](super::Table::add_columns).
|
||||
pub struct AddColumnsBuilder {
|
||||
parent: Arc<dyn BaseTable>,
|
||||
transform: Option<NewColumnTransform>,
|
||||
read_columns: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for AddColumnsBuilder {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("AddColumnsBuilder")
|
||||
.field("parent", &self.parent)
|
||||
.field("has_transform", &self.transform.is_some())
|
||||
.field("read_columns", &self.read_columns)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl AddColumnsBuilder {
|
||||
pub(crate) fn new(parent: Arc<dyn BaseTable>) -> Self {
|
||||
Self {
|
||||
parent,
|
||||
transform: None,
|
||||
read_columns: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set how the new columns' values are produced. Required.
|
||||
pub fn transform(mut self, transform: NewColumnTransform) -> Self {
|
||||
self.transform = Some(transform);
|
||||
self
|
||||
}
|
||||
|
||||
/// Limit which existing columns a [`NewColumnTransform::BatchUDF`] mapper
|
||||
/// receives. Every other transform determines what it reads, so setting
|
||||
/// this alongside one is an error rather than a silent no-op.
|
||||
pub fn read_columns(mut self, columns: impl IntoIterator<Item = impl Into<String>>) -> Self {
|
||||
self.read_columns = Some(columns.into_iter().map(Into::into).collect());
|
||||
self
|
||||
}
|
||||
|
||||
/// Add the columns.
|
||||
pub async fn execute(self) -> Result<AddColumnsResult> {
|
||||
let Self {
|
||||
parent,
|
||||
transform,
|
||||
read_columns,
|
||||
} = self;
|
||||
|
||||
let Some(transform) = transform else {
|
||||
return Err(Error::InvalidInput {
|
||||
message: "add_columns requires a transform".into(),
|
||||
});
|
||||
};
|
||||
|
||||
if read_columns.is_some() && !matches!(transform, NewColumnTransform::BatchUDF(_)) {
|
||||
return Err(Error::InvalidInput {
|
||||
message: "read_columns applies only to a BatchUDF transform; \
|
||||
every other transform determines what it reads"
|
||||
.into(),
|
||||
});
|
||||
}
|
||||
|
||||
parent.add_columns(transform, read_columns).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::{Int32Array, RecordBatch, record_batch};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use lance::dataset::{BatchUDF, NewColumnTransform};
|
||||
|
||||
use crate::Table;
|
||||
use crate::connect;
|
||||
|
||||
async fn table_with_two_columns(name: &str) -> Table {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let batch = record_batch!(("x", Int32, [1, 2, 3]), ("y", Int32, [10, 20, 30])).unwrap();
|
||||
conn.create_table(name, batch).execute().await.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_requires_a_transform() {
|
||||
let table = table_with_two_columns("no_transform").await;
|
||||
let err = table.add_columns().execute().await.unwrap_err();
|
||||
assert!(
|
||||
err.to_string().contains("requires a transform"),
|
||||
"got: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_columns_with_sql_expressions_is_rejected() {
|
||||
let table = table_with_two_columns("read_cols_sql").await;
|
||||
let err = table
|
||||
.add_columns()
|
||||
.transform(NewColumnTransform::SqlExpressions(vec![(
|
||||
"doubled".into(),
|
||||
"x * 2".into(),
|
||||
)]))
|
||||
.read_columns(["x"])
|
||||
.execute()
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("BatchUDF"), "got: {err}");
|
||||
|
||||
let schema = table.schema().await.unwrap();
|
||||
assert!(
|
||||
schema.field_with_name("doubled").is_err(),
|
||||
"a rejected call must not commit"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_columns_limits_what_a_batch_udf_sees() {
|
||||
let table = table_with_two_columns("read_cols_udf").await;
|
||||
|
||||
let output_schema = Arc::new(Schema::new(vec![Field::new("sum", DataType::Int32, true)]));
|
||||
let mapper_schema = output_schema.clone();
|
||||
let udf = BatchUDF {
|
||||
mapper: Box::new(move |batch: &RecordBatch| {
|
||||
assert!(batch.column_by_name("x").is_some());
|
||||
assert!(batch.column_by_name("y").is_none(), "y was not requested");
|
||||
let x = batch["x"].as_any().downcast_ref::<Int32Array>().unwrap();
|
||||
let doubled: Int32Array = x.iter().map(|v| v.map(|v| v * 2)).collect();
|
||||
Ok(RecordBatch::try_new(
|
||||
mapper_schema.clone(),
|
||||
vec![Arc::new(doubled)],
|
||||
)?)
|
||||
}),
|
||||
output_schema,
|
||||
result_checkpoint: None,
|
||||
};
|
||||
|
||||
table
|
||||
.add_columns()
|
||||
.transform(NewColumnTransform::BatchUDF(udf))
|
||||
.read_columns(["x"])
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let schema = table.schema().await.unwrap();
|
||||
assert!(schema.field_with_name("sum").is_ok());
|
||||
}
|
||||
}
|
||||
@@ -258,6 +258,7 @@ mod tests {
|
||||
FixedSizeListArray, Float32Array, Int32Array, LargeStringArray, ListArray, RecordBatch,
|
||||
RecordBatchIterator, record_batch,
|
||||
};
|
||||
use arrow_buffer::OffsetBuffer;
|
||||
use arrow_schema::{ArrowError, DataType, Field, Schema};
|
||||
use futures::TryStreamExt;
|
||||
use lance::dataset::{WriteMode, WriteParams};
|
||||
@@ -576,10 +577,12 @@ mod tests {
|
||||
|
||||
// Add a new physical column AFTER the embedding column.
|
||||
table
|
||||
.add_columns(
|
||||
NewColumnTransform::SqlExpressions(vec![("score".into(), "42.0".into())]),
|
||||
None,
|
||||
)
|
||||
.add_columns()
|
||||
.transform(NewColumnTransform::SqlExpressions(vec![(
|
||||
"score".into(),
|
||||
"42.0".into(),
|
||||
)]))
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -683,7 +686,9 @@ mod tests {
|
||||
true,
|
||||
)]));
|
||||
table
|
||||
.add_columns(NewColumnTransform::AllNulls(nested_schema), None)
|
||||
.add_columns()
|
||||
.transform(NewColumnTransform::AllNulls(nested_schema))
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -881,6 +886,63 @@ mod tests {
|
||||
assert_eq!(row_count, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_add_rejects_nan_multivectors() {
|
||||
let vector_type =
|
||||
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4);
|
||||
let schema = Arc::new(Schema::new(vec![Field::new(
|
||||
"embedding",
|
||||
DataType::List(Arc::new(Field::new("item", vector_type.clone(), true))),
|
||||
false,
|
||||
)]));
|
||||
|
||||
let db = connect("memory://").execute().await.unwrap();
|
||||
let table = db
|
||||
.create_empty_table("nan_multivector_test", schema.clone())
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let vectors = FixedSizeListArray::try_new(
|
||||
Arc::new(Field::new("item", DataType::Float32, true)),
|
||||
4,
|
||||
Arc::new(Float32Array::from(vec![
|
||||
0.1,
|
||||
0.2,
|
||||
0.3,
|
||||
0.4,
|
||||
0.5,
|
||||
f32::NAN,
|
||||
0.7,
|
||||
0.8,
|
||||
])),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let multivectors = ListArray::try_new(
|
||||
Arc::new(Field::new("item", vector_type, true)),
|
||||
OffsetBuffer::from_lengths([2]),
|
||||
Arc::new(vectors),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let batch = RecordBatch::try_new(schema, vec![Arc::new(multivectors)]).unwrap();
|
||||
|
||||
let err = table.add(batch.clone()).execute().await.unwrap_err();
|
||||
assert!(
|
||||
err.to_string().contains("NaN"),
|
||||
"Expected error mentioning NaN values, but got: {err:?}"
|
||||
);
|
||||
|
||||
table
|
||||
.add(batch)
|
||||
.on_nan_vectors(NaNVectorBehavior::Keep)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(table.count_rows(None).await.unwrap(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_add_subschema() {
|
||||
let data = record_batch!(("id", Int64, [4, 5]), ("text", Utf8, ["foo", "bar"])).unwrap();
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
use std::sync::{Arc, LazyLock};
|
||||
|
||||
use arrow_array::{Array, FixedSizeListArray};
|
||||
use arrow_array::{Array, FixedSizeListArray, ListArray};
|
||||
use arrow_schema::{DataType, Field, FieldRef};
|
||||
use datafusion_common::config::ConfigOptions;
|
||||
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
|
||||
@@ -19,16 +19,20 @@ use crate::{Error, Result};
|
||||
static REJECT_NAN_UDF: LazyLock<Arc<datafusion_expr::ScalarUDF>> =
|
||||
LazyLock::new(|| Arc::new(datafusion_expr::ScalarUDF::from(RejectNanUdf::new())));
|
||||
|
||||
/// Returns true if the field is a vector column: FixedSizeList<Float16/32/64>.
|
||||
/// Returns true if the field is a vector or multivector column.
|
||||
fn is_vector_field(field: &Field) -> bool {
|
||||
if let DataType::FixedSizeList(child, _) = field.data_type() {
|
||||
matches!(
|
||||
child.data_type(),
|
||||
DataType::Float16 | DataType::Float32 | DataType::Float64
|
||||
)
|
||||
} else {
|
||||
false
|
||||
fn is_vector_data_type(data_type: &DataType) -> bool {
|
||||
match data_type {
|
||||
DataType::FixedSizeList(child, _) => matches!(
|
||||
child.data_type(),
|
||||
DataType::Float16 | DataType::Float32 | DataType::Float64
|
||||
),
|
||||
DataType::List(child) => is_vector_data_type(child.data_type()),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
is_vector_data_type(field.data_type())
|
||||
}
|
||||
|
||||
/// Wraps the input plan with a projection that checks vector columns for NaN values.
|
||||
@@ -69,8 +73,8 @@ pub fn reject_nan_vectors(input: Arc<dyn ExecutionPlan>) -> Result<Arc<dyn Execu
|
||||
Ok(Arc::new(projection))
|
||||
}
|
||||
|
||||
/// A scalar UDF that passes through FixedSizeList arrays unchanged, but errors
|
||||
/// if any float values in the list are NaN.
|
||||
/// A scalar UDF that passes through vector arrays unchanged, but errors if any
|
||||
/// float values in the vector or multivector are NaN.
|
||||
#[derive(Debug, Hash, PartialEq, Eq)]
|
||||
struct RejectNanUdf {
|
||||
signature: Signature,
|
||||
@@ -113,44 +117,54 @@ impl ScalarUDFImpl for RejectNanUdf {
|
||||
}
|
||||
|
||||
fn check_no_nans(array: &dyn Array) -> datafusion_common::Result<()> {
|
||||
let fsl = array
|
||||
.as_any()
|
||||
.downcast_ref::<FixedSizeListArray>()
|
||||
.ok_or_else(|| {
|
||||
datafusion_common::DataFusionError::Internal(
|
||||
"reject_nan expected FixedSizeList".to_string(),
|
||||
)
|
||||
})?;
|
||||
|
||||
// Only inspect elements that are both in a valid parent row and non-null
|
||||
// themselves. Values backing null parent rows or null child elements may
|
||||
// contain garbage (including NaN) per the Arrow spec.
|
||||
let has_nan = (0..fsl.len()).filter(|i| fsl.is_valid(*i)).any(|i| {
|
||||
let row = fsl.value(i);
|
||||
match row.data_type() {
|
||||
DataType::Float16 => row
|
||||
fn contains_nan(array: &dyn Array) -> datafusion_common::Result<bool> {
|
||||
match array.data_type() {
|
||||
DataType::Float16 => Ok(array
|
||||
.as_any()
|
||||
.downcast_ref::<arrow_array::Float16Array>()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|v| v.is_some_and(|v| v.is_nan())),
|
||||
DataType::Float32 => row
|
||||
.any(|v| v.is_some_and(|v| v.is_nan()))),
|
||||
DataType::Float32 => Ok(array
|
||||
.as_any()
|
||||
.downcast_ref::<arrow_array::Float32Array>()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|v| v.is_some_and(|v| v.is_nan())),
|
||||
DataType::Float64 => row
|
||||
.any(|v| v.is_some_and(|v| v.is_nan()))),
|
||||
DataType::Float64 => Ok(array
|
||||
.as_any()
|
||||
.downcast_ref::<arrow_array::Float64Array>()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|v| v.is_some_and(|v| v.is_nan())),
|
||||
_ => false,
|
||||
.any(|v| v.is_some_and(|v| v.is_nan()))),
|
||||
DataType::FixedSizeList(_, _) => {
|
||||
let lists = array.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
|
||||
for i in (0..lists.len()).filter(|i| lists.is_valid(*i)) {
|
||||
if contains_nan(lists.value(i).as_ref())? {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
DataType::List(_) => {
|
||||
let lists = array.as_any().downcast_ref::<ListArray>().unwrap();
|
||||
for i in (0..lists.len()).filter(|i| lists.is_valid(*i)) {
|
||||
if contains_nan(lists.value(i).as_ref())? {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
data_type => Err(datafusion_common::DataFusionError::Internal(format!(
|
||||
"reject_nan expected a vector or multivector, got {data_type}"
|
||||
))),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if has_nan {
|
||||
// Only inspect elements that are both in a valid parent row and non-null
|
||||
// themselves. Values backing null parent rows or null child elements may
|
||||
// contain garbage (including NaN) per the Arrow spec.
|
||||
if contains_nan(array)? {
|
||||
return Err(datafusion_common::DataFusionError::ArrowError(
|
||||
Box::new(arrow_schema::ArrowError::ComputeError(
|
||||
"Vector column contains NaN values".to_string(),
|
||||
@@ -166,6 +180,7 @@ fn check_no_nans(array: &dyn Array) -> datafusion_common::Result<()> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use arrow_array::Float32Array;
|
||||
use arrow_buffer::OffsetBuffer;
|
||||
|
||||
#[test]
|
||||
fn test_passes_clean_vectors() {
|
||||
@@ -191,6 +206,46 @@ mod tests {
|
||||
assert!(check_no_nans(&fsl).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rejects_nan_multivectors() {
|
||||
let vectors = FixedSizeListArray::try_new(
|
||||
Arc::new(Field::new("item", DataType::Float32, true)),
|
||||
2,
|
||||
Arc::new(Float32Array::from(vec![1.0, 2.0, 3.0, f32::NAN])),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let multivectors = ListArray::try_new(
|
||||
Arc::new(Field::new("item", vectors.data_type().clone(), true)),
|
||||
OffsetBuffer::from_lengths([2]),
|
||||
Arc::new(vectors),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(check_no_nans(&multivectors).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_skips_null_multivector_rows() {
|
||||
let vectors = FixedSizeListArray::try_new(
|
||||
Arc::new(Field::new("item", DataType::Float32, true)),
|
||||
2,
|
||||
Arc::new(Float32Array::from(vec![f32::NAN, f32::NAN, 1.0, 2.0])),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let multivectors = ListArray::try_new(
|
||||
Arc::new(Field::new("item", vectors.data_type().clone(), true)),
|
||||
OffsetBuffer::from_lengths([1, 1]),
|
||||
Arc::new(vectors),
|
||||
Some(vec![false, true].into()),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(check_no_nans(&multivectors).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_skips_null_rows() {
|
||||
// Values backing null rows may contain NaN per the Arrow spec.
|
||||
@@ -254,6 +309,15 @@ mod tests {
|
||||
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float64, true)), 4),
|
||||
false,
|
||||
)));
|
||||
assert!(is_vector_field(&Field::new(
|
||||
"v",
|
||||
DataType::List(Arc::new(Field::new(
|
||||
"item",
|
||||
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4,),
|
||||
true,
|
||||
))),
|
||||
false,
|
||||
)));
|
||||
assert!(!is_vector_field(&Field::new("id", DataType::Int32, false)));
|
||||
assert!(!is_vector_field(&Field::new(
|
||||
"v",
|
||||
|
||||
@@ -193,10 +193,12 @@ mod tests {
|
||||
|
||||
// Add a computed column
|
||||
let result = table
|
||||
.add_columns(
|
||||
NewColumnTransform::SqlExpressions(vec![("doubled".into(), "id * 2".into())]),
|
||||
None,
|
||||
)
|
||||
.add_columns()
|
||||
.transform(NewColumnTransform::SqlExpressions(vec![(
|
||||
"doubled".into(),
|
||||
"id * 2".into(),
|
||||
)]))
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -251,13 +253,12 @@ mod tests {
|
||||
|
||||
// Add multiple columns at once
|
||||
table
|
||||
.add_columns(
|
||||
NewColumnTransform::SqlExpressions(vec![
|
||||
("y".into(), "x + 1".into()),
|
||||
("z".into(), "x * x".into()),
|
||||
]),
|
||||
None,
|
||||
)
|
||||
.add_columns()
|
||||
.transform(NewColumnTransform::SqlExpressions(vec![
|
||||
("y".into(), "x + 1".into()),
|
||||
("z".into(), "x * x".into()),
|
||||
]))
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -283,10 +284,12 @@ mod tests {
|
||||
|
||||
// Add a column with a constant value
|
||||
table
|
||||
.add_columns(
|
||||
NewColumnTransform::SqlExpressions(vec![("constant".into(), "42".into())]),
|
||||
None,
|
||||
)
|
||||
.add_columns()
|
||||
.transform(NewColumnTransform::SqlExpressions(vec![(
|
||||
"constant".into(),
|
||||
"42".into(),
|
||||
)]))
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -659,10 +662,12 @@ mod tests {
|
||||
|
||||
// Add column increments version
|
||||
let add_result = table
|
||||
.add_columns(
|
||||
NewColumnTransform::SqlExpressions(vec![("c".into(), "a + b".into())]),
|
||||
None,
|
||||
)
|
||||
.add_columns()
|
||||
.transform(NewColumnTransform::SqlExpressions(vec![(
|
||||
"c".into(),
|
||||
"a + b".into(),
|
||||
)]))
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(add_result.version > v1);
|
||||
|
||||
@@ -9,14 +9,17 @@ use arrow_array::{
|
||||
};
|
||||
use arrow_schema::{DataType, Field, Fields, Schema};
|
||||
use futures::TryStreamExt;
|
||||
use lance::Dataset;
|
||||
use lance_encoding::version::LanceFileVersion;
|
||||
use lancedb::{
|
||||
Connection, Error, Result, Table,
|
||||
blob::{BlobRangeRequest, blob},
|
||||
connect, connect_namespace,
|
||||
database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS,
|
||||
database::listing::{
|
||||
ListingDatabaseOptions, NewTableConfig, OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS,
|
||||
},
|
||||
query::{ExecutableQuery, QueryBase},
|
||||
table::{AddDataMode, CompactionOptions, OptimizeAction},
|
||||
table::{AddDataMode, CompactionOptions, OptimizeAction, OptimizeStats},
|
||||
};
|
||||
use tempfile::tempdir;
|
||||
|
||||
@@ -1075,3 +1078,252 @@ async fn fetch_blob_files_aligns_across_fragments_with_nulls_and_dups() -> Resul
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rows exercising the null/empty interleavings from
|
||||
/// <https://github.com/lancedb/lancedb/issues/3744>: a payload, a null, a valid
|
||||
/// empty value, then payloads whose descriptors a fragment rewrite used to zero.
|
||||
fn null_empty_input_batch() -> RecordBatch {
|
||||
let owned = [
|
||||
Some(dedicated_blob_bytes(1)),
|
||||
None,
|
||||
Some(Vec::new()),
|
||||
Some(dedicated_blob_bytes(4)),
|
||||
Some(dedicated_blob_bytes(5)),
|
||||
Some(dedicated_blob_bytes(6)),
|
||||
];
|
||||
let payloads: Vec<Option<&[u8]>> = owned.iter().map(|payload| payload.as_deref()).collect();
|
||||
binary_input_batch(&[1, 2, 3, 4, 5, 6], &payloads)
|
||||
}
|
||||
|
||||
/// One `(id, Some((payload length, first byte)))` per live row, or `(id, None)`
|
||||
/// for a null blob. Comparing lengths and first bytes keeps failure output
|
||||
/// readable where comparing whole payloads would not.
|
||||
type BlobSummary = Vec<(i64, Option<(usize, Option<u8>)>)>;
|
||||
|
||||
/// The rows [`null_empty_input_batch`] leaves behind after `id IN (1, 4)` is
|
||||
/// deleted: a null, a valid empty value, and the two payloads that follow them.
|
||||
fn expected_null_empty_survivors() -> BlobSummary {
|
||||
vec![
|
||||
(2, None),
|
||||
(3, Some((0, None))),
|
||||
(5, Some((DEDICATED_BLOB_LEN, Some(5)))),
|
||||
(6, Some((DEDICATED_BLOB_LEN, Some(6)))),
|
||||
]
|
||||
}
|
||||
|
||||
/// `optimize()` only rewrites a fragment when lance's compaction planner selects
|
||||
/// it — here because the delete pushes the fragment past
|
||||
/// `materialize_deletions_threshold` (0.1 by default; these tests delete 2 of 6
|
||||
/// rows). Without this check, a planner or threshold change upstream would leave
|
||||
/// both regression tests green while no rewrite happened at all.
|
||||
fn assert_compacted(stats: &OptimizeStats) {
|
||||
let metrics = stats
|
||||
.compaction
|
||||
.as_ref()
|
||||
.expect("OptimizeAction::All runs compaction");
|
||||
assert!(
|
||||
metrics.fragments_removed >= 1,
|
||||
"optimize() rewrote no fragment, so this test proves nothing: {metrics:?}"
|
||||
);
|
||||
}
|
||||
|
||||
fn summarize(rows: &[(i64, Option<Vec<u8>>)]) -> BlobSummary {
|
||||
rows.iter()
|
||||
.map(|(id, payload)| {
|
||||
(
|
||||
*id,
|
||||
payload
|
||||
.as_ref()
|
||||
.map(|bytes| (bytes.len(), bytes.first().copied())),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn sorted_id_rowid(table: &Table) -> Result<Vec<(i64, u64)>> {
|
||||
let mut pairs = collect_id_rowid(table).await?;
|
||||
pairs.sort_by_key(|(id, _)| *id);
|
||||
Ok(pairs)
|
||||
}
|
||||
|
||||
/// `{position, size}` descriptors of a legacy v1 blob column, keyed by `id`.
|
||||
async fn v1_blob_descriptors(table: &Table) -> Result<Vec<(i64, Option<(u64, u64)>)>> {
|
||||
let batches = table
|
||||
.query()
|
||||
.execute()
|
||||
.await?
|
||||
.try_collect::<Vec<_>>()
|
||||
.await?;
|
||||
let batch = arrow_select::concat::concat_batches(&batches[0].schema(), &batches).unwrap();
|
||||
let ids = batch
|
||||
.column_by_name("id")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<Int64Array>()
|
||||
.unwrap();
|
||||
let descriptors = batch
|
||||
.column_by_name("image")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<StructArray>()
|
||||
.expect("v1 blob column reads back as a descriptor struct");
|
||||
let position = descriptors
|
||||
.column_by_name("position")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<UInt64Array>()
|
||||
.unwrap();
|
||||
let size = descriptors
|
||||
.column_by_name("size")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<UInt64Array>()
|
||||
.unwrap();
|
||||
let mut rows: Vec<(i64, Option<(u64, u64)>)> = (0..batch.num_rows())
|
||||
.map(|row| {
|
||||
let descriptor =
|
||||
(!descriptors.is_null(row)).then(|| (position.value(row), size.value(row)));
|
||||
(ids.value(row), descriptor)
|
||||
})
|
||||
.collect();
|
||||
rows.sort_by_key(|(id, _)| *id);
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Payload bytes of every live row of a legacy v1 blob column, keyed by `id`.
|
||||
/// [`Table::fetch_blobs`] rejects v1 columns, so read them through lance.
|
||||
async fn v1_blob_payloads(dataset_uri: &str, table: &Table) -> Result<Vec<(i64, Option<Vec<u8>>)>> {
|
||||
let pairs = sorted_id_rowid(table).await?;
|
||||
let row_ids: Vec<u64> = pairs.iter().map(|(_, row_id)| *row_id).collect();
|
||||
let dataset = Arc::new(Dataset::open(dataset_uri).await?);
|
||||
let files = dataset.take_blobs(&row_ids, "image").await?;
|
||||
assert_eq!(
|
||||
files.len(),
|
||||
pairs.len(),
|
||||
"take_blobs returned {} handles for {} live rows",
|
||||
files.len(),
|
||||
pairs.len()
|
||||
);
|
||||
let mut rows = Vec::with_capacity(pairs.len());
|
||||
for ((id, _), file) in pairs.iter().zip(files) {
|
||||
let payload = match file {
|
||||
Some(file) => Some(file.read().await?.to_vec()),
|
||||
None => None,
|
||||
};
|
||||
rows.push((*id, payload));
|
||||
}
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Length and first byte of every live blob v2 value, keyed by `id`.
|
||||
async fn blob_v2_values(table: &Table) -> Result<BlobSummary> {
|
||||
let pairs = sorted_id_rowid(table).await?;
|
||||
let row_ids: Vec<u64> = pairs.iter().map(|(_, row_id)| *row_id).collect();
|
||||
let bytes = table.fetch_blobs("image", &row_ids).await?;
|
||||
Ok(pairs
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(slot, (id, _))| {
|
||||
let value = (!bytes.is_null(slot))
|
||||
.then(|| (bytes.value(slot).len(), bytes.value(slot).first().copied()));
|
||||
(*id, value)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Regression test for [#3744]: on storage 2.0 (legacy v1 descriptors),
|
||||
/// compaction rewrote every payload following a null or empty value in the same
|
||||
/// fragment as `{position: 0, size: 0}`, so the payload bytes read back as `b""`
|
||||
/// and the new fragment no longer referenced them at all.
|
||||
///
|
||||
/// [#3744]: https://github.com/lancedb/lancedb/issues/3744
|
||||
#[tokio::test]
|
||||
async fn optimize_preserves_v1_blob_payloads_with_null_and_empty() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
let db_uri = tmp.path().to_str().unwrap().to_string();
|
||||
let db = connect(&db_uri)
|
||||
.database_options(&ListingDatabaseOptions {
|
||||
new_table_config: NewTableConfig {
|
||||
data_storage_version: Some(LanceFileVersion::V2_0),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
})
|
||||
.execute()
|
||||
.await?;
|
||||
let legacy = Field::new("image", DataType::LargeBinary, true).with_metadata(
|
||||
std::collections::HashMap::from([("lance-encoding:blob".to_string(), "true".to_string())]),
|
||||
);
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::Int64, false),
|
||||
legacy,
|
||||
]));
|
||||
let table = db.create_empty_table("t", schema).execute().await?;
|
||||
table.add(null_empty_input_batch()).execute().await?;
|
||||
assert_eq!(
|
||||
storage_format_version(&table).await,
|
||||
LanceFileVersion::V2_0.resolve(),
|
||||
"v1 blob descriptors only exist below storage 2.2"
|
||||
);
|
||||
let dataset_uri = table.uri().await?;
|
||||
|
||||
// Any rewrite triggers it; deleting rows is the shape from the issue.
|
||||
table.delete("id IN (1, 4)").await?;
|
||||
let descriptors_before = v1_blob_descriptors(&table).await?;
|
||||
let before = v1_blob_payloads(&dataset_uri, &table).await?;
|
||||
assert_eq!(
|
||||
summarize(&before),
|
||||
expected_null_empty_survivors(),
|
||||
"test setup no longer produces the null/empty/payload mix"
|
||||
);
|
||||
|
||||
let stats = table.optimize(OptimizeAction::All).await?;
|
||||
assert_compacted(&stats);
|
||||
|
||||
let descriptors_after = v1_blob_descriptors(&table).await?;
|
||||
let after = v1_blob_payloads(&dataset_uri, &table).await?;
|
||||
assert_eq!(
|
||||
summarize(&after),
|
||||
summarize(&before),
|
||||
"optimize() lost blob payloads; descriptors before={descriptors_before:?} after={descriptors_after:?}"
|
||||
);
|
||||
assert!(after == before, "optimize() changed blob payload bytes");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Regression test for the blob v2 half of [#3744]: compaction rewrote a valid
|
||||
/// empty value as null, destroying the null-vs-empty distinction.
|
||||
///
|
||||
/// [#3744]: https://github.com/lancedb/lancedb/issues/3744
|
||||
#[tokio::test]
|
||||
async fn optimize_preserves_blob_v2_null_and_empty_distinction() -> Result<()> {
|
||||
let tmp = tempdir().unwrap();
|
||||
let db = connect(tmp.path().to_str().unwrap()).execute().await?;
|
||||
let table = db
|
||||
.create_empty_table("t", blob_table_schema())
|
||||
.execute()
|
||||
.await?;
|
||||
table.add(null_empty_input_batch()).execute().await?;
|
||||
assert!(
|
||||
storage_format_version(&table).await >= LanceFileVersion::V2_2,
|
||||
"blob v2 columns require storage >= 2.2"
|
||||
);
|
||||
|
||||
table.delete("id IN (1, 4)").await?;
|
||||
let before = blob_v2_values(&table).await?;
|
||||
assert_eq!(
|
||||
before,
|
||||
expected_null_empty_survivors(),
|
||||
"test setup no longer produces the null/empty/payload mix"
|
||||
);
|
||||
|
||||
let stats = table.optimize(OptimizeAction::All).await?;
|
||||
assert_compacted(&stats);
|
||||
|
||||
assert_eq!(
|
||||
blob_v2_values(&table).await?,
|
||||
before,
|
||||
"optimize() changed blob v2 values"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user